From 754e02939d5152a22c94a47fd8d26065d9468ffa Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Wed, 8 Jan 2025 14:23:03 -0800 Subject: [PATCH 01/10] test CI --- .../source-pokeapi/integration_tests/abnormal_state.json | 3 +++ airbyte-integrations/connectors/source-pokeapi/metadata.yaml | 2 +- docs/integrations/sources/pokeapi.md | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-pokeapi/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-pokeapi/integration_tests/abnormal_state.json index 52b0f2c2118f..0481e3798c8a 100644 --- a/airbyte-integrations/connectors/source-pokeapi/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-pokeapi/integration_tests/abnormal_state.json @@ -1,5 +1,8 @@ { "todo-stream-name": { "todo-field-name": "todo-abnormal-value" + }, + "fake-stream-name": { + "fake-field": "fake-value" } } diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index acc4519cfcb6..20d8e5185898 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -19,7 +19,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6371b14b-bc68-4236-bfbd-468e8df8e968 - dockerImageTag: 0.3.7 + dockerImageTag: 0.3.8 dockerRepository: airbyte/source-pokeapi githubIssueLabel: source-pokeapi icon: pokeapi.svg diff --git a/docs/integrations/sources/pokeapi.md b/docs/integrations/sources/pokeapi.md index af2be216fe47..57bc6eea7464 100644 --- a/docs/integrations/sources/pokeapi.md +++ b/docs/integrations/sources/pokeapi.md @@ -39,6 +39,7 @@ The PokéAPI uses the same [JSONSchema](https://json-schema.org/understanding-js | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------- | +| 0.3.8 | 2025-01-08 | []() | Fake update | | 0.3.7 | 2024-12-28 | [50708](https://github.com/airbytehq/airbyte/pull/50708) | Update dependencies | | 0.3.6 | 2024-12-21 | [50261](https://github.com/airbytehq/airbyte/pull/50261) | Update dependencies | | 0.3.5 | 2024-12-14 | [49689](https://github.com/airbytehq/airbyte/pull/49689) | Update dependencies | From db165a6d402fc1785b97b8ffda3e6583b8a17716 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Wed, 8 Jan 2025 14:56:34 -0800 Subject: [PATCH 02/10] comment-out if condition on python-setup --- .github/actions/install-airbyte-ci/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install-airbyte-ci/action.yml b/.github/actions/install-airbyte-ci/action.yml index d0563331c805..84fd00e28cca 100644 --- a/.github/actions/install-airbyte-ci/action.yml +++ b/.github/actions/install-airbyte-ci/action.yml @@ -54,7 +54,7 @@ runs: - name: Install Python 3.10 id: install-python-3-10 uses: actions/setup-python@v4 - if: steps.determine-install-mode.outputs.install-mode == 'source' + # if: steps.determine-install-mode.outputs.install-mode == 'source' with: python-version: "3.10" token: ${{ inputs.github_token }} From 53ae75625c5bec263af7bea27850381fe9865a4b Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Wed, 8 Jan 2025 15:12:18 -0800 Subject: [PATCH 03/10] use new runner definition with fixed ubuntu version --- .github/workflows/connectors_tests.yml | 2 +- .github/workflows/connectors_version_increment_check.yml | 2 +- .github/workflows/gradle.yml | 2 +- .github/workflows/live_tests.yml | 2 +- .github/workflows/publish-bulk-cdk.yml | 2 +- .github/workflows/publish-java-cdk-command.yml | 2 +- .github/workflows/regression_tests.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/connectors_tests.yml b/.github/workflows/connectors_tests.yml index 4ff74942bab2..08b12420b4fe 100644 --- a/.github/workflows/connectors_tests.yml +++ b/.github/workflows/connectors_tests.yml @@ -78,7 +78,7 @@ jobs: # If the condition is not met the job will be skipped (it will not fail) if: (github.event_name == 'pull_request' && needs.changes.outputs.connectors == 'true' && github.event.pull_request.head.repo.fork != true) || github.event_name == 'workflow_dispatch' name: Connectors CI - runs-on: connector-test-large + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings timeout-minutes: 360 # 6 hours steps: - name: Checkout Airbyte diff --git a/.github/workflows/connectors_version_increment_check.yml b/.github/workflows/connectors_version_increment_check.yml index cd35b22de657..ae3312e70857 100644 --- a/.github/workflows/connectors_version_increment_check.yml +++ b/.github/workflows/connectors_version_increment_check.yml @@ -21,7 +21,7 @@ on: jobs: connectors_ci: name: Connectors Version Increment Check - runs-on: connector-test-large + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings if: github.event.pull_request.head.repo.fork != true timeout-minutes: 22 steps: diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 7081860cf9a4..1c2661bd3993 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -49,7 +49,7 @@ jobs: # Any revision upwards should be based on a performance analysis of gradle scans. # See https://github.com/airbytehq/airbyte/pull/36055 for an example of this, # which explains why which we went down from 64 cores to 16. - runs-on: connector-test-large + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings name: Gradle Check timeout-minutes: 30 steps: diff --git a/.github/workflows/live_tests.yml b/.github/workflows/live_tests.yml index cbe33c48b635..b82777b161ca 100644 --- a/.github/workflows/live_tests.yml +++ b/.github/workflows/live_tests.yml @@ -44,7 +44,7 @@ on: jobs: live_tests: name: Live Tests - runs-on: connector-test-large + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings timeout-minutes: 360 # 6 hours steps: - name: Checkout Airbyte diff --git a/.github/workflows/publish-bulk-cdk.yml b/.github/workflows/publish-bulk-cdk.yml index a1ef1e716c85..a21092c24973 100644 --- a/.github/workflows/publish-bulk-cdk.yml +++ b/.github/workflows/publish-bulk-cdk.yml @@ -25,7 +25,7 @@ env: jobs: publish-bulk-cdk: name: Publish Bulk CDK - runs-on: connector-test-large + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings timeout-minutes: 30 steps: - name: Checkout Airbyte diff --git a/.github/workflows/publish-java-cdk-command.yml b/.github/workflows/publish-java-cdk-command.yml index 182cb2c59613..9a707d468012 100644 --- a/.github/workflows/publish-java-cdk-command.yml +++ b/.github/workflows/publish-java-cdk-command.yml @@ -61,7 +61,7 @@ env: jobs: publish-cdk: name: Publish Java CDK - runs-on: connector-test-large + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings timeout-minutes: 30 steps: - name: Link comment to Workflow Run diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 4bd5fd68bb4b..a56d21f636a7 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -44,7 +44,7 @@ on: jobs: regression_tests: name: Regression Tests - runs-on: connector-test-large + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings timeout-minutes: 360 # 6 hours steps: - name: Checkout Airbyte From 9a16a00acf36858623535967db9824f0c8c1f557 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Wed, 8 Jan 2025 15:23:57 -0800 Subject: [PATCH 04/10] retry with airbyte-ci forced to run as python library --- .github/actions/install-airbyte-ci/action.yml | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/actions/install-airbyte-ci/action.yml b/.github/actions/install-airbyte-ci/action.yml index 84fd00e28cca..12cb0f362aa2 100644 --- a/.github/actions/install-airbyte-ci/action.yml +++ b/.github/actions/install-airbyte-ci/action.yml @@ -30,21 +30,26 @@ runs: - name: "Determine how Airbyte CI should be installed" shell: bash id: determine-install-mode + run: + echo "install-mode=source" >> $GITHUB_OUTPUT + echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV + # When the PR is from a fork, we always install from binary - if: inputs.is_fork == 'false' - run: | - if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then - echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source." - echo "install-mode=source" >> $GITHUB_OUTPUT - echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV - else - echo "install-mode=binary" >> $GITHUB_OUTPUT - echo "SENTRY_ENVIRONMENT=production" >> $GITHUB_ENV - fi + # if: inputs.is_fork == 'false' + # run: | + # if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then + # echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source." + # echo "install-mode=source" >> $GITHUB_OUTPUT + # echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV + # else + # echo "install-mode=binary" >> $GITHUB_OUTPUT + # echo "SENTRY_ENVIRONMENT=production" >> $GITHUB_ENV + # fi - name: Install Airbyte CI from binary id: install-airbyte-ci-binary - if: steps.determine-install-mode.outputs.install-mode == 'binary' || ${{ inputs.is_fork }} == 'true' + if: false + # if: steps.determine-install-mode.outputs.install-mode == 'binary' || ${{ inputs.is_fork }} == 'true' shell: bash run: | curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin @@ -69,6 +74,10 @@ runs: pipx ensurepath pipx install ${{ inputs.path_to_airbyte_ci_source }} + - name: Debug `airbyte-ci` installed version + run: | + airbyte-ci --version + - name: Get dagger engine image name id: get-dagger-engine-image-name shell: bash From 5e613a62976e8f2d60da1405db239782ac26ca57 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Wed, 8 Jan 2025 15:27:12 -0800 Subject: [PATCH 05/10] retry with shell def --- .github/actions/install-airbyte-ci/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/install-airbyte-ci/action.yml b/.github/actions/install-airbyte-ci/action.yml index 12cb0f362aa2..9652c718b076 100644 --- a/.github/actions/install-airbyte-ci/action.yml +++ b/.github/actions/install-airbyte-ci/action.yml @@ -74,7 +74,8 @@ runs: pipx ensurepath pipx install ${{ inputs.path_to_airbyte_ci_source }} - - name: Debug `airbyte-ci` installed version + - name: Print installed `airbyte-ci` version + shell: bash run: | airbyte-ci --version From db9d5832d2be376f6e42e4585610b31a88f27f3e Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Wed, 8 Jan 2025 15:33:00 -0800 Subject: [PATCH 06/10] always install from source --- .github/actions/install-airbyte-ci/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/install-airbyte-ci/action.yml b/.github/actions/install-airbyte-ci/action.yml index 9652c718b076..f7646aa7e568 100644 --- a/.github/actions/install-airbyte-ci/action.yml +++ b/.github/actions/install-airbyte-ci/action.yml @@ -30,7 +30,7 @@ runs: - name: "Determine how Airbyte CI should be installed" shell: bash id: determine-install-mode - run: + run: | echo "install-mode=source" >> $GITHUB_OUTPUT echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV @@ -66,7 +66,8 @@ runs: - name: Install Airbyte CI from source id: install-airbyte-ci-source - if: steps.determine-install-mode.outputs.install-mode == 'source' + if: true + # if: steps.determine-install-mode.outputs.install-mode == 'source' shell: bash run: | pip install --upgrade pip From 80d689b104e9f5bdf10708d11a23c58a14eed1f5 Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:40:33 -0500 Subject: [PATCH 07/10] Update pokeapi.md --- docs/integrations/sources/pokeapi.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/integrations/sources/pokeapi.md b/docs/integrations/sources/pokeapi.md index 57bc6eea7464..af2be216fe47 100644 --- a/docs/integrations/sources/pokeapi.md +++ b/docs/integrations/sources/pokeapi.md @@ -39,7 +39,6 @@ The PokéAPI uses the same [JSONSchema](https://json-schema.org/understanding-js | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------- | -| 0.3.8 | 2025-01-08 | []() | Fake update | | 0.3.7 | 2024-12-28 | [50708](https://github.com/airbytehq/airbyte/pull/50708) | Update dependencies | | 0.3.6 | 2024-12-21 | [50261](https://github.com/airbytehq/airbyte/pull/50261) | Update dependencies | | 0.3.5 | 2024-12-14 | [49689](https://github.com/airbytehq/airbyte/pull/49689) | Update dependencies | From a3994325a21e1adb59ec34e8315d066374d0ba97 Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:41:37 -0500 Subject: [PATCH 08/10] Update abnormal_state.json --- .../source-pokeapi/integration_tests/abnormal_state.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-pokeapi/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-pokeapi/integration_tests/abnormal_state.json index 0481e3798c8a..52b0f2c2118f 100644 --- a/airbyte-integrations/connectors/source-pokeapi/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-pokeapi/integration_tests/abnormal_state.json @@ -1,8 +1,5 @@ { "todo-stream-name": { "todo-field-name": "todo-abnormal-value" - }, - "fake-stream-name": { - "fake-field": "fake-value" } } From 8b1a348cded1badbc247261796306106e913953e Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:42:03 -0500 Subject: [PATCH 09/10] Update metadata.yaml --- airbyte-integrations/connectors/source-pokeapi/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index 20d8e5185898..acc4519cfcb6 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -19,7 +19,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6371b14b-bc68-4236-bfbd-468e8df8e968 - dockerImageTag: 0.3.8 + dockerImageTag: 0.3.7 dockerRepository: airbyte/source-pokeapi githubIssueLabel: source-pokeapi icon: pokeapi.svg From cce645896fdd07ecb1825c1190bb81b05378f64b Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Wed, 8 Jan 2025 15:44:32 -0800 Subject: [PATCH 10/10] Apply suggestions from code review --- .github/workflows/connectors_tests.yml | 2 +- .github/workflows/connectors_version_increment_check.yml | 2 +- .github/workflows/gradle.yml | 2 +- .github/workflows/live_tests.yml | 2 +- .github/workflows/publish-bulk-cdk.yml | 2 +- .github/workflows/publish-java-cdk-command.yml | 2 +- .github/workflows/regression_tests.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/connectors_tests.yml b/.github/workflows/connectors_tests.yml index 08b12420b4fe..1262e36b85c1 100644 --- a/.github/workflows/connectors_tests.yml +++ b/.github/workflows/connectors_tests.yml @@ -78,7 +78,7 @@ jobs: # If the condition is not met the job will be skipped (it will not fail) if: (github.event_name == 'pull_request' && needs.changes.outputs.connectors == 'true' && github.event.pull_request.head.repo.fork != true) || github.event_name == 'workflow_dispatch' name: Connectors CI - runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings timeout-minutes: 360 # 6 hours steps: - name: Checkout Airbyte diff --git a/.github/workflows/connectors_version_increment_check.yml b/.github/workflows/connectors_version_increment_check.yml index ae3312e70857..efb38521c3b8 100644 --- a/.github/workflows/connectors_version_increment_check.yml +++ b/.github/workflows/connectors_version_increment_check.yml @@ -21,7 +21,7 @@ on: jobs: connectors_ci: name: Connectors Version Increment Check - runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings if: github.event.pull_request.head.repo.fork != true timeout-minutes: 22 steps: diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 1c2661bd3993..b730e2f230e9 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -49,7 +49,7 @@ jobs: # Any revision upwards should be based on a performance analysis of gradle scans. # See https://github.com/airbytehq/airbyte/pull/36055 for an example of this, # which explains why which we went down from 64 cores to 16. - runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings name: Gradle Check timeout-minutes: 30 steps: diff --git a/.github/workflows/live_tests.yml b/.github/workflows/live_tests.yml index b82777b161ca..aa017eb3843c 100644 --- a/.github/workflows/live_tests.yml +++ b/.github/workflows/live_tests.yml @@ -44,7 +44,7 @@ on: jobs: live_tests: name: Live Tests - runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings timeout-minutes: 360 # 6 hours steps: - name: Checkout Airbyte diff --git a/.github/workflows/publish-bulk-cdk.yml b/.github/workflows/publish-bulk-cdk.yml index a21092c24973..2d89ac71a249 100644 --- a/.github/workflows/publish-bulk-cdk.yml +++ b/.github/workflows/publish-bulk-cdk.yml @@ -25,7 +25,7 @@ env: jobs: publish-bulk-cdk: name: Publish Bulk CDK - runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings timeout-minutes: 30 steps: - name: Checkout Airbyte diff --git a/.github/workflows/publish-java-cdk-command.yml b/.github/workflows/publish-java-cdk-command.yml index 9a707d468012..82a9ffac5636 100644 --- a/.github/workflows/publish-java-cdk-command.yml +++ b/.github/workflows/publish-java-cdk-command.yml @@ -61,7 +61,7 @@ env: jobs: publish-cdk: name: Publish Java CDK - runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings timeout-minutes: 30 steps: - name: Link comment to Workflow Run diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index a56d21f636a7..9008e5d0eae5 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -44,7 +44,7 @@ on: jobs: regression_tests: name: Regression Tests - runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings + runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings timeout-minutes: 360 # 6 hours steps: - name: Checkout Airbyte