diff --git a/.builders/scripts/files_to_remove.toml b/.builders/scripts/files_to_remove.toml index 2907bfe7c987f..8c79a65fe2bd4 100644 --- a/.builders/scripts/files_to_remove.toml +++ b/.builders/scripts/files_to_remove.toml @@ -56,4 +56,7 @@ excluded_paths = [ "!botocore/data/rds", # Riakcs integration monitors s3. "!botocore/data/s3", + # Top-level JSON files (endpoints.json, partitions.json, _retry.json, etc.) + # are required by botocore to resolve AWS endpoints and handle retries. + "!botocore/data/*.json", ] diff --git a/.github/workflows/resolve-build-deps.yaml b/.github/workflows/resolve-build-deps.yaml index 26553000058ce..886f2585c231f 100644 --- a/.github/workflows/resolve-build-deps.yaml +++ b/.github/workflows/resolve-build-deps.yaml @@ -2,6 +2,11 @@ name: Resolve Dependencies and Build Wheels on: workflow_dispatch: + inputs: + builder_changed: + description: 'Rebuild the builder image (use when .builders/ changed)' + type: boolean + default: false pull_request: branches: - master @@ -74,6 +79,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} FILES_CHANGED: ${{ steps.changed-files.outputs.files_changed }} + INPUT_BUILDER_CHANGED: ${{ inputs.builder_changed }} test: name: Run tests @@ -167,7 +173,7 @@ jobs: env: OUT_DIR: output/${{ matrix.job.image }} - BUILDER_IMAGE: ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }} + BUILDER_IMAGE: ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }}-botocore-endpoint-fix DOCKER: docker steps: @@ -208,17 +214,6 @@ jobs: digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json) python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest --constraints .constraints/constraints.txt - - name: Publish image - if: github.event_name == 'push' && needs.check-should-run.outputs.builder_changed == 'true' - run: ${DOCKER} push ${{ env.BUILDER_IMAGE }} - - - name: Save new image digest - if: github.event_name == 'push' && needs.check-should-run.outputs.builder_changed == 'true' - run: >- - ${DOCKER} inspect --format "{{index .RepoDigests 0}}" ${{ env.BUILDER_IMAGE }} - | cut -d '@' -f 2 - > ${{ env.OUT_DIR }}/image_digest - - name: Persist current image digest if: needs.check-should-run.outputs.builder_changed == 'false' run: >- @@ -323,7 +318,7 @@ jobs: publish: name: Publish artifacts and update lockfiles via PR - if: needs.check-should-run.outputs.should_run_build == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.')))) + if: needs.check-should-run.outputs.should_run_build == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.') || github.ref_name == 'nubtron/custom-build/botocore-endpoint-fix'))) needs: - build - build-macos @@ -404,6 +399,7 @@ jobs: uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0 with: token: ${{ steps.token-generator.outputs.token }} + base: nubtron/custom-build/botocore-endpoint-fix title: Update dependency resolution commit-message: Update dependency resolution branch: bot/update-dependency-resolution diff --git a/.github/workflows/scripts/resolve_deps_check_should_run.sh b/.github/workflows/scripts/resolve_deps_check_should_run.sh index b036997b869ab..f2ed458bdf792 100755 --- a/.github/workflows/scripts/resolve_deps_check_should_run.sh +++ b/.github/workflows/scripts/resolve_deps_check_should_run.sh @@ -1,7 +1,7 @@ #!/bin/bash if [ "$GITHUB_EVENT_NAME" == 'workflow_dispatch' ]; then - builder_changed="false" + builder_changed="${INPUT_BUILDER_CHANGED:-false}" should_run_build="true" else