Fix botocore endpoint resolution by preserving top-level data files

.builders/scripts/files_to_remove.toml

@@ -56,4 +56,7 @@ excluded_paths = [
   "!botocore/data/rds",
   # Riakcs integration monitors s3.
   "!botocore/data/s3",
+  # Top-level JSON files (endpoints.json, partitions.json, _retry.json, etc.)
+  # are required by botocore to resolve AWS endpoints and handle retries.
+  "!botocore/data/*.json",
 ]

.github/workflows/resolve-build-deps.yaml

@@ -2,6 +2,11 @@ name: Resolve Dependencies and Build Wheels
 
 on:
   workflow_dispatch:
+    inputs:
+      builder_changed:
+        description: 'Rebuild the builder image (use when .builders/ changed)'
+        type: boolean
+        default: false
   pull_request:
     branches:
     - master
@@ -74,6 +79,7 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           FILES_CHANGED: ${{ steps.changed-files.outputs.files_changed }}
+          INPUT_BUILDER_CHANGED: ${{ inputs.builder_changed }}
 
   test:
     name: Run tests
@@ -167,7 +173,7 @@ jobs:
 
     env:
       OUT_DIR: output/${{ matrix.job.image }}
-      BUILDER_IMAGE: ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }}
+      BUILDER_IMAGE: ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }}-botocore-endpoint-fix
       DOCKER: docker
 
     steps:
@@ -208,17 +214,6 @@ jobs:
         digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json)
         python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest --constraints .constraints/constraints.txt
 
-    - name: Publish image
-      if: github.event_name == 'push' && needs.check-should-run.outputs.builder_changed == 'true'
-      run: ${DOCKER} push ${{ env.BUILDER_IMAGE }}
-
-    - name: Save new image digest
-      if: github.event_name == 'push' && needs.check-should-run.outputs.builder_changed == 'true'
-      run: >-
-        ${DOCKER} inspect --format "{{index .RepoDigests 0}}" ${{ env.BUILDER_IMAGE }}
-        | cut -d '@' -f 2
-        > ${{ env.OUT_DIR }}/image_digest
-
     - name: Persist current image digest
       if: needs.check-should-run.outputs.builder_changed == 'false'
       run: >-
@@ -323,7 +318,7 @@ jobs:
 
   publish:
     name: Publish artifacts and update lockfiles via PR
-    if: needs.check-should-run.outputs.should_run_build == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.'))))
+    if: needs.check-should-run.outputs.should_run_build == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.') || github.ref_name == 'nubtron/custom-build/botocore-endpoint-fix')))
     needs:
     - build
     - build-macos
@@ -404,6 +399,7 @@ jobs:
       uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
       with:
         token: ${{ steps.token-generator.outputs.token }}
+        base: nubtron/custom-build/botocore-endpoint-fix
         title: Update dependency resolution
         commit-message: Update dependency resolution
         branch: bot/update-dependency-resolution

.github/workflows/scripts/resolve_deps_check_should_run.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 if [ "$GITHUB_EVENT_NAME" == 'workflow_dispatch' ]; then
-  builder_changed="false"
+  builder_changed="${INPUT_BUILDER_CHANGED:-false}"
   should_run_build="true"
 else