fix: throttling issue (#654)

* fix: throttling issue

* fix: slow down docker-retry backoff to 1/2/5 min with jitter

public.ecr.aws / Docker Hub rate-limit windows recover on a minute-scale
timer, so the previous 5s..60s exponential ladder tended to exhaust its
retry budget before the throttle slot reopened. Replace with a fixed
1min / 2min / 5min ladder + additive [0,30]s jitter.

---------

Co-authored-by: Davide Melfi <darklight3it@gmail.com>

Author: maxday

.github/workflows/runtime-interface-client_pr.yml

@@ -136,6 +136,15 @@ jobs:
               platform: linux/arm64/v8
     name: "integration-test (${{ matrix.buildspec }} / ${{ matrix.arch.label }})"
     steps:
+    - name: Jitter start (stagger matrix shards to spare public.ecr.aws rate limit)
+      # Every shard is scheduled within milliseconds of the others and hits
+      # public.ecr.aws / docker hub inside the same rate-limit window, so they
+      # collectively trip the throttle.
+      run: |
+        jitter=$((RANDOM % 61))
+        echo "Sleeping ${jitter}s to stagger this shard"
+        sleep "${jitter}"
+
     - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
 
     - name: Set up Docker Buildx

aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/docker-retry.sh

@@ -3,9 +3,14 @@
 
 set -uo pipefail
 
-MAX_ATTEMPTS="${RETRY_MAX_ATTEMPTS:-5}"
-BASE_DELAY="${RETRY_BASE_DELAY:-5}"
-MAX_DELAY="${RETRY_MAX_DELAY:-60}"
+# Ladder of base wait times (in seconds) between successive attempts.
+# Default: 1 min, 2 min, 5 min — 4 attempts total (initial + 3 retries).
+# public.ecr.aws / Docker Hub rate-limit windows recover on a minute-scale
+# timer, so short exponential backoff (5s..60s) tends to exhaust its budget
+# before the rate-limit slot reopens.
+IFS=' ' read -r -a DELAYS <<< "${RETRY_DELAYS:-60 120 300}"
+JITTER_MAX="${RETRY_JITTER_MAX:-30}"
+MAX_ATTEMPTS=$(( ${#DELAYS[@]} + 1 ))
 
 if (( $# == 0 )); then
     >&2 echo "usage: docker-retry.sh <command> [args...]"
@@ -22,14 +27,13 @@ while true; do
         exit "$status"
     fi
 
-    # Exponential backoff: BASE_DELAY * 2^(attempt-1), capped at MAX_DELAY.
-    backoff=$(( BASE_DELAY * (2 ** (attempt - 1)) ))
-    (( backoff > MAX_DELAY )) && backoff=$MAX_DELAY
-    # Full jitter: wait a random duration in [0, backoff] so concurrent jobs
-    # spread out instead of retrying at the same moment.
-    delay=$(( RANDOM % (backoff + 1) ))
+    # Base delay from the ladder + additive jitter in [0, JITTER_MAX] so
+    # concurrent shards don't retry in the exact same second.
+    base=${DELAYS[$((attempt - 1))]}
+    jitter=$(( RANDOM % (JITTER_MAX + 1) ))
+    delay=$(( base + jitter ))
 
-    >&2 echo "docker-retry: '$*' failed (exit ${status}); attempt ${attempt}/${MAX_ATTEMPTS}, retrying in ${delay}s."
+    >&2 echo "docker-retry: '$*' failed (exit ${status}); attempt ${attempt}/${MAX_ATTEMPTS}, retrying in ${delay}s (${base}s + ${jitter}s jitter)."
     sleep "$delay"
     attempt=$(( attempt + 1 ))
 done

aws-lambda-java-runtime-interface-client/test/integration/codebuild-local/test_one.sh

@@ -62,7 +62,10 @@ main() {
     mkdir -p "$ARTIFACTS_DIR"
 
     LOCAL_AGENT_IMAGE="$(get_local_agent_image)"
-    "$(dirname "$0")"/docker-retry.sh docker pull "$LOCAL_AGENT_IMAGE" || true
+
+    if ! docker image inspect "$LOCAL_AGENT_IMAGE" >/dev/null 2>&1; then
+        "$(dirname "$0")"/docker-retry.sh docker pull "$LOCAL_AGENT_IMAGE"
+    fi
 
     # Run CodeBuild local agent.
     "$(dirname "$0")"/codebuild_build.sh \