ci: rebase #3235 onto main + shard the compressed allow-list runs

.github/actions/build-packages/action.yml

@@ -4,8 +4,27 @@ description: "Compiles and builds Go services."
 runs:
   using: "composite"
   steps:
-    - name: Setup Go
-      uses: ./.github/actions/go-setup-cache
+    - name: Set up Go
+      uses: actions/setup-go@v6
+      with:
+        go-version-file: go.work
+        # Caching is handled explicitly below. setup-go keys its cache purely
+        # on the dependency files, so it could never store a build cache for
+        # the optimized (DEBUG_GCFLAGS="") binaries this job builds - every
+        # run would restore stale artifacts and recompile everything.
+        cache: false
+
+    - name: Restore Go caches
+      uses: actions/cache@v5
+      with:
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
+        # The "opt" salt separates this cache from setup-go caches populated
+        # by other jobs with debugger-friendly build flags.
+        key: integration-go-opt-v1-${{ runner.os }}-${{ hashFiles('go.work', 'packages/*/go.mod', 'packages/*/go.sum', 'tests/integration/go.mod', 'tests/integration/go.sum') }}
+        restore-keys: |
+          integration-go-opt-v1-${{ runner.os }}-
 
     # Built concurrently: the runner has 32 cores and 128 GB, while the two
     # slow builds (api, orchestrator) are mostly serial link steps that leave

.github/actions/build-sandbox-template/action.yml

@@ -26,6 +26,9 @@ inputs:
 runs:
   using: "composite"
   steps:
+    # Note: this cannot run concurrently with a running orchestrator -
+    # create-build manages its own sandbox network state (slot pools,
+    # namespaces) and the two processes conflict, wedging the provisioning VM.
     - name: Build Sandbox Template
       env:
         TEMPLATE_ID: "2j6ly824owf4awgai1xo"

.github/actions/start-databases/action.yml

@@ -0,0 +1,57 @@
+name: "Start Databases"
+description: "Kicks off the PostgreSQL, ClickHouse, and Redis containers in the background. Run this as early as possible: pulls and boots then overlap with the Go builds, and the start-services action later waits for health before migrating."
+
+runs:
+  using: "composite"
+  steps:
+    - name: Start Database Containers
+      env:
+        CLICKHOUSE_USERNAME: "e2b"
+        CLICKHOUSE_PASSWORD: "clickity-clicky-click"
+        CLICKHOUSE_PORT: "9000"
+        CLICKHOUSE_DATABASE: "default"
+      run: |
+        # Everything is backgrounded so the image pulls (with rate-limit
+        # retries, see scripts/pull-retry.sh) and container boots overlap
+        # with the (minutes-long) package builds that follow. Failures are
+        # not silent: a container whose pull or boot failed here never turns
+        # healthy, and start-services waits on container health with a
+        # bounded timeout.
+        (
+          ./scripts/pull-retry.sh postgres:18
+          docker run -d --name postgres \
+                        -e POSTGRES_USER=postgres \
+                        -e POSTGRES_PASSWORD=local \
+                        -e POSTGRES_DB=mydatabase \
+                        -p 5432:5432 \
+                        --health-cmd="pg_isready -U postgres" \
+                        --health-interval=5s \
+                        --health-timeout=2s \
+                        --health-retries=5 \
+                        postgres:18
+        ) &
+
+        (
+          ./scripts/pull-retry.sh redis:8
+          docker run -d --name redis \
+                -p 6379:6379 \
+                --health-cmd="redis-cli ping" \
+                --health-interval=5s \
+                --health-timeout=2s \
+                --health-retries=5 \
+                redis:8
+        ) &
+
+        (
+          ./scripts/pull-retry.sh clickhouse/clickhouse-server:25.4.5.24
+          make -C packages/clickhouse run
+        ) &
+
+        # Warm-up only: this discards its exit status by design (the step
+        # must not block on it). The guarantee lives in start-services,
+        # which re-runs pull-retry.sh for this image synchronously before
+        # starting the otel collector. The tag comes from the Makefile that
+        # runs the image, so a version bump there can't strand these pulls.
+        OTEL_IMAGE=$(grep -oE 'otel/opentelemetry-collector-contrib:[0-9.]+' packages/otel-collector/Makefile | head -1)
+        ./scripts/pull-retry.sh "$OTEL_IMAGE" &
+      shell: bash

.github/actions/start-services/action.yml

@@ -30,41 +30,42 @@ inputs:
 runs:
   using: "composite"
   steps:
-    # Anonymous Docker Hub pulls from shared runners intermittently fail with
-    # "unauthorized: authentication required" (Hub rate limiting); the implicit
-    # pulls inside the service steps then abort the whole job. Pull everything
-    # up front with retries so a transient 401/429 can't kill the run.
-    - name: Pre-pull Docker Hub images
-      shell: bash
-      run: |
-        for image in postgres:latest redis:latest otel/opentelemetry-collector-contrib:0.146.0 clickhouse/clickhouse-server:25.4.5.24; do
-          for attempt in 1 2 3 4 5; do
-            docker pull --quiet "$image" && break
-            if [ "$attempt" = 5 ]; then echo "giving up on $image"; exit 1; fi
-            echo "pull of $image failed (attempt $attempt), retrying in $((attempt * 10))s..."
-            sleep $((attempt * 10))
-          done
-        done
-
-    - name: Run PostgreSQL Database
+    # The postgres/clickhouse/redis containers were already kicked off in the
+    # background by the start-databases action (early in the job, so their
+    # pulls and boots overlapped with the package builds). Here we wait for
+    # database health and migrate. ClickHouse migrations run goose directly
+    # on the host (migrate-host): building the migrator image took ~2 min per
+    # job even with layer caching, for a container whose only purpose is to
+    # run goose against migrations that get volume-mounted anyway.
+    - name: Wait for Databases and Migrate
       env:
         TESTS_E2B_API_KEY: "e2b_5ec17bd3933af21f80dc10bba686691c4fcd7057"
         TESTS_E2B_ACCESS_TOKEN: "sk_e2b_17bd3933af21f80dc10bba686691c4fcd7057123"
         TESTS_SANDBOX_TEAM_ID: "834777bd-9956-45ca-b088-9bac9290e2ac"
         TESTS_SANDBOX_USER_ID: "2a5a9fc5-db8d-4af7-ac9e-d0f9272463bc"
+        CLICKHOUSE_USERNAME: "e2b"
+        CLICKHOUSE_PASSWORD: "clickity-clicky-click"
+        CLICKHOUSE_PORT: "9000"
+        CLICKHOUSE_DATABASE: "default"
       run: |
-        docker run -d --name postgres \
-                      -e POSTGRES_USER=postgres \
-                      -e POSTGRES_PASSWORD=local \
-                      -e POSTGRES_DB=mydatabase \
-                      -p 5432:5432 \
-                      --health-cmd="pg_isready -U postgres" \
-                      --health-interval=5s \
-                      --health-timeout=2s \
-                      --health-retries=5 \
-                      postgres:latest
-        while [ "$(docker inspect -f '{{.State.Health.Status}}' postgres 2>/dev/null)" != "healthy" ]; do echo "Waiting for PostgreSQL to be healthy..."; sleep 2; done
-        echo "PostgreSQL is healthy!"
+        wait_healthy() {
+          local name="$1" timeout="${2:-120}" i
+          for ((i = 0; i < timeout; i += 2)); do
+            if [ "$(docker inspect -f '{{.State.Health.Status}}' "$name" 2>/dev/null)" = "healthy" ]; then
+              echo "$name is healthy!"
+              return 0
+            fi
+            echo "Waiting for $name to be healthy..."
+            sleep 2
+          done
+          echo "::error::$name did not become healthy within ${timeout}s"
+          docker logs "$name" 2>&1 | tail -50 || true
+          return 1
+        }
+
+        wait_healthy postgres 120
+        wait_healthy clickhouse 120
+        wait_healthy redis 60
 
         # Install extensions
         docker exec postgres psql -U postgres -d mydatabase -c "CREATE SCHEMA extensions; CREATE EXTENSION IF NOT EXISTS pgcrypto SCHEMA extensions;"
@@ -74,60 +75,18 @@ runs:
         echo "TESTS_E2B_ACCESS_TOKEN=${TESTS_E2B_ACCESS_TOKEN}" >> .env.test
         echo "TESTS_SANDBOX_TEAM_ID=${TESTS_SANDBOX_TEAM_ID}" >> .env.test
         echo "TESTS_SANDBOX_USER_ID=${TESTS_SANDBOX_USER_ID}" >> .env.test
-        set -x
-        make migrate
-        make -C tests/integration seed
-      shell: bash
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v4
-
-    - name: Build codegen image with caching
-      uses: docker/build-push-action@v7
-      with:
-        context: packages/clickhouse
-        file: packages/clickhouse/Dockerfile
-        tags: clickhouse-migrator:latest
-        load: true # makes the image available for `docker run`
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
-
-    - name: Run Clickhouse
-      env:
-        CLICKHOUSE_USERNAME: "e2b"
-        CLICKHOUSE_PASSWORD: "clickity-clicky-click"
-        CLICKHOUSE_PORT: "9000"
-        CLICKHOUSE_DATABASE: "default"
-      run: |
-        echo "CLICKHOUSE_MIGRATOR_IMAGE=clickhouse-migrator" >> .env.test
+        echo "REDIS_URL=localhost:6379" >> .env.test
         echo "CLICKHOUSE_USERNAME=${CLICKHOUSE_USERNAME}" >> .env.test
         echo "CLICKHOUSE_PORT=${CLICKHOUSE_PORT}" >> .env.test
         echo "CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}" >> .env.test
         echo "CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE}" >> .env.test
         echo "CLICKHOUSE_CONNECTION_STRING=clickhouse://${CLICKHOUSE_USERNAME}:${CLICKHOUSE_PASSWORD}@$localhost:${CLICKHOUSE_PORT}/${CLICKHOUSE_DATABASE}" >> .env.test
 
-        make -C packages/clickhouse run &
-        while [ "$(docker inspect -f '{{.State.Health.Status}}' clickhouse 2>/dev/null)" != "healthy" ]; do echo "Waiting for Clickhouse to be healthy..."; sleep 2; done
-        echo "Clickhouse is healthy!"
-
-        # We build the image in separate step to cache it and avoid rebuilding it every time
-        make -C packages/clickhouse migrate-without-build
-      shell: bash
-
-    - name: Run Redis
-      run: |
-        docker run -d --name redis \
-              -p 6379:6379 \
-              --health-cmd="redis-cli ping" \
-              --health-interval=5s \
-              --health-timeout=2s \
-              --health-retries=5 \
-              redis:latest
-
-        while [ "$(docker inspect -f '{{.State.Health.Status}}' redis 2>/dev/null)" != "healthy" ]; do echo "Waiting for Redis to be healthy..."; sleep 2; done
-        echo "Redis is healthy!"
+        set -x
+        make migrate
+        make -C tests/integration seed
 
-        echo "REDIS_URL=localhost:6379" >> .env.test
+        make -C packages/clickhouse migrate-host
       shell: bash
 
     - name: Start Services
@@ -162,6 +121,14 @@ runs:
         PERSISTENT_VOLUME_MOUNTS="test-volume-type:${test_volume_dir}"
         export PERSISTENT_VOLUME_MOUNTS
 
+        # The backgrounded warm-up pull in start-databases discards its exit
+        # status; this synchronous call is the actual guarantee (near-instant
+        # when the warm-up succeeded) so a rate-limited pull retries instead
+        # of failing the collector's 30s health window. The tag comes from
+        # the Makefile that runs the image.
+        OTEL_IMAGE=$(grep -oE 'otel/opentelemetry-collector-contrib:[0-9.]+' packages/otel-collector/Makefile | head -1)
+        ./scripts/pull-retry.sh "$OTEL_IMAGE"
+
         echo "Start otel-collector"
         ./scripts/start-service.sh "OtelCollector" packages/otel-collector run ~/logs/otel-collector.log http://localhost:13133/healthz
 

.github/workflows/integration_tests.yml

@@ -12,6 +12,11 @@ on:
         description: "Whether to run integration tests"
         required: false
         default: true
+      full-matrix:
+        type: boolean
+        description: "When true (main), run all three compression configs unsharded. When false (PRs), split the full-suite uncompressed config into package shards on parallel runners to cut wall time; the compressed configs run their allow-list unsharded either way."
+        required: false
+        default: true
     secrets:
       CODECOV_TOKEN: { required: false }
 jobs:
@@ -25,36 +30,62 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - name: uncompressed
-            compress_enabled: "false"
-            compress_type: ""
-            compress_level: ""
-            compress_workers: ""
-            dedup_mode: "default"
-            disable_memfd: "true"
-          - name: zstd1
-            compress_enabled: "true"
-            compress_type: "zstd"
-            compress_level: "1"
-            compress_workers: "8"
-            dedup_mode: "best_effort"
-            disable_memfd: "false"
-          - name: lz4
-            compress_enabled: "true"
-            compress_type: "lz4"
-            compress_level: "0"
-            compress_workers: "8"
-            dedup_mode: "direct_io"
-            disable_memfd: "false"
+        # Coverage is split along two independent axes:
+        #
+        #   - Which tests run per config: the uncompressed config runs the
+        #     whole suite; the compressed configs (zstd1/lz4) only re-run the
+        #     allow-listed tests whose subject is writing or reading back a
+        #     snapshot, which is all the compression knobs can affect (see
+        #     scripts/compression-tests.tsv).
+        #   - Sharding for wall time (PRs only): each host saturates at
+        #     go test -parallel=4 (Firecracker boots + compression), so the
+        #     only way to cut wall time is more hosts. The full-suite
+        #     uncompressed config is split into package shards - the
+        #     TestTemplateBuild* tests (real FC builds) in two halves by name
+        #     prefix, api/sandboxes plus the light packages on their own
+        #     runner, everything else on the last one. The compressed
+        #     allow-list runs are split at the package level: the templates
+        #     package (real-build snapshot tests) vs everything else. Shards
+        #     are resolved by TEST_SHARD in tests/integration/Makefile.
+        #
+        # On push to main (full-matrix), all three configs run unsharded, one
+        # job per config, as before.
+        include: >-
+          ${{ fromJSON(inputs.full-matrix
+            && '[
+                  {"name": "uncompressed", "shard": "all", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"},
+                  {"name": "zstd1", "shard": "all", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
+                  {"name": "lz4", "shard": "all", "compress_enabled": "true", "compress_type": "lz4", "compress_level": "0", "compress_workers": "8", "dedup_mode": "direct_io", "disable_memfd": "false"}
+                ]'
+            || '[
+                  {"name": "uncompressed-templates-1", "shard": "templates-builds-1", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"},
+                  {"name": "uncompressed-templates-2", "shard": "templates-builds-2", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"},
+                  {"name": "uncompressed-sandboxes", "shard": "sandboxes", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"},
+                  {"name": "uncompressed-rest", "shard": "rest", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"},
+                  {"name": "zstd1-templates", "shard": "templates", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
+                  {"name": "zstd1-other", "shard": "no-templates", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
+                  {"name": "lz4-templates", "shard": "templates", "compress_enabled": "true", "compress_type": "lz4", "compress_level": "0", "compress_workers": "8", "dedup_mode": "direct_io", "disable_memfd": "false"},
+                  {"name": "lz4-other", "shard": "no-templates", "compress_enabled": "true", "compress_type": "lz4", "compress_level": "0", "compress_workers": "8", "dedup_mode": "direct_io", "disable_memfd": "false"}
+                ]') }}
     env:
       # Surfaced as env so upload steps can gate on presence (skipped on fork PRs).
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      # Build race-instrumented but optimized binaries: nobody attaches a
+      # debugger in CI, and -N -l slows both compilation and every test at
+      # runtime. Set at the job level so the build-packages step and the
+      # rebuilds inside the run/run-debug targets use identical flags (the
+      # second build is then a Go build cache no-op).
+      DEBUG_GCFLAGS: ""
 
     steps:
       - name: Checkout Code
         uses: actions/checkout@v5
 
+      # Kick off the database containers first: their image pulls and boots
+      # run in the background and overlap with the package builds below.
+      - name: Start Databases
+        uses: ./.github/actions/start-databases
+
       # The compressed configs only run what the allow-list names, so verify
       # its stale-entry guard still bites before depending on it.
       - name: Check Test Allow-list
@@ -67,6 +98,10 @@ jobs:
       - name: Initialize Host
         uses: ./.github/actions/host-init
 
+      # Must run before Start Services: create-build manages its own sandbox
+      # network state (slot pools, namespaces) and conflicts with a running
+      # orchestrator doing the same - overlapping them wedges the
+      # provisioning VM until its deadline.
       - name: Build Template
         uses: ./.github/actions/build-sandbox-template
         with:
@@ -97,6 +132,15 @@ jobs:
           # snapshot, which is all the compression knobs can affect — the
           # allow-list documents the criteria and the code paths involved.
           TESTS_ONLY: ${{ matrix.compress_enabled == 'true' && 'scripts/compression-tests.tsv' || '' }}
+          # Keep go test -parallel at 4: the host is the bottleneck, not
+          # client-side concurrency. Measured on the zstd1 config: at 8 the
+          # per-test times inflated 70-140% (net wall win of only ~1 min) and
+          # 10 template builds flaked on timeouts; at 12 concurrent FC builds
+          # (x8 zstd workers each) starved envd inits into "syncing took too
+          # long" failures across 87 tests. Wall time is cut by sharding
+          # across runners instead (see the matrix above).
+          TEST_PARALLELISM: "4"
+          TEST_SHARD: ${{ matrix.shard }}
         run: |
           # Run the integration tests
           make test-integration

.github/workflows/pull-request.yml

@@ -58,10 +58,13 @@ jobs:
               - '.tool-versions'
               # invoked by the start-services action to launch each service
               - 'scripts/start-service.sh'
+              # invoked by start-databases and start-services to pull images
+              - 'scripts/pull-retry.sh'
               - '.github/workflows/integration_tests.yml'
               - '.github/actions/build-packages/**'
               - '.github/actions/build-sandbox-template/**'
               - '.github/actions/host-init/**'
+              - '.github/actions/start-databases/**'
               - '.github/actions/start-services/**'
               - '.github/actions/go-setup-cache/**'
             lint-inputs:
@@ -164,6 +167,10 @@ jobs:
       # Only publish the results for same-repo PRs
       publish: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
       run-tests: ${{ contains(fromJSON(needs.detect-changes.outputs.changed-scopes), 'integration-inputs') && needs.detect-changes.outputs.docs-only != 'true' }}
+      # PRs shard the full-suite uncompressed config across parallel runners
+      # for wall time (the compressed configs run their allow-list unsharded
+      # either way); push to main runs all three configs unsharded.
+      full-matrix: false
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   publish-test-results: