From ea47dab30b7c550a5418828279838a829edd2226 Mon Sep 17 00:00:00 2001
From: Jakub Dobry <jakub.dobry8@gmail.com>
Date: Wed, 8 Jul 2026 16:13:49 -0700
Subject: [PATCH 1/8] ci: speed up integration tests

PR wall time for integration tests is ~21 min; on a recent successful run
the zstd1 shard spent 2m26s building packages, ~4m on host init/template/
services, and 14m07s running tests. Several compounding causes:

- The full suite (incl. the whole build+services pipeline) ran 3x per PR
  in the compression matrix, though most tests don't depend on
  compression/dedup/memfd settings. PRs now run only the production-like
  zstd1 config via a new full-matrix workflow input; push to main keeps
  all three configs.
- go test ran with -parallel=4 while the harness allows 100 concurrent
  sandboxes; api/sandboxes alone is ~1250s of test time squeezed through
  4 slots. Parallelism is now tunable via TEST_PARALLELISM (4 locally,
  12 in CI).
- All debug builds used -race -gcflags=all="-N -l"; disabling
  optimizations slows compilation and every test at runtime (envd runs
  inside every sandbox). -N -l is now opt-out via DEBUG_GCFLAGS, empty
  in CI, and set at the job level so the rebuilds inside run/run-debug
  targets become Go build cache no-ops.
- postgres/clickhouse/redis started serially, each with an unbounded
  sleep-2 health loop, on unpinned images. They now start concurrently
  (overlapping the migrator image build), with bounded health waits and
  pinned tags.
- 7 template tests built full Firecracker templates (~60-160s each) only
  to exercise metadata/authz paths (visibility updates, cross-team 403/
  404, default-tag deletion guard) that don't depend on build state.
  They now use a new RequestTemplateWithoutBuild helper (single API
  call): the env row, waiting build, alias, and default tag assignment
  all exist at request time, and PATCH/list/delete don't filter on
  build status.
- envd process tests slept a fixed 10s/5s waiting for processes to
  start; they now poll the process list with EventuallyWithT.
---
 .github/actions/start-services/action.yml     | 93 +++++++++++--------
 .github/workflows/integration_tests.yml       | 52 ++++++-----
 .github/workflows/pull-request.yml            |  3 +
 packages/api/Makefile                         |  8 +-
 packages/client-proxy/Makefile                |  8 +-
 packages/envd/Makefile                        |  8 +-
 packages/orchestrator/Makefile                |  8 +-
 tests/integration/Makefile                    | 12 ++-
 .../api/templates/delete_template_test.go     | 15 +--
 .../tests/api/templates/template_tags_test.go |  5 +-
 .../api/templates/template_update_test.go     | 14 +--
 .../internal/tests/envd/process_test.go       | 28 +++---
 tests/integration/internal/utils/template.go  | 16 ++++
 13 files changed, 169 insertions(+), 101 deletions(-)

diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml
index 5e57f3033b..b44dbaca88 100644
--- a/.github/actions/start-services/action.yml
+++ b/.github/actions/start-services/action.yml
@@ -30,12 +30,15 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Run PostgreSQL Database
+    # Start all database containers up front without waiting, so they boot
+    # concurrently (and overlap with the migrator image build below) instead
+    # of serially blocking on each other's health checks.
+    - name: Start Database Containers
       env:
-        TESTS_E2B_API_KEY: "e2b_5ec17bd3933af21f80dc10bba686691c4fcd7057"
-        TESTS_E2B_ACCESS_TOKEN: "sk_e2b_17bd3933af21f80dc10bba686691c4fcd7057123"
-        TESTS_SANDBOX_TEAM_ID: "834777bd-9956-45ca-b088-9bac9290e2ac"
-        TESTS_SANDBOX_USER_ID: "2a5a9fc5-db8d-4af7-ac9e-d0f9272463bc"
+        CLICKHOUSE_USERNAME: "e2b"
+        CLICKHOUSE_PASSWORD: "clickity-clicky-click"
+        CLICKHOUSE_PORT: "9000"
+        CLICKHOUSE_DATABASE: "default"
       run: |
         docker run -d --name postgres \
                       -e POSTGRES_USER=postgres \
@@ -46,21 +49,17 @@ runs:
                       --health-interval=5s \
                       --health-timeout=2s \
                       --health-retries=5 \
-                      postgres:latest
-        while [ "$(docker inspect -f '{{.State.Health.Status}}' postgres 2>/dev/null)" != "healthy" ]; do echo "Waiting for PostgreSQL to be healthy..."; sleep 2; done
-        echo "PostgreSQL is healthy!"
+                      postgres:18
 
-        # Install extensions
-        docker exec postgres psql -U postgres -d mydatabase -c "CREATE SCHEMA extensions; CREATE EXTENSION IF NOT EXISTS pgcrypto SCHEMA extensions;"
-        echo "extensions installed"
+        docker run -d --name redis \
+              -p 6379:6379 \
+              --health-cmd="redis-cli ping" \
+              --health-interval=5s \
+              --health-timeout=2s \
+              --health-retries=5 \
+              redis:8
 
-        echo "TESTS_E2B_API_KEY=${TESTS_E2B_API_KEY}" >> .env.test
-        echo "TESTS_E2B_ACCESS_TOKEN=${TESTS_E2B_ACCESS_TOKEN}" >> .env.test
-        echo "TESTS_SANDBOX_TEAM_ID=${TESTS_SANDBOX_TEAM_ID}" >> .env.test
-        echo "TESTS_SANDBOX_USER_ID=${TESTS_SANDBOX_USER_ID}" >> .env.test
-        set -x
-        make migrate
-        make -C tests/integration seed
+        make -C packages/clickhouse run &
       shell: bash
 
     - name: Set up Docker Buildx
@@ -76,13 +75,45 @@ runs:
         cache-from: type=gha
         cache-to: type=gha,mode=max
 
-    - name: Run Clickhouse
+    - name: Wait for Databases and Migrate
       env:
+        TESTS_E2B_API_KEY: "e2b_5ec17bd3933af21f80dc10bba686691c4fcd7057"
+        TESTS_E2B_ACCESS_TOKEN: "sk_e2b_17bd3933af21f80dc10bba686691c4fcd7057123"
+        TESTS_SANDBOX_TEAM_ID: "834777bd-9956-45ca-b088-9bac9290e2ac"
+        TESTS_SANDBOX_USER_ID: "2a5a9fc5-db8d-4af7-ac9e-d0f9272463bc"
         CLICKHOUSE_USERNAME: "e2b"
         CLICKHOUSE_PASSWORD: "clickity-clicky-click"
         CLICKHOUSE_PORT: "9000"
         CLICKHOUSE_DATABASE: "default"
       run: |
+        wait_healthy() {
+          local name="$1" timeout="${2:-120}" i
+          for ((i = 0; i < timeout; i += 2)); do
+            if [ "$(docker inspect -f '{{.State.Health.Status}}' "$name" 2>/dev/null)" = "healthy" ]; then
+              echo "$name is healthy!"
+              return 0
+            fi
+            echo "Waiting for $name to be healthy..."
+            sleep 2
+          done
+          echo "::error::$name did not become healthy within ${timeout}s"
+          docker logs "$name" 2>&1 | tail -50 || true
+          return 1
+        }
+
+        wait_healthy postgres 120
+        wait_healthy clickhouse 120
+        wait_healthy redis 60
+
+        # Install extensions
+        docker exec postgres psql -U postgres -d mydatabase -c "CREATE SCHEMA extensions; CREATE EXTENSION IF NOT EXISTS pgcrypto SCHEMA extensions;"
+        echo "extensions installed"
+
+        echo "TESTS_E2B_API_KEY=${TESTS_E2B_API_KEY}" >> .env.test
+        echo "TESTS_E2B_ACCESS_TOKEN=${TESTS_E2B_ACCESS_TOKEN}" >> .env.test
+        echo "TESTS_SANDBOX_TEAM_ID=${TESTS_SANDBOX_TEAM_ID}" >> .env.test
+        echo "TESTS_SANDBOX_USER_ID=${TESTS_SANDBOX_USER_ID}" >> .env.test
+        echo "REDIS_URL=localhost:6379" >> .env.test
         echo "CLICKHOUSE_MIGRATOR_IMAGE=clickhouse-migrator" >> .env.test
         echo "CLICKHOUSE_USERNAME=${CLICKHOUSE_USERNAME}" >> .env.test
         echo "CLICKHOUSE_PORT=${CLICKHOUSE_PORT}" >> .env.test
@@ -90,30 +121,14 @@ runs:
         echo "CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE}" >> .env.test
         echo "CLICKHOUSE_CONNECTION_STRING=clickhouse://${CLICKHOUSE_USERNAME}:${CLICKHOUSE_PASSWORD}@$localhost:${CLICKHOUSE_PORT}/${CLICKHOUSE_DATABASE}" >> .env.test
 
-        make -C packages/clickhouse run &
-        while [ "$(docker inspect -f '{{.State.Health.Status}}' clickhouse 2>/dev/null)" != "healthy" ]; do echo "Waiting for Clickhouse to be healthy..."; sleep 2; done
-        echo "Clickhouse is healthy!"
+        set -x
+        make migrate
+        make -C tests/integration seed
 
-        # We build the image in separate step to cache it and avoid rebuilding it every time
+        # We build the image in a separate step to cache it and avoid rebuilding it every time
         make -C packages/clickhouse migrate-without-build
       shell: bash
 
-    - name: Run Redis
-      run: |
-        docker run -d --name redis \
-              -p 6379:6379 \
-              --health-cmd="redis-cli ping" \
-              --health-interval=5s \
-              --health-timeout=2s \
-              --health-retries=5 \
-              redis:latest
-
-        while [ "$(docker inspect -f '{{.State.Health.Status}}' redis 2>/dev/null)" != "healthy" ]; do echo "Waiting for Redis to be healthy..."; sleep 2; done
-        echo "Redis is healthy!"
-
-        echo "REDIS_URL=localhost:6379" >> .env.test
-      shell: bash
-
     - name: Start Services
       env:
         ENVD_TIMEOUT: "60s"
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 6d00df773b..c35fef709e 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -12,6 +12,11 @@ on:
         description: "Whether to run integration tests"
         required: false
         default: true
+      full-matrix:
+        type: boolean
+        description: "Run all compression configs. When false, only the production-like zstd1 config runs (used on PRs to save ~2/3 of runner time; the full matrix still runs on main)."
+        required: false
+        default: true
     secrets:
       CODECOV_TOKEN: { required: false }
 jobs:
@@ -25,31 +30,29 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - name: uncompressed
-            compress_enabled: "false"
-            compress_type: ""
-            compress_level: ""
-            compress_workers: ""
-            dedup_mode: "default"
-            disable_memfd: "true"
-          - name: zstd1
-            compress_enabled: "true"
-            compress_type: "zstd"
-            compress_level: "1"
-            compress_workers: "8"
-            dedup_mode: "best_effort"
-            disable_memfd: "false"
-          - name: lz4
-            compress_enabled: "true"
-            compress_type: "lz4"
-            compress_level: "0"
-            compress_workers: "8"
-            dedup_mode: "direct_io"
-            disable_memfd: "false"
+        # The full 3-config matrix (uncompressed/zstd1/lz4) runs on push to
+        # main. PRs run only zstd1 (the production-like config) since most
+        # tests don't depend on compression/dedup/memfd settings, and each
+        # shard repeats the whole build+services+suite pipeline (~20 min).
+        include: >-
+          ${{ fromJSON(inputs.full-matrix
+            && '[
+                  {"name": "uncompressed", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"},
+                  {"name": "zstd1", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
+                  {"name": "lz4", "compress_enabled": "true", "compress_type": "lz4", "compress_level": "0", "compress_workers": "8", "dedup_mode": "direct_io", "disable_memfd": "false"}
+                ]'
+            || '[
+                  {"name": "zstd1", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"}
+                ]') }}
     env:
       # Surfaced as env so upload steps can gate on presence (skipped on fork PRs).
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      # Build race-instrumented but optimized binaries: nobody attaches a
+      # debugger in CI, and -N -l slows both compilation and every test at
+      # runtime. Set at the job level so the build-packages step and the
+      # rebuilds inside the run/run-debug targets use identical flags (the
+      # second build is then a Go build cache no-op).
+      DEBUG_GCFLAGS: ""
 
     steps:
       - name: Checkout Code
@@ -86,6 +89,11 @@ jobs:
           TESTS_ORCHESTRATOR_HOST: "localhost:5008"
           TESTS_ENVD_PROXY: "http://localhost:3002"
           TESTS_CLIENT_PROXY: "http://localhost:3002"
+          # Nearly all tests are t.Parallel() and gated by a 100-slot sandbox
+          # semaphore in the harness; the go test default of -parallel=4 was
+          # the throughput bottleneck (e.g. api/sandboxes is ~1250s of test
+          # time squeezed through 4 slots). Tune down if flakiness increases.
+          TEST_PARALLELISM: "12"
         run: |
           # Run the integration tests
           make test-integration
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index e6bbbd2039..aa3c3ffb7f 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -164,6 +164,9 @@ jobs:
       # Only publish the results for same-repo PRs
       publish: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
       run-tests: ${{ contains(fromJSON(needs.detect-changes.outputs.changed-scopes), 'integration-inputs') && needs.detect-changes.outputs.docs-only != 'true' }}
+      # PRs run only the production-like zstd1 config; the full compression
+      # matrix runs on push to main.
+      full-matrix: false
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   publish-test-results:
diff --git a/packages/api/Makefile b/packages/api/Makefile
index 64fbbd91e4..1de78d953b 100644
--- a/packages/api/Makefile
+++ b/packages/api/Makefile
@@ -25,11 +25,17 @@ build:
 	$(eval EXPECTED_MIGRATION_TIMESTAMP ?= $(expectedMigration))
 	CGO_ENABLED=0 go build -o bin/api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" .
 
+# Compiler flags for debug builds. The default disables optimizations and
+# inlining so binaries are debugger-friendly. CI overrides this with an empty
+# value: race-instrumented but optimized binaries compile faster and speed up
+# every integration test at runtime.
+DEBUG_GCFLAGS ?= -gcflags=all="-N -l"
+
 .PHONY: build-debug
 build-debug:
 	$(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD))
 	$(eval EXPECTED_MIGRATION_TIMESTAMP ?= $(expectedMigration))
-	CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" .
+	CGO_ENABLED=1 go build -race $(DEBUG_GCFLAGS) -o bin/api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" .
 
 .PHONY: run
 run:
diff --git a/packages/client-proxy/Makefile b/packages/client-proxy/Makefile
index dd3510b122..949922a3aa 100644
--- a/packages/client-proxy/Makefile
+++ b/packages/client-proxy/Makefile
@@ -42,10 +42,16 @@ build:
 	$(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD))
 	CGO_ENABLED=0 GOOS=linux GOARCH=$(BUILD_ARCH) go build -o bin/client-proxy -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" .
 
+# Compiler flags for debug builds. The default disables optimizations and
+# inlining so binaries are debugger-friendly. CI overrides this with an empty
+# value: race-instrumented but optimized binaries compile faster and speed up
+# every integration test at runtime.
+DEBUG_GCFLAGS ?= -gcflags=all="-N -l"
+
 .PHONY: build-debug
 build-debug:
 	$(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD))
-	CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/client-proxy -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" .
+	CGO_ENABLED=1 go build -race $(DEBUG_GCFLAGS) -o bin/client-proxy -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" .
 
 .PHONY: build-and-upload
 build-and-upload:
diff --git a/packages/envd/Makefile b/packages/envd/Makefile
index de1723032d..ead1aed115 100644
--- a/packages/envd/Makefile
+++ b/packages/envd/Makefile
@@ -52,8 +52,14 @@ endif
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=$(BUILD_ARCH) go build -trimpath -buildvcs=false -a -o bin/envd ${PROD_LDFLAGS}
 
+# Compiler flags for debug builds. The default disables optimizations and
+# inlining so binaries are debugger-friendly. CI overrides this with an empty
+# value: race-instrumented but optimized binaries compile faster and speed up
+# every integration test at runtime (this binary runs inside every sandbox).
+DEBUG_GCFLAGS ?= -gcflags=all="-N -l"
+
 build-debug:
-	CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/debug/envd ${LDFLAGS}
+	CGO_ENABLED=1 go build -race $(DEBUG_GCFLAGS) -o bin/debug/envd ${LDFLAGS}
 
 start-docker:
 	make build
diff --git a/packages/orchestrator/Makefile b/packages/orchestrator/Makefile
index 473a72aec5..aea2ddb731 100644
--- a/packages/orchestrator/Makefile
+++ b/packages/orchestrator/Makefile
@@ -44,9 +44,15 @@ build-local:
 	CGO_ENABLED=1 GOOS=linux GOARCH=$(BUILD_ARCH) go build -o bin/orchestrator -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" .
 	CGO_ENABLED=1 GOOS=linux GOARCH=$(BUILD_ARCH) go build -o bin/clean-nfs-cache -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" ./cmd/clean-nfs-cache
 
+# Compiler flags for debug builds. The default disables optimizations and
+# inlining so binaries are debugger-friendly. CI overrides this with an empty
+# value: race-instrumented but optimized binaries compile faster and speed up
+# every integration test at runtime.
+DEBUG_GCFLAGS ?= -gcflags=all="-N -l"
+
 .PHONY: build-debug
 build-debug:
-	CGO_ENABLED=1 GOOS=linux GOARCH=$(BUILD_ARCH) go build -race -gcflags=all="-N -l" -o bin/orchestrator .
+	CGO_ENABLED=1 GOOS=linux GOARCH=$(BUILD_ARCH) go build -race $(DEBUG_GCFLAGS) -o bin/orchestrator .
 
 # Builds the dummy orchestrator. It only implements the orchestrator.proto
 # SandboxService surface against an in-memory store, and is meant for local
diff --git a/tests/integration/Makefile b/tests/integration/Makefile
index b1492223b3..41ce9384e2 100644
--- a/tests/integration/Makefile
+++ b/tests/integration/Makefile
@@ -1,6 +1,12 @@
 ENV := $(shell cat ../../.last_used_env || echo "not-set")
 -include ../../.env.${ENV}
 
+# Max in-flight parallel tests per package (go test -parallel). The harness
+# itself allows up to 100 concurrent sandboxes (utils.MaxConcurrentSandboxes),
+# so this is usually the throughput bottleneck. CI overrides this to a higher
+# value; the default stays conservative for local runs.
+TEST_PARALLELISM ?= 4
+
 .PHONY: generate
 generate:
 	go generate ./...
@@ -39,9 +45,9 @@ test/%:
 		*.go:*) \
 			BASE=$${TEST_PATH%%:*}; \
 			TEST_FN=$${TEST_PATH#*:}; \
-			go tool gotestsum --rerun-fails=1 --packages="$$BASE" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=4 -timeout=20m -run "$${TEST_FN}" ;; \
-		*.go) go tool gotestsum --rerun-fails=1 --packages="$$TEST_PATH" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=4 -timeout=20m ;; \
-		*) go tool gotestsum --rerun-fails=1 --packages="$$TEST_PATH/..." --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=4 -timeout=20m ;; \
+			go tool gotestsum --rerun-fails=1 --packages="$$BASE" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=$(TEST_PARALLELISM) -timeout=20m -run "$${TEST_FN}" ;; \
+		*.go) go tool gotestsum --rerun-fails=1 --packages="$$TEST_PATH" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=$(TEST_PARALLELISM) -timeout=20m ;; \
+		*) go tool gotestsum --rerun-fails=1 --packages="$$TEST_PATH/..." --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=$(TEST_PARALLELISM) -timeout=20m ;; \
 	esac
 
 .PHONY: connect-orchestrator
diff --git a/tests/integration/internal/tests/api/templates/delete_template_test.go b/tests/integration/internal/tests/api/templates/delete_template_test.go
index 940a1c7ccf..f69ff03cf4 100644
--- a/tests/integration/internal/tests/api/templates/delete_template_test.go
+++ b/tests/integration/internal/tests/api/templates/delete_template_test.go
@@ -48,18 +48,9 @@ func TestDeleteTemplateFromAnotherTeamAPIKey(t *testing.T) {
 	teamID := testutils.CreateTeamWithUser(t, db, "foreign-team", userID.String())
 	apiKey := testutils.CreateAPIKey(t, t.Context(), setup.GetAPIClient(), userID.String(), teamID)
 
-	res := buildTemplate(t, alias, api.TemplateBuildStartV2{
-		Force:     new(ForceBaseBuild),
-		FromImage: new("ubuntu:22.04"),
-		Steps: new([]api.TemplateStep{
-			{
-				Type:  "RUN",
-				Force: new(true),
-				Args:  new([]string{"echo 'Hello, World!'"}),
-			},
-		}),
-	}, defaultBuildLogHandler(t))
-	require.True(t, res)
+	// The foreign team's lookup never sees the template regardless of build
+	// state, so requesting the template (no build) is enough for this test.
+	testutils.RequestTemplateWithoutBuild(t, alias, setup.WithAPIKey())
 
 	c := setup.GetAPIClient()
 	deleteRes, err := c.DeleteTemplatesTemplateIDWithResponse(
diff --git a/tests/integration/internal/tests/api/templates/template_tags_test.go b/tests/integration/internal/tests/api/templates/template_tags_test.go
index 45d647152b..411c1e2971 100644
--- a/tests/integration/internal/tests/api/templates/template_tags_test.go
+++ b/tests/integration/internal/tests/api/templates/template_tags_test.go
@@ -90,8 +90,9 @@ func TestTemplateTagDeleteLatestNotAllowed(t *testing.T) {
 
 	c := setup.GetAPIClient()
 
-	// Build a template to work with
-	template := testutils.BuildSimpleTemplate(t, "test-tag-delete-latest", setup.WithAPIKey())
+	// The default-tag deletion guard fires before any build lookup, so a
+	// template without a build is enough for this test.
+	template := testutils.RequestTemplateWithoutBuild(t, "test-tag-delete-latest", setup.WithAPIKey())
 
 	// Try to delete the 'default' tag - should fail
 	deleteResp, err := c.DeleteTemplatesTagsWithResponse(ctx, api.DeleteTemplateTagsRequest{
diff --git a/tests/integration/internal/tests/api/templates/template_update_test.go b/tests/integration/internal/tests/api/templates/template_update_test.go
index b4b4aa32bc..f72f5b203f 100644
--- a/tests/integration/internal/tests/api/templates/template_update_test.go
+++ b/tests/integration/internal/tests/api/templates/template_update_test.go
@@ -14,8 +14,8 @@ import (
 
 func TestUpdateTemplateVisibilityToPublicWithAPIKey(t *testing.T) {
 	t.Parallel()
-	// Create a test template
-	template := testutils.BuildSimpleTemplate(t, "test-update-public-api-key", setup.WithAPIKey())
+	// Create a test template (metadata only, no build needed)
+	template := testutils.RequestTemplateWithoutBuild(t, "test-update-public-api-key", setup.WithAPIKey())
 
 	c := setup.GetAPIClient()
 
@@ -53,8 +53,8 @@ func TestUpdateTemplateVisibilityToPublicWithAPIKey(t *testing.T) {
 
 func TestUpdateTemplateVisibilityToPrivateWithAPIKey(t *testing.T) {
 	t.Parallel()
-	// Create a test template
-	template := testutils.BuildSimpleTemplate(t, "test-update-private-api-key", setup.WithAPIKey())
+	// Create a test template (metadata only, no build needed)
+	template := testutils.RequestTemplateWithoutBuild(t, "test-update-private-api-key", setup.WithAPIKey())
 
 	c := setup.GetAPIClient()
 
@@ -104,7 +104,7 @@ func TestUpdateTemplateVisibilityToPrivateWithAPIKey(t *testing.T) {
 func TestUpdateTemplateWithInvalidAPIKey(t *testing.T) {
 	t.Parallel()
 	// Create a test template with valid API key
-	template := testutils.BuildSimpleTemplate(t, "test-update-invalid-key", setup.WithAPIKey())
+	template := testutils.RequestTemplateWithoutBuild(t, "test-update-invalid-key", setup.WithAPIKey())
 
 	c := setup.GetAPIClient()
 
@@ -145,7 +145,7 @@ func TestUpdateNonExistentTemplateWithAPIKey(t *testing.T) {
 func TestUpdateTemplateWithAPIKey(t *testing.T) {
 	t.Parallel()
 	// Create a test template with API key first
-	template := testutils.BuildSimpleTemplate(t, "test-update-api-key", setup.WithAPIKey())
+	template := testutils.RequestTemplateWithoutBuild(t, "test-update-api-key", setup.WithAPIKey())
 
 	c := setup.GetAPIClient()
 
@@ -202,7 +202,7 @@ func TestUpdateTemplateNotOwnedByTeam(t *testing.T) {
 	team2APIKey := testutils.CreateAPIKey(t, ctx, c, user1ID.String(), team2ID)
 
 	// Create a template
-	template := testutils.BuildSimpleTemplate(t, "test-update-template-cross-team", setup.WithAPIKey())
+	template := testutils.RequestTemplateWithoutBuild(t, "test-update-template-cross-team", setup.WithAPIKey())
 	team1TemplateID := template.TemplateID
 
 	// Try to update team1's template using team2's API key - should fail
diff --git a/tests/integration/internal/tests/envd/process_test.go b/tests/integration/internal/tests/envd/process_test.go
index 93ca59aa8c..1492d43e92 100644
--- a/tests/integration/internal/tests/envd/process_test.go
+++ b/tests/integration/internal/tests/envd/process_test.go
@@ -83,16 +83,18 @@ func TestCommandKillNextApp(t *testing.T) {
 		}
 	}()
 
-	// Wait for the next dev to start and list processes
-	time.Sleep(10 * time.Second)
-
+	// Wait for the next dev to show up in the process list
 	listReq := connect.NewRequest(&process.ListRequest{})
 	setup.SetSandboxHeader(t, listReq.Header(), sbx.SandboxID)
 	setup.SetUserHeader(t, listReq.Header(), "user")
-	listResp, err := envdClient.ProcessClient.List(ctx, listReq)
-	require.NoError(t, err)
 
-	assert.Len(t, listResp.Msg.GetProcesses(), 1, "Expected one process (next dev) running")
+	var listResp *connect.Response[process.ListResponse]
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		var err error
+		listResp, err = envdClient.ProcessClient.List(ctx, listReq)
+		require.NoError(c, err)
+		assert.Len(c, listResp.Msg.GetProcesses(), 1, "Expected one process (next dev) running")
+	}, 30*time.Second, 500*time.Millisecond)
 
 	// Kill all processes
 	for _, proc := range listResp.Msg.GetProcesses() {
@@ -160,16 +162,18 @@ func TestCommandKillWithAnd(t *testing.T) {
 		}
 	}()
 
-	// Step 2: Wait for the command to start
-	time.Sleep(5 * time.Second)
-
+	// Step 2: Wait for the command to show up in the process list
 	listReq := connect.NewRequest(&process.ListRequest{})
 	setup.SetSandboxHeader(t, listReq.Header(), sbx.SandboxID)
 	setup.SetUserHeader(t, listReq.Header(), "user")
-	listResp, err := envdClient.ProcessClient.List(ctx, listReq)
-	require.NoError(t, err)
 
-	assert.Len(t, listResp.Msg.GetProcesses(), 1, "Expected one process running")
+	var listResp *connect.Response[process.ListResponse]
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		var err error
+		listResp, err = envdClient.ProcessClient.List(ctx, listReq)
+		require.NoError(c, err)
+		assert.Len(c, listResp.Msg.GetProcesses(), 1, "Expected one process running")
+	}, 30*time.Second, 500*time.Millisecond)
 
 	// Kill all processes
 	for _, proc := range listResp.Msg.GetProcesses() {
diff --git a/tests/integration/internal/utils/template.go b/tests/integration/internal/utils/template.go
index 803db729e6..e5ad2e036f 100644
--- a/tests/integration/internal/utils/template.go
+++ b/tests/integration/internal/utils/template.go
@@ -234,3 +234,19 @@ func BuildSimpleTemplate(tb testing.TB, name string, reqEditors ...api.RequestEd
 
 	return BuildTemplate(tb, opts)
 }
+
+// RequestTemplateWithoutBuild creates a template without starting (or waiting
+// for) its build. The template row, a build in the "waiting" state, the alias,
+// and the default tag assignment all exist as soon as the request returns, so
+// the template can be listed, updated (e.g. visibility), and deleted.
+//
+// Use this instead of BuildSimpleTemplate in tests that only exercise template
+// metadata or authorization and never need a runnable build - a real build
+// boots a Firecracker VM and takes minutes, while this is a single API call.
+func RequestTemplateWithoutBuild(tb testing.TB, name string, reqEditors ...api.RequestEditorFn) *api.TemplateRequestResponseV3 {
+	tb.Helper()
+
+	reqEditors = append(reqEditors, setup.WithTestsUserAgent())
+
+	return requestTemplateBuild(tb, name, nil, nil, nil, reqEditors...)
+}

From 7f69548f901eb46064bcca4d62f220fe194aa4b3 Mon Sep 17 00:00:00 2001
From: Jakub Dobry <jakub.dobry8@gmail.com>
Date: Wed, 8 Jul 2026 16:52:43 -0700
Subject: [PATCH 2/8] ci: dial integration test parallelism back to 8

At -parallel=12 the zstd1 shard failed 87 tests: concurrent Firecracker
template builds (each with 8 zstd frame-encode workers) starved envd
inits inside freshly booted sandboxes, surfacing as 'failed to init
envd: syncing took too long' (44 occurrences in the orchestrator log,
no OOM/data races).
---
 .github/workflows/integration_tests.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index c35fef709e..d7862accbc 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -92,8 +92,10 @@ jobs:
           # Nearly all tests are t.Parallel() and gated by a 100-slot sandbox
           # semaphore in the harness; the go test default of -parallel=4 was
           # the throughput bottleneck (e.g. api/sandboxes is ~1250s of test
-          # time squeezed through 4 slots). Tune down if flakiness increases.
-          TEST_PARALLELISM: "12"
+          # time squeezed through 4 slots). 12 starved the host: concurrent
+          # FC template builds (x8 zstd workers each) starved envd inits into
+          # "syncing took too long" failures. Tune down if flakiness returns.
+          TEST_PARALLELISM: "8"
         run: |
           # Run the integration tests
           make test-integration

From b4498214ba5a1e2b3a8df6e78611e24eb33a9009 Mon Sep 17 00:00:00 2001
From: Jakub Dobry <jakub.dobry8@gmail.com>
Date: Wed, 8 Jul 2026 17:19:07 -0700
Subject: [PATCH 3/8] ci: keep integration test parallelism at 4

Measured on the zstd1 shard: at -parallel=8 per-test times inflated
70-140% (api/templates 3232s vs 1898s, api/sandboxes 2994s vs 1248s of
summed test time) for a net wall-clock win of ~1 min, with 10 template
build flakes rescued by --rerun-fails; at 12, envd inits starved into
'syncing took too long' across 87 tests. The host saturates server-side
(FC boots + zstd compression), so client concurrency beyond 4 only adds
contention. The TEST_PARALLELISM knob stays for local runs and future
tuning.
---
 .github/workflows/integration_tests.yml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index d7862accbc..312c201926 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -89,13 +89,14 @@ jobs:
           TESTS_ORCHESTRATOR_HOST: "localhost:5008"
           TESTS_ENVD_PROXY: "http://localhost:3002"
           TESTS_CLIENT_PROXY: "http://localhost:3002"
-          # Nearly all tests are t.Parallel() and gated by a 100-slot sandbox
-          # semaphore in the harness; the go test default of -parallel=4 was
-          # the throughput bottleneck (e.g. api/sandboxes is ~1250s of test
-          # time squeezed through 4 slots). 12 starved the host: concurrent
-          # FC template builds (x8 zstd workers each) starved envd inits into
-          # "syncing took too long" failures. Tune down if flakiness returns.
-          TEST_PARALLELISM: "8"
+          # Keep go test -parallel at 4: the host is the bottleneck, not
+          # client-side concurrency. Measured on the zstd1 shard: at 8 the
+          # per-test times inflated 70-140% (net wall win of only ~1 min) and
+          # 10 template builds flaked on timeouts; at 12 concurrent FC builds
+          # (x8 zstd workers each) starved envd inits into "syncing took too
+          # long" failures across 87 tests. Revisit if the compression path
+          # gets cheaper or the runners get more CPU.
+          TEST_PARALLELISM: "4"
         run: |
           # Run the integration tests
           make test-integration

From 8db6a2ed685664617232b77a87835bbcbcdef900 Mon Sep 17 00:00:00 2001
From: Jakub Dobry <jakub.dobry8@gmail.com>
Date: Wed, 8 Jul 2026 17:28:36 -0700
Subject: [PATCH 4/8] ci: shard integration suite across parallel runners for
 wall time

Each CI host saturates at go test -parallel=4 (Firecracker boots + zstd
compression), so client-side concurrency can't cut wall time - more
hosts can. PRs now split the zstd1 config into three package shards on
parallel runners: api/templates (~1900s of summed test time) and
api/sandboxes (~1250s) each get a dedicated runner, everything else
shares the third. Shard resolution lives in tests/integration/Makefile
(TEST_SHARD=templates|sandboxes|rest|all); the rest shard is computed
via go list so new packages are picked up automatically. Push to main
still runs the full suite per compression config.

Also start the postgres/clickhouse/redis containers in the background
right after checkout (new start-databases action), so their image pulls
and boots overlap with the ~3 min package build instead of serializing
in front of the migrations.
---
 .github/actions/start-databases/action.yml | 38 +++++++++++++++++++
 .github/actions/start-services/action.yml  | 36 ++----------------
 .github/workflows/integration_tests.yml    | 39 +++++++++++++------
 .github/workflows/pull-request.yml         |  6 ++-
 Makefile                                   |  2 +-
 tests/integration/Makefile                 | 44 ++++++++++++++++++++--
 6 files changed, 114 insertions(+), 51 deletions(-)
 create mode 100644 .github/actions/start-databases/action.yml

diff --git a/.github/actions/start-databases/action.yml b/.github/actions/start-databases/action.yml
new file mode 100644
index 0000000000..5caeab23e3
--- /dev/null
+++ b/.github/actions/start-databases/action.yml
@@ -0,0 +1,38 @@
+name: "Start Databases"
+description: "Kicks off the PostgreSQL, ClickHouse, and Redis containers in the background. Run this as early as possible: pulls and boots then overlap with the Go builds, and the start-services action later waits for health before migrating."
+
+runs:
+  using: "composite"
+  steps:
+    - name: Start Database Containers
+      env:
+        CLICKHOUSE_USERNAME: "e2b"
+        CLICKHOUSE_PASSWORD: "clickity-clicky-click"
+        CLICKHOUSE_PORT: "9000"
+        CLICKHOUSE_DATABASE: "default"
+      run: |
+        # Everything is backgrounded so the image pulls and container boots
+        # overlap with the (minutes-long) package builds that follow. The
+        # start-services action waits for container health with a bounded
+        # timeout, which also surfaces any startup failure from here.
+        docker run -d --name postgres \
+                      -e POSTGRES_USER=postgres \
+                      -e POSTGRES_PASSWORD=local \
+                      -e POSTGRES_DB=mydatabase \
+                      -p 5432:5432 \
+                      --health-cmd="pg_isready -U postgres" \
+                      --health-interval=5s \
+                      --health-timeout=2s \
+                      --health-retries=5 \
+                      postgres:18 &
+
+        docker run -d --name redis \
+              -p 6379:6379 \
+              --health-cmd="redis-cli ping" \
+              --health-interval=5s \
+              --health-timeout=2s \
+              --health-retries=5 \
+              redis:8 &
+
+        make -C packages/clickhouse run &
+      shell: bash
diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml
index b44dbaca88..1676587569 100644
--- a/.github/actions/start-services/action.yml
+++ b/.github/actions/start-services/action.yml
@@ -30,38 +30,10 @@ inputs:
 runs:
   using: "composite"
   steps:
-    # Start all database containers up front without waiting, so they boot
-    # concurrently (and overlap with the migrator image build below) instead
-    # of serially blocking on each other's health checks.
-    - name: Start Database Containers
-      env:
-        CLICKHOUSE_USERNAME: "e2b"
-        CLICKHOUSE_PASSWORD: "clickity-clicky-click"
-        CLICKHOUSE_PORT: "9000"
-        CLICKHOUSE_DATABASE: "default"
-      run: |
-        docker run -d --name postgres \
-                      -e POSTGRES_USER=postgres \
-                      -e POSTGRES_PASSWORD=local \
-                      -e POSTGRES_DB=mydatabase \
-                      -p 5432:5432 \
-                      --health-cmd="pg_isready -U postgres" \
-                      --health-interval=5s \
-                      --health-timeout=2s \
-                      --health-retries=5 \
-                      postgres:18
-
-        docker run -d --name redis \
-              -p 6379:6379 \
-              --health-cmd="redis-cli ping" \
-              --health-interval=5s \
-              --health-timeout=2s \
-              --health-retries=5 \
-              redis:8
-
-        make -C packages/clickhouse run &
-      shell: bash
-
+    # The postgres/clickhouse/redis containers were already kicked off in the
+    # background by the start-databases action (early in the job, so their
+    # pulls and boots overlapped with the package builds). Here we build the
+    # migrator image, wait for database health, and migrate.
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v4
 
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 312c201926..d7aa8d8641 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -14,7 +14,7 @@ on:
         default: true
       full-matrix:
         type: boolean
-        description: "Run all compression configs. When false, only the production-like zstd1 config runs (used on PRs to save ~2/3 of runner time; the full matrix still runs on main)."
+        description: "When true (main), run the whole suite once per compression config. When false (PRs), run only the production-like zstd1 config, split into package shards on parallel runners to cut wall time."
         required: false
         default: true
     secrets:
@@ -30,19 +30,28 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # The full 3-config matrix (uncompressed/zstd1/lz4) runs on push to
-        # main. PRs run only zstd1 (the production-like config) since most
-        # tests don't depend on compression/dedup/memfd settings, and each
-        # shard repeats the whole build+services+suite pipeline (~20 min).
+        # On push to main, the full 3-config matrix (uncompressed/zstd1/lz4)
+        # runs the whole suite per config, as before.
+        #
+        # On PRs, only the production-like zstd1 config runs (most tests don't
+        # depend on compression/dedup/memfd settings), but the suite is split
+        # into package shards on parallel runners. Each host saturates at
+        # go test -parallel=4 (Firecracker boots + zstd compression), so the
+        # only way to cut wall time is more hosts: the two heaviest packages
+        # (api/templates, api/sandboxes) get dedicated runners and everything
+        # else shares the third. Shards are resolved by TEST_SHARD in
+        # tests/integration/Makefile.
         include: >-
           ${{ fromJSON(inputs.full-matrix
             && '[
-                  {"name": "uncompressed", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"},
-                  {"name": "zstd1", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
-                  {"name": "lz4", "compress_enabled": "true", "compress_type": "lz4", "compress_level": "0", "compress_workers": "8", "dedup_mode": "direct_io", "disable_memfd": "false"}
+                  {"name": "uncompressed", "shard": "all", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"},
+                  {"name": "zstd1", "shard": "all", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
+                  {"name": "lz4", "shard": "all", "compress_enabled": "true", "compress_type": "lz4", "compress_level": "0", "compress_workers": "8", "dedup_mode": "direct_io", "disable_memfd": "false"}
                 ]'
             || '[
-                  {"name": "zstd1", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"}
+                  {"name": "zstd1-templates", "shard": "templates", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
+                  {"name": "zstd1-sandboxes", "shard": "sandboxes", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
+                  {"name": "zstd1-rest", "shard": "rest", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"}
                 ]') }}
     env:
       # Surfaced as env so upload steps can gate on presence (skipped on fork PRs).
@@ -58,6 +67,11 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v5
 
+      # Kick off the database containers first: their image pulls and boots
+      # run in the background and overlap with the package builds below.
+      - name: Start Databases
+        uses: ./.github/actions/start-databases
+
       - name: Build Packages
         uses: ./.github/actions/build-packages
 
@@ -90,13 +104,14 @@ jobs:
           TESTS_ENVD_PROXY: "http://localhost:3002"
           TESTS_CLIENT_PROXY: "http://localhost:3002"
           # Keep go test -parallel at 4: the host is the bottleneck, not
-          # client-side concurrency. Measured on the zstd1 shard: at 8 the
+          # client-side concurrency. Measured on the zstd1 config: at 8 the
           # per-test times inflated 70-140% (net wall win of only ~1 min) and
           # 10 template builds flaked on timeouts; at 12 concurrent FC builds
           # (x8 zstd workers each) starved envd inits into "syncing took too
-          # long" failures across 87 tests. Revisit if the compression path
-          # gets cheaper or the runners get more CPU.
+          # long" failures across 87 tests. Wall time is cut by sharding
+          # across runners instead (see the matrix above).
           TEST_PARALLELISM: "4"
+          TEST_SHARD: ${{ matrix.shard }}
         run: |
           # Run the integration tests
           make test-integration
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index aa3c3ffb7f..baad4e3e38 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -62,6 +62,7 @@ jobs:
               - '.github/actions/build-packages/**'
               - '.github/actions/build-sandbox-template/**'
               - '.github/actions/host-init/**'
+              - '.github/actions/start-databases/**'
               - '.github/actions/start-services/**'
               - '.github/actions/go-setup-cache/**'
             lint-inputs:
@@ -164,8 +165,9 @@ jobs:
       # Only publish the results for same-repo PRs
       publish: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
       run-tests: ${{ contains(fromJSON(needs.detect-changes.outputs.changed-scopes), 'integration-inputs') && needs.detect-changes.outputs.docs-only != 'true' }}
-      # PRs run only the production-like zstd1 config; the full compression
-      # matrix runs on push to main.
+      # PRs run only the production-like zstd1 config, sharded by package
+      # across parallel runners for wall time; the full compression matrix
+      # runs on push to main.
       full-matrix: false
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
diff --git a/Makefile b/Makefile
index 1e167898e7..3abcdc1c6b 100644
--- a/Makefile
+++ b/Makefile
@@ -184,7 +184,7 @@ test:
 
 .PHONY: test-integration
 test-integration:
-	$(MAKE) -C tests/integration test
+	$(MAKE) -C tests/integration test-shard
 
 .PHONY: connect-orchestrator
 connect-orchestrator:
diff --git a/tests/integration/Makefile b/tests/integration/Makefile
index 41ce9384e2..14de27664a 100644
--- a/tests/integration/Makefile
+++ b/tests/integration/Makefile
@@ -1,12 +1,30 @@
 ENV := $(shell cat ../../.last_used_env || echo "not-set")
 -include ../../.env.${ENV}
 
-# Max in-flight parallel tests per package (go test -parallel). The harness
-# itself allows up to 100 concurrent sandboxes (utils.MaxConcurrentSandboxes),
-# so this is usually the throughput bottleneck. CI overrides this to a higher
-# value; the default stays conservative for local runs.
+# Max in-flight parallel tests per package (go test -parallel). Measurements
+# on the CI hosts showed they saturate server-side (Firecracker boots + zstd
+# compression) right around 4; higher values only inflate per-test times and
+# flake template builds. Kept as a knob for local runs and future tuning.
 TEST_PARALLELISM ?= 4
 
+# Package shards so CI can split the suite across parallel runners; the wall
+# clock is bound by the host (see TEST_PARALLELISM above), so the only way to
+# go faster is more hosts. Shards are sized by measured test time: the two
+# heaviest packages (api/templates ~1900s, api/sandboxes ~1250s of summed test
+# time) each get their own runner, everything else shares the third.
+TEST_SHARD ?= all
+ifeq ($(TEST_SHARD),templates)
+TEST_PACKAGES := ./internal/tests/api/templates/...
+else ifeq ($(TEST_SHARD),sandboxes)
+TEST_PACKAGES := ./internal/tests/api/sandboxes/...
+else ifeq ($(TEST_SHARD),rest)
+# Everything except the two dedicated shards; computed via go list so new
+# packages are picked up automatically.
+TEST_PACKAGES = $(shell go list ./internal/tests/... | grep -v -e '/tests/api/templates' -e '/tests/api/sandboxes')
+else
+TEST_PACKAGES := ./internal/tests/...
+endif
+
 .PHONY: generate
 generate:
 	go generate ./...
@@ -27,6 +45,24 @@ seed:
 		go run seed.go
 	@echo "Done"
 
+# CI entrypoint: run one package shard (or the whole suite with the default
+# TEST_SHARD=all). Kept separate from the test/% pattern target below, which
+# supports arbitrary path/function selection for local development.
+.PHONY: test-shard
+test-shard:
+	@if [ -z "$(strip $(TEST_PACKAGES))" ]; then echo "no packages resolved for TEST_SHARD=$(TEST_SHARD)"; exit 1; fi
+	@export POSTGRES_CONNECTION_STRING=$(POSTGRES_CONNECTION_STRING); \
+	export TESTS_API_SERVER_URL=$(TESTS_API_SERVER_URL); \
+	export TESTS_ORCHESTRATOR_HOST=$(TESTS_ORCHESTRATOR_HOST); \
+	export TESTS_ENVD_PROXY=$(TESTS_ENVD_PROXY); \
+	export TESTS_SANDBOX_TEMPLATE_ID=$(TESTS_SANDBOX_TEMPLATE_ID); \
+	export TESTS_E2B_API_KEY=$(TESTS_E2B_API_KEY); \
+	export TESTS_E2B_ACCESS_TOKEN=$(TESTS_E2B_ACCESS_TOKEN); \
+	export TESTS_SANDBOX_TEAM_ID=$(TESTS_SANDBOX_TEAM_ID); \
+	export TESTS_SANDBOX_USER_ID=$(TESTS_SANDBOX_USER_ID); \
+	go test -v ./internal/main_test.go -count=1 && \
+	go tool gotestsum --rerun-fails=1 --packages="$(strip $(TEST_PACKAGES))" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=$(TEST_PARALLELISM) -timeout=20m
+
 .PHONY: test
 test: test/.
 test/%:

From 5dd3c68d9e3bbca268f4baa9e34bd2498f124b99 Mon Sep 17 00:00:00 2001
From: Jakub Dobry <jakub.dobry8@gmail.com>
Date: Wed, 8 Jul 2026 18:04:10 -0700
Subject: [PATCH 5/8] ci: get integration test wall time under 10 minutes

The previous long pole (zstd1-templates, 14.7 min) was ~7 min of setup
plus ~7 min of tests. Both halves shrink:

- Split api/templates across two runners by test-name prefix
  (^TestTemplateBuild = 417s vs the rest = 460s of measured test time),
  since each host saturates at -parallel=4.
- Build packages concurrently instead of serially (2m48s -> bounded by
  the slowest binary) and give the integration jobs their own salted Go
  cache: setup-go keys purely on dependency files, so it kept restoring
  stale -N -l artifacts and forced full recompiles every run.
- Stop building the clickhouse migrator image (1m54s per job even with
  gha layer caching): its only job is running goose against migrations
  that get volume-mounted anyway, so run goose directly on the host via
  a new migrate-host target (pinned by go.mod tool directive).
- Run the sandbox template build (~40s) in the background overlapping
  service startup; tests gate on a wait step that surfaces the log and
  exit status.
---
 .github/actions/build-packages/action.yml     | 53 ++++++++++++---
 .../actions/build-sandbox-template/action.yml | 64 ++++++++++++++++---
 .github/actions/start-services/action.yml     | 24 ++-----
 .github/workflows/integration_tests.yml       | 24 +++++--
 packages/clickhouse/Makefile                  | 11 ++++
 tests/integration/Makefile                    | 20 ++++--
 6 files changed, 150 insertions(+), 46 deletions(-)

diff --git a/.github/actions/build-packages/action.yml b/.github/actions/build-packages/action.yml
index ffa1730dd1..91b0aa4775 100644
--- a/.github/actions/build-packages/action.yml
+++ b/.github/actions/build-packages/action.yml
@@ -4,15 +4,52 @@ description: "Compiles and builds Go services."
 runs:
   using: "composite"
   steps:
-    - name: Setup Go
-      uses: ./.github/actions/go-setup-cache
+    - name: Set up Go
+      uses: actions/setup-go@v6
+      with:
+        go-version-file: go.work
+        # Caching is handled explicitly below. setup-go keys its cache purely
+        # on the dependency files, so it could never store a build cache for
+        # the optimized (DEBUG_GCFLAGS="") binaries this job builds - every
+        # run would restore stale artifacts and recompile everything.
+        cache: false
+
+    - name: Restore Go caches
+      uses: actions/cache@v5
+      with:
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
+        # The "opt" salt separates this cache from setup-go caches populated
+        # by other jobs with debugger-friendly build flags.
+        key: integration-go-opt-v1-${{ runner.os }}-${{ hashFiles('go.work', 'packages/*/go.mod', 'packages/*/go.sum', 'tests/integration/go.mod', 'tests/integration/go.sum') }}
+        restore-keys: |
+          integration-go-opt-v1-${{ runner.os }}-
 
     - name: Build Packages
       run: |
-        make -C tests/integration build-debug
-        make -C packages/db build-debug
-        make -C packages/orchestrator build-debug
-        make -C packages/api build-debug
-        make -C packages/envd build-debug
-        make -C packages/client-proxy build-debug
+        # The builds are independent, so run them concurrently: the Go build
+        # cache and module cache are safe for concurrent use and shared
+        # dependencies compile only once.
+        pkgs="tests/integration packages/db packages/orchestrator packages/api packages/envd packages/client-proxy"
+
+        pids=""
+        for pkg in $pkgs; do
+          log="/tmp/build-$(basename "$pkg").log"
+          make -C "$pkg" build-debug > "$log" 2>&1 &
+          pids="$pids $!:$pkg"
+        done
+
+        status=0
+        for entry in $pids; do
+          pid="${entry%%:*}"
+          pkg="${entry#*:}"
+          if ! wait "$pid"; then
+            echo "::error::build failed for $pkg"
+            status=1
+          fi
+          echo "===== $pkg ====="
+          cat "/tmp/build-$(basename "$pkg").log"
+        done
+        exit $status
       shell: bash
diff --git a/.github/actions/build-sandbox-template/action.yml b/.github/actions/build-sandbox-template/action.yml
index 08109a29ca..641d7e02cc 100644
--- a/.github/actions/build-sandbox-template/action.yml
+++ b/.github/actions/build-sandbox-template/action.yml
@@ -22,11 +22,16 @@ inputs:
     description: "Base template rootfs disk size in MB. Larger than the create-build default to leave headroom for disk-heavy build steps after the build-reserved-disk-space-mb reservation."
     required: false
     default: "2048"
+  mode:
+    description: "sync (default): build and wait. start: kick off the build in the background so it overlaps with later steps (e.g. service startup). wait: block until a previously started background build finishes and fail if it failed."
+    required: false
+    default: "sync"
 
 runs:
   using: "composite"
   steps:
     - name: Build Sandbox Template
+      if: ${{ inputs.mode == 'sync' || inputs.mode == 'start' }}
       env:
         TEMPLATE_ID: "2j6ly824owf4awgai1xo"
         KERNEL_VERSION: "vmlinux-6.1.158-c1a568c"
@@ -49,12 +54,55 @@ runs:
         echo "COMPRESS_LEVEL=${COMPRESS_LEVEL}" >> .env.test
         echo "COMPRESS_FRAME_ENCODE_WORKERS=${COMPRESS_FRAME_ENCODE_WORKERS}" >> .env.test
 
-        sudo -E make -C packages/orchestrator build-template \
-          ARTIFACTS_REGISTRY_PROVIDER=Local \
-          STORAGE_PROVIDER=Local \
-          TEMPLATE_ID=${TEMPLATE_ID} \
-          BUILD_ID=${BUILD_ID} \
-          KERNEL_VERSION=${KERNEL_VERSION} \
-          FC_VERSION=${FC_VERSION} \
-          DISK_SIZE_MB=${DISK_SIZE_MB}
+        build_template() {
+          sudo -E make -C packages/orchestrator build-template \
+            ARTIFACTS_REGISTRY_PROVIDER=Local \
+            STORAGE_PROVIDER=Local \
+            TEMPLATE_ID=${TEMPLATE_ID} \
+            BUILD_ID=${BUILD_ID} \
+            KERNEL_VERSION=${KERNEL_VERSION} \
+            FC_VERSION=${FC_VERSION} \
+            DISK_SIZE_MB=${DISK_SIZE_MB}
+        }
+
+        if [ "${{ inputs.mode }}" = "start" ]; then
+          # Run the build in the background so it overlaps with later steps.
+          # The .env.test entries above are written synchronously, so steps
+          # that only need the IDs (e.g. seeding) can proceed immediately;
+          # a later mode=wait invocation gates on the template artifacts.
+          mkdir -p ~/logs
+          rm -f /tmp/template-build.done /tmp/template-build.ok
+          (
+            if build_template > ~/logs/template-build.log 2>&1; then
+              touch /tmp/template-build.ok
+            fi
+            touch /tmp/template-build.done
+          ) &
+          echo "Template build started in the background"
+        else
+          build_template
+        fi
+      shell: bash
+
+    - name: Wait for Sandbox Template Build
+      if: ${{ inputs.mode == 'wait' }}
+      run: |
+        echo "Waiting for the background template build to finish..."
+        for ((i = 0; i < 600; i++)); do
+          [ -f /tmp/template-build.done ] && break
+          sleep 1
+        done
+
+        echo "===== template build log ====="
+        cat ~/logs/template-build.log || true
+
+        if [ ! -f /tmp/template-build.done ]; then
+          echo "::error::template build did not finish within 10 minutes"
+          exit 1
+        fi
+        if [ ! -f /tmp/template-build.ok ]; then
+          echo "::error::template build failed, see log above"
+          exit 1
+        fi
+        echo "Template build finished successfully"
       shell: bash
diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml
index 1676587569..d2db53659d 100644
--- a/.github/actions/start-services/action.yml
+++ b/.github/actions/start-services/action.yml
@@ -32,21 +32,11 @@ runs:
   steps:
     # The postgres/clickhouse/redis containers were already kicked off in the
     # background by the start-databases action (early in the job, so their
-    # pulls and boots overlapped with the package builds). Here we build the
-    # migrator image, wait for database health, and migrate.
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v4
-
-    - name: Build codegen image with caching
-      uses: docker/build-push-action@v7
-      with:
-        context: packages/clickhouse
-        file: packages/clickhouse/Dockerfile
-        tags: clickhouse-migrator:latest
-        load: true # makes the image available for `docker run`
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
-
+    # pulls and boots overlapped with the package builds). Here we wait for
+    # database health and migrate. ClickHouse migrations run goose directly
+    # on the host (migrate-host): building the migrator image took ~2 min per
+    # job even with layer caching, for a container whose only purpose is to
+    # run goose against migrations that get volume-mounted anyway.
     - name: Wait for Databases and Migrate
       env:
         TESTS_E2B_API_KEY: "e2b_5ec17bd3933af21f80dc10bba686691c4fcd7057"
@@ -86,7 +76,6 @@ runs:
         echo "TESTS_SANDBOX_TEAM_ID=${TESTS_SANDBOX_TEAM_ID}" >> .env.test
         echo "TESTS_SANDBOX_USER_ID=${TESTS_SANDBOX_USER_ID}" >> .env.test
         echo "REDIS_URL=localhost:6379" >> .env.test
-        echo "CLICKHOUSE_MIGRATOR_IMAGE=clickhouse-migrator" >> .env.test
         echo "CLICKHOUSE_USERNAME=${CLICKHOUSE_USERNAME}" >> .env.test
         echo "CLICKHOUSE_PORT=${CLICKHOUSE_PORT}" >> .env.test
         echo "CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}" >> .env.test
@@ -97,8 +86,7 @@ runs:
         make migrate
         make -C tests/integration seed
 
-        # We build the image in a separate step to cache it and avoid rebuilding it every time
-        make -C packages/clickhouse migrate-without-build
+        make -C packages/clickhouse migrate-host
       shell: bash
 
     - name: Start Services
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index d7aa8d8641..50d0abb966 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -35,12 +35,12 @@ jobs:
         #
         # On PRs, only the production-like zstd1 config runs (most tests don't
         # depend on compression/dedup/memfd settings), but the suite is split
-        # into package shards on parallel runners. Each host saturates at
+        # into shards on parallel runners. Each host saturates at
         # go test -parallel=4 (Firecracker boots + zstd compression), so the
-        # only way to cut wall time is more hosts: the two heaviest packages
-        # (api/templates, api/sandboxes) get dedicated runners and everything
-        # else shares the third. Shards are resolved by TEST_SHARD in
-        # tests/integration/Makefile.
+        # only way to cut wall time is more hosts: api/templates is split in
+        # two halves by test-name prefix, api/sandboxes gets its own runner,
+        # and everything else shares the last one. Shards are resolved by
+        # TEST_SHARD in tests/integration/Makefile.
         include: >-
           ${{ fromJSON(inputs.full-matrix
             && '[
@@ -49,7 +49,8 @@ jobs:
                   {"name": "lz4", "shard": "all", "compress_enabled": "true", "compress_type": "lz4", "compress_level": "0", "compress_workers": "8", "dedup_mode": "direct_io", "disable_memfd": "false"}
                 ]'
             || '[
-                  {"name": "zstd1-templates", "shard": "templates", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
+                  {"name": "zstd1-templates-builds", "shard": "templates-builds", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
+                  {"name": "zstd1-templates-misc", "shard": "templates-misc", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
                   {"name": "zstd1-sandboxes", "shard": "sandboxes", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
                   {"name": "zstd1-rest", "shard": "rest", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"}
                 ]') }}
@@ -78,9 +79,13 @@ jobs:
       - name: Initialize Host
         uses: ./.github/actions/host-init
 
-      - name: Build Template
+      # Kick off the template build in the background: it doesn't need the
+      # services, and the services don't need the template, so it overlaps
+      # with service startup below. Tests gate on the wait step.
+      - name: Build Template (background)
         uses: ./.github/actions/build-sandbox-template
         with:
+          mode: start
           compress_enabled: ${{ matrix.compress_enabled }}
           compress_type: ${{ matrix.compress_type }}
           compress_level: ${{ matrix.compress_level }}
@@ -96,6 +101,11 @@ jobs:
           dedup_mode: ${{ matrix.dedup_mode }}
           disable_memfd: ${{ matrix.disable_memfd }}
 
+      - name: Wait for Template Build
+        uses: ./.github/actions/build-sandbox-template
+        with:
+          mode: wait
+
       - name: Run Integration Tests
         env:
           TESTS_API_SERVER_URL: "http://localhost:3000"
diff --git a/packages/clickhouse/Makefile b/packages/clickhouse/Makefile
index e9d132b497..dbc7d9558e 100644
--- a/packages/clickhouse/Makefile
+++ b/packages/clickhouse/Makefile
@@ -49,6 +49,17 @@ migrate-local:
 	GOOSE_DBSTRING="clickhouse://clickhouse:clickhouse@localhost:9000/default" go tool goose -table "_migrations" -dir "migrations" clickhouse up
 	@echo "Done"
 
+# Like migrate-without-build but runs goose directly on the host instead of
+# inside the migrator image, using the CLICKHOUSE_* environment credentials.
+# Used by CI, where building the migrator image costs ~2 minutes per job for
+# no benefit (the migrations directory is mounted over the baked-in copy at
+# runtime anyway).
+.PHONY: migrate-host
+migrate-host:
+	@echo "Applying Clickhouse migrations *$(notdir $@)*"
+	@GOOSE_DBSTRING="clickhouse://$(CLICKHOUSE_USERNAME):$(CLICKHOUSE_PASSWORD)@localhost:$(CLICKHOUSE_PORT)/$(CLICKHOUSE_DATABASE)" go tool goose -table "_migrations" -dir "migrations" clickhouse up
+	@echo "Done"
+
 .PHONY: build
 build:
 	$(eval COMMIT_SHA := $(shell git rev-parse --short HEAD))
diff --git a/tests/integration/Makefile b/tests/integration/Makefile
index 14de27664a..75cd0dd646 100644
--- a/tests/integration/Makefile
+++ b/tests/integration/Makefile
@@ -9,16 +9,26 @@ TEST_PARALLELISM ?= 4
 
 # Package shards so CI can split the suite across parallel runners; the wall
 # clock is bound by the host (see TEST_PARALLELISM above), so the only way to
-# go faster is more hosts. Shards are sized by measured test time: the two
-# heaviest packages (api/templates ~1900s, api/sandboxes ~1250s of summed test
-# time) each get their own runner, everything else shares the third.
+# go faster is more hosts. Shards are sized by measured test time: the
+# api/templates package is heavy enough (~880s of summed test time, dominated
+# by real Firecracker builds) that it is split in two roughly equal halves by
+# test-name prefix (^TestTemplateBuild = 417s vs the rest = 460s measured);
+# api/sandboxes (~1250s summed) gets its own runner, and everything else
+# shares the last one.
 TEST_SHARD ?= all
+TEST_RUN_FLAGS :=
 ifeq ($(TEST_SHARD),templates)
 TEST_PACKAGES := ./internal/tests/api/templates/...
+else ifeq ($(TEST_SHARD),templates-builds)
+TEST_PACKAGES := ./internal/tests/api/templates/...
+TEST_RUN_FLAGS := -run '^TestTemplateBuild'
+else ifeq ($(TEST_SHARD),templates-misc)
+TEST_PACKAGES := ./internal/tests/api/templates/...
+TEST_RUN_FLAGS := -skip '^TestTemplateBuild'
 else ifeq ($(TEST_SHARD),sandboxes)
 TEST_PACKAGES := ./internal/tests/api/sandboxes/...
 else ifeq ($(TEST_SHARD),rest)
-# Everything except the two dedicated shards; computed via go list so new
+# Everything except the dedicated shards; computed via go list so new
 # packages are picked up automatically.
 TEST_PACKAGES = $(shell go list ./internal/tests/... | grep -v -e '/tests/api/templates' -e '/tests/api/sandboxes')
 else
@@ -61,7 +71,7 @@ test-shard:
 	export TESTS_SANDBOX_TEAM_ID=$(TESTS_SANDBOX_TEAM_ID); \
 	export TESTS_SANDBOX_USER_ID=$(TESTS_SANDBOX_USER_ID); \
 	go test -v ./internal/main_test.go -count=1 && \
-	go tool gotestsum --rerun-fails=1 --packages="$(strip $(TEST_PACKAGES))" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=$(TEST_PARALLELISM) -timeout=20m
+	go tool gotestsum --rerun-fails=1 --packages="$(strip $(TEST_PACKAGES))" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=$(TEST_PARALLELISM) -timeout=20m $(TEST_RUN_FLAGS)
 
 .PHONY: test
 test: test/.

From a8311d9280c4aae145d5876b76dfbfebdb339f35 Mon Sep 17 00:00:00 2001
From: Jakub Dobry <jakub.dobry8@gmail.com>
Date: Wed, 8 Jul 2026 18:18:11 -0700
Subject: [PATCH 6/8] ci: template build must precede service startup

The background overlap wedged every job: create-build manages its own
sandbox network state (slot pools, namespaces) and conflicts with the
orchestrator starting up concurrently, so the provisioning VM never
came ready and hit its 5-minute deadline. The overlap only saved ~40s;
run it synchronously before Start Services as before.
---
 .../actions/build-sandbox-template/action.yml | 67 +++----------------
 .github/workflows/integration_tests.yml       | 15 ++---
 2 files changed, 16 insertions(+), 66 deletions(-)

diff --git a/.github/actions/build-sandbox-template/action.yml b/.github/actions/build-sandbox-template/action.yml
index 641d7e02cc..fc7d78971e 100644
--- a/.github/actions/build-sandbox-template/action.yml
+++ b/.github/actions/build-sandbox-template/action.yml
@@ -22,16 +22,14 @@ inputs:
     description: "Base template rootfs disk size in MB. Larger than the create-build default to leave headroom for disk-heavy build steps after the build-reserved-disk-space-mb reservation."
     required: false
     default: "2048"
-  mode:
-    description: "sync (default): build and wait. start: kick off the build in the background so it overlaps with later steps (e.g. service startup). wait: block until a previously started background build finishes and fail if it failed."
-    required: false
-    default: "sync"
 
 runs:
   using: "composite"
   steps:
+    # Note: this cannot run concurrently with a running orchestrator -
+    # create-build manages its own sandbox network state (slot pools,
+    # namespaces) and the two processes conflict, wedging the provisioning VM.
     - name: Build Sandbox Template
-      if: ${{ inputs.mode == 'sync' || inputs.mode == 'start' }}
       env:
         TEMPLATE_ID: "2j6ly824owf4awgai1xo"
         KERNEL_VERSION: "vmlinux-6.1.158-c1a568c"
@@ -54,55 +52,12 @@ runs:
         echo "COMPRESS_LEVEL=${COMPRESS_LEVEL}" >> .env.test
         echo "COMPRESS_FRAME_ENCODE_WORKERS=${COMPRESS_FRAME_ENCODE_WORKERS}" >> .env.test
 
-        build_template() {
-          sudo -E make -C packages/orchestrator build-template \
-            ARTIFACTS_REGISTRY_PROVIDER=Local \
-            STORAGE_PROVIDER=Local \
-            TEMPLATE_ID=${TEMPLATE_ID} \
-            BUILD_ID=${BUILD_ID} \
-            KERNEL_VERSION=${KERNEL_VERSION} \
-            FC_VERSION=${FC_VERSION} \
-            DISK_SIZE_MB=${DISK_SIZE_MB}
-        }
-
-        if [ "${{ inputs.mode }}" = "start" ]; then
-          # Run the build in the background so it overlaps with later steps.
-          # The .env.test entries above are written synchronously, so steps
-          # that only need the IDs (e.g. seeding) can proceed immediately;
-          # a later mode=wait invocation gates on the template artifacts.
-          mkdir -p ~/logs
-          rm -f /tmp/template-build.done /tmp/template-build.ok
-          (
-            if build_template > ~/logs/template-build.log 2>&1; then
-              touch /tmp/template-build.ok
-            fi
-            touch /tmp/template-build.done
-          ) &
-          echo "Template build started in the background"
-        else
-          build_template
-        fi
-      shell: bash
-
-    - name: Wait for Sandbox Template Build
-      if: ${{ inputs.mode == 'wait' }}
-      run: |
-        echo "Waiting for the background template build to finish..."
-        for ((i = 0; i < 600; i++)); do
-          [ -f /tmp/template-build.done ] && break
-          sleep 1
-        done
-
-        echo "===== template build log ====="
-        cat ~/logs/template-build.log || true
-
-        if [ ! -f /tmp/template-build.done ]; then
-          echo "::error::template build did not finish within 10 minutes"
-          exit 1
-        fi
-        if [ ! -f /tmp/template-build.ok ]; then
-          echo "::error::template build failed, see log above"
-          exit 1
-        fi
-        echo "Template build finished successfully"
+        sudo -E make -C packages/orchestrator build-template \
+          ARTIFACTS_REGISTRY_PROVIDER=Local \
+          STORAGE_PROVIDER=Local \
+          TEMPLATE_ID=${TEMPLATE_ID} \
+          BUILD_ID=${BUILD_ID} \
+          KERNEL_VERSION=${KERNEL_VERSION} \
+          FC_VERSION=${FC_VERSION} \
+          DISK_SIZE_MB=${DISK_SIZE_MB}
       shell: bash
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 50d0abb966..22e01a2da7 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -79,13 +79,13 @@ jobs:
       - name: Initialize Host
         uses: ./.github/actions/host-init
 
-      # Kick off the template build in the background: it doesn't need the
-      # services, and the services don't need the template, so it overlaps
-      # with service startup below. Tests gate on the wait step.
-      - name: Build Template (background)
+      # Must run before Start Services: create-build manages its own sandbox
+      # network state (slot pools, namespaces) and conflicts with a running
+      # orchestrator doing the same - overlapping them wedges the
+      # provisioning VM until its deadline.
+      - name: Build Template
         uses: ./.github/actions/build-sandbox-template
         with:
-          mode: start
           compress_enabled: ${{ matrix.compress_enabled }}
           compress_type: ${{ matrix.compress_type }}
           compress_level: ${{ matrix.compress_level }}
@@ -101,11 +101,6 @@ jobs:
           dedup_mode: ${{ matrix.dedup_mode }}
           disable_memfd: ${{ matrix.disable_memfd }}
 
-      - name: Wait for Template Build
-        uses: ./.github/actions/build-sandbox-template
-        with:
-          mode: wait
-
       - name: Run Integration Tests
         env:
           TESTS_API_SERVER_URL: "http://localhost:3000"

From 4bb1fe9a661f81c46e4bad9f3a27af8dcb4f1615 Mon Sep 17 00:00:00 2001
From: Jakub Dobry <jakub.dobry8@gmail.com>
Date: Thu, 9 Jul 2026 11:29:42 -0700
Subject: [PATCH 7/8] ci: rebalance integration shards to fit the 10-minute
 budget

The templates-builds shard was 11.6 min: its 18 tests are all real FC
builds (~1400s of summed test+subtest time) that barely parallelize on
one host. Split them in two balanced halves by name prefix
(^TestTemplateBuild(ENV|COPY|RUN|W) = ~680s vs the rest = ~740s) and
fold the non-build template tests into the rest shard (separate
packages run concurrently within one gotestsum invocation; the -skip
only affects the templates package since no other package has
TestTemplateBuild* tests). Still four runners total.
---
 .github/workflows/integration_tests.yml | 13 ++++++-----
 tests/integration/Makefile              | 29 +++++++++++++++----------
 2 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 22e01a2da7..157f61f6d6 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -37,10 +37,11 @@ jobs:
         # depend on compression/dedup/memfd settings), but the suite is split
         # into shards on parallel runners. Each host saturates at
         # go test -parallel=4 (Firecracker boots + zstd compression), so the
-        # only way to cut wall time is more hosts: api/templates is split in
-        # two halves by test-name prefix, api/sandboxes gets its own runner,
-        # and everything else shares the last one. Shards are resolved by
-        # TEST_SHARD in tests/integration/Makefile.
+        # only way to cut wall time is more hosts: the TestTemplateBuild*
+        # tests (real FC builds) are split in two halves by name prefix,
+        # api/sandboxes gets its own runner, and everything else shares the
+        # last one. Shards are resolved by TEST_SHARD in
+        # tests/integration/Makefile.
         include: >-
           ${{ fromJSON(inputs.full-matrix
             && '[
@@ -49,8 +50,8 @@ jobs:
                   {"name": "lz4", "shard": "all", "compress_enabled": "true", "compress_type": "lz4", "compress_level": "0", "compress_workers": "8", "dedup_mode": "direct_io", "disable_memfd": "false"}
                 ]'
             || '[
-                  {"name": "zstd1-templates-builds", "shard": "templates-builds", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
-                  {"name": "zstd1-templates-misc", "shard": "templates-misc", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
+                  {"name": "zstd1-templates-1", "shard": "templates-builds-1", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
+                  {"name": "zstd1-templates-2", "shard": "templates-builds-2", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
                   {"name": "zstd1-sandboxes", "shard": "sandboxes", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"},
                   {"name": "zstd1-rest", "shard": "rest", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"}
                 ]') }}
diff --git a/tests/integration/Makefile b/tests/integration/Makefile
index 75cd0dd646..f252340695 100644
--- a/tests/integration/Makefile
+++ b/tests/integration/Makefile
@@ -9,28 +9,35 @@ TEST_PARALLELISM ?= 4
 
 # Package shards so CI can split the suite across parallel runners; the wall
 # clock is bound by the host (see TEST_PARALLELISM above), so the only way to
-# go faster is more hosts. Shards are sized by measured test time: the
-# api/templates package is heavy enough (~880s of summed test time, dominated
-# by real Firecracker builds) that it is split in two roughly equal halves by
-# test-name prefix (^TestTemplateBuild = 417s vs the rest = 460s measured);
-# api/sandboxes (~1250s summed) gets its own runner, and everything else
-# shares the last one.
+# go faster is more hosts. Shards are sized by measured test time:
+#
+#   - The ^TestTemplateBuild tests (real Firecracker builds, ~1400s of summed
+#     test+subtest time) are split into two roughly equal halves by name
+#     prefix: (ENV|COPY|RUN|W)* = ~680s vs the rest = ~740s. New
+#     TestTemplateBuild* tests fall into shard 2 unless they match the
+#     shard 1 prefix.
+#   - api/sandboxes gets its own runner.
+#   - Everything else (including the non-build template tests) shares the
+#     last runner; the -skip only affects the templates package since no
+#     other package has TestTemplateBuild* tests.
+TEMPLATE_BUILDS_SHARD1_RE := ^TestTemplateBuild(ENV|COPY|RUN|W)
 TEST_SHARD ?= all
 TEST_RUN_FLAGS :=
 ifeq ($(TEST_SHARD),templates)
 TEST_PACKAGES := ./internal/tests/api/templates/...
-else ifeq ($(TEST_SHARD),templates-builds)
+else ifeq ($(TEST_SHARD),templates-builds-1)
 TEST_PACKAGES := ./internal/tests/api/templates/...
-TEST_RUN_FLAGS := -run '^TestTemplateBuild'
-else ifeq ($(TEST_SHARD),templates-misc)
+TEST_RUN_FLAGS := -run '$(TEMPLATE_BUILDS_SHARD1_RE)'
+else ifeq ($(TEST_SHARD),templates-builds-2)
 TEST_PACKAGES := ./internal/tests/api/templates/...
-TEST_RUN_FLAGS := -skip '^TestTemplateBuild'
+TEST_RUN_FLAGS := -run '^TestTemplateBuild' -skip '$(TEMPLATE_BUILDS_SHARD1_RE)'
 else ifeq ($(TEST_SHARD),sandboxes)
 TEST_PACKAGES := ./internal/tests/api/sandboxes/...
 else ifeq ($(TEST_SHARD),rest)
 # Everything except the dedicated shards; computed via go list so new
 # packages are picked up automatically.
-TEST_PACKAGES = $(shell go list ./internal/tests/... | grep -v -e '/tests/api/templates' -e '/tests/api/sandboxes')
+TEST_PACKAGES = $(shell go list ./internal/tests/... | grep -v -e '/tests/api/sandboxes')
+TEST_RUN_FLAGS := -skip '^TestTemplateBuild'
 else
 TEST_PACKAGES := ./internal/tests/...
 endif

From 3715e062d8e412654ee833c6d774b3569012d9ed Mon Sep 17 00:00:00 2001
From: Jakub Dobry <jakub.dobry8@gmail.com>
Date: Thu, 9 Jul 2026 11:45:37 -0700
Subject: [PATCH 8/8] ci: move light packages onto the sandboxes runner

The rest shard came in at 9.9 min - too close to the 10-minute budget.
Move proxies, api/metrics, and api/volumes from rest (261s of tests) to
the sandboxes runner (160s) to even the two out at roughly 210s each.
---
 tests/integration/Makefile | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tests/integration/Makefile b/tests/integration/Makefile
index f252340695..4e09abb6d8 100644
--- a/tests/integration/Makefile
+++ b/tests/integration/Makefile
@@ -16,11 +16,13 @@ TEST_PARALLELISM ?= 4
 #     prefix: (ENV|COPY|RUN|W)* = ~680s vs the rest = ~740s. New
 #     TestTemplateBuild* tests fall into shard 2 unless they match the
 #     shard 1 prefix.
-#   - api/sandboxes gets its own runner.
+#   - api/sandboxes gets its own runner, topped up with the light packages
+#     (proxies, api/metrics, api/volumes) to balance against the rest shard.
 #   - Everything else (including the non-build template tests) shares the
 #     last runner; the -skip only affects the templates package since no
 #     other package has TestTemplateBuild* tests.
 TEMPLATE_BUILDS_SHARD1_RE := ^TestTemplateBuild(ENV|COPY|RUN|W)
+SANDBOXES_SHARD_PACKAGES := ./internal/tests/api/sandboxes/... ./internal/tests/api/metrics/... ./internal/tests/api/volumes/... ./internal/tests/proxies/...
 TEST_SHARD ?= all
 TEST_RUN_FLAGS :=
 ifeq ($(TEST_SHARD),templates)
@@ -32,11 +34,11 @@ else ifeq ($(TEST_SHARD),templates-builds-2)
 TEST_PACKAGES := ./internal/tests/api/templates/...
 TEST_RUN_FLAGS := -run '^TestTemplateBuild' -skip '$(TEMPLATE_BUILDS_SHARD1_RE)'
 else ifeq ($(TEST_SHARD),sandboxes)
-TEST_PACKAGES := ./internal/tests/api/sandboxes/...
+TEST_PACKAGES := $(SANDBOXES_SHARD_PACKAGES)
 else ifeq ($(TEST_SHARD),rest)
-# Everything except the dedicated shards; computed via go list so new
-# packages are picked up automatically.
-TEST_PACKAGES = $(shell go list ./internal/tests/... | grep -v -e '/tests/api/sandboxes')
+# Everything not covered by the dedicated shards; computed via go list so
+# new packages are picked up automatically.
+TEST_PACKAGES = $(shell go list ./internal/tests/... | grep -v -e '/tests/api/sandboxes' -e '/tests/api/metrics' -e '/tests/api/volumes' -e '/tests/proxies')
 TEST_RUN_FLAGS := -skip '^TestTemplateBuild'
 else
 TEST_PACKAGES := ./internal/tests/...