diff --git a/.github/actions/build-packages/action.yml b/.github/actions/build-packages/action.yml index 767fbb8316..48ac2299a0 100644 --- a/.github/actions/build-packages/action.yml +++ b/.github/actions/build-packages/action.yml @@ -4,8 +4,27 @@ description: "Compiles and builds Go services." runs: using: "composite" steps: - - name: Setup Go - uses: ./.github/actions/go-setup-cache + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.work + # Caching is handled explicitly below. setup-go keys its cache purely + # on the dependency files, so it could never store a build cache for + # the optimized (DEBUG_GCFLAGS="") binaries this job builds - every + # run would restore stale artifacts and recompile everything. + cache: false + + - name: Restore Go caches + uses: actions/cache@v5 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + # The "opt" salt separates this cache from setup-go caches populated + # by other jobs with debugger-friendly build flags. + key: integration-go-opt-v1-${{ runner.os }}-${{ hashFiles('go.work', 'packages/*/go.mod', 'packages/*/go.sum', 'tests/integration/go.mod', 'tests/integration/go.sum') }} + restore-keys: | + integration-go-opt-v1-${{ runner.os }}- # Built concurrently: the runner has 32 cores and 128 GB, while the two # slow builds (api, orchestrator) are mostly serial link steps that leave diff --git a/.github/actions/build-sandbox-template/action.yml b/.github/actions/build-sandbox-template/action.yml index 08109a29ca..fc7d78971e 100644 --- a/.github/actions/build-sandbox-template/action.yml +++ b/.github/actions/build-sandbox-template/action.yml @@ -26,6 +26,9 @@ inputs: runs: using: "composite" steps: + # Note: this cannot run concurrently with a running orchestrator - + # create-build manages its own sandbox network state (slot pools, + # namespaces) and the two processes conflict, wedging the provisioning VM. - name: Build Sandbox Template env: TEMPLATE_ID: "2j6ly824owf4awgai1xo" diff --git a/.github/actions/start-databases/action.yml b/.github/actions/start-databases/action.yml new file mode 100644 index 0000000000..29310ed69c --- /dev/null +++ b/.github/actions/start-databases/action.yml @@ -0,0 +1,57 @@ +name: "Start Databases" +description: "Kicks off the PostgreSQL, ClickHouse, and Redis containers in the background. Run this as early as possible: pulls and boots then overlap with the Go builds, and the start-services action later waits for health before migrating." + +runs: + using: "composite" + steps: + - name: Start Database Containers + env: + CLICKHOUSE_USERNAME: "e2b" + CLICKHOUSE_PASSWORD: "clickity-clicky-click" + CLICKHOUSE_PORT: "9000" + CLICKHOUSE_DATABASE: "default" + run: | + # Everything is backgrounded so the image pulls (with rate-limit + # retries, see scripts/pull-retry.sh) and container boots overlap + # with the (minutes-long) package builds that follow. Failures are + # not silent: a container whose pull or boot failed here never turns + # healthy, and start-services waits on container health with a + # bounded timeout. + ( + ./scripts/pull-retry.sh postgres:18 + docker run -d --name postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=local \ + -e POSTGRES_DB=mydatabase \ + -p 5432:5432 \ + --health-cmd="pg_isready -U postgres" \ + --health-interval=5s \ + --health-timeout=2s \ + --health-retries=5 \ + postgres:18 + ) & + + ( + ./scripts/pull-retry.sh redis:8 + docker run -d --name redis \ + -p 6379:6379 \ + --health-cmd="redis-cli ping" \ + --health-interval=5s \ + --health-timeout=2s \ + --health-retries=5 \ + redis:8 + ) & + + ( + ./scripts/pull-retry.sh clickhouse/clickhouse-server:25.4.5.24 + make -C packages/clickhouse run + ) & + + # Warm-up only: this discards its exit status by design (the step + # must not block on it). The guarantee lives in start-services, + # which re-runs pull-retry.sh for this image synchronously before + # starting the otel collector. The tag comes from the Makefile that + # runs the image, so a version bump there can't strand these pulls. + OTEL_IMAGE=$(grep -oE 'otel/opentelemetry-collector-contrib:[0-9.]+' packages/otel-collector/Makefile | head -1) + ./scripts/pull-retry.sh "$OTEL_IMAGE" & + shell: bash diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index 84ee7b9074..bc8745b540 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -30,41 +30,42 @@ inputs: runs: using: "composite" steps: - # Anonymous Docker Hub pulls from shared runners intermittently fail with - # "unauthorized: authentication required" (Hub rate limiting); the implicit - # pulls inside the service steps then abort the whole job. Pull everything - # up front with retries so a transient 401/429 can't kill the run. - - name: Pre-pull Docker Hub images - shell: bash - run: | - for image in postgres:latest redis:latest otel/opentelemetry-collector-contrib:0.146.0 clickhouse/clickhouse-server:25.4.5.24; do - for attempt in 1 2 3 4 5; do - docker pull --quiet "$image" && break - if [ "$attempt" = 5 ]; then echo "giving up on $image"; exit 1; fi - echo "pull of $image failed (attempt $attempt), retrying in $((attempt * 10))s..." - sleep $((attempt * 10)) - done - done - - - name: Run PostgreSQL Database + # The postgres/clickhouse/redis containers were already kicked off in the + # background by the start-databases action (early in the job, so their + # pulls and boots overlapped with the package builds). Here we wait for + # database health and migrate. ClickHouse migrations run goose directly + # on the host (migrate-host): building the migrator image took ~2 min per + # job even with layer caching, for a container whose only purpose is to + # run goose against migrations that get volume-mounted anyway. + - name: Wait for Databases and Migrate env: TESTS_E2B_API_KEY: "e2b_5ec17bd3933af21f80dc10bba686691c4fcd7057" TESTS_E2B_ACCESS_TOKEN: "sk_e2b_17bd3933af21f80dc10bba686691c4fcd7057123" TESTS_SANDBOX_TEAM_ID: "834777bd-9956-45ca-b088-9bac9290e2ac" TESTS_SANDBOX_USER_ID: "2a5a9fc5-db8d-4af7-ac9e-d0f9272463bc" + CLICKHOUSE_USERNAME: "e2b" + CLICKHOUSE_PASSWORD: "clickity-clicky-click" + CLICKHOUSE_PORT: "9000" + CLICKHOUSE_DATABASE: "default" run: | - docker run -d --name postgres \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=local \ - -e POSTGRES_DB=mydatabase \ - -p 5432:5432 \ - --health-cmd="pg_isready -U postgres" \ - --health-interval=5s \ - --health-timeout=2s \ - --health-retries=5 \ - postgres:latest - while [ "$(docker inspect -f '{{.State.Health.Status}}' postgres 2>/dev/null)" != "healthy" ]; do echo "Waiting for PostgreSQL to be healthy..."; sleep 2; done - echo "PostgreSQL is healthy!" + wait_healthy() { + local name="$1" timeout="${2:-120}" i + for ((i = 0; i < timeout; i += 2)); do + if [ "$(docker inspect -f '{{.State.Health.Status}}' "$name" 2>/dev/null)" = "healthy" ]; then + echo "$name is healthy!" + return 0 + fi + echo "Waiting for $name to be healthy..." + sleep 2 + done + echo "::error::$name did not become healthy within ${timeout}s" + docker logs "$name" 2>&1 | tail -50 || true + return 1 + } + + wait_healthy postgres 120 + wait_healthy clickhouse 120 + wait_healthy redis 60 # Install extensions docker exec postgres psql -U postgres -d mydatabase -c "CREATE SCHEMA extensions; CREATE EXTENSION IF NOT EXISTS pgcrypto SCHEMA extensions;" @@ -74,60 +75,18 @@ runs: echo "TESTS_E2B_ACCESS_TOKEN=${TESTS_E2B_ACCESS_TOKEN}" >> .env.test echo "TESTS_SANDBOX_TEAM_ID=${TESTS_SANDBOX_TEAM_ID}" >> .env.test echo "TESTS_SANDBOX_USER_ID=${TESTS_SANDBOX_USER_ID}" >> .env.test - set -x - make migrate - make -C tests/integration seed - shell: bash - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - - name: Build codegen image with caching - uses: docker/build-push-action@v7 - with: - context: packages/clickhouse - file: packages/clickhouse/Dockerfile - tags: clickhouse-migrator:latest - load: true # makes the image available for `docker run` - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Run Clickhouse - env: - CLICKHOUSE_USERNAME: "e2b" - CLICKHOUSE_PASSWORD: "clickity-clicky-click" - CLICKHOUSE_PORT: "9000" - CLICKHOUSE_DATABASE: "default" - run: | - echo "CLICKHOUSE_MIGRATOR_IMAGE=clickhouse-migrator" >> .env.test + echo "REDIS_URL=localhost:6379" >> .env.test echo "CLICKHOUSE_USERNAME=${CLICKHOUSE_USERNAME}" >> .env.test echo "CLICKHOUSE_PORT=${CLICKHOUSE_PORT}" >> .env.test echo "CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}" >> .env.test echo "CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE}" >> .env.test echo "CLICKHOUSE_CONNECTION_STRING=clickhouse://${CLICKHOUSE_USERNAME}:${CLICKHOUSE_PASSWORD}@$localhost:${CLICKHOUSE_PORT}/${CLICKHOUSE_DATABASE}" >> .env.test - make -C packages/clickhouse run & - while [ "$(docker inspect -f '{{.State.Health.Status}}' clickhouse 2>/dev/null)" != "healthy" ]; do echo "Waiting for Clickhouse to be healthy..."; sleep 2; done - echo "Clickhouse is healthy!" - - # We build the image in separate step to cache it and avoid rebuilding it every time - make -C packages/clickhouse migrate-without-build - shell: bash - - - name: Run Redis - run: | - docker run -d --name redis \ - -p 6379:6379 \ - --health-cmd="redis-cli ping" \ - --health-interval=5s \ - --health-timeout=2s \ - --health-retries=5 \ - redis:latest - - while [ "$(docker inspect -f '{{.State.Health.Status}}' redis 2>/dev/null)" != "healthy" ]; do echo "Waiting for Redis to be healthy..."; sleep 2; done - echo "Redis is healthy!" + set -x + make migrate + make -C tests/integration seed - echo "REDIS_URL=localhost:6379" >> .env.test + make -C packages/clickhouse migrate-host shell: bash - name: Start Services @@ -162,6 +121,14 @@ runs: PERSISTENT_VOLUME_MOUNTS="test-volume-type:${test_volume_dir}" export PERSISTENT_VOLUME_MOUNTS + # The backgrounded warm-up pull in start-databases discards its exit + # status; this synchronous call is the actual guarantee (near-instant + # when the warm-up succeeded) so a rate-limited pull retries instead + # of failing the collector's 30s health window. The tag comes from + # the Makefile that runs the image. + OTEL_IMAGE=$(grep -oE 'otel/opentelemetry-collector-contrib:[0-9.]+' packages/otel-collector/Makefile | head -1) + ./scripts/pull-retry.sh "$OTEL_IMAGE" + echo "Start otel-collector" ./scripts/start-service.sh "OtelCollector" packages/otel-collector run ~/logs/otel-collector.log http://localhost:13133/healthz diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index fe4757ea11..14198a2e45 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -12,6 +12,11 @@ on: description: "Whether to run integration tests" required: false default: true + full-matrix: + type: boolean + description: "When true (main), run all three compression configs unsharded. When false (PRs), split the full-suite uncompressed config into package shards on parallel runners to cut wall time; the compressed configs run their allow-list unsharded either way." + required: false + default: true secrets: CODECOV_TOKEN: { required: false } jobs: @@ -25,36 +30,62 @@ jobs: strategy: fail-fast: false matrix: - include: - - name: uncompressed - compress_enabled: "false" - compress_type: "" - compress_level: "" - compress_workers: "" - dedup_mode: "default" - disable_memfd: "true" - - name: zstd1 - compress_enabled: "true" - compress_type: "zstd" - compress_level: "1" - compress_workers: "8" - dedup_mode: "best_effort" - disable_memfd: "false" - - name: lz4 - compress_enabled: "true" - compress_type: "lz4" - compress_level: "0" - compress_workers: "8" - dedup_mode: "direct_io" - disable_memfd: "false" + # Coverage is split along two independent axes: + # + # - Which tests run per config: the uncompressed config runs the + # whole suite; the compressed configs (zstd1/lz4) only re-run the + # allow-listed tests whose subject is writing or reading back a + # snapshot, which is all the compression knobs can affect (see + # scripts/compression-tests.tsv). + # - Sharding for wall time (PRs only): each host saturates at + # go test -parallel=4 (Firecracker boots + compression), so the + # only way to cut wall time is more hosts. The full-suite + # uncompressed config is split into package shards - the + # TestTemplateBuild* tests (real FC builds) in two halves by name + # prefix, api/sandboxes plus the light packages on their own + # runner, everything else on the last one. The compressed + # allow-list runs are split at the package level: the templates + # package (real-build snapshot tests) vs everything else. Shards + # are resolved by TEST_SHARD in tests/integration/Makefile. + # + # On push to main (full-matrix), all three configs run unsharded, one + # job per config, as before. + include: >- + ${{ fromJSON(inputs.full-matrix + && '[ + {"name": "uncompressed", "shard": "all", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"}, + {"name": "zstd1", "shard": "all", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"}, + {"name": "lz4", "shard": "all", "compress_enabled": "true", "compress_type": "lz4", "compress_level": "0", "compress_workers": "8", "dedup_mode": "direct_io", "disable_memfd": "false"} + ]' + || '[ + {"name": "uncompressed-templates-1", "shard": "templates-builds-1", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"}, + {"name": "uncompressed-templates-2", "shard": "templates-builds-2", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"}, + {"name": "uncompressed-sandboxes", "shard": "sandboxes", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"}, + {"name": "uncompressed-rest", "shard": "rest", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"}, + {"name": "zstd1-templates", "shard": "templates", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"}, + {"name": "zstd1-other", "shard": "no-templates", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"}, + {"name": "lz4-templates", "shard": "templates", "compress_enabled": "true", "compress_type": "lz4", "compress_level": "0", "compress_workers": "8", "dedup_mode": "direct_io", "disable_memfd": "false"}, + {"name": "lz4-other", "shard": "no-templates", "compress_enabled": "true", "compress_type": "lz4", "compress_level": "0", "compress_workers": "8", "dedup_mode": "direct_io", "disable_memfd": "false"} + ]') }} env: # Surfaced as env so upload steps can gate on presence (skipped on fork PRs). CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + # Build race-instrumented but optimized binaries: nobody attaches a + # debugger in CI, and -N -l slows both compilation and every test at + # runtime. Set at the job level so the build-packages step and the + # rebuilds inside the run/run-debug targets use identical flags (the + # second build is then a Go build cache no-op). + DEBUG_GCFLAGS: "" steps: - name: Checkout Code uses: actions/checkout@v5 + # Kick off the database containers first: their image pulls and boots + # run in the background and overlap with the package builds below. + - name: Start Databases + uses: ./.github/actions/start-databases + # The compressed configs only run what the allow-list names, so verify # its stale-entry guard still bites before depending on it. - name: Check Test Allow-list @@ -67,6 +98,10 @@ jobs: - name: Initialize Host uses: ./.github/actions/host-init + # Must run before Start Services: create-build manages its own sandbox + # network state (slot pools, namespaces) and conflicts with a running + # orchestrator doing the same - overlapping them wedges the + # provisioning VM until its deadline. - name: Build Template uses: ./.github/actions/build-sandbox-template with: @@ -97,6 +132,15 @@ jobs: # snapshot, which is all the compression knobs can affect — the # allow-list documents the criteria and the code paths involved. TESTS_ONLY: ${{ matrix.compress_enabled == 'true' && 'scripts/compression-tests.tsv' || '' }} + # Keep go test -parallel at 4: the host is the bottleneck, not + # client-side concurrency. Measured on the zstd1 config: at 8 the + # per-test times inflated 70-140% (net wall win of only ~1 min) and + # 10 template builds flaked on timeouts; at 12 concurrent FC builds + # (x8 zstd workers each) starved envd inits into "syncing took too + # long" failures across 87 tests. Wall time is cut by sharding + # across runners instead (see the matrix above). + TEST_PARALLELISM: "4" + TEST_SHARD: ${{ matrix.shard }} run: | # Run the integration tests make test-integration diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index e6bbbd2039..4191a91ce5 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -58,10 +58,13 @@ jobs: - '.tool-versions' # invoked by the start-services action to launch each service - 'scripts/start-service.sh' + # invoked by start-databases and start-services to pull images + - 'scripts/pull-retry.sh' - '.github/workflows/integration_tests.yml' - '.github/actions/build-packages/**' - '.github/actions/build-sandbox-template/**' - '.github/actions/host-init/**' + - '.github/actions/start-databases/**' - '.github/actions/start-services/**' - '.github/actions/go-setup-cache/**' lint-inputs: @@ -164,6 +167,10 @@ jobs: # Only publish the results for same-repo PRs publish: ${{ github.event.pull_request.head.repo.full_name == github.repository }} run-tests: ${{ contains(fromJSON(needs.detect-changes.outputs.changed-scopes), 'integration-inputs') && needs.detect-changes.outputs.docs-only != 'true' }} + # PRs shard the full-suite uncompressed config across parallel runners + # for wall time (the compressed configs run their allow-list unsharded + # either way); push to main runs all three configs unsharded. + full-matrix: false secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} publish-test-results: diff --git a/Makefile b/Makefile index 1e167898e7..3abcdc1c6b 100644 --- a/Makefile +++ b/Makefile @@ -184,7 +184,7 @@ test: .PHONY: test-integration test-integration: - $(MAKE) -C tests/integration test + $(MAKE) -C tests/integration test-shard .PHONY: connect-orchestrator connect-orchestrator: diff --git a/packages/api/Makefile b/packages/api/Makefile index a8d72dbfc8..ab964b8826 100644 --- a/packages/api/Makefile +++ b/packages/api/Makefile @@ -46,11 +46,17 @@ build: $(eval EXPECTED_MIGRATION_TIMESTAMP ?= $(expectedMigration)) CGO_ENABLED=0 go build -o bin/api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" . +# Compiler flags for debug builds. The default disables optimizations and +# inlining so binaries are debugger-friendly. CI overrides this with an empty +# value: race-instrumented but optimized binaries compile faster and speed up +# every integration test at runtime. +DEBUG_GCFLAGS ?= -gcflags=all="-N -l" + .PHONY: build-debug build-debug: $(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD)) $(eval EXPECTED_MIGRATION_TIMESTAMP ?= $(expectedMigration)) - CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" . + CGO_ENABLED=1 go build -race $(DEBUG_GCFLAGS) -o bin/api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" . .PHONY: run run: diff --git a/packages/clickhouse/Makefile b/packages/clickhouse/Makefile index e9d132b497..dbc7d9558e 100644 --- a/packages/clickhouse/Makefile +++ b/packages/clickhouse/Makefile @@ -49,6 +49,17 @@ migrate-local: GOOSE_DBSTRING="clickhouse://clickhouse:clickhouse@localhost:9000/default" go tool goose -table "_migrations" -dir "migrations" clickhouse up @echo "Done" +# Like migrate-without-build but runs goose directly on the host instead of +# inside the migrator image, using the CLICKHOUSE_* environment credentials. +# Used by CI, where building the migrator image costs ~2 minutes per job for +# no benefit (the migrations directory is mounted over the baked-in copy at +# runtime anyway). +.PHONY: migrate-host +migrate-host: + @echo "Applying Clickhouse migrations *$(notdir $@)*" + @GOOSE_DBSTRING="clickhouse://$(CLICKHOUSE_USERNAME):$(CLICKHOUSE_PASSWORD)@localhost:$(CLICKHOUSE_PORT)/$(CLICKHOUSE_DATABASE)" go tool goose -table "_migrations" -dir "migrations" clickhouse up + @echo "Done" + .PHONY: build build: $(eval COMMIT_SHA := $(shell git rev-parse --short HEAD)) diff --git a/packages/client-proxy/Makefile b/packages/client-proxy/Makefile index dd3510b122..949922a3aa 100644 --- a/packages/client-proxy/Makefile +++ b/packages/client-proxy/Makefile @@ -42,10 +42,16 @@ build: $(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD)) CGO_ENABLED=0 GOOS=linux GOARCH=$(BUILD_ARCH) go build -o bin/client-proxy -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" . +# Compiler flags for debug builds. The default disables optimizations and +# inlining so binaries are debugger-friendly. CI overrides this with an empty +# value: race-instrumented but optimized binaries compile faster and speed up +# every integration test at runtime. +DEBUG_GCFLAGS ?= -gcflags=all="-N -l" + .PHONY: build-debug build-debug: $(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD)) - CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/client-proxy -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" . + CGO_ENABLED=1 go build -race $(DEBUG_GCFLAGS) -o bin/client-proxy -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" . .PHONY: build-and-upload build-and-upload: diff --git a/packages/envd/Makefile b/packages/envd/Makefile index ae6b3c1f53..d4ee046777 100644 --- a/packages/envd/Makefile +++ b/packages/envd/Makefile @@ -9,6 +9,12 @@ LDFLAGS=-ldflags "-X=main.commitSHA=$(BUILD)" PROD_LDFLAGS=-ldflags "-X=main.commitSHA=$(BUILD) -s -w -buildid=" DEBUG_LDFLAGS=-ldflags "-X=main.commitSHA=$(BUILD) -linkmode external -extldflags=-static" +# Compiler flags for debug builds. The default disables optimizations and +# inlining so binaries are debugger-friendly. CI overrides this with an empty +# value: race-instrumented but optimized binaries compile faster and speed up +# every integration test at runtime. +DEBUG_GCFLAGS ?= -gcflags=all="-N -l" + AWS_BUCKET_PREFIX ?= $(PREFIX)$(AWS_ACCOUNT_ID)- GCP_BUCKET_PREFIX ?= $(GCP_PROJECT_ID)- ENVD_UPLOAD_MODE ?= latest @@ -77,7 +83,7 @@ build: # link. osusergo/netgo keep user and DNS lookups pure Go, as in the CGO_ENABLED=0 # build above, so only the race instrumentation differs from what ships. build-debug: - CGO_ENABLED=1 go build -race -tags osusergo,netgo -gcflags=all="-N -l" -o bin/debug/envd ${DEBUG_LDFLAGS} + CGO_ENABLED=1 go build -race -tags osusergo,netgo $(DEBUG_GCFLAGS) -o bin/debug/envd ${DEBUG_LDFLAGS} start-docker: make build diff --git a/packages/orchestrator/Makefile b/packages/orchestrator/Makefile index a2ef8d6735..5ed0a8d0db 100644 --- a/packages/orchestrator/Makefile +++ b/packages/orchestrator/Makefile @@ -67,9 +67,15 @@ build-local: CGO_ENABLED=1 GOOS=linux GOARCH=$(BUILD_ARCH) go build -o bin/orchestrator -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" . CGO_ENABLED=1 GOOS=linux GOARCH=$(BUILD_ARCH) go build -o bin/clean-nfs-cache -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" ./cmd/clean-nfs-cache +# Compiler flags for debug builds. The default disables optimizations and +# inlining so binaries are debugger-friendly. CI overrides this with an empty +# value: race-instrumented but optimized binaries compile faster and speed up +# every integration test at runtime. +DEBUG_GCFLAGS ?= -gcflags=all="-N -l" + .PHONY: build-debug build-debug: - CGO_ENABLED=1 GOOS=linux GOARCH=$(BUILD_ARCH) go build -race -gcflags=all="-N -l" -o bin/orchestrator . + CGO_ENABLED=1 GOOS=linux GOARCH=$(BUILD_ARCH) go build -race $(DEBUG_GCFLAGS) -o bin/orchestrator . # Builds the dummy orchestrator. It only implements the orchestrator.proto # SandboxService surface against an in-memory store, and is meant for local diff --git a/scripts/pull-retry.sh b/scripts/pull-retry.sh new file mode 100755 index 0000000000..7975ae0507 --- /dev/null +++ b/scripts/pull-retry.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Pull a Docker image with retries. Anonymous Docker Hub pulls from shared +# runners intermittently fail with "unauthorized: authentication required" +# (Hub rate limiting); an implicit pull inside docker run then aborts the +# whole job. Near-instant when the image is already present, so it is safe +# to call both as a background warm-up and as a synchronous guarantee. + +set -uo pipefail + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +IMAGE="$1" + +for attempt in 1 2 3 4 5; do + docker pull --quiet "$IMAGE" && exit 0 + if [ "$attempt" = 5 ]; then + echo "::error::giving up on pulling $IMAGE" + exit 1 + fi + echo "pull of $IMAGE failed (attempt $attempt), retrying in $((attempt * 10))s..." + sleep $((attempt * 10)) +done diff --git a/tests/integration/Makefile b/tests/integration/Makefile index e492cd097f..728c8cebe9 100644 --- a/tests/integration/Makefile +++ b/tests/integration/Makefile @@ -1,6 +1,66 @@ ENV := $(shell cat ../../.last_used_env || echo "not-set") -include ../../.env.${ENV} +# Max in-flight parallel tests per package (go test -parallel). Measurements +# on the CI hosts showed they saturate server-side (Firecracker boots + zstd +# compression) right around 4; higher values only inflate per-test times and +# flake template builds. Kept as a knob for local runs and future tuning. +TEST_PARALLELISM ?= 4 + +# Package shards so CI can split the suite across parallel runners; the wall +# clock is bound by the host (see TEST_PARALLELISM above), so the only way to +# go faster is more hosts. Shards are sized by measured test time: +# +# - The ^TestTemplateBuild tests (real Firecracker builds, ~1015s of summed +# test+subtest time measured uncompressed on run 30634340619) are split +# into two roughly equal halves by name prefix: the shard-1 prefixes sum +# to ~504s vs ~511s for the rest. New TestTemplateBuild* tests fall into +# shard 2 unless they match the shard 1 prefix; rebalance by moving a +# prefix when the shards' test steps drift apart. +# - api/sandboxes gets its own runner, topped up with the light packages +# (proxies, api/metrics, api/volumes) to balance against the rest shard. +# - Everything else (including the non-build template tests) shares the +# last runner; the -skip only affects the templates package since no +# other package has TestTemplateBuild* tests. +TEMPLATE_BUILDS_SHARD1_RE := ^TestTemplateBuild(ENV|COPY|RUN|W|Cache|Fuse|Unsupported) +SANDBOXES_SHARD_PACKAGES := ./internal/tests/api/sandboxes/... ./internal/tests/api/metrics/... ./internal/tests/api/volumes/... ./internal/tests/proxies/... +TEST_SHARD ?= all +TEST_RUN_FLAGS := +# TESTS_ONLY (the compression allow-list) composes with package-pure shards: +# its selection becomes a -run regex, so it cannot combine with the +# name-split shards that carry their own -run (go test keeps only the last +# -run flag, which would silently drop one of the two selections). +TESTS_ONLY_OK := yes +ifeq ($(TEST_SHARD),templates) +TEST_PACKAGES := ./internal/tests/api/templates/... +else ifeq ($(TEST_SHARD),no-templates) +# Package-level complement of the templates shard; computed via go list so +# new packages are picked up automatically. Used to split the compressed +# allow-list runs: templates (real-build snapshot tests) vs everything else. +TEST_PACKAGES = $(shell go list ./internal/tests/... | grep -v -e '/tests/api/templates') +else ifeq ($(TEST_SHARD),templates-builds-1) +TEST_PACKAGES := ./internal/tests/api/templates/... +TEST_RUN_FLAGS := -run '$(TEMPLATE_BUILDS_SHARD1_RE)' +TESTS_ONLY_OK := no +else ifeq ($(TEST_SHARD),templates-builds-2) +TEST_PACKAGES := ./internal/tests/api/templates/... +TEST_RUN_FLAGS := -run '^TestTemplateBuild' -skip '$(TEMPLATE_BUILDS_SHARD1_RE)' +TESTS_ONLY_OK := no +else ifeq ($(TEST_SHARD),sandboxes) +TEST_PACKAGES := $(SANDBOXES_SHARD_PACKAGES) +else ifeq ($(TEST_SHARD),rest) +# Everything not covered by the dedicated shards; computed via go list so +# new packages are picked up automatically. +TEST_PACKAGES = $(shell go list ./internal/tests/... | grep -v -e '/tests/api/sandboxes' -e '/tests/api/metrics' -e '/tests/api/volumes' -e '/tests/proxies') +TEST_RUN_FLAGS := -skip '^TestTemplateBuild' +else ifeq ($(TEST_SHARD),all) +TEST_PACKAGES := ./internal/tests/... +else +# Unknown shard names resolve to nothing so the guard in test-shard fails +# loudly instead of silently running the whole suite. +TEST_PACKAGES := +endif + .PHONY: generate generate: go generate ./... @@ -43,6 +103,30 @@ check-tests-allowlist: done; \ echo "allow-list resolves; stale named and wildcard entries are rejected" +# CI entrypoint: run one package shard (or the whole suite with the default +# TEST_SHARD=all). TESTS_ONLY narrows the run to the allow-list within the +# shard's packages; the guard below rejects the name-split shards whose own +# -run flag would silently collide with the allow-list's. +.PHONY: test-shard +test-shard: + @if [ -z "$(strip $(TEST_PACKAGES))" ]; then echo "no packages resolved for TEST_SHARD=$(TEST_SHARD)"; exit 1; fi + @if [ -n "$(TESTS_ONLY)" ] && [ "$(TESTS_ONLY_OK)" != "yes" ]; then echo "TESTS_ONLY cannot compose with name-split shard $(TEST_SHARD) (-run flags collide)"; exit 1; fi + @export POSTGRES_CONNECTION_STRING=$(POSTGRES_CONNECTION_STRING); \ + export TESTS_API_SERVER_URL=$(TESTS_API_SERVER_URL); \ + export TESTS_ORCHESTRATOR_HOST=$(TESTS_ORCHESTRATOR_HOST); \ + export TESTS_ENVD_PROXY=$(TESTS_ENVD_PROXY); \ + export TESTS_SANDBOX_TEMPLATE_ID=$(TESTS_SANDBOX_TEMPLATE_ID); \ + export TESTS_E2B_API_KEY=$(TESTS_E2B_API_KEY); \ + export TESTS_E2B_ACCESS_TOKEN=$(TESTS_E2B_ACCESS_TOKEN); \ + export TESTS_SANDBOX_TEAM_ID=$(TESTS_SANDBOX_TEAM_ID); \ + export TESTS_SANDBOX_USER_ID=$(TESTS_SANDBOX_USER_ID); \ + go test -v ./internal/main_test.go -count=1 && \ + RUN=""; \ + if [ -n "$(TESTS_ONLY)" ]; then RUN=$$(./scripts/select-tests.sh --only $(TESTS_ONLY) ./internal/tests/...) || exit 1; fi; \ + if [ -n "$${RUN}" ]; then set -- -run "$${RUN}"; else set --; fi; \ + go tool gotestsum --rerun-fails=1 --packages="$(strip $(TEST_PACKAGES))" --format standard-verbose --junitfile=test-results.xml \ + -- -count=1 -parallel=$(TEST_PARALLELISM) -timeout=20m $(TEST_RUN_FLAGS) "$$@" + .PHONY: test test: test/. test/%: @@ -67,7 +151,7 @@ test/%: if [ -n "$(TESTS_ONLY)" ]; then RUN=$$(./scripts/select-tests.sh --only $(TESTS_ONLY) "$${PACKAGES}") || exit 1; fi; \ if [ -n "$${RUN}" ]; then set -- -run "$${RUN}"; else set --; fi; \ go tool gotestsum --rerun-fails=1 --packages="$${PACKAGES}" --format standard-verbose --junitfile=test-results.xml \ - -- -count=1 -parallel=4 -timeout=20m "$$@" + -- -count=1 -parallel=$(TEST_PARALLELISM) -timeout=20m "$$@" .PHONY: connect-orchestrator connect-orchestrator: diff --git a/tests/integration/internal/tests/api/templates/delete_template_test.go b/tests/integration/internal/tests/api/templates/delete_template_test.go index 940a1c7ccf..f69ff03cf4 100644 --- a/tests/integration/internal/tests/api/templates/delete_template_test.go +++ b/tests/integration/internal/tests/api/templates/delete_template_test.go @@ -48,18 +48,9 @@ func TestDeleteTemplateFromAnotherTeamAPIKey(t *testing.T) { teamID := testutils.CreateTeamWithUser(t, db, "foreign-team", userID.String()) apiKey := testutils.CreateAPIKey(t, t.Context(), setup.GetAPIClient(), userID.String(), teamID) - res := buildTemplate(t, alias, api.TemplateBuildStartV2{ - Force: new(ForceBaseBuild), - FromImage: new("ubuntu:22.04"), - Steps: new([]api.TemplateStep{ - { - Type: "RUN", - Force: new(true), - Args: new([]string{"echo 'Hello, World!'"}), - }, - }), - }, defaultBuildLogHandler(t)) - require.True(t, res) + // The foreign team's lookup never sees the template regardless of build + // state, so requesting the template (no build) is enough for this test. + testutils.RequestTemplateWithoutBuild(t, alias, setup.WithAPIKey()) c := setup.GetAPIClient() deleteRes, err := c.DeleteTemplatesTemplateIDWithResponse( diff --git a/tests/integration/internal/tests/api/templates/template_tags_test.go b/tests/integration/internal/tests/api/templates/template_tags_test.go index 45d647152b..411c1e2971 100644 --- a/tests/integration/internal/tests/api/templates/template_tags_test.go +++ b/tests/integration/internal/tests/api/templates/template_tags_test.go @@ -90,8 +90,9 @@ func TestTemplateTagDeleteLatestNotAllowed(t *testing.T) { c := setup.GetAPIClient() - // Build a template to work with - template := testutils.BuildSimpleTemplate(t, "test-tag-delete-latest", setup.WithAPIKey()) + // The default-tag deletion guard fires before any build lookup, so a + // template without a build is enough for this test. + template := testutils.RequestTemplateWithoutBuild(t, "test-tag-delete-latest", setup.WithAPIKey()) // Try to delete the 'default' tag - should fail deleteResp, err := c.DeleteTemplatesTagsWithResponse(ctx, api.DeleteTemplateTagsRequest{ diff --git a/tests/integration/internal/tests/api/templates/template_update_test.go b/tests/integration/internal/tests/api/templates/template_update_test.go index b4b4aa32bc..f72f5b203f 100644 --- a/tests/integration/internal/tests/api/templates/template_update_test.go +++ b/tests/integration/internal/tests/api/templates/template_update_test.go @@ -14,8 +14,8 @@ import ( func TestUpdateTemplateVisibilityToPublicWithAPIKey(t *testing.T) { t.Parallel() - // Create a test template - template := testutils.BuildSimpleTemplate(t, "test-update-public-api-key", setup.WithAPIKey()) + // Create a test template (metadata only, no build needed) + template := testutils.RequestTemplateWithoutBuild(t, "test-update-public-api-key", setup.WithAPIKey()) c := setup.GetAPIClient() @@ -53,8 +53,8 @@ func TestUpdateTemplateVisibilityToPublicWithAPIKey(t *testing.T) { func TestUpdateTemplateVisibilityToPrivateWithAPIKey(t *testing.T) { t.Parallel() - // Create a test template - template := testutils.BuildSimpleTemplate(t, "test-update-private-api-key", setup.WithAPIKey()) + // Create a test template (metadata only, no build needed) + template := testutils.RequestTemplateWithoutBuild(t, "test-update-private-api-key", setup.WithAPIKey()) c := setup.GetAPIClient() @@ -104,7 +104,7 @@ func TestUpdateTemplateVisibilityToPrivateWithAPIKey(t *testing.T) { func TestUpdateTemplateWithInvalidAPIKey(t *testing.T) { t.Parallel() // Create a test template with valid API key - template := testutils.BuildSimpleTemplate(t, "test-update-invalid-key", setup.WithAPIKey()) + template := testutils.RequestTemplateWithoutBuild(t, "test-update-invalid-key", setup.WithAPIKey()) c := setup.GetAPIClient() @@ -145,7 +145,7 @@ func TestUpdateNonExistentTemplateWithAPIKey(t *testing.T) { func TestUpdateTemplateWithAPIKey(t *testing.T) { t.Parallel() // Create a test template with API key first - template := testutils.BuildSimpleTemplate(t, "test-update-api-key", setup.WithAPIKey()) + template := testutils.RequestTemplateWithoutBuild(t, "test-update-api-key", setup.WithAPIKey()) c := setup.GetAPIClient() @@ -202,7 +202,7 @@ func TestUpdateTemplateNotOwnedByTeam(t *testing.T) { team2APIKey := testutils.CreateAPIKey(t, ctx, c, user1ID.String(), team2ID) // Create a template - template := testutils.BuildSimpleTemplate(t, "test-update-template-cross-team", setup.WithAPIKey()) + template := testutils.RequestTemplateWithoutBuild(t, "test-update-template-cross-team", setup.WithAPIKey()) team1TemplateID := template.TemplateID // Try to update team1's template using team2's API key - should fail diff --git a/tests/integration/internal/tests/envd/process_test.go b/tests/integration/internal/tests/envd/process_test.go index 93ca59aa8c..1492d43e92 100644 --- a/tests/integration/internal/tests/envd/process_test.go +++ b/tests/integration/internal/tests/envd/process_test.go @@ -83,16 +83,18 @@ func TestCommandKillNextApp(t *testing.T) { } }() - // Wait for the next dev to start and list processes - time.Sleep(10 * time.Second) - + // Wait for the next dev to show up in the process list listReq := connect.NewRequest(&process.ListRequest{}) setup.SetSandboxHeader(t, listReq.Header(), sbx.SandboxID) setup.SetUserHeader(t, listReq.Header(), "user") - listResp, err := envdClient.ProcessClient.List(ctx, listReq) - require.NoError(t, err) - assert.Len(t, listResp.Msg.GetProcesses(), 1, "Expected one process (next dev) running") + var listResp *connect.Response[process.ListResponse] + require.EventuallyWithT(t, func(c *assert.CollectT) { + var err error + listResp, err = envdClient.ProcessClient.List(ctx, listReq) + require.NoError(c, err) + assert.Len(c, listResp.Msg.GetProcesses(), 1, "Expected one process (next dev) running") + }, 30*time.Second, 500*time.Millisecond) // Kill all processes for _, proc := range listResp.Msg.GetProcesses() { @@ -160,16 +162,18 @@ func TestCommandKillWithAnd(t *testing.T) { } }() - // Step 2: Wait for the command to start - time.Sleep(5 * time.Second) - + // Step 2: Wait for the command to show up in the process list listReq := connect.NewRequest(&process.ListRequest{}) setup.SetSandboxHeader(t, listReq.Header(), sbx.SandboxID) setup.SetUserHeader(t, listReq.Header(), "user") - listResp, err := envdClient.ProcessClient.List(ctx, listReq) - require.NoError(t, err) - assert.Len(t, listResp.Msg.GetProcesses(), 1, "Expected one process running") + var listResp *connect.Response[process.ListResponse] + require.EventuallyWithT(t, func(c *assert.CollectT) { + var err error + listResp, err = envdClient.ProcessClient.List(ctx, listReq) + require.NoError(c, err) + assert.Len(c, listResp.Msg.GetProcesses(), 1, "Expected one process running") + }, 30*time.Second, 500*time.Millisecond) // Kill all processes for _, proc := range listResp.Msg.GetProcesses() { diff --git a/tests/integration/internal/utils/template.go b/tests/integration/internal/utils/template.go index 803db729e6..e5ad2e036f 100644 --- a/tests/integration/internal/utils/template.go +++ b/tests/integration/internal/utils/template.go @@ -234,3 +234,19 @@ func BuildSimpleTemplate(tb testing.TB, name string, reqEditors ...api.RequestEd return BuildTemplate(tb, opts) } + +// RequestTemplateWithoutBuild creates a template without starting (or waiting +// for) its build. The template row, a build in the "waiting" state, the alias, +// and the default tag assignment all exist as soon as the request returns, so +// the template can be listed, updated (e.g. visibility), and deleted. +// +// Use this instead of BuildSimpleTemplate in tests that only exercise template +// metadata or authorization and never need a runnable build - a real build +// boots a Firecracker VM and takes minutes, while this is a single API call. +func RequestTemplateWithoutBuild(tb testing.TB, name string, reqEditors ...api.RequestEditorFn) *api.TemplateRequestResponseV3 { + tb.Helper() + + reqEditors = append(reqEditors, setup.WithTestsUserAgent()) + + return requestTemplateBuild(tb, name, nil, nil, nil, reqEditors...) +}