diff --git a/.github/actions/build-packages/action.yml b/.github/actions/build-packages/action.yml index ffa1730dd1..91b0aa4775 100644 --- a/.github/actions/build-packages/action.yml +++ b/.github/actions/build-packages/action.yml @@ -4,15 +4,52 @@ description: "Compiles and builds Go services." runs: using: "composite" steps: - - name: Setup Go - uses: ./.github/actions/go-setup-cache + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.work + # Caching is handled explicitly below. setup-go keys its cache purely + # on the dependency files, so it could never store a build cache for + # the optimized (DEBUG_GCFLAGS="") binaries this job builds - every + # run would restore stale artifacts and recompile everything. + cache: false + + - name: Restore Go caches + uses: actions/cache@v5 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + # The "opt" salt separates this cache from setup-go caches populated + # by other jobs with debugger-friendly build flags. + key: integration-go-opt-v1-${{ runner.os }}-${{ hashFiles('go.work', 'packages/*/go.mod', 'packages/*/go.sum', 'tests/integration/go.mod', 'tests/integration/go.sum') }} + restore-keys: | + integration-go-opt-v1-${{ runner.os }}- - name: Build Packages run: | - make -C tests/integration build-debug - make -C packages/db build-debug - make -C packages/orchestrator build-debug - make -C packages/api build-debug - make -C packages/envd build-debug - make -C packages/client-proxy build-debug + # The builds are independent, so run them concurrently: the Go build + # cache and module cache are safe for concurrent use and shared + # dependencies compile only once. + pkgs="tests/integration packages/db packages/orchestrator packages/api packages/envd packages/client-proxy" + + pids="" + for pkg in $pkgs; do + log="/tmp/build-$(basename "$pkg").log" + make -C "$pkg" build-debug > "$log" 2>&1 & + pids="$pids $!:$pkg" + done + + status=0 + for entry in $pids; do + pid="${entry%%:*}" + pkg="${entry#*:}" + if ! wait "$pid"; then + echo "::error::build failed for $pkg" + status=1 + fi + echo "===== $pkg =====" + cat "/tmp/build-$(basename "$pkg").log" + done + exit $status shell: bash diff --git a/.github/actions/build-sandbox-template/action.yml b/.github/actions/build-sandbox-template/action.yml index 08109a29ca..fc7d78971e 100644 --- a/.github/actions/build-sandbox-template/action.yml +++ b/.github/actions/build-sandbox-template/action.yml @@ -26,6 +26,9 @@ inputs: runs: using: "composite" steps: + # Note: this cannot run concurrently with a running orchestrator - + # create-build manages its own sandbox network state (slot pools, + # namespaces) and the two processes conflict, wedging the provisioning VM. - name: Build Sandbox Template env: TEMPLATE_ID: "2j6ly824owf4awgai1xo" diff --git a/.github/actions/start-databases/action.yml b/.github/actions/start-databases/action.yml new file mode 100644 index 0000000000..5caeab23e3 --- /dev/null +++ b/.github/actions/start-databases/action.yml @@ -0,0 +1,38 @@ +name: "Start Databases" +description: "Kicks off the PostgreSQL, ClickHouse, and Redis containers in the background. Run this as early as possible: pulls and boots then overlap with the Go builds, and the start-services action later waits for health before migrating." + +runs: + using: "composite" + steps: + - name: Start Database Containers + env: + CLICKHOUSE_USERNAME: "e2b" + CLICKHOUSE_PASSWORD: "clickity-clicky-click" + CLICKHOUSE_PORT: "9000" + CLICKHOUSE_DATABASE: "default" + run: | + # Everything is backgrounded so the image pulls and container boots + # overlap with the (minutes-long) package builds that follow. The + # start-services action waits for container health with a bounded + # timeout, which also surfaces any startup failure from here. + docker run -d --name postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=local \ + -e POSTGRES_DB=mydatabase \ + -p 5432:5432 \ + --health-cmd="pg_isready -U postgres" \ + --health-interval=5s \ + --health-timeout=2s \ + --health-retries=5 \ + postgres:18 & + + docker run -d --name redis \ + -p 6379:6379 \ + --health-cmd="redis-cli ping" \ + --health-interval=5s \ + --health-timeout=2s \ + --health-retries=5 \ + redis:8 & + + make -C packages/clickhouse run & + shell: bash diff --git a/.github/actions/start-services/action.yml b/.github/actions/start-services/action.yml index 5e57f3033b..d2db53659d 100644 --- a/.github/actions/start-services/action.yml +++ b/.github/actions/start-services/action.yml @@ -30,25 +30,42 @@ inputs: runs: using: "composite" steps: - - name: Run PostgreSQL Database + # The postgres/clickhouse/redis containers were already kicked off in the + # background by the start-databases action (early in the job, so their + # pulls and boots overlapped with the package builds). Here we wait for + # database health and migrate. ClickHouse migrations run goose directly + # on the host (migrate-host): building the migrator image took ~2 min per + # job even with layer caching, for a container whose only purpose is to + # run goose against migrations that get volume-mounted anyway. + - name: Wait for Databases and Migrate env: TESTS_E2B_API_KEY: "e2b_5ec17bd3933af21f80dc10bba686691c4fcd7057" TESTS_E2B_ACCESS_TOKEN: "sk_e2b_17bd3933af21f80dc10bba686691c4fcd7057123" TESTS_SANDBOX_TEAM_ID: "834777bd-9956-45ca-b088-9bac9290e2ac" TESTS_SANDBOX_USER_ID: "2a5a9fc5-db8d-4af7-ac9e-d0f9272463bc" + CLICKHOUSE_USERNAME: "e2b" + CLICKHOUSE_PASSWORD: "clickity-clicky-click" + CLICKHOUSE_PORT: "9000" + CLICKHOUSE_DATABASE: "default" run: | - docker run -d --name postgres \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=local \ - -e POSTGRES_DB=mydatabase \ - -p 5432:5432 \ - --health-cmd="pg_isready -U postgres" \ - --health-interval=5s \ - --health-timeout=2s \ - --health-retries=5 \ - postgres:latest - while [ "$(docker inspect -f '{{.State.Health.Status}}' postgres 2>/dev/null)" != "healthy" ]; do echo "Waiting for PostgreSQL to be healthy..."; sleep 2; done - echo "PostgreSQL is healthy!" + wait_healthy() { + local name="$1" timeout="${2:-120}" i + for ((i = 0; i < timeout; i += 2)); do + if [ "$(docker inspect -f '{{.State.Health.Status}}' "$name" 2>/dev/null)" = "healthy" ]; then + echo "$name is healthy!" + return 0 + fi + echo "Waiting for $name to be healthy..." + sleep 2 + done + echo "::error::$name did not become healthy within ${timeout}s" + docker logs "$name" 2>&1 | tail -50 || true + return 1 + } + + wait_healthy postgres 120 + wait_healthy clickhouse 120 + wait_healthy redis 60 # Install extensions docker exec postgres psql -U postgres -d mydatabase -c "CREATE SCHEMA extensions; CREATE EXTENSION IF NOT EXISTS pgcrypto SCHEMA extensions;" @@ -58,60 +75,18 @@ runs: echo "TESTS_E2B_ACCESS_TOKEN=${TESTS_E2B_ACCESS_TOKEN}" >> .env.test echo "TESTS_SANDBOX_TEAM_ID=${TESTS_SANDBOX_TEAM_ID}" >> .env.test echo "TESTS_SANDBOX_USER_ID=${TESTS_SANDBOX_USER_ID}" >> .env.test - set -x - make migrate - make -C tests/integration seed - shell: bash - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - - name: Build codegen image with caching - uses: docker/build-push-action@v7 - with: - context: packages/clickhouse - file: packages/clickhouse/Dockerfile - tags: clickhouse-migrator:latest - load: true # makes the image available for `docker run` - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Run Clickhouse - env: - CLICKHOUSE_USERNAME: "e2b" - CLICKHOUSE_PASSWORD: "clickity-clicky-click" - CLICKHOUSE_PORT: "9000" - CLICKHOUSE_DATABASE: "default" - run: | - echo "CLICKHOUSE_MIGRATOR_IMAGE=clickhouse-migrator" >> .env.test + echo "REDIS_URL=localhost:6379" >> .env.test echo "CLICKHOUSE_USERNAME=${CLICKHOUSE_USERNAME}" >> .env.test echo "CLICKHOUSE_PORT=${CLICKHOUSE_PORT}" >> .env.test echo "CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}" >> .env.test echo "CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE}" >> .env.test echo "CLICKHOUSE_CONNECTION_STRING=clickhouse://${CLICKHOUSE_USERNAME}:${CLICKHOUSE_PASSWORD}@$localhost:${CLICKHOUSE_PORT}/${CLICKHOUSE_DATABASE}" >> .env.test - make -C packages/clickhouse run & - while [ "$(docker inspect -f '{{.State.Health.Status}}' clickhouse 2>/dev/null)" != "healthy" ]; do echo "Waiting for Clickhouse to be healthy..."; sleep 2; done - echo "Clickhouse is healthy!" - - # We build the image in separate step to cache it and avoid rebuilding it every time - make -C packages/clickhouse migrate-without-build - shell: bash - - - name: Run Redis - run: | - docker run -d --name redis \ - -p 6379:6379 \ - --health-cmd="redis-cli ping" \ - --health-interval=5s \ - --health-timeout=2s \ - --health-retries=5 \ - redis:latest - - while [ "$(docker inspect -f '{{.State.Health.Status}}' redis 2>/dev/null)" != "healthy" ]; do echo "Waiting for Redis to be healthy..."; sleep 2; done - echo "Redis is healthy!" + set -x + make migrate + make -C tests/integration seed - echo "REDIS_URL=localhost:6379" >> .env.test + make -C packages/clickhouse migrate-host shell: bash - name: Start Services diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 6d00df773b..157f61f6d6 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -12,6 +12,11 @@ on: description: "Whether to run integration tests" required: false default: true + full-matrix: + type: boolean + description: "When true (main), run the whole suite once per compression config. When false (PRs), run only the production-like zstd1 config, split into package shards on parallel runners to cut wall time." + required: false + default: true secrets: CODECOV_TOKEN: { required: false } jobs: @@ -25,42 +30,60 @@ jobs: strategy: fail-fast: false matrix: - include: - - name: uncompressed - compress_enabled: "false" - compress_type: "" - compress_level: "" - compress_workers: "" - dedup_mode: "default" - disable_memfd: "true" - - name: zstd1 - compress_enabled: "true" - compress_type: "zstd" - compress_level: "1" - compress_workers: "8" - dedup_mode: "best_effort" - disable_memfd: "false" - - name: lz4 - compress_enabled: "true" - compress_type: "lz4" - compress_level: "0" - compress_workers: "8" - dedup_mode: "direct_io" - disable_memfd: "false" + # On push to main, the full 3-config matrix (uncompressed/zstd1/lz4) + # runs the whole suite per config, as before. + # + # On PRs, only the production-like zstd1 config runs (most tests don't + # depend on compression/dedup/memfd settings), but the suite is split + # into shards on parallel runners. Each host saturates at + # go test -parallel=4 (Firecracker boots + zstd compression), so the + # only way to cut wall time is more hosts: the TestTemplateBuild* + # tests (real FC builds) are split in two halves by name prefix, + # api/sandboxes gets its own runner, and everything else shares the + # last one. Shards are resolved by TEST_SHARD in + # tests/integration/Makefile. + include: >- + ${{ fromJSON(inputs.full-matrix + && '[ + {"name": "uncompressed", "shard": "all", "compress_enabled": "false", "compress_type": "", "compress_level": "", "compress_workers": "", "dedup_mode": "default", "disable_memfd": "true"}, + {"name": "zstd1", "shard": "all", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"}, + {"name": "lz4", "shard": "all", "compress_enabled": "true", "compress_type": "lz4", "compress_level": "0", "compress_workers": "8", "dedup_mode": "direct_io", "disable_memfd": "false"} + ]' + || '[ + {"name": "zstd1-templates-1", "shard": "templates-builds-1", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"}, + {"name": "zstd1-templates-2", "shard": "templates-builds-2", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"}, + {"name": "zstd1-sandboxes", "shard": "sandboxes", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"}, + {"name": "zstd1-rest", "shard": "rest", "compress_enabled": "true", "compress_type": "zstd", "compress_level": "1", "compress_workers": "8", "dedup_mode": "best_effort", "disable_memfd": "false"} + ]') }} env: # Surfaced as env so upload steps can gate on presence (skipped on fork PRs). CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + # Build race-instrumented but optimized binaries: nobody attaches a + # debugger in CI, and -N -l slows both compilation and every test at + # runtime. Set at the job level so the build-packages step and the + # rebuilds inside the run/run-debug targets use identical flags (the + # second build is then a Go build cache no-op). + DEBUG_GCFLAGS: "" steps: - name: Checkout Code uses: actions/checkout@v5 + # Kick off the database containers first: their image pulls and boots + # run in the background and overlap with the package builds below. + - name: Start Databases + uses: ./.github/actions/start-databases + - name: Build Packages uses: ./.github/actions/build-packages - name: Initialize Host uses: ./.github/actions/host-init + # Must run before Start Services: create-build manages its own sandbox + # network state (slot pools, namespaces) and conflicts with a running + # orchestrator doing the same - overlapping them wedges the + # provisioning VM until its deadline. - name: Build Template uses: ./.github/actions/build-sandbox-template with: @@ -86,6 +109,15 @@ jobs: TESTS_ORCHESTRATOR_HOST: "localhost:5008" TESTS_ENVD_PROXY: "http://localhost:3002" TESTS_CLIENT_PROXY: "http://localhost:3002" + # Keep go test -parallel at 4: the host is the bottleneck, not + # client-side concurrency. Measured on the zstd1 config: at 8 the + # per-test times inflated 70-140% (net wall win of only ~1 min) and + # 10 template builds flaked on timeouts; at 12 concurrent FC builds + # (x8 zstd workers each) starved envd inits into "syncing took too + # long" failures across 87 tests. Wall time is cut by sharding + # across runners instead (see the matrix above). + TEST_PARALLELISM: "4" + TEST_SHARD: ${{ matrix.shard }} run: | # Run the integration tests make test-integration diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index e6bbbd2039..baad4e3e38 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -62,6 +62,7 @@ jobs: - '.github/actions/build-packages/**' - '.github/actions/build-sandbox-template/**' - '.github/actions/host-init/**' + - '.github/actions/start-databases/**' - '.github/actions/start-services/**' - '.github/actions/go-setup-cache/**' lint-inputs: @@ -164,6 +165,10 @@ jobs: # Only publish the results for same-repo PRs publish: ${{ github.event.pull_request.head.repo.full_name == github.repository }} run-tests: ${{ contains(fromJSON(needs.detect-changes.outputs.changed-scopes), 'integration-inputs') && needs.detect-changes.outputs.docs-only != 'true' }} + # PRs run only the production-like zstd1 config, sharded by package + # across parallel runners for wall time; the full compression matrix + # runs on push to main. + full-matrix: false secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} publish-test-results: diff --git a/Makefile b/Makefile index 1e167898e7..3abcdc1c6b 100644 --- a/Makefile +++ b/Makefile @@ -184,7 +184,7 @@ test: .PHONY: test-integration test-integration: - $(MAKE) -C tests/integration test + $(MAKE) -C tests/integration test-shard .PHONY: connect-orchestrator connect-orchestrator: diff --git a/packages/api/Makefile b/packages/api/Makefile index 64fbbd91e4..1de78d953b 100644 --- a/packages/api/Makefile +++ b/packages/api/Makefile @@ -25,11 +25,17 @@ build: $(eval EXPECTED_MIGRATION_TIMESTAMP ?= $(expectedMigration)) CGO_ENABLED=0 go build -o bin/api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" . +# Compiler flags for debug builds. The default disables optimizations and +# inlining so binaries are debugger-friendly. CI overrides this with an empty +# value: race-instrumented but optimized binaries compile faster and speed up +# every integration test at runtime. +DEBUG_GCFLAGS ?= -gcflags=all="-N -l" + .PHONY: build-debug build-debug: $(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD)) $(eval EXPECTED_MIGRATION_TIMESTAMP ?= $(expectedMigration)) - CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" . + CGO_ENABLED=1 go build -race $(DEBUG_GCFLAGS) -o bin/api -ldflags "-X=main.commitSHA=$(COMMIT_SHA) -X=main.expectedMigrationTimestamp=$(EXPECTED_MIGRATION_TIMESTAMP)" . .PHONY: run run: diff --git a/packages/clickhouse/Makefile b/packages/clickhouse/Makefile index e9d132b497..dbc7d9558e 100644 --- a/packages/clickhouse/Makefile +++ b/packages/clickhouse/Makefile @@ -49,6 +49,17 @@ migrate-local: GOOSE_DBSTRING="clickhouse://clickhouse:clickhouse@localhost:9000/default" go tool goose -table "_migrations" -dir "migrations" clickhouse up @echo "Done" +# Like migrate-without-build but runs goose directly on the host instead of +# inside the migrator image, using the CLICKHOUSE_* environment credentials. +# Used by CI, where building the migrator image costs ~2 minutes per job for +# no benefit (the migrations directory is mounted over the baked-in copy at +# runtime anyway). +.PHONY: migrate-host +migrate-host: + @echo "Applying Clickhouse migrations *$(notdir $@)*" + @GOOSE_DBSTRING="clickhouse://$(CLICKHOUSE_USERNAME):$(CLICKHOUSE_PASSWORD)@localhost:$(CLICKHOUSE_PORT)/$(CLICKHOUSE_DATABASE)" go tool goose -table "_migrations" -dir "migrations" clickhouse up + @echo "Done" + .PHONY: build build: $(eval COMMIT_SHA := $(shell git rev-parse --short HEAD)) diff --git a/packages/client-proxy/Makefile b/packages/client-proxy/Makefile index dd3510b122..949922a3aa 100644 --- a/packages/client-proxy/Makefile +++ b/packages/client-proxy/Makefile @@ -42,10 +42,16 @@ build: $(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD)) CGO_ENABLED=0 GOOS=linux GOARCH=$(BUILD_ARCH) go build -o bin/client-proxy -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" . +# Compiler flags for debug builds. The default disables optimizations and +# inlining so binaries are debugger-friendly. CI overrides this with an empty +# value: race-instrumented but optimized binaries compile faster and speed up +# every integration test at runtime. +DEBUG_GCFLAGS ?= -gcflags=all="-N -l" + .PHONY: build-debug build-debug: $(eval COMMIT_SHA ?= $(shell git rev-parse --short HEAD)) - CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/client-proxy -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" . + CGO_ENABLED=1 go build -race $(DEBUG_GCFLAGS) -o bin/client-proxy -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" . .PHONY: build-and-upload build-and-upload: diff --git a/packages/envd/Makefile b/packages/envd/Makefile index de1723032d..ead1aed115 100644 --- a/packages/envd/Makefile +++ b/packages/envd/Makefile @@ -52,8 +52,14 @@ endif build: CGO_ENABLED=0 GOOS=linux GOARCH=$(BUILD_ARCH) go build -trimpath -buildvcs=false -a -o bin/envd ${PROD_LDFLAGS} +# Compiler flags for debug builds. The default disables optimizations and +# inlining so binaries are debugger-friendly. CI overrides this with an empty +# value: race-instrumented but optimized binaries compile faster and speed up +# every integration test at runtime (this binary runs inside every sandbox). +DEBUG_GCFLAGS ?= -gcflags=all="-N -l" + build-debug: - CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/debug/envd ${LDFLAGS} + CGO_ENABLED=1 go build -race $(DEBUG_GCFLAGS) -o bin/debug/envd ${LDFLAGS} start-docker: make build diff --git a/packages/orchestrator/Makefile b/packages/orchestrator/Makefile index 473a72aec5..aea2ddb731 100644 --- a/packages/orchestrator/Makefile +++ b/packages/orchestrator/Makefile @@ -44,9 +44,15 @@ build-local: CGO_ENABLED=1 GOOS=linux GOARCH=$(BUILD_ARCH) go build -o bin/orchestrator -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" . CGO_ENABLED=1 GOOS=linux GOARCH=$(BUILD_ARCH) go build -o bin/clean-nfs-cache -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" ./cmd/clean-nfs-cache +# Compiler flags for debug builds. The default disables optimizations and +# inlining so binaries are debugger-friendly. CI overrides this with an empty +# value: race-instrumented but optimized binaries compile faster and speed up +# every integration test at runtime. +DEBUG_GCFLAGS ?= -gcflags=all="-N -l" + .PHONY: build-debug build-debug: - CGO_ENABLED=1 GOOS=linux GOARCH=$(BUILD_ARCH) go build -race -gcflags=all="-N -l" -o bin/orchestrator . + CGO_ENABLED=1 GOOS=linux GOARCH=$(BUILD_ARCH) go build -race $(DEBUG_GCFLAGS) -o bin/orchestrator . # Builds the dummy orchestrator. It only implements the orchestrator.proto # SandboxService surface against an in-memory store, and is meant for local diff --git a/tests/integration/Makefile b/tests/integration/Makefile index b1492223b3..4e09abb6d8 100644 --- a/tests/integration/Makefile +++ b/tests/integration/Makefile @@ -1,6 +1,49 @@ ENV := $(shell cat ../../.last_used_env || echo "not-set") -include ../../.env.${ENV} +# Max in-flight parallel tests per package (go test -parallel). Measurements +# on the CI hosts showed they saturate server-side (Firecracker boots + zstd +# compression) right around 4; higher values only inflate per-test times and +# flake template builds. Kept as a knob for local runs and future tuning. +TEST_PARALLELISM ?= 4 + +# Package shards so CI can split the suite across parallel runners; the wall +# clock is bound by the host (see TEST_PARALLELISM above), so the only way to +# go faster is more hosts. Shards are sized by measured test time: +# +# - The ^TestTemplateBuild tests (real Firecracker builds, ~1400s of summed +# test+subtest time) are split into two roughly equal halves by name +# prefix: (ENV|COPY|RUN|W)* = ~680s vs the rest = ~740s. New +# TestTemplateBuild* tests fall into shard 2 unless they match the +# shard 1 prefix. +# - api/sandboxes gets its own runner, topped up with the light packages +# (proxies, api/metrics, api/volumes) to balance against the rest shard. +# - Everything else (including the non-build template tests) shares the +# last runner; the -skip only affects the templates package since no +# other package has TestTemplateBuild* tests. +TEMPLATE_BUILDS_SHARD1_RE := ^TestTemplateBuild(ENV|COPY|RUN|W) +SANDBOXES_SHARD_PACKAGES := ./internal/tests/api/sandboxes/... ./internal/tests/api/metrics/... ./internal/tests/api/volumes/... ./internal/tests/proxies/... +TEST_SHARD ?= all +TEST_RUN_FLAGS := +ifeq ($(TEST_SHARD),templates) +TEST_PACKAGES := ./internal/tests/api/templates/... +else ifeq ($(TEST_SHARD),templates-builds-1) +TEST_PACKAGES := ./internal/tests/api/templates/... +TEST_RUN_FLAGS := -run '$(TEMPLATE_BUILDS_SHARD1_RE)' +else ifeq ($(TEST_SHARD),templates-builds-2) +TEST_PACKAGES := ./internal/tests/api/templates/... +TEST_RUN_FLAGS := -run '^TestTemplateBuild' -skip '$(TEMPLATE_BUILDS_SHARD1_RE)' +else ifeq ($(TEST_SHARD),sandboxes) +TEST_PACKAGES := $(SANDBOXES_SHARD_PACKAGES) +else ifeq ($(TEST_SHARD),rest) +# Everything not covered by the dedicated shards; computed via go list so +# new packages are picked up automatically. +TEST_PACKAGES = $(shell go list ./internal/tests/... | grep -v -e '/tests/api/sandboxes' -e '/tests/api/metrics' -e '/tests/api/volumes' -e '/tests/proxies') +TEST_RUN_FLAGS := -skip '^TestTemplateBuild' +else +TEST_PACKAGES := ./internal/tests/... +endif + .PHONY: generate generate: go generate ./... @@ -21,6 +64,24 @@ seed: go run seed.go @echo "Done" +# CI entrypoint: run one package shard (or the whole suite with the default +# TEST_SHARD=all). Kept separate from the test/% pattern target below, which +# supports arbitrary path/function selection for local development. +.PHONY: test-shard +test-shard: + @if [ -z "$(strip $(TEST_PACKAGES))" ]; then echo "no packages resolved for TEST_SHARD=$(TEST_SHARD)"; exit 1; fi + @export POSTGRES_CONNECTION_STRING=$(POSTGRES_CONNECTION_STRING); \ + export TESTS_API_SERVER_URL=$(TESTS_API_SERVER_URL); \ + export TESTS_ORCHESTRATOR_HOST=$(TESTS_ORCHESTRATOR_HOST); \ + export TESTS_ENVD_PROXY=$(TESTS_ENVD_PROXY); \ + export TESTS_SANDBOX_TEMPLATE_ID=$(TESTS_SANDBOX_TEMPLATE_ID); \ + export TESTS_E2B_API_KEY=$(TESTS_E2B_API_KEY); \ + export TESTS_E2B_ACCESS_TOKEN=$(TESTS_E2B_ACCESS_TOKEN); \ + export TESTS_SANDBOX_TEAM_ID=$(TESTS_SANDBOX_TEAM_ID); \ + export TESTS_SANDBOX_USER_ID=$(TESTS_SANDBOX_USER_ID); \ + go test -v ./internal/main_test.go -count=1 && \ + go tool gotestsum --rerun-fails=1 --packages="$(strip $(TEST_PACKAGES))" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=$(TEST_PARALLELISM) -timeout=20m $(TEST_RUN_FLAGS) + .PHONY: test test: test/. test/%: @@ -39,9 +100,9 @@ test/%: *.go:*) \ BASE=$${TEST_PATH%%:*}; \ TEST_FN=$${TEST_PATH#*:}; \ - go tool gotestsum --rerun-fails=1 --packages="$$BASE" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=4 -timeout=20m -run "$${TEST_FN}" ;; \ - *.go) go tool gotestsum --rerun-fails=1 --packages="$$TEST_PATH" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=4 -timeout=20m ;; \ - *) go tool gotestsum --rerun-fails=1 --packages="$$TEST_PATH/..." --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=4 -timeout=20m ;; \ + go tool gotestsum --rerun-fails=1 --packages="$$BASE" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=$(TEST_PARALLELISM) -timeout=20m -run "$${TEST_FN}" ;; \ + *.go) go tool gotestsum --rerun-fails=1 --packages="$$TEST_PATH" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=$(TEST_PARALLELISM) -timeout=20m ;; \ + *) go tool gotestsum --rerun-fails=1 --packages="$$TEST_PATH/..." --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=$(TEST_PARALLELISM) -timeout=20m ;; \ esac .PHONY: connect-orchestrator diff --git a/tests/integration/internal/tests/api/templates/delete_template_test.go b/tests/integration/internal/tests/api/templates/delete_template_test.go index 940a1c7ccf..f69ff03cf4 100644 --- a/tests/integration/internal/tests/api/templates/delete_template_test.go +++ b/tests/integration/internal/tests/api/templates/delete_template_test.go @@ -48,18 +48,9 @@ func TestDeleteTemplateFromAnotherTeamAPIKey(t *testing.T) { teamID := testutils.CreateTeamWithUser(t, db, "foreign-team", userID.String()) apiKey := testutils.CreateAPIKey(t, t.Context(), setup.GetAPIClient(), userID.String(), teamID) - res := buildTemplate(t, alias, api.TemplateBuildStartV2{ - Force: new(ForceBaseBuild), - FromImage: new("ubuntu:22.04"), - Steps: new([]api.TemplateStep{ - { - Type: "RUN", - Force: new(true), - Args: new([]string{"echo 'Hello, World!'"}), - }, - }), - }, defaultBuildLogHandler(t)) - require.True(t, res) + // The foreign team's lookup never sees the template regardless of build + // state, so requesting the template (no build) is enough for this test. + testutils.RequestTemplateWithoutBuild(t, alias, setup.WithAPIKey()) c := setup.GetAPIClient() deleteRes, err := c.DeleteTemplatesTemplateIDWithResponse( diff --git a/tests/integration/internal/tests/api/templates/template_tags_test.go b/tests/integration/internal/tests/api/templates/template_tags_test.go index 45d647152b..411c1e2971 100644 --- a/tests/integration/internal/tests/api/templates/template_tags_test.go +++ b/tests/integration/internal/tests/api/templates/template_tags_test.go @@ -90,8 +90,9 @@ func TestTemplateTagDeleteLatestNotAllowed(t *testing.T) { c := setup.GetAPIClient() - // Build a template to work with - template := testutils.BuildSimpleTemplate(t, "test-tag-delete-latest", setup.WithAPIKey()) + // The default-tag deletion guard fires before any build lookup, so a + // template without a build is enough for this test. + template := testutils.RequestTemplateWithoutBuild(t, "test-tag-delete-latest", setup.WithAPIKey()) // Try to delete the 'default' tag - should fail deleteResp, err := c.DeleteTemplatesTagsWithResponse(ctx, api.DeleteTemplateTagsRequest{ diff --git a/tests/integration/internal/tests/api/templates/template_update_test.go b/tests/integration/internal/tests/api/templates/template_update_test.go index b4b4aa32bc..f72f5b203f 100644 --- a/tests/integration/internal/tests/api/templates/template_update_test.go +++ b/tests/integration/internal/tests/api/templates/template_update_test.go @@ -14,8 +14,8 @@ import ( func TestUpdateTemplateVisibilityToPublicWithAPIKey(t *testing.T) { t.Parallel() - // Create a test template - template := testutils.BuildSimpleTemplate(t, "test-update-public-api-key", setup.WithAPIKey()) + // Create a test template (metadata only, no build needed) + template := testutils.RequestTemplateWithoutBuild(t, "test-update-public-api-key", setup.WithAPIKey()) c := setup.GetAPIClient() @@ -53,8 +53,8 @@ func TestUpdateTemplateVisibilityToPublicWithAPIKey(t *testing.T) { func TestUpdateTemplateVisibilityToPrivateWithAPIKey(t *testing.T) { t.Parallel() - // Create a test template - template := testutils.BuildSimpleTemplate(t, "test-update-private-api-key", setup.WithAPIKey()) + // Create a test template (metadata only, no build needed) + template := testutils.RequestTemplateWithoutBuild(t, "test-update-private-api-key", setup.WithAPIKey()) c := setup.GetAPIClient() @@ -104,7 +104,7 @@ func TestUpdateTemplateVisibilityToPrivateWithAPIKey(t *testing.T) { func TestUpdateTemplateWithInvalidAPIKey(t *testing.T) { t.Parallel() // Create a test template with valid API key - template := testutils.BuildSimpleTemplate(t, "test-update-invalid-key", setup.WithAPIKey()) + template := testutils.RequestTemplateWithoutBuild(t, "test-update-invalid-key", setup.WithAPIKey()) c := setup.GetAPIClient() @@ -145,7 +145,7 @@ func TestUpdateNonExistentTemplateWithAPIKey(t *testing.T) { func TestUpdateTemplateWithAPIKey(t *testing.T) { t.Parallel() // Create a test template with API key first - template := testutils.BuildSimpleTemplate(t, "test-update-api-key", setup.WithAPIKey()) + template := testutils.RequestTemplateWithoutBuild(t, "test-update-api-key", setup.WithAPIKey()) c := setup.GetAPIClient() @@ -202,7 +202,7 @@ func TestUpdateTemplateNotOwnedByTeam(t *testing.T) { team2APIKey := testutils.CreateAPIKey(t, ctx, c, user1ID.String(), team2ID) // Create a template - template := testutils.BuildSimpleTemplate(t, "test-update-template-cross-team", setup.WithAPIKey()) + template := testutils.RequestTemplateWithoutBuild(t, "test-update-template-cross-team", setup.WithAPIKey()) team1TemplateID := template.TemplateID // Try to update team1's template using team2's API key - should fail diff --git a/tests/integration/internal/tests/envd/process_test.go b/tests/integration/internal/tests/envd/process_test.go index 93ca59aa8c..1492d43e92 100644 --- a/tests/integration/internal/tests/envd/process_test.go +++ b/tests/integration/internal/tests/envd/process_test.go @@ -83,16 +83,18 @@ func TestCommandKillNextApp(t *testing.T) { } }() - // Wait for the next dev to start and list processes - time.Sleep(10 * time.Second) - + // Wait for the next dev to show up in the process list listReq := connect.NewRequest(&process.ListRequest{}) setup.SetSandboxHeader(t, listReq.Header(), sbx.SandboxID) setup.SetUserHeader(t, listReq.Header(), "user") - listResp, err := envdClient.ProcessClient.List(ctx, listReq) - require.NoError(t, err) - assert.Len(t, listResp.Msg.GetProcesses(), 1, "Expected one process (next dev) running") + var listResp *connect.Response[process.ListResponse] + require.EventuallyWithT(t, func(c *assert.CollectT) { + var err error + listResp, err = envdClient.ProcessClient.List(ctx, listReq) + require.NoError(c, err) + assert.Len(c, listResp.Msg.GetProcesses(), 1, "Expected one process (next dev) running") + }, 30*time.Second, 500*time.Millisecond) // Kill all processes for _, proc := range listResp.Msg.GetProcesses() { @@ -160,16 +162,18 @@ func TestCommandKillWithAnd(t *testing.T) { } }() - // Step 2: Wait for the command to start - time.Sleep(5 * time.Second) - + // Step 2: Wait for the command to show up in the process list listReq := connect.NewRequest(&process.ListRequest{}) setup.SetSandboxHeader(t, listReq.Header(), sbx.SandboxID) setup.SetUserHeader(t, listReq.Header(), "user") - listResp, err := envdClient.ProcessClient.List(ctx, listReq) - require.NoError(t, err) - assert.Len(t, listResp.Msg.GetProcesses(), 1, "Expected one process running") + var listResp *connect.Response[process.ListResponse] + require.EventuallyWithT(t, func(c *assert.CollectT) { + var err error + listResp, err = envdClient.ProcessClient.List(ctx, listReq) + require.NoError(c, err) + assert.Len(c, listResp.Msg.GetProcesses(), 1, "Expected one process running") + }, 30*time.Second, 500*time.Millisecond) // Kill all processes for _, proc := range listResp.Msg.GetProcesses() { diff --git a/tests/integration/internal/utils/template.go b/tests/integration/internal/utils/template.go index 803db729e6..e5ad2e036f 100644 --- a/tests/integration/internal/utils/template.go +++ b/tests/integration/internal/utils/template.go @@ -234,3 +234,19 @@ func BuildSimpleTemplate(tb testing.TB, name string, reqEditors ...api.RequestEd return BuildTemplate(tb, opts) } + +// RequestTemplateWithoutBuild creates a template without starting (or waiting +// for) its build. The template row, a build in the "waiting" state, the alias, +// and the default tag assignment all exist as soon as the request returns, so +// the template can be listed, updated (e.g. visibility), and deleted. +// +// Use this instead of BuildSimpleTemplate in tests that only exercise template +// metadata or authorization and never need a runnable build - a real build +// boots a Firecracker VM and takes minutes, while this is a single API call. +func RequestTemplateWithoutBuild(tb testing.TB, name string, reqEditors ...api.RequestEditorFn) *api.TemplateRequestResponseV3 { + tb.Helper() + + reqEditors = append(reqEditors, setup.WithTestsUserAgent()) + + return requestTemplateBuild(tb, name, nil, nil, nil, reqEditors...) +}