From ba5045a44896dcefbec229bf7913836e6e971e9b Mon Sep 17 00:00:00 2001 From: saratomaz Date: Tue, 17 Feb 2026 17:06:45 +0000 Subject: [PATCH 01/18] Add docker environment for containerized tests --- docker/Dockerfile | 23 ++++++++++++ docker/Dockerfile.dockerignore | 66 ++++++++++++++++++++++++++++++++++ docker/README.md | 59 ++++++++++++++++++++++++++++++ docker/docker-compose.yaml | 30 ++++++++++++++++ 4 files changed, 178 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile.dockerignore create mode 100644 docker/README.md create mode 100644 docker/docker-compose.yaml diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..7d2d10a9d --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,23 @@ +# Dockerfile for cardano-node-tests (Antithesis/Moog driver image) +# +# Minimal image: nix is configured and the repo is copied in. +# All heavy setup (cardano binaries, Python venv) happens at runtime +# via regression.sh, which manages its own nix environment through its shebang. +# +# Build and push to GHCR before submitting to Moog: +# docker build -f docker/Dockerfile -t ghcr.io/intersectmbo/cardano-node-tests-antithesis:latest . +# docker push ghcr.io/intersectmbo/cardano-node-tests-antithesis:latest + +FROM nixos/nix:2.25.5 + +ARG GIT_REVISION +ENV GIT_REVISION=${GIT_REVISION} + +RUN mkdir -p /etc/nix && \ + echo "extra-substituters = https://cache.iog.io" >> /etc/nix/nix.conf && \ + echo "extra-trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" >> /etc/nix/nix.conf && \ + echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf && \ + echo "accept-flake-config = true" >> /etc/nix/nix.conf + +WORKDIR /work +COPY . /work/ diff --git a/docker/Dockerfile.dockerignore b/docker/Dockerfile.dockerignore new file mode 100644 index 000000000..ea2e6c752 --- /dev/null +++ b/docker/Dockerfile.dockerignore @@ -0,0 +1,66 @@ +# Ignore unnecessary files during Docker build + +# Git +.git/ +.gitignore + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg-info/ +dist/ +build/ +*.egg + +# Virtual environments +.venv/ +venv/ +ENV/ +env/ + +# Testing artifacts +run_workdir/ +.artifacts/ +.cli_coverage/ +.reports/ +allure-results/ +allure-results.tar.xz +testrun-report.* +*.log +*.json.log + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# Nix +result +result-* + +# Documentation +docs/_build/ +*.md + +# Temporary files +*.tmp +*.bak +.DS_Store + +# Scripts output +scripts/destination/ +scripts/destination_working/ + +# Coverage +.coverage +htmlcov/ +cli_coverage.json +requirements_coverage.json + +# CI specific +.bin/ diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..f9b78d17c --- /dev/null +++ b/docker/README.md @@ -0,0 +1,59 @@ +# Docker setup for cardano-node-tests (Antithesis/Moog) + +This directory contains the driver image and compose file for submitting +`cardano-node-tests` to Antithesis via the Moog platform. + +## How it works + +- `Dockerfile` — minimal image: configures Nix and copies the repo into + `/work/`. No binaries are pre-built; `regression.sh` handles all setup at + runtime via its own Nix shebang. Requires `GIT_REVISION` build arg (the + current commit hash) so pytest can identify the revision without a `.git` + directory inside the image. +- `docker-compose.yaml` — single `driver` service for Moog submission. + +## Workflow + +### 1. Build and push the image + +```bash +docker build -f docker/Dockerfile \ + --build-arg GIT_REVISION=$(git rev-parse HEAD) \ + -t ghcr.io/intersectmbo/cardano-node-tests-antithesis:latest . + +docker push ghcr.io/intersectmbo/cardano-node-tests-antithesis:latest +``` + +### 2. Validate the compose locally + +```bash +docker compose -f docker/docker-compose.yaml config +docker compose -f docker/docker-compose.yaml up --build +``` + +### 3. Submit to Moog + +```bash +moog requester create-test \ + --platform github \ + --username saratomaz \ + --repository IntersectMBO/cardano-node-tests \ + --directory ./docker \ + --commit $(git rev-parse HEAD) \ + --try 1 \ + --duration 2 +``` + +## Environment variables + +| Variable | Default | Description | +|-------------------|------------|------------------------------------------| +| `NODE_REV` | `master` | cardano-node git revision | +| `CARDANO_CLI_REV` | (built-in) | cardano-cli revision, empty = use node's | +| `DBSYNC_REV` | (disabled) | db-sync revision, empty = disabled | +| `RUN_TARGET` | `tests` | `tests`, `testpr`, or `testnets` | +| `MARKEXPR` | | pytest marker expression | +| `CLUSTERS_COUNT` | | number of local cluster instances | +| `CLUSTER_ERA` | | e.g. `conway` | +| `PROTOCOL_VERSION`| | e.g. `11` | +| `UTXO_BACKEND` | | e.g. `disk`, `mem` | diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 000000000..06f73d541 --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,30 @@ +# Docker Compose for Antithesis/Moog test submission. +# +# Submit to Moog: +# moog requester create-test \ +# --platform github --username \ +# --repository IntersectMBO/cardano-node-tests \ +# --directory ./docker \ +# --commit --try 1 --duration 2 +# +# Validate locally before submitting: +# docker compose -f docker/docker-compose.yaml config +# docker compose -f docker/docker-compose.yaml up --build + +services: + driver: + image: ghcr.io/intersectmbo/cardano-node-tests-antithesis:latest + build: + context: .. + dockerfile: docker/Dockerfile + command: ["/work/.github/regression.sh"] + environment: + - NODE_REV=${NODE_REV:-master} + - CARDANO_CLI_REV=${CARDANO_CLI_REV:-} + - DBSYNC_REV=${DBSYNC_REV:-} + - RUN_TARGET=${RUN_TARGET:-tests} + - MARKEXPR=${MARKEXPR:-} + - CLUSTERS_COUNT=${CLUSTERS_COUNT:-} + - CLUSTER_ERA=${CLUSTER_ERA:-} + - PROTOCOL_VERSION=${PROTOCOL_VERSION:-} + - UTXO_BACKEND=${UTXO_BACKEND:-} From fcfdc6a0df10e6b55573a663b3c0295e124841ff Mon Sep 17 00:00:00 2001 From: saratomaz Date: Fri, 27 Mar 2026 12:57:39 +0000 Subject: [PATCH 02/18] Adapt docker to antithesis --- docker/Dockerfile | 74 ++++++++++++++++++++++++++++++++------ docker/Dockerfile.config | 13 +++++++ docker/README.md | 73 +++++++++++++++++++++++++------------ docker/antithesis_run.sh | 49 +++++++++++++++++++++++++ docker/docker-compose.yaml | 26 ++++++++------ runner/regression.sh | 38 ++++++++++++++------ 6 files changed, 217 insertions(+), 56 deletions(-) create mode 100644 docker/Dockerfile.config create mode 100755 docker/antithesis_run.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 7d2d10a9d..0bae1dfe6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,23 +1,75 @@ -# Dockerfile for cardano-node-tests (Antithesis/Moog driver image) +# Dockerfile for cardano-node-tests (Antithesis-compatible driver image) # -# Minimal image: nix is configured and the repo is copied in. -# All heavy setup (cardano binaries, Python venv) happens at runtime -# via regression.sh, which manages its own nix environment through its shebang. +# All heavy dependencies are baked in at image build time so the container +# runs without any network access (required by Antithesis environments). # -# Build and push to GHCR before submitting to Moog: -# docker build -f docker/Dockerfile -t ghcr.io/intersectmbo/cardano-node-tests-antithesis:latest . -# docker push ghcr.io/intersectmbo/cardano-node-tests-antithesis:latest +# Build args: +# GIT_REVISION — git commit hash stored as $GIT_REVISION in the image +# NODE_REV — cardano-node git ref to pre-build (default: master) +# +# Build and push to GHCR before submitting to Antithesis: +# docker build -f docker/Dockerfile \ +# --build-arg GIT_REVISION=$(git rev-parse HEAD) \ +# --build-arg NODE_REV=master \ +# -t ghcr.io/saratomaz/cardano-node-tests-antithesis:latest . +# docker push ghcr.io/saratomaz/cardano-node-tests-antithesis:latest FROM nixos/nix:2.25.5 ARG GIT_REVISION +ARG NODE_REV=master + ENV GIT_REVISION=${GIT_REVISION} +# Store the baked-in node revision for reference at runtime. +ENV BAKED_NODE_REV=${NODE_REV} +# Configure Nix with IOG binary cache and required experimental features. RUN mkdir -p /etc/nix && \ - echo "extra-substituters = https://cache.iog.io" >> /etc/nix/nix.conf && \ - echo "extra-trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" >> /etc/nix/nix.conf && \ - echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf && \ - echo "accept-flake-config = true" >> /etc/nix/nix.conf + printf 'extra-substituters = https://cache.iog.io\n\ +extra-trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=\n\ +experimental-features = nix-command flakes\n\ +accept-flake-config = true\n' >> /etc/nix/nix.conf WORKDIR /work COPY . /work/ + +# Pre-build cardano-node, cardano-submit-api, cardano-cli, and bech32 into /opt/cardano/. +# NODE_REV is fixed at image build time — no network access is needed at runtime. +RUN mkdir -p /opt/cardano && \ + nix build \ + --accept-flake-config --no-write-lock-file \ + "github://github.com/IntersectMBO/cardano-node?ref=${NODE_REV}#cardano-node" \ + -o /opt/cardano/cardano-node && \ + nix build \ + --accept-flake-config --no-write-lock-file \ + "github://github.com/IntersectMBO/cardano-node?ref=${NODE_REV}#cardano-submit-api" \ + -o /opt/cardano/cardano-submit-api && \ + nix build \ + --accept-flake-config --no-write-lock-file \ + "github://github.com/IntersectMBO/cardano-node?ref=${NODE_REV}#cardano-cli" \ + -o /opt/cardano/cardano-cli && \ + nix build \ + --accept-flake-config --no-write-lock-file \ + "github://github.com/IntersectMBO/cardano-node?ref=${NODE_REV}#bech32" \ + -o /opt/cardano/bech32 + +# Pre-warm the testenv dev shell (pulls nixpkgs, postgres, uv, python313 into the +# nix store) and create the Python venv at /opt/tests-venv with all project +# dependencies installed. This is the same step regression.sh does at runtime +# but done here so no pip/uv network calls are needed in the Antithesis env. +RUN nix develop --accept-flake-config .#testenv --command \ + bash -c 'python3 -m venv /opt/tests-venv --prompt tests-venv && \ + . /opt/tests-venv/bin/activate && \ + cd /work && \ + uv sync --active --no-dev' + +# Pre-warm the base dev shell (bash, coreutils, git, jq, …) so its store +# paths are cached and the regression.sh shebang resolves offline. +RUN nix develop --accept-flake-config .#base --command true + +# Create the Antithesis test driver directory and install the entry-point. +# singleton_driver_* files are run once per test run by Antithesis. +RUN mkdir -p /opt/antithesis/test/v1/quickstart && \ + cp /work/docker/antithesis_run.sh \ + /opt/antithesis/test/v1/quickstart/singleton_driver_regression.sh && \ + chmod +x /opt/antithesis/test/v1/quickstart/singleton_driver_regression.sh diff --git a/docker/Dockerfile.config b/docker/Dockerfile.config new file mode 100644 index 000000000..ce0461dd7 --- /dev/null +++ b/docker/Dockerfile.config @@ -0,0 +1,13 @@ +# Config image for Antithesis. +# +# Contains only the docker-compose.yaml that tells Antithesis how to run +# the services. Must be pushed to the Antithesis registry alongside the +# driver image. +# +# Build: +# docker build -f docker/Dockerfile.config \ +# -t us-central1-docker.pkg.dev//antithesis/config:latest . +# docker push us-central1-docker.pkg.dev//antithesis/config:latest + +FROM scratch +COPY docker/docker-compose.yaml /docker-compose.yaml diff --git a/docker/README.md b/docker/README.md index f9b78d17c..2cb54c300 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,54 +1,81 @@ -# Docker setup for cardano-node-tests (Antithesis/Moog) +# Docker setup for cardano-node-tests (Antithesis) -This directory contains the driver image and compose file for submitting -`cardano-node-tests` to Antithesis via the Moog platform. +This directory contains the driver image and compose files for submitting +`cardano-node-tests` to Antithesis. ## How it works -- `Dockerfile` — minimal image: configures Nix and copies the repo into - `/work/`. No binaries are pre-built; `regression.sh` handles all setup at - runtime via its own Nix shebang. Requires `GIT_REVISION` build arg (the - current commit hash) so pytest can identify the revision without a `.git` - directory inside the image. -- `docker-compose.yaml` — single `driver` service for Moog submission. +Antithesis environments have **no internet access** at runtime, so all +dependencies are baked into the image at build time: + +- `Dockerfile` — builds the driver image. At build time it: + 1. Pre-builds `cardano-node`, `cardano-submit-api`, `cardano-cli`, and + `bech32` from `NODE_REV` into `/opt/cardano/` via `nix build`. + 2. Pre-warms the `testenv` dev shell and creates the Python venv at + `/opt/tests-venv/` with all project dependencies installed. + 3. Pre-warms the `base` dev shell so the `regression.sh` shebang resolves + from the local nix store without network access. + 4. Installs `antithesis_run.sh` as the Antithesis test driver at + `/opt/antithesis/test/v1/quickstart/singleton_driver_regression.sh`. + +- `antithesis_run.sh` — container entrypoint that: + 1. Forces nix into offline mode (`offline = true`). + 2. Exports `CARDANO_PREBUILT_DIR=/opt/cardano` and `_VENV_DIR=/opt/tests-venv` + so `regression.sh` skips all downloads and uses the pre-built artefacts. + 3. Emits the Antithesis `setup_complete` lifecycle signal. + 4. Hands off to `.github/regression.sh`. + +- `Dockerfile.config` — builds the Antithesis config image (`FROM scratch`) + containing only `docker-compose.yaml`. + +- `docker-compose.yaml` — single `driver` service. ## Workflow -### 1. Build and push the image +### 1. Build and push the driver image ```bash docker build -f docker/Dockerfile \ --build-arg GIT_REVISION=$(git rev-parse HEAD) \ - -t ghcr.io/intersectmbo/cardano-node-tests-antithesis:latest . + --build-arg NODE_REV=master \ + -t ghcr.io/saratomaz/cardano-node-tests-antithesis:latest . -docker push ghcr.io/intersectmbo/cardano-node-tests-antithesis:latest +docker push ghcr.io/saratomaz/cardano-node-tests-antithesis:latest ``` -### 2. Validate the compose locally +`NODE_REV` is locked at build time — the same binaries are used every run +regardless of what is on the `master` branch when the container starts. + +### 2. Build and push the config image + +```bash +docker build -f docker/Dockerfile.config \ + -t us-central1-docker.pkg.dev//antithesis/config:latest . + +docker push us-central1-docker.pkg.dev//antithesis/config:latest +``` + +### 3. Validate locally (internet-connected build, isolated network at runtime) ```bash docker compose -f docker/docker-compose.yaml config docker compose -f docker/docker-compose.yaml up --build ``` -### 3. Submit to Moog +To fully simulate the Antithesis no-internet constraint, run inside an +isolated network namespace on Linux: ```bash -moog requester create-test \ - --platform github \ - --username saratomaz \ - --repository IntersectMBO/cardano-node-tests \ - --directory ./docker \ - --commit $(git rev-parse HEAD) \ - --try 1 \ - --duration 2 +unshare -n docker compose -f docker/docker-compose.yaml up ``` ## Environment variables +`NODE_REV` is baked into the image at build time and must **not** be set at +runtime. All other variables are passed through docker-compose as before. + | Variable | Default | Description | |-------------------|------------|------------------------------------------| -| `NODE_REV` | `master` | cardano-node git revision | | `CARDANO_CLI_REV` | (built-in) | cardano-cli revision, empty = use node's | | `DBSYNC_REV` | (disabled) | db-sync revision, empty = disabled | | `RUN_TARGET` | `tests` | `tests`, `testpr`, or `testnets` | diff --git a/docker/antithesis_run.sh b/docker/antithesis_run.sh new file mode 100755 index 000000000..d4a6282e3 --- /dev/null +++ b/docker/antithesis_run.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Antithesis entrypoint for cardano-node-tests. +# +# Runs the full test suite without any network access by: +# 1. Forcing nix into offline mode (all store paths were pre-built into +# the image by docker/Dockerfile). +# 2. Pointing regression.sh at the pre-built cardano binaries and Python +# venv so it skips all download / build steps. +# 3. Emitting the Antithesis `setup_complete` lifecycle signal before +# starting pytest. +# +# This file is installed at: +# /opt/antithesis/test/v1/quickstart/singleton_driver_regression.sh +# and is also usable directly as the docker-compose `command`. + +set -Eeuo pipefail + +# --------------------------------------------------------------------------- +# 1. Force nix offline — all required store paths were pre-built into the +# image. This prevents nix from attempting any network calls at runtime, +# which would fail inside the Antithesis environment. +# --------------------------------------------------------------------------- +echo "offline = true" >> /etc/nix/nix.conf + +# --------------------------------------------------------------------------- +# 2. Tell regression.sh to use the pre-built binaries and Python venv that +# were baked into the image at docker build time. +# --------------------------------------------------------------------------- +export CARDANO_PREBUILT_DIR=/opt/cardano +export _VENV_DIR=/opt/tests-venv + +# --------------------------------------------------------------------------- +# 3. Emit the Antithesis setup_complete signal. +# Written as JSONL to $ANTITHESIS_OUTPUT_DIR/sdk.jsonl. +# Antithesis begins fault injection / test orchestration after receiving +# this message. +# --------------------------------------------------------------------------- +_output_dir="${ANTITHESIS_OUTPUT_DIR:-/tmp/antithesis}" +mkdir -p "$_output_dir" +printf '{"antithesis_setup": {"status": "complete", "details": {"info": ["cardano-node-tests driver ready, node_rev=%s"]}}}\n' \ + "${BAKED_NODE_REV:-unknown}" >> "$_output_dir/sdk.jsonl" +unset _output_dir + +# --------------------------------------------------------------------------- +# 4. Hand off to regression.sh. The shebang in that script will invoke +# `nix develop .#base` which now resolves entirely from the local nix +# store (offline = true). +# --------------------------------------------------------------------------- +exec /work/.github/regression.sh diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 06f73d541..d8e82da32 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,25 +1,29 @@ -# Docker Compose for Antithesis/Moog test submission. +# Docker Compose for Antithesis test submission. # -# Submit to Moog: -# moog requester create-test \ -# --platform github --username \ -# --repository IntersectMBO/cardano-node-tests \ -# --directory ./docker \ -# --commit --try 1 --duration 2 +# The driver image must be pre-built with all cardano binaries and the Python +# venv baked in (see docker/Dockerfile). No internet access is available at +# runtime inside the Antithesis environment. # -# Validate locally before submitting: +# Push images to the Antithesis registry before submitting: +# docker push us-central1-docker.pkg.dev//antithesis/cardano-node-tests:latest +# docker push us-central1-docker.pkg.dev//antithesis/config:latest +# +# Validate locally (requires internet — use an isolated netns to simulate +# the Antithesis environment): # docker compose -f docker/docker-compose.yaml config # docker compose -f docker/docker-compose.yaml up --build services: driver: - image: ghcr.io/intersectmbo/cardano-node-tests-antithesis:latest + image: ghcr.io/saratomaz/cardano-node-tests-antithesis:latest build: context: .. dockerfile: docker/Dockerfile - command: ["/work/.github/regression.sh"] + # antithesis_run.sh sets nix offline, exports pre-built paths, emits + # setup_complete, then hands off to regression.sh. + command: ["/work/docker/antithesis_run.sh"] environment: - - NODE_REV=${NODE_REV:-master} + # NODE_REV is baked into the image at build time; do not override here. - CARDANO_CLI_REV=${CARDANO_CLI_REV:-} - DBSYNC_REV=${DBSYNC_REV:-} - RUN_TARGET=${RUN_TARGET:-tests} diff --git a/runner/regression.sh b/runner/regression.sh index 2fb42ae84..c0c76732d 100755 --- a/runner/regression.sh +++ b/runner/regression.sh @@ -130,16 +130,25 @@ case "${CARDANO_CLI_REV:-}" in esac # setup cardano-node binaries -case "${NODE_REV:-}" in - "" | "none" ) - NODE_REV=master - ;; -esac -# shellcheck disable=SC1091 -. runner/source_cardano_node.sh -cardano_bins_build_all "$NODE_REV" "${CARDANO_CLI_REV:-}" -PATH_PREPEND="$(cardano_bins_print_path_prepend "${CARDANO_CLI_REV:-}")${PATH_PREPEND}" -export PATH_PREPEND +if [ -n "${CARDANO_PREBUILT_DIR:-}" ]; then + # Pre-built binaries were baked into the image (e.g. for Antithesis). + # Skip all nix builds and point PATH_PREPEND at the pre-built directories. + _d="${CARDANO_PREBUILT_DIR}" + PATH_PREPEND="${_d}/cardano-node/bin:${_d}/cardano-submit-api/bin:${_d}/cardano-cli/bin:${_d}/bech32/bin:${PATH_PREPEND}" + export PATH_PREPEND + unset _d +else + case "${NODE_REV:-}" in + "" | "none" ) + NODE_REV=master + ;; + esac + # shellcheck disable=SC1091 + . runner/source_cardano_node.sh + cardano_bins_build_all "$NODE_REV" "${CARDANO_CLI_REV:-}" + PATH_PREPEND="$(cardano_bins_print_path_prepend "${CARDANO_CLI_REV:-}")${PATH_PREPEND}" + export PATH_PREPEND +fi # optimize nix store if running in GitHub Actions if [ -n "${GITHUB_ACTIONS:-}" ]; then @@ -290,7 +299,14 @@ nix develop --accept-flake-config .#testenv --command bash -c ' echo "::group::Python venv setup" printf "start: %(%H:%M:%S)T\n" -1 - . runner/setup_venv.sh clean + # When _VENV_DIR points to a pre-built venv (e.g. baked into the image for + # Antithesis), skip the `clean` flag so the existing venv is reused as-is + # without re-downloading packages. + if [ -n "${_VENV_DIR:-}" ] && [ -e "${_VENV_DIR}" ]; then + . runner/setup_venv.sh + else + . runner/setup_venv.sh clean + fi echo "::endgroup::" # end group for "Python venv setup" echo "::group::🧪 Testrun" From 5d78f92454268c3cb6f573e8888bf5e7dfd61330 Mon Sep 17 00:00:00 2001 From: saratomaz Date: Mon, 13 Apr 2026 15:43:03 +0100 Subject: [PATCH 03/18] Fix exit code from antithesis script --- docker/antithesis_run.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docker/antithesis_run.sh b/docker/antithesis_run.sh index d4a6282e3..613f898d8 100755 --- a/docker/antithesis_run.sh +++ b/docker/antithesis_run.sh @@ -45,5 +45,15 @@ unset _output_dir # 4. Hand off to regression.sh. The shebang in that script will invoke # `nix develop .#base` which now resolves entirely from the local nix # store (offline = true). -# --------------------------------------------------------------------------- -exec /work/.github/regression.sh +# +# Do not exec directly: Antithesis treats any non-zero container exit +# code (other than 137/143) as an error property violation. Test +# failures are expected and communicated via SDK assertions, not the +# process exit code. Always exit 0 so the container is not flagged. +# --------------------------------------------------------------------------- +set +e +/work/.github/regression.sh +_rc=$? +set -e +echo "regression.sh finished with exit code ${_rc}" +exit 0 From 0dec94537b12f2cc994342abd5bd2033af7ad413 Mon Sep 17 00:00:00 2001 From: saratomaz Date: Thu, 16 Apr 2026 10:30:55 +0100 Subject: [PATCH 04/18] fix docker antithesis, add antithesis-net bridge network to compose --- docker/docker-compose.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index d8e82da32..d30f59fbe 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -13,6 +13,10 @@ # docker compose -f docker/docker-compose.yaml config # docker compose -f docker/docker-compose.yaml up --build +networks: + antithesis-net: + driver: bridge + services: driver: image: ghcr.io/saratomaz/cardano-node-tests-antithesis:latest @@ -22,6 +26,8 @@ services: # antithesis_run.sh sets nix offline, exports pre-built paths, emits # setup_complete, then hands off to regression.sh. command: ["/work/docker/antithesis_run.sh"] + networks: + - antithesis-net environment: # NODE_REV is baked into the image at build time; do not override here. - CARDANO_CLI_REV=${CARDANO_CLI_REV:-} From ede7c30a1bf8a4fe6691c2d7cbaabfef78270fe2 Mon Sep 17 00:00:00 2001 From: saratomaz Date: Thu, 16 Apr 2026 16:03:13 +0100 Subject: [PATCH 05/18] Docker antithesis - split into node and driver containers --- docker/Dockerfile | 5 +- docker/README.md | 6 +- docker/antithesis_run.sh | 79 ++++++++++++++++++++++---- docker/docker-compose.yaml | 56 +++++++++++++++++-- docker/node_run.sh | 112 +++++++++++++++++++++++++++++++++++++ runner/regression.sh | 3 +- 6 files changed, 241 insertions(+), 20 deletions(-) create mode 100755 docker/node_run.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 0bae1dfe6..81f2b4f24 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -67,9 +67,10 @@ RUN nix develop --accept-flake-config .#testenv --command \ # paths are cached and the regression.sh shebang resolves offline. RUN nix develop --accept-flake-config .#base --command true -# Create the Antithesis test driver directory and install the entry-point. +# Create the Antithesis test driver directory and install the entry-points. # singleton_driver_* files are run once per test run by Antithesis. RUN mkdir -p /opt/antithesis/test/v1/quickstart && \ cp /work/docker/antithesis_run.sh \ /opt/antithesis/test/v1/quickstart/singleton_driver_regression.sh && \ - chmod +x /opt/antithesis/test/v1/quickstart/singleton_driver_regression.sh + chmod +x /opt/antithesis/test/v1/quickstart/singleton_driver_regression.sh && \ + chmod +x /work/docker/node_run.sh diff --git a/docker/README.md b/docker/README.md index 2cb54c300..c38088a94 100644 --- a/docker/README.md +++ b/docker/README.md @@ -28,7 +28,11 @@ dependencies are baked into the image at build time: - `Dockerfile.config` — builds the Antithesis config image (`FROM scratch`) containing only `docker-compose.yaml`. -- `docker-compose.yaml` — single `driver` service. +- `docker-compose.yaml` — two services: `node` (cardano-node cluster) and + `driver` (pytest). Both share a `cluster-state` Docker volume so the + driver accesses the node sockets without going over the network. An HTTP + health check on port 8090 provides cross-container traffic that satisfies + the Antithesis "Containers joined the Antithesis network" property. ## Workflow diff --git a/docker/antithesis_run.sh b/docker/antithesis_run.sh index 613f898d8..bcfdf4f69 100755 --- a/docker/antithesis_run.sh +++ b/docker/antithesis_run.sh @@ -1,17 +1,27 @@ #!/usr/bin/env bash -# Antithesis entrypoint for cardano-node-tests. +# Antithesis driver container entrypoint. # # Runs the full test suite without any network access by: # 1. Forcing nix into offline mode (all store paths were pre-built into # the image by docker/Dockerfile). # 2. Pointing regression.sh at the pre-built cardano binaries and Python # venv so it skips all download / build steps. -# 3. Emitting the Antithesis `setup_complete` lifecycle signal before -# starting pytest. +# 3. When NODE_HOST is set (multi-container mode): waiting for the node +# container's health check on port 8090 before running tests, and +# setting DEV_CLUSTER_RUNNING=1 so pytest uses the pre-running cluster +# instead of starting its own. +# 4. Emitting the Antithesis setup_complete lifecycle signal. +# 5. Handing off to regression.sh. +# +# Multi-container environment variables (set in docker-compose): +# NODE_HOST Hostname of the node container (default: unset). +# NODE_PORT Health check port on the node container (default: 8090). +# CLUSTER_STATE_DIR Mount point of the shared cluster-state volume +# (default: /cluster-state). # # This file is installed at: # /opt/antithesis/test/v1/quickstart/singleton_driver_regression.sh -# and is also usable directly as the docker-compose `command`. +# and is also usable directly as the docker-compose command. set -Eeuo pipefail @@ -29,20 +39,67 @@ echo "offline = true" >> /etc/nix/nix.conf export CARDANO_PREBUILT_DIR=/opt/cardano export _VENV_DIR=/opt/tests-venv -# --------------------------------------------------------------------------- -# 3. Emit the Antithesis setup_complete signal. -# Written as JSONL to $ANTITHESIS_OUTPUT_DIR/sdk.jsonl. -# Antithesis begins fault injection / test orchestration after receiving -# this message. -# --------------------------------------------------------------------------- _output_dir="${ANTITHESIS_OUTPUT_DIR:-/tmp/antithesis}" mkdir -p "$_output_dir" + +# --------------------------------------------------------------------------- +# 3. Multi-container mode: wait for the node container and configure the +# driver to use the pre-running cluster. +# +# When NODE_HOST is set the driver polls the node's HTTP health endpoint +# (port 8090) until it responds "ready". This HTTP traffic is what makes +# both containers visible on the Antithesis network bridge. +# +# DEV_CLUSTER_RUNNING=1 tells pytest to skip cluster startup/shutdown and +# use the cluster already started by the node container. +# CARDANO_NODE_SOCKET_PATH_CI is pre-set to the shared volume socket path +# so regression.sh does not override it with its default workdir path. +# --------------------------------------------------------------------------- +if [ -n "${NODE_HOST:-}" ]; then + _node_port="${NODE_PORT:-8090}" + echo "Waiting for ${NODE_HOST}:${_node_port} to report ready..." + + _ready=0 + for _i in $(seq 1 120); do + # Use the venv's Python directly — python3 is not in PATH outside a nix shell. + _resp="$("${_VENV_DIR}/bin/python3" -c " +import urllib.request, sys +try: + r = urllib.request.urlopen('http://${NODE_HOST}:${_node_port}/', timeout=5) + sys.stdout.write(r.read().decode()) +except Exception: + pass +" 2>/dev/null || true)" + if [ "$_resp" = "ready" ]; then + _ready=1 + break + fi + echo " attempt ${_i}/120: node reports '${_resp:-no response}', retrying in 5s..." + sleep 5 + done + + if [ "$_ready" -ne 1 ]; then + echo "ERROR: node container did not become ready within 10 minutes" >&2 + exit 1 + fi + echo "Node is ready." + + CLUSTER_STATE_DIR="${CLUSTER_STATE_DIR:-/cluster-state}" + export DEV_CLUSTER_RUNNING=1 + export CLUSTERS_COUNT="${CLUSTERS_COUNT:-1}" + # Pre-set so regression.sh does not overwrite with its default workdir path. + export CARDANO_NODE_SOCKET_PATH_CI="${CLUSTER_STATE_DIR}/state-cluster0/bft1.socket" +fi + +# --------------------------------------------------------------------------- +# 4. Emit the Antithesis setup_complete signal. +# --------------------------------------------------------------------------- printf '{"antithesis_setup": {"status": "complete", "details": {"info": ["cardano-node-tests driver ready, node_rev=%s"]}}}\n' \ "${BAKED_NODE_REV:-unknown}" >> "$_output_dir/sdk.jsonl" unset _output_dir # --------------------------------------------------------------------------- -# 4. Hand off to regression.sh. The shebang in that script will invoke +# 5. Hand off to regression.sh. The shebang in that script will invoke # `nix develop .#base` which now resolves entirely from the local nix # store (offline = true). # diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index d30f59fbe..a86e1facb 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,7 +1,18 @@ # Docker Compose for Antithesis test submission. # -# The driver image must be pre-built with all cardano binaries and the Python -# venv baked in (see docker/Dockerfile). No internet access is available at +# Two services share a cluster-state volume: +# +# node — starts the cardano-node cluster (system under test). +# Serves a health check on port 8090 so the driver can detect +# when the cluster is ready. The traffic between driver and node +# over the antithesis-net bridge satisfies the Antithesis +# "Containers joined the Antithesis network" property. +# +# driver — waits for the node health check, then runs the pytest test +# suite against the pre-running cluster via DEV_CLUSTER_RUNNING=1. +# +# Both images must be pre-built with all cardano binaries and the Python venv +# baked in (see docker/Dockerfile). No internet access is available at # runtime inside the Antithesis environment. # # Push images to the Antithesis registry before submitting: @@ -17,24 +28,59 @@ networks: antithesis-net: driver: bridge +volumes: + cluster-state: + services: + node: + image: ghcr.io/saratomaz/cardano-node-tests-antithesis:latest + build: + context: .. + dockerfile: docker/Dockerfile + command: ["/work/docker/node_run.sh"] + networks: + - antithesis-net + volumes: + - cluster-state:/cluster-state + environment: + - CLUSTER_STATE_DIR=/cluster-state + - TESTNET_VARIANT=${TESTNET_VARIANT:-conway_fast} + healthcheck: + test: + - "CMD" + - "/opt/tests-venv/bin/python3" + - "-c" + - "import urllib.request; exit(0 if urllib.request.urlopen('http://localhost:8090/', timeout=5).read() == b'ready' else 1)" + interval: 15s + timeout: 6s + retries: 60 + start_period: 60s + driver: image: ghcr.io/saratomaz/cardano-node-tests-antithesis:latest build: context: .. dockerfile: docker/Dockerfile - # antithesis_run.sh sets nix offline, exports pre-built paths, emits - # setup_complete, then hands off to regression.sh. + # antithesis_run.sh sets nix offline, waits for the node health check, + # exports DEV_CLUSTER_RUNNING=1, emits setup_complete, then hands off + # to regression.sh. command: ["/work/docker/antithesis_run.sh"] networks: - antithesis-net + depends_on: + - node + volumes: + - cluster-state:/cluster-state environment: + - CLUSTER_STATE_DIR=/cluster-state + - NODE_HOST=node + - NODE_PORT=8090 # NODE_REV is baked into the image at build time; do not override here. - CARDANO_CLI_REV=${CARDANO_CLI_REV:-} - DBSYNC_REV=${DBSYNC_REV:-} - RUN_TARGET=${RUN_TARGET:-tests} - MARKEXPR=${MARKEXPR:-} - - CLUSTERS_COUNT=${CLUSTERS_COUNT:-} + - CLUSTERS_COUNT=${CLUSTERS_COUNT:-1} - CLUSTER_ERA=${CLUSTER_ERA:-} - PROTOCOL_VERSION=${PROTOCOL_VERSION:-} - UTXO_BACKEND=${UTXO_BACKEND:-} diff --git a/docker/node_run.sh b/docker/node_run.sh new file mode 100755 index 000000000..4d6ef6ab8 --- /dev/null +++ b/docker/node_run.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Antithesis node container entrypoint. +# +# 1. Starts the cardano-node cluster on the shared 'cluster-state' volume so +# the driver container can reach the node sockets without going over the +# network (Unix socket on a shared Docker volume). +# 2. Serves a lightweight HTTP health check on port 8090 over the Antithesis +# network bridge. Returns "ready" once the cluster socket exists. +# This cross-container HTTP traffic satisfies the Antithesis +# "Containers joined the Antithesis network" property. +# +# Environment variables: +# CLUSTER_STATE_DIR Mount point of the shared cluster-state volume +# (default: /cluster-state). +# TESTNET_VARIANT Cluster variant passed to prepare_cluster_scripts +# (default: conway_fast). + +set -Eeuo pipefail + +# --------------------------------------------------------------------------- +# 1. Force nix offline — all store paths are pre-built into the image. +# --------------------------------------------------------------------------- +echo "offline = true" >> /etc/nix/nix.conf + +# --------------------------------------------------------------------------- +# 2. Point at pre-built binaries and Python venv. +# All variables are exported so the inner nix shell inherits them. +# --------------------------------------------------------------------------- +export CARDANO_PREBUILT_DIR=/opt/cardano +export _VENV_DIR=/opt/tests-venv +export _PATH_PREPEND="/opt/cardano/cardano-node/bin:/opt/cardano/cardano-submit-api/bin:/opt/cardano/cardano-cli/bin:/opt/cardano/bech32/bin" + +# --------------------------------------------------------------------------- +# 3. Cluster state lives on the shared volume so the driver can read sockets. +# --------------------------------------------------------------------------- +CLUSTER_STATE_DIR="${CLUSTER_STATE_DIR:-/cluster-state}" +export _INSTANCE_NUM=0 +export _STATE_CLUSTER="${CLUSTER_STATE_DIR}/state-cluster${_INSTANCE_NUM}" +export _SCRIPTS_DEST="${CLUSTER_STATE_DIR}/startup_scripts" +export CLUSTER_STATE_DIR + +# Local clusters (conway_fast, etc.) use bft1.socket. +export CARDANO_NODE_SOCKET_PATH="${_STATE_CLUSTER}/bft1.socket" + +export _output_dir="${ANTITHESIS_OUTPUT_DIR:-/tmp/antithesis}" +mkdir -p "$_output_dir" "${CLUSTER_STATE_DIR}" + +# --------------------------------------------------------------------------- +# 4. Health check server on port 8090 (Antithesis network bridge traffic). +# Returns HTTP 200 "ready" once the cluster socket file exists, +# 503 "starting" while the cluster is still coming up. +# Uses the venv Python directly — python3 is not in PATH outside a nix shell. +# --------------------------------------------------------------------------- +"${_VENV_DIR}/bin/python3" -c " +import os, socket as _s +_sock_path = os.environ.get('CARDANO_NODE_SOCKET_PATH', '') +server = _s.socket(_s.AF_INET, _s.SOCK_STREAM) +server.setsockopt(_s.SOL_SOCKET, _s.SO_REUSEADDR, 1) +server.bind(('0.0.0.0', 8090)) +server.listen(64) +while True: + conn, _ = server.accept() + ready = os.path.exists(_sock_path) + body = b'ready' if ready else b'starting' + status = b'200 OK' if ready else b'503 Service Unavailable' + conn.sendall(b'HTTP/1.1 ' + status + b'\r\nContent-Length: ' + str(len(body)).encode() + b'\r\n\r\n' + body) + conn.close() +" & +_health_pid=$! +trap 'kill "$_health_pid" 2>/dev/null || true' EXIT + +# --------------------------------------------------------------------------- +# 5. Prepare cluster startup scripts and run the cluster. +# The inner script uses single quotes so the outer shell does NOT expand +# variables — the nix shell inherits all exported vars above and the inner +# bash expands them from its environment. This avoids PATH corruption from +# nested quoting (single quotes inside a double-quoted string are literal +# and prevent $PATH from being expanded in the inner shell). +# --------------------------------------------------------------------------- +export _testnet_variant="${TESTNET_VARIANT:-conway_fast}" + +set +e +# shellcheck disable=SC2016 +nix develop --accept-flake-config .#testenv --command bash -c ' + set -euo pipefail + . "$_VENV_DIR/bin/activate" + export PATH="$_PATH_PREPEND:$PATH" + + # Instantiate cluster scripts for instance $_INSTANCE_NUM into the + # shared volume. --clean removes any previous attempt. + python3 -m cardano_node_tests.prepare_cluster_scripts \ + --dest-dir "$_SCRIPTS_DEST" \ + --testnet-variant "$_testnet_variant" \ + --instance-num "$_INSTANCE_NUM" \ + --clean + + # start-cluster must run from the parent of the state-cluster directory. + cd "$CLUSTER_STATE_DIR" + "$_SCRIPTS_DEST/start-cluster" + + # shellcheck disable=SC2016 + printf '"'"'{"antithesis_setup": {"status": "complete", "details": {"info": ["cardano-node cluster ready, socket=%s"]}}}\n'"'"' \ + "$CARDANO_NODE_SOCKET_PATH" >> "$_output_dir/sdk.jsonl" + + # Keep the cluster alive until the container is stopped. + tail -f /dev/null +' +_rc=$? +set -e + +echo "node_run.sh exiting with code ${_rc}" +exit 0 diff --git a/runner/regression.sh b/runner/regression.sh index c0c76732d..bd7993a23 100755 --- a/runner/regression.sh +++ b/runner/regression.sh @@ -90,7 +90,8 @@ if [ "${CI_CONSENSUS_MODE:-}" = "Genesis" ]; then export USE_GENESIS_MODE=true fi -export CARDANO_NODE_SOCKET_PATH_CI="$WORKDIR/state-cluster0/bft1.socket" +CARDANO_NODE_SOCKET_PATH_CI="${CARDANO_NODE_SOCKET_PATH_CI:-$WORKDIR/state-cluster0/bft1.socket}" +export CARDANO_NODE_SOCKET_PATH_CI # assume we run tests on testnet when `BOOTSTRAP_DIR` is set if [ -n "${BOOTSTRAP_DIR:-}" ]; then From d57c073ad73de1d57b80bf0338148a8091e17ebc Mon Sep 17 00:00:00 2001 From: saratomaz Date: Tue, 28 Apr 2026 11:16:38 +0100 Subject: [PATCH 06/18] fix antithesis driver, always exits with code 0 --- docker/antithesis_run.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/antithesis_run.sh b/docker/antithesis_run.sh index bcfdf4f69..b5a7af062 100755 --- a/docker/antithesis_run.sh +++ b/docker/antithesis_run.sh @@ -80,7 +80,9 @@ except Exception: if [ "$_ready" -ne 1 ]; then echo "ERROR: node container did not become ready within 10 minutes" >&2 - exit 1 + printf '{"antithesis_assert": {"type": "always", "condition": false, "display_name": "Node became ready", "message": "Node container did not become ready within 10 minutes", "details": {"node_host": "%s", "node_port": "%s"}, "location": {"function": "antithesis_run.sh", "file": "antithesis_run.sh", "begin_line": 1, "begin_column": 1, "class": ""}}}\n' \ + "${NODE_HOST:-}" "${_node_port:-8090}" >> "${ANTITHESIS_OUTPUT_DIR:-/tmp/antithesis}/sdk.jsonl" + exit 0 fi echo "Node is ready." From 0c9e5cc5831780566a74487fe78806816708f9f4 Mon Sep 17 00:00:00 2001 From: saratomaz Date: Tue, 5 May 2026 13:25:14 +0100 Subject: [PATCH 07/18] fix antithesis - ensure driver runs to completion --- docker/docker-compose.yaml | 3 ++- runner/run_tests.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index a86e1facb..43cc9a66c 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -79,7 +79,8 @@ services: - CARDANO_CLI_REV=${CARDANO_CLI_REV:-} - DBSYNC_REV=${DBSYNC_REV:-} - RUN_TARGET=${RUN_TARGET:-tests} - - MARKEXPR=${MARKEXPR:-} + - MARKEXPR=${MARKEXPR:-smoke} + - SESSION_TIMEOUT=${SESSION_TIMEOUT:-1h} - CLUSTERS_COUNT=${CLUSTERS_COUNT:-1} - CLUSTER_ERA=${CLUSTER_ERA:-} - PROTOCOL_VERSION=${PROTOCOL_VERSION:-} diff --git a/runner/run_tests.sh b/runner/run_tests.sh index 2bdf3cec5..b7281e9b2 100755 --- a/runner/run_tests.sh +++ b/runner/run_tests.sh @@ -58,7 +58,7 @@ EOF run_pytest() { if [ -n "${SESSION_TIMEOUT:-}" ]; then - local -a timeout_arr=( "--foreground" "--signal=INT" "--kill-after=0" "$SESSION_TIMEOUT" ) + local -a timeout_arr=( "--foreground" "--signal=INT" "--kill-after=120" "$SESSION_TIMEOUT" ) echo "Running: PYTEST_ADDOPTS='${PYTEST_ADDOPTS:-}' timeout ${timeout_arr[*]} pytest $*" timeout "${timeout_arr[@]}" pytest "$@" else From 0e02ceba153902b92f40947314795c9927866685 Mon Sep 17 00:00:00 2001 From: saratomaz Date: Thu, 7 May 2026 09:54:38 +0100 Subject: [PATCH 08/18] Emit an Antithesis SDK assertion for every test failure --- cardano_node_tests/tests/conftest.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cardano_node_tests/tests/conftest.py b/cardano_node_tests/tests/conftest.py index 1d8f385d6..a5a7c34bf 100644 --- a/cardano_node_tests/tests/conftest.py +++ b/cardano_node_tests/tests/conftest.py @@ -202,6 +202,32 @@ def pytest_keyboard_interrupt() -> None: (session_basetemp / INTERRUPTED_NAME).touch() +def pytest_runtest_logreport(report: tp.Any) -> None: + """Emit an Antithesis SDK assertion for every test failure.""" + if report.when != "call" or not report.failed: + return + sdk_file = pl.Path(os.environ.get("ANTITHESIS_OUTPUT_DIR", "/tmp/antithesis")) / "sdk.jsonl" + sdk_file.parent.mkdir(parents=True, exist_ok=True) + assertion = { + "antithesis_assert": { + "type": "always", + "condition": False, + "display_name": report.nodeid, + "message": str(report.longrepr)[:500] if report.longrepr else "", + "details": {}, + "location": { + "function": report.nodeid, + "file": str(report.fspath), + "begin_line": 0, + "begin_column": 0, + "class": "", + }, + } + } + with sdk_file.open("a") as f: + f.write(json.dumps(assertion) + "\n") + + @pytest.fixture(scope="session") def init_pytest_temp_dirs(tmp_path_factory: TempPathFactory) -> None: """Init `PytestTempDirs`.""" From b8bb201f823f7b15d0523b8fa2174ab4a7a71269 Mon Sep 17 00:00:00 2001 From: saratomaz Date: Thu, 7 May 2026 09:55:41 +0100 Subject: [PATCH 09/18] Start local submit_api in driver so tests can reach it via localhost --- docker/antithesis_run.sh | 48 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docker/antithesis_run.sh b/docker/antithesis_run.sh index b5a7af062..bb261b265 100755 --- a/docker/antithesis_run.sh +++ b/docker/antithesis_run.sh @@ -10,6 +10,11 @@ # container's health check on port 8090 before running tests, and # setting DEV_CLUSTER_RUNNING=1 so pytest uses the pre-running cluster # instead of starting its own. +# 3b. Starting a local cardano-submit-api in the driver container so that +# tests using submit_api can reach it via localhost. The test framework +# hard-codes http://localhost: for submit_api; since submit_api in +# the node container binds to 127.0.0.1 there (unreachable here), we +# run our own instance against the shared cluster-state socket. # 4. Emitting the Antithesis setup_complete lifecycle signal. # 5. Handing off to regression.sh. # @@ -91,7 +96,50 @@ except Exception: export CLUSTERS_COUNT="${CLUSTERS_COUNT:-1}" # Pre-set so regression.sh does not overwrite with its default workdir path. export CARDANO_NODE_SOCKET_PATH_CI="${CLUSTER_STATE_DIR}/state-cluster0/bft1.socket" + + # ------------------------------------------------------------------------- + # 3b. Start a local cardano-submit-api so tests can reach it via localhost. + # + # start-cluster already generated state-cluster0/run-cardano-submit-api + # with the correct port, socket path, and testnet magic substituted in. + # We run it from CLUSTER_STATE_DIR so its relative paths resolve, and + # put the pre-built binary on PATH. + # ------------------------------------------------------------------------- + _submit_api_script="${CLUSTER_STATE_DIR}/state-cluster0/run-cardano-submit-api" + if [ -x "$_submit_api_script" ]; then + export PATH="/opt/cardano/cardano-submit-api/bin:${PATH}" + # Derive the port the same way cluster_scripts.py does: + # base = PORTS_BASE + instance_num*10 (instance 0 → base = PORTS_BASE) + # submit_api = base + ports_per_instance - 1 - 2 = base + 7 + _submit_api_port=$(( ${PORTS_BASE:-23000} + 7 )) + + echo "Starting local cardano-submit-api on port ${_submit_api_port}..." + (cd "${CLUSTER_STATE_DIR}" && exec "${_submit_api_script}") & + _submit_api_pid=$! + # Kill it when this script exits so no orphan is left behind. + trap 'kill "${_submit_api_pid}" 2>/dev/null || true' EXIT + + # Wait up to 30 s for the port to open. + _sa_ready=0 + for _i in $(seq 1 30); do + if (echo >/dev/tcp/127.0.0.1/"${_submit_api_port}") 2>/dev/null; then + _sa_ready=1 + echo "Local submit_api is ready." + break + fi + echo " waiting for local submit_api (${_i}/30)..." + sleep 1 + done + if [ "$_sa_ready" -ne 1 ]; then + echo "WARNING: local submit_api did not start within 30 s; submit_api tests will fail." >&2 + fi + unset _submit_api_port _sa_ready _i + else + echo "WARNING: ${_submit_api_script} not found; submit_api tests will fail." >&2 + fi + unset _submit_api_script fi +# _submit_api_pid is intentionally kept in scope so the EXIT trap above can kill it. # --------------------------------------------------------------------------- # 4. Emit the Antithesis setup_complete signal. From 7bcd51e1f800fa3ba24cf493c9354b6691d5cb7b Mon Sep 17 00:00:00 2001 From: saratomaz Date: Fri, 8 May 2026 13:55:20 +0100 Subject: [PATCH 10/18] docker - fix ping tests in multi-container mode via TCP proxy --- docker/antithesis_run.sh | 54 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/docker/antithesis_run.sh b/docker/antithesis_run.sh index bb261b265..711933639 100755 --- a/docker/antithesis_run.sh +++ b/docker/antithesis_run.sh @@ -15,6 +15,10 @@ # hard-codes http://localhost: for submit_api; since submit_api in # the node container binds to 127.0.0.1 there (unreachable here), we # run our own instance against the shared cluster-state socket. +# 3c. Starting a TCP proxy (Python, no extra packages) that forwards +# localhost: → NODE_HOST:. cardano-cli ping +# hardcodes --host localhost, so without this the ping tests fail in +# multi-container mode. # 4. Emitting the Antithesis setup_complete lifecycle signal. # 5. Handing off to regression.sh. # @@ -116,8 +120,6 @@ except Exception: echo "Starting local cardano-submit-api on port ${_submit_api_port}..." (cd "${CLUSTER_STATE_DIR}" && exec "${_submit_api_script}") & _submit_api_pid=$! - # Kill it when this script exits so no orphan is left behind. - trap 'kill "${_submit_api_pid}" 2>/dev/null || true' EXIT # Wait up to 30 s for the port to open. _sa_ready=0 @@ -138,8 +140,54 @@ except Exception: echo "WARNING: ${_submit_api_script} not found; submit_api tests will fail." >&2 fi unset _submit_api_script + + # ------------------------------------------------------------------------- + # 3c. Start a TCP proxy so ping tests using --host localhost can reach the + # cardano-node P2P port in the node container. + # + # cardano-cli ping hardcodes --host localhost --port . In + # multi-container mode the node's TCP P2P port lives in the node + # container and is unreachable on the driver's localhost. We forward + # localhost: → NODE_HOST: using a small Python + # proxy backed by the pre-built venv Python (no extra packages needed). + # + # pool1 port = PORTS_BASE + 5 (see cardonnay local_scripts.py). + # ------------------------------------------------------------------------- + _pool1_port=$(( ${PORTS_BASE:-23000} + 5 )) + echo "Starting TCP proxy localhost:${_pool1_port} → ${NODE_HOST}:${_pool1_port}..." + "${_VENV_DIR}/bin/python3" -c " +import socket, threading +def relay(a, b): + try: + while True: + d = a.recv(4096) + if not d: break + b.sendall(d) + except OSError: pass + finally: + for s in (a, b): + try: s.close() + except OSError: pass +def fwd(src, host, port): + try: dst = socket.create_connection((host, port), timeout=10) + except OSError: src.close(); return + for args in ((src, dst), (dst, src)): + threading.Thread(target=relay, args=args, daemon=True).start() +srv = socket.socket() +srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +srv.bind(('127.0.0.1', ${_pool1_port})) +srv.listen(32) +while True: + conn, _ = srv.accept() + threading.Thread(target=fwd, args=(conn, '${NODE_HOST}', ${_pool1_port}), daemon=True).start() +" & + _proxy_pid=$! + unset _pool1_port + + # Kill both background processes when this script exits. + trap 'kill "${_submit_api_pid:-}" "${_proxy_pid:-}" 2>/dev/null || true' EXIT fi -# _submit_api_pid is intentionally kept in scope so the EXIT trap above can kill it. +# _submit_api_pid and _proxy_pid are intentionally kept in scope so the EXIT trap above can kill them. # --------------------------------------------------------------------------- # 4. Emit the Antithesis setup_complete signal. From a4eaefba9ed9a07e495ff5ca0d04dbf239fa9a80 Mon Sep 17 00:00:00 2001 From: saratomaz Date: Mon, 11 May 2026 15:17:46 +0100 Subject: [PATCH 11/18] improve Antithesis SDK assertion error detail --- cardano_node_tests/tests/conftest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cardano_node_tests/tests/conftest.py b/cardano_node_tests/tests/conftest.py index a5a7c34bf..8c5d37591 100644 --- a/cardano_node_tests/tests/conftest.py +++ b/cardano_node_tests/tests/conftest.py @@ -208,13 +208,16 @@ def pytest_runtest_logreport(report: tp.Any) -> None: return sdk_file = pl.Path(os.environ.get("ANTITHESIS_OUTPUT_DIR", "/tmp/antithesis")) / "sdk.jsonl" sdk_file.parent.mkdir(parents=True, exist_ok=True) + longrepr = report.longrepr + reprcrash = getattr(longrepr, "reprcrash", None) + exc_message = reprcrash.message if reprcrash else (str(longrepr)[:2000] if longrepr else "") assertion = { "antithesis_assert": { "type": "always", "condition": False, "display_name": report.nodeid, - "message": str(report.longrepr)[:500] if report.longrepr else "", - "details": {}, + "message": exc_message, + "details": {"traceback": str(longrepr)[-2000:] if longrepr else ""}, "location": { "function": report.nodeid, "file": str(report.fspath), From 4c80a2be978b8fb253b9361b533cf28c7a9884fa Mon Sep 17 00:00:00 2001 From: saratomaz Date: Tue, 12 May 2026 14:17:06 +0100 Subject: [PATCH 12/18] Ignore SIGINT in antithesis driver --- docker/antithesis_run.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/antithesis_run.sh b/docker/antithesis_run.sh index 711933639..e55cb6fc2 100755 --- a/docker/antithesis_run.sh +++ b/docker/antithesis_run.sh @@ -205,7 +205,13 @@ unset _output_dir # code (other than 137/143) as an error property violation. Test # failures are expected and communicated via SDK assertions, not the # process exit code. Always exit 0 so the container is not flagged. +# +# Ignore SIGINT in this driver so that SESSION_TIMEOUT's `timeout +# --foreground --signal=INT` does not kill PID 1 before exit 0 is +# reached. regression.sh sets its own trap and handles SIGINT +# independently. # --------------------------------------------------------------------------- +trap '' SIGINT set +e /work/.github/regression.sh _rc=$? From 12e9ad7af5a558822bc2888d84058e9d64f3991b Mon Sep 17 00:00:00 2001 From: saratomaz Date: Tue, 12 May 2026 15:50:10 +0100 Subject: [PATCH 13/18] Implement antithesis assertions --- cardano_node_tests/tests/test_tx_basic.py | 29 ++++++++++++---- cardano_node_tests/utils/antithesis.py | 42 +++++++++++++++++++++++ docker/Dockerfile | 3 +- 3 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 cardano_node_tests/utils/antithesis.py diff --git a/cardano_node_tests/tests/test_tx_basic.py b/cardano_node_tests/tests/test_tx_basic.py index 53acce774..24c3a28b1 100644 --- a/cardano_node_tests/tests/test_tx_basic.py +++ b/cardano_node_tests/tests/test_tx_basic.py @@ -15,6 +15,7 @@ from cardano_node_tests.tests import common from cardano_node_tests.tests import issues from cardano_node_tests.tests import tx_common +from cardano_node_tests.utils import antithesis from cardano_node_tests.utils import cluster_nodes from cardano_node_tests.utils import clusterlib_utils from cardano_node_tests.utils import dbsync_utils @@ -181,13 +182,27 @@ def test_transfer_funds( ) out_utxos = cluster.g_query.get_utxo(tx_raw_output=tx_output) - assert ( - clusterlib.filter_utxos(utxos=out_utxos, address=src_addr.address)[0].amount - == clusterlib.calculate_utxos_balance(tx_output.txins) - tx_output.fee - amount - ), f"Incorrect balance for source address `{src_addr.address}`" - assert ( - clusterlib.filter_utxos(utxos=out_utxos, address=dst_addr.address)[0].amount == amount - ), f"Incorrect balance for destination address `{dst_addr.address}`" + + src_actual = clusterlib.filter_utxos(utxos=out_utxos, address=src_addr.address)[0].amount + src_expected = clusterlib.calculate_utxos_balance(tx_output.txins) - tx_output.fee - amount + antithesis.always( + src_actual == src_expected, + "Source balance decreased by transfer amount and fee", + {"src_addr": src_addr.address, "expected": src_expected, "actual": src_actual}, + ) + assert src_actual == src_expected, ( + f"Incorrect balance for source address `{src_addr.address}`" + ) + + dst_actual = clusterlib.filter_utxos(utxos=out_utxos, address=dst_addr.address)[0].amount + antithesis.always( + dst_actual == amount, + "Destination received exact transfer amount", + {"dst_addr": dst_addr.address, "expected": amount, "actual": dst_actual}, + ) + assert dst_actual == amount, ( + f"Incorrect balance for destination address `{dst_addr.address}`" + ) common.check_missing_utxos(cluster_obj=cluster, utxos=out_utxos) diff --git a/cardano_node_tests/utils/antithesis.py b/cardano_node_tests/utils/antithesis.py new file mode 100644 index 000000000..24c9df2fb --- /dev/null +++ b/cardano_node_tests/utils/antithesis.py @@ -0,0 +1,42 @@ +"""Antithesis SDK wrappers. + +All functions are no-ops when the ``antithesis`` package is not installed, +so tests that use them run normally outside the Antithesis environment. +Install the package only inside the Antithesis Docker image — do not add it +to pyproject.toml. +""" + +import typing as tp + +try: + import antithesis.assertions as _ant + + def always(condition: bool, message: str, details: tp.Mapping[str, tp.Any]) -> None: + """Assert *condition* is true on every invocation.""" + _ant.always(condition, message, details) + + def sometimes(condition: bool, message: str, details: tp.Mapping[str, tp.Any]) -> None: + """Assert *condition* is true at least once across all calls.""" + _ant.sometimes(condition, message, details) + + def reachable(message: str, details: tp.Mapping[str, tp.Any]) -> None: + """Assert this code location is reached at least once.""" + _ant.reachable(message, details) + + def unreachable(message: str, details: tp.Mapping[str, tp.Any]) -> None: + """Assert this code location is never reached.""" + _ant.unreachable(message, details) + +except ImportError: + + def always(condition: bool, message: str, details: tp.Mapping[str, tp.Any]) -> None: # type: ignore[misc] + pass + + def sometimes(condition: bool, message: str, details: tp.Mapping[str, tp.Any]) -> None: # type: ignore[misc] + pass + + def reachable(message: str, details: tp.Mapping[str, tp.Any]) -> None: # type: ignore[misc] + pass + + def unreachable(message: str, details: tp.Mapping[str, tp.Any]) -> None: # type: ignore[misc] + pass diff --git a/docker/Dockerfile b/docker/Dockerfile index 81f2b4f24..44053f09c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -61,7 +61,8 @@ RUN nix develop --accept-flake-config .#testenv --command \ bash -c 'python3 -m venv /opt/tests-venv --prompt tests-venv && \ . /opt/tests-venv/bin/activate && \ cd /work && \ - uv sync --active --no-dev' + uv sync --active --no-dev && \ + pip install "antithesis>=0.2.0,<0.3.0"' # Pre-warm the base dev shell (bash, coreutils, git, jq, …) so its store # paths are cached and the regression.sh shebang resolves offline. From 9e3646008968587b3495ec2200f43ba942c42c48 Mon Sep 17 00:00:00 2001 From: saratomaz Date: Tue, 12 May 2026 16:02:18 +0100 Subject: [PATCH 14/18] Rename docker dir to docker-antithesis --- {docker => docker-antithesis}/Dockerfile | 6 +++--- {docker => docker-antithesis}/Dockerfile.config | 0 .../Dockerfile.dockerignore | 0 {docker => docker-antithesis}/README.md | 10 +++++----- {docker => docker-antithesis}/antithesis_run.sh | 2 +- {docker => docker-antithesis}/docker-compose.yaml | 14 +++++++------- {docker => docker-antithesis}/node_run.sh | 0 7 files changed, 16 insertions(+), 16 deletions(-) rename {docker => docker-antithesis}/Dockerfile (95%) rename {docker => docker-antithesis}/Dockerfile.config (100%) rename {docker => docker-antithesis}/Dockerfile.dockerignore (100%) rename {docker => docker-antithesis}/README.md (92%) rename {docker => docker-antithesis}/antithesis_run.sh (99%) rename {docker => docker-antithesis}/docker-compose.yaml (86%) rename {docker => docker-antithesis}/node_run.sh (100%) diff --git a/docker/Dockerfile b/docker-antithesis/Dockerfile similarity index 95% rename from docker/Dockerfile rename to docker-antithesis/Dockerfile index 44053f09c..e5609855d 100644 --- a/docker/Dockerfile +++ b/docker-antithesis/Dockerfile @@ -8,7 +8,7 @@ # NODE_REV — cardano-node git ref to pre-build (default: master) # # Build and push to GHCR before submitting to Antithesis: -# docker build -f docker/Dockerfile \ +# docker build -f docker-antithesis/Dockerfile \ # --build-arg GIT_REVISION=$(git rev-parse HEAD) \ # --build-arg NODE_REV=master \ # -t ghcr.io/saratomaz/cardano-node-tests-antithesis:latest . @@ -71,7 +71,7 @@ RUN nix develop --accept-flake-config .#base --command true # Create the Antithesis test driver directory and install the entry-points. # singleton_driver_* files are run once per test run by Antithesis. RUN mkdir -p /opt/antithesis/test/v1/quickstart && \ - cp /work/docker/antithesis_run.sh \ + cp /work/docker-antithesis/antithesis_run.sh \ /opt/antithesis/test/v1/quickstart/singleton_driver_regression.sh && \ chmod +x /opt/antithesis/test/v1/quickstart/singleton_driver_regression.sh && \ - chmod +x /work/docker/node_run.sh + chmod +x /work/docker-antithesis/node_run.sh diff --git a/docker/Dockerfile.config b/docker-antithesis/Dockerfile.config similarity index 100% rename from docker/Dockerfile.config rename to docker-antithesis/Dockerfile.config diff --git a/docker/Dockerfile.dockerignore b/docker-antithesis/Dockerfile.dockerignore similarity index 100% rename from docker/Dockerfile.dockerignore rename to docker-antithesis/Dockerfile.dockerignore diff --git a/docker/README.md b/docker-antithesis/README.md similarity index 92% rename from docker/README.md rename to docker-antithesis/README.md index c38088a94..4070cb408 100644 --- a/docker/README.md +++ b/docker-antithesis/README.md @@ -39,7 +39,7 @@ dependencies are baked into the image at build time: ### 1. Build and push the driver image ```bash -docker build -f docker/Dockerfile \ +docker build -f docker-antithesis/Dockerfile \ --build-arg GIT_REVISION=$(git rev-parse HEAD) \ --build-arg NODE_REV=master \ -t ghcr.io/saratomaz/cardano-node-tests-antithesis:latest . @@ -53,7 +53,7 @@ regardless of what is on the `master` branch when the container starts. ### 2. Build and push the config image ```bash -docker build -f docker/Dockerfile.config \ +docker build -f docker-antithesis/Dockerfile.config \ -t us-central1-docker.pkg.dev//antithesis/config:latest . docker push us-central1-docker.pkg.dev//antithesis/config:latest @@ -62,15 +62,15 @@ docker push us-central1-docker.pkg.dev//antithesis/config:latest ### 3. Validate locally (internet-connected build, isolated network at runtime) ```bash -docker compose -f docker/docker-compose.yaml config -docker compose -f docker/docker-compose.yaml up --build +docker compose -f docker-antithesis/docker-compose.yaml config +docker compose -f docker-antithesis/docker-compose.yaml up --build ``` To fully simulate the Antithesis no-internet constraint, run inside an isolated network namespace on Linux: ```bash -unshare -n docker compose -f docker/docker-compose.yaml up +unshare -n docker compose -f docker-antithesis/docker-compose.yaml up ``` ## Environment variables diff --git a/docker/antithesis_run.sh b/docker-antithesis/antithesis_run.sh similarity index 99% rename from docker/antithesis_run.sh rename to docker-antithesis/antithesis_run.sh index e55cb6fc2..ae861dc8d 100755 --- a/docker/antithesis_run.sh +++ b/docker-antithesis/antithesis_run.sh @@ -3,7 +3,7 @@ # # Runs the full test suite without any network access by: # 1. Forcing nix into offline mode (all store paths were pre-built into -# the image by docker/Dockerfile). +# the image by docker-antithesis/Dockerfile). # 2. Pointing regression.sh at the pre-built cardano binaries and Python # venv so it skips all download / build steps. # 3. When NODE_HOST is set (multi-container mode): waiting for the node diff --git a/docker/docker-compose.yaml b/docker-antithesis/docker-compose.yaml similarity index 86% rename from docker/docker-compose.yaml rename to docker-antithesis/docker-compose.yaml index 43cc9a66c..b8a8bddca 100644 --- a/docker/docker-compose.yaml +++ b/docker-antithesis/docker-compose.yaml @@ -12,7 +12,7 @@ # suite against the pre-running cluster via DEV_CLUSTER_RUNNING=1. # # Both images must be pre-built with all cardano binaries and the Python venv -# baked in (see docker/Dockerfile). No internet access is available at +# baked in (see docker-antithesis/Dockerfile). No internet access is available at # runtime inside the Antithesis environment. # # Push images to the Antithesis registry before submitting: @@ -21,8 +21,8 @@ # # Validate locally (requires internet — use an isolated netns to simulate # the Antithesis environment): -# docker compose -f docker/docker-compose.yaml config -# docker compose -f docker/docker-compose.yaml up --build +# docker compose -f docker-antithesis/docker-compose.yaml config +# docker compose -f docker-antithesis/docker-compose.yaml up --build networks: antithesis-net: @@ -36,8 +36,8 @@ services: image: ghcr.io/saratomaz/cardano-node-tests-antithesis:latest build: context: .. - dockerfile: docker/Dockerfile - command: ["/work/docker/node_run.sh"] + dockerfile: docker-antithesis/Dockerfile + command: ["/work/docker-antithesis/node_run.sh"] networks: - antithesis-net volumes: @@ -60,11 +60,11 @@ services: image: ghcr.io/saratomaz/cardano-node-tests-antithesis:latest build: context: .. - dockerfile: docker/Dockerfile + dockerfile: docker-antithesis/Dockerfile # antithesis_run.sh sets nix offline, waits for the node health check, # exports DEV_CLUSTER_RUNNING=1, emits setup_complete, then hands off # to regression.sh. - command: ["/work/docker/antithesis_run.sh"] + command: ["/work/docker-antithesis/antithesis_run.sh"] networks: - antithesis-net depends_on: diff --git a/docker/node_run.sh b/docker-antithesis/node_run.sh similarity index 100% rename from docker/node_run.sh rename to docker-antithesis/node_run.sh From 807ab0419b6af5b2caf0b45883bf21df886db32b Mon Sep 17 00:00:00 2001 From: saratomaz Date: Wed, 13 May 2026 10:19:03 +0100 Subject: [PATCH 15/18] Parse submit_api port from generated script --- docker-antithesis/antithesis_run.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker-antithesis/antithesis_run.sh b/docker-antithesis/antithesis_run.sh index ae861dc8d..a84a0dbd9 100755 --- a/docker-antithesis/antithesis_run.sh +++ b/docker-antithesis/antithesis_run.sh @@ -112,10 +112,9 @@ except Exception: _submit_api_script="${CLUSTER_STATE_DIR}/state-cluster0/run-cardano-submit-api" if [ -x "$_submit_api_script" ]; then export PATH="/opt/cardano/cardano-submit-api/bin:${PATH}" - # Derive the port the same way cluster_scripts.py does: - # base = PORTS_BASE + instance_num*10 (instance 0 → base = PORTS_BASE) - # submit_api = base + ports_per_instance - 1 - 2 = base + 7 - _submit_api_port=$(( ${PORTS_BASE:-23000} + 7 )) + # Parse the port directly from the generated script so we are not + # tied to a specific cluster variant's port formula. + _submit_api_port=$(grep -oE -- '--port [0-9]+' "$_submit_api_script" | grep -oE '[0-9]+' | head -1) echo "Starting local cardano-submit-api on port ${_submit_api_port}..." (cd "${CLUSTER_STATE_DIR}" && exec "${_submit_api_script}") & From 6db8a140086679907df5af88f37bd5466b16d940 Mon Sep 17 00:00:00 2001 From: saratomaz Date: Mon, 18 May 2026 10:09:53 +0100 Subject: [PATCH 16/18] Clean up previous cluster state --- docker-antithesis/node_run.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker-antithesis/node_run.sh b/docker-antithesis/node_run.sh index 4d6ef6ab8..f609ddad7 100755 --- a/docker-antithesis/node_run.sh +++ b/docker-antithesis/node_run.sh @@ -45,6 +45,15 @@ export CARDANO_NODE_SOCKET_PATH="${_STATE_CLUSTER}/bft1.socket" export _output_dir="${ANTITHESIS_OUTPUT_DIR:-/tmp/antithesis}" mkdir -p "$_output_dir" "${CLUSTER_STATE_DIR}" +# --------------------------------------------------------------------------- +# Clean up any stale cluster state left by a previous container run on the +# same Docker volume. The previous node process is gone (fresh container), +# so only the socket file and lock files remain — start-cluster refuses to +# run if it finds them. Remove before starting the health check so the +# health endpoint never serves a stale socket as "ready". +# --------------------------------------------------------------------------- +rm -rf "${_STATE_CLUSTER:?}" "${_SCRIPTS_DEST:?}" + # --------------------------------------------------------------------------- # 4. Health check server on port 8090 (Antithesis network bridge traffic). # Returns HTTP 200 "ready" once the cluster socket file exists, From 226cfdf1466a39838d29515cb36ce56b6cb0f7b1 Mon Sep 17 00:00:00 2001 From: saratomaz Date: Mon, 18 May 2026 12:32:32 +0100 Subject: [PATCH 17/18] Fix http server --- docker-antithesis/antithesis_run.sh | 40 ++++++++++++++++++++++++----- docker-antithesis/node_run.sh | 9 +++++++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/docker-antithesis/antithesis_run.sh b/docker-antithesis/antithesis_run.sh index a84a0dbd9..b85e404d4 100755 --- a/docker-antithesis/antithesis_run.sh +++ b/docker-antithesis/antithesis_run.sh @@ -134,7 +134,7 @@ except Exception: if [ "$_sa_ready" -ne 1 ]; then echo "WARNING: local submit_api did not start within 30 s; submit_api tests will fail." >&2 fi - unset _submit_api_port _sa_ready _i + unset _sa_ready _i else echo "WARNING: ${_submit_api_script} not found; submit_api tests will fail." >&2 fi @@ -147,8 +147,7 @@ except Exception: # cardano-cli ping hardcodes --host localhost --port . In # multi-container mode the node's TCP P2P port lives in the node # container and is unreachable on the driver's localhost. We forward - # localhost: → NODE_HOST: using a small Python - # proxy backed by the pre-built venv Python (no extra packages needed). + # localhost: → NODE_HOST:. # # pool1 port = PORTS_BASE + 5 (see cardonnay local_scripts.py). # ------------------------------------------------------------------------- @@ -183,10 +182,39 @@ while True: _proxy_pid=$! unset _pool1_port - # Kill both background processes when this script exits. - trap 'kill "${_submit_api_pid:-}" "${_proxy_pid:-}" 2>/dev/null || true' EXIT + # ------------------------------------------------------------------------- + # 3d. Start a local HTTP file server for anchor URLs. + # + # cardano-cli transaction build fetches and verifies anchor hashes at + # http://localhost:/p/. The cluster's webserver + # runs in the node container but binds to 127.0.0.1 there (Python 3.11+ + # http.server default), making it unreachable from the driver container. + # Since the webserver directory lives on the shared cluster-state volume, + # running our own http.server in the driver container serves the same + # files without going over the network bridge. + # + # webserver port = submit_api port + 2 (last_port in cardonnay; + # submit_api = last_port - 2). + # ------------------------------------------------------------------------- + # Derive webserver port from the already-parsed submit_api port. + # Fall back to PORTS_BASE + 99 (correct for a 3-pool cluster). + _webserver_port=$(( ${_submit_api_port:-$(( ${PORTS_BASE:-23000} + 97 ))} + 2 )) + unset _submit_api_port + + _webserver_dir="${CLUSTER_STATE_DIR}/state-cluster0/webserver" + mkdir -p "${_webserver_dir}" + echo "Starting local HTTP file server on port ${_webserver_port} (dir: ${_webserver_dir})..." + "${_VENV_DIR}/bin/python3" -m http.server \ + --bind 127.0.0.1 \ + --directory "${_webserver_dir}" \ + "${_webserver_port}" & + _webserver_pid=$! + unset _webserver_port _webserver_dir + + # Kill all background processes when this script exits. + trap 'kill "${_submit_api_pid:-}" "${_proxy_pid:-}" "${_webserver_pid:-}" 2>/dev/null || true' EXIT fi -# _submit_api_pid and _proxy_pid are intentionally kept in scope so the EXIT trap above can kill them. +# _submit_api_pid, _proxy_pid, _webserver_pid are kept in scope for the EXIT trap above. # --------------------------------------------------------------------------- # 4. Emit the Antithesis setup_complete signal. diff --git a/docker-antithesis/node_run.sh b/docker-antithesis/node_run.sh index f609ddad7..343e5f21d 100755 --- a/docker-antithesis/node_run.sh +++ b/docker-antithesis/node_run.sh @@ -103,6 +103,15 @@ nix develop --accept-flake-config .#testenv --command bash -c ' --instance-num "$_INSTANCE_NUM" \ --clean + # Patch pool1 to bind on all interfaces so the driver container can reach + # its P2P port over the Docker bridge network. By default cardonnay + # generates --host-addr 127.0.0.1 for all local cluster nodes; pool1 is + # the one the ping tests connect to via TCP. + if [ -f "$_SCRIPTS_DEST/cardano-node-pool1" ]; then + sed -i "s/--host-addr 127.0.0.1/--host-addr 0.0.0.0/g" \ + "$_SCRIPTS_DEST/cardano-node-pool1" + fi + # start-cluster must run from the parent of the state-cluster directory. cd "$CLUSTER_STATE_DIR" "$_SCRIPTS_DEST/start-cluster" From ac3222e6bf420b625fbd23e28d5d284b455638fc Mon Sep 17 00:00:00 2001 From: saratomaz Date: Wed, 20 May 2026 11:50:56 +0100 Subject: [PATCH 18/18] Review readme file --- docker-antithesis/README.md | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/docker-antithesis/README.md b/docker-antithesis/README.md index 4070cb408..e8767244f 100644 --- a/docker-antithesis/README.md +++ b/docker-antithesis/README.md @@ -22,8 +22,16 @@ dependencies are baked into the image at build time: 1. Forces nix into offline mode (`offline = true`). 2. Exports `CARDANO_PREBUILT_DIR=/opt/cardano` and `_VENV_DIR=/opt/tests-venv` so `regression.sh` skips all downloads and uses the pre-built artefacts. - 3. Emits the Antithesis `setup_complete` lifecycle signal. - 4. Hands off to `.github/regression.sh`. + 3. In multi-container mode (when `NODE_HOST` is set): + - Polls `NODE_HOST:NODE_PORT` until the node cluster reports ready. + - Starts a local `cardano-submit-api` in the driver container so that + submit-api tests can reach it via `localhost`. + - Starts a TCP proxy forwarding `localhost:` → + `NODE_HOST:` so `cardano-cli ping` tests work. + - Starts a local HTTP file server for anchor URLs used by governance + tests (`cardano-cli transaction build` fetches anchor hashes via HTTP). + 4. Emits the Antithesis `setup_complete` lifecycle signal. + 5. Hands off to `.github/regression.sh`. - `Dockerfile.config` — builds the Antithesis config image (`FROM scratch`) containing only `docker-compose.yaml`. @@ -63,7 +71,8 @@ docker push us-central1-docker.pkg.dev//antithesis/config:latest ```bash docker compose -f docker-antithesis/docker-compose.yaml config -docker compose -f docker-antithesis/docker-compose.yaml up --build +docker compose -f docker-antithesis/docker-compose.yaml up --build \ + --abort-on-container-exit --exit-code-from driver ``` To fully simulate the Antithesis no-internet constraint, run inside an @@ -78,13 +87,15 @@ unshare -n docker compose -f docker-antithesis/docker-compose.yaml up `NODE_REV` is baked into the image at build time and must **not** be set at runtime. All other variables are passed through docker-compose as before. -| Variable | Default | Description | -|-------------------|------------|------------------------------------------| -| `CARDANO_CLI_REV` | (built-in) | cardano-cli revision, empty = use node's | -| `DBSYNC_REV` | (disabled) | db-sync revision, empty = disabled | -| `RUN_TARGET` | `tests` | `tests`, `testpr`, or `testnets` | -| `MARKEXPR` | | pytest marker expression | -| `CLUSTERS_COUNT` | | number of local cluster instances | -| `CLUSTER_ERA` | | e.g. `conway` | -| `PROTOCOL_VERSION`| | e.g. `11` | -| `UTXO_BACKEND` | | e.g. `disk`, `mem` | +| Variable | Default | Description | +|--------------------|----------------|------------------------------------------------| +| `CARDANO_CLI_REV` | (built-in) | cardano-cli revision, empty = use node's | +| `DBSYNC_REV` | (disabled) | db-sync revision, empty = disabled | +| `RUN_TARGET` | `tests` | `tests`, `testpr`, or `testnets` | +| `MARKEXPR` | `smoke` | pytest marker expression | +| `SESSION_TIMEOUT` | `1h` | wall-clock limit passed to `timeout(1)` | +| `TESTNET_VARIANT` | `conway_fast` | cluster variant for `prepare_cluster_scripts` | +| `CLUSTERS_COUNT` | `1` | number of local cluster instances | +| `CLUSTER_ERA` | | e.g. `conway` | +| `PROTOCOL_VERSION` | | e.g. `11` | +| `UTXO_BACKEND` | | e.g. `disk`, `mem` |