diff --git a/tests/e2e/lib/container_mesh.bash b/tests/e2e/lib/container_mesh.bash index 8f066084..749ea52f 100644 --- a/tests/e2e/lib/container_mesh.bash +++ b/tests/e2e/lib/container_mesh.bash @@ -325,11 +325,16 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then export KUBERNETES_CLUSTER_NAME="sam-wi-test" export KUBECONTEXT="kind-${KUBERNETES_CLUSTER_NAME}" - if ! kind get clusters | grep -q "^${KUBERNETES_CLUSTER_NAME}$"; then + # Recreating is the default so a local run matches CI, which builds a cluster + # per run. Reuse carries over database PVCs, control plane signing keys and + # bootstrap tokens; E2E_REUSE_CLUSTER=1 trades that fidelity for a faster loop. + local reused_cluster=0 + if [[ "${E2E_REUSE_CLUSTER:-0}" == "1" ]] && kind get clusters | grep -q "^${KUBERNETES_CLUSTER_NAME}$"; then + kind export kubeconfig --name "${KUBERNETES_CLUSTER_NAME}" + reused_cluster=1 + else kind delete cluster --name "${KUBERNETES_CLUSTER_NAME}" >/dev/null 2>&1 || true kind create cluster --name "${KUBERNETES_CLUSTER_NAME}" --config=tests/e2e/fixtures/kind-cluster.yaml - else - kind export kubeconfig --name "${KUBERNETES_CLUSTER_NAME}" fi kind load docker-image sam-control-plane:local --name "${KUBERNETES_CLUSTER_NAME}" @@ -380,6 +385,12 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then mesh_wait_for_rollout statefulset/sam-db mesh_wait_for_rollout deployment/sam-control-plane mesh_wait_for_job job/sam-bootstrap + # A router surviving a reinstall holds a biscuit no current control plane key + # verifies and a bootstrap token past its 24h default, so it can never lease + # again. Restarting re-enrolls it against the state this run just installed. + if [[ "${reused_cluster}" == "1" ]]; then + kubectl --context="${KUBECONTEXT}" rollout restart statefulset/sam-router + fi mesh_wait_for_rollout statefulset/sam-router local i @@ -393,6 +404,22 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then router_peer_id=$(kubectl --context="${KUBECONTEXT}" logs "sam-router-0" | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1 || true) [[ -n "${router_peer_id}" ]] + # The router pod reports Ready before its lease reaches the control plane, and + # a node's /register serves router addresses from that lease, so a node + # started in between enrolls against an empty list and exits. + local router_node_ip + router_node_ip=$(docker inspect -f "{{(index .NetworkSettings.Networks \"${MESH_NETWORK:-kind}\").IPAddress}}" \ + "$(kubectl --context="${KUBECONTEXT}" get pod sam-router-0 -o jsonpath='{.spec.nodeName}')") + local lease_deadline=$((SECONDS + 60)) + until docker run --rm --network "${MESH_NETWORK:-kind}" python:3.12 \ + curl -sf --max-time 5 "http://${router_node_ip}:8080/info" 2>/dev/null | grep -qaF "${router_peer_id}"; do + if ((SECONDS >= lease_deadline)); then + echo "router lease did not reach the control plane within 60s" >&2 + return 1 + fi + sleep 1 + done + echo "${router_peer_id}" > "/tmp/sam-wi-test-router-peer-id" return 0 } diff --git a/tests/e2e/relay.bats b/tests/e2e/relay.bats index 32533e3f..316ba65e 100644 --- a/tests/e2e/relay.bats +++ b/tests/e2e/relay.bats @@ -81,24 +81,35 @@ teardown() { mesh_start_node "2" "--log-level=debug" MESH_NETWORK=$OLD_NET - run mesh_wait_for_log "${MESH_PREFIX}-node-1" "PeerID:" 20 + # Gate on MCP actually answering: get_mesh_info polls below go through it, and + # a not-yet-listening sidecar burns discovery budget on empty responses. + run mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 30 [[ "$status" -eq 0 ]] - run mesh_wait_for_log "${MESH_PREFIX}-node-2" "PeerID:" 20 + run mesh_wait_for_mcp_ready "1" 30 [[ "$status" -eq 0 ]] - # Node 1 should eventually see 1 peer (node 2) besides the router - run mesh_wait_for_node_count "1" 1 30 - [[ "$status" -eq 0 ]] - - # Node 2 should eventually see 1 peer (node 1) besides the router OLD_NET=$MESH_NETWORK MESH_NETWORK=$MESH_NETWORK_2 - run mesh_wait_for_node_count "2" 1 30 + run mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 30 + [[ "$status" -eq 0 ]] + run mesh_wait_for_mcp_ready "2" 30 [[ "$status" -eq 0 ]] MESH_NETWORK=$OLD_NET - local node1_peer_id + local node1_peer_id node2_peer_id node1_peer_id=$(docker logs "${MESH_PREFIX}-node-1" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r') + node2_peer_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r') + + # Assert on the specific peer, not on set size: a relayed connection can leave + # extra entries in connected_peers, so "length - 1" is not a stable count. + run mesh_wait_for_peer_connection "1" "${node2_peer_id}" 60 + [[ "$status" -eq 0 ]] + + OLD_NET=$MESH_NETWORK + MESH_NETWORK=$MESH_NETWORK_2 + run mesh_wait_for_peer_connection "2" "${node1_peer_id}" 60 + [[ "$status" -eq 0 ]] + MESH_NETWORK=$OLD_NET # 1. Setup HTTP Service on Node 1 side (default network) docker run -d \