From 99d3f71b2a4513f760b6a92615d785366306e48b Mon Sep 17 00:00:00 2001 From: Prashant Yadav Date: Tue, 4 Aug 2026 22:26:55 -0700 Subject: [PATCH 1/3] feat(local-cre): runtime CRE-settings override helper for e2e tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ApplyCRESettings, a test helper that overrides CRE settings on a DON at runtime — without restarting the topology — by proposing a `cresettings` job to every node of the DON (applied live via loop.AtomicSettings.Store) and restoring the pre-test baseline on cleanup. Delivery reuses the existing deployment changeset (ProposeJobSpec{Template: CRESettings}) + node approval; overrides are merged onto each DON's boot CL_CRE_SETTINGS baseline and rendered as a scoped TOML document. Includes scope-varied smoke tests (global, org, workflow, multi-scope) that demonstrate apply and explicit/auto cleanup. See core/scripts/cre/environment/docs/cresettings-runtime-override-proposal.md. Co-Authored-By: Claude Opus 4.8 --- .../docs/cresettings-override-README.md | 300 +++++++++++ system-tests/lib/cre/don/jobs/jobs.go | 6 +- system-tests/tests/go.mod | 4 +- .../smoke/cre/cresettings_override_test.go | 95 ++++ .../test-helpers/cresettings_override.go | 468 ++++++++++++++++++ 5 files changed, 869 insertions(+), 4 deletions(-) create mode 100644 core/scripts/cre/environment/docs/cresettings-override-README.md create mode 100644 system-tests/tests/smoke/cre/cresettings_override_test.go create mode 100644 system-tests/tests/test-helpers/cresettings_override.go diff --git a/core/scripts/cre/environment/docs/cresettings-override-README.md b/core/scripts/cre/environment/docs/cresettings-override-README.md new file mode 100644 index 00000000000..e760e30e862 --- /dev/null +++ b/core/scripts/cre/environment/docs/cresettings-override-README.md @@ -0,0 +1,300 @@ +# CRE Settings Override (Local CRE e2e tests) + +A test helper for **overriding CRE settings inside a Local CRE e2e test at runtime** — +without tearing down and restarting the topology — with **automatic revert** when the +test ends. + +- Helper: [`system-tests/tests/test-helpers/cresettings_override.go`](../../../../system-tests/tests/test-helpers/cresettings_override.go) +- Example tests: [`system-tests/tests/smoke/cre/cresettings_override_test.go`](../../../../system-tests/tests/smoke/cre/cresettings_override_test.go) + +## TL;DR + +```go +func Test_CRE_MyThing(t *testing.T) { + testEnv := t_helpers.SetupTestEnvironmentWithPerTestKeys(t, t_helpers.GetDefaultTestConfig(t)) + + // Scope the override to THIS test's workflow so it can't affect any other test; + // it auto-reverts when the test ends. + t_helpers.ApplyCRESettings(t, testEnv, t_helpers.CRESettingsOverrides{ + Workflow: map[string]map[string]string{ + myWorkflowID: {"PerWorkflow.HTTPAction.CallLimit": "1"}, // myWorkflowID: hex, no 0x + }, + }) + + // ... trigger the workflow and assert its behaviour under the override ... +} +``` + +That's the whole happy path: one call, and cleanup is automatic. + +Prefer scoping to your own **workflow** (or **owner**) as above — it isolates the change +to your workflow. Use `Global` only when you truly mean an environment-wide change (see +[Choosing a scope](#choosing-a-scope--isolation)). + +## When to use it + +Use it when a test needs a settings value different from the topology default — a rate +limit, a size bound, a feature gate, a concurrency limit — and you don't want a whole +separate topology just to bake that value into `CL_CRE_SETTINGS`. + +Don't reach for it to change things that aren't CRE settings (node TOML, capability +config, chain config); those aren't in the settings schema and this won't touch them. + +## How it mirrors what we do in CLD + +This is the in-test analogue of the production settings rollout in `chainlink-deployments` +(the `cre-limit-change` flow). Same mechanism, smaller blast radius: + +| Production rollout (CLD) | This helper (in a test) | +|---|---| +| Edit `settings/*.toml`, regenerate the compiled `settings.toml` | Pass a `CRESettingsOverrides{}` | +| Durable pipeline emits `job_propose_arbitrary` → `jobName: CRESettings`, `template: cre-settings`, `donName: all-nodes` | Calls the **same** `ProposeJobSpec{Template: CRESettings}` changeset, targeting every node of every DON | +| Sign + execute the proposal | Auto-approves the proposal on each node | +| Nodes apply it live via `loop.AtomicSettings.Store` (no restart) | Same — no restart | +| Change persists until the next rollout | **Auto-reverts** to the pre-test baseline on cleanup | + +Same job type, same all-nodes delivery, same live-apply path — just scoped to one test +and reverted afterwards. If you understand the CLD settings rollout, you understand this. + +## The API + +```go +type CRESettingsOverrides struct { + Global map[string]string // [global] — applies to everything + Org map[string]map[string]string // [org.] — keyed by org id + Owner map[string]map[string]string // [owner.] — keyed by workflow-owner hex (no 0x) + Workflow map[string]map[string]string // [workflow.] — keyed by workflow id hex (no 0x) +} + +// Apply to every DON, register auto-revert, and return a handle. +func ApplyCRESettings(t *testing.T, env *TestEnvironment, o CRESettingsOverrides) *CRESettingsHandle + +func (h *CRESettingsHandle) Reset(t *testing.T) // revert now (also happens automatically) +func (h *CRESettingsHandle) AppliedTOML(donName string) string // the document that was applied +func (h *CRESettingsHandle) BaselineTOML(donName string) string // the document it reverts to +``` + +- **Keys** are the dotted setting path exactly as in the schema + (`PerWorkflow.HTTPAction.CallLimit`, `PerOrg.BaseTriggerRetransmitEnabled`, …). +- **Values** are always strings. +- The canonical list of settings and their formats lives in + `chainlink-common/pkg/settings/cresettings/defaults.toml`. + +## Choosing a scope & isolation + +Settings resolve **most-specific-first**. For a given workflow a node looks up: + +``` +workflow. → owner. → org. → global → compiled default +``` + +**Prefer the narrowest scope.** Because the environment is shared across the suite, the +scope you pick is also your **blast radius**: + +- **Workflow / Owner** — the override only resolves for *your* workflow (or owner). Even if + something went wrong, no other test's workflow is affected. **This is the default you + should reach for.** +- **Org** — affects every workflow in that org. +- **Global** — affects **every workflow in the environment**. It's the deliberate + environment-wide escape hatch; use it only when you actually mean that. + +> **Where do the IDs come from?** From your deployment step: the org id your workflow runs +> under, and the workflow-owner / workflow-id hex (**without** the `0x` prefix) you get when +> you register/deploy the workflow. `Global` needs no id. + +### Examples + +**Workflow — one workflow only (preferred)** + +```go +t_helpers.ApplyCRESettings(t, testEnv, t_helpers.CRESettingsOverrides{ + Workflow: map[string]map[string]string{ + workflowID: { // workflowID has NO 0x prefix + "PerWorkflow.HTTPAction.CallLimit": "9", + "PerWorkflow.HTTPTrigger.RateLimit": "every5s:2", + }, + }, +}) +``` + +**Owner — one workflow owner only** + +```go +t_helpers.ApplyCRESettings(t, testEnv, t_helpers.CRESettingsOverrides{ + Owner: map[string]map[string]string{ + ownerHex: {"PerOwner.WorkflowExecutionConcurrencyLimit": "5"}, // ownerHex has NO 0x prefix + }, +}) +``` + +**Org — one org only** + +```go +t_helpers.ApplyCRESettings(t, testEnv, t_helpers.CRESettingsOverrides{ + Org: map[string]map[string]string{ + orgID: { + "PerOrg.BaseTriggerRetransmitEnabled": "false", + "PerOrg.WorkflowExecutionConcurrencyLimit": "42", + }, + }, +}) +``` + +**Global — the whole environment (escape hatch)** + +```go +t_helpers.ApplyCRESettings(t, testEnv, t_helpers.CRESettingsOverrides{ + Global: map[string]string{ + "PerWorkflow.HTTPAction.CallLimit": "7", + "PerWorkflow.ExecutionConcurrencyLimit": "3", + }, +}) +``` + +**Several scopes at once (merged)** + +```go +t_helpers.ApplyCRESettings(t, testEnv, t_helpers.CRESettingsOverrides{ + Workflow: map[string]map[string]string{workflowID: {"PerWorkflow.ExecutionConcurrencyLimit": "1"}}, + Org: map[string]map[string]string{orgID: {"PerOrg.BaseTriggerRetransmitEnabled": "false"}}, + Global: map[string]string{"PerWorkflow.HTTPAction.CallLimit": "11"}, +}) +``` + +All scopes merge into a single document (layered on each DON's baseline) and are delivered +together. To see exactly what got produced, log `h.AppliedTOML("workflow")`. + +## When does the change take effect? (live vs. restart) + +Every setting is read through the same live `AtomicSettings`, but *when* a consumer reads +it differs — this is the single most important thing to get right: + +| Class | Effect of a mid-test override | Examples | +|---|---|---| +| **Immediate** (read per operation) | Next call sees it | gates (`RemoteExecutableWorkflowDONBindingEnabled`, `ExecutionTimestampsEnabled`, `ChainAllowed`), bounds/limits (`HTTPAction.CallLimit`, `ExecutionConcurrencyLimit`, `ChainRead.CallLimit`), timeouts | +| **~5s lag** (poller-resized) | Applies within a few seconds | rate limits (`HTTPTrigger.RateLimit`, gateway rates), queue caps | +| **Registration-time** | Only affects workflows registered *after* the override | trigger subscription/registration limits, WASM size checks, workflow-admission limits | + +**Rule of thumb:** +- Per-execution limits/gates → `ApplyCRESettings`, then trigger the workflow. +- Registration-time settings → **`ApplyCRESettings` *before* you deploy/register the + workflow.** Applying them afterwards leaves the already-registered workflow on the old + value, and it will look like nothing happened. + +## It applies to every DON (you don't choose) + +Overrides are delivered to **every node of every DON**, mirroring CLD's all-nodes rollout. +This is deliberate: a setting may be enforced on the workflow nodes, the capabilities +nodes, *or* the gateway nodes, so a partial rollout could silently fail to take effect. You +say *what* to change; the helper makes sure it lands everywhere it could be read. +(Internally it still merges each DON's own boot baseline, since DONs can boot with +different `CL_CRE_SETTINGS` — but that's handled for you.) + +## Run these serially — the override guard + +Because overrides mutate settings on the shared environment, **only one override may be +active at a time**, and settings-override tests must run **serially**. Don't call +`t.Parallel()` in them, and don't add them to the `CRE_TEST_PARALLEL_ENABLED` set. (The CRE +suite already runs environment-using scenarios serially by default, so this is the norm, +not a special case.) + +The helper enforces it. If a second override starts while another is still active, it +**fails fast** with an actionable message: + +``` +a CRE settings override from "Test_CRE_Other" is still active on the shared environment. +Settings-override tests mutate shared state and must run serially: remove t.Parallel() +from this test (and do not add it to the CRE_TEST_PARALLEL_ENABLED set). ... +``` + +**If you see this:** make the failing test serial — remove its `t.Parallel()` and keep it +out of the parallel set. Re-applying settings *within one test* is fine: call `h.Reset(t)` +before applying again, or just apply again from the same test (the guard keys on the owning +test, so it won't trip on itself). + +The guard coordinates override-vs-override. What keeps an *unrelated* concurrent test from +seeing your change is running serially (the default) **plus** scoping narrowly — see +[Choosing a scope & isolation](#choosing-a-scope--isolation). + +## How cleanup works + +- `ApplyCRESettings` captures each DON's **boot baseline** (its `CL_CRE_SETTINGS`) up front + and registers a `t.Cleanup` that re-applies it. **You don't have to do anything** — the + settings revert when the test ends, whether it passes or fails. +- Deleting the settings job does **not** revert (a node keeps the last values it stored), so + cleanup works by **re-applying the baseline**, not by removing the job. The helper does + this for you. +- Overrides are **not additive**: each delivery fully replaces the getter, so the helper + always sends *baseline ⊕ your overrides*. Two consequences worth knowing: + - Omitting a key is fine — it just falls back to its compiled default. + - The DON's boot settings are always preserved, because they're part of the baseline. + +### Reverting early — `Handle.Reset(t)` + +Call `Reset` to revert **inside the test body** — e.g. to assert behaviour before *and* +after, or to A/B two settings in one test: + +```go +h := t_helpers.ApplyCRESettings(t, testEnv, t_helpers.CRESettingsOverrides{ + Global: map[string]string{"PerWorkflow.HTTPAction.CallLimit": "1"}, +}) +// ... assert the tight limit is enforced ... + +h.Reset(t) // back to baseline now +// ... assert normal behaviour is restored ... +``` + +`Reset` is idempotent and turns the automatic cleanup into a no-op, so it's always safe to +call (and safe to *not* call — cleanup still runs). + +## Advanced usage + +- **Inspect the exact documents:** `h.AppliedTOML(donName)` / `h.BaselineTOML(donName)` + return the TOML that was applied / will be restored (`donName` is e.g. `"workflow"`). + Handy for `t.Logf` and debugging. +- **A/B within one test:** apply → assert → `Reset` → apply a different value → assert. + Because each apply layers on the *baseline* (not the previous override), values never + accumulate. +- **Combine scopes** freely in one call; they merge into one document. + +## Mistakes & failure modes + +The helper is fail-loud for anything it can detect, and it validates your overrides against +the schema **before** delivering anything. + +| Mistake | What happens | +|---|---| +| Unknown / misspelled setting key | **Fails the test** at `ApplyCRESettings` (`unknown fields …`). Nothing is delivered. | +| Value in the wrong format for the setting (`"banana"` for an int limit, a malformed rate, …) | **Fails the test** at `ApplyCRESettings` (`invalid toml settings …`) — every value is parsed against its setting type. Nothing is delivered. | +| Non-string value | Impossible — the API only accepts `string` values. | +| `0x`-prefixed org / owner / workflow id | **Fails the test** at `ApplyCRESettings` — ids must be given without the `0x` prefix. | +| Environment not running / a node unreachable / proposal rejected | **Fails the test** at `ApplyCRESettings` — propose/approve errors are surfaced. | +| Revert fails during cleanup | **Surfaced**, not swallowed — `t.Error` from the automatic cleanup, or a hard failure from an explicit `Reset`. | +| Best-effort convergence log can't read container logs | **Ignored** — it's only a visibility aid; the authoritative success signal is that every node approved the job. | +| Flipping a **registration-time** setting *after* the workflow is registered | **Silently no-op** for that workflow (see the live-vs-restart table). Apply it *before* deploying the workflow. | +| Overriding a scope nothing matches (e.g. an org id no running workflow uses) | Delivered fine, simply never consulted. Not an error. | +| Two override tests running **concurrently** on the shared environment | **Fails fast** at `ApplyCRESettings` — the second override is refused while the first is active, with an actionable message. Keep override tests serial; do **not** call `t.Parallel()`. Re-applying within one test is fine (`Reset` first, or apply again from the same test). | +| An *unrelated* concurrent test caught by a `Global` override's blast radius | **Not caught by the guard** — it only coordinates override-vs-override. Prevented by running serially (the CRE default) and by scoping narrowly (Workflow/Owner). | + +**Short version:** authoring mistakes (bad keys / values / ids) and overlapping override +tests **fail loudly**. The only quiet risks left are timing ones — flipping a +registration-time setting too late, or a `Global` override touching an unrelated +concurrent test — and both are avoided by scoping narrowly and running serially (the +default). + +## Requirements & running + +These are e2e tests and need a running Local CRE environment (see +`docs/local-cre/system-tests/running-tests.md`). With the environment up: + +```bash +go test ./system-tests/tests/smoke/cre -run '^Test_CRE_CRESettings_' -timeout 20m -v +``` + +## Under the hood (pointers) + +You don't need any of this to use the helper, but if you want to trace it: + +- Live apply on the node: `core/services/cresettings/delegate.go` → `loop.AtomicSettings.Store` (`chainlink-common/pkg/loop/settings.go`). +- Delivery changeset: `deployment/cre/jobs` (`ProposeJobSpec{Template: CRESettings}`); validation: `deployment/cre/jobs/settings.go` (`VerifyCRESettings`). +- Settings schema, scopes and resolution: `chainlink-common/pkg/settings/cresettings` and `chainlink-common/pkg/settings`. diff --git a/system-tests/lib/cre/don/jobs/jobs.go b/system-tests/lib/cre/don/jobs/jobs.go index 4eac058a6f3..d90f30d0fa0 100644 --- a/system-tests/lib/cre/don/jobs/jobs.go +++ b/system-tests/lib/cre/don/jobs/jobs.go @@ -142,8 +142,10 @@ func accept(ctx context.Context, node *cre.Node, proposalID, jobSpec string) err err = approveJobProposalSpec(ctx, node, proposalID) } if err != nil { - // Workflow specs get auto approved - if strings.Contains(err.Error(), "cannot approve an approved spec") && strings.Contains(jobSpec, `type = "workflow"`) { + // Workflow and CRE settings specs get auto-approved by the node on proposal, so a + // subsequent explicit approve races into an already-approved spec — tolerate that. + if strings.Contains(err.Error(), "cannot approve an approved spec") && + (strings.Contains(jobSpec, `type = "workflow"`) || strings.Contains(jobSpec, `type = "cresettings"`)) { return nil } fmt.Println("Failed jobspec proposal for node ", node.Name) diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index 648454568d0..30b694947fa 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -305,7 +305,7 @@ require ( cosmossdk.io/math v1.4.0 // indirect cosmossdk.io/store v1.1.1 // indirect cosmossdk.io/x/tx v0.13.7 // indirect - dario.cat/mergo v1.0.2 // indirect + dario.cat/mergo v1.0.2 filippo.io/bigmod v0.1.0 // indirect filippo.io/edwards25519 v1.2.0 // indirect filippo.io/nistec v0.0.4 // indirect @@ -601,7 +601,7 @@ require ( github.com/otiai10/mint v1.6.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.3.1 // indirect + github.com/pelletier/go-toml/v2 v2.3.1 github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect github.com/philhofer/fwd v1.2.0 // indirect diff --git a/system-tests/tests/smoke/cre/cresettings_override_test.go b/system-tests/tests/smoke/cre/cresettings_override_test.go new file mode 100644 index 00000000000..b5628aeb918 --- /dev/null +++ b/system-tests/tests/smoke/cre/cresettings_override_test.go @@ -0,0 +1,95 @@ +package cre + +import ( + "testing" + + t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers" +) + +// Test_CRE_CRESettings_Override exercises the runtime CRE-settings override helper at +// every scope — workflow, org, global, and several scopes merged — without restarting +// the topology, and reverts each one. It doubles as executable documentation for +// system-tests/tests/test-helpers/cresettings_override.go. +// See core/scripts/cre/environment/docs/cresettings-override-README.md. +// +// Scope guidance (mirrored in the README): prefer the narrowest scope. Workflow/Owner +// scope isolates the override to your own workflow; Global changes it for the WHOLE +// environment and is the deliberate env-wide escape hatch. +// +// It is a single test (one environment spin-up) whose sections run SERIALLY: overrides +// mutate settings on the shared environment, so they must not run concurrently. The +// helper enforces this and fails fast if two overrides overlap. +// +// Requirements: a running Local CRE environment (default topology). Run with: +// +// go test ./system-tests/tests/smoke/cre -run '^Test_CRE_CRESettings_Override$' -timeout 20m -v +// +// NOTE: the org/workflow IDs below are illustrative. In a real test you pass the actual +// org id / workflow id your deployed workflow runs under (obtainable from the deployment +// step). The delivery + cleanup mechanics are identical regardless of the ID value; only +// the isolation guarantee depends on using a real ID. + +// creSettingsDON is only the DON whose applied/baseline document the test prints for +// readability — overrides are applied to ALL DONs. It is the workflow DON of the default +// topology (configs/workflow-gateway-capabilities-don.toml). +const creSettingsDON = "workflow" + +// Illustrative scoped identifiers (must not be 0x-prefixed). +const ( + exampleOrgID = "cresettingstestorg01" + exampleWorkflowID = "abababababababababababababababababababababababababababababababab" // 62-char hex-like id +) + +//nolint:paralleltest // mutates settings on the shared environment; must run serially +func Test_CRE_CRESettings_Override(t *testing.T) { + testEnv := t_helpers.SetupTestEnvironmentWithPerTestKeys(t, t_helpers.GetDefaultTestConfig(t)) + + // 1) Workflow scope (preferred) — isolated to a single workflow. Reverted explicitly + // so the full apply -> revert cycle is visible in the output. + t.Log("=== workflow-scoped override ===") + wf := t_helpers.ApplyCRESettings(t, testEnv, t_helpers.CRESettingsOverrides{ + Workflow: map[string]map[string]string{ + exampleWorkflowID: { + "PerWorkflow.HTTPAction.CallLimit": "9", + "PerWorkflow.HTTPTrigger.RateLimit": "every5s:2", + }, + }, + }) + t.Logf("[workflow] applied for DON %q:\n%s", creSettingsDON, wf.AppliedTOML(creSettingsDON)) + wf.Reset(t) + t.Logf("[workflow] reverted DON %q to baseline:\n%s", creSettingsDON, wf.BaselineTOML(creSettingsDON)) + + // 2) Org scope. + t.Log("=== org-scoped override ===") + org := t_helpers.ApplyCRESettings(t, testEnv, t_helpers.CRESettingsOverrides{ + Org: map[string]map[string]string{ + exampleOrgID: { + "PerOrg.BaseTriggerRetransmitEnabled": "false", + "PerOrg.WorkflowExecutionConcurrencyLimit": "42", + }, + }, + }) + t.Logf("[org=%s] applied for DON %q:\n%s", exampleOrgID, creSettingsDON, org.AppliedTOML(creSettingsDON)) + org.Reset(t) + + // 3) Global scope — the env-wide escape hatch (affects every workflow). + t.Log("=== global-scoped override (env-wide) ===") + global := t_helpers.ApplyCRESettings(t, testEnv, t_helpers.CRESettingsOverrides{ + Global: map[string]string{ + "PerWorkflow.HTTPAction.CallLimit": "7", + "PerWorkflow.ExecutionConcurrencyLimit": "3", + }, + }) + t.Logf("[global] applied for DON %q:\n%s", creSettingsDON, global.AppliedTOML(creSettingsDON)) + global.Reset(t) + + // 4) Multiple scopes merged in one call. Left to auto-revert via t.Cleanup, to + // exercise the automatic cleanup path as well. + t.Log("=== multi-scope override (merged; auto-revert on cleanup) ===") + multi := t_helpers.ApplyCRESettings(t, testEnv, t_helpers.CRESettingsOverrides{ + Workflow: map[string]map[string]string{exampleWorkflowID: {"PerWorkflow.ExecutionConcurrencyLimit": "1"}}, + Org: map[string]map[string]string{exampleOrgID: {"PerOrg.BaseTriggerRetransmitEnabled": "false"}}, + Global: map[string]string{"PerWorkflow.HTTPAction.CallLimit": "11"}, + }) + t.Logf("[multi-scope] applied for DON %q:\n%s", creSettingsDON, multi.AppliedTOML(creSettingsDON)) +} diff --git a/system-tests/tests/test-helpers/cresettings_override.go b/system-tests/tests/test-helpers/cresettings_override.go new file mode 100644 index 00000000000..751b39aa356 --- /dev/null +++ b/system-tests/tests/test-helpers/cresettings_override.go @@ -0,0 +1,468 @@ +package helpers + +// Runtime CRE-settings overrides for Local CRE e2e tests. +// +// A node applies CRE settings live — without a restart — when it receives a job of +// type `cresettings`: the node's delegate calls loop.AtomicSettings.Store, which +// hot-swaps the in-memory settings getter (see core/services/cresettings/delegate.go +// and chainlink-common/pkg/loop/settings.go). This is the same mechanism prod uses to +// roll out limit changes (the durable-pipeline `cre-settings` job proposed to +// all-nodes). Here we reuse the deployment changeset to propose that job to every node +// of a DON at test time, and restore the pre-test baseline on cleanup. +// +// Design notes / gotchas (see the usage guide in +// core/scripts/cre/environment/docs/cresettings-override-README.md): +// - Each Store fully REPLACES the settings getter (updates are not cumulative), so +// the delivered document must be the DON's boot baseline MERGED with the overrides. +// A bare-diff document would silently drop the DON's boot CL_CRE_SETTINGS. +// - Deleting the settings job does NOT revert the settings, so cleanup must re-apply +// the captured baseline explicitly. +// - The document must be TOML (JSON is rejected), scoped ([global]/[org.]/...), +// with all values quoted strings. +// - Overrides must reach ALL nodes of an OCR DON; a partial rollout makes nodes +// disagree. Approve() below only returns once every targeted node accepted the job. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + "sync" + "testing" + "time" + + "dario.cat/mergo" + "github.com/moby/moby/client" + "github.com/pelletier/go-toml/v2" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-testing-framework/framework" + + cre_jobs "github.com/smartcontractkit/chainlink/deployment/cre/jobs" + cre_jobs_ops "github.com/smartcontractkit/chainlink/deployment/cre/jobs/operations" + job_types "github.com/smartcontractkit/chainlink/deployment/cre/jobs/types" + "github.com/smartcontractkit/chainlink/deployment/cre/pkg/offchain" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/jobs" + ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration" +) + +const ( + // creSettingsExternalJobID is the fixed external job id shared by every CRESettings + // job (at most one per node). Mirrors deployment/cre/jobs/pkg/cre_settings_job.go so + // cancel-by-external-id targets the right proposal. + creSettingsExternalJobID = "8561c20c-7d06-421e-a155-3baf21b1622b" + + // creSettingsUpdateLogMarker is what the node's cresettings delegate logs when it + // applies an update (core/services/cresettings/delegate.go: `Updated settings`). + // We scan for it plus the doc hash as best-effort proof that a node converged. + creSettingsUpdateLogMarker = "Updated settings" + + // how long to wait (best effort) for every targeted node to log the applied hash. + creSettingsConvergenceTimeout = 60 * time.Second + + // per-delivery timeout for the propose+approve round trip. + creSettingsDeliveryTimeout = 2 * time.Minute +) + +// Only one CRE settings override may be active at a time. Overrides mutate settings on +// the SHARED Local CRE environment, so two override tests running concurrently would +// clobber each other's document. These package-level vars enforce that and fail the +// second test fast with an actionable message. Re-application by the SAME test is +// allowed (keyed on the owning test name), so apply -> Reset -> apply within one test, +// or two applies from the same test, are both fine. +var ( + creSettingsActiveMu sync.Mutex + creSettingsActiveOwner string // t.Name() of the test currently holding an override; "" if none +) + +func claimCRESettingsOverride(owner string) error { + creSettingsActiveMu.Lock() + defer creSettingsActiveMu.Unlock() + if creSettingsActiveOwner != "" && creSettingsActiveOwner != owner { + return fmt.Errorf( + "a CRE settings override from %q is still active on the shared environment.\n"+ + "Settings-override tests mutate shared state and must run serially: remove t.Parallel() "+ + "from this test (and do not add it to the CRE_TEST_PARALLEL_ENABLED set). If you meant to "+ + "change settings within one test, call handle.Reset(t) before applying again.\n"+ + "See core/scripts/cre/environment/docs/cresettings-override-README.md", + creSettingsActiveOwner) + } + creSettingsActiveOwner = owner + return nil +} + +func releaseCRESettingsOverride(owner string) { + creSettingsActiveMu.Lock() + defer creSettingsActiveMu.Unlock() + if creSettingsActiveOwner == owner { + creSettingsActiveOwner = "" + } +} + +// CRESettingsOverrides is a scoped set of CRE settings overrides to apply for the +// duration of a test. All values are strings, as required by the settings schema. +// Keys are dotted setting paths, e.g. "PerWorkflow.HTTPAction.CallLimit" or +// "PerOrg.BaseTriggerRetransmitEnabled". +type CRESettingsOverrides struct { + // Global applies to every org/owner/workflow (the [global] scope). + Global map[string]string + // Org is keyed by org id (the [org.] scope). + Org map[string]map[string]string + // Owner is keyed by workflow-owner hex WITHOUT the 0x prefix (the [owner.] scope). + Owner map[string]map[string]string + // Workflow is keyed by workflow id hex WITHOUT the 0x prefix (the [workflow.] scope). + Workflow map[string]map[string]string +} + +// CRESettingsHandle tracks an applied override so a test can revert it. ApplyCRESettings +// already registers a t.Cleanup that reverts automatically; Reset lets a test revert +// early (e.g. to assert post-revert behaviour within the test body). +type CRESettingsHandle struct { + env *ttypes.TestEnvironment + owner string // t.Name() of the owning test; released back to the guard on restore + targets []creSettingsTarget + reverted bool +} + +type creSettingsTarget struct { + don *cre.Don + baselineTOML string + baselineHash string + appliedTOML string + appliedHash string +} + +// ApplyCRESettings overrides CRE settings across the whole environment at runtime, +// without restarting the topology, and registers a t.Cleanup that restores the pre-test +// baseline when the test finishes. +// +// Settings are applied to EVERY DON — mirroring prod, which delivers to all-nodes. A +// given setting may be enforced on the workflow, capabilities, or gateway nodes, so +// applying to only a subset could silently fail to take effect; the user therefore does +// not choose which DONs are targeted. +// +// For each DON it captures the DON's boot CL_CRE_SETTINGS as the baseline, merges the +// overrides on top, and proposes+approves a `cresettings` job to every node of the DON. +// Approve only returns once every node accepted the job, so a successful call means the +// environment converged on the new settings. +// +// It fails the test (require) if delivery to any DON fails. A best-effort log-scan +// confirmation is emitted via t.Logf for visibility. +func ApplyCRESettings(t *testing.T, env *ttypes.TestEnvironment, o CRESettingsOverrides) *CRESettingsHandle { + t.Helper() + + require.NotNil(t, env, "test environment must not be nil") + require.NotNil(t, env.CreEnvironment, "CreEnvironment must not be nil") + require.NotNil(t, env.CreEnvironment.CldfEnvironment, "CldfEnvironment must not be nil") + require.NotNil(t, env.Dons, "Dons must not be nil") + + // The same overrides are layered onto every targeted DON; only the baseline differs. + overrideDoc := overridesToDoc(o) + require.NotEmpty(t, overrideDoc, "no overrides provided") + + targets := env.Dons.List() + require.NotEmpty(t, targets, "no DONs found in the environment") + + // Claim the single-active-override slot before touching anything. Fails fast (with an + // actionable message) if another test's override is still active on the shared env. + owner := t.Name() + if err := claimCRESettingsOverride(owner); err != nil { + require.FailNow(t, err.Error()) + } + + h := &CRESettingsHandle{env: env, owner: owner} + // Register cleanup up front, so the override is reverted and the slot released even if + // a delivery below fails partway through. + t.Cleanup(func() { h.restore(t, false /* not fatal: the test already finished */) }) + + for _, don := range targets { + baselineJSON := bootSettingsForDON(env, don.Name) + baselineTOML, baselineHash := renderSettings(t, baselineJSON, nil) + appliedTOML, appliedHash := renderSettings(t, baselineJSON, overrideDoc) + + t.Logf("[cresettings] DON %q: applying override (hash %s) over baseline (hash %s)", + don.Name, shortHash(appliedHash), shortHash(baselineHash)) + + err := deliverCRESettings(env, don, appliedTOML) + require.NoErrorf(t, err, "failed to deliver CRE settings override to DON %q", don.Name) + + h.targets = append(h.targets, creSettingsTarget{ + don: don, + baselineTOML: baselineTOML, + baselineHash: baselineHash, + appliedTOML: appliedTOML, + appliedHash: appliedHash, + }) + } + + // Best-effort confirmation that every node actually logged the applied settings. + for _, tg := range h.targets { + logSettingsConvergence(t, tg.don, tg.appliedHash, creSettingsConvergenceTimeout) + } + + return h +} + +// Reset restores the baseline settings on all targeted DONs immediately and waits +// (best effort) for convergence. Safe to call multiple times; the automatic cleanup +// becomes a no-op afterwards. +func (h *CRESettingsHandle) Reset(t *testing.T) { + t.Helper() + h.restore(t, true /* fatal: called from the test body */) +} + +// AppliedTOML returns the settings document applied to the named DON (its boot +// baseline merged with the overrides). Empty if the DON was not targeted. Useful for +// logging exactly what a test applied at each scope. +func (h *CRESettingsHandle) AppliedTOML(donName string) string { + for _, tg := range h.targets { + if tg.don.Name == donName { + return tg.appliedTOML + } + } + return "" +} + +// BaselineTOML returns the pre-test baseline document captured for the named DON, i.e. +// what the settings are restored to on cleanup. Empty if the DON was not targeted. +func (h *CRESettingsHandle) BaselineTOML(donName string) string { + for _, tg := range h.targets { + if tg.don.Name == donName { + return tg.baselineTOML + } + } + return "" +} + +func (h *CRESettingsHandle) restore(t *testing.T, fatal bool) { + t.Helper() + if h.reverted { + return + } + h.reverted = true + // Release the single-active-override slot once we've reverted, so the next test can + // claim it. Runs exactly once (guarded by h.reverted above). + defer releaseCRESettingsOverride(h.owner) + + for _, tg := range h.targets { + t.Logf("[cresettings] DON %q: reverting to baseline (hash %s)", tg.don.Name, shortHash(tg.baselineHash)) + err := deliverCRESettings(h.env, tg.don, tg.baselineTOML) + if err != nil { + if fatal { + require.NoErrorf(t, err, "failed to revert CRE settings on DON %q", tg.don.Name) + } else { + t.Errorf("[cresettings] failed to revert CRE settings on DON %q: %v", tg.don.Name, err) + } + continue + } + logSettingsConvergence(t, tg.don, tg.baselineHash, creSettingsConvergenceTimeout) + } +} + +// deliverCRESettings cancels any active settings job on every node of the DON, then +// proposes and approves a new one carrying settingsTOML. Approve returns only after +// every targeted node accepted the proposal. +func deliverCRESettings(env *ttypes.TestEnvironment, don *cre.Don, settingsTOML string) error { + ctx, cancel := context.WithTimeout(context.Background(), creSettingsDeliveryTimeout) + defer cancel() + + // At most one CRESettings job may be active per node; cancel any existing proposal + // first (mirrors `env swap capability`). Best-effort: on the first apply there is + // nothing to cancel. + for _, node := range don.Nodes { + if _, err := node.CancelProposalsByExternalJobID(ctx, []string{creSettingsExternalJobID}); err != nil { + framework.L.Warn(). + Str("don", don.Name). + Str("node", node.JobDistributorDetails.NodeID). + Err(err). + Msg("[cresettings] could not cancel existing settings proposal (continuing)") + } + } + + input := cre_jobs.ProposeJobSpecInput{ + Domain: offchain.ProductLabel, + Environment: env.CreEnvironment.CldfEnvironment.Name, + DONName: don.Name, + JobName: "cre-settings", + ExtraLabels: map[string]string{cre.CapabilityLabelKey: "cre-settings-override"}, + DONFilters: []offchain.TargetDONFilter{ + {Key: offchain.FilterKeyDONName, Value: don.Name}, + }, + Template: job_types.CRESettings, + Inputs: job_types.JobSpecInput{"settings": settingsTOML}, + } + + if err := (cre_jobs.ProposeJobSpec{}).VerifyPreconditions(*env.CreEnvironment.CldfEnvironment, input); err != nil { + return fmt.Errorf("verify settings job preconditions: %w", err) + } + + out, err := (cre_jobs.ProposeJobSpec{}).Apply(*env.CreEnvironment.CldfEnvironment, input) + if err != nil { + return fmt.Errorf("propose settings job: %w", err) + } + + // Collect the per-node proposed specs so we can approve them on each node. + specs := make(map[string][]string) + for _, r := range out.Reports { + o, ok := r.Output.(cre_jobs_ops.ProposeCRESettingsJobsOutput) + if !ok { + return fmt.Errorf("unexpected settings job report output type: %T", r.Output) + } + if mErr := mergo.Merge(&specs, o.Specs, mergo.WithAppendSlice); mErr != nil { + return fmt.Errorf("merge settings job specs: %w", mErr) + } + } + if len(specs) == 0 { + return fmt.Errorf("settings job proposal produced no specs for DON %q", don.Name) + } + + if err := jobs.Approve(ctx, env.CreEnvironment.CldfEnvironment.Offchain, env.Dons, specs); err != nil { + return fmt.Errorf("approve settings job: %w", err) + } + return nil +} + +// overridesToDoc converts the scoped overrides into a nested map matching the settings +// document shape: {global:{...}, org:{:{...}}, owner:{...}, workflow:{...}}. Dotted +// keys ("PerWorkflow.HTTPAction.CallLimit") are expanded into nested tables so the +// marshalled TOML is scoped correctly. +func overridesToDoc(o CRESettingsOverrides) map[string]any { + doc := map[string]any{} + for k, v := range o.Global { + setNested(doc, append([]string{"global"}, strings.Split(k, ".")...), v) + } + addScoped := func(scope string, byID map[string]map[string]string) { + for id, m := range byID { + for k, v := range m { + setNested(doc, append([]string{scope, id}, strings.Split(k, ".")...), v) + } + } + } + addScoped("org", o.Org) + addScoped("owner", o.Owner) + addScoped("workflow", o.Workflow) + return doc +} + +func setNested(m map[string]any, path []string, val string) { + for i := 0; i < len(path)-1; i++ { + child, ok := m[path[i]].(map[string]any) + if !ok { + child = map[string]any{} + m[path[i]] = child + } + m = child + } + m[path[len(path)-1]] = val +} + +// renderSettings merges overrideDoc (may be nil) onto the DON's boot baseline (a +// CL_CRE_SETTINGS JSON string, may be empty) and returns the resulting settings TOML +// plus its sha256 hash (the same hash the node computes and logs). +func renderSettings(t *testing.T, baselineJSON string, overrideDoc map[string]any) (string, string) { + t.Helper() + base := map[string]any{} + if strings.TrimSpace(baselineJSON) != "" { + require.NoErrorf(t, json.Unmarshal([]byte(baselineJSON), &base), + "invalid baseline CL_CRE_SETTINGS json: %s", baselineJSON) + } + if overrideDoc != nil { + deepMergeInto(base, overrideDoc) + } + if len(base) == 0 { + // Empty document => node resets to compiled defaults. + return "", hashString("") + } + b, err := toml.Marshal(base) + require.NoError(t, err, "failed to marshal settings toml") + s := string(b) + return s, hashString(s) +} + +// deepMergeInto recursively merges src into dst (src wins on leaves). Both are nested +// map[string]any documents; overlapping sub-tables are merged, not replaced. +func deepMergeInto(dst, src map[string]any) { + for k, sv := range src { + if sm, ok := sv.(map[string]any); ok { + if dm, ok := dst[k].(map[string]any); ok { + deepMergeInto(dm, sm) + continue + } + } + dst[k] = sv + } +} + +func bootSettingsForDON(env *ttypes.TestEnvironment, donName string) string { + if env.Config == nil { + return "" + } + for _, nodeSet := range env.Config.NodeSets { + if nodeSet != nil && nodeSet.Input != nil && nodeSet.Name == donName { + return nodeSet.EnvVars["CL_CRE_SETTINGS"] + } + } + return "" +} + +// logSettingsConvergence polls container logs (best effort) and reports how many nodes +// of the DON have logged the given settings hash. It never fails the test — the +// authoritative signal that the settings were delivered is that Approve succeeded. +func logSettingsConvergence(t *testing.T, don *cre.Don, hash string, timeout time.Duration) { + t.Helper() + want := len(don.Nodes) + deadline := time.Now().Add(timeout) + for { + got := countContainersWithSettingsHash(hash) + if got >= want { + t.Logf("[cresettings] DON %q: %d/%d nodes logged settings hash %s", don.Name, got, want, shortHash(hash)) + return + } + if time.Now().After(deadline) { + t.Logf("[cresettings] DON %q: only %d/%d nodes logged settings hash %s within %s "+ + "(delivery via Approve already succeeded; log-scan is best-effort)", + don.Name, got, want, shortHash(hash), timeout) + return + } + time.Sleep(3 * time.Second) + } +} + +// countContainersWithSettingsHash returns the number of containers whose logs contain +// the settings-update marker together with the given hash. +func countContainersWithSettingsHash(hash string) int { + logStreams, err := framework.StreamContainerLogs( + client.ContainerListOptions{All: true}, + client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true}, + ) + if err != nil { + framework.L.Warn().Err(err).Msg("[cresettings] could not stream container logs") + return 0 + } + count := 0 + for _, reader := range logStreams { + content, readErr := readContainerLogs(reader) // closes reader + if readErr != nil { + continue + } + if strings.Contains(content, creSettingsUpdateLogMarker) && strings.Contains(content, hash) { + count++ + } + } + return count +} + +func hashString(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +func shortHash(h string) string { + if len(h) <= 12 { + return h + } + return h[:12] +} From 02acb1a019adea9f25e40bd67f5bc07935b170fa Mon Sep 17 00:00:00 2001 From: Prashant Yadav Date: Wed, 5 Aug 2026 15:07:42 -0700 Subject: [PATCH 2/3] fix(local-cre): deliver CRE-settings overrides only to DONs with worker nodes --- .../test-helpers/cresettings_override.go | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/system-tests/tests/test-helpers/cresettings_override.go b/system-tests/tests/test-helpers/cresettings_override.go index 751b39aa356..b22fa5944cf 100644 --- a/system-tests/tests/test-helpers/cresettings_override.go +++ b/system-tests/tests/test-helpers/cresettings_override.go @@ -139,10 +139,10 @@ type creSettingsTarget struct { // without restarting the topology, and registers a t.Cleanup that restores the pre-test // baseline when the test finishes. // -// Settings are applied to EVERY DON — mirroring prod, which delivers to all-nodes. A -// given setting may be enforced on the workflow, capabilities, or gateway nodes, so -// applying to only a subset could silently fail to take effect; the user therefore does -// not choose which DONs are targeted. +// Settings are applied to every DON that has worker (plugin) nodes — the workflow and +// capabilities DONs — since CRE settings are enforced there and the delivery changeset +// targets type=plugin nodes. Bootstrap-only DONs (e.g. bootstrap-gateway) have no such +// nodes and are skipped. The user does not choose which DONs are targeted. // // For each DON it captures the DON's boot CL_CRE_SETTINGS as the baseline, merges the // overrides on top, and proposes+approves a `cresettings` job to every node of the DON. @@ -163,8 +163,19 @@ func ApplyCRESettings(t *testing.T, env *ttypes.TestEnvironment, o CRESettingsOv overrideDoc := overridesToDoc(o) require.NotEmpty(t, overrideDoc, "no overrides provided") - targets := env.Dons.List() - require.NotEmpty(t, targets, "no DONs found in the environment") + // CRE settings are enforced on worker (plugin) nodes, and the delivery changeset filters + // proposals to type=plugin — so bootstrap-only DONs (e.g. bootstrap-gateway) have no + // matching nodes. Deliver only to DONs that have worker nodes. + require.NotEmpty(t, env.Dons.List(), "no DONs found in the environment") + targets := make([]*cre.Don, 0) + for _, don := range env.Dons.List() { + if don.WorkersCount() > 0 { + targets = append(targets, don) + } else { + t.Logf("[cresettings] skipping DON %q (no worker nodes)", don.Name) + } + } + require.NotEmpty(t, targets, "no DONs with worker nodes found in the environment") // Claim the single-active-override slot before touching anything. Fails fast (with an // actionable message) if another test's override is still active on the shared env. From 3dc2a67f053604edd8e24b1d430889dabc6d6eb6 Mon Sep 17 00:00:00 2001 From: Prashant Yadav Date: Wed, 5 Aug 2026 16:07:38 -0700 Subject: [PATCH 3/3] fix(local-cre): deliver CRE settings by propose-only (rely on node auto-approve) --- system-tests/tests/go.mod | 2 +- .../test-helpers/cresettings_override.go | 70 ++++--------------- 2 files changed, 13 insertions(+), 59 deletions(-) diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index 30b694947fa..f5d86cdba00 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -305,7 +305,7 @@ require ( cosmossdk.io/math v1.4.0 // indirect cosmossdk.io/store v1.1.1 // indirect cosmossdk.io/x/tx v0.13.7 // indirect - dario.cat/mergo v1.0.2 + dario.cat/mergo v1.0.2 // indirect filippo.io/bigmod v0.1.0 // indirect filippo.io/edwards25519 v1.2.0 // indirect filippo.io/nistec v0.0.4 // indirect diff --git a/system-tests/tests/test-helpers/cresettings_override.go b/system-tests/tests/test-helpers/cresettings_override.go index b22fa5944cf..46436b683b6 100644 --- a/system-tests/tests/test-helpers/cresettings_override.go +++ b/system-tests/tests/test-helpers/cresettings_override.go @@ -23,7 +23,6 @@ package helpers // disagree. Approve() below only returns once every targeted node accepted the job. import ( - "context" "crypto/sha256" "encoding/hex" "encoding/json" @@ -33,7 +32,6 @@ import ( "testing" "time" - "dario.cat/mergo" "github.com/moby/moby/client" "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/require" @@ -41,20 +39,13 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/framework" cre_jobs "github.com/smartcontractkit/chainlink/deployment/cre/jobs" - cre_jobs_ops "github.com/smartcontractkit/chainlink/deployment/cre/jobs/operations" job_types "github.com/smartcontractkit/chainlink/deployment/cre/jobs/types" "github.com/smartcontractkit/chainlink/deployment/cre/pkg/offchain" "github.com/smartcontractkit/chainlink/system-tests/lib/cre" - "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/jobs" ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration" ) const ( - // creSettingsExternalJobID is the fixed external job id shared by every CRESettings - // job (at most one per node). Mirrors deployment/cre/jobs/pkg/cre_settings_job.go so - // cancel-by-external-id targets the right proposal. - creSettingsExternalJobID = "8561c20c-7d06-421e-a155-3baf21b1622b" - // creSettingsUpdateLogMarker is what the node's cresettings delegate logs when it // applies an update (core/services/cresettings/delegate.go: `Updated settings`). // We scan for it plus the doc hash as best-effort proof that a node converged. @@ -62,9 +53,6 @@ const ( // how long to wait (best effort) for every targeted node to log the applied hash. creSettingsConvergenceTimeout = 60 * time.Second - - // per-delivery timeout for the propose+approve round trip. - creSettingsDeliveryTimeout = 2 * time.Minute ) // Only one CRE settings override may be active at a time. Overrides mutate settings on @@ -145,12 +133,11 @@ type creSettingsTarget struct { // nodes and are skipped. The user does not choose which DONs are targeted. // // For each DON it captures the DON's boot CL_CRE_SETTINGS as the baseline, merges the -// overrides on top, and proposes+approves a `cresettings` job to every node of the DON. -// Approve only returns once every node accepted the job, so a successful call means the -// environment converged on the new settings. +// overrides on top, and proposes a `cresettings` job to the DON's worker nodes, which +// auto-approve and apply it live. // -// It fails the test (require) if delivery to any DON fails. A best-effort log-scan -// confirmation is emitted via t.Logf for visibility. +// It fails the test (require) if proposing to any DON fails. Application is confirmed +// best-effort via the nodes' "Updated settings" logs, emitted via t.Logf for visibility. func ApplyCRESettings(t *testing.T, env *ttypes.TestEnvironment, o CRESettingsOverrides) *CRESettingsHandle { t.Helper() @@ -273,26 +260,14 @@ func (h *CRESettingsHandle) restore(t *testing.T, fatal bool) { } } -// deliverCRESettings cancels any active settings job on every node of the DON, then -// proposes and approves a new one carrying settingsTOML. Approve returns only after -// every targeted node accepted the proposal. +// deliverCRESettings proposes a `cresettings` job carrying settingsTOML to the DON's +// worker nodes. CRE nodes auto-approve the settings job and apply it live, so proposing +// is sufficient — we deliberately do NOT cancel the previous job or explicitly approve +// the new one. With the fixed settings-job UUID, an explicit cancel/approve corrupts the +// JD proposal history on repeated deliveries (e.g. reverting to the same baseline twice, +// which failed with "no job proposal found"). Application is confirmed best-effort via +// the nodes' "Updated settings" logs (see logSettingsConvergence). func deliverCRESettings(env *ttypes.TestEnvironment, don *cre.Don, settingsTOML string) error { - ctx, cancel := context.WithTimeout(context.Background(), creSettingsDeliveryTimeout) - defer cancel() - - // At most one CRESettings job may be active per node; cancel any existing proposal - // first (mirrors `env swap capability`). Best-effort: on the first apply there is - // nothing to cancel. - for _, node := range don.Nodes { - if _, err := node.CancelProposalsByExternalJobID(ctx, []string{creSettingsExternalJobID}); err != nil { - framework.L.Warn(). - Str("don", don.Name). - Str("node", node.JobDistributorDetails.NodeID). - Err(err). - Msg("[cresettings] could not cancel existing settings proposal (continuing)") - } - } - input := cre_jobs.ProposeJobSpecInput{ Domain: offchain.ProductLabel, Environment: env.CreEnvironment.CldfEnvironment.Name, @@ -309,30 +284,9 @@ func deliverCRESettings(env *ttypes.TestEnvironment, don *cre.Don, settingsTOML if err := (cre_jobs.ProposeJobSpec{}).VerifyPreconditions(*env.CreEnvironment.CldfEnvironment, input); err != nil { return fmt.Errorf("verify settings job preconditions: %w", err) } - - out, err := (cre_jobs.ProposeJobSpec{}).Apply(*env.CreEnvironment.CldfEnvironment, input) - if err != nil { + if _, err := (cre_jobs.ProposeJobSpec{}).Apply(*env.CreEnvironment.CldfEnvironment, input); err != nil { return fmt.Errorf("propose settings job: %w", err) } - - // Collect the per-node proposed specs so we can approve them on each node. - specs := make(map[string][]string) - for _, r := range out.Reports { - o, ok := r.Output.(cre_jobs_ops.ProposeCRESettingsJobsOutput) - if !ok { - return fmt.Errorf("unexpected settings job report output type: %T", r.Output) - } - if mErr := mergo.Merge(&specs, o.Specs, mergo.WithAppendSlice); mErr != nil { - return fmt.Errorf("merge settings job specs: %w", mErr) - } - } - if len(specs) == 0 { - return fmt.Errorf("settings job proposal produced no specs for DON %q", don.Name) - } - - if err := jobs.Approve(ctx, env.CreEnvironment.CldfEnvironment.Offchain, env.Dons, specs); err != nil { - return fmt.Errorf("approve settings job: %w", err) - } return nil }