diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index fc5edd2..99355c8 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -58,4 +58,13 @@ inference-time descent** into distribution-free reliability certificates. Read a wash where geometry already wins (arith-IRED +0.007 4/5) and slightly worse on fixed-bowl cells. Aggregation is feature-set-aware (`feature_set_of`, canonical tables default `feature_set="base"`) so richer never pollutes T1/T2/T4. JAX for spectrum/connectivity lives in `curvature.py` (inv. 1). +- Phase 4l certificates on firm footing ✅ (T6/T7): put the guarantees on a 5-seed footing. + `run_halting_sweep.py` + `aggregate.{halting_rows,aggregate_halting_cell}` → **T6**: arith halting + saves 57.1%±1.7% (was single-seed 57.8%), graph 80.8%±5.2%, both risk ≤ α, within-budget 5/5. + `[sweep] include_ood=true` + OOD aggregation block → **T7**: guarantee breaks under shift over 5 + seeds (arith ID risk 0.077→OOD 0.764, 0/5; graph 0.010→0.168, 4/5). Full-fold graph cert run: LTT + certifies 1.1% cov at α=0.1 but **22.5% at α=0.15 / 48.5% at α=0.20, validity holds at every α** — + the "certifies zero" was an operating-point artifact of graph's ~30% error floor, not a broken + certificate. Aggregation dedups halting rows by seed (drops the stale Phase-4b H1 config-hash row); + `plotting._selective_row`/`_halting_row` prefer arithmetic so full-fold graph rows don't flip F2/F4/F6. - Next: E3/E4 (a 3rd/4th task to characterize *when* geometry beats softmax) · Modal · scale-up. diff --git a/analysis/aggregate.py b/analysis/aggregate.py index 696c0b2..04a60c9 100644 --- a/analysis/aggregate.py +++ b/analysis/aggregate.py @@ -93,6 +93,18 @@ def aggregate_cell(rows_for_k: list[dict]) -> dict: dadd = [m["delta_aurc_geom_adds"] for m in ms] if has_add else [] add_mean, add_std = _ms([d[0] for d in dadd]) if has_add else (float("nan"), float("nan")) add_ci = sum(1 for d in dadd if d[1] > 0.0) + # OOD stress (Phase 4l): the ID-calibrated threshold applied to the shifted fold. Only rows run + # with include_ood carry these; pre-4l / selective-sweep rows fall through (has_ood=False). + has_ood = all("ood_ltt" in m and "accuracy_ood" in m for m in ms) + _nan = (float("nan"), float("nan")) + if has_ood: + acc_ood = _ms([m["accuracy_ood"] for m in ms]) + risk_ood = _ms([m["ood_ltt"]["selective_risk"] for m in ms]) + cov_ood = _ms([m["ood_ltt"]["coverage"] for m in ms]) + ood_within = sum(1 for m in ms if m["ood_ltt"].get("risk_within_budget")) + else: + acc_ood = risk_ood = cov_ood = _nan + ood_within = 0 return { "task": rows_for_k[0].get("task"), "k_restarts": _k(rows_for_k[0]), @@ -126,6 +138,12 @@ def aggregate_cell(rows_for_k: list[dict]) -> dict: "ltt_coverage": _ms([m["ltt"]["coverage"] for m in ms]), "ltt_selective_risk": _ms([m["ltt"]["selective_risk"] for m in ms]), "ltt_abstain_rate": _ms([m["ltt"]["abstain_rate"] for m in ms]), + # OOD stress (Phase 4l); NaN + has_ood=False when the cell has no include_ood rows. + "has_ood": has_ood, + "accuracy_ood": acc_ood, + "ood_selective_risk": risk_ood, + "ood_coverage": cov_ood, + "seeds_ood_within_budget": ood_within, } @@ -172,3 +190,60 @@ def _key(cell: list[dict]) -> tuple: return (has_add, len(cell), max(r.get("timestamp", "") for r in cell)) return aggregate_cell(max(cells, key=_key)) + + +def halting_rows(rows: list[dict] | None = None, task: str | None = None) -> list[dict]: + """Latest ``split=='halting'`` row per ``(config_hash, seed)``, optionally one ``task``. + + The halting rows are excluded from ``selective_rows`` (different split); this is their + multi-seed counterpart for the adaptive-halting (CRC) guarantee (Phase 4l). + """ + if rows is None: + rows = read_all() + latest: dict[tuple[str, int], dict] = {} + for r in rows: + if r.get("split") == "halting" and (task is None or r.get("task") == task): + latest[(r["config_hash"], r["seed"])] = r # later rows win + return list(latest.values()) + + +def halting_tasks_present(rows: list[dict] | None = None) -> list[str]: + """Sorted distinct task names among halting rows.""" + return sorted({r.get("task") for r in halting_rows(rows)} - {None}) + + +def aggregate_halting_cell(rows_for_task: list[dict]) -> dict: + """Aggregate per-seed halting rows into the multi-seed CRC verdict (compute saved + risk). + + Dedups to the latest row per seed by timestamp — so a stale single-seed run under an older + config schema (e.g. the original Phase-4b H1 seed-0 row, whose config_hash has since drifted) + does not double-count against the current seed sweep. + """ + by_seed: dict[int, dict] = {} + for r in sorted(rows_for_task, key=lambda r: r.get("timestamp", "")): + by_seed[r["seed"]] = r + rows_for_task = list(by_seed.values()) + ms = [r["metrics"] for r in rows_for_task] + n = len(ms) + taus = [m["tau_hat"] for m in ms if m.get("tau_hat") is not None] + return { + "task": rows_for_task[0].get("task"), + "n_seeds": n, + "seeds": sorted(r["seed"] for r in rows_for_task), + "run_ids": [r["run_id"] for r in rows_for_task], + "alpha": ms[0].get("alpha"), + "tau_hat": _ms(taus) if taus else (float("nan"), float("nan")), + "n_tau_feasible": len(taus), + "compute_saved": _ms([m["compute_saved"] for m in ms]), + "halting_risk": _ms([m["halting_risk"] for m in ms]), + "full_accuracy": _ms([m["full_accuracy"] for m in ms]), + "accuracy_drop": _ms([m["accuracy_drop"] for m in ms]), + "seeds_within_budget": sum(1 for m in ms if m.get("risk_within_budget")), + "seeds_risk_monotone": sum(1 for m in ms if m.get("risk_monotone_in_tau")), + } + + +def aggregate_halting(rows: list[dict] | None = None) -> dict[str, dict]: + """``{task: aggregate_halting_cell(...)}`` for every task with halting rows.""" + return {t: aggregate_halting_cell(halting_rows(rows, task=t)) + for t in halting_tasks_present(rows)} diff --git a/analysis/figures/F2_risk_coverage.png b/analysis/figures/F2_risk_coverage.png index f98c466..e618929 100644 Binary files a/analysis/figures/F2_risk_coverage.png and b/analysis/figures/F2_risk_coverage.png differ diff --git a/analysis/figures/F3_coverage_validity.png b/analysis/figures/F3_coverage_validity.png index 5884f20..d7c6f43 100644 Binary files a/analysis/figures/F3_coverage_validity.png and b/analysis/figures/F3_coverage_validity.png differ diff --git a/analysis/figures/F4_halting_pareto.png b/analysis/figures/F4_halting_pareto.png index 68613ea..23fec14 100644 Binary files a/analysis/figures/F4_halting_pareto.png and b/analysis/figures/F4_halting_pareto.png differ diff --git a/analysis/figures/F5_feature_diagnostics.png b/analysis/figures/F5_feature_diagnostics.png index 6e66445..27aaec2 100644 Binary files a/analysis/figures/F5_feature_diagnostics.png and b/analysis/figures/F5_feature_diagnostics.png differ diff --git a/analysis/figures/F6_ood_stress.png b/analysis/figures/F6_ood_stress.png index 1e36b70..440803f 100644 Binary files a/analysis/figures/F6_ood_stress.png and b/analysis/figures/F6_ood_stress.png differ diff --git a/analysis/figures/S1_k_restart_lift.png b/analysis/figures/S1_k_restart_lift.png index b88bbe2..b596be6 100644 Binary files a/analysis/figures/S1_k_restart_lift.png and b/analysis/figures/S1_k_restart_lift.png differ diff --git a/analysis/make_tables.py b/analysis/make_tables.py index ec08a7a..4356189 100644 --- a/analysis/make_tables.py +++ b/analysis/make_tables.py @@ -14,6 +14,7 @@ try: # sibling import when run as a script (analysis/ on sys.path) from aggregate import ( aggregate_by_k, + aggregate_halting, feature_set_of, headline_cell, objective_of, @@ -23,6 +24,7 @@ except ImportError: # pragma: no cover from analysis.aggregate import ( aggregate_by_k, + aggregate_halting, feature_set_of, headline_cell, objective_of, @@ -193,6 +195,59 @@ def _t5(rows: list[dict]) -> str: "tab:richer", f"run_ids={all_ids}") +def _t6(rows: list[dict]) -> str: + """Adaptive halting (CRC guarantee) over seeds — H1 on a multi-seed footing (Phase 4l). + + Per task: the CRC-chosen agreement threshold, compute saved, halting risk vs the budget alpha, + end-task accuracy drop, and how many seeds keep the risk within budget — all mean$\\pm$std. + """ + cells = aggregate_halting(rows) + body, all_ids = [], [] + for task, a in cells.items(): + body.append([ + _esc(task), f"${a['alpha']}$", _pm(a["tau_hat"], 2), _pm(a["compute_saved"], 3), + _pm(a["halting_risk"], 3), _pm(a["accuracy_drop"], 3), + f"{a['seeds_within_budget']}/{a['n_seeds']}", + ]) + all_ids += a["run_ids"] + header = ["task", r"$\alpha$", r"$\hat\tau$", "compute saved", "halting risk", + "acc drop", "seeds $\\le\\alpha$"] + return _tabular( + header, body, "lcccccc", + "Adaptive halting (CRC), mean$\\pm$std over seeds: the basin-agreement threshold " + "$\\hat\\tau$ is calibrated on a disjoint fold so the halting risk (disagreement with the " + "full-budget answer) stays within the budget $\\alpha$. ``compute saved`` is the fraction " + "of descent steps skipped; ``seeds $\\le\\alpha$`` counts seeds with test risk in budget.", + "tab:halting", f"run_ids={all_ids}") + + +def _t7(rows: list[dict]) -> str: + """OOD stress over seeds — the guarantee holds ID and breaks under shift (Phase 4l). + + Per task: ID vs OOD accuracy, and the ID-calibrated LTT threshold's selective risk on the ID + fold vs the shifted fold, with a count of seeds where OOD risk stays within budget. + """ + body, all_ids = [], [] + for task in tasks_present(rows): + a = headline_cell(rows, task=task) + if not a.get("has_ood"): + continue + body.append([ + _esc(task), _pm(a["accuracy_id"], 2), _pm(a["accuracy_ood"], 2), + _pm(a["ltt_selective_risk"], 3), _pm(a["ood_selective_risk"], 3), + f"{a['seeds_ood_within_budget']}/{a['n_seeds']}", + ]) + all_ids += a["run_ids"] + header = ["task", "acc ID", "acc OOD", "sel.\\ risk ID", "sel.\\ risk OOD", "seeds OOD ok"] + return _tabular( + header, body, "lccccc", + "OOD stress (mean$\\pm$std over seeds): the ID-calibrated selective-prediction threshold " + "applied to a shifted fold. The selective-risk guarantee holds in-distribution but breaks " + "under shift (OOD risk exceeds the budget), motivating abstention; ``seeds OOD ok`` counts " + "seeds whose OOD risk still stays within budget.", + "tab:oodstress", f"run_ids={all_ids}") + + def _t3(rows: list[dict]) -> str: sel = selective_rows(rows) env = sel[-1]["env"] if sel else {} @@ -235,6 +290,12 @@ def _has_body(tex: str) -> bool: t5 = _t5(rows) # only when richer-geometry rows exist if _has_body(t5): outputs["T5_richer_geometry.tex"] = t5 + t6 = _t6(rows) # only when split='halting' rows exist + if _has_body(t6): + outputs["T6_halting.tex"] = t6 + t7 = _t7(rows) # only when include_ood rows exist + if _has_body(t7): + outputs["T7_ood_stress.tex"] = t7 for name, tex in outputs.items(): (TABLE_DIR / name).write_text(tex) print(f"[tables] wrote {TABLE_DIR / name}") diff --git a/configs/experiments/graph_halting.toml b/configs/experiments/graph_halting.toml new file mode 100644 index 0000000..08d7fd7 --- /dev/null +++ b/configs/experiments/graph_halting.toml @@ -0,0 +1,36 @@ +# Adaptive-halting experiment on graph shortest-path (Phase 4l — a 2nd task for the CRC guarantee). +# Same task/inference/train as the graph selective E2 config, but per-step decoding is recorded so +# the basin-agreement halting threshold can be CRC-calibrated. Folds kept modest to bound decode +# memory on CPU (mirrors arithmetic_halting). +[run] +task = "graph_planning" +seed = 0 +notes = "graph adaptive-halting run" + +[task.graph_planning] +n_nodes = 7 +ood_n_nodes = 10 +edge_prob = 0.4 +max_len = 4 + +[model] +latent_dim = 32 +hidden_dim = 128 +context_dim = 64 + +[inference] +k_restarts = 12 +steps = 50 + +[train] +epochs = 25 # graph needs more epochs than arithmetic to reach the 70-85% regime +n_train = 8000 +n_neg = 4 + +[eval] +n_eval = 800 # test fold (per-step decode kept small) + +[conformal] +alpha = 0.1 # halting error budget: E[early answer != full answer] <= alpha +delta = 0.05 +n_calib = 800 # calibration fold diff --git a/configs/sweeps/arith_halting_seeds.toml b/configs/sweeps/arith_halting_seeds.toml new file mode 100644 index 0000000..09f5a2f --- /dev/null +++ b/configs/sweeps/arith_halting_seeds.toml @@ -0,0 +1,9 @@ +# Phase 4l: 5-seed adaptive-halting (CRC) on arithmetic — the multi-seed version of the H1 headline +# (~58% compute saved at halting risk <= alpha), which was single-seed. Uses run_halting_sweep.py +# (each cell trains, CRC-calibrates, appends a split="halting" row). aggregate.aggregate_halting + +# make_tables T6 summarise the compute-saved / halting-risk mean±std. +[sweep] +base = "configs/experiments/arithmetic_halting.toml" + +[sweep.grid] +"run.seed" = [0, 1, 2, 3, 4] diff --git a/configs/sweeps/arith_ood_seeds.toml b/configs/sweeps/arith_ood_seeds.toml new file mode 100644 index 0000000..d53e3b9 --- /dev/null +++ b/configs/sweeps/arith_ood_seeds.toml @@ -0,0 +1,9 @@ +# Phase 4l: 5-seed FULL-FOLD arithmetic selective prediction with include_ood=true, so the OOD +# stress result (F6: ID risk vs OOD risk under the ID-calibrated threshold) gets a multi-seed +# mean±std instead of a single seed. Matches graph_cert_seeds so T7 is 5-seed on both tasks. +[sweep] +base = "configs/experiments/arithmetic_selective.toml" +include_ood = true + +[sweep.grid] +"run.seed" = [0, 1, 2, 3, 4] diff --git a/configs/sweeps/graph_cert_seeds.toml b/configs/sweeps/graph_cert_seeds.toml new file mode 100644 index 0000000..f6087ab --- /dev/null +++ b/configs/sweeps/graph_cert_seeds.toml @@ -0,0 +1,10 @@ +# Phase 4l THE CERT RUN: 5-seed FULL-FOLD graph selective prediction (n_calib=1500), so LTT +# abstention can actually certify nonzero coverage on graph — the reduced-fold seed sweeps +# (n_calib=800) certify zero. include_ood=true also yields 5-seed graph OOD stress (T7). No n_calib +# override: it inherits graph_selective.toml's full folds. Heaviest cell set in the project. +[sweep] +base = "configs/experiments/graph_selective.toml" +include_ood = true + +[sweep.grid] +"run.seed" = [0, 1, 2, 3, 4] diff --git a/configs/sweeps/graph_halting_seeds.toml b/configs/sweeps/graph_halting_seeds.toml new file mode 100644 index 0000000..b4d7fa1 --- /dev/null +++ b/configs/sweeps/graph_halting_seeds.toml @@ -0,0 +1,8 @@ +# Phase 4l: 5-seed adaptive-halting (CRC) on graph shortest-path — a 2nd task for the halting +# guarantee (secondary to the arithmetic headline). Uses run_halting_sweep.py -> split="halting" +# rows -> aggregate_halting -> T6. +[sweep] +base = "configs/experiments/graph_halting.toml" + +[sweep.grid] +"run.seed" = [0, 1, 2, 3, 4] diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index cc6d943..a41f1f7 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -2,6 +2,24 @@ Short, dated, append-only. Newest first. +## 2026-07-30 — Phase 4l: certificates on firm footing (multi-seed guarantees + graph certification) +**Finding/decision:** put the project's core contribution — the distribution-free certificates — on a +5-seed footing. (1) **Halting:** new `experiments/run_halting_sweep.py` + `aggregate.halting_rows`/ +`aggregate_halting_cell` (split="halting" was excluded from `selective_rows`). Arith saves +57.1%±1.7% (single-seed H1 was 57.8%), graph 80.8%±5.2%; both risk ≤ α, within budget 5/5 (T6). +(2) **OOD:** `[sweep] include_ood=true` in `run_sweep` + an OOD aggregation block; the selective +guarantee breaks under shift over 5 seeds (arith ID 0.077 → OOD 0.764, 0/5; graph 0.010 → 0.168, +4/5) (T7). (3) **Graph certification:** a full-fold run (n_calib=1500) certifies only 1.1% coverage +at α=0.1 but 22.5% at α=0.15 and 48.5% at α=0.20, with achieved risk ≤ target at every α — so the +old "certifies zero coverage" note was an **operating-point artifact** of graph's ~30% error floor, +not a broken certificate. **Bugs fixed:** (a) halting aggregation dedups by seed (the stale Phase-4b +H1 seed-0 row survived under a drifted config_hash, giving n=6); (b) `plotting._selective_row` and +`_halting_row` now prefer arithmetic, since the new full-fold (n_test=1500) graph rows would +otherwise tie/flip the arithmetic figures F2/F4/F6 to graph. **Why record it:** every guarantee +number the paper states is now multi-seed, and the graph certificate is shown valid at its honest α. +**Reversible?** Additive runner + aggregation + a sweep flag; default `include_ood=False` preserves +all prior sweeps. + ## 2026-07-29 — Phase 4k: the graph redundancy is a task property, not thin features **Finding/decision:** the 4j redundancy verdict rested on a thin curvature description (only `λmax`/`tr(H)`), so we added two opt-in richer feature groups — the **full Hessian spectrum** diff --git a/docs/EXPERIMENTS.md b/docs/EXPERIMENTS.md index 968c96d..41ed54c 100644 --- a/docs/EXPERIMENTS.md +++ b/docs/EXPERIMENTS.md @@ -232,6 +232,52 @@ contribute there, even with the richest geometry we can extract. The base rows r numbers exactly, confirming richer geometry is purely additive and non-breaking (default stays the canonical 14 features; aggregation is feature-set-aware so richer never pollutes T1/T2/T4). +### Certificates on firm footing — multi-seed guarantees + graph certification (Phase 4l → T6/T7) + +The certificates are the contribution, but three of their numbers were single-seed or vacuous. This +phase puts all three on a 5-seed footing (new `run_halting_sweep.py`; `[sweep] include_ood=true`; +objective/feature-set-style aggregation for halting + OOD; full-fold graph cert run). Additive — no +change to the geometry features or the falsification test. + +**Adaptive halting (CRC), 5 seeds — T6.** The H1 headline replicates and extends: + +| task | τ̂ | compute saved | halting risk (≤ α=0.1) | seeds ≤ α | +|------|-----|---------------|------------------------|-----------| +| arithmetic | $0.917$ | $0.571 \pm 0.017$ | $0.012 \pm 0.005$ | $5/5$ | +| graph | $0.85\pm0.03$ | $\mathbf{0.808 \pm 0.052}$ | $0.065 \pm 0.032$ | $5/5$ | + +The single-seed 57.8% arithmetic number is now $57.1\%\pm1.7\%$ over 5 seeds (within budget every +seed); on graph the CRC halt saves **~81%** of compute at risk $0.065 \le 0.1$ — the halting +guarantee holds on both tasks across seeds. + +**OOD stress, 5 seeds — T7.** The ID-calibrated selective threshold applied to the shifted fold: + +| task | acc ID | acc OOD | sel. risk ID | sel. risk OOD | seeds OOD ≤ α | +|------|--------|---------|--------------|---------------|---------------| +| arithmetic | $0.83$ | $0.21$ | $0.077$ | $\mathbf{0.764}$ | $0/5$ | +| graph | $0.70$ | $0.33$ | $0.010$ | $0.168$ | $4/5$ | + +The guarantee holds ID and **breaks under shift** on both tasks (arithmetic catastrophically: +$0.077\to0.764$; graph mildly: $0.010\to0.168$) — 5-seed confirmation of F6, and the argument for +treating an abstention as a routing signal. + +**Graph certification — the certificate is valid on graph, at its honest operating point.** At the +full fold ($n_\text{calib}=1500$) the graph LTT abstention still certifies almost nothing at +α=0.1 (coverage $0.011$), because graph's ~30% base error leaves no confident slice with true error +below 10%. But sweeping α (from the same `coverage_validity` block; mean over 5 seeds): + +| α | certified coverage | achieved risk | +|-----|--------------------|---------------| +| $0.10$ | $0.011$ | $0.010$ | +| $0.15$ | $0.225$ | $0.078$ | +| $0.20$ | $0.485$ | $0.157$ | +| $0.30$ | $0.895$ | $0.263$ | + +Achieved risk sits **at or below target at every α** — validity holds; the earlier "certifies zero +coverage" was an α=0.1 operating-point artifact, not a broken certificate. Graph certifies **22.5% +coverage at α=0.15** and **48.5% at α=0.20**, versus arithmetic's 74% at α=0.10 — the honest cost of +a harder task's higher error floor, not a failure. (F3 shows the arithmetic α-sweep on the diagonal.) + ## Operating regime Tune each task's difficulty so base accuracy is **70–85%**. Fully-solved tasks saturate AURC and diff --git a/docs/SESSION_HANDOFF.md b/docs/SESSION_HANDOFF.md index 3c43261..34fa828 100644 --- a/docs/SESSION_HANDOFF.md +++ b/docs/SESSION_HANDOFF.md @@ -2,7 +2,7 @@ **Start here.** Rolling state of the project between sessions. -## Current state (2026-07-29) +## Current state (2026-07-30) - **Phase 0 (skeleton): ✅** repo tree, `uv`/`pyproject.toml`, `Makefile`, CI, config system (`edc.config`), deterministic seeding (`edc.seeding`), append-only ledger (`edc.ledger`), @@ -99,9 +99,21 @@ geometry already wins (arith-IRED +0.007, 4/5, unchanged) and slightly worse on the fixed-bowl basin-center cells (extra features add noise). Aggregation is feature-set-aware (`feature_set_of`; `headline_cell`/`by_k` default `feature_set="base"`) so richer never pollutes T1/T2/T4. 109 tests. -- **Open question / next:** a 3rd/4th task (E3/E4, e.g. logic/Sudoku) to characterize *when* geometry - beats softmax now that feature poverty is ruled out; harden honesty gaps (multi-seed halting/OOD, - MC-dropout baseline, a full-fold graph run that actually certifies); Modal; scale-up. +- **Phase 4l (certificates on firm footing — multi-seed guarantees + graph certification): ✅** put + the project's core contribution (the certificates) on a 5-seed footing. New + `experiments/run_halting_sweep.py` + `aggregate.{halting_rows, aggregate_halting_cell}` → **T6**: + arith halting saves **57.1%±1.7%** (was single-seed 57.8%), graph **80.8%±5.2%**, both halting risk + ≤ α, within budget 5/5. `[sweep] include_ood=true` (run_sweep) + an OOD aggregation block → **T7**: + the selective guarantee breaks under shift over 5 seeds (arith ID sel-risk 0.077 → OOD 0.764, 0/5; + graph 0.010 → 0.168, 4/5). Full-fold graph cert run (`graph_cert_seeds.toml`, n_calib=1500): LTT + certifies only **1.1% coverage at α=0.1** but **22.5% at α=0.15, 48.5% at α=0.20; validity holds at + every α** — the old "certifies zero" was an operating-point artifact of graph's ~30% error floor, + not a broken certificate. Fixed a stale-row bug (halting dedups by seed, dropping the Phase-4b H1 + row under a drifted config_hash) and made `plotting._selective_row`/`_halting_row` prefer arithmetic + so the new full-fold (n_test=1500) graph rows don't flip F2/F4/F6. 115 tests. +- **Open question / next:** a 3rd/4th task (E3/E4, e.g. logic/parity) to characterize *when* geometry + beats softmax now that feature poverty is ruled out; a deep-ensemble baseline (the one remaining + deferred baseline); Modal; scale-up. ## What is real vs stub diff --git a/experiments/run_halting_sweep.py b/experiments/run_halting_sweep.py new file mode 100644 index 0000000..506b951 --- /dev/null +++ b/experiments/run_halting_sweep.py @@ -0,0 +1,71 @@ +"""Expand a halting sweep grid into many adaptive-halting runs. [Phase 4l] + + PYTHONPATH=src python experiments/run_halting_sweep.py configs/sweeps/arith_halting_seeds.toml + +The multi-seed counterpart of ``run_halting.py``: same sweep-TOML format as ``run_sweep`` (a +``base`` config + optional ``[sweep.override]`` + ``[sweep.grid]``), but each cell trains, CRC- +calibrates the halting threshold, and appends one ``split="halting"`` row via ``evaluate_halting`` — +so the compute-saved / halting-risk guarantee gets a multi-seed mean±std instead of a single seed. +``analysis/aggregate.halting_rows`` + ``aggregate_halting_cell`` summarise the rows (T6). +""" + +from __future__ import annotations + +import datetime as _dt +import sys +import uuid +from pathlib import Path + +from edc.config import load_from_dict +from edc.eval.evaluate_halting import evaluate_halting +from edc.ledger import RunRecord, append +from edc.registry import build_task + +try: # sys.path[0] is experiments/ when run as a script; fall back to package form. + from run_sweep import cell_config_dict, load_sweep +except ImportError: # pragma: no cover + from experiments.run_sweep import cell_config_dict, load_sweep + + +def run_halting_and_append(cfg, task) -> dict: + """Train + CRC-calibrate + evaluate halting for one cell, append a ``split='halting'`` row.""" + m = evaluate_halting(cfg, task) + return append(RunRecord( + run_id=uuid.uuid4().hex[:12], + timestamp=_dt.datetime.now(_dt.UTC).isoformat(), + resolved_config=cfg.to_dict(), + task=task.name, + split="halting", + seed=cfg.run.seed, + metrics=m, + )) + + +def main(argv: list[str]) -> int: + if not argv: + print("usage: run_halting_sweep.py ", file=sys.stderr) + return 2 + base, override, cells, _opts = load_sweep(argv[0]) + + print(f"[edc] halting sweep '{Path(argv[0]).name}': {len(cells)} cells " + f"(override={override or '{}'}) -> appending to results/ledger.jsonl") + for i, cell in enumerate(cells, 1): + d = cell_config_dict(base, override, cell) + cfg = load_from_dict(d) + task_kwargs = d.get("task", {}).get(cfg.run.task, {}) + task = build_task(cfg.run.task, **task_kwargs) + print(f"\n[edc] cell {i}/{len(cells)}: {cell}") + row = run_halting_and_append(cfg, task) + m = row["metrics"] + tau = m["tau_hat"] + tau_s = f"{tau:.3f}" if tau is not None else "None" + print(f"[edc] -> run_id={row['run_id']} tau_hat={tau_s} saved={m['compute_saved']:.3f} " + f"risk={m['halting_risk']:.3f} within_budget={m['risk_within_budget']}") + from edc.config import REPO_ROOT + ledger = REPO_ROOT / "results/ledger.jsonl" + print(f"\n[edc] halting sweep complete: {len(cells)} rows appended -> {ledger}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/experiments/run_sweep.py b/experiments/run_sweep.py index a0562f7..4b9c5ed 100644 --- a/experiments/run_sweep.py +++ b/experiments/run_sweep.py @@ -51,14 +51,19 @@ def expand_grid(grid: dict[str, list]) -> list[dict]: return [dict(zip(keys, combo, strict=True)) for combo in combos] -def load_sweep(path: str | Path) -> tuple[dict, dict, list[dict]]: - """Return ``(base_resolved_dict, override, cells)`` from a sweep TOML.""" +def load_sweep(path: str | Path) -> tuple[dict, dict, list[dict], dict]: + """Return ``(base_resolved_dict, override, cells, opts)`` from a sweep TOML. + + ``opts`` carries sweep-level flags that are not config keys — currently ``include_ood`` (default + False), which turns the OOD fold back on for a sweep that needs multi-seed OOD/certification. + """ with open(path, "rb") as f: spec = tomllib.load(f)["sweep"] base = load_config(spec["base"]).to_dict() override = spec.get("override", {}) cells = expand_grid(spec.get("grid", {})) - return base, override, cells + opts = {"include_ood": bool(spec.get("include_ood", False))} + return base, override, cells, opts def cell_config_dict(base: dict, override: dict, cell: dict) -> dict: @@ -73,18 +78,19 @@ def main(argv: list[str]) -> int: if not argv: print("usage: run_sweep.py ", file=sys.stderr) return 2 - base, override, cells = load_sweep(argv[0]) + base, override, cells, opts = load_sweep(argv[0]) from edc.config import load_from_dict # local: keep module import surface small + include_ood = opts["include_ood"] print(f"[edc] sweep '{Path(argv[0]).name}': {len(cells)} cells " - f"(override={override or '{}'}) -> appending to results/ledger.jsonl") + f"(override={override or '{}'}, include_ood={include_ood}) -> results/ledger.jsonl") for i, cell in enumerate(cells, 1): d = cell_config_dict(base, override, cell) cfg = load_from_dict(d) task_kwargs = d.get("task", {}).get(cfg.run.task, {}) task = build_task(cfg.run.task, **task_kwargs) print(f"\n[edc] cell {i}/{len(cells)}: {cell}") - row = run_and_append(cfg, task, include_ood=False) + row = run_and_append(cfg, task, include_ood=include_ood) print(f"[edc] -> run_id={row['run_id']}") ledger = REPO_ROOT / "results/ledger.jsonl" print(f"\n[edc] sweep complete: {len(cells)} rows appended -> {ledger}") diff --git a/paper/main.tex b/paper/main.tex index 3886051..2b847d0 100644 --- a/paper/main.tex +++ b/paper/main.tex @@ -32,7 +32,8 @@ terminal energy used by Energy-Based Transformers as a nonconformity score (paired-bootstrap $\Delta$AURC CI excluding zero, $5/5$ seeds each), the advantage is caused by the restart geometry (it grows with the number of restarts; a single descent barely separates) and driven mainly by basin -agreement, and halting saves ${\sim}58\%$ of inference compute at a guaranteed error budget. Against a stronger, standard softmax-confidence baseline (maximum softmax probability, temperature +agreement, and adaptive halting saves $57\%$ (arithmetic) to $81\%$ (graph) of inference compute at a +guaranteed error budget, holding within budget on both tasks across $5$ seeds. Against a stronger, standard softmax-confidence baseline (maximum softmax probability, temperature scaling, entropy), geometry only \emph{ties} (arithmetic) or \emph{loses} (graph) under a supervised basin-center reasoner---but when the reasoner is a genuinely learned, multi-basin landscape, geometry \emph{beats} softmax confidence on arithmetic ($4/5$ seeds). This does not generalize: on diff --git a/paper/sections/experiments.tex b/paper/sections/experiments.tex index 54c5b1e..61f35cc 100644 --- a/paper/sections/experiments.tex +++ b/paper/sections/experiments.tex @@ -41,17 +41,30 @@ \section{Experiments} energy statistics the vector happens to contain. Figure~\ref{fig:f5} shows the corresponding per-feature separation. -\paragraph{The guarantees behave as designed (Figs.~\ref{fig:f3}--\ref{fig:f6}).} In-distribution, -Learn-then-Test selective prediction is valid: achieved selective risk sits on or below the nominal -target across the $\alpha$-sweep (Fig.~\ref{fig:f3}). Conformal-Risk-Control adaptive halting -(Fig.~\ref{fig:f4}) chooses a basin-agreement threshold that uses $42\%$ of the step budget---$58\%$ -compute saved---at a halting risk of $0.9\% \le \alpha{=}10\%$ with no end-task accuracy loss, and -the halting risk was empirically monotone in the threshold (CRC's assumption). Under distribution -shift the guarantee \emph{breaks by design} (Fig.~\ref{fig:f6}): the ID-calibrated threshold, applied -to OOD, achieves $76\%$ selective risk against a $10\%$ target---exchangeability is violated, so the -certificate is void. This is precisely the argument for abstention / shift detection in critical -systems: the guarantees are marginal under exchangeability, and we show them fail loudly when it -does not hold. +\paragraph{The guarantees behave as designed, over 5 seeds (Figs.~\ref{fig:f3}--\ref{fig:f6}, +Tables~\ref{tab:halting}--\ref{tab:oodstress}).} In-distribution, Learn-then-Test selective +prediction is valid: achieved selective risk sits on or below the nominal target across the +$\alpha$-sweep (Fig.~\ref{fig:f3}). Conformal-Risk-Control adaptive halting +(Fig.~\ref{fig:f4}, Table~\ref{tab:halting}) chooses a basin-agreement threshold that saves +$57.1\%\pm1.7\%$ of the step budget on arithmetic at a halting risk of $1.2\% \le \alpha{=}10\%$ +(within budget $5/5$ seeds, no end-task accuracy loss), and $80.8\%\pm5.2\%$ on graph at risk +$6.5\% \le 10\%$ ($5/5$)---the halting guarantee holds on both tasks across seeds, and the halting +risk was empirically monotone in the threshold (CRC's assumption). Under distribution shift the +selective guarantee \emph{breaks by design} over 5 seeds (Fig.~\ref{fig:f6}, +Table~\ref{tab:oodstress}): the ID-calibrated threshold applied to OOD achieves $76.4\%$ selective +risk against a $10\%$ target on arithmetic ($0/5$ within budget) and $16.8\%$ on graph +($4/5$)---exchangeability is violated, so the certificate is void. This is precisely the argument for +abstention / shift detection in critical systems: the guarantees are marginal under exchangeability, +and we show them fail loudly when it does not hold. + +On graph, the LTT abstention certifies almost no coverage at $\alpha{=}0.1$ (mean $1.1\%$ over 5 +seeds at the full $n_\text{calib}=1500$ fold): the task's ${\sim}30\%$ base error leaves no confident +slice with true error below $10\%$. This is an operating-point effect, not a broken certificate---% +validity holds at every $\alpha$ (achieved risk $\le$ target), and relaxing the budget certifies real +coverage: $22.5\%$ at $\alpha{=}0.15$ (achieved risk $0.078$) and $48.5\%$ at $\alpha{=}0.20$ +($0.157$), versus arithmetic's $74\%$ at $\alpha{=}0.10$. The honest reading is that a harder task's +higher error floor pushes the usable operating point to a larger $\alpha$, not that the guarantee +fails. \paragraph{Key caveat: geometry does not beat softmax confidence (Table~\ref{tab:main}).} When the baseline set is widened from scalar energy to a standard softmax-confidence score (MSP, @@ -113,6 +126,8 @@ \section{Experiments} well-calibrated softmax already captures what even the richest descent geometry would add. \input{tables/T1_main} +\input{tables/T6_halting} +\input{tables/T7_ood_stress} \input{tables/T4_complementarity} \input{tables/T5_richer_geometry} \input{tables/T2b_feature_ablation} @@ -138,8 +153,8 @@ \section{Experiments} \includegraphics[width=0.48\linewidth]{F3_coverage_validity.png}\hfill \includegraphics[width=0.48\linewidth]{F4_halting_pareto.png} \caption{\textbf{Left (F3):} LTT selective-risk validity---achieved risk on/below the diagonal. - \textbf{Right (F4):} CRC adaptive halting saves $\sim$58\% compute at a guaranteed risk with no - accuracy loss.} + \textbf{Right (F4):} CRC adaptive halting (arithmetic) saves ${\sim}57\%$ compute at a guaranteed + risk with no accuracy loss; the 5-seed numbers for both tasks are in Table~\ref{tab:halting}.} \label{fig:f3}\label{fig:f4} \end{figure} \begin{figure}[t]\centering diff --git a/paper/tables/T1_main.tex b/paper/tables/T1_main.tex index 20c45da..8572481 100644 --- a/paper/tables/T1_main.tex +++ b/paper/tables/T1_main.tex @@ -1,4 +1,4 @@ -% auto-generated by analysis/make_tables.py — do not edit. run_ids=['aefb55fc7987', '8cfc5a222d83', '8ab8e76e3b44', '7d6cba43dccf', '10f375729230', '7802e0276803', 'd80ff5b236eb', '198178fd3f60', '8867921abd20', 'c81e3990d838'] +% auto-generated by analysis/make_tables.py — do not edit. run_ids=['59f889f657d5', 'b04ffc85ccc3', '354783efb5d1', '5bd7ea239e4f', '52b5abb0b7f5', '2165fc5b4ecc', '496414a8f2cd', '9202e8561acd', '8bc3880fccd3', 'fadafc6f2be1'] \begin{table}[t] \centering \caption{Main selective-prediction results across reasoning families (mean$\pm$std over seeds): geometry's $\Delta$AURC over the best raw-energy baseline and over the best of all baselines (energy, MSP, temperature-scaled MSP, predictive entropy). $\Delta$AURC $>0$ with the CI clearing 0 means geometry wins; ``seeds win`` counts seeds beating the best overall baseline.} @@ -8,8 +8,8 @@ \toprule task & $K$ & acc & AURC geom & $\Delta$AURC vs energy & $\Delta$AURC vs best base & seeds win & LTT cov \\ \midrule - arithmetic & 12 & $0.83 \pm 0.02$ & $0.059 \pm 0.018$ & $0.085 \pm 0.036$ & $-0.011 \pm 0.006$ & 0/5 & $0.55 \pm 0.30$ \\ - graph\_planning & 12 & $0.70 \pm 0.02$ & $0.159 \pm 0.012$ & $0.111 \pm 0.054$ & $-0.034 \pm 0.008$ & 0/5 & $0.00 \pm 0.00$ \\ + arithmetic & 12 & $0.83 \pm 0.02$ & $0.059 \pm 0.018$ & $0.086 \pm 0.038$ & $-0.008 \pm 0.008$ & 0/5 & $0.74 \pm 0.08$ \\ + graph\_planning & 12 & $0.70 \pm 0.02$ & $0.159 \pm 0.016$ & $0.103 \pm 0.045$ & $-0.030 \pm 0.004$ & 0/5 & $0.01 \pm 0.02$ \\ \bottomrule \end{tabular}} \end{table} diff --git a/paper/tables/T2_kablation.tex b/paper/tables/T2_kablation.tex index 7f2cbc1..bfd7136 100644 --- a/paper/tables/T2_kablation.tex +++ b/paper/tables/T2_kablation.tex @@ -1,4 +1,4 @@ -% auto-generated by analysis/make_tables.py — do not edit. run_ids=['acd437459961', '852e712d4157', '8b6b1348cf28', '56ea729e5ea5', '8b8e1e4a7381', 'a4170c27c22b', 'af69f3e49ad2', '25320822fdfd', '9519934cb397', '978eda331152', 'b73a5b1750f2', 'f14b18cfaee4', '04b6026d0afc', '3dc77bab6f6f', '8c4a180e7c9e', 'db2094e35b89', '7a85d96687d2', '7f822ee20bdb', 'e7cfb7fd624f', '539e9c3f0431', '1bfe6b304e0f', 'f63036690d31', '46752558c5df', 'b8dc1a068df5', 'aa60425c10e1', 'c948f8130a3d', '4a5576e4c244', '70741801c041', 'b0d0d7f2f513', '45d9d36ce766', 'f301fe7ebb4d', '39101f30bcb7', 'aefb55fc7987', '8cfc5a222d83', '8ab8e76e3b44', '7d6cba43dccf', '10f375729230', '8d6b3f5b24a2', 'b5b8cfd55dee', '1a2ccd4ef091', 'ff953d4a0e15', 'e14af222ab26'] +% auto-generated by analysis/make_tables.py — do not edit. run_ids=['acd437459961', '852e712d4157', '8b6b1348cf28', '56ea729e5ea5', '8b8e1e4a7381', 'a4170c27c22b', 'af69f3e49ad2', '25320822fdfd', '9519934cb397', '978eda331152', 'b73a5b1750f2', 'f14b18cfaee4', '04b6026d0afc', '3dc77bab6f6f', '8c4a180e7c9e', 'db2094e35b89', '7a85d96687d2', '7f822ee20bdb', 'e7cfb7fd624f', '539e9c3f0431', '1bfe6b304e0f', 'f63036690d31', '46752558c5df', 'b8dc1a068df5', 'aa60425c10e1', 'c948f8130a3d', '4a5576e4c244', '70741801c041', 'b0d0d7f2f513', '45d9d36ce766', 'f301fe7ebb4d', '39101f30bcb7', 'aefb55fc7987', '8cfc5a222d83', '8ab8e76e3b44', '7d6cba43dccf', '10f375729230', '59f889f657d5', 'b04ffc85ccc3', '354783efb5d1', '5bd7ea239e4f', '52b5abb0b7f5', '8d6b3f5b24a2', 'b5b8cfd55dee', '1a2ccd4ef091', 'ff953d4a0e15', 'e14af222ab26'] \begin{table}[t] \centering \caption{Restart ablation: does landscape geometry across $K$ restarts add signal over a single descent ($K{=}1\approx$ EBT)? Lift is $\Delta$AURC $>0$ with the CI clearing 0.} @@ -12,7 +12,7 @@ 2 & $0.103 \pm 0.036$ & $0.146 \pm 0.048$ & $0.043 \pm 0.024$ & 3/5 & $0.18 \pm 0.36$ \\ 4 & $0.082 \pm 0.038$ & $0.144 \pm 0.046$ & $0.062 \pm 0.035$ & 4/5 & $0.29 \pm 0.36$ \\ 8 & $0.076 \pm 0.029$ & $0.145 \pm 0.053$ & $0.070 \pm 0.033$ & 5/5 & $0.54 \pm 0.29$ \\ - 12 & $0.049 \pm 0.026$ & $0.154 \pm 0.063$ & $0.105 \pm 0.053$ & 17/17 & $0.66 \pm 0.28$ \\ + 12 & $0.051 \pm 0.025$ & $0.152 \pm 0.061$ & $0.100 \pm 0.050$ & 22/22 & $0.68 \pm 0.25$ \\ 16 & $0.051 \pm 0.021$ & $0.134 \pm 0.050$ & $0.083 \pm 0.034$ & 5/5 & $0.68 \pm 0.11$ \\ \bottomrule \end{tabular}} diff --git a/paper/tables/T2b_feature_ablation.tex b/paper/tables/T2b_feature_ablation.tex index 282ff84..adcb3f7 100644 --- a/paper/tables/T2b_feature_ablation.tex +++ b/paper/tables/T2b_feature_ablation.tex @@ -8,11 +8,11 @@ \toprule subset (AURC, lower=better) & arithmetic & graph\_planning \\ \midrule - full & $0.059 \pm 0.018$ & $0.159 \pm 0.012$ \\ - drop\_basin & $0.110 \pm 0.030$ & $0.175 \pm 0.017$ \\ - drop\_energy & $0.062 \pm 0.018$ & $0.162 \pm 0.014$ \\ - drop\_curv & $0.063 \pm 0.020$ & $0.160 \pm 0.012$ \\ - drop\_dynamics & $0.064 \pm 0.024$ & $0.187 \pm 0.024$ \\ + full & $0.059 \pm 0.018$ & $0.159 \pm 0.016$ \\ + drop\_basin & $0.116 \pm 0.037$ & $0.175 \pm 0.016$ \\ + drop\_energy & $0.066 \pm 0.019$ & $0.163 \pm 0.019$ \\ + drop\_curv & $0.062 \pm 0.020$ & $0.160 \pm 0.018$ \\ + drop\_dynamics & $0.065 \pm 0.023$ & $0.190 \pm 0.028$ \\ \bottomrule \end{tabular}} \end{table} diff --git a/paper/tables/T3_repro.tex b/paper/tables/T3_repro.tex index 16b7979..281a3c4 100644 --- a/paper/tables/T3_repro.tex +++ b/paper/tables/T3_repro.tex @@ -1,4 +1,4 @@ -% auto-generated by analysis/make_tables.py — do not edit. n_rows=82 +% auto-generated by analysis/make_tables.py — do not edit. n_rows=92 \begin{table}[t] \centering \caption{Reproducibility appendix. Env: python 3.12.13, jax 0.11.0 (cpu). Every row regenerates from seed + config via \texttt{run\_sweep}.} @@ -41,6 +41,8 @@ c468dfa50c2c & 0 & 12 & 1e7f9a8e1d3b & 9367d60 \\ ec2457a78f8d & 0 & 12 & f821965bd47a & 9367d60 \\ abb4dcb4ea84 & 0 & 12 & b333edd88366 & 9367d60 \\ + 59f889f657d5 & 0 & 12 & 646eeb651c32 & 67dc7a2 \\ + 2165fc5b4ecc & 0 & 12 & c5e806bfe9f7 & 67dc7a2 \\ 452500308f3b & 1 & 12 & 60da27bc4d66 & 5207145 \\ b8dc1a068df5 & 1 & 12 & e65ef9e88f92 & 5207145 \\ b0d0d7f2f513 & 1 & 12 & 372c61dccb87 & 3e52940 \\ @@ -52,6 +54,8 @@ 69bcb88d896d & 1 & 12 & b199710e8a3d & 9367d60 \\ 34bc023358b8 & 1 & 12 & f71a346333e0 & 9367d60 \\ 6c7eb2cdb623 & 1 & 12 & a146d7ac1641 & 9367d60 \\ + b04ffc85ccc3 & 1 & 12 & c8e9150407ea & 67dc7a2 \\ + 496414a8f2cd & 1 & 12 & 593dfa402e05 & 67dc7a2 \\ b1e83969abe5 & 2 & 12 & ff22f3433ff5 & 5207145 \\ aa60425c10e1 & 2 & 12 & cacbb6ece843 & 5207145 \\ 45d9d36ce766 & 2 & 12 & c7aaaa71b4ba & 3e52940 \\ @@ -63,6 +67,8 @@ affc597f3e34 & 2 & 12 & cd311d01e1a0 & 9367d60 \\ 28ac1614501c & 2 & 12 & d133e8a33968 & 9367d60 \\ 9fdd2103f70b & 2 & 12 & a55d25d137ca & 9367d60 \\ + 354783efb5d1 & 2 & 12 & 3c2fcb2e5dde & 67dc7a2 \\ + 9202e8561acd & 2 & 12 & 882cd7965583 & 67dc7a2 \\ 70d97e02bed0 & 3 & 12 & fbed14eeb756 & 5207145 \\ c948f8130a3d & 3 & 12 & 054f01c6c637 & 5207145 \\ f301fe7ebb4d & 3 & 12 & 49d35c26cae8 & 3e52940 \\ @@ -74,6 +80,8 @@ 37776ebe2a5b & 3 & 12 & f586dabb6126 & 9367d60 \\ 39b5492337a7 & 3 & 12 & a93a0c192b97 & 9367d60 \\ cb3102b09662 & 3 & 12 & a0a621e5442c & 9367d60 \\ + 5bd7ea239e4f & 3 & 12 & 0c62cba9cde6 & 67dc7a2 \\ + 8bc3880fccd3 & 3 & 12 & e71a6b480861 & 67dc7a2 \\ 46bb116f8c7c & 4 & 12 & 609a4152cf1e & 5207145 \\ 4a5576e4c244 & 4 & 12 & 0b564d7f9e23 & 5207145 \\ 39101f30bcb7 & 4 & 12 & 83b44608e95e & 3e52940 \\ @@ -85,6 +93,8 @@ e7b91099dfc5 & 4 & 12 & d5fa1e138f15 & 9367d60 \\ 61d34e2aec3a & 4 & 12 & e76acc6a32ac & 9367d60 \\ a405cf7b867a & 4 & 12 & 23346bbdcf92 & 9367d60 \\ + 52b5abb0b7f5 & 4 & 12 & 4a05527e96db & 67dc7a2 \\ + fadafc6f2be1 & 4 & 12 & 68757964fe31 & 67dc7a2 \\ 8d6b3f5b24a2 & 0 & 16 & 3c20300a5e78 & 758d5f2 \\ b5b8cfd55dee & 1 & 16 & 22b25365d745 & 758d5f2 \\ 1a2ccd4ef091 & 2 & 16 & b6cea5e21b36 & 758d5f2 \\ diff --git a/paper/tables/T4_complementarity.tex b/paper/tables/T4_complementarity.tex index 5e07d78..e8ab310 100644 --- a/paper/tables/T4_complementarity.tex +++ b/paper/tables/T4_complementarity.tex @@ -1,4 +1,4 @@ -% auto-generated by analysis/make_tables.py — do not edit. run_ids=['aefb55fc7987', '8cfc5a222d83', '8ab8e76e3b44', '7d6cba43dccf', '10f375729230', '70741801c041', 'b0d0d7f2f513', '45d9d36ce766', 'f301fe7ebb4d', '39101f30bcb7', '7802e0276803', 'd80ff5b236eb', '198178fd3f60', '8867921abd20', 'c81e3990d838', '2a91bbeda6e4', 'a4a1a13b8234', 'a009b8f5ff16', 'c85bba3fffca', '6d2a4e347994'] +% auto-generated by analysis/make_tables.py — do not edit. run_ids=['59f889f657d5', 'b04ffc85ccc3', '354783efb5d1', '5bd7ea239e4f', '52b5abb0b7f5', '70741801c041', 'b0d0d7f2f513', '45d9d36ce766', 'f301fe7ebb4d', '39101f30bcb7', '2165fc5b4ecc', '496414a8f2cd', '9202e8561acd', '8bc3880fccd3', 'fadafc6f2be1', '2a91bbeda6e4', 'a4a1a13b8234', 'a009b8f5ff16', 'c85bba3fffca', '6d2a4e347994'] \begin{table}[t] \centering \caption{Complementarity (mean$\pm$std over seeds): head-to-head geometry vs the best softmax baseline, and whether geometry \emph{adds} conditional signal on top of a learned softmax (a mapper on [geometry $\cup$ softmax] vs [softmax]). $>0$ with the CI clearing 0 (``seeds add'') means geometry is complementary, even where it ties head-to-head.} @@ -8,9 +8,9 @@ \toprule task & reasoner & $\Delta$AURC vs softmax & $\Delta$AURC geom \emph{adds} & seeds add \\ \midrule - arithmetic & basin\_center & $-0.011 \pm 0.006$ & $0.001 \pm 0.002$ & 0/5 \\ + arithmetic & basin\_center & $-0.008 \pm 0.008$ & $0.004 \pm 0.002$ & 3/5 \\ arithmetic & ired & $0.007 \pm 0.005$ & $0.007 \pm 0.006$ & 4/5 \\ - graph\_planning & basin\_center & $-0.034 \pm 0.008$ & $-0.006 \pm 0.004$ & 0/5 \\ + graph\_planning & basin\_center & $-0.030 \pm 0.004$ & $-0.002 \pm 0.001$ & 0/5 \\ graph\_planning & ired & $0.001 \pm 0.003$ & $0.001 \pm 0.003$ & 0/5 \\ \bottomrule \end{tabular}} diff --git a/paper/tables/T5_richer_geometry.tex b/paper/tables/T5_richer_geometry.tex index e1554e5..1739122 100644 --- a/paper/tables/T5_richer_geometry.tex +++ b/paper/tables/T5_richer_geometry.tex @@ -1,4 +1,4 @@ -% auto-generated by analysis/make_tables.py — do not edit. run_ids=['aefb55fc7987', '8cfc5a222d83', '8ab8e76e3b44', '7d6cba43dccf', '10f375729230', 'afd5e1407276', 'eb3b1db3b4e5', '7c7410c6f66f', 'd117be743bac', '682227a040a9', '70741801c041', 'b0d0d7f2f513', '45d9d36ce766', 'f301fe7ebb4d', '39101f30bcb7', 'ec2457a78f8d', '34bc023358b8', '28ac1614501c', '39b5492337a7', '61d34e2aec3a', '7802e0276803', 'd80ff5b236eb', '198178fd3f60', '8867921abd20', 'c81e3990d838', 'c468dfa50c2c', '69bcb88d896d', 'affc597f3e34', '37776ebe2a5b', 'e7b91099dfc5', '2a91bbeda6e4', 'a4a1a13b8234', 'a009b8f5ff16', 'c85bba3fffca', '6d2a4e347994', 'abb4dcb4ea84', '6c7eb2cdb623', '9fdd2103f70b', 'cb3102b09662', 'a405cf7b867a'] +% auto-generated by analysis/make_tables.py — do not edit. run_ids=['59f889f657d5', 'b04ffc85ccc3', '354783efb5d1', '5bd7ea239e4f', '52b5abb0b7f5', 'afd5e1407276', 'eb3b1db3b4e5', '7c7410c6f66f', 'd117be743bac', '682227a040a9', '70741801c041', 'b0d0d7f2f513', '45d9d36ce766', 'f301fe7ebb4d', '39101f30bcb7', 'ec2457a78f8d', '34bc023358b8', '28ac1614501c', '39b5492337a7', '61d34e2aec3a', '2165fc5b4ecc', '496414a8f2cd', '9202e8561acd', '8bc3880fccd3', 'fadafc6f2be1', 'c468dfa50c2c', '69bcb88d896d', 'affc597f3e34', '37776ebe2a5b', 'e7b91099dfc5', '2a91bbeda6e4', 'a4a1a13b8234', 'a009b8f5ff16', 'c85bba3fffca', '6d2a4e347994', 'abb4dcb4ea84', '6c7eb2cdb623', '9fdd2103f70b', 'cb3102b09662', 'a405cf7b867a'] \begin{table}[t] \centering \caption{Richer geometry (mean$\pm$std over seeds): the base 14-feature vector vs the base plus the full Hessian spectrum and mode-connectivity barriers. If ``richer`` does not raise $\Delta$AURC vs softmax or the conditional ``geom adds`` on the graph cell, the graph redundancy is a task property, not an artifact of thin curvature features.} @@ -8,11 +8,11 @@ \toprule task & reasoner & features & AURC geom & $\Delta$AURC vs softmax & $\Delta$AURC geom \emph{adds} & seeds add \\ \midrule - arithmetic & basin\_center & base & $0.059 \pm 0.018$ & $-0.011 \pm 0.006$ & $0.001 \pm 0.002$ & 0/5 \\ + arithmetic & basin\_center & base & $0.059 \pm 0.018$ & $-0.008 \pm 0.008$ & $0.004 \pm 0.002$ & 3/5 \\ arithmetic & basin\_center & richer & $0.061 \pm 0.020$ & $-0.012 \pm 0.007$ & $-0.002 \pm 0.005$ & 1/5 \\ arithmetic & ired & base & $0.028 \pm 0.028$ & $0.007 \pm 0.005$ & $0.007 \pm 0.006$ & 4/5 \\ arithmetic & ired & richer & $0.028 \pm 0.029$ & $0.008 \pm 0.006$ & $0.007 \pm 0.006$ & 4/5 \\ - graph\_planning & basin\_center & base & $0.159 \pm 0.012$ & $-0.034 \pm 0.008$ & $-0.006 \pm 0.004$ & 0/5 \\ + graph\_planning & basin\_center & base & $0.159 \pm 0.016$ & $-0.030 \pm 0.004$ & $-0.002 \pm 0.001$ & 0/5 \\ graph\_planning & basin\_center & richer & $0.162 \pm 0.014$ & $-0.037 \pm 0.011$ & $-0.009 \pm 0.006$ & 0/5 \\ graph\_planning & ired & base & $0.056 \pm 0.005$ & $0.001 \pm 0.003$ & $0.001 \pm 0.003$ & 0/5 \\ graph\_planning & ired & richer & $0.055 \pm 0.005$ & $0.002 \pm 0.002$ & $-0.002 \pm 0.008$ & 0/5 \\ diff --git a/paper/tables/T6_halting.tex b/paper/tables/T6_halting.tex new file mode 100644 index 0000000..efba281 --- /dev/null +++ b/paper/tables/T6_halting.tex @@ -0,0 +1,15 @@ +% auto-generated by analysis/make_tables.py — do not edit. run_ids=['a6c36d85615f', '5b496a841123', '24d91758f427', '19cb5e9f64c6', '2a54cf585cb3', 'eaebf4a2b9f3', '20d1395ffa7a', '70cee2dae378', '470896cedd8b', 'ed919980f35e'] +\begin{table}[t] + \centering + \caption{Adaptive halting (CRC), mean$\pm$std over seeds: the basin-agreement threshold $\hat\tau$ is calibrated on a disjoint fold so the halting risk (disagreement with the full-budget answer) stays within the budget $\alpha$. ``compute saved`` is the fraction of descent steps skipped; ``seeds $\le\alpha$`` counts seeds with test risk in budget.} + \label{tab:halting} + \resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{% + \begin{tabular}{lcccccc} + \toprule + task & $\alpha$ & $\hat\tau$ & compute saved & halting risk & acc drop & seeds $\le\alpha$ \\ + \midrule + arithmetic & $0.1$ & $0.92 \pm 0.00$ & $0.571 \pm 0.017$ & $0.012 \pm 0.005$ & $-0.007 \pm 0.006$ & 5/5 \\ + graph\_planning & $0.1$ & $0.85 \pm 0.03$ & $0.808 \pm 0.052$ & $0.065 \pm 0.032$ & $0.011 \pm 0.007$ & 5/5 \\ + \bottomrule + \end{tabular}} +\end{table} diff --git a/paper/tables/T7_ood_stress.tex b/paper/tables/T7_ood_stress.tex new file mode 100644 index 0000000..e55f7ee --- /dev/null +++ b/paper/tables/T7_ood_stress.tex @@ -0,0 +1,15 @@ +% auto-generated by analysis/make_tables.py — do not edit. run_ids=['59f889f657d5', 'b04ffc85ccc3', '354783efb5d1', '5bd7ea239e4f', '52b5abb0b7f5', '2165fc5b4ecc', '496414a8f2cd', '9202e8561acd', '8bc3880fccd3', 'fadafc6f2be1'] +\begin{table}[t] + \centering + \caption{OOD stress (mean$\pm$std over seeds): the ID-calibrated selective-prediction threshold applied to a shifted fold. The selective-risk guarantee holds in-distribution but breaks under shift (OOD risk exceeds the budget), motivating abstention; ``seeds OOD ok`` counts seeds whose OOD risk still stays within budget.} + \label{tab:oodstress} + \resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{% + \begin{tabular}{lccccc} + \toprule + task & acc ID & acc OOD & sel.\ risk ID & sel.\ risk OOD & seeds OOD ok \\ + \midrule + arithmetic & $0.83 \pm 0.02$ & $0.21 \pm 0.01$ & $0.077 \pm 0.004$ & $0.764 \pm 0.012$ & 0/5 \\ + graph\_planning & $0.70 \pm 0.02$ & $0.33 \pm 0.05$ & $0.010 \pm 0.019$ & $0.168 \pm 0.336$ & 4/5 \\ + \bottomrule + \end{tabular}} +\end{table} diff --git a/pyproject.toml b/pyproject.toml index f12bb4c..71abfc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "energy-descent-certificates" -version = "0.0.4" +version = "0.0.5" description = "Distribution-free selective prediction and adaptive halting from the geometry of an energy-based reasoner's inference-time descent." readme = "README.md" requires-python = ">=3.12,<3.13" diff --git a/results/ledger.jsonl b/results/ledger.jsonl index 15ff1ff..969af07 100644 --- a/results/ledger.jsonl +++ b/results/ledger.jsonl @@ -112,3 +112,23 @@ {"run_id": "9fdd2103f70b", "timestamp": "2026-07-28T23:18:15.037116+00:00", "git_sha": "9367d60", "config_hash": "a55d25d137ca", "config": {"run": {"seed": 2, "task": "graph_planning", "notes": "stronger IRED learned-landscape reasoner on graph"}, "model": {"latent_dim": 48, "hidden_dim": 256, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.01, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "annealed", "anneal_levels": 20, "anneal_steps_per_level": 10, "anneal_step_max": 0.5, "anneal_step_min": 0.003}, "train": {"epochs": 110, "batch_size": 128, "lr": 0.001, "n_train": 12000, "n_neg": 4, "neg_noise": 0.5, "objective": "ired", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.2, "ired_decode_weight": 4.0, "ired_stat_weight": 8.0}, "eval": {"n_eval": 700, "richer_geometry": true}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 800}, "task": {"graph_planning": {"n_nodes": 7, "ood_n_nodes": 10, "edge_prob": 0.4, "max_len": 4}}}, "task": "graph_planning", "split": "selective", "seed": 2, "metrics": {"n_fit": 700, "n_calib": 800, "n_test": 700, "k_restarts": 12, "objective": "ired", "sampler": "annealed", "feature_set": "richer", "accuracy_id": 0.8214285714285714, "base_error": 0.17857142857142858, "final_train_loss": 0.1651451736688614, "feature_names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual", "spectrum/lmin_mean", "spectrum/lmin_best", "spectrum/negfrac_mean", "spectrum/negfrac_best", "spectrum/effrank_mean", "spectrum/effrank_best", "spectrum/logdet_mean", "spectrum/logdet_best", "connect/barrier_mean", "connect/barrier_max", "connect/connected_frac"], "aurc": {"geometry": 0.060251437947040715, "rho_basin": 0.09443079176745177, "energy_min": 0.19082820106844728, "energy_mean": 0.1866157548243378, "energy_std": 0.17904397128304222, "msp": 0.07817313287375449, "temp_msp": 0.061490269632624606, "entropy": 0.07723791360586213, "softmax_learned": 0.06164043542026263, "geom_softmax": 0.061066118791798824}, "temperature": 7.072506528454141, "best_energy_baseline": "energy_std", "best_baseline": "temp_msp", "delta_aurc_vs_energy_min": [0.13057676312140656, 0.09222572919121763, 0.17134522211586267], "delta_aurc_vs_best_energy": [0.11879253333600151, 0.08170908894261716, 0.1550689073356496], "delta_aurc_vs_best_baseline": [0.001238831685583891, -0.01255199296098033, 0.014540953603940372], "delta_aurc_geom_adds": [0.0005743166284638071, -0.011961335862339472, 0.01325108482993284], "geometry_wins": true, "geometry_wins_vs_baseline": false, "geometry_adds_over_softmax": false, "risk_coverage": {"geometry": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.07142857142857142, 0.03571428571428571, 0.023809523809523808, 0.017857142857142856, 0.014285714285714285, 0.011904761904761902, 0.010204081632653062, 0.008928571428571428, 0.007936507936507936, 0.0071428571428571435, 0.006493506493506494, 0.005952380952380952, 0.005494505494505495, 0.00510204081632653, 0.0047619047619047615, 0.004464285714285714, 0.004201680672268907, 0.007936507936507934, 0.007518796992481203, 0.007142857142857143, 0.006802721088435373, 0.006493506493506494, 0.012422360248447204, 0.023809523809523805, 0.03428571428571429, 0.04120879120879121, 0.04497354497354497, 0.05102040816326531, 0.0541871921182266, 0.05714285714285714, 0.06451612903225806, 0.07142857142857142, 0.07575757575757576, 0.07983193277310924, 0.09183673469387754, 0.09920634920634919, 0.10617760617760617, 0.11466165413533834, 0.12637362637362637, 0.13035714285714287, 0.13588850174216024, 0.14285714285714282, 0.1461794019933555, 0.1525974025974026, 0.15555555555555556, 0.16304347826086957, 0.16413373860182368, 0.168154761904762, 0.17346938775510204, 0.17857142857142858]}, "rho_basin": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.07726269315673287, 0.07726269315673287, 0.07726269315673294, 0.07726269315673297, 0.077262693156733, 0.077262693156733, 0.07726269315673301, 0.07726269315673302, 0.07726269315673302, 0.07726269315673302, 0.07726269315673304, 0.07726269315673304, 0.07726269315673304, 0.07726269315673304, 0.07726269315673304, 0.07726269315673304, 0.07726269315673304, 0.07726269315673304, 0.07726269315673304, 0.07726269315673304, 0.07726269315673304, 0.07726269315673304, 0.07726269315673304, 0.07726269315673304, 0.07726269315673305, 0.07726269315673305, 0.07726269315673305, 0.07726269315673305, 0.07726269315673305, 0.07726269315673305, 0.07726269315673305, 0.07726269315673305, 0.08130081300813018, 0.08727881396461024, 0.09291521486643432, 0.09823848238482372, 0.10327400571302986, 0.10804450149764624, 0.11257035647279512, 0.11686991869918655, 0.1209597461828272, 0.12714609653385106, 0.133417708168538, 0.13940424654710287, 0.14512471655328707, 0.15167480035492367, 0.1598458532349103, 0.16721491228070112, 0.17346938775510137, 0.17857142857142796]}, "energy_min": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.5, 0.35714285714285715, 0.2857142857142857, 0.25, 0.24285714285714285, 0.23809523809523805, 0.21428571428571433, 0.21428571428571427, 0.20634920634920634, 0.1928571428571429, 0.19480519480519481, 0.19047619047619047, 0.1813186813186813, 0.17857142857142858, 0.1857142857142857, 0.17410714285714285, 0.17647058823529413, 0.1706349206349206, 0.16917293233082706, 0.17142857142857143, 0.16666666666666663, 0.1590909090909091, 0.16459627329192547, 0.16369047619047616, 0.16285714285714287, 0.16483516483516483, 0.164021164021164, 0.16326530612244897, 0.15763546798029554, 0.15714285714285714, 0.16129032258064516, 0.16071428571428573, 0.16233766233766234, 0.16596638655462184, 0.16530612244897958, 0.16666666666666663, 0.1640926640926641, 0.16541353383458646, 0.16483516483516483, 0.16428571428571428, 0.1655052264808362, 0.16496598639455792, 0.17109634551495018, 0.17694805194805194, 0.1761904761904762, 0.17391304347826086, 0.17629179331306988, 0.17559523809523805, 0.17346938775510204, 0.17857142857142858]}, "energy_mean": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.6428571428571429, 0.35714285714285715, 0.2619047619047619, 0.23214285714285715, 0.24285714285714285, 0.21428571428571425, 0.19387755102040818, 0.17857142857142858, 0.18253968253968253, 0.1857142857142856, 0.17532467532467533, 0.17857142857142858, 0.17032967032967034, 0.16326530612244897, 0.15238095238095237, 0.15178571428571427, 0.14705882352941177, 0.15476190476190474, 0.15413533834586465, 0.15357142857142858, 0.15646258503401358, 0.1525974025974026, 0.15527950310559005, 0.1607142857142857, 0.15428571428571428, 0.1565934065934066, 0.16137566137566137, 0.16071428571428573, 0.15517241379310343, 0.1523809523809524, 0.15668202764976957, 0.15848214285714285, 0.16017316017316016, 0.15756302521008403, 0.15918367346938772, 0.1607142857142857, 0.16216216216216217, 0.16165413533834586, 0.16117216117216118, 0.16428571428571428, 0.16550522648083635, 0.17006802721088432, 0.16943521594684385, 0.17045454545454544, 0.16666666666666666, 0.16770186335403728, 0.16717325227963523, 0.168154761904762, 0.17346938775510204, 0.17857142857142858]}, "energy_std": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.35714285714285715, 0.25, 0.19047619047619047, 0.21428571428571427, 0.18571428571428572, 0.19047619047619044, 0.17346938775510207, 0.17857142857142858, 0.18253968253968253, 0.1785714285714286, 0.18181818181818182, 0.17857142857142858, 0.17582417582417584, 0.1683673469387755, 0.1714285714285714, 0.17410714285714285, 0.1722689075630252, 0.17460317460317457, 0.17293233082706766, 0.17142857142857143, 0.17006802721088432, 0.16233766233766234, 0.15838509316770186, 0.16369047619047616, 0.16285714285714287, 0.16758241758241757, 0.1693121693121693, 0.17091836734693877, 0.17241379310344845, 0.1738095238095238, 0.17511520737327188, 0.171875, 0.16883116883116883, 0.16596638655462184, 0.16530612244897974, 0.16666666666666663, 0.16602316602316602, 0.16541353383458646, 0.1684981684981685, 0.16607142857142856, 0.16376306620209055, 0.16496598639455778, 0.16611295681063123, 0.1672077922077922, 0.16825396825396827, 0.16925465838509315, 0.16869300911854102, 0.1681547619047619, 0.1749271137026239, 0.17857142857142858]}, "msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.03571428571428571, 0.047619047619047616, 0.03571428571428571, 0.04285714285714286, 0.03571428571428582, 0.04081632653061225, 0.044642857142857144, 0.03968253968253968, 0.042857142857142864, 0.03896103896103896, 0.03571428571428571, 0.038461538461538464, 0.04081632653061224, 0.03809523809523809, 0.044642857142857144, 0.04201680672268908, 0.05158730158730158, 0.05263157894736842, 0.05714285714285714, 0.054421768707482984, 0.05194805194805195, 0.055900621118012424, 0.06249999999999999, 0.06, 0.07417582417582418, 0.07142857142857142, 0.07142857142857142, 0.07142857142857141, 0.0761904761904762, 0.07603686635944701, 0.078125, 0.08441558441558442, 0.09453781512605042, 0.09795918367346952, 0.10515873015873015, 0.11003861003861004, 0.11842105263157894, 0.12087912087912088, 0.125, 0.1289198606271778, 0.13265306122448992, 0.1362126245847176, 0.1396103896103896, 0.14603174603174604, 0.14906832298136646, 0.15653495440729495, 0.16369047619047628, 0.16909620991253643, 0.17857142857142858]}, "temp_msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.07142857142857142, 0.03571428571428571, 0.023809523809523808, 0.017857142857142856, 0.014285714285714285, 0.011904761904761902, 0.010204081632653062, 0.008928571428571428, 0.007936507936507936, 0.0071428571428571435, 0.006493506493506494, 0.005952380952380952, 0.005494505494505495, 0.00510204081632653, 0.0047619047619047615, 0.004464285714285714, 0.004201680672268907, 0.007936507936507934, 0.007518796992481203, 0.007142857142857143, 0.010204081632653059, 0.00974025974025974, 0.009316770186335404, 0.02678571428571428, 0.03428571428571429, 0.04120879120879121, 0.05291005291005291, 0.061224489795918366, 0.06403940886699507, 0.07142857142857142, 0.08525345622119816, 0.08482142857142858, 0.08874458874458875, 0.09033613445378151, 0.09591836734693891, 0.10317460317460315, 0.11196911196911197, 0.11654135338345864, 0.12087912087912088, 0.12678571428571428, 0.12717770034843218, 0.13605442176870747, 0.1362126245847176, 0.1396103896103896, 0.14603174603174604, 0.14751552795031056, 0.15501519756838916, 0.16369047619047616, 0.16909620991253643, 0.17857142857142858]}, "entropy": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.03571428571428571, 0.047619047619047616, 0.03571428571428571, 0.02857142857142857, 0.035714285714285705, 0.04081632653061225, 0.044642857142857144, 0.03968253968253968, 0.03571428571428572, 0.03896103896103896, 0.03571428571428571, 0.038461538461538464, 0.04081632653061224, 0.03809523809523809, 0.04017857142857143, 0.04201680672268908, 0.04761904761904761, 0.05263157894736842, 0.05357142857142857, 0.054421768707482984, 0.05194805194805195, 0.052795031055900624, 0.059523809523809514, 0.06, 0.06868131868131869, 0.07142857142857142, 0.07142857142857142, 0.07142857142857141, 0.0761904761904762, 0.07603686635944701, 0.078125, 0.08441558441558442, 0.09453781512605042, 0.09795918367346952, 0.10515873015873015, 0.11003861003861004, 0.11842105263157894, 0.12087912087912088, 0.12678571428571428, 0.13066202090592333, 0.13265306122448992, 0.1362126245847176, 0.1396103896103896, 0.14603174603174604, 0.14906832298136646, 0.15653495440729495, 0.16369047619047628, 0.16909620991253643, 0.17857142857142858]}, "softmax_learned": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.07142857142857142, 0.03571428571428571, 0.023809523809523808, 0.017857142857142856, 0.014285714285714285, 0.011904761904761902, 0.010204081632653062, 0.008928571428571428, 0.007936507936507936, 0.0071428571428571435, 0.006493506493506494, 0.005952380952380952, 0.005494505494505495, 0.00510204081632653, 0.0047619047619047615, 0.004464285714285714, 0.004201680672268907, 0.007936507936507934, 0.007518796992481203, 0.007142857142857143, 0.010204081632653059, 0.00974025974025974, 0.009316770186335404, 0.02678571428571428, 0.03428571428571429, 0.04120879120879121, 0.05291005291005291, 0.061224489795918366, 0.06403940886699507, 0.07142857142857142, 0.08525345622119816, 0.08482142857142858, 0.08874458874458875, 0.09033613445378151, 0.09591836734693891, 0.10317460317460315, 0.11196911196911197, 0.11654135338345864, 0.12087912087912088, 0.12678571428571428, 0.12717770034843218, 0.13605442176870747, 0.1362126245847176, 0.1396103896103896, 0.14603174603174604, 0.14751552795031056, 0.15501519756838902, 0.1651785714285714, 0.1749271137026239, 0.17857142857142858]}, "geom_softmax": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.03571428571428571, 0.023809523809523808, 0.017857142857142856, 0.014285714285714285, 0.011904761904761902, 0.010204081632653062, 0.008928571428571428, 0.007936507936507936, 0.0071428571428571435, 0.006493506493506494, 0.005952380952380952, 0.005494505494505495, 0.00510204081632653, 0.0047619047619047615, 0.004464285714285714, 0.004201680672268907, 0.007936507936507934, 0.007518796992481203, 0.007142857142857143, 0.006802721088435373, 0.006493506493506494, 0.012422360248447204, 0.020833333333333447, 0.03428571428571429, 0.04120879120879121, 0.05026455026455026, 0.05612244897959184, 0.059113300492611015, 0.06904761904761905, 0.07142857142857142, 0.08258928571428571, 0.09307359307359307, 0.09663865546218488, 0.10000000000000014, 0.10515873015873015, 0.11003861003861004, 0.11842105263157894, 0.12454212454212454, 0.13035714285714287, 0.13240418118466896, 0.13775510204081629, 0.14285714285714285, 0.14935064935064934, 0.15396825396825398, 0.15372670807453417, 0.16109422492401212, 0.16964285714285712, 0.17346938775510204, 0.17857142857142858]}}, "ltt": {"alpha": 0.1, "delta": 0.05, "lambda_hat": 0.16349649886293133, "coverage": 0.5314285714285715, "abstain_rate": 0.4685714285714286, "selective_risk": 0.0456989247311828, "selective_accuracy": 0.9543010752688172, "risk_within_budget": true}, "coverage_validity": {"alpha_grid": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "achieved_risk": [0.0, 0.006993006993006993, 0.0456989247311828, 0.1053639846743295, 0.16566265060240964, 0.17621776504297995], "coverage": [0.0, 0.4085714285714286, 0.5314285714285715, 0.7457142857142857, 0.9485714285714286, 0.9971428571428571]}, "feature_diagnostics": {"names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual", "spectrum/lmin_mean", "spectrum/lmin_best", "spectrum/negfrac_mean", "spectrum/negfrac_best", "spectrum/effrank_mean", "spectrum/effrank_best", "spectrum/logdet_mean", "spectrum/logdet_best", "connect/barrier_mean", "connect/barrier_max", "connect/connected_frac"], "auroc": {"basin/rho": 0.7423443478260869, "basin/entropy": 0.2577530434782609, "basin/dispersion": 0.48076521739130434, "energy/mean": 0.45377391304347825, "energy/min": 0.4752, "energy/std": 0.46964869565217393, "curv/lmax_mean": 0.49575652173913043, "curv/lmax_best": 0.4976904347826087, "curv/trace_mean": 0.5, "curv/trace_best": 0.5, "dynamics/steps": 0.5, "dynamics/monotonic": 0.5954643478260869, "dynamics/drop": 0.7773495652173913, "dynamics/residual": 0.4896973913043478, "spectrum/lmin_mean": 0.5, "spectrum/lmin_best": 0.5, "spectrum/negfrac_mean": 0.5, "spectrum/negfrac_best": 0.5, "spectrum/effrank_mean": 0.5, "spectrum/effrank_best": 0.5, "spectrum/logdet_mean": 0.5, "spectrum/logdet_best": 0.5, "connect/barrier_mean": 0.4774539130434783, "connect/barrier_max": 0.47738434782608696, "connect/connected_frac": 0.5224904347826087}, "hist": {"basin/rho": {"edges": [0.4166666666666667, 0.44583333333333336, 0.47500000000000003, 0.5041666666666667, 0.5333333333333333, 0.5625, 0.5916666666666667, 0.6208333333333333, 0.65, 0.6791666666666667, 0.7083333333333333, 0.7375, 0.7666666666666666, 0.7958333333333334, 0.825, 0.8541666666666666, 0.8833333333333333, 0.9125, 0.9416666666666667, 0.9708333333333332, 1.0], "correct_counts": [0, 0, 3, 0, 0, 5, 0, 0, 10, 0, 0, 13, 0, 0, 38, 0, 0, 88, 0, 418], "incorrect_counts": [1, 0, 2, 0, 0, 3, 0, 0, 9, 0, 0, 15, 0, 0, 25, 0, 0, 35, 0, 35]}, "basin/entropy": {"edges": [0.0, 0.05387781635334003, 0.10775563270668007, 0.16163344906002008, 0.21551126541336013, 0.2693890817667002, 0.32326689812004017, 0.3771447144733802, 0.43102253082672026, 0.4849003471800603, 0.5387781635334004, 0.5926559798867403, 0.6465337962400803, 0.7004116125934204, 0.7542894289467604, 0.8081672453001005, 0.8620450616534405, 0.9159228780067805, 0.9698006943601206, 1.0236785107134607, 1.0775563270668007], "correct_counts": [418, 0, 0, 0, 0, 88, 0, 0, 36, 0, 13, 10, 6, 2, 0, 0, 0, 1, 1, 0], "incorrect_counts": [35, 0, 0, 0, 0, 35, 0, 0, 24, 0, 14, 8, 5, 1, 0, 2, 0, 0, 0, 1]}, "basin/dispersion": {"edges": [1.98346868203938, 2.1969464865932773, 2.4104242911471747, 2.623902095701072, 2.8373799002549696, 3.050857704808867, 3.2643355093627644, 3.477813313916662, 3.6912911184705592, 3.9047689230244567, 4.118246727578354, 4.3317245321322515, 4.545202336686149, 4.758680141240046, 4.972157945793944, 5.185635750347841, 5.399113554901739, 5.612591359455636, 5.8260691640095335, 6.039546968563431, 6.253024773117328], "correct_counts": [312, 249, 1, 1, 0, 1, 0, 0, 0, 1, 2, 0, 2, 2, 0, 1, 1, 1, 0, 1], "incorrect_counts": [62, 53, 0, 0, 0, 1, 2, 0, 0, 2, 0, 2, 1, 1, 1, 0, 0, 0, 0, 0]}, "energy/mean": {"edges": [-2.283770283063253, -2.1011410554250083, -1.9185118277867637, -1.735882600148519, -1.5532533725102744, -1.3706241448720298, -1.1879949172337851, -1.0053656895955405, -0.8227364619572959, -0.6401072343190513, -0.4574780066808066, -0.274848779042562, -0.09221955140431737, 0.09040967623392726, 0.2730389038721719, 0.4556681315104165, 0.6382973591486611, 0.8209265867869058, 1.0035558144251504, 1.186185042063395, 1.3688142697016399], "correct_counts": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 514, 44, 1, 5, 2, 1, 1], "incorrect_counts": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 96, 10, 3, 4, 3, 0, 0]}, "energy/min": {"edges": [-2.531198501586914, -2.3657448172569273, -2.200291132926941, -2.0348374485969543, -1.8693837642669677, -1.7039300799369812, -1.5384763956069945, -1.373022711277008, -1.2075690269470214, -1.0421153426170349, -0.8766616582870483, -0.7112079739570616, -0.5457542896270751, -0.38030060529708853, -0.21484692096710178, -0.04939323663711548, 0.11606044769287127, 0.281514132022858, 0.4469678163528443, 0.6124215006828311, 0.7778751850128174], "correct_counts": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 320, 236, 15, 1, 0], "incorrect_counts": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 60, 55, 2, 1, 4]}, "energy/std": {"edges": [0.020960167890197088, 0.10303025054062304, 0.185100333191049, 0.26717041584147494, 0.3492404984919009, 0.43131058114232684, 0.5133806637927528, 0.5954507464431787, 0.6775208290936047, 0.7595909117440306, 0.8416609943944565, 0.9237310770448826, 1.0058011596953085, 1.0878712423457346, 1.1699413249961605, 1.2520114076465865, 1.3340814902970124, 1.4161515729474383, 1.4982216555978642, 1.5802917382482902, 1.662361820898716], "correct_counts": [562, 1, 2, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 3, 1, 1, 0, 1, 1], "incorrect_counts": [117, 0, 3, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]}, "curv/lmax_mean": {"edges": [0.1999999893208345, 0.19999999056259793, 0.19999999180436134, 0.19999999304612479, 0.19999999428788823, 0.19999999552965164, 0.19999999677141508, 0.19999999801317853, 0.19999999925494194, 0.20000000049670538, 0.20000000173846882, 0.20000000298023224, 0.20000000422199568, 0.2000000054637591, 0.20000000670552254, 0.20000000794728598, 0.2000000091890494, 0.20000001043081284, 0.20000001167257628, 0.2000000129143397, 0.20000001415610313], "correct_counts": [5, 0, 3, 25, 0, 12, 69, 0, 74, 140, 0, 58, 52, 51, 38, 18, 14, 9, 1, 6], "incorrect_counts": [0, 0, 1, 4, 0, 4, 21, 0, 9, 29, 0, 14, 9, 18, 8, 3, 4, 1, 0, 0]}, "curv/lmax_best": {"edges": [0.19999995827674866, 0.19999996200203896, 0.19999996572732925, 0.19999996945261955, 0.19999997317790985, 0.19999997690320015, 0.19999998062849045, 0.19999998435378075, 0.19999998807907104, 0.19999999180436134, 0.19999999552965164, 0.19999999925494194, 0.20000000298023224, 0.20000000670552254, 0.20000001043081284, 0.20000001415610313, 0.20000001788139343, 0.20000002160668373, 0.20000002533197403, 0.20000002905726433, 0.20000003278255463], "correct_counts": [2, 0, 0, 0, 5, 0, 0, 0, 221, 0, 0, 0, 149, 0, 0, 0, 186, 0, 0, 12], "incorrect_counts": [0, 0, 0, 0, 1, 0, 0, 0, 48, 0, 0, 0, 32, 0, 0, 0, 43, 0, 0, 1]}, "curv/trace_mean": {"edges": [9.600000381469727, 9.650000381469727, 9.700000381469726, 9.750000381469727, 9.800000381469726, 9.850000381469727, 9.900000381469727, 9.950000381469726, 10.000000381469727, 10.050000381469726, 10.100000381469727, 10.150000381469727, 10.200000381469726, 10.250000381469727, 10.300000381469726, 10.350000381469727, 10.400000381469727, 10.450000381469726, 10.500000381469727, 10.550000381469726, 10.600000381469727], "correct_counts": [575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "curv/trace_best": {"edges": [9.600000381469727, 9.650000381469727, 9.700000381469726, 9.750000381469727, 9.800000381469726, 9.850000381469727, 9.900000381469727, 9.950000381469726, 10.000000381469727, 10.050000381469726, 10.100000381469727, 10.150000381469727, 10.200000381469726, 10.250000381469727, 10.300000381469726, 10.350000381469727, 10.400000381469727, 10.450000381469726, 10.500000381469727, 10.550000381469726, 10.600000381469727], "correct_counts": [575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/steps": {"edges": [1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7000000000000002, 1.75, 1.8, 1.85, 1.9, 1.9500000000000002, 2.0], "correct_counts": [575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/monotonic": {"edges": [0.5341666666666667, 0.5398958333333334, 0.545625, 0.5513541666666667, 0.5570833333333334, 0.5628124999999999, 0.5685416666666666, 0.5742708333333333, 0.58, 0.5857291666666666, 0.5914583333333333, 0.5971875, 0.6029166666666667, 0.6086458333333333, 0.614375, 0.6201041666666667, 0.6258333333333332, 0.6315624999999999, 0.6372916666666666, 0.6430208333333333, 0.6487499999999999], "correct_counts": [1, 9, 47, 95, 139, 148, 80, 38, 13, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1], "incorrect_counts": [3, 5, 12, 25, 31, 31, 14, 1, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/drop": {"edges": [5.8192191918691, 6.054377051194509, 6.289534910519918, 6.524692769845327, 6.759850629170735, 6.995008488496144, 7.230166347821553, 7.4653242071469625, 7.700482066472372, 7.93563992579778, 8.17079778512319, 8.405955644448598, 8.641113503774008, 8.876271363099416, 9.111429222424825, 9.346587081750235, 9.581744941075643, 9.816902800401053, 10.052060659726461, 10.28721851905187, 10.52237637837728], "correct_counts": [12, 12, 29, 49, 54, 41, 29, 18, 4, 5, 5, 18, 40, 53, 55, 63, 42, 31, 12, 3], "incorrect_counts": [7, 8, 13, 28, 18, 19, 19, 4, 4, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0]}, "dynamics/residual": {"edges": [0.280464556068182, 0.2857741178944707, 0.2910836797207594, 0.2963932415470481, 0.30170280337333677, 0.3070123651996255, 0.3123219270259142, 0.3176314888522029, 0.3229410506784916, 0.32825061250478027, 0.333560174331069, 0.3388697361573577, 0.3441792979836464, 0.3494888598099351, 0.35479842163622377, 0.3601079834625125, 0.3654175452888012, 0.3707271071150899, 0.3760366689413786, 0.38134623076766727, 0.386655792593956], "correct_counts": [2, 9, 38, 63, 99, 132, 113, 66, 36, 11, 1, 2, 1, 0, 0, 0, 1, 0, 0, 1], "incorrect_counts": [0, 2, 3, 15, 23, 32, 27, 12, 6, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/lmin_mean": {"edges": [0.20000000298023224, 0.2500000029802322, 0.3000000029802322, 0.35000000298023226, 0.40000000298023225, 0.45000000298023224, 0.5000000029802323, 0.5500000029802323, 0.6000000029802323, 0.6500000029802322, 0.7000000029802322, 0.7500000029802323, 0.8000000029802323, 0.8500000029802323, 0.9000000029802323, 0.9500000029802322, 1.0000000029802323, 1.0500000029802323, 1.1000000029802321, 1.1500000029802324, 1.2000000029802322], "correct_counts": [575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/lmin_best": {"edges": [0.20000000298023224, 0.2500000029802322, 0.3000000029802322, 0.35000000298023226, 0.40000000298023225, 0.45000000298023224, 0.5000000029802323, 0.5500000029802323, 0.6000000029802323, 0.6500000029802322, 0.7000000029802322, 0.7500000029802323, 0.8000000029802323, 0.8500000029802323, 0.9000000029802323, 0.9500000029802322, 1.0000000029802323, 1.0500000029802323, 1.1000000029802321, 1.1500000029802324, 1.2000000029802322], "correct_counts": [575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/negfrac_mean": {"edges": [0.0, 0.05, 0.1, 0.15000000000000002, 0.2, 0.25, 0.30000000000000004, 0.35000000000000003, 0.4, 0.45, 0.5, 0.55, 0.6000000000000001, 0.65, 0.7000000000000001, 0.75, 0.8, 0.8500000000000001, 0.9, 0.9500000000000001, 1.0], "correct_counts": [575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/negfrac_best": {"edges": [0.0, 0.05, 0.1, 0.15000000000000002, 0.2, 0.25, 0.30000000000000004, 0.35000000000000003, 0.4, 0.45, 0.5, 0.55, 0.6000000000000001, 0.65, 0.7000000000000001, 0.75, 0.8, 0.8500000000000001, 0.9, 0.9500000000000001, 1.0], "correct_counts": [575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/effrank_mean": {"edges": [47.99999975000001, 48.049999750000005, 48.09999975000001, 48.149999750000006, 48.19999975000001, 48.24999975000001, 48.299999750000005, 48.34999975000001, 48.399999750000006, 48.44999975000001, 48.49999975000001, 48.549999750000005, 48.59999975000001, 48.649999750000006, 48.69999975000001, 48.74999975000001, 48.799999750000005, 48.84999975000001, 48.899999750000006, 48.94999975000001, 48.99999975000001], "correct_counts": [575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/effrank_best": {"edges": [47.99999975000001, 48.049999750000005, 48.09999975000001, 48.149999750000006, 48.19999975000001, 48.24999975000001, 48.299999750000005, 48.34999975000001, 48.399999750000006, 48.44999975000001, 48.49999975000001, 48.549999750000005, 48.59999975000001, 48.649999750000006, 48.69999975000001, 48.74999975000001, 48.799999750000005, 48.84999975000001, 48.899999750000006, 48.94999975000001, 48.99999975000001], "correct_counts": [575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/logdet_mean": {"edges": [-77.25301668158117, -77.20301668158118, -77.15301668158118, -77.10301668158117, -77.05301668158117, -77.00301668158117, -76.95301668158118, -76.90301668158118, -76.85301668158117, -76.80301668158117, -76.75301668158117, -76.70301668158118, -76.65301668158118, -76.60301668158117, -76.55301668158117, -76.50301668158117, -76.45301668158118, -76.40301668158118, -76.35301668158117, -76.30301668158117, -76.25301668158117], "correct_counts": [575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/logdet_best": {"edges": [-77.25301668158119, -77.20301668158119, -77.15301668158119, -77.10301668158118, -77.05301668158118, -77.00301668158119, -76.95301668158119, -76.90301668158119, -76.85301668158118, -76.80301668158118, -76.75301668158119, -76.70301668158119, -76.65301668158119, -76.60301668158118, -76.55301668158118, -76.50301668158119, -76.45301668158119, -76.40301668158119, -76.35301668158118, -76.30301668158118, -76.25301668158119], "correct_counts": [575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "connect/barrier_mean": {"edges": [0.0, 0.055023192275654186, 0.11004638455130837, 0.16506957682696255, 0.22009276910261674, 0.27511596137827093, 0.3301391536539251, 0.3851623459295793, 0.4401855382052335, 0.49520873048088765, 0.5502319227565419, 0.6052551150321961, 0.6602783073078502, 0.7153014995835044, 0.7703246918591586, 0.8253478841348127, 0.880371076410467, 0.9353942686861212, 0.9904174609617753, 1.0454406532374296, 1.1004638455130837], "correct_counts": [567, 3, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [119, 2, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]}, "connect/barrier_max": {"edges": [0.0, 0.06335251331329346, 0.12670502662658692, 0.1900575399398804, 0.25341005325317384, 0.3167625665664673, 0.3801150798797608, 0.44346759319305423, 0.5068201065063477, 0.5701726198196412, 0.6335251331329346, 0.6968776464462281, 0.7602301597595216, 0.823582673072815, 0.8869351863861085, 0.9502876996994019, 1.0136402130126954, 1.0769927263259889, 1.1403452396392824, 1.2036977529525756, 1.2670502662658691], "correct_counts": [564, 1, 0, 2, 0, 1, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 1, 0, 0], "incorrect_counts": [117, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2, 0, 0, 1]}, "connect/connected_frac": {"edges": [0.09090909090909091, 0.13636363636363635, 0.18181818181818182, 0.22727272727272727, 0.2727272727272727, 0.31818181818181823, 0.36363636363636365, 0.40909090909090906, 0.4545454545454546, 0.5, 0.5454545454545455, 0.5909090909090909, 0.6363636363636364, 0.6818181818181819, 0.7272727272727273, 0.7727272727272728, 0.8181818181818182, 0.8636363636363636, 0.9090909090909092, 0.9545454545454546, 1.0], "correct_counts": [0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 2, 0, 3, 2, 0, 564], "incorrect_counts": [1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 1, 0, 117]}}}, "feature_ablation": {"full": 0.060251437947040715, "drop_basin": 0.06621243031414509, "basin_only": 0.09591115960898094, "drop_energy": 0.06067653000248025, "energy_only": 0.18899204074897005, "drop_curv": 0.05970888894980699, "curv_only": 0.17780760616067234, "drop_dynamics": 0.11094637528088547, "dynamics_only": 0.06495557418904481, "drop_spectrum": 0.060251437947040715, "spectrum_only": 0.1785714285714291, "drop_connect": 0.059672699284536494, "connect_only": 0.17190996073703077}, "ece_geometry": 0.04131107511114491}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} {"run_id": "cb3102b09662", "timestamp": "2026-07-28T23:18:50.308360+00:00", "git_sha": "9367d60", "config_hash": "a0a621e5442c", "config": {"run": {"seed": 3, "task": "graph_planning", "notes": "stronger IRED learned-landscape reasoner on graph"}, "model": {"latent_dim": 48, "hidden_dim": 256, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.01, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "annealed", "anneal_levels": 20, "anneal_steps_per_level": 10, "anneal_step_max": 0.5, "anneal_step_min": 0.003}, "train": {"epochs": 110, "batch_size": 128, "lr": 0.001, "n_train": 12000, "n_neg": 4, "neg_noise": 0.5, "objective": "ired", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.2, "ired_decode_weight": 4.0, "ired_stat_weight": 8.0}, "eval": {"n_eval": 700, "richer_geometry": true}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 800}, "task": {"graph_planning": {"n_nodes": 7, "ood_n_nodes": 10, "edge_prob": 0.4, "max_len": 4}}}, "task": "graph_planning", "split": "selective", "seed": 3, "metrics": {"n_fit": 700, "n_calib": 800, "n_test": 700, "k_restarts": 12, "objective": "ired", "sampler": "annealed", "feature_set": "richer", "accuracy_id": 0.8285714285714286, "base_error": 0.17142857142857143, "final_train_loss": 0.16593526303768158, "feature_names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual", "spectrum/lmin_mean", "spectrum/lmin_best", "spectrum/negfrac_mean", "spectrum/negfrac_best", "spectrum/effrank_mean", "spectrum/effrank_best", "spectrum/logdet_mean", "spectrum/logdet_best", "connect/barrier_mean", "connect/barrier_max", "connect/connected_frac"], "aurc": {"geometry": 0.0533799136280693, "rho_basin": 0.11253061398624546, "energy_min": 0.21096449789187388, "energy_mean": 0.21638804114554994, "energy_std": 0.16204516813051956, "msp": 0.14599198362345006, "temp_msp": 0.057174909505143715, "entropy": 0.14494194070453414, "softmax_learned": 0.05731160565959381, "geom_softmax": 0.05430276688183326}, "temperature": 7.043475628034012, "best_energy_baseline": "energy_std", "best_baseline": "temp_msp", "delta_aurc_vs_energy_min": [0.1575845842638046, 0.1181093627407647, 0.19629026325032864], "delta_aurc_vs_best_energy": [0.10866525450245026, 0.07712510933316896, 0.1425734539285489], "delta_aurc_vs_best_baseline": [0.0037949958770744155, -0.0030469980404765977, 0.010411379788991282], "delta_aurc_geom_adds": [0.00300883877776055, -0.0031222374730943523, 0.008727410350044167], "geometry_wins": true, "geometry_wins_vs_baseline": false, "geometry_adds_over_softmax": false, "risk_coverage": {"geometry": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01020408163265306, 0.009523809523809523, 0.008928571428571428, 0.008403361344537815, 0.007936507936507934, 0.007518796992481203, 0.007142857142857143, 0.006802721088435373, 0.006493506493506494, 0.006211180124223602, 0.014880952380952493, 0.022857142857142857, 0.027472527472527472, 0.037037037037037035, 0.04336734693877551, 0.04926108374384236, 0.05952380952380952, 0.06451612903225806, 0.08035714285714286, 0.08441558441558442, 0.08823529411764706, 0.09387755102040815, 0.0912698412698414, 0.10617760617760617, 0.10714285714285714, 0.11721611721611722, 0.12321428571428572, 0.12369337979094075, 0.13095238095238093, 0.13953488372093023, 0.1444805194805195, 0.14603174603174604, 0.14596273291925466, 0.1474164133738603, 0.1562500000000001, 0.16326530612244897, 0.17142857142857143]}, "rho_basin": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.10282258064516127, 0.10282258064516134, 0.10282258064516135, 0.10282258064516127, 0.10282258064516123, 0.10282258064516127, 0.10282258064516137, 0.10282258064516143, 0.10282258064516149, 0.10282258064516153, 0.10282258064516156, 0.10282258064516145, 0.10282258064516135, 0.10282258064516127, 0.1028225806451612, 0.10282258064516113, 0.10282258064516107, 0.10282258064516102, 0.10282258064516098, 0.10282258064516094, 0.1028225806451609, 0.10282258064516087, 0.10282258064516082, 0.1028225806451608, 0.10282258064516077, 0.10282258064516075, 0.10282258064516073, 0.1028225806451607, 0.10282258064516069, 0.10282258064516067, 0.10282258064516066, 0.10282258064516063, 0.10282258064516062, 0.1028225806451606, 0.10282258064516059, 0.10582595870206428, 0.11085864625687578, 0.1156264555193288, 0.12014976174268162, 0.12444690265486667, 0.12853442693718908, 0.13242730720606752, 0.13613912327639346, 0.14047805642633135, 0.14540229885057374, 0.15011244377810992, 0.15462215700660203, 0.15968406593406492, 0.16472303206996985, 0.17142857142857038]}, "energy_min": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.35714285714285715, 0.39285714285714285, 0.3333333333333333, 0.32142857142857145, 0.2714285714285714, 0.23809523809523817, 0.23469387755102047, 0.24107142857142858, 0.23015873015873015, 0.22142857142857145, 0.21428571428571427, 0.2261904761904762, 0.22527472527472528, 0.2193877551020408, 0.2095238095238095, 0.20535714285714285, 0.21428571428571427, 0.21825396825396823, 0.20676691729323307, 0.20357142857142857, 0.19727891156462582, 0.19805194805194806, 0.19875776397515527, 0.1964285714285714, 0.19428571428571428, 0.19230769230769232, 0.1931216931216931, 0.18877551020408162, 0.18965517241379307, 0.1880952380952381, 0.18663594470046083, 0.19196428571428573, 0.18831168831168832, 0.18277310924369747, 0.17755102040816323, 0.17460317460317457, 0.17567567567567569, 0.17481203007518797, 0.17032967032967034, 0.17142857142857143, 0.1689895470383275, 0.16496598639455778, 0.1644518272425249, 0.16233766233766234, 0.16031746031746033, 0.15683229813664595, 0.15349544072948326, 0.150297619047619, 0.16034985422740525, 0.17142857142857143]}, "energy_mean": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.5, 0.39285714285714285, 0.40476190476190477, 0.4107142857142857, 0.35714285714285715, 0.3214285714285714, 0.28571428571428575, 0.25892857142857145, 0.25396825396825395, 0.24285714285714288, 0.22727272727272727, 0.20833333333333334, 0.1978021978021978, 0.19387755102040816, 0.19999999999999998, 0.19196428571428573, 0.20588235294117646, 0.19841269841269837, 0.20300751879699247, 0.19642857142857142, 0.2040816326530612, 0.19805194805194806, 0.1956521739130435, 0.1964285714285714, 0.2, 0.20054945054945056, 0.19576719576719576, 0.19642857142857142, 0.18965517241379326, 0.1880952380952381, 0.18202764976958524, 0.18080357142857142, 0.18614718614718614, 0.18277310924369747, 0.18163265306122447, 0.17857142857142855, 0.17374517374517376, 0.17293233082706766, 0.17032967032967034, 0.16964285714285715, 0.1655052264808362, 0.16496598639455778, 0.16279069767441862, 0.1590909090909091, 0.15714285714285714, 0.15527950310559005, 0.15197568389057747, 0.1532738095238095, 0.16034985422740525, 0.17142857142857143]}, "energy_std": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.21428571428571427, 0.17857142857142858, 0.19047619047619047, 0.19642857142857142, 0.18571428571428572, 0.16666666666666663, 0.15306122448979595, 0.13392857142857142, 0.12698412698412698, 0.1285714285714286, 0.12987012987012986, 0.13690476190476192, 0.13736263736263737, 0.15306122448979592, 0.1571428571428571, 0.15625, 0.1638655462184874, 0.16269841269841284, 0.16165413533834586, 0.16071428571428573, 0.15986394557823141, 0.16558441558441558, 0.16770186335403728, 0.17559523809523817, 0.17714285714285713, 0.17582417582417584, 0.1746031746031746, 0.17091836734693877, 0.1650246305418719, 0.16666666666666666, 0.16820276497695852, 0.16741071428571427, 0.1645021645021645, 0.16176470588235295, 0.16326530612244894, 0.16666666666666663, 0.16795366795366795, 0.16729323308270677, 0.16666666666666666, 0.16607142857142856, 0.16724738675958187, 0.16666666666666663, 0.16611295681063123, 0.16233766233766234, 0.16031746031746033, 0.16459627329192547, 0.16261398176291791, 0.16220238095238093, 0.1661807580174927, 0.17142857142857143]}, "msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.14285714285714285, 0.25, 0.23809523809523808, 0.19642857142857142, 0.17142857142857143, 0.15476190476190474, 0.15306122448979595, 0.16071428571428573, 0.15079365079365079, 0.1642857142857143, 0.16233766233766234, 0.15476190476190477, 0.15934065934065933, 0.16326530612244897, 0.15238095238095237, 0.14732142857142858, 0.14285714285714285, 0.13888888888888887, 0.14285714285714285, 0.1357142857142857, 0.13605442176870747, 0.13636363636363635, 0.13664596273291926, 0.1369047619047619, 0.13142857142857142, 0.12912087912087913, 0.13227513227513227, 0.13520408163265307, 0.13300492610837436, 0.12857142857142856, 0.1313364055299539, 0.13392857142857142, 0.13203463203463203, 0.13025210084033614, 0.1285714285714287, 0.12698412698412695, 0.12934362934362933, 0.12969924812030076, 0.13003663003663005, 0.13035714285714287, 0.13414634146341475, 0.13775510204081629, 0.14285714285714285, 0.1444805194805195, 0.1492063492063492, 0.15062111801242237, 0.15349544072948326, 0.15922619047619044, 0.1661807580174927, 0.17142857142857143]}, "temp_msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0047619047619047615, 0.004464285714285714, 0.004201680672268907, 0.003968253968253967, 0.0037593984962406013, 0.0035714285714285713, 0.006802721088435373, 0.00974025974025974, 0.018633540372670808, 0.02083333333333333, 0.022857142857142857, 0.03571428571428571, 0.047619047619047616, 0.061224489795918366, 0.07389162561576373, 0.08095238095238096, 0.08294930875576037, 0.08482142857142858, 0.09307359307359307, 0.09663865546218488, 0.1020408163265306, 0.11111111111111124, 0.11969111969111969, 0.11842105263157894, 0.1227106227106227, 0.12321428571428572, 0.12543554006968638, 0.13095238095238093, 0.1378737541528239, 0.1396103896103896, 0.14761904761904762, 0.15217391304347827, 0.15501519756838902, 0.1607142857142857, 0.16472303206997085, 0.17142857142857143]}, "entropy": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.21428571428571427, 0.21428571428571427, 0.21428571428571427, 0.19642857142857142, 0.17142857142857143, 0.15476190476190474, 0.14285714285714288, 0.16071428571428573, 0.15873015873015872, 0.1642857142857143, 0.15584415584415584, 0.15476190476190477, 0.15934065934065933, 0.15306122448979592, 0.15238095238095237, 0.14732142857142858, 0.14285714285714285, 0.13888888888888887, 0.13909774436090225, 0.1357142857142857, 0.12925170068027222, 0.1331168831168831, 0.13043478260869565, 0.1339285714285714, 0.13142857142857142, 0.12912087912087913, 0.12962962962962962, 0.1326530612244898, 0.13300492610837436, 0.12857142857142856, 0.12903225806451613, 0.13392857142857142, 0.13203463203463203, 0.13025210084033614, 0.12857142857142853, 0.12698412698412695, 0.12934362934362933, 0.12781954887218044, 0.13003663003663005, 0.13035714285714287, 0.1324041811846691, 0.13775510204081629, 0.14119601328903655, 0.1444805194805195, 0.1492063492063492, 0.15062111801242237, 0.15349544072948326, 0.15922619047619044, 0.1661807580174927, 0.17142857142857143]}, "softmax_learned": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0047619047619047615, 0.004464285714285714, 0.004201680672268907, 0.003968253968253967, 0.0037593984962406013, 0.0035714285714285713, 0.006802721088435373, 0.00974025974025974, 0.018633540372670808, 0.02083333333333333, 0.022857142857142857, 0.03571428571428571, 0.047619047619047616, 0.061224489795918366, 0.07389162561576373, 0.08095238095238096, 0.08294930875576037, 0.08482142857142858, 0.09307359307359307, 0.09663865546218488, 0.1020408163265306, 0.11111111111111124, 0.11969111969111969, 0.11842105263157894, 0.1227106227106227, 0.12321428571428572, 0.12543554006968638, 0.13095238095238093, 0.1378737541528239, 0.14123376623376624, 0.15079365079365079, 0.15527950310559005, 0.1565349544072948, 0.1607142857142857, 0.16472303206997085, 0.17142857142857143]}, "geom_softmax": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004464285714285714, 0.008403361344537815, 0.007936507936507934, 0.007518796992481203, 0.007142857142857143, 0.006802721088435373, 0.006493506493506494, 0.006211180124223602, 0.008928571428571426, 0.02, 0.03021978021978022, 0.042328042328042326, 0.04846938775510204, 0.0566502463054189, 0.06904761904761905, 0.07603686635944701, 0.08258928571428571, 0.08874458874458875, 0.09243697478991597, 0.09795918367346952, 0.10912698412698411, 0.10810810810810811, 0.11466165413533834, 0.11721611721611722, 0.12321428571428572, 0.12543554006968652, 0.13265306122448992, 0.1345514950166113, 0.13636363636363635, 0.1365079365079365, 0.14130434782608695, 0.14893617021276606, 0.1562500000000001, 0.16034985422740525, 0.17142857142857143]}}, "ltt": {"alpha": 0.1, "delta": 0.05, "lambda_hat": 0.11405049303851267, "coverage": 0.5271428571428571, "abstain_rate": 0.47285714285714286, "selective_risk": 0.02981029810298103, "selective_accuracy": 0.9701897018970189, "risk_within_budget": true}, "coverage_validity": {"alpha_grid": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "achieved_risk": [0.0, 0.0, 0.02981029810298103, 0.07623318385650224, 0.14660493827160495, 0.17142857142857143], "coverage": [0.0, 0.0, 0.5271428571428571, 0.6371428571428571, 0.9257142857142857, 1.0]}, "feature_diagnostics": {"names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual", "spectrum/lmin_mean", "spectrum/lmin_best", "spectrum/negfrac_mean", "spectrum/negfrac_best", "spectrum/effrank_mean", "spectrum/effrank_best", "spectrum/logdet_mean", "spectrum/logdet_best", "connect/barrier_mean", "connect/barrier_max", "connect/connected_frac"], "auroc": {"basin/rho": 0.6801724137931034, "basin/entropy": 0.3199640804597701, "basin/dispersion": 0.5014798850574713, "energy/mean": 0.5422413793103448, "energy/min": 0.541882183908046, "energy/std": 0.475617816091954, "curv/lmax_mean": 0.5129310344827587, "curv/lmax_best": 0.500926724137931, "curv/trace_mean": 0.5, "curv/trace_best": 0.5, "dynamics/steps": 0.5, "dynamics/monotonic": 0.5025215517241379, "dynamics/drop": 0.7858045977011494, "dynamics/residual": 0.5214655172413794, "spectrum/lmin_mean": 0.5, "spectrum/lmin_best": 0.5, "spectrum/negfrac_mean": 0.5, "spectrum/negfrac_best": 0.5, "spectrum/effrank_mean": 0.5, "spectrum/effrank_best": 0.5, "spectrum/logdet_mean": 0.5, "spectrum/logdet_best": 0.5, "connect/barrier_mean": 0.48670258620689655, "connect/barrier_max": 0.4867744252873563, "connect/connected_frac": 0.5132686781609196}, "hist": {"basin/rho": {"edges": [0.5, 0.525, 0.55, 0.575, 0.6, 0.625, 0.65, 0.675, 0.7, 0.725, 0.75, 0.775, 0.8, 0.825, 0.8500000000000001, 0.875, 0.9, 0.925, 0.95, 0.9750000000000001, 1.0], "correct_counts": [2, 0, 0, 1, 0, 0, 8, 0, 0, 0, 7, 0, 0, 37, 0, 0, 80, 0, 0, 445], "incorrect_counts": [1, 0, 0, 4, 0, 0, 4, 0, 0, 0, 6, 0, 0, 21, 0, 0, 33, 0, 0, 51]}, "basin/entropy": {"edges": [0.0, 0.041197960825054114, 0.08239592165010823, 0.12359388247516234, 0.16479184330021646, 0.20598980412527057, 0.24718776495032468, 0.2883857257753788, 0.3295836866004329, 0.370781647425487, 0.41197960825054114, 0.45317756907559525, 0.49437552990064937, 0.5355734907257035, 0.5767714515507576, 0.6179694123758117, 0.6591673732008658, 0.7003653340259199, 0.741563294850974, 0.7827612556760282, 0.8239592165010823], "correct_counts": [445, 0, 0, 0, 0, 0, 80, 0, 0, 0, 36, 0, 0, 8, 0, 8, 3, 0, 0, 0], "incorrect_counts": [51, 0, 0, 0, 0, 0, 33, 0, 0, 0, 21, 0, 0, 6, 0, 3, 5, 0, 0, 1]}, "basin/dispersion": {"edges": [1.9691264667768913, 2.1556350537882016, 2.342143640799512, 2.5286522278108228, 2.7151608148221333, 2.901669401833444, 3.0881779888447545, 3.274686575856065, 3.4611951628673756, 3.647703749878686, 3.8342123368899967, 4.020720923901307, 4.207229510912617, 4.3937380979239276, 4.580246684935238, 4.766755271946549, 4.953263858957859, 5.13977244596917, 5.32628103298048, 5.512789619991791, 5.6992982070031015], "correct_counts": [154, 409, 8, 0, 1, 0, 2, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 1], "incorrect_counts": [42, 67, 4, 0, 1, 0, 0, 2, 0, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0]}, "energy/mean": {"edges": [0.05226107438405355, 0.1975594937801361, 0.3428579131762186, 0.4881563325723011, 0.6334547519683837, 0.7787531713644663, 0.9240515907605488, 1.0693500101566313, 1.2146484295527138, 1.3599468489487962, 1.505245268344879, 1.6505436877409614, 1.7958421071370438, 1.9411405265331265, 2.0864389459292094, 2.2317373653252917, 2.3770357847213743, 2.522334204117457, 2.6676326235135392, 2.812931042909622, 2.9582294623057046], "correct_counts": [230, 333, 7, 2, 1, 1, 2, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1], "incorrect_counts": [60, 40, 3, 3, 4, 0, 0, 4, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1]}, "energy/min": {"edges": [-0.02320098876953125, 0.11365594863891601, 0.25051288604736327, 0.3873698234558105, 0.5242267608642578, 0.6610836982727051, 0.7979406356811523, 0.9347975730895995, 1.0716545104980468, 1.208511447906494, 1.3453683853149414, 1.4822253227233886, 1.6190822601318358, 1.7559391975402832, 1.8927961349487303, 2.0296530723571777, 2.166510009765625, 2.303366947174072, 2.4402238845825193, 2.577080821990967, 2.713937759399414], "correct_counts": [179, 378, 17, 0, 2, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0], "incorrect_counts": [46, 55, 2, 4, 3, 1, 0, 1, 3, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1]}, "energy/std": {"edges": [0.01770560819923182, 0.11276130864966308, 0.20781700910009432, 0.3028727095505256, 0.39792841000095686, 0.4929841104513881, 0.5880398109018193, 0.6830955113522507, 0.7781512118026819, 0.8732069122531131, 0.9682626127035444, 1.0633183131539756, 1.1583740136044067, 1.253429714054838, 1.3484854145052694, 1.4435411149557005, 1.5385968154061318, 1.6336525158565631, 1.7287082163069942, 1.8237639167574256, 1.9188196172078569], "correct_counts": [571, 0, 2, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1], "incorrect_counts": [114, 0, 1, 1, 0, 2, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "curv/lmax_mean": {"edges": [0.1999999905625979, 0.19999999174227318, 0.19999999292194842, 0.1999999941016237, 0.19999999528129894, 0.19999999646097422, 0.19999999764064946, 0.19999999882032474, 0.19999999999999998, 0.20000000117967526, 0.2000000023593505, 0.20000000353902578, 0.20000000471870105, 0.2000000058983763, 0.20000000707805157, 0.20000000825772682, 0.2000000094374021, 0.20000001061707734, 0.2000000117967526, 0.20000001297642786, 0.20000001415610313], "correct_counts": [1, 3, 7, 16, 20, 36, 45, 67, 70, 70, 62, 61, 40, 31, 28, 15, 4, 3, 0, 1], "incorrect_counts": [0, 2, 1, 1, 5, 8, 14, 6, 20, 17, 13, 11, 5, 8, 3, 2, 3, 0, 1, 0]}, "curv/lmax_best": {"edges": [0.19999995827674866, 0.19999996274709703, 0.19999996721744537, 0.19999997168779374, 0.19999997615814208, 0.19999998062849045, 0.19999998509883882, 0.19999998956918716, 0.19999999403953553, 0.19999999850988387, 0.20000000298023224, 0.2000000074505806, 0.20000001192092895, 0.20000001639127732, 0.20000002086162566, 0.20000002533197403, 0.2000000298023224, 0.20000003427267074, 0.2000000387430191, 0.20000004321336745, 0.20000004768371582], "correct_counts": [1, 0, 0, 15, 0, 0, 216, 0, 0, 0, 160, 0, 0, 179, 0, 0, 7, 0, 0, 2], "incorrect_counts": [0, 0, 0, 2, 0, 0, 46, 0, 0, 0, 35, 0, 0, 34, 0, 0, 3, 0, 0, 0]}, "curv/trace_mean": {"edges": [9.600000381469727, 9.650000381469727, 9.700000381469726, 9.750000381469727, 9.800000381469726, 9.850000381469727, 9.900000381469727, 9.950000381469726, 10.000000381469727, 10.050000381469726, 10.100000381469727, 10.150000381469727, 10.200000381469726, 10.250000381469727, 10.300000381469726, 10.350000381469727, 10.400000381469727, 10.450000381469726, 10.500000381469727, 10.550000381469726, 10.600000381469727], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "curv/trace_best": {"edges": [9.600000381469727, 9.650000381469727, 9.700000381469726, 9.750000381469727, 9.800000381469726, 9.850000381469727, 9.900000381469727, 9.950000381469726, 10.000000381469727, 10.050000381469726, 10.100000381469727, 10.150000381469727, 10.200000381469726, 10.250000381469727, 10.300000381469726, 10.350000381469727, 10.400000381469727, 10.450000381469726, 10.500000381469727, 10.550000381469726, 10.600000381469727], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/steps": {"edges": [1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7000000000000002, 1.75, 1.8, 1.85, 1.9, 1.9500000000000002, 2.0], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/monotonic": {"edges": [0.5375, 0.5408124999999999, 0.544125, 0.5474375, 0.55075, 0.5540624999999999, 0.557375, 0.5606875, 0.564, 0.5673124999999999, 0.5706249999999999, 0.5739375, 0.5772499999999999, 0.5805624999999999, 0.5838749999999999, 0.5871875, 0.5904999999999999, 0.5938124999999999, 0.5971249999999999, 0.6004375, 0.6037499999999999], "correct_counts": [2, 3, 15, 24, 59, 66, 78, 108, 91, 57, 44, 20, 9, 1, 2, 1, 0, 0, 0, 0], "incorrect_counts": [1, 4, 6, 0, 7, 18, 17, 26, 12, 10, 7, 1, 3, 2, 0, 1, 1, 1, 0, 3]}, "dynamics/drop": {"edges": [5.1263887484868365, 5.376814742883046, 5.627240737279256, 5.877666731675466, 6.128092726071675, 6.378518720467885, 6.628944714864096, 6.879370709260305, 7.129796703656515, 7.380222698052725, 7.6306486924489345, 7.881074686845144, 8.131500681241354, 8.381926675637564, 8.632352670033773, 8.882778664429983, 9.133204658826193, 9.383630653222403, 9.634056647618612, 9.884482642014822, 10.134908636411032], "correct_counts": [1, 0, 0, 2, 8, 27, 45, 52, 44, 40, 20, 13, 15, 53, 64, 68, 64, 31, 23, 10], "incorrect_counts": [0, 0, 0, 3, 2, 18, 17, 30, 18, 12, 10, 7, 1, 0, 0, 2, 0, 0, 0, 0]}, "dynamics/residual": {"edges": [0.2833298866947492, 0.2860603225727876, 0.28879075845082597, 0.2915211943288644, 0.2942516302069028, 0.2969820660849412, 0.2997125019629796, 0.302442937841018, 0.30517337371905645, 0.30790380959709485, 0.31063424547513324, 0.31336468135317164, 0.3160951172312101, 0.3188255531092485, 0.3215559889872869, 0.3242864248653253, 0.32701686074336367, 0.3297472966214021, 0.3324777324994405, 0.3352081683774789, 0.3379386042555173], "correct_counts": [2, 2, 3, 11, 16, 30, 39, 49, 59, 71, 63, 60, 54, 42, 34, 24, 10, 6, 3, 2], "incorrect_counts": [0, 1, 1, 2, 2, 6, 8, 16, 14, 15, 12, 7, 12, 10, 3, 4, 2, 2, 2, 1]}, "spectrum/lmin_mean": {"edges": [0.20000000298023224, 0.2500000029802322, 0.3000000029802322, 0.35000000298023226, 0.40000000298023225, 0.45000000298023224, 0.5000000029802323, 0.5500000029802323, 0.6000000029802323, 0.6500000029802322, 0.7000000029802322, 0.7500000029802323, 0.8000000029802323, 0.8500000029802323, 0.9000000029802323, 0.9500000029802322, 1.0000000029802323, 1.0500000029802323, 1.1000000029802321, 1.1500000029802324, 1.2000000029802322], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/lmin_best": {"edges": [0.20000000298023224, 0.2500000029802322, 0.3000000029802322, 0.35000000298023226, 0.40000000298023225, 0.45000000298023224, 0.5000000029802323, 0.5500000029802323, 0.6000000029802323, 0.6500000029802322, 0.7000000029802322, 0.7500000029802323, 0.8000000029802323, 0.8500000029802323, 0.9000000029802323, 0.9500000029802322, 1.0000000029802323, 1.0500000029802323, 1.1000000029802321, 1.1500000029802324, 1.2000000029802322], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/negfrac_mean": {"edges": [0.0, 0.05, 0.1, 0.15000000000000002, 0.2, 0.25, 0.30000000000000004, 0.35000000000000003, 0.4, 0.45, 0.5, 0.55, 0.6000000000000001, 0.65, 0.7000000000000001, 0.75, 0.8, 0.8500000000000001, 0.9, 0.9500000000000001, 1.0], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/negfrac_best": {"edges": [0.0, 0.05, 0.1, 0.15000000000000002, 0.2, 0.25, 0.30000000000000004, 0.35000000000000003, 0.4, 0.45, 0.5, 0.55, 0.6000000000000001, 0.65, 0.7000000000000001, 0.75, 0.8, 0.8500000000000001, 0.9, 0.9500000000000001, 1.0], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/effrank_mean": {"edges": [47.99999975000001, 48.049999750000005, 48.09999975000001, 48.149999750000006, 48.19999975000001, 48.24999975000001, 48.299999750000005, 48.34999975000001, 48.399999750000006, 48.44999975000001, 48.49999975000001, 48.549999750000005, 48.59999975000001, 48.649999750000006, 48.69999975000001, 48.74999975000001, 48.799999750000005, 48.84999975000001, 48.899999750000006, 48.94999975000001, 48.99999975000001], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/effrank_best": {"edges": [47.99999975000001, 48.049999750000005, 48.09999975000001, 48.149999750000006, 48.19999975000001, 48.24999975000001, 48.299999750000005, 48.34999975000001, 48.399999750000006, 48.44999975000001, 48.49999975000001, 48.549999750000005, 48.59999975000001, 48.649999750000006, 48.69999975000001, 48.74999975000001, 48.799999750000005, 48.84999975000001, 48.899999750000006, 48.94999975000001, 48.99999975000001], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/logdet_mean": {"edges": [-77.25301668158117, -77.20301668158118, -77.15301668158118, -77.10301668158117, -77.05301668158117, -77.00301668158117, -76.95301668158118, -76.90301668158118, -76.85301668158117, -76.80301668158117, -76.75301668158117, -76.70301668158118, -76.65301668158118, -76.60301668158117, -76.55301668158117, -76.50301668158117, -76.45301668158118, -76.40301668158118, -76.35301668158117, -76.30301668158117, -76.25301668158117], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/logdet_best": {"edges": [-77.25301668158119, -77.20301668158119, -77.15301668158119, -77.10301668158118, -77.05301668158118, -77.00301668158119, -76.95301668158119, -76.90301668158119, -76.85301668158118, -76.80301668158118, -76.75301668158119, -76.70301668158119, -76.65301668158119, -76.60301668158118, -76.55301668158118, -76.50301668158119, -76.45301668158119, -76.40301668158119, -76.35301668158118, -76.30301668158118, -76.25301668158119], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "connect/barrier_mean": {"edges": [0.0, 0.05873445651747964, 0.11746891303495928, 0.17620336955243893, 0.23493782606991856, 0.2936722825873982, 0.35240673910487785, 0.41114119562235746, 0.4698756521398371, 0.5286101086573167, 0.5873445651747964, 0.646079021692276, 0.7048134782097557, 0.7635479347272354, 0.8222823912447149, 0.8810168477621946, 0.9397513042796742, 0.9984857607971539, 1.0572202173146334, 1.115954673832113, 1.1746891303495928], "correct_counts": [577, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [115, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]}, "connect/barrier_max": {"edges": [0.0, 0.06394378542900085, 0.1278875708580017, 0.19183135628700254, 0.2557751417160034, 0.3197189271450043, 0.3836627125740051, 0.44760649800300595, 0.5115502834320068, 0.5754940688610076, 0.6394378542900085, 0.7033816397190094, 0.7673254251480102, 0.8312692105770111, 0.8952129960060119, 0.9591567814350128, 1.0231005668640136, 1.0870443522930144, 1.1509881377220152, 1.2149319231510163, 1.278875708580017], "correct_counts": [572, 0, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0], "incorrect_counts": [115, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1]}, "connect/connected_frac": {"edges": [0.0, 0.05, 0.1, 0.15000000000000002, 0.2, 0.25, 0.30000000000000004, 0.35000000000000003, 0.4, 0.45, 0.5, 0.55, 0.6000000000000001, 0.65, 0.7000000000000001, 0.75, 0.8, 0.8500000000000001, 0.9, 0.9500000000000001, 1.0], "correct_counts": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 3, 0, 4, 571], "incorrect_counts": [1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 2, 0, 0, 115]}}}, "feature_ablation": {"full": 0.0533799136280693, "drop_basin": 0.05512073495619045, "basin_only": 0.12163708444636095, "drop_energy": 0.0540551948406156, "energy_only": 0.21717465689606066, "drop_curv": 0.05368458197556656, "curv_only": 0.1688173605877835, "drop_dynamics": 0.12406978604857082, "dynamics_only": 0.05622078593396844, "drop_spectrum": 0.0533799136280693, "spectrum_only": 0.17142857142857193, "drop_connect": 0.05324936333102736, "connect_only": 0.1676576914051575}, "ece_geometry": 0.042492555746301765}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} {"run_id": "a405cf7b867a", "timestamp": "2026-07-28T23:19:30.542724+00:00", "git_sha": "9367d60", "config_hash": "23346bbdcf92", "config": {"run": {"seed": 4, "task": "graph_planning", "notes": "stronger IRED learned-landscape reasoner on graph"}, "model": {"latent_dim": 48, "hidden_dim": 256, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.01, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "annealed", "anneal_levels": 20, "anneal_steps_per_level": 10, "anneal_step_max": 0.5, "anneal_step_min": 0.003}, "train": {"epochs": 110, "batch_size": 128, "lr": 0.001, "n_train": 12000, "n_neg": 4, "neg_noise": 0.5, "objective": "ired", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.2, "ired_decode_weight": 4.0, "ired_stat_weight": 8.0}, "eval": {"n_eval": 700, "richer_geometry": true}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 800}, "task": {"graph_planning": {"n_nodes": 7, "ood_n_nodes": 10, "edge_prob": 0.4, "max_len": 4}}}, "task": "graph_planning", "split": "selective", "seed": 4, "metrics": {"n_fit": 700, "n_calib": 800, "n_test": 700, "k_restarts": 12, "objective": "ired", "sampler": "annealed", "feature_set": "richer", "accuracy_id": 0.8285714285714286, "base_error": 0.17142857142857143, "final_train_loss": 0.1806420087814331, "feature_names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual", "spectrum/lmin_mean", "spectrum/lmin_best", "spectrum/negfrac_mean", "spectrum/negfrac_best", "spectrum/effrank_mean", "spectrum/effrank_best", "spectrum/logdet_mean", "spectrum/logdet_best", "connect/barrier_mean", "connect/barrier_max", "connect/connected_frac"], "aurc": {"geometry": 0.0499999909178207, "rho_basin": 0.10276071740829948, "energy_min": 0.14142815403252237, "energy_mean": 0.13715414350531202, "energy_std": 0.16317105464147816, "msp": 0.09655790013556867, "temp_msp": 0.053993209301107216, "entropy": 0.09625911916651533, "softmax_learned": 0.053985762260104715, "geom_softmax": 0.04996327543640174}, "temperature": 7.166651351401238, "best_energy_baseline": "energy_mean", "best_baseline": "temp_msp", "delta_aurc_vs_energy_min": [0.09142816311470167, 0.05939135802345537, 0.12372412125974012], "delta_aurc_vs_best_energy": [0.08715415258749132, 0.05679752406767961, 0.1182631632770357], "delta_aurc_vs_best_baseline": [0.003993218383286513, -0.0029830289649739033, 0.010870978934784942], "delta_aurc_geom_adds": [0.004022486823702973, -0.00434027542554494, 0.012002982463869206], "geometry_wins": true, "geometry_wins_vs_baseline": false, "geometry_adds_over_softmax": false, "risk_coverage": {"geometry": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004201680672268907, 0.003968253968253967, 0.0037593984962406013, 0.0035714285714285713, 0.0034013605442176865, 0.006493506493506494, 0.009316770186335404, 0.008928571428571426, 0.02, 0.02197802197802198, 0.0291005291005291, 0.04081632653061224, 0.046798029556650425, 0.05476190476190476, 0.06451612903225806, 0.06696428571428571, 0.07575757575757576, 0.07983193277310924, 0.08367346938775509, 0.0853174603174603, 0.0945945945945946, 0.09962406015037593, 0.1043956043956044, 0.11428571428571428, 0.1219512195121951, 0.12585034013605453, 0.13122923588039867, 0.1396103896103896, 0.14603174603174604, 0.14906832298136646, 0.15501519756838916, 0.15624999999999997, 0.16472303206997085, 0.17142857142857143]}, "rho_basin": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.09110169491525424, 0.09110169491525429, 0.0911016949152543, 0.0911016949152543, 0.0911016949152543, 0.0911016949152543, 0.09110169491525431, 0.09110169491525431, 0.09110169491525431, 0.09110169491525431, 0.09110169491525431, 0.09110169491525431, 0.09110169491525424, 0.09110169491525413, 0.09110169491525402, 0.09110169491525393, 0.09110169491525386, 0.09110169491525377, 0.09110169491525372, 0.09110169491525366, 0.0911016949152536, 0.09110169491525355, 0.09110169491525351, 0.09110169491525347, 0.09110169491525344, 0.09110169491525341, 0.09110169491525338, 0.09110169491525334, 0.09110169491525331, 0.0911016949152533, 0.09110169491525327, 0.09110169491525325, 0.09110169491525323, 0.09250700280111943, 0.0972448979591827, 0.10171957671957575, 0.10595238095237998, 0.10996240601503664, 0.11376678876678782, 0.11738095238095152, 0.12081881533100979, 0.12409297052154146, 0.12922442433268372, 0.13490819525302244, 0.1403393541324572, 0.14553437566930785, 0.15012554513017012, 0.15424430641821898, 0.16107871720116568, 0.17142857142857088]}, "energy_min": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.35714285714285715, 0.21428571428571427, 0.16666666666666666, 0.17857142857142858, 0.14285714285714285, 0.13095238095238093, 0.12244897959183655, 0.125, 0.1111111111111111, 0.1214285714285713, 0.12987012987012986, 0.125, 0.12087912087912088, 0.11734693877551021, 0.11428571428571425, 0.11160714285714286, 0.12605042016806722, 0.12698412698412695, 0.12406015037593984, 0.12142857142857143, 0.11904761904761915, 0.12012987012987013, 0.12422360248447205, 0.12202380952380962, 0.12285714285714286, 0.12912087912087913, 0.12962962962962962, 0.125, 0.12807881773399013, 0.1261904761904762, 0.12211981566820276, 0.13169642857142858, 0.13203463203463203, 0.13655462184873948, 0.14081632653061238, 0.14285714285714282, 0.14092664092664092, 0.15225563909774437, 0.15567765567765568, 0.15178571428571427, 0.15679442508710797, 0.15476190476190474, 0.15282392026578073, 0.1525974025974026, 0.1523809523809524, 0.15062111801242237, 0.1519756838905776, 0.1577380952380952, 0.16326530612244897, 0.17142857142857143]}, "energy_mean": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.21428571428571427, 0.14285714285714285, 0.14285714285714285, 0.14285714285714285, 0.14285714285714285, 0.14285714285714282, 0.14285714285714288, 0.13392857142857142, 0.1349206349206349, 0.1285714285714286, 0.12337662337662338, 0.11904761904761904, 0.11538461538461539, 0.11734693877551021, 0.11428571428571425, 0.10714285714285714, 0.1092436974789916, 0.11904761904761903, 0.12406015037593984, 0.12142857142857143, 0.1258503401360544, 0.1266233766233766, 0.13043478260869565, 0.12797619047619044, 0.12285714285714286, 0.12362637362637363, 0.12433862433862433, 0.11989795918367346, 0.12561576354679801, 0.1261904761904762, 0.12903225806451613, 0.12946428571428573, 0.1277056277056277, 0.13025210084033614, 0.13265306122448978, 0.13690476190476206, 0.14285714285714285, 0.14285714285714285, 0.14285714285714285, 0.14642857142857144, 0.14634146341463425, 0.1462585034013605, 0.14950166112956811, 0.15097402597402598, 0.15396825396825398, 0.15217391304347827, 0.15501519756838902, 0.1577380952380952, 0.16326530612244897, 0.17142857142857143]}, "energy_std": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.2857142857142857, 0.21428571428571427, 0.23809523809523808, 0.21428571428571427, 0.18571428571428572, 0.17857142857142855, 0.15306122448979595, 0.16071428571428573, 0.16666666666666666, 0.15714285714285717, 0.16233766233766234, 0.17261904761904762, 0.17582417582417584, 0.17346938775510204, 0.16666666666666663, 0.16071428571428573, 0.15966386554621848, 0.15079365079365076, 0.15413533834586465, 0.15, 0.1462585034013605, 0.14285714285714285, 0.14906832298136646, 0.15476190476190474, 0.15142857142857144, 0.1510989010989011, 0.15079365079365079, 0.14795918367346939, 0.14532019704433494, 0.14761904761904762, 0.15668202764976957, 0.15848214285714285, 0.16017316017316016, 0.15966386554621848, 0.15918367346938772, 0.1607142857142857, 0.16216216216216217, 0.16917293233082706, 0.17032967032967034, 0.16607142857142856, 0.16376306620209055, 0.16156462585034012, 0.16279069767441862, 0.16233766233766234, 0.16031746031746033, 0.16770186335403728, 0.16413373860182368, 0.16369047619047616, 0.1661807580174927, 0.17142857142857143]}, "msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.07142857142857142, 0.07142857142857142, 0.07142857142857142, 0.05357142857142857, 0.04285714285714286, 0.04761904761904773, 0.07142857142857144, 0.07142857142857142, 0.07142857142857142, 0.07142857142857144, 0.06493506493506493, 0.07142857142857142, 0.08241758241758242, 0.08163265306122448, 0.07619047619047636, 0.08482142857142858, 0.08823529411764706, 0.08333333333333331, 0.08270676691729323, 0.08214285714285714, 0.08163265306122447, 0.07792207792207792, 0.08074534161490683, 0.08333333333333331, 0.08285714285714285, 0.08791208791208792, 0.09788359788359788, 0.09438775510204081, 0.09852216748768472, 0.09523809523809523, 0.0944700460829493, 0.09598214285714286, 0.09956709956709957, 0.10294117647058823, 0.11020408163265305, 0.11507936507936506, 0.11776061776061776, 0.11654135338345864, 0.11904761904761904, 0.11964285714285715, 0.12543554006968652, 0.12925170068027222, 0.1378737541528239, 0.13636363636363635, 0.13968253968253969, 0.14285714285714285, 0.1458966565349545, 0.1532738095238095, 0.16326530612244897, 0.17142857142857143]}, "temp_msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0037593984962406013, 0.0035714285714285713, 0.006802721088435373, 0.006493506493506494, 0.006211180124223602, 0.008928571428571426, 0.008571428571428572, 0.016483516483516484, 0.023809523809523808, 0.04846938775510204, 0.0566502463054189, 0.06904761904761905, 0.07834101382488479, 0.08705357142857142, 0.09740259740259741, 0.09873949579831932, 0.10816326530612244, 0.10912698412698411, 0.11583011583011583, 0.12030075187969924, 0.12087912087912088, 0.12142857142857143, 0.12717770034843204, 0.13435374149659862, 0.13953488372093023, 0.14123376623376624, 0.13968253968253969, 0.14596273291925466, 0.14893617021276592, 0.15624999999999997, 0.1618075801749271, 0.17142857142857143]}, "entropy": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.07142857142857142, 0.07142857142857142, 0.07142857142857142, 0.05357142857142857, 0.04285714285714286, 0.04761904761904773, 0.07142857142857144, 0.07142857142857142, 0.07142857142857142, 0.07142857142857144, 0.06493506493506493, 0.07142857142857142, 0.08241758241758242, 0.08163265306122448, 0.07619047619047618, 0.08482142857142858, 0.08823529411764706, 0.08333333333333331, 0.08270676691729323, 0.08214285714285714, 0.08163265306122447, 0.07792207792207792, 0.08074534161490683, 0.08333333333333331, 0.08285714285714285, 0.08791208791208792, 0.09788359788359788, 0.09438775510204081, 0.09852216748768472, 0.09523809523809523, 0.0944700460829493, 0.09598214285714286, 0.09956709956709957, 0.10084033613445378, 0.1081632653061226, 0.11507936507936506, 0.11776061776061776, 0.11654135338345864, 0.11904761904761904, 0.11964285714285715, 0.12543554006968652, 0.12925170068027222, 0.1378737541528239, 0.137987012987013, 0.13968253968253969, 0.14285714285714285, 0.1458966565349545, 0.15327380952380965, 0.16326530612244897, 0.17142857142857143]}, "softmax_learned": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0037593984962406013, 0.0035714285714285713, 0.006802721088435373, 0.006493506493506494, 0.006211180124223602, 0.008928571428571426, 0.008571428571428572, 0.016483516483516484, 0.023809523809523808, 0.04846938775510204, 0.0566502463054189, 0.06904761904761905, 0.07834101382488479, 0.08705357142857142, 0.09740259740259741, 0.09873949579831932, 0.10816326530612244, 0.10912698412698411, 0.11583011583011583, 0.12030075187969924, 0.12087912087912088, 0.12142857142857143, 0.12717770034843204, 0.13435374149659862, 0.13953488372093023, 0.14123376623376624, 0.13968253968253969, 0.14596273291925466, 0.1474164133738603, 0.15624999999999997, 0.16326530612244897, 0.17142857142857143]}, "geom_softmax": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0047619047619047615, 0.008928571428571428, 0.008403361344537815, 0.007936507936507934, 0.007518796992481203, 0.010714285714285714, 0.010204081632653059, 0.012987012987012988, 0.015527950310559006, 0.014880952380952378, 0.02, 0.024725274725274724, 0.03439153439153439, 0.04336734693877551, 0.04679802955665024, 0.05238095238095238, 0.055299539170506916, 0.06696428571428571, 0.06926406926406926, 0.07352941176470588, 0.07959183673469386, 0.0853174603174603, 0.08687258687258688, 0.09398496240601503, 0.10073260073260074, 0.10892857142857143, 0.11846689895470382, 0.12074829931972787, 0.12458471760797342, 0.13474025974025974, 0.14126984126984127, 0.14751552795031056, 0.1534954407294834, 0.15922619047619044, 0.16472303206997085, 0.17142857142857143]}}, "ltt": {"alpha": 0.1, "delta": 0.05, "lambda_hat": 0.16840793323263656, "coverage": 0.5757142857142857, "abstain_rate": 0.42428571428571427, "selective_risk": 0.04714640198511166, "selective_accuracy": 0.9528535980148883, "risk_within_budget": true}, "coverage_validity": {"alpha_grid": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "achieved_risk": [0.0, 0.0, 0.04714640198511166, 0.0706401766004415, 0.14664586583463338, 0.17167381974248927], "coverage": [0.0, 0.0, 0.5757142857142857, 0.6471428571428571, 0.9157142857142857, 0.9985714285714286]}, "feature_diagnostics": {"names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual", "spectrum/lmin_mean", "spectrum/lmin_best", "spectrum/negfrac_mean", "spectrum/negfrac_best", "spectrum/effrank_mean", "spectrum/effrank_best", "spectrum/logdet_mean", "spectrum/logdet_best", "connect/barrier_mean", "connect/barrier_max", "connect/connected_frac"], "auroc": {"basin/rho": 0.7098060344827586, "basin/entropy": 0.2908979885057471, "basin/dispersion": 0.4298419540229885, "energy/mean": 0.3820402298850575, "energy/min": 0.39098419540229884, "energy/std": 0.4610488505747126, "curv/lmax_mean": 0.47841954022988503, "curv/lmax_best": 0.484683908045977, "curv/trace_mean": 0.5, "curv/trace_best": 0.5, "dynamics/steps": 0.5, "dynamics/monotonic": 0.5555316091954023, "dynamics/drop": 0.7991379310344827, "dynamics/residual": 0.4871551724137931, "spectrum/lmin_mean": 0.5, "spectrum/lmin_best": 0.5, "spectrum/negfrac_mean": 0.5, "spectrum/negfrac_best": 0.5, "spectrum/effrank_mean": 0.5, "spectrum/effrank_best": 0.5, "spectrum/logdet_mean": 0.5, "spectrum/logdet_best": 0.5, "connect/barrier_mean": 0.47932471264367815, "connect/barrier_max": 0.479382183908046, "connect/connected_frac": 0.5207183908045977}, "hist": {"basin/rho": {"edges": [0.5, 0.525, 0.55, 0.575, 0.6, 0.625, 0.65, 0.675, 0.7, 0.725, 0.75, 0.775, 0.8, 0.825, 0.8500000000000001, 0.875, 0.9, 0.925, 0.95, 0.9750000000000001, 1.0], "correct_counts": [0, 0, 0, 2, 0, 0, 9, 0, 0, 0, 15, 0, 0, 36, 0, 0, 89, 0, 0, 429], "incorrect_counts": [2, 0, 0, 5, 0, 0, 9, 0, 0, 0, 8, 0, 0, 22, 0, 0, 31, 0, 0, 43]}, "basin/entropy": {"edges": [0.0, 0.03607318433462558, 0.07214636866925116, 0.10821955300387673, 0.1442927373385023, 0.1803659216731279, 0.21643910600775346, 0.25251229034237904, 0.2885854746770046, 0.3246586590116302, 0.3607318433462558, 0.3968050276808814, 0.4328782120155069, 0.4689513963501325, 0.5050245806847581, 0.5410977650193837, 0.5771709493540093, 0.6132441336886348, 0.6493173180232604, 0.685390502357886, 0.7214636866925116], "correct_counts": [429, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 35, 0, 0, 13, 0, 9, 2, 3], "incorrect_counts": [43, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 21, 0, 0, 9, 0, 9, 5, 2]}, "basin/dispersion": {"edges": [2.006589668932686, 2.1440741312539653, 2.281558593575245, 2.419043055896524, 2.5565275182178033, 2.6940119805390825, 2.831496442860362, 2.9689809051816414, 3.1064653675029206, 3.2439498298242, 3.3814342921454794, 3.5189187544667586, 3.6564032167880383, 3.7938876791093175, 3.9313721414305967, 4.068856603751876, 4.206341066073156, 4.343825528394435, 4.481309990715714, 4.618794453036994, 4.756278915358273], "correct_counts": [141, 392, 38, 0, 1, 0, 0, 2, 2, 2, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0], "incorrect_counts": [23, 68, 21, 0, 0, 0, 1, 0, 3, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 2]}, "energy/mean": {"edges": [-1.6460671226183574, -1.4175087143977483, -1.1889503061771394, -0.9603918979565302, -0.7318334897359212, -0.5032750815153122, -0.27471667329470306, -0.04615826507409415, 0.18240014314651498, 0.4109585513671241, 0.639516959587733, 0.8680753678083419, 1.0966337760289513, 1.3251921842495602, 1.5537505924701691, 1.7823090006907785, 2.0108674089113876, 2.2394258171319965, 2.4679842253526054, 2.6965426335732143, 2.9251010417938232], "correct_counts": [1, 0, 0, 0, 0, 0, 0, 439, 127, 5, 0, 3, 0, 1, 1, 0, 2, 1, 0, 0], "incorrect_counts": [0, 0, 0, 0, 0, 0, 1, 72, 33, 2, 1, 3, 1, 1, 0, 1, 1, 2, 0, 2]}, "energy/min": {"edges": [-2.360767364501953, -2.1069081664085387, -1.8530489683151246, -1.5991897702217104, -1.345330572128296, -1.0914713740348816, -0.8376121759414674, -0.5837529778480532, -0.32989377975463885, -0.07603458166122445, 0.17782461643218994, 0.4316838145256039, 0.6855430126190183, 0.9394022107124327, 1.1932614088058466, 1.447120606899261, 1.7009798049926754, 1.9548390030860894, 2.208698201179504, 2.462557399272918, 2.716416597366333], "correct_counts": [1, 0, 0, 0, 0, 0, 0, 0, 0, 553, 15, 4, 2, 1, 0, 2, 1, 1, 0, 0], "incorrect_counts": [0, 0, 0, 0, 0, 0, 0, 0, 1, 97, 9, 3, 2, 2, 1, 1, 1, 1, 0, 2]}, "energy/std": {"edges": [0.019892553899172424, 0.06891081471924504, 0.11792907553931765, 0.1669473363593903, 0.2159655971794629, 0.2649838579995355, 0.31400211881960816, 0.36302037963968076, 0.41203864045975336, 0.46105690127982596, 0.5100751620998986, 0.5590934229199712, 0.6081116837400439, 0.6571299445601164, 0.7061482053801891, 0.7551664662002616, 0.8041847270203343, 0.8532029878404069, 0.9022212486604795, 0.9512395094805521, 1.0002577703006248], "correct_counts": [560, 11, 1, 0, 2, 0, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "incorrect_counts": [110, 4, 1, 2, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0]}, "curv/lmax_mean": {"edges": [0.1999999893208345, 0.19999999050050976, 0.199999991680185, 0.19999999285986025, 0.19999999403953553, 0.1999999952192108, 0.19999999639888605, 0.1999999975785613, 0.19999999875823657, 0.19999999993791184, 0.2000000011175871, 0.20000000229726234, 0.2000000034769376, 0.20000000465661288, 0.20000000583628813, 0.20000000701596338, 0.20000000819563865, 0.20000000937531393, 0.20000001055498917, 0.20000001173466442, 0.2000000129143397], "correct_counts": [1, 3, 1, 7, 13, 20, 28, 51, 65, 66, 76, 76, 55, 44, 30, 22, 14, 5, 1, 2], "incorrect_counts": [0, 0, 1, 1, 1, 6, 1, 9, 13, 17, 20, 13, 9, 14, 10, 4, 1, 0, 0, 0]}, "curv/lmax_best": {"edges": [0.19999995827674866, 0.19999996274709703, 0.19999996721744537, 0.19999997168779374, 0.19999997615814208, 0.19999998062849045, 0.19999998509883882, 0.19999998956918716, 0.19999999403953553, 0.19999999850988387, 0.20000000298023224, 0.2000000074505806, 0.20000001192092895, 0.20000001639127732, 0.20000002086162566, 0.20000002533197403, 0.2000000298023224, 0.20000003427267074, 0.2000000387430191, 0.20000004321336745, 0.20000004768371582], "correct_counts": [2, 0, 0, 8, 0, 0, 233, 0, 0, 0, 139, 0, 0, 185, 0, 0, 12, 0, 0, 1], "incorrect_counts": [0, 0, 0, 2, 0, 0, 44, 0, 0, 0, 31, 0, 0, 41, 0, 0, 2, 0, 0, 0]}, "curv/trace_mean": {"edges": [9.600000381469727, 9.650000381469727, 9.700000381469726, 9.750000381469727, 9.800000381469726, 9.850000381469727, 9.900000381469727, 9.950000381469726, 10.000000381469727, 10.050000381469726, 10.100000381469727, 10.150000381469727, 10.200000381469726, 10.250000381469727, 10.300000381469726, 10.350000381469727, 10.400000381469727, 10.450000381469726, 10.500000381469727, 10.550000381469726, 10.600000381469727], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "curv/trace_best": {"edges": [9.600000381469727, 9.650000381469727, 9.700000381469726, 9.750000381469727, 9.800000381469726, 9.850000381469727, 9.900000381469727, 9.950000381469726, 10.000000381469727, 10.050000381469726, 10.100000381469727, 10.150000381469727, 10.200000381469726, 10.250000381469727, 10.300000381469726, 10.350000381469727, 10.400000381469727, 10.450000381469726, 10.500000381469727, 10.550000381469726, 10.600000381469727], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/steps": {"edges": [1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7000000000000002, 1.75, 1.8, 1.85, 1.9, 1.9500000000000002, 2.0], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/monotonic": {"edges": [0.5308333333333334, 0.5355000000000001, 0.5401666666666667, 0.5448333333333334, 0.5495, 0.5541666666666667, 0.5588333333333334, 0.5635, 0.5681666666666667, 0.5728333333333333, 0.5775, 0.5821666666666667, 0.5868333333333333, 0.5915, 0.5961666666666666, 0.6008333333333333, 0.6055, 0.6101666666666666, 0.6148333333333333, 0.6194999999999999, 0.6241666666666666], "correct_counts": [0, 2, 5, 20, 61, 102, 134, 108, 76, 42, 22, 4, 1, 1, 0, 0, 1, 0, 0, 1], "incorrect_counts": [1, 0, 0, 7, 17, 24, 27, 22, 8, 10, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/drop": {"edges": [5.520331422487895, 5.784909941752752, 6.049488461017609, 6.314066980282466, 6.5786454995473225, 6.84322401881218, 7.107802538077037, 7.372381057341894, 7.636959576606751, 7.901538095871608, 8.166116615136465, 8.430695134401322, 8.69527365366618, 8.959852172931036, 9.224430692195892, 9.48900921146075, 9.753587730725606, 10.018166249990465, 10.28274476925532, 10.547323288520179, 10.811901807785034], "correct_counts": [0, 4, 2, 16, 36, 51, 49, 40, 18, 16, 15, 33, 58, 61, 73, 53, 29, 14, 7, 5], "incorrect_counts": [2, 1, 3, 8, 16, 29, 27, 11, 10, 8, 2, 2, 0, 1, 0, 0, 0, 0, 0, 0]}, "dynamics/residual": {"edges": [0.28685005381703377, 0.29140506281207007, 0.2959600718071063, 0.3005150808021426, 0.3050700897971789, 0.30962509879221517, 0.3141801077872515, 0.3187351167822878, 0.323290125777324, 0.3278451347723603, 0.33240014376739657, 0.3369551527624329, 0.3415101617574692, 0.3460651707525054, 0.3506201797475417, 0.35517518874257803, 0.3597301977376143, 0.3642852067326506, 0.3688402157276869, 0.37339522472272313, 0.37795023371775943], "correct_counts": [10, 24, 45, 83, 110, 108, 99, 60, 31, 2, 5, 0, 2, 0, 0, 0, 0, 0, 0, 1], "incorrect_counts": [0, 4, 13, 17, 22, 21, 17, 14, 8, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/lmin_mean": {"edges": [0.20000000298023224, 0.2500000029802322, 0.3000000029802322, 0.35000000298023226, 0.40000000298023225, 0.45000000298023224, 0.5000000029802323, 0.5500000029802323, 0.6000000029802323, 0.6500000029802322, 0.7000000029802322, 0.7500000029802323, 0.8000000029802323, 0.8500000029802323, 0.9000000029802323, 0.9500000029802322, 1.0000000029802323, 1.0500000029802323, 1.1000000029802321, 1.1500000029802324, 1.2000000029802322], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/lmin_best": {"edges": [0.20000000298023224, 0.2500000029802322, 0.3000000029802322, 0.35000000298023226, 0.40000000298023225, 0.45000000298023224, 0.5000000029802323, 0.5500000029802323, 0.6000000029802323, 0.6500000029802322, 0.7000000029802322, 0.7500000029802323, 0.8000000029802323, 0.8500000029802323, 0.9000000029802323, 0.9500000029802322, 1.0000000029802323, 1.0500000029802323, 1.1000000029802321, 1.1500000029802324, 1.2000000029802322], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/negfrac_mean": {"edges": [0.0, 0.05, 0.1, 0.15000000000000002, 0.2, 0.25, 0.30000000000000004, 0.35000000000000003, 0.4, 0.45, 0.5, 0.55, 0.6000000000000001, 0.65, 0.7000000000000001, 0.75, 0.8, 0.8500000000000001, 0.9, 0.9500000000000001, 1.0], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/negfrac_best": {"edges": [0.0, 0.05, 0.1, 0.15000000000000002, 0.2, 0.25, 0.30000000000000004, 0.35000000000000003, 0.4, 0.45, 0.5, 0.55, 0.6000000000000001, 0.65, 0.7000000000000001, 0.75, 0.8, 0.8500000000000001, 0.9, 0.9500000000000001, 1.0], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/effrank_mean": {"edges": [47.99999975000001, 48.049999750000005, 48.09999975000001, 48.149999750000006, 48.19999975000001, 48.24999975000001, 48.299999750000005, 48.34999975000001, 48.399999750000006, 48.44999975000001, 48.49999975000001, 48.549999750000005, 48.59999975000001, 48.649999750000006, 48.69999975000001, 48.74999975000001, 48.799999750000005, 48.84999975000001, 48.899999750000006, 48.94999975000001, 48.99999975000001], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/effrank_best": {"edges": [47.99999975000001, 48.049999750000005, 48.09999975000001, 48.149999750000006, 48.19999975000001, 48.24999975000001, 48.299999750000005, 48.34999975000001, 48.399999750000006, 48.44999975000001, 48.49999975000001, 48.549999750000005, 48.59999975000001, 48.649999750000006, 48.69999975000001, 48.74999975000001, 48.799999750000005, 48.84999975000001, 48.899999750000006, 48.94999975000001, 48.99999975000001], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/logdet_mean": {"edges": [-77.25301668158117, -77.20301668158118, -77.15301668158118, -77.10301668158117, -77.05301668158117, -77.00301668158117, -76.95301668158118, -76.90301668158118, -76.85301668158117, -76.80301668158117, -76.75301668158117, -76.70301668158118, -76.65301668158118, -76.60301668158117, -76.55301668158117, -76.50301668158117, -76.45301668158118, -76.40301668158118, -76.35301668158117, -76.30301668158117, -76.25301668158117], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "spectrum/logdet_best": {"edges": [-77.25301668158119, -77.20301668158119, -77.15301668158119, -77.10301668158118, -77.05301668158118, -77.00301668158119, -76.95301668158119, -76.90301668158119, -76.85301668158118, -76.80301668158118, -76.75301668158119, -76.70301668158119, -76.65301668158119, -76.60301668158118, -76.55301668158118, -76.50301668158119, -76.45301668158119, -76.40301668158119, -76.35301668158118, -76.30301668158118, -76.25301668158119], "correct_counts": [580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "connect/barrier_mean": {"edges": [0.0, 0.06370989842848344, 0.1274197968569669, 0.19112969528545032, 0.2548395937139338, 0.31854949214241723, 0.38225939057090064, 0.4459692889993841, 0.5096791874278676, 0.573389085856351, 0.6370989842848345, 0.7008088827133179, 0.7645187811418013, 0.8282286795702848, 0.8919385779987682, 0.9556484764272517, 1.019358374855735, 1.0830682732842185, 1.146778171712702, 1.2104880701411855, 1.274197968569669], "correct_counts": [578, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], "incorrect_counts": [118, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "connect/barrier_max": {"edges": [0.0, 0.06671607494354248, 0.13343214988708496, 0.20014822483062744, 0.2668642997741699, 0.3335803747177124, 0.4002964496612549, 0.46701252460479736, 0.5337285995483398, 0.6004446744918823, 0.6671607494354248, 0.7338768243789673, 0.8005928993225098, 0.8673089742660522, 0.9340250492095947, 1.0007411241531372, 1.0674571990966797, 1.1341732740402222, 1.2008893489837646, 1.2676054239273071, 1.3343214988708496], "correct_counts": [576, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1], "incorrect_counts": [115, 1, 0, 1, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "connect/connected_frac": {"edges": [0.0, 0.05, 0.1, 0.15000000000000002, 0.2, 0.25, 0.30000000000000004, 0.35000000000000003, 0.4, 0.45, 0.5, 0.55, 0.6000000000000001, 0.65, 0.7000000000000001, 0.75, 0.8, 0.8500000000000001, 0.9, 0.9500000000000001, 1.0], "correct_counts": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 575], "incorrect_counts": [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 2, 114]}}}, "feature_ablation": {"full": 0.0499999909178207, "drop_basin": 0.05322165715841172, "basin_only": 0.10375895158755924, "drop_energy": 0.049739566895181775, "energy_only": 0.13920001795584305, "drop_curv": 0.05021424448265369, "curv_only": 0.17253571329725742, "drop_dynamics": 0.09321471121667474, "dynamics_only": 0.05282089227280329, "drop_spectrum": 0.0499999909178207, "spectrum_only": 0.17142857142857193, "drop_connect": 0.049927555157745766, "connect_only": 0.16551065695700648}, "ece_geometry": 0.040849909596787144}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "a6c36d85615f", "timestamp": "2026-07-30T11:17:03.568524+00:00", "git_sha": "67dc7a2", "config_hash": "f7c0c557ea97", "config": {"run": {"seed": 0, "task": "arithmetic", "notes": "F4 adaptive-halting run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 7, "batch_size": 128, "lr": 0.001, "n_train": 6000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 800, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 800}, "task": {"arithmetic": {"modular": false, "n_operands": 6, "ood_n_operands": 6, "max_operand": 7, "ood_max_operand": 9}}}, "task": "arithmetic", "split": "halting", "seed": 0, "metrics": {"k_restarts": 12, "steps": 50, "alpha": 0.1, "n_calib": 800, "n_test": 800, "final_train_loss": 1.010847568511963, "tau_hat": 0.9167, "compute_used": 0.42152500000000004, "compute_saved": 0.578475, "halting_risk": 0.00875, "risk_within_budget": true, "full_accuracy": 0.80375, "halted_accuracy": 0.8075, "accuracy_drop": -0.003750000000000031, "risk_monotone_in_tau": true, "calib_risk_by_tau": [0.9025, 0.845, 0.845, 0.71, 0.48375, 0.48375, 0.37625, 0.19, 0.19, 0.13875, 0.00875, 0.00875], "calib_compute_by_tau": [0.0, 0.013049999999999886, 0.013049999999999886, 0.03759999999999957, 0.09074999999999989, 0.09074999999999989, 0.1236749999999998, 0.20574999999999993, 0.20574999999999993, 0.2681749999999995, 0.4182, 0.4182], "tau_sweep": {"taus": [0.0833, 0.1667, 0.25, 0.3333, 0.4167, 0.5, 0.5833, 0.6667, 0.75, 0.8333, 0.9167, 1.0], "compute_used": [0.0, 0.013375000000000001, 0.013375000000000001, 0.0383, 0.09365000000000002, 0.09365000000000002, 0.12260000000000001, 0.20592500000000002, 0.20592500000000002, 0.25880000000000003, 0.42152500000000004, 0.42152500000000004], "accuracy": [0.0875, 0.14375, 0.14375, 0.2375, 0.455, 0.455, 0.56, 0.69125, 0.69125, 0.7425, 0.8075, 0.8075], "disagreement": [0.91625, 0.85625, 0.85625, 0.7425, 0.4975, 0.4975, 0.37375, 0.20875, 0.20875, 0.14875, 0.00875, 0.00875]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "5b496a841123", "timestamp": "2026-07-30T11:17:05.368848+00:00", "git_sha": "67dc7a2", "config_hash": "b6cd6bcd6e19", "config": {"run": {"seed": 1, "task": "arithmetic", "notes": "F4 adaptive-halting run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 7, "batch_size": 128, "lr": 0.001, "n_train": 6000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 800, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 800}, "task": {"arithmetic": {"modular": false, "n_operands": 6, "ood_n_operands": 6, "max_operand": 7, "ood_max_operand": 9}}}, "task": "arithmetic", "split": "halting", "seed": 1, "metrics": {"k_restarts": 12, "steps": 50, "alpha": 0.1, "n_calib": 800, "n_test": 800, "final_train_loss": 0.9732173085212708, "tau_hat": 0.9167, "compute_used": 0.45080000000000003, "compute_saved": 0.5491999999999999, "halting_risk": 0.0175, "risk_within_budget": true, "full_accuracy": 0.80375, "halted_accuracy": 0.8175, "accuracy_drop": -0.01375000000000004, "risk_monotone_in_tau": true, "calib_risk_by_tau": [0.88625, 0.84625, 0.84625, 0.73375, 0.465, 0.465, 0.35125, 0.21875, 0.21875, 0.12625, 0.015, 0.015], "calib_compute_by_tau": [0.0, 0.009049999999999954, 0.009049999999999954, 0.03192499999999969, 0.09209999999999983, 0.09209999999999983, 0.12292499999999983, 0.20739999999999958, 0.20739999999999958, 0.27014999999999956, 0.43754999999999983, 0.43754999999999983], "tau_sweep": {"taus": [0.0833, 0.1667, 0.25, 0.3333, 0.4167, 0.5, 0.5833, 0.6667, 0.75, 0.8333, 0.9167, 1.0], "compute_used": [0.0, 0.010074999999999999, 0.010074999999999999, 0.0333, 0.09310000000000002, 0.09310000000000002, 0.12170000000000002, 0.21402500000000002, 0.21402500000000002, 0.27865, 0.45080000000000003, 0.45080000000000003], "accuracy": [0.095, 0.135, 0.135, 0.235, 0.4575, 0.4575, 0.5575, 0.7175, 0.7175, 0.76625, 0.8175, 0.8175], "disagreement": [0.9025, 0.8575, 0.8575, 0.74625, 0.50375, 0.50375, 0.38875, 0.2125, 0.2125, 0.1475, 0.0175, 0.0175]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "24d91758f427", "timestamp": "2026-07-30T11:17:07.226291+00:00", "git_sha": "67dc7a2", "config_hash": "4b65b46d70e6", "config": {"run": {"seed": 2, "task": "arithmetic", "notes": "F4 adaptive-halting run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 7, "batch_size": 128, "lr": 0.001, "n_train": 6000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 800, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 800}, "task": {"arithmetic": {"modular": false, "n_operands": 6, "ood_n_operands": 6, "max_operand": 7, "ood_max_operand": 9}}}, "task": "arithmetic", "split": "halting", "seed": 2, "metrics": {"k_restarts": 12, "steps": 50, "alpha": 0.1, "n_calib": 800, "n_test": 800, "final_train_loss": 0.9444285035133362, "tau_hat": 0.9167, "compute_used": 0.417825, "compute_saved": 0.582175, "halting_risk": 0.0175, "risk_within_budget": true, "full_accuracy": 0.86125, "halted_accuracy": 0.875, "accuracy_drop": -0.01375000000000004, "risk_monotone_in_tau": true, "calib_risk_by_tau": [0.91375, 0.8575, 0.8575, 0.71875, 0.4675, 0.4675, 0.35125, 0.20625, 0.20625, 0.13625, 0.01375, 0.01375], "calib_compute_by_tau": [0.0, 0.015349999999999841, 0.015349999999999841, 0.042149999999999584, 0.09782499999999984, 0.09782499999999984, 0.13077499999999975, 0.2109499999999998, 0.2109499999999998, 0.27007499999999957, 0.41997499999999993, 0.41997499999999993], "tau_sweep": {"taus": [0.0833, 0.1667, 0.25, 0.3333, 0.4167, 0.5, 0.5833, 0.6667, 0.75, 0.8333, 0.9167, 1.0], "compute_used": [0.0, 0.0146, 0.0146, 0.03995, 0.097375, 0.097375, 0.12955000000000003, 0.20885, 0.20885, 0.266425, 0.417825, 0.417825], "accuracy": [0.0775, 0.13875, 0.13875, 0.27375, 0.5125, 0.5125, 0.6025, 0.73, 0.73, 0.78, 0.875, 0.875], "disagreement": [0.9225, 0.86625, 0.86625, 0.7275, 0.45, 0.45, 0.3525, 0.23625, 0.23625, 0.1675, 0.0175, 0.0175]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "19cb5e9f64c6", "timestamp": "2026-07-30T11:17:08.943784+00:00", "git_sha": "67dc7a2", "config_hash": "e5e215c688c3", "config": {"run": {"seed": 3, "task": "arithmetic", "notes": "F4 adaptive-halting run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 7, "batch_size": 128, "lr": 0.001, "n_train": 6000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 800, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 800}, "task": {"arithmetic": {"modular": false, "n_operands": 6, "ood_n_operands": 6, "max_operand": 7, "ood_max_operand": 9}}}, "task": "arithmetic", "split": "halting", "seed": 3, "metrics": {"k_restarts": 12, "steps": 50, "alpha": 0.1, "n_calib": 800, "n_test": 800, "final_train_loss": 0.943260669708252, "tau_hat": 0.9167, "compute_used": 0.447575, "compute_saved": 0.5524249999999999, "halting_risk": 0.01, "risk_within_budget": true, "full_accuracy": 0.84, "halted_accuracy": 0.83875, "accuracy_drop": 0.0012499999999999734, "risk_monotone_in_tau": true, "calib_risk_by_tau": [0.92375, 0.83875, 0.83875, 0.73, 0.48375, 0.48375, 0.37625, 0.2, 0.2, 0.12625, 0.01375, 0.01375], "calib_compute_by_tau": [0.0, 0.02069999999999977, 0.02069999999999977, 0.04744999999999969, 0.10197499999999976, 0.10197499999999976, 0.13589999999999983, 0.22402499999999953, 0.22402499999999953, 0.28324999999999995, 0.4625250000000001, 0.4625250000000001], "tau_sweep": {"taus": [0.0833, 0.1667, 0.25, 0.3333, 0.4167, 0.5, 0.5833, 0.6667, 0.75, 0.8333, 0.9167, 1.0], "compute_used": [0.0, 0.021325000000000004, 0.021325000000000004, 0.046150000000000004, 0.09740000000000001, 0.09740000000000001, 0.12895, 0.215375, 0.215375, 0.28347500000000003, 0.447575, 0.447575], "accuracy": [0.0875, 0.18, 0.18, 0.30625, 0.49625, 0.49625, 0.61125, 0.75875, 0.75875, 0.78125, 0.83875, 0.83875], "disagreement": [0.89875, 0.8075, 0.8075, 0.6825, 0.47125, 0.47125, 0.3475, 0.19625, 0.19625, 0.135, 0.01, 0.01]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "2a54cf585cb3", "timestamp": "2026-07-30T11:17:10.656072+00:00", "git_sha": "67dc7a2", "config_hash": "315846db967f", "config": {"run": {"seed": 4, "task": "arithmetic", "notes": "F4 adaptive-halting run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 7, "batch_size": 128, "lr": 0.001, "n_train": 6000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 800, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 800}, "task": {"arithmetic": {"modular": false, "n_operands": 6, "ood_n_operands": 6, "max_operand": 7, "ood_max_operand": 9}}}, "task": "arithmetic", "split": "halting", "seed": 4, "metrics": {"k_restarts": 12, "steps": 50, "alpha": 0.1, "n_calib": 800, "n_test": 800, "final_train_loss": 0.8476110100746155, "tau_hat": 0.9167, "compute_used": 0.407875, "compute_saved": 0.592125, "halting_risk": 0.00625, "risk_within_budget": true, "full_accuracy": 0.84, "halted_accuracy": 0.84625, "accuracy_drop": -0.006249999999999978, "risk_monotone_in_tau": true, "calib_risk_by_tau": [0.90375, 0.83, 0.83, 0.72375, 0.485, 0.485, 0.3775, 0.18, 0.18, 0.1225, 0.01125, 0.01125], "calib_compute_by_tau": [0.0, 0.01557499999999983, 0.01557499999999983, 0.03847499999999953, 0.08722499999999994, 0.08722499999999994, 0.11397499999999977, 0.19922499999999954, 0.19922499999999954, 0.2511749999999996, 0.39365, 0.39365], "tau_sweep": {"taus": [0.0833, 0.1667, 0.25, 0.3333, 0.4167, 0.5, 0.5833, 0.6667, 0.75, 0.8333, 0.9167, 1.0], "compute_used": [0.0, 0.015425, 0.015425, 0.0375, 0.08922500000000001, 0.08922500000000001, 0.12187500000000001, 0.20292500000000005, 0.20292500000000005, 0.2681, 0.407875, 0.407875], "accuracy": [0.07875, 0.14875, 0.14875, 0.2475, 0.44875, 0.44875, 0.535, 0.71625, 0.71625, 0.7725, 0.84625, 0.84625], "disagreement": [0.91375, 0.8425, 0.8425, 0.7175, 0.50125, 0.50125, 0.40375, 0.19375, 0.19375, 0.12375, 0.00625, 0.00625]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "eaebf4a2b9f3", "timestamp": "2026-07-30T11:17:52.284763+00:00", "git_sha": "67dc7a2", "config_hash": "fccc394c29a9", "config": {"run": {"seed": 0, "task": "graph_planning", "notes": "graph adaptive-halting run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 25, "batch_size": 128, "lr": 0.001, "n_train": 8000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 800, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 800}, "task": {"graph_planning": {"n_nodes": 7, "ood_n_nodes": 10, "edge_prob": 0.4, "max_len": 4}}}, "task": "graph_planning", "split": "halting", "seed": 0, "metrics": {"k_restarts": 12, "steps": 50, "alpha": 0.1, "n_calib": 800, "n_test": 800, "final_train_loss": 0.227946937084198, "tau_hat": 0.8333, "compute_used": 0.168025, "compute_saved": 0.831975, "halting_risk": 0.0825, "risk_within_budget": true, "full_accuracy": 0.715, "halted_accuracy": 0.70625, "accuracy_drop": 0.008749999999999925, "risk_monotone_in_tau": true, "calib_risk_by_tau": [0.4575, 0.4575, 0.4575, 0.455, 0.3575, 0.3575, 0.2775, 0.14, 0.14, 0.08875, 0.005, 0.005], "calib_compute_by_tau": [0.0, 0.0, 0.0, 0.0005750000000000002, 0.025024999999999804, 0.025024999999999804, 0.05142499999999985, 0.12042500000000007, 0.12042500000000007, 0.16520000000000032, 0.28642499999999965, 0.28642499999999965], "tau_sweep": {"taus": [0.0833, 0.1667, 0.25, 0.3333, 0.4167, 0.5, 0.5833, 0.6667, 0.75, 0.8333, 0.9167, 1.0], "compute_used": [0.0, 0.0, 0.0, 0.000625, 0.024675000000000002, 0.024675000000000002, 0.0509, 0.11850000000000001, 0.11850000000000001, 0.168025, 0.28750000000000003, 0.28750000000000003], "accuracy": [0.53625, 0.53625, 0.53625, 0.53625, 0.58625, 0.58625, 0.61875, 0.6825, 0.6825, 0.70625, 0.71625, 0.71625], "disagreement": [0.4375, 0.4375, 0.4375, 0.43625, 0.34625, 0.34625, 0.2725, 0.12125, 0.12125, 0.0825, 0.005, 0.005]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "59f889f657d5", "timestamp": "2026-07-30T11:17:53.773421+00:00", "git_sha": "67dc7a2", "config_hash": "646eeb651c32", "config": {"run": {"seed": 0, "task": "arithmetic", "notes": "E1 selective-prediction main run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 7, "batch_size": 128, "lr": 0.001, "n_train": 6000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 1500, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 1500}, "task": {"arithmetic": {"modular": false, "n_operands": 6, "ood_n_operands": 6, "max_operand": 7, "ood_max_operand": 9}}}, "task": "arithmetic", "split": "selective", "seed": 0, "metrics": {"n_fit": 1500, "n_calib": 1500, "n_test": 1500, "k_restarts": 12, "objective": "basin_center", "sampler": "langevin", "feature_set": "base", "accuracy_id": 0.81, "base_error": 0.19, "final_train_loss": 1.010847568511963, "feature_names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "aurc": {"geometry": 0.0711432604851103, "rho_basin": 0.1192048662021191, "energy_min": 0.1447332231633481, "energy_mean": 0.14862598663924864, "energy_std": 0.18501378708598182, "msp": 0.06933811307974048, "temp_msp": 0.08261243554210007, "entropy": 0.07082020691013495, "softmax_learned": 0.06450094181149578, "geom_softmax": 0.05673560229226548}, "temperature": 0.4068744349087813, "best_energy_baseline": "energy_min", "best_baseline": "msp", "delta_aurc_vs_energy_min": [0.07358996267823781, 0.05406001270195325, 0.09343612485819916], "delta_aurc_vs_best_energy": [0.07358996267823781, 0.05406001270195325, 0.09343612485819916], "delta_aurc_vs_best_baseline": [-0.001805147405369814, -0.015438010117724282, 0.011245405317148913], "delta_aurc_geom_adds": [0.007765339519230301, 0.003572430837825001, 0.012225885708889614], "geometry_wins": true, "geometry_wins_vs_baseline": false, "geometry_adds_over_softmax": true, "risk_coverage": {"geometry": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.05, 0.03333333333333333, 0.025, 0.02666666666666667, 0.033333333333333326, 0.052380952380952396, 0.05, 0.04814814814814815, 0.046666666666666676, 0.04242424242424243, 0.041666666666666664, 0.04358974358974359, 0.04047619047619048, 0.04444444444444444, 0.04375, 0.043137254901960784, 0.04444444444444444, 0.043859649122807015, 0.045, 0.04285714285714285, 0.04242424242424243, 0.04057971014492753, 0.04166666666666666, 0.044, 0.04358974358974359, 0.04938271604938271, 0.05238095238095238, 0.0586206896551724, 0.06444444444444444, 0.06451612903225806, 0.06458333333333334, 0.06666666666666667, 0.07058823529411765, 0.07714285714285712, 0.08148148148148163, 0.08558558558558559, 0.08947368421052632, 0.10170940170940171, 0.11, 0.11788617886178875, 0.12380952380952392, 0.12945736434108526, 0.13636363636363635, 0.14444444444444443, 0.1492753623188406, 0.15673758865248238, 0.16805555555555565, 0.1782312925170068, 0.19]}, "rho_basin": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.10957642725598533, 0.10957642725598518, 0.10957642725598526, 0.1095764272559854, 0.10957642725598545, 0.10957642725598521, 0.10957642725598504, 0.10957642725598492, 0.10957642725598482, 0.10957642725598482, 0.10957642725598507, 0.10957642725598529, 0.10957642725598546, 0.10957642725598561, 0.10957642725598575, 0.10957642725598586, 0.10957642725598597, 0.10957642725598606, 0.10957642725598614, 0.1095764272559862, 0.10957642725598626, 0.10957642725598632, 0.10957642725598638, 0.10957642725598643, 0.10957642725598647, 0.10957642725598653, 0.10957642725598656, 0.1095764272559866, 0.10957642725598664, 0.10957642725598667, 0.1095764272559867, 0.10957642725598672, 0.10957642725598675, 0.10957642725598678, 0.10957642725598679, 0.10957642725598682, 0.11300813008130228, 0.11709456568250055, 0.12097144048363749, 0.12465447154471758, 0.12967479674796975, 0.1350907029478481, 0.14025470653377864, 0.14618844696969926, 0.15196759259259482, 0.15961352657005046, 0.1675650118203331, 0.17561111111111322, 0.18333333333333537, 0.19000000000000197]}, "energy_min": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.16666666666666666, 0.16666666666666666, 0.15833333333333333, 0.16, 0.15555555555555553, 0.14285714285714268, 0.1375, 0.14074074074074075, 0.1399999999999999, 0.13636363636363635, 0.1361111111111111, 0.13333333333333333, 0.13095238095238096, 0.12888888888888886, 0.13541666666666666, 0.13725490196078433, 0.13518518518518516, 0.13859649122807016, 0.14333333333333334, 0.14126984126984124, 0.1409090909090909, 0.14202898550724638, 0.1444444444444444, 0.14533333333333334, 0.14743589743589744, 0.14938271604938272, 0.15, 0.14712643678160917, 0.15, 0.14623655913978495, 0.14375, 0.1404040404040404, 0.13725490196078433, 0.13619047619047617, 0.13425925925925924, 0.13423423423423422, 0.13508771929824562, 0.13931623931623932, 0.1425, 0.14552845528455283, 0.14841269841269839, 0.15426356589147286, 0.15757575757575756, 0.16296296296296298, 0.16666666666666666, 0.17163120567375897, 0.17638888888888887, 0.18503401360544217, 0.19]}, "energy_mean": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.1, 0.15, 0.17777777777777778, 0.14166666666666666, 0.15333333333333332, 0.14999999999999997, 0.14761904761904765, 0.14166666666666666, 0.14814814814814814, 0.1466666666666667, 0.14545454545454545, 0.1388888888888889, 0.13333333333333333, 0.1357142857142857, 0.13999999999999999, 0.14375, 0.1411764705882353, 0.1444444444444444, 0.14385964912280702, 0.14333333333333334, 0.13968253968253966, 0.15, 0.15072463768115943, 0.15138888888888885, 0.14933333333333335, 0.15512820512820513, 0.15432098765432098, 0.15119047619047618, 0.14942528735632182, 0.14777777777777779, 0.14731182795698924, 0.14270833333333333, 0.1404040404040404, 0.13823529411764707, 0.1352380952380952, 0.13425925925925924, 0.13603603603603603, 0.13771929824561405, 0.14017094017094017, 0.14083333333333334, 0.1422764227642276, 0.146031746031746, 0.15193798449612403, 0.1590909090909091, 0.16296296296296298, 0.17028985507246377, 0.1730496453900709, 0.17638888888888898, 0.18435374149659864, 0.19]}, "energy_std": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.23333333333333334, 0.18333333333333332, 0.16666666666666666, 0.19166666666666668, 0.16666666666666666, 0.1611111111111111, 0.16666666666666669, 0.15833333333333333, 0.15925925925925927, 0.16666666666666669, 0.1696969696969697, 0.18611111111111112, 0.18461538461538463, 0.19523809523809524, 0.1911111111111111, 0.19166666666666668, 0.18627450980392157, 0.18888888888888886, 0.18947368421052632, 0.18833333333333332, 0.18888888888888886, 0.19242424242424241, 0.19130434782608696, 0.18888888888888886, 0.192, 0.18974358974358974, 0.18518518518518517, 0.18214285714285713, 0.18160919540229883, 0.17888888888888888, 0.18494623655913978, 0.18645833333333334, 0.18686868686868688, 0.18627450980392157, 0.1876190476190476, 0.18703703703703717, 0.1873873873873874, 0.18859649122807018, 0.18803418803418803, 0.1925, 0.19430894308943086, 0.19444444444444442, 0.1937984496124031, 0.19242424242424241, 0.18962962962962962, 0.18840579710144928, 0.18936170212765954, 0.19097222222222218, 0.19115646258503402, 0.19]}, "msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.011111111111111112, 0.03333333333333333, 0.03333333333333333, 0.033333333333333326, 0.03333333333333334, 0.029166666666666667, 0.02962962962962963, 0.026666666666666672, 0.024242424242424242, 0.03611111111111111, 0.03333333333333333, 0.03333333333333333, 0.033333333333333326, 0.0375, 0.0392156862745098, 0.04074074074074074, 0.042105263157894736, 0.04833333333333333, 0.04761904761904761, 0.051515151515151514, 0.04927536231884058, 0.049999999999999996, 0.052, 0.052564102564102565, 0.05308641975308642, 0.055952380952380955, 0.06436781609195401, 0.06777777777777778, 0.07311827956989247, 0.07395833333333333, 0.08282828282828283, 0.08529411764705883, 0.08952380952380966, 0.09537037037037036, 0.1009009009009009, 0.10526315789473684, 0.1094017094017094, 0.11583333333333333, 0.12357723577235771, 0.12619047619047616, 0.13023255813953488, 0.13787878787878788, 0.1437037037037037, 0.1471014492753623, 0.15602836879432622, 0.1659722222222222, 0.17755102040816326, 0.19]}, "temp_msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.016666666666666666, 0.03333333333333333, 0.041666666666666664, 0.03333333333333333, 0.03888888888888888, 0.038095238095238106, 0.0375, 0.04814814814814815, 0.046666666666666676, 0.06363636363636363, 0.06111111111111111, 0.05897435897435897, 0.06190476190476191, 0.06222222222222221, 0.06458333333333334, 0.06666666666666667, 0.06296296296296294, 0.06666666666666667, 0.06666666666666667, 0.06349206349206347, 0.06363636363636363, 0.06521739130434782, 0.06805555555555554, 0.068, 0.07179487179487179, 0.07407407407407407, 0.07976190476190476, 0.08390804597701168, 0.08555555555555555, 0.08602150537634409, 0.08645833333333333, 0.0898989898989899, 0.0892156862745098, 0.09333333333333348, 0.10092592592592592, 0.1045045045045045, 0.10789473684210527, 0.11196581196581197, 0.11583333333333333, 0.1219512195121951, 0.1269841269841271, 0.1325581395348837, 0.14166666666666666, 0.1474074074074074, 0.15434782608695652, 0.16099290780141856, 0.16736111111111118, 0.1782312925170068, 0.19]}, "entropy": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.011111111111111112, 0.008333333333333333, 0.02666666666666667, 0.02222222222222222, 0.02380952380952381, 0.025, 0.025925925925925925, 0.023333333333333338, 0.03636363636363636, 0.03333333333333333, 0.03333333333333333, 0.0380952380952381, 0.039999999999999994, 0.041666666666666664, 0.041176470588235294, 0.04444444444444444, 0.043859649122807015, 0.05, 0.057142857142857134, 0.06060606060606061, 0.06231884057971015, 0.06666666666666665, 0.068, 0.06923076923076923, 0.07037037037037037, 0.07261904761904762, 0.07241379310344827, 0.07111111111111111, 0.07526881720430108, 0.07604166666666666, 0.07575757575757576, 0.0784313725490196, 0.0809523809523811, 0.08425925925925924, 0.08828828828828829, 0.09385964912280702, 0.10085470085470086, 0.10583333333333333, 0.11707317073170745, 0.123015873015873, 0.13178294573643412, 0.14166666666666666, 0.1474074074074074, 0.15869565217391304, 0.1638297872340425, 0.17013888888888887, 0.17959183673469387, 0.19]}, "softmax_learned": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.011111111111111112, 0.016666666666666666, 0.02666666666666667, 0.033333333333333326, 0.028571428571428577, 0.025, 0.025925925925925925, 0.026666666666666672, 0.024242424242424242, 0.022222222222222223, 0.02564102564102564, 0.023809523809523808, 0.02888888888888888, 0.03125, 0.03333333333333333, 0.03518518518518518, 0.03508771929824561, 0.03666666666666667, 0.04126984126984126, 0.03939393939393939, 0.042028985507246375, 0.04305555555555555, 0.044, 0.04230769230769231, 0.04691358024691358, 0.04880952380952381, 0.0528735632183908, 0.058888888888888886, 0.06344086021505377, 0.06666666666666667, 0.07474747474747474, 0.0784313725490196, 0.0819047619047619, 0.09074074074074073, 0.0945945945945946, 0.10087719298245613, 0.10598290598290598, 0.11416666666666667, 0.12032520325203251, 0.1238095238095238, 0.1310077519379845, 0.14015151515151514, 0.14148148148148149, 0.15, 0.15673758865248225, 0.16736111111111118, 0.17687074829931973, 0.19]}, "geom_softmax": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.02, 0.02222222222222222, 0.019047619047619053, 0.016666666666666666, 0.022222222222222223, 0.026666666666666672, 0.02727272727272727, 0.025, 0.023076923076923078, 0.02142857142857143, 0.019999999999999997, 0.01875, 0.0196078431372549, 0.018518518518518514, 0.021052631578947368, 0.023333333333333334, 0.023809523809523937, 0.02878787878787879, 0.02753623188405797, 0.029166666666666664, 0.032, 0.03333333333333333, 0.037037037037037035, 0.04404761904761905, 0.04712643678160919, 0.04777777777777778, 0.04946236559139785, 0.05520833333333333, 0.05858585858585859, 0.06470588235294118, 0.0676190476190476, 0.07592592592592591, 0.08378378378378379, 0.09473684210526316, 0.10085470085470086, 0.10666666666666667, 0.11382113821138223, 0.11984126984126983, 0.12868217054263567, 0.13484848484848486, 0.13925925925925925, 0.1463768115942029, 0.15744680851063841, 0.16736111111111118, 0.17959183673469387, 0.19]}}, "ltt": {"alpha": 0.1, "delta": 0.05, "lambda_hat": 0.19374220945381396, "coverage": 0.6933333333333334, "abstain_rate": 0.30666666666666664, "selective_risk": 0.075, "selective_accuracy": 0.925, "risk_within_budget": true}, "coverage_validity": {"alpha_grid": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "achieved_risk": [0.0, 0.0, 0.075, 0.1295577967416602, 0.17525773195876287, 0.1879598662207358], "coverage": [0.0, 0.0, 0.6933333333333334, 0.8593333333333333, 0.97, 0.9966666666666667]}, "feature_diagnostics": {"names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "auroc": {"basin/rho": 0.7041700960219479, "basin/entropy": 0.2946790845426323, "basin/dispersion": 0.5298130098909826, "energy/mean": 0.3736336726590138, "energy/min": 0.36821890116237094, "energy/std": 0.49345462421485814, "curv/lmax_mean": 0.4698808750270739, "curv/lmax_best": 0.4712165186629124, "curv/trace_mean": 0.4452761533463288, "curv/trace_best": 0.4445397444227854, "dynamics/steps": 0.5, "dynamics/monotonic": 0.341111833080644, "dynamics/drop": 0.3597834091401343, "dynamics/residual": 0.5355483358602267}, "hist": {"basin/rho": {"edges": [0.3333333333333333, 0.36666666666666664, 0.4, 0.43333333333333335, 0.4666666666666667, 0.5, 0.5333333333333333, 0.5666666666666667, 0.6000000000000001, 0.6333333333333333, 0.6666666666666667, 0.7000000000000001, 0.7333333333333334, 0.7666666666666667, 0.8, 0.8333333333333335, 0.8666666666666667, 0.9000000000000001, 0.9333333333333333, 0.9666666666666668, 1.0], "correct_counts": [2, 0, 0, 0, 0, 14, 0, 22, 0, 28, 0, 0, 38, 0, 54, 0, 0, 90, 0, 967], "incorrect_counts": [1, 0, 2, 0, 0, 14, 0, 28, 0, 32, 0, 0, 26, 0, 30, 0, 0, 33, 0, 119]}, "basin/entropy": {"edges": [0.0, 0.06789889274936621, 0.13579778549873242, 0.20369667824809862, 0.27159557099746484, 0.33949446374683107, 0.40739335649619723, 0.47529224924556346, 0.5431911419949297, 0.6110900347442959, 0.6789889274936621, 0.7468878202430284, 0.8147867129923945, 0.8826856057417607, 0.9505844984911269, 1.0184833912404931, 1.0863822839898594, 1.1542811767392256, 1.2221800694885918, 1.290078962237958, 1.3579778549873243], "correct_counts": [967, 0, 0, 0, 90, 0, 53, 0, 37, 25, 35, 0, 3, 1, 1, 1, 1, 0, 0, 1], "incorrect_counts": [119, 0, 0, 0, 33, 0, 26, 0, 27, 29, 35, 0, 3, 2, 6, 3, 2, 0, 0, 0]}, "basin/dispersion": {"edges": [1.592531476672375, 1.614116358729778, 1.6357012407871807, 1.6572861228445837, 1.6788710049019866, 1.7004558869593893, 1.7220407690167923, 1.7436256510741952, 1.7652105331315981, 1.7867954151890009, 1.8083802972464038, 1.8299651793038068, 1.8515500613612095, 1.8731349434186124, 1.8947198254760154, 1.9163047075334183, 1.937889589590821, 1.959474471648224, 1.981059353705627, 2.0026442357630296, 2.0242291178204326], "correct_counts": [3, 2, 8, 9, 23, 47, 59, 82, 96, 177, 167, 139, 126, 93, 70, 52, 31, 17, 10, 4], "incorrect_counts": [0, 0, 2, 1, 5, 11, 19, 21, 37, 36, 38, 29, 30, 20, 17, 12, 5, 1, 1, 0]}, "energy/mean": {"edges": [-0.30860641847054165, -0.24559052648643653, -0.18257463450233144, -0.11955874251822635, -0.056542850534121225, 0.006473041449983896, 0.06948893343408896, 0.13250482541819408, 0.1955207174022992, 0.25853660938640427, 0.32155250137050945, 0.3845683933546145, 0.4475842853387196, 0.5106001773228248, 0.5736160693069299, 0.6366319612910349, 0.69964785327514, 0.7626637452592451, 0.8256796372433501, 0.8886955292274554, 0.9517114212115606], "correct_counts": [10, 53, 216, 277, 172, 82, 46, 41, 37, 37, 39, 36, 43, 41, 43, 21, 18, 2, 0, 1], "incorrect_counts": [2, 11, 35, 49, 32, 8, 2, 2, 4, 7, 14, 6, 21, 33, 19, 24, 12, 3, 1, 0]}, "energy/min": {"edges": [-0.6984671354293823, -0.628644996881485, -0.5588228583335877, -0.48900071978569026, -0.41917858123779295, -0.34935644268989563, -0.27953430414199826, -0.20971216559410089, -0.13989002704620357, -0.07006788849830625, -0.00024574995040893555, 0.06957638859748849, 0.1393985271453858, 0.20922066569328313, 0.27904280424118055, 0.34886494278907776, 0.4186870813369752, 0.4885092198848726, 0.5583313584327698, 0.6281534969806672, 0.6979756355285645], "correct_counts": [5, 25, 85, 195, 264, 157, 92, 53, 48, 43, 42, 45, 44, 51, 33, 14, 13, 6, 0, 0], "incorrect_counts": [0, 3, 21, 25, 47, 32, 10, 2, 5, 6, 12, 17, 21, 28, 22, 19, 11, 3, 0, 1]}, "energy/std": {"edges": [0.058812152065186904, 0.07638791223625852, 0.09396367240733014, 0.11153943257840175, 0.12911519274947336, 0.14669095292054496, 0.1642667130916166, 0.1818424732626882, 0.1994182334337598, 0.21699399360483146, 0.23456975377590306, 0.25214551394697465, 0.2697212741180463, 0.2872970342891179, 0.3048727944601895, 0.32244855463126115, 0.34002431480233275, 0.35760007497340435, 0.375175835144476, 0.3927515953155476, 0.4103273554866192], "correct_counts": [3, 6, 20, 52, 77, 135, 180, 211, 168, 133, 90, 68, 39, 16, 9, 5, 1, 0, 1, 1], "incorrect_counts": [0, 3, 4, 12, 10, 38, 42, 44, 44, 39, 19, 10, 16, 1, 1, 1, 0, 1, 0, 0]}, "curv/lmax_mean": {"edges": [1.0000369946161907, 1.0007940803964934, 1.001551166176796, 1.0023082519570987, 1.0030653377374015, 1.003822423517704, 1.0045795092980068, 1.0053365950783095, 1.006093680858612, 1.0068507666389148, 1.0076078524192176, 1.0083649381995201, 1.0091220239798229, 1.0098791097601254, 1.0106361955404282, 1.011393281320731, 1.0121503671010335, 1.0129074528813362, 1.013664538661639, 1.0144216244419415, 1.0151787102222443], "correct_counts": [617, 173, 79, 69, 52, 45, 45, 33, 29, 19, 20, 8, 6, 5, 6, 2, 6, 0, 0, 1], "incorrect_counts": [119, 74, 29, 22, 10, 10, 4, 5, 3, 4, 3, 1, 0, 0, 0, 1, 0, 0, 0, 0]}, "curv/lmax_best": {"edges": [1.0, 1.0017322301864624, 1.0034644603729248, 1.0051966905593872, 1.0069289207458496, 1.008661150932312, 1.0103933811187744, 1.0121256113052368, 1.0138578414916992, 1.0155900716781616, 1.017322301864624, 1.0190545320510864, 1.0207867622375488, 1.0225189924240112, 1.0242512226104736, 1.025983452796936, 1.0277156829833984, 1.0294479131698608, 1.0311801433563232, 1.0329123735427856, 1.034644603729248], "correct_counts": [934, 93, 63, 35, 27, 11, 9, 7, 9, 7, 3, 3, 2, 5, 3, 2, 1, 0, 0, 1], "incorrect_counts": [213, 31, 16, 6, 6, 3, 3, 3, 2, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0]}, "curv/trace_mean": {"edges": [32.00269635518392, 32.00753688812256, 32.0123774210612, 32.01721795399984, 32.02205848693848, 32.02689901987712, 32.03173955281576, 32.036580085754395, 32.04142061869303, 32.046261151631676, 32.05110168457031, 32.05594221750895, 32.06078275044759, 32.06562328338623, 32.07046381632487, 32.075304349263504, 32.08014488220215, 32.084985415140785, 32.08982594807942, 32.094666481018066, 32.0995070139567], "correct_counts": [175, 275, 151, 82, 84, 56, 49, 58, 42, 42, 57, 38, 42, 21, 20, 15, 4, 2, 1, 1], "incorrect_counts": [9, 50, 53, 31, 37, 20, 19, 17, 9, 8, 7, 13, 4, 5, 2, 0, 1, 0, 0, 0]}, "curv/trace_best": {"edges": [32.0009880065918, 32.010338592529294, 32.0196891784668, 32.029039764404295, 32.0383903503418, 32.0477409362793, 32.057091522216794, 32.0664421081543, 32.075792694091795, 32.0851432800293, 32.0944938659668, 32.103844451904294, 32.1131950378418, 32.122545623779295, 32.1318962097168, 32.1412467956543, 32.150597381591794, 32.1599479675293, 32.169298553466795, 32.1786491394043, 32.1879997253418], "correct_counts": [402, 281, 150, 106, 83, 59, 46, 28, 17, 9, 10, 6, 3, 5, 2, 2, 2, 3, 0, 1], "incorrect_counts": [66, 67, 55, 31, 17, 13, 8, 8, 7, 6, 3, 2, 0, 1, 1, 0, 0, 0, 0, 0]}, "dynamics/steps": {"edges": [1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7000000000000002, 1.75, 1.8, 1.85, 1.9, 1.9500000000000002, 2.0], "correct_counts": [1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/monotonic": {"edges": [0.7133333333333333, 0.7193333333333333, 0.7253333333333333, 0.7313333333333333, 0.7373333333333333, 0.7433333333333333, 0.7493333333333333, 0.7553333333333333, 0.7613333333333333, 0.7673333333333333, 0.7733333333333334, 0.7793333333333334, 0.7853333333333334, 0.7913333333333334, 0.7973333333333334, 0.8033333333333335, 0.8093333333333335, 0.8153333333333335, 0.8213333333333335, 0.8273333333333335, 0.8333333333333335], "correct_counts": [1, 0, 2, 23, 32, 66, 131, 145, 240, 187, 167, 125, 55, 25, 12, 3, 1, 0, 0, 0], "incorrect_counts": [1, 0, 0, 3, 8, 14, 10, 17, 33, 38, 46, 39, 19, 20, 14, 11, 6, 3, 2, 1]}, "dynamics/drop": {"edges": [63.47809874266386, 76.39078911232451, 89.30347948198516, 102.21616985164583, 115.12886022130648, 128.04155059096712, 140.9542409606278, 153.86693133028845, 166.7796216999491, 179.69231206960976, 192.6050024392704, 205.51769280893106, 218.43038317859174, 231.3430735482524, 244.25576391791304, 257.1684542875737, 270.08114465723435, 282.993835026895, 295.90652539655565, 308.8192157662163, 321.73190613587695], "correct_counts": [10, 31, 108, 203, 266, 281, 168, 89, 28, 12, 10, 3, 1, 2, 2, 0, 0, 1, 0, 0], "incorrect_counts": [1, 7, 23, 29, 45, 36, 31, 27, 17, 10, 15, 12, 4, 11, 5, 4, 0, 4, 1, 3]}, "dynamics/residual": {"edges": [1.1409447093804677, 1.1559197435776392, 1.1708947777748107, 1.1858698119719822, 1.2008448461691539, 1.2158198803663254, 1.2307949145634969, 1.2457699487606684, 1.2607449829578399, 1.2757200171550114, 1.2906950513521829, 1.3056700855493546, 1.320645119746526, 1.3356201539436976, 1.350595188140869, 1.3655702223380406, 1.380545256535212, 1.3955202907323838, 1.4104953249295553, 1.4254703591267268, 1.4404453933238983], "correct_counts": [2, 4, 6, 19, 33, 43, 85, 102, 112, 156, 148, 145, 122, 97, 61, 43, 24, 7, 4, 2], "incorrect_counts": [0, 2, 2, 1, 11, 8, 21, 34, 36, 27, 34, 40, 24, 26, 10, 6, 3, 0, 0, 0]}}}, "feature_ablation": {"full": 0.0711432604851103, "drop_basin": 0.11290811853904308, "basin_only": 0.13271270080190306, "drop_energy": 0.08423482839718313, "energy_only": 0.14821380110222312, "drop_curv": 0.07388292606950923, "curv_only": 0.14046656429951282, "drop_dynamics": 0.07747284186509383, "dynamics_only": 0.15568347575014724}, "ece_geometry": 0.03153101157848922, "accuracy_ood": 0.20533333333333334, "aurc_ood": {"geometry": 0.8035703588171388, "rho_basin": 0.7533412924431383, "energy_min": 0.7509755512657853, "energy_mean": 0.740760170602411, "energy_std": 0.7992759846101417, "msp": 0.7106997166086145, "temp_msp": 0.709349201412843, "entropy": 0.7172185327581629, "softmax_learned": 0.7078058975428306, "geom_softmax": 0.7203531164404758}, "ood_ltt": {"alpha": 0.1, "lambda_hat": 0.19374220945381396, "selective_risk": 0.7619532044760936, "coverage": 0.6553333333333333, "risk_within_budget": false}, "ood_validity": {"target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "id_risk": [0.0, 0.0, 0.075, 0.1295577967416602, 0.17525773195876287, 0.1879598662207358], "ood_risk": [0.0, 0.0, 0.7619532044760936, 0.7915726109857035, 0.7924914675767918, 0.7949231796927188], "id_coverage": [0.0, 0.0, 0.6933333333333334, 0.8593333333333333, 0.97, 0.9966666666666667], "ood_coverage": [0.0, 0.0, 0.6553333333333333, 0.886, 0.9766666666666667, 0.998]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "2165fc5b4ecc", "timestamp": "2026-07-30T11:17:57.810793+00:00", "git_sha": "67dc7a2", "config_hash": "c5e806bfe9f7", "config": {"run": {"seed": 0, "task": "graph_planning", "notes": "E2 graph selective-prediction"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 25, "batch_size": 128, "lr": 0.001, "n_train": 8000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 1500, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 1500}, "task": {"graph_planning": {"n_nodes": 7, "ood_n_nodes": 10, "edge_prob": 0.4, "max_len": 4}}}, "task": "graph_planning", "split": "selective", "seed": 0, "metrics": {"n_fit": 1500, "n_calib": 1500, "n_test": 1500, "k_restarts": 12, "objective": "basin_center", "sampler": "langevin", "feature_set": "base", "accuracy_id": 0.7153333333333334, "base_error": 0.2846666666666667, "final_train_loss": 0.227946937084198, "feature_names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "aurc": {"geometry": 0.14828452971658584, "rho_basin": 0.22590771079860786, "energy_min": 0.37002240610013937, "energy_mean": 0.38689280922170594, "energy_std": 0.2757969578417908, "msp": 0.11876501845457642, "temp_msp": 0.11800947069600645, "entropy": 0.11733858388676534, "softmax_learned": 0.118432975750678, "geom_softmax": 0.1213234913882991}, "temperature": 3.0183027222710757, "best_energy_baseline": "energy_std", "best_baseline": "entropy", "delta_aurc_vs_energy_min": [0.22173787638355352, 0.18892196301826353, 0.2546275576199255], "delta_aurc_vs_best_energy": [0.12751242812520497, 0.09889745209899486, 0.15448230768652413], "delta_aurc_vs_best_baseline": [-0.030945945829820506, -0.04266843039923084, -0.020834827103470434], "delta_aurc_geom_adds": [-0.002890515637621105, -0.006592549629494021, 0.0005597097649683881], "geometry_wins": true, "geometry_wins_vs_baseline": false, "geometry_adds_over_softmax": false, "risk_coverage": {"geometry": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.06666666666666667, 0.05, 0.044444444444444446, 0.05, 0.04666666666666667, 0.06666666666666678, 0.07142857142857144, 0.07083333333333333, 0.08518518518518518, 0.08666666666666668, 0.07878787878787878, 0.08888888888888889, 0.10256410256410256, 0.10952380952380952, 0.11111111111111109, 0.11041666666666666, 0.11568627450980393, 0.12037037037037035, 0.1280701754385965, 0.13, 0.13015873015873014, 0.13484848484848486, 0.13478260869565217, 0.138888888888889, 0.14533333333333334, 0.14358974358974358, 0.1469135802469136, 0.14642857142857144, 0.14827586206896548, 0.15333333333333332, 0.15591397849462366, 0.1625, 0.1717171717171717, 0.1784313725490196, 0.18952380952380948, 0.1916666666666668, 0.1963963963963964, 0.2, 0.20598290598290597, 0.21166666666666667, 0.2203252032520325, 0.23015873015873012, 0.2364341085271318, 0.24166666666666667, 0.2511111111111111, 0.2565217391304348, 0.2645390070921985, 0.27083333333333326, 0.27755102040816326, 0.2846666666666667]}, "rho_basin": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.21985233798195233, 0.21985233798195258, 0.2198523379819524, 0.2198523379819521, 0.21985233798195206, 0.21985233798195253, 0.21985233798195283, 0.21985233798195308, 0.21985233798195328, 0.21985233798195325, 0.21985233798195275, 0.2198523379819523, 0.21985233798195197, 0.21985233798195167, 0.2198523379819514, 0.21985233798195117, 0.21985233798195095, 0.21985233798195078, 0.2198523379819506, 0.21985233798195086, 0.2198523379819514, 0.21985233798195186, 0.2198523379819523, 0.21985233798195272, 0.21985233798195308, 0.21985233798195342, 0.21985233798195372, 0.21985233798195403, 0.2198523379819543, 0.21985233798195455, 0.21985233798195478, 0.219852337981955, 0.21985233798195522, 0.21985233798195541, 0.2198523379819556, 0.21985233798195578, 0.21985233798195594, 0.2198523379819561, 0.21985233798195614, 0.21985233798195555, 0.2225706542779741, 0.22974300831443945, 0.23658176448874357, 0.24213688610240544, 0.2467177522349954, 0.25484057971014695, 0.26473758865248487, 0.27293836805555816, 0.2785034013605472, 0.2846666666666696]}, "energy_min": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.4, 0.48333333333333334, 0.4666666666666667, 0.48333333333333334, 0.47333333333333333, 0.44999999999999996, 0.44285714285714295, 0.42916666666666664, 0.4148148148148148, 0.4133333333333334, 0.4, 0.4027777777777778, 0.3974358974358974, 0.3952380952380952, 0.3955555555555555, 0.3875, 0.3862745098039216, 0.38888888888888884, 0.38070175438596493, 0.38, 0.3714285714285714, 0.3696969696969697, 0.36666666666666664, 0.3666666666666666, 0.364, 0.3628205128205128, 0.3567901234567901, 0.3595238095238095, 0.36436781609195396, 0.3611111111111111, 0.3548387096774194, 0.346875, 0.3434343434343434, 0.3392156862745098, 0.33619047619047615, 0.3324074074074074, 0.3279279279279279, 0.32456140350877194, 0.3213675213675214, 0.32083333333333336, 0.3186991869918699, 0.3142857142857144, 0.31085271317829455, 0.30757575757575756, 0.30444444444444446, 0.29927536231884055, 0.2964539007092198, 0.2930555555555555, 0.2891156462585034, 0.2846666666666667]}, "energy_mean": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.5, 0.5333333333333333, 0.5333333333333333, 0.49166666666666664, 0.4866666666666667, 0.48888888888888893, 0.49047619047619034, 0.4625, 0.4444444444444444, 0.4366666666666667, 0.43636363636363634, 0.425, 0.41794871794871796, 0.4023809523809524, 0.4066666666666668, 0.3958333333333333, 0.4, 0.3981481481481481, 0.3929824561403509, 0.39166666666666666, 0.38730158730158726, 0.39090909090909093, 0.3927536231884058, 0.38888888888888884, 0.38666666666666666, 0.37948717948717947, 0.37777777777777777, 0.3726190476190476, 0.36896551724137927, 0.3622222222222222, 0.35698924731182796, 0.35104166666666664, 0.3434343434343434, 0.34019607843137256, 0.33809523809523806, 0.3342592592592592, 0.3324324324324324, 0.3307017543859649, 0.32564102564102565, 0.3225, 0.3227642276422765, 0.3166666666666666, 0.31317829457364343, 0.3090909090909091, 0.30592592592592593, 0.30144927536231886, 0.29574468085106376, 0.2909722222222222, 0.28775510204081634, 0.2846666666666667]}, "energy_std": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.26666666666666666, 0.26666666666666666, 0.26666666666666666, 0.25833333333333336, 0.28, 0.27222222222222214, 0.2666666666666667, 0.2791666666666667, 0.2777777777777778, 0.2733333333333332, 0.2818181818181818, 0.2833333333333333, 0.28205128205128205, 0.28095238095238095, 0.2888888888888888, 0.28125, 0.27450980392156865, 0.274074074074074, 0.2736842105263158, 0.28, 0.2793650793650793, 0.2787878787878788, 0.27391304347826084, 0.26944444444444454, 0.268, 0.2717948717948718, 0.2654320987654321, 0.26666666666666666, 0.2701149425287358, 0.27, 0.26881720430107525, 0.26979166666666665, 0.2696969696969697, 0.2735294117647059, 0.27238095238095233, 0.2703703703703703, 0.27297297297297296, 0.27631578947368424, 0.27863247863247864, 0.2775, 0.2796747967479674, 0.2793650793650793, 0.2813953488372093, 0.2825757575757576, 0.2837037037037037, 0.28405797101449276, 0.2843971631205675, 0.28472222222222227, 0.282312925170068, 0.2846666666666667]}, "msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.011111111111111112, 0.016666666666666666, 0.02666666666666667, 0.02222222222222222, 0.02380952380952381, 0.020833333333333332, 0.02962962962962963, 0.030000000000000002, 0.03636363636363636, 0.03611111111111111, 0.046153846153846156, 0.047619047619047616, 0.05333333333333333, 0.05625, 0.056862745098039215, 0.0574074074074074, 0.06315789473684211, 0.06666666666666667, 0.07777777777777777, 0.08787878787878788, 0.08985507246376812, 0.09722222222222221, 0.108, 0.11794871794871795, 0.1271604938271605, 0.13214285714285715, 0.1333333333333333, 0.14333333333333334, 0.15053763440860216, 0.16041666666666668, 0.16363636363636364, 0.1676470588235294, 0.17238095238095236, 0.17777777777777792, 0.1837837837837838, 0.193859649122807, 0.20256410256410257, 0.20916666666666667, 0.21300813008130093, 0.22222222222222232, 0.2310077519379845, 0.23712121212121212, 0.24814814814814815, 0.25507246376811593, 0.2645390070921985, 0.26736111111111105, 0.2755102040816326, 0.2846666666666667]}, "temp_msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.011111111111111112, 0.025, 0.02666666666666667, 0.02222222222222222, 0.02380952380952381, 0.029166666666666667, 0.03333333333333333, 0.030000000000000002, 0.030303030303030304, 0.041666666666666664, 0.046153846153846156, 0.05476190476190476, 0.055555555555555546, 0.05625, 0.054901960784313725, 0.059259259259259255, 0.06666666666666667, 0.07333333333333333, 0.08253968253968266, 0.0893939393939394, 0.09420289855072464, 0.09999999999999999, 0.10666666666666667, 0.11025641025641025, 0.12345679012345678, 0.12857142857142856, 0.13448275862068962, 0.1411111111111111, 0.14516129032258066, 0.14895833333333333, 0.15858585858585858, 0.16568627450980392, 0.1714285714285714, 0.1759259259259259, 0.1837837837837838, 0.19210526315789472, 0.19914529914529913, 0.20833333333333334, 0.21463414634146352, 0.21746031746031758, 0.2255813953488372, 0.23257575757575757, 0.23925925925925925, 0.24782608695652175, 0.25673758865248236, 0.26597222222222233, 0.2761904761904762, 0.2846666666666667]}, "entropy": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.011111111111111112, 0.016666666666666666, 0.02666666666666667, 0.02222222222222222, 0.02380952380952381, 0.020833333333333332, 0.02962962962962963, 0.030000000000000002, 0.03636363636363636, 0.03611111111111111, 0.046153846153846156, 0.047619047619047616, 0.05333333333333333, 0.05625, 0.056862745098039215, 0.055555555555555546, 0.06315789473684211, 0.06833333333333333, 0.07777777777777777, 0.08787878787878788, 0.08840579710144927, 0.09583333333333333, 0.10666666666666667, 0.11666666666666667, 0.1259259259259259, 0.13095238095238096, 0.13563218390804596, 0.14, 0.14838709677419354, 0.159375, 0.1606060606060606, 0.16470588235294117, 0.17047619047619045, 0.17777777777777776, 0.1864864864864865, 0.19210526315789472, 0.19658119658119658, 0.2025, 0.20813008130081298, 0.21825396825396837, 0.22325581395348837, 0.23106060606060605, 0.23703703703703705, 0.24710144927536232, 0.2574468085106382, 0.2666666666666668, 0.2761904761904762, 0.2846666666666667]}, "softmax_learned": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.011111111111111112, 0.025, 0.02666666666666667, 0.02222222222222222, 0.02380952380952381, 0.029166666666666667, 0.03333333333333333, 0.030000000000000002, 0.030303030303030304, 0.041666666666666664, 0.046153846153846156, 0.05476190476190476, 0.055555555555555546, 0.05625, 0.054901960784313725, 0.059259259259259255, 0.06666666666666667, 0.075, 0.08095238095238108, 0.09090909090909091, 0.09710144927536232, 0.1000000000000001, 0.10666666666666667, 0.11025641025641025, 0.12222222222222222, 0.12738095238095237, 0.13218390804597718, 0.1388888888888889, 0.14408602150537633, 0.15104166666666666, 0.15757575757575756, 0.16470588235294117, 0.1714285714285714, 0.1759259259259259, 0.18288288288288287, 0.18859649122807018, 0.20085470085470086, 0.21, 0.2138211382113822, 0.21825396825396837, 0.22790697674418606, 0.2356060606060606, 0.24222222222222223, 0.2514492753623188, 0.2595744680851065, 0.2666666666666668, 0.2761904761904762, 0.2846666666666667]}, "geom_softmax": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.03333333333333333, 0.03333333333333333, 0.025, 0.02666666666666667, 0.03888888888888888, 0.03333333333333334, 0.03333333333333333, 0.03333333333333333, 0.036666666666666674, 0.04242424242424243, 0.044444444444444446, 0.04871794871794872, 0.05238095238095238, 0.048888888888888885, 0.04791666666666667, 0.052941176470588235, 0.06296296296296294, 0.06666666666666667, 0.07833333333333334, 0.08412698412698412, 0.08636363636363636, 0.09565217391304348, 0.10138888888888888, 0.10666666666666667, 0.11153846153846154, 0.12098765432098765, 0.1261904761904762, 0.13103448275862067, 0.14, 0.14623655913978495, 0.15208333333333332, 0.15858585858585858, 0.16176470588235295, 0.16952380952380966, 0.17499999999999996, 0.18468468468468469, 0.19210526315789472, 0.19743589743589743, 0.2075, 0.21626016260162598, 0.2214285714285714, 0.2317829457364341, 0.23939393939393938, 0.25037037037037035, 0.25579710144927537, 0.2652482269503547, 0.27083333333333326, 0.27687074829931974, 0.2846666666666667]}}, "ltt": {"alpha": 0.1, "delta": 0.05, "lambda_hat": null, "coverage": 0.0, "abstain_rate": 1.0, "selective_risk": 0.0, "selective_accuracy": null, "risk_within_budget": true}, "coverage_validity": {"alpha_grid": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "achieved_risk": [0.0, 0.0, 0.0, 0.07346938775510205, 0.15090090090090091, 0.2565217391304348], "coverage": [0.0, 0.0, 0.0, 0.16333333333333333, 0.592, 0.92]}, "feature_diagnostics": {"names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "auroc": {"basin/rho": 0.6313428828974335, "basin/entropy": 0.36828280270903224, "basin/dispersion": 0.4898869636009263, "energy/mean": 0.656499429252397, "energy/min": 0.6382769751904857, "energy/std": 0.4808925052000236, "curv/lmax_mean": 0.32037165163225084, "curv/lmax_best": 0.37439951459171356, "curv/trace_mean": 0.31464016709918347, "curv/trace_best": 0.34222484618188403, "dynamics/steps": 0.5, "dynamics/monotonic": 0.6361162098867017, "dynamics/drop": 0.6684338380211755, "dynamics/residual": 0.47719956086264737}, "hist": {"basin/rho": {"edges": [0.3333333333333333, 0.36666666666666664, 0.4, 0.43333333333333335, 0.4666666666666667, 0.5, 0.5333333333333333, 0.5666666666666667, 0.6000000000000001, 0.6333333333333333, 0.6666666666666667, 0.7000000000000001, 0.7333333333333334, 0.7666666666666667, 0.8, 0.8333333333333335, 0.8666666666666667, 0.9000000000000001, 0.9333333333333333, 0.9666666666666668, 1.0], "correct_counts": [0, 0, 2, 0, 0, 9, 0, 14, 0, 11, 0, 0, 14, 0, 32, 0, 0, 40, 0, 951], "incorrect_counts": [1, 0, 0, 0, 0, 15, 0, 16, 0, 21, 0, 0, 36, 0, 26, 0, 0, 44, 0, 268]}, "basin/entropy": {"edges": [0.0, 0.06648306744273791, 0.13296613488547582, 0.19944920232821373, 0.26593226977095163, 0.33241533721368954, 0.39889840465642745, 0.46538147209916536, 0.5318645395419033, 0.5983476069846412, 0.6648306744273791, 0.731313741870117, 0.7977968093128549, 0.8642798767555928, 0.9307629441983307, 0.9972460116410686, 1.0637290790838065, 1.1302121465265444, 1.1966952139692824, 1.2631782814120203, 1.3296613488547582], "correct_counts": [951, 0, 0, 0, 40, 0, 31, 0, 14, 11, 23, 0, 0, 0, 1, 1, 1, 0, 0, 0], "incorrect_counts": [268, 0, 0, 0, 44, 0, 25, 0, 34, 17, 28, 0, 3, 5, 1, 1, 0, 0, 0, 1]}, "basin/dispersion": {"edges": [1.6005140780864282, 1.6224848783774093, 1.6444556786683906, 1.6664264789593717, 1.688397279250353, 1.710368079541334, 1.7323388798323154, 1.7543096801232965, 1.7762804804142778, 1.7982512807052589, 1.8202220809962402, 1.8421928812872213, 1.8641636815782023, 1.8861344818691836, 1.9081052821601647, 1.930076082451146, 1.9520468827421271, 1.9740176830331084, 1.9959884833240895, 2.017959283615071, 2.039930083906052], "correct_counts": [0, 4, 7, 11, 23, 46, 61, 84, 113, 167, 135, 114, 105, 69, 50, 41, 20, 17, 4, 2], "incorrect_counts": [2, 0, 5, 3, 8, 14, 28, 30, 47, 54, 54, 56, 39, 35, 29, 16, 3, 2, 2, 0]}, "energy/mean": {"edges": [0.5080794406433901, 0.5479599892472228, 0.5878405378510555, 0.6277210864548882, 0.6676016350587209, 0.7074821836625537, 0.7473627322663863, 0.787243280870219, 0.8271238294740517, 0.8670043780778844, 0.9068849266817172, 0.9467654752855499, 0.9866460238893826, 1.0265265724932153, 1.066407121097048, 1.1062876697008808, 1.1461682183047135, 1.1860487669085462, 1.225929315512379, 1.2658098641162117, 1.3056904127200444], "correct_counts": [2, 1, 2, 6, 18, 39, 57, 97, 121, 125, 130, 149, 116, 82, 52, 39, 17, 14, 4, 2], "incorrect_counts": [1, 1, 5, 6, 20, 34, 44, 48, 64, 69, 40, 40, 27, 15, 3, 2, 4, 3, 1, 0]}, "energy/min": {"edges": [0.20125961303710938, 0.2415154755115509, 0.2817713379859924, 0.322027200460434, 0.3622830629348755, 0.402538925409317, 0.4427947878837586, 0.4830506503582001, 0.5233065128326416, 0.5635623753070831, 0.6038182377815247, 0.6440741002559662, 0.6843299627304078, 0.7245858252048493, 0.7648416876792908, 0.8050975501537323, 0.8453534126281739, 0.8856092751026154, 0.9258651375770569, 0.9661210000514985, 1.00637686252594], "correct_counts": [2, 3, 12, 10, 23, 44, 81, 91, 107, 115, 120, 113, 109, 99, 49, 39, 25, 18, 8, 5], "incorrect_counts": [2, 1, 8, 14, 23, 30, 46, 53, 52, 51, 50, 29, 33, 17, 9, 4, 3, 2, 0, 0]}, "energy/std": {"edges": [0.06508645292670491, 0.08258405015316497, 0.10008164737962504, 0.1175792446060851, 0.13507684183254515, 0.15257443905900522, 0.1700720362854653, 0.18756963351192535, 0.2050672307383854, 0.22256482796484545, 0.24006242519130555, 0.2575600224177656, 0.27505761964422565, 0.2925552168706857, 0.3100528140971458, 0.32755041132360585, 0.3450480085500659, 0.36254560577652595, 0.380043203002986, 0.3975408002294461, 0.41503839745590615], "correct_counts": [3, 8, 30, 47, 85, 138, 177, 162, 170, 93, 57, 54, 33, 6, 6, 3, 0, 0, 0, 1], "incorrect_counts": [1, 3, 11, 15, 35, 59, 64, 53, 69, 43, 33, 21, 8, 4, 2, 4, 0, 1, 0, 1]}, "curv/lmax_mean": {"edges": [0.9988154868284861, 0.9990696673591931, 0.9993238478899001, 0.9995780284206072, 0.9998322089513143, 1.0000863894820213, 1.0003405700127284, 1.0005947505434354, 1.0008489310741424, 1.0011031116048494, 1.0013572921355565, 1.0016114726662635, 1.0018656531969705, 1.0021198337276775, 1.0023740142583848, 1.0026281947890918, 1.0028823753197988, 1.0031365558505059, 1.003390736381213, 1.00364491691192, 1.003899097442627], "correct_counts": [4, 18, 97, 334, 405, 123, 49, 25, 8, 4, 2, 2, 1, 0, 0, 1, 0, 0, 0, 0], "incorrect_counts": [1, 2, 18, 62, 138, 113, 51, 19, 11, 5, 4, 1, 1, 0, 0, 0, 0, 0, 0, 1]}, "curv/lmax_best": {"edges": [0.9955618977546692, 0.99611676633358, 0.9966716349124909, 0.9972265034914016, 0.9977813720703125, 0.9983362406492233, 0.9988911092281342, 0.999445977807045, 1.0000008463859558, 1.0005557149648667, 1.0011105835437775, 1.0016654521226882, 1.0022203207015992, 1.00277518928051, 1.0033300578594209, 1.0038849264383316, 1.0044397950172423, 1.0049946635961533, 1.005549532175064, 1.006104400753975, 1.0066592693328857], "correct_counts": [1, 0, 2, 5, 11, 19, 79, 651, 258, 21, 14, 8, 3, 0, 1, 0, 0, 0, 0, 0], "incorrect_counts": [1, 0, 1, 0, 1, 4, 19, 183, 173, 23, 12, 6, 0, 1, 0, 0, 0, 2, 0, 1]}, "curv/trace_mean": {"edges": [31.957900683085125, 31.961792182922366, 31.965683682759604, 31.969575182596845, 31.973466682434083, 31.977358182271324, 31.98124968210856, 31.985141181945803, 31.98903268178304, 31.992924181620282, 31.99681568145752, 32.000707181294764, 32.004598681132, 32.00849018096924, 32.01238168080648, 32.01627318064372, 32.02016468048096, 32.0240561803182, 32.027947680155435, 32.03183917999268, 32.03573067982992], "correct_counts": [3, 3, 7, 25, 38, 49, 99, 128, 127, 145, 164, 120, 66, 46, 20, 12, 9, 6, 4, 2], "incorrect_counts": [0, 1, 0, 0, 1, 4, 16, 25, 45, 51, 54, 72, 61, 46, 25, 16, 9, 0, 1, 0]}, "curv/trace_best": {"edges": [31.9183292388916, 31.92680730819702, 31.935285377502442, 31.94376344680786, 31.952241516113283, 31.9607195854187, 31.96919765472412, 31.97767572402954, 31.98615379333496, 31.994631862640382, 32.0031099319458, 32.01158800125122, 32.02006607055664, 32.02854413986206, 32.03702220916748, 32.0455002784729, 32.05397834777832, 32.06245641708374, 32.07093448638916, 32.07941255569458, 32.087890625], "correct_counts": [1, 0, 3, 4, 11, 21, 52, 128, 281, 358, 140, 45, 19, 4, 2, 2, 1, 0, 0, 1], "incorrect_counts": [0, 0, 0, 0, 1, 4, 5, 19, 77, 159, 98, 42, 15, 6, 0, 0, 1, 0, 0, 0]}, "dynamics/steps": {"edges": [1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7000000000000002, 1.75, 1.8, 1.85, 1.9, 1.9500000000000002, 2.0], "correct_counts": [1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/monotonic": {"edges": [0.6383333333333333, 0.645, 0.6516666666666666, 0.6583333333333333, 0.6649999999999999, 0.6716666666666666, 0.6783333333333333, 0.6849999999999999, 0.6916666666666667, 0.6983333333333333, 0.705, 0.7116666666666667, 0.7183333333333333, 0.725, 0.7316666666666666, 0.7383333333333333, 0.745, 0.7516666666666666, 0.7583333333333333, 0.7649999999999999, 0.7716666666666666], "correct_counts": [0, 2, 6, 12, 52, 94, 96, 161, 117, 142, 138, 65, 79, 37, 37, 21, 7, 5, 1, 1], "incorrect_counts": [1, 3, 8, 16, 36, 47, 54, 77, 64, 43, 40, 18, 11, 3, 3, 1, 1, 1, 0, 0]}, "dynamics/drop": {"edges": [13.273132185141245, 17.449723252654074, 21.626314320166905, 25.802905387679733, 29.97949645519256, 34.15608752270539, 38.33267859021822, 42.509269657731046, 46.68586072524388, 50.86245179275671, 55.03904286026954, 59.21563392778236, 63.392224995295194, 67.56881606280803, 71.74540713032086, 75.92199819783369, 80.09858926534652, 84.27518033285935, 88.45177140037218, 92.62836246788501, 96.80495353539784], "correct_counts": [35, 165, 236, 186, 132, 96, 75, 48, 29, 25, 19, 5, 10, 1, 3, 4, 2, 1, 0, 1], "incorrect_counts": [52, 120, 93, 70, 48, 20, 15, 1, 1, 2, 2, 2, 0, 0, 0, 1, 0, 0, 0, 0]}, "dynamics/residual": {"edges": [1.1351244548956554, 1.1498689969380698, 1.1646135389804841, 1.1793580810228985, 1.1941026230653127, 1.208847165107727, 1.2235917071501414, 1.2383362491925558, 1.2530807912349702, 1.2678253332773846, 1.282569875319799, 1.297314417362213, 1.3120589594046275, 1.3268035014470418, 1.3415480434894562, 1.3562925855318706, 1.371037127574285, 1.3857816696166991, 1.4005262116591135, 1.4152707537015279, 1.4300152957439423], "correct_counts": [2, 0, 5, 14, 20, 39, 68, 81, 95, 120, 140, 113, 122, 101, 58, 54, 17, 16, 4, 4], "incorrect_counts": [2, 1, 0, 5, 8, 16, 17, 34, 38, 34, 59, 59, 43, 41, 32, 16, 14, 6, 1, 1]}}}, "feature_ablation": {"full": 0.14828452971658584, "drop_basin": 0.1651095026755807, "basin_only": 0.21983508381035266, "drop_energy": 0.1476320101970345, "energy_only": 0.18812416405697532, "drop_curv": 0.1481249587041615, "curv_only": 0.17486262874155167, "drop_dynamics": 0.1564704847843828, "dynamics_only": 0.17978757695863193}, "ece_geometry": 0.048452947907751065, "accuracy_ood": 0.3546666666666667, "aurc_ood": {"geometry": 0.634405040392116, "rho_basin": 0.6277002900945788, "energy_min": 0.7263364569173539, "energy_mean": 0.728125986539371, "energy_std": 0.6284172651454782, "msp": 0.605943901894947, "temp_msp": 0.5966869880651136, "entropy": 0.6037165831832931, "softmax_learned": 0.5966414374218213, "geom_softmax": 0.6175133507470368}, "ood_ltt": {"alpha": 0.1, "lambda_hat": null, "selective_risk": 0.0, "coverage": 0.0, "risk_within_budget": true}, "ood_validity": {"target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "id_risk": [0.0, 0.0, 0.0, 0.07346938775510205, 0.15090090090090091, 0.2565217391304348], "ood_risk": [0.0, 0.0, 0.0, 0.6439169139465876, 0.6160804020100502, 0.635846372688478], "id_coverage": [0.0, 0.0, 0.0, 0.16333333333333333, 0.592, 0.92], "ood_coverage": [0.0, 0.0, 0.0, 0.22466666666666665, 0.6633333333333333, 0.9373333333333334]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "20d1395ffa7a", "timestamp": "2026-07-30T11:17:58.833773+00:00", "git_sha": "67dc7a2", "config_hash": "1c0de51a1390", "config": {"run": {"seed": 1, "task": "graph_planning", "notes": "graph adaptive-halting run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 25, "batch_size": 128, "lr": 0.001, "n_train": 8000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 800, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 800}, "task": {"graph_planning": {"n_nodes": 7, "ood_n_nodes": 10, "edge_prob": 0.4, "max_len": 4}}}, "task": "graph_planning", "split": "halting", "seed": 1, "metrics": {"k_restarts": 12, "steps": 50, "alpha": 0.1, "n_calib": 800, "n_test": 800, "final_train_loss": 0.2080264687538147, "tau_hat": 0.9167, "compute_used": 0.29487500000000005, "compute_saved": 0.705125, "halting_risk": 0.0025, "risk_within_budget": true, "full_accuracy": 0.68875, "halted_accuracy": 0.68875, "accuracy_drop": 0.0, "risk_monotone_in_tau": true, "calib_risk_by_tau": [0.47875, 0.47875, 0.47875, 0.4775, 0.4025, 0.4025, 0.335, 0.1975, 0.1975, 0.115, 0.01, 0.01], "calib_compute_by_tau": [0.0, 0.0, 0.0, 0.00027499999999999996, 0.016949999999999875, 0.016949999999999875, 0.039174999999999655, 0.11260000000000017, 0.11260000000000017, 0.16875000000000026, 0.31139999999999984, 0.31139999999999984], "tau_sweep": {"taus": [0.0833, 0.1667, 0.25, 0.3333, 0.4167, 0.5, 0.5833, 0.6667, 0.75, 0.8333, 0.9167, 1.0], "compute_used": [0.0, 0.0, 0.0, 0.000275, 0.017725, 0.017725, 0.03785, 0.11257500000000001, 0.11257500000000001, 0.165775, 0.29487500000000005, 0.29487500000000005], "accuracy": [0.47625, 0.47625, 0.47625, 0.47625, 0.5175, 0.5175, 0.56875, 0.64125, 0.64125, 0.66625, 0.68875, 0.68875], "disagreement": [0.47125, 0.47125, 0.47125, 0.47125, 0.405, 0.405, 0.33125, 0.1675, 0.1675, 0.08125, 0.0025, 0.0025]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "b04ffc85ccc3", "timestamp": "2026-07-30T11:17:59.455068+00:00", "git_sha": "67dc7a2", "config_hash": "c8e9150407ea", "config": {"run": {"seed": 1, "task": "arithmetic", "notes": "E1 selective-prediction main run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 7, "batch_size": 128, "lr": 0.001, "n_train": 6000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 1500, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 1500}, "task": {"arithmetic": {"modular": false, "n_operands": 6, "ood_n_operands": 6, "max_operand": 7, "ood_max_operand": 9}}}, "task": "arithmetic", "split": "selective", "seed": 1, "metrics": {"n_fit": 1500, "n_calib": 1500, "n_test": 1500, "k_restarts": 12, "objective": "basin_center", "sampler": "langevin", "feature_set": "base", "accuracy_id": 0.8093333333333333, "base_error": 0.19066666666666668, "final_train_loss": 0.9732173085212708, "feature_names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "aurc": {"geometry": 0.07534184983249899, "rho_basin": 0.10651266796902968, "energy_min": 0.19613627208106035, "energy_mean": 0.18655715870175713, "energy_std": 0.1850079661273594, "msp": 0.056516158696690806, "temp_msp": 0.06264464287256487, "entropy": 0.07361179491321607, "softmax_learned": 0.05595378790404033, "geom_softmax": 0.0522583408533078}, "temperature": 0.3758208374823171, "best_energy_baseline": "energy_std", "best_baseline": "msp", "delta_aurc_vs_energy_min": [0.12079442224856136, 0.09713330903618891, 0.14596535431672475], "delta_aurc_vs_best_energy": [0.10966611629486041, 0.08345583515537251, 0.135750761601739], "delta_aurc_vs_best_baseline": [-0.018825691135808183, -0.031267766857238775, -0.007356400682959861], "delta_aurc_geom_adds": [0.003695447050732527, 0.0006793574386630384, 0.006965369236499613], "geometry_wins": true, "geometry_wins_vs_baseline": false, "geometry_adds_over_softmax": true, "risk_coverage": {"geometry": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.06666666666666667, 0.03333333333333333, 0.022222222222222223, 0.03333333333333333, 0.04666666666666667, 0.03888888888888888, 0.038095238095238106, 0.05, 0.05555555555555555, 0.06333333333333335, 0.05757575757575758, 0.05277777777777778, 0.05128205128205128, 0.05, 0.05111111111111111, 0.052083333333333336, 0.052941176470588235, 0.05185185185185184, 0.05263157894736842, 0.051666666666666666, 0.055555555555555546, 0.05454545454545454, 0.05217391304347826, 0.05138888888888888, 0.04933333333333333, 0.05384615384615385, 0.05308641975308642, 0.055952380952380955, 0.0586206896551724, 0.06444444444444444, 0.06774193548387097, 0.06979166666666667, 0.07171717171717172, 0.07450980392156863, 0.07619047619047618, 0.08055555555555555, 0.0891891891891892, 0.09298245614035087, 0.10341880341880341, 0.11, 0.11626016260162614, 0.11984126984126996, 0.1263565891472868, 0.13636363636363635, 0.1474074074074074, 0.15434782608695652, 0.1617021276595746, 0.17013888888888898, 0.18095238095238095, 0.19066666666666668]}, "rho_basin": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.09414024975984632, 0.09414024975984626, 0.09414024975984618, 0.09414024975984615, 0.09414024975984613, 0.09414024975984611, 0.0941402497598461, 0.09414024975984608, 0.09414024975984608, 0.09414024975984607, 0.09414024975984607, 0.09414024975984607, 0.09414024975984607, 0.09414024975984606, 0.09414024975984606, 0.09414024975984606, 0.09414024975984606, 0.09414024975984606, 0.09414024975984606, 0.09414024975984606, 0.09414024975984606, 0.09414024975984606, 0.09414024975984604, 0.09414024975984604, 0.09414024975984604, 0.09414024975984604, 0.09414024975984604, 0.09414024975984604, 0.09414024975984604, 0.09414024975984604, 0.09414024975984604, 0.09414024975984604, 0.09414024975984604, 0.09414024975984604, 0.09550476190476165, 0.09988888888888856, 0.1040360360360356, 0.10796491228070125, 0.11169230769230713, 0.11560307017543801, 0.12016474112109492, 0.12450918964076814, 0.13166112956810566, 0.139383116883116, 0.14806314511232435, 0.15803041102399493, 0.16711125224939005, 0.17513474295190584, 0.18291074005359584, 0.19066666666666512]}, "energy_min": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.26666666666666666, 0.2833333333333333, 0.24444444444444444, 0.20833333333333334, 0.19333333333333333, 0.19444444444444442, 0.20000000000000004, 0.2, 0.1962962962962963, 0.1866666666666667, 0.18787878787878787, 0.2, 0.19743589743589743, 0.19285714285714287, 0.19555555555555554, 0.18958333333333333, 0.19215686274509805, 0.18888888888888886, 0.18596491228070175, 0.18833333333333332, 0.18888888888888886, 0.19545454545454546, 0.19130434782608696, 0.19305555555555554, 0.19333333333333333, 0.191025641025641, 0.18888888888888888, 0.18571428571428572, 0.18275862068965515, 0.18555555555555556, 0.1827956989247312, 0.18125, 0.18585858585858586, 0.18529411764705883, 0.18190476190476204, 0.1851851851851853, 0.18468468468468469, 0.1850877192982456, 0.18888888888888888, 0.18833333333333332, 0.18699186991869915, 0.18968253968253965, 0.18992248062015504, 0.19090909090909092, 0.19037037037037038, 0.19130434782608696, 0.19219858156028366, 0.19166666666666676, 0.19115646258503402, 0.19066666666666668]}, "energy_mean": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.16666666666666666, 0.15, 0.17777777777777778, 0.20833333333333334, 0.18, 0.17777777777777776, 0.17619047619047623, 0.19583333333333333, 0.1962962962962963, 0.20000000000000004, 0.19393939393939394, 0.18333333333333332, 0.1794871794871795, 0.17857142857142858, 0.18000000000000013, 0.17708333333333334, 0.1843137254901961, 0.1833333333333333, 0.18771929824561404, 0.19, 0.18888888888888886, 0.19545454545454546, 0.1927536231884058, 0.19166666666666665, 0.18666666666666668, 0.18461538461538463, 0.18518518518518517, 0.18571428571428572, 0.18965517241379307, 0.18555555555555556, 0.1870967741935484, 0.184375, 0.18282828282828284, 0.18529411764705883, 0.1876190476190476, 0.1898148148148148, 0.19099099099099098, 0.19210526315789472, 0.19145299145299147, 0.19, 0.1926829268292684, 0.19047619047619044, 0.1883720930232558, 0.18712121212121213, 0.18814814814814815, 0.19202898550724637, 0.19007092198581557, 0.19027777777777774, 0.19047619047619047, 0.19066666666666668]}, "energy_std": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.13333333333333333, 0.11666666666666667, 0.12222222222222222, 0.16666666666666666, 0.19333333333333333, 0.19444444444444442, 0.20952380952380958, 0.20833333333333334, 0.1962962962962963, 0.1833333333333332, 0.18484848484848485, 0.18333333333333332, 0.18974358974358974, 0.18095238095238095, 0.1844444444444444, 0.1875, 0.19019607843137254, 0.188888888888889, 0.18771929824561404, 0.185, 0.18730158730158727, 0.18636363636363637, 0.1855072463768116, 0.1847222222222222, 0.18666666666666668, 0.18846153846153846, 0.18888888888888888, 0.19166666666666668, 0.19195402298850572, 0.19111111111111112, 0.1913978494623656, 0.19375, 0.19494949494949496, 0.19411764705882353, 0.1933333333333333, 0.19259259259259257, 0.1900900900900901, 0.1868421052631579, 0.18717948717948718, 0.1875, 0.18780487804878046, 0.188888888888889, 0.19069767441860466, 0.1893939393939394, 0.18888888888888888, 0.1891304347826087, 0.18865248226950362, 0.1895833333333334, 0.19183673469387755, 0.19066666666666668]}, "msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.005555555555555555, 0.009523809523809526, 0.008333333333333333, 0.007407407407407408, 0.010000000000000002, 0.012121212121212121, 0.013888888888888888, 0.015384615384615385, 0.016666666666666666, 0.015555555555555552, 0.01875, 0.01764705882352941, 0.02037037037037037, 0.021052631578947368, 0.021666666666666667, 0.02222222222222222, 0.024242424242424242, 0.02608695652173913, 0.027777777777777773, 0.029333333333333333, 0.03333333333333333, 0.0345679012345679, 0.039285714285714285, 0.041379310344827766, 0.04888888888888889, 0.053763440860215055, 0.057291666666666664, 0.06363636363636363, 0.06862745098039216, 0.07809523809523808, 0.08425925925925924, 0.0936936936936937, 0.10087719298245613, 0.10598290598290598, 0.1125, 0.1195121951219512, 0.12857142857142853, 0.13953488372093023, 0.1462121212121212, 0.15333333333333332, 0.16014492753623188, 0.16595744680851074, 0.17500000000000007, 0.18299319727891156, 0.19066666666666668]}, "temp_msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.016666666666666666, 0.011111111111111112, 0.008333333333333333, 0.006666666666666667, 0.005555555555555555, 0.004761904761904763, 0.008333333333333333, 0.007407407407407408, 0.013333333333333336, 0.012121212121212121, 0.013888888888888888, 0.017948717948717947, 0.01904761904761905, 0.026666666666666665, 0.03125, 0.03529411764705882, 0.03518518518518518, 0.03684210526315789, 0.035, 0.03809523809523809, 0.03939393939393939, 0.042028985507246375, 0.041666666666666775, 0.04533333333333334, 0.047435897435897434, 0.05308641975308642, 0.05357142857142857, 0.06091954022988504, 0.06444444444444444, 0.06451612903225806, 0.065625, 0.07373737373737374, 0.0784313725490196, 0.0857142857142857, 0.0898148148148148, 0.0972972972972973, 0.10087719298245613, 0.10683760683760683, 0.11166666666666666, 0.11707317073170745, 0.12380952380952392, 0.1310077519379845, 0.14015151515151514, 0.14814814814814814, 0.1572463768115942, 0.1652482269503547, 0.1722222222222223, 0.1816326530612245, 0.19066666666666668]}, "entropy": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.005555555555555555, 0.014285714285714289, 0.016666666666666666, 0.014814814814814815, 0.020000000000000004, 0.024242424242424242, 0.027777777777777776, 0.03333333333333333, 0.04047619047619048, 0.039999999999999994, 0.041666666666666664, 0.047058823529411764, 0.04814814814814814, 0.04912280701754386, 0.05333333333333334, 0.053968253968253964, 0.05303030303030303, 0.05217391304347826, 0.056944444444444554, 0.06266666666666666, 0.06538461538461539, 0.06666666666666667, 0.06785714285714285, 0.07586206896551742, 0.08555555555555555, 0.08494623655913978, 0.09166666666666666, 0.09494949494949495, 0.1, 0.10285714285714284, 0.10648148148148147, 0.10990990990990991, 0.11578947368421053, 0.12051282051282051, 0.1275, 0.1333333333333333, 0.13571428571428581, 0.14573643410852713, 0.15, 0.1562962962962963, 0.16304347826086957, 0.16666666666666663, 0.17430555555555552, 0.18231292517006803, 0.19066666666666668]}, "softmax_learned": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.005555555555555555, 0.004761904761904763, 0.008333333333333333, 0.007407407407407408, 0.013333333333333336, 0.012121212121212121, 0.013888888888888888, 0.01282051282051282, 0.016666666666666666, 0.015555555555555552, 0.01875, 0.0196078431372549, 0.02037037037037037, 0.021052631578947368, 0.023333333333333334, 0.02222222222222222, 0.025757575757575757, 0.02608695652173913, 0.027777777777777773, 0.032, 0.03333333333333333, 0.03950617283950617, 0.04404761904761905, 0.045977011494252866, 0.04888888888888889, 0.05161290322580645, 0.058333333333333334, 0.06262626262626263, 0.06862745098039216, 0.07333333333333332, 0.08055555555555555, 0.0891891891891892, 0.09385964912280702, 0.09829059829059829, 0.11, 0.1227642276422764, 0.12936507936507935, 0.13488372093023257, 0.1409090909090909, 0.1511111111111111, 0.15942028985507245, 0.1659574468085106, 0.1736111111111112, 0.18231292517006803, 0.19066666666666668]}, "geom_softmax": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.003703703703703704, 0.003333333333333334, 0.006060606060606061, 0.008333333333333333, 0.010256410256410256, 0.014285714285714285, 0.015555555555555552, 0.016666666666666666, 0.01764705882352941, 0.016666666666666663, 0.01929824561403509, 0.02, 0.02063492063492063, 0.019696969696969695, 0.01884057971014493, 0.024999999999999998, 0.028, 0.029487179487179487, 0.02962962962962963, 0.030952380952380953, 0.03563218390804597, 0.043333333333333335, 0.046236559139784944, 0.052083333333333336, 0.0595959595959596, 0.06470588235294118, 0.06857142857142873, 0.07777777777777777, 0.08558558558558559, 0.09210526315789473, 0.09914529914529914, 0.10666666666666667, 0.11707317073170731, 0.12698412698412695, 0.1325581395348837, 0.13787878787878788, 0.14814814814814814, 0.15579710144927536, 0.16382978723404265, 0.17013888888888887, 0.17959183673469387, 0.19066666666666668]}}, "ltt": {"alpha": 0.1, "delta": 0.05, "lambda_hat": 0.1463238254140143, "coverage": 0.638, "abstain_rate": 0.362, "selective_risk": 0.07001044932079414, "selective_accuracy": 0.9299895506792059, "risk_within_budget": true}, "coverage_validity": {"alpha_grid": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "achieved_risk": [0.0, 0.0, 0.07001044932079414, 0.11437908496732026, 0.15036496350364964, 0.18904475617902472], "coverage": [0.0, 0.0, 0.638, 0.816, 0.9133333333333333, 0.998]}, "feature_diagnostics": {"names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "auroc": {"basin/rho": 0.7400893998917063, "basin/entropy": 0.25900623264708933, "basin/dispersion": 0.49065390951717147, "energy/mean": 0.4921515881153443, "energy/min": 0.4963422080390779, "energy/std": 0.49589866476192673, "curv/lmax_mean": 0.5486183914931856, "curv/lmax_best": 0.5253496503496503, "curv/trace_mean": 0.5459758528127556, "curv/trace_best": 0.5448453934862502, "dynamics/steps": 0.5, "dynamics/monotonic": 0.42523991659082266, "dynamics/drop": 0.39914862731996176, "dynamics/residual": 0.5023185216760175}, "hist": {"basin/rho": {"edges": [0.3333333333333333, 0.36666666666666664, 0.4, 0.43333333333333335, 0.4666666666666667, 0.5, 0.5333333333333333, 0.5666666666666667, 0.6000000000000001, 0.6333333333333333, 0.6666666666666667, 0.7000000000000001, 0.7333333333333334, 0.7666666666666667, 0.8, 0.8333333333333335, 0.8666666666666667, 0.9000000000000001, 0.9333333333333333, 0.9666666666666668, 1.0], "correct_counts": [1, 0, 0, 0, 0, 14, 0, 30, 0, 24, 0, 0, 37, 0, 53, 0, 0, 112, 0, 943], "incorrect_counts": [0, 0, 1, 0, 0, 19, 0, 37, 0, 37, 0, 0, 33, 0, 23, 0, 0, 38, 0, 98]}, "basin/entropy": {"edges": [0.0, 0.05493061443340548, 0.10986122886681096, 0.16479184330021646, 0.21972245773362192, 0.2746530721670274, 0.3295836866004329, 0.3845143010338384, 0.43944491546724385, 0.4943755299006493, 0.5493061443340548, 0.6042367587674603, 0.6591673732008658, 0.7140979876342712, 0.7690286020676768, 0.8239592165010822, 0.8788898309344877, 0.9338204453678932, 0.9887510598012986, 1.0436816742347041, 1.0986122886681096], "correct_counts": [943, 0, 0, 0, 0, 112, 0, 0, 53, 0, 37, 23, 38, 0, 0, 1, 5, 1, 0, 1], "incorrect_counts": [98, 0, 0, 0, 0, 38, 0, 0, 22, 0, 32, 33, 49, 2, 0, 4, 4, 1, 2, 1]}, "basin/dispersion": {"edges": [1.6209795073409872, 1.6420014502019364, 1.663023393062886, 1.6840453359238352, 1.7050672787847847, 1.726089221645734, 1.7471111645066832, 1.7681331073676327, 1.789155050228582, 1.8101769930895315, 1.8311989359504808, 1.8522208788114303, 1.8732428216723795, 1.8942647645333288, 1.9152867073942783, 1.9363086502552276, 1.9573305931161769, 1.9783525359771263, 1.9993744788380756, 2.020396421699025, 2.0414183645599744], "correct_counts": [2, 8, 14, 32, 47, 65, 98, 126, 152, 145, 133, 112, 97, 76, 42, 37, 11, 9, 6, 2], "incorrect_counts": [1, 2, 1, 6, 13, 15, 17, 35, 31, 30, 45, 28, 18, 18, 10, 6, 4, 3, 2, 1]}, "energy/mean": {"edges": [-0.2875423381725947, -0.26474122231205305, -0.2419401064515114, -0.21913899059096975, -0.19633787473042807, -0.1735367588698864, -0.15073564300934475, -0.1279345271488031, -0.10513341128826142, -0.08233229542771975, -0.0595311795671781, -0.03673006370663645, -0.013928947846094775, 0.008872168014446902, 0.03167328387498852, 0.0544743997355302, 0.07727551559607188, 0.10007663145661355, 0.12287774731715523, 0.14567886317769685, 0.16847997903823853], "correct_counts": [7, 20, 30, 58, 83, 135, 149, 163, 127, 126, 94, 86, 52, 33, 25, 14, 8, 4, 0, 0], "incorrect_counts": [1, 4, 6, 16, 21, 25, 40, 32, 31, 35, 26, 11, 19, 6, 6, 4, 2, 0, 0, 1]}, "energy/min": {"edges": [-0.7031644582748413, -0.6723987430334091, -0.641633027791977, -0.6108673125505447, -0.5801015973091126, -0.5493358820676804, -0.5185701668262481, -0.487804451584816, -0.45703873634338377, -0.4262730211019516, -0.3955073058605194, -0.36474159061908723, -0.33397587537765505, -0.3032101601362229, -0.27244444489479064, -0.24167872965335846, -0.21091301441192628, -0.1801472991704941, -0.14938158392906187, -0.11861586868762974, -0.08785015344619751], "correct_counts": [1, 3, 16, 12, 41, 73, 86, 119, 149, 155, 154, 128, 105, 75, 38, 31, 20, 3, 4, 1], "incorrect_counts": [0, 3, 4, 7, 8, 13, 20, 30, 32, 36, 31, 33, 27, 21, 11, 5, 0, 3, 2, 0]}, "energy/std": {"edges": [0.07042911362104956, 0.08610065979939398, 0.1017722059777384, 0.1174437521560828, 0.1331152983344272, 0.14878684451277163, 0.16445839069111604, 0.18012993686946044, 0.19580148304780487, 0.21147302922614927, 0.2271445754044937, 0.2428161215828381, 0.2584876677611825, 0.2741592139395269, 0.28983076011787134, 0.3055023062962158, 0.3211738524745602, 0.33684539865290464, 0.35251694483124896, 0.3681884910095934, 0.3838600371879378], "correct_counts": [3, 14, 19, 49, 89, 135, 163, 181, 155, 142, 98, 72, 49, 20, 3, 8, 8, 2, 1, 3], "incorrect_counts": [0, 3, 4, 9, 30, 27, 34, 45, 44, 20, 30, 17, 11, 8, 3, 0, 0, 1, 0, 0]}, "curv/lmax_mean": {"edges": [1.0000324149926503, 1.0006146644552547, 1.0011969139178594, 1.0017791633804638, 1.0023614128430685, 1.002943662305673, 1.0035259117682773, 1.004108161230882, 1.0046904106934864, 1.005272660156091, 1.0058549096186955, 1.0064371590813, 1.0070194085439046, 1.007601658006509, 1.0081839074691137, 1.0087661569317181, 1.0093484063943226, 1.0099306558569272, 1.0105129053195316, 1.0110951547821363, 1.0116774042447407], "correct_counts": [561, 240, 130, 87, 59, 46, 26, 18, 12, 9, 4, 7, 8, 1, 2, 2, 0, 0, 1, 1], "incorrect_counts": [154, 60, 27, 18, 11, 6, 4, 1, 1, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0]}, "curv/lmax_best": {"edges": [1.0, 1.0020527243614197, 1.0041054487228394, 1.006158173084259, 1.0082108974456787, 1.0102636218070984, 1.012316346168518, 1.0143690705299377, 1.0164217948913574, 1.018474519252777, 1.0205272436141968, 1.0225799679756165, 1.0246326923370361, 1.0266854166984558, 1.0287381410598755, 1.0307908654212952, 1.0328435897827148, 1.0348963141441345, 1.0369490385055542, 1.0390017628669739, 1.0410544872283936], "correct_counts": [1013, 104, 42, 23, 9, 5, 3, 4, 3, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 1], "incorrect_counts": [254, 19, 7, 3, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "curv/trace_mean": {"edges": [32.001986503601074, 32.00630712509155, 32.01062774658203, 32.01494836807251, 32.01926898956299, 32.02358961105347, 32.027910232543945, 32.032230854034424, 32.0365514755249, 32.04087209701538, 32.04519271850586, 32.04951333999634, 32.053833961486816, 32.058154582977295, 32.06247520446777, 32.06679582595825, 32.07111644744873, 32.07543706893921, 32.07975769042969, 32.084078311920166, 32.088398933410645], "correct_counts": [158, 234, 163, 124, 97, 88, 75, 63, 68, 49, 38, 15, 18, 8, 9, 2, 2, 0, 1, 2], "incorrect_counts": [35, 70, 54, 25, 19, 20, 20, 15, 10, 9, 2, 2, 2, 1, 1, 0, 0, 0, 0, 1]}, "curv/trace_best": {"edges": [32.00096893310547, 32.00789184570313, 32.01481475830078, 32.02173767089844, 32.02866058349609, 32.03558349609375, 32.04250640869141, 32.04942932128906, 32.05635223388672, 32.06327514648437, 32.07019805908203, 32.07712097167969, 32.08404388427734, 32.090966796875, 32.09788970947265, 32.10481262207031, 32.11173553466797, 32.118658447265624, 32.12558135986328, 32.132504272460935, 32.139427185058594], "correct_counts": [338, 281, 171, 124, 100, 56, 36, 33, 22, 17, 9, 7, 3, 5, 4, 1, 4, 2, 0, 1], "incorrect_counts": [81, 87, 43, 31, 12, 11, 5, 6, 4, 1, 1, 1, 2, 1, 0, 0, 0, 0, 0, 0]}, "dynamics/steps": {"edges": [1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7000000000000002, 1.75, 1.8, 1.85, 1.9, 1.9500000000000002, 2.0], "correct_counts": [1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/monotonic": {"edges": [0.7166666666666667, 0.7220833333333333, 0.7275, 0.7329166666666667, 0.7383333333333334, 0.74375, 0.7491666666666668, 0.7545833333333334, 0.7600000000000001, 0.7654166666666667, 0.7708333333333335, 0.7762500000000001, 0.7816666666666667, 0.7870833333333335, 0.7925000000000001, 0.7979166666666668, 0.8033333333333335, 0.8087500000000002, 0.8141666666666668, 0.8195833333333336, 0.8250000000000002], "correct_counts": [1, 2, 5, 25, 29, 69, 103, 144, 199, 162, 148, 138, 95, 45, 25, 11, 6, 3, 2, 2], "incorrect_counts": [0, 0, 2, 7, 9, 13, 14, 39, 30, 31, 29, 22, 25, 20, 17, 7, 10, 5, 4, 2]}, "dynamics/drop": {"edges": [62.5166342407465, 77.29895427798232, 92.08127431521814, 106.86359435245394, 121.64591438968976, 136.4282344269256, 151.2105544641614, 165.9928745013972, 180.77519453863303, 195.55751457586885, 210.33983461310467, 225.1221546503405, 239.90447468757628, 254.6867947248121, 269.4691147620479, 284.25143479928374, 299.03375483651956, 313.8160748737554, 328.5983949109912, 343.380714948227, 358.16303498546284], "correct_counts": [17, 72, 212, 278, 288, 178, 67, 37, 30, 12, 6, 4, 2, 6, 2, 2, 0, 1, 0, 0], "incorrect_counts": [2, 9, 39, 66, 39, 38, 24, 11, 4, 7, 10, 8, 5, 7, 4, 7, 2, 1, 2, 1]}, "dynamics/residual": {"edges": [1.1406713624795277, 1.1573103273908296, 1.1739492923021315, 1.1905882572134334, 1.2072272221247355, 1.2238661870360374, 1.2405051519473393, 1.2571441168586412, 1.2737830817699431, 1.290422046681245, 1.307061011592547, 1.323699976503849, 1.340338941415151, 1.3569779063264529, 1.3736168712377548, 1.3902558361490567, 1.4068948010603588, 1.4235337659716607, 1.4401727308829626, 1.4568116957942645, 1.4734506607055664], "correct_counts": [1, 6, 14, 25, 53, 88, 111, 153, 153, 165, 149, 107, 96, 53, 25, 8, 5, 1, 0, 1], "incorrect_counts": [0, 1, 0, 4, 10, 23, 35, 34, 38, 36, 45, 21, 17, 7, 9, 4, 1, 0, 1, 0]}}}, "feature_ablation": {"full": 0.07534184983249899, "drop_basin": 0.15063825991789662, "basin_only": 0.10286261887290694, "drop_energy": 0.08021799152104715, "energy_only": 0.19386361374456082, "drop_curv": 0.07785615271208078, "curv_only": 0.16390990398067096, "drop_dynamics": 0.08761931255420763, "dynamics_only": 0.15113347189816922}, "ece_geometry": 0.027494786480900396, "accuracy_ood": 0.20933333333333334, "aurc_ood": {"geometry": 0.7968967969651003, "rho_basin": 0.7527208068366574, "energy_min": 0.7530102393631208, "energy_mean": 0.7621969907261918, "energy_std": 0.7900061103185938, "msp": 0.6957160882123623, "temp_msp": 0.6999530402417576, "entropy": 0.698412184022287, "softmax_learned": 0.6954866419202574, "geom_softmax": 0.7147067020732829}, "ood_ltt": {"alpha": 0.1, "lambda_hat": 0.1463238254140143, "selective_risk": 0.75, "coverage": 0.616, "risk_within_budget": false}, "ood_validity": {"target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "id_risk": [0.0, 0.0, 0.07001044932079414, 0.11437908496732026, 0.15036496350364964, 0.18904475617902472], "ood_risk": [0.0, 0.0, 0.75, 0.7721311475409836, 0.7871428571428571, 0.7906666666666666], "id_coverage": [0.0, 0.0, 0.638, 0.816, 0.9133333333333333, 0.998], "ood_coverage": [0.0, 0.0, 0.616, 0.8133333333333334, 0.9333333333333333, 1.0]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "354783efb5d1", "timestamp": "2026-07-30T11:18:04.745595+00:00", "git_sha": "67dc7a2", "config_hash": "3c2fcb2e5dde", "config": {"run": {"seed": 2, "task": "arithmetic", "notes": "E1 selective-prediction main run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 7, "batch_size": 128, "lr": 0.001, "n_train": 6000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 1500, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 1500}, "task": {"arithmetic": {"modular": false, "n_operands": 6, "ood_n_operands": 6, "max_operand": 7, "ood_max_operand": 9}}}, "task": "arithmetic", "split": "selective", "seed": 2, "metrics": {"n_fit": 1500, "n_calib": 1500, "n_test": 1500, "k_restarts": 12, "objective": "basin_center", "sampler": "langevin", "feature_set": "base", "accuracy_id": 0.8713333333333333, "base_error": 0.12866666666666668, "final_train_loss": 0.9444285035133362, "feature_names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "aurc": {"geometry": 0.024274182941440476, "rho_basin": 0.05598821987398549, "energy_min": 0.04412798365125574, "energy_mean": 0.042606873866458325, "energy_std": 0.1207663666916667, "msp": 0.024891583299795343, "temp_msp": 0.027860819750315315, "entropy": 0.032137841946629477, "softmax_learned": 0.02315351961620342, "geom_softmax": 0.021598978976821472}, "temperature": 0.2805001582613758, "best_energy_baseline": "energy_mean", "best_baseline": "msp", "delta_aurc_vs_energy_min": [0.019853800709815263, 0.00990417624353011, 0.032291575515518305], "delta_aurc_vs_best_energy": [0.01833269092501785, 0.007161408501466717, 0.03183435108421057], "delta_aurc_vs_best_baseline": [0.0006174003583548672, -0.003978080304717403, 0.004496550989570287], "delta_aurc_geom_adds": [0.0015545406393819468, -1.6751835250081416e-05, 0.003164056614505232], "geometry_wins": true, "geometry_wins_vs_baseline": false, "geometry_adds_over_softmax": false, "risk_coverage": {"geometry": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.011111111111111112, 0.008333333333333333, 0.006666666666666667, 0.005555555555555555, 0.004761904761904763, 0.004166666666666667, 0.003703703703703704, 0.003333333333333334, 0.006060606060606061, 0.005555555555555556, 0.007692307692307693, 0.007142857142857143, 0.006666666666666666, 0.00625, 0.00784313725490196, 0.007407407407407407, 0.007017543859649123, 0.006666666666666667, 0.006349206349206348, 0.006060606060606061, 0.008695652173913044, 0.008333333333333331, 0.008, 0.008974358974358974, 0.008641975308641974, 0.008333333333333333, 0.008045977011494251, 0.008888888888888889, 0.00967741935483871, 0.0125, 0.014141414141414142, 0.014705882352941176, 0.016190476190476345, 0.018518518518518514, 0.021621621621621623, 0.02894736842105263, 0.03247863247863248, 0.04, 0.04552845528455284, 0.054761904761904755, 0.06434108527131784, 0.07348484848484849, 0.08148148148148149, 0.09130434782608696, 0.10283687943262422, 0.11180555555555555, 0.11972789115646258, 0.12866666666666668]}, "rho_basin": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0470588235294118, 0.04705882352941174, 0.047058823529411695, 0.047058823529411674, 0.04705882352941166, 0.047058823529411646, 0.04705882352941164, 0.04705882352941163, 0.04705882352941163, 0.047058823529411625, 0.047058823529411625, 0.047058823529411625, 0.047058823529411625, 0.047058823529411625, 0.04705882352941162, 0.04705882352941162, 0.04705882352941162, 0.04705882352941162, 0.04705882352941162, 0.04705882352941162, 0.04705882352941161, 0.04705882352941161, 0.04705882352941162, 0.04705882352941162, 0.04705882352941162, 0.04705882352941162, 0.04705882352941162, 0.04705882352941161, 0.04705882352941161, 0.04705882352941161, 0.04705882352941161, 0.04705882352941161, 0.04705882352941161, 0.04705882352941161, 0.04705882352941161, 0.04705882352941161, 0.047714381047714215, 0.05152696556205304, 0.05514403292181039, 0.05858024691357986, 0.0618488407106289, 0.06746031746031694, 0.07403100775193731, 0.08030303030302952, 0.08962962962962887, 0.09840579710144855, 0.10482269503546043, 0.11097222222222178, 0.11687074829931939, 0.1286666666666665]}, "energy_min": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.06666666666666667, 0.03333333333333333, 0.022222222222222223, 0.016666666666666666, 0.02, 0.027777777777777773, 0.03333333333333334, 0.029166666666666667, 0.02962962962962963, 0.026666666666666672, 0.024242424242424242, 0.022222222222222223, 0.02564102564102564, 0.023809523809523808, 0.031111111111111103, 0.029166666666666667, 0.027450980392156862, 0.027777777777777773, 0.028070175438596492, 0.028333333333333332, 0.028571428571428567, 0.030303030303030304, 0.030434782608695653, 0.03055555555555555, 0.030666666666666665, 0.03205128205128205, 0.03333333333333333, 0.03214285714285714, 0.033333333333333326, 0.03666666666666667, 0.03978494623655914, 0.03854166666666667, 0.03939393939393939, 0.04215686274509804, 0.041904761904761896, 0.04444444444444444, 0.047747747747747746, 0.05087719298245614, 0.0547008547008547, 0.06, 0.06422764227642275, 0.06587301587301586, 0.06976744186046512, 0.07803030303030303, 0.08518518518518518, 0.09565217391304348, 0.10212765957446807, 0.10972222222222232, 0.11836734693877551, 0.12866666666666668]}, "energy_mean": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.016666666666666666, 0.022222222222222223, 0.025, 0.02666666666666667, 0.027777777777777773, 0.028571428571428577, 0.025, 0.022222222222222223, 0.023333333333333338, 0.024242424242424242, 0.022222222222222223, 0.023076923076923078, 0.02142857142857143, 0.019999999999999997, 0.022916666666666665, 0.021568627450980392, 0.02037037037037037, 0.021052631578947368, 0.02, 0.02063492063492063, 0.022727272727272728, 0.02463768115942029, 0.027777777777777773, 0.02666666666666667, 0.026923076923076925, 0.027160493827160494, 0.030952380952380953, 0.032183908045977004, 0.03111111111111111, 0.03225806451612903, 0.035416666666666666, 0.03636363636363636, 0.0392156862745098, 0.04095238095238095, 0.043518518518518665, 0.04594594594594595, 0.04912280701754386, 0.05042735042735043, 0.058333333333333334, 0.06341463414634145, 0.06825396825396837, 0.07364341085271318, 0.0803030303030303, 0.08444444444444445, 0.09202898550724638, 0.09929078014184409, 0.10624999999999998, 0.11904761904761904, 0.12866666666666668]}, "energy_std": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.1, 0.15, 0.13333333333333333, 0.1, 0.12, 0.11111111111111109, 0.11428571428571431, 0.11666666666666667, 0.11851851851851852, 0.11333333333333334, 0.11818181818181818, 0.11944444444444445, 0.12564102564102564, 0.12142857142857143, 0.12222222222222219, 0.12291666666666666, 0.11960784313725491, 0.11666666666666665, 0.11228070175438597, 0.11, 0.11111111111111122, 0.11212121212121212, 0.11014492753623188, 0.11527777777777777, 0.12266666666666666, 0.12307692307692308, 0.12345679012345678, 0.12380952380952381, 0.12298850574712641, 0.12555555555555556, 0.12365591397849462, 0.12395833333333334, 0.12323232323232323, 0.12254901960784313, 0.12285714285714285, 0.12314814814814813, 0.12162162162162163, 0.12192982456140351, 0.12478632478632479, 0.12583333333333332, 0.12439024390243901, 0.1238095238095238, 0.12713178294573643, 0.1303030303030303, 0.1288888888888889, 0.1289855072463768, 0.1283687943262411, 0.12777777777777774, 0.12857142857142856, 0.12866666666666668]}, "msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.002222222222222222, 0.0020833333333333333, 0.00196078431372549, 0.0018518518518518517, 0.0017543859649122807, 0.0016666666666666668, 0.001587301587301587, 0.0015151515151515152, 0.0014492753623188406, 0.0027777777777777775, 0.004, 0.0038461538461538464, 0.0049382716049382715, 0.007142857142857143, 0.008045977011494251, 0.008888888888888889, 0.00967741935483871, 0.016666666666666666, 0.022222222222222223, 0.024509803921568627, 0.030476190476190473, 0.03703703703703703, 0.043243243243243246, 0.04736842105263158, 0.05042735042735043, 0.059166666666666666, 0.060162601626016256, 0.06507936507936507, 0.07131782945736434, 0.07803030303030303, 0.08592592592592592, 0.09492753623188406, 0.10070921985815602, 0.10833333333333343, 0.11904761904761904, 0.12866666666666668]}, "temp_msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004761904761904565, 0.004166666666666667, 0.003703703703703704, 0.006666666666666668, 0.006060606060606061, 0.008333333333333333, 0.010256410256410256, 0.009523809523809525, 0.01111111111111111, 0.010416666666666666, 0.00980392156862745, 0.009259259259259257, 0.008771929824561403, 0.008333333333333333, 0.007936507936507934, 0.007575757575757576, 0.008695652173913044, 0.008333333333333331, 0.008, 0.010256410256410256, 0.013580246913580247, 0.015476190476190477, 0.018390804597701337, 0.02, 0.02258064516129032, 0.022916666666666665, 0.024242424242424242, 0.024509803921568627, 0.026666666666666665, 0.029629629629629627, 0.03423423423423423, 0.03859649122807018, 0.04358974358974359, 0.0525, 0.05609756097560988, 0.06269841269841268, 0.06666666666666667, 0.07424242424242425, 0.0837037037037037, 0.0927536231884058, 0.10496453900709218, 0.11597222222222221, 0.11972789115646258, 0.12866666666666668]}, "entropy": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004761904761904762, 0.004444444444444444, 0.010416666666666666, 0.00980392156862745, 0.009259259259259257, 0.010526315789473684, 0.013333333333333334, 0.012698412698412697, 0.015151515151515152, 0.014492753623188406, 0.016666666666666663, 0.018666666666666668, 0.02435897435897436, 0.030864197530864196, 0.034523809523809526, 0.03448275862068965, 0.03666666666666667, 0.04086021505376344, 0.041666666666666664, 0.04040404040404041, 0.041176470588235294, 0.04476190476190475, 0.045370370370370366, 0.04864864864864865, 0.05, 0.05128205128205128, 0.056666666666666664, 0.060162601626016256, 0.06825396825396837, 0.07441860465116279, 0.07803030303030303, 0.08592592592592592, 0.09130434782608696, 0.09716312056737587, 0.10763888888888888, 0.11700680272108843, 0.12866666666666668]}, "softmax_learned": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.8462688879574732e-16, 0.0020833333333333333, 0.00196078431372549, 0.0018518518518518517, 0.0017543859649122807, 0.0016666666666666668, 0.001587301587301587, 0.0015151515151515152, 0.0014492753623188406, 0.0013888888888888887, 0.0026666666666666666, 0.0038461538461538464, 0.003703703703703704, 0.005952380952380952, 0.005747126436781608, 0.006666666666666667, 0.007526881720430108, 0.010416666666666666, 0.01616161616161616, 0.018627450980392157, 0.023809523809523805, 0.0287037037037037, 0.03153153153153153, 0.03508771929824561, 0.042735042735042736, 0.05, 0.05609756097560975, 0.06587301587301586, 0.06976744186046512, 0.07803030303030303, 0.08592592592592592, 0.09420289855072464, 0.10141843971631216, 0.11249999999999999, 0.12108843537414966, 0.12866666666666668]}, "geom_softmax": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.002380952380952381, 0.002222222222222222, 0.0020833333333333333, 0.00196078431372549, 0.0037037037037037034, 0.0035087719298245615, 0.0033333333333333335, 0.003174603174603174, 0.0030303030303030303, 0.004347826086956522, 0.004166666666666666, 0.004, 0.0038461538461538464, 0.0049382716049382715, 0.005952380952380952, 0.005747126436781608, 0.006666666666666667, 0.0064516129032258064, 0.007291666666666667, 0.010101010101010102, 0.012745098039215686, 0.016190476190476186, 0.01851851851851867, 0.024324324324324326, 0.028070175438596492, 0.03247863247863248, 0.04416666666666667, 0.05040650406504078, 0.05873015873015872, 0.06511627906976744, 0.07196969696969698, 0.08074074074074074, 0.09130434782608696, 0.10283687943262422, 0.11041666666666665, 0.12040816326530612, 0.12866666666666668]}}, "ltt": {"alpha": 0.1, "delta": 0.05, "lambda_hat": 0.42350354070874574, "coverage": 0.8913333333333333, "abstain_rate": 0.10866666666666666, "selective_risk": 0.07778608825729244, "selective_accuracy": 0.9222139117427075, "risk_within_budget": true}, "coverage_validity": {"alpha_grid": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "achieved_risk": [0.0, 0.03186907838070629, 0.07778608825729244, 0.12750333778371162, 0.12750333778371162, 0.12750333778371162], "coverage": [0.0, 0.774, 0.8913333333333333, 0.9986666666666667, 0.9986666666666667, 0.9986666666666667]}, "feature_diagnostics": {"names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "auroc": {"basin/rho": 0.7907897292775846, "basin/entropy": 0.2093569500220019, "basin/dispersion": 0.4859485195301505, "energy/mean": 0.16196962549206942, "energy/min": 0.17429465096273156, "energy/std": 0.47604964896075735, "curv/lmax_mean": 0.43994671973550153, "curv/lmax_best": 0.5017086156249133, "curv/trace_mean": 0.4422876420707946, "curv/trace_best": 0.4850169077625064, "dynamics/steps": 0.5, "dynamics/monotonic": 0.2952654300676707, "dynamics/drop": 0.2783695604774609, "dynamics/residual": 0.46715374765610446}, "hist": {"basin/rho": {"edges": [0.4166666666666667, 0.44583333333333336, 0.47500000000000003, 0.5041666666666667, 0.5333333333333333, 0.5625, 0.5916666666666667, 0.6208333333333333, 0.65, 0.6791666666666667, 0.7083333333333333, 0.7375, 0.7666666666666666, 0.7958333333333334, 0.825, 0.8541666666666666, 0.8833333333333333, 0.9125, 0.9416666666666667, 0.9708333333333332, 1.0], "correct_counts": [0, 0, 4, 0, 0, 27, 0, 0, 33, 0, 0, 29, 0, 0, 52, 0, 0, 109, 0, 1053], "incorrect_counts": [2, 0, 16, 0, 0, 18, 0, 0, 22, 0, 0, 29, 0, 0, 28, 0, 0, 26, 0, 52]}, "basin/entropy": {"edges": [0.0, 0.05387781635334003, 0.10775563270668007, 0.16163344906002008, 0.21551126541336013, 0.2693890817667002, 0.32326689812004017, 0.3771447144733802, 0.43102253082672026, 0.4849003471800603, 0.5387781635334004, 0.5926559798867403, 0.6465337962400803, 0.7004116125934204, 0.7542894289467604, 0.8081672453001005, 0.8620450616534405, 0.9159228780067805, 0.9698006943601206, 1.0236785107134607, 1.0775563270668007], "correct_counts": [1053, 0, 0, 0, 0, 109, 0, 0, 50, 0, 30, 30, 28, 1, 0, 3, 3, 0, 0, 0], "incorrect_counts": [52, 0, 0, 0, 0, 26, 0, 0, 27, 0, 28, 20, 32, 2, 0, 1, 1, 1, 1, 2]}, "basin/dispersion": {"edges": [1.5319602421529375, 1.557597310257325, 1.5832343783617129, 1.6088714464661005, 1.6345085145704883, 1.6601455826748759, 1.6857826507792635, 1.7114197188836513, 1.737056786988039, 1.7626938550924267, 1.7883309231968143, 1.813967991301202, 1.8396050594055897, 1.8652421275099773, 1.8908791956143651, 1.9165162637187527, 1.9421533318231403, 1.9677903999275281, 1.9934274680319157, 2.0190645361363035, 2.044701604240691], "correct_counts": [0, 0, 0, 4, 13, 13, 50, 72, 111, 158, 164, 177, 192, 126, 117, 61, 28, 16, 2, 3], "incorrect_counts": [1, 0, 0, 0, 2, 3, 3, 11, 21, 19, 23, 25, 26, 27, 10, 9, 6, 4, 2, 1]}, "energy/mean": {"edges": [-0.30340700844923657, -0.2387943930923939, -0.1741817777355512, -0.10956916237870853, -0.044956547021865856, 0.01965606833497685, 0.0842686836918195, 0.14888129904866215, 0.21349391440550486, 0.27810652976234757, 0.34271914511919027, 0.40733176047603287, 0.4719443758328756, 0.5365569911897183, 0.6011696065465608, 0.6657822219034035, 0.7303948372602462, 0.7950074526170889, 0.8596200679739316, 0.9242326833307744, 0.988845298687617], "correct_counts": [2, 39, 170, 267, 275, 152, 74, 55, 57, 38, 33, 34, 23, 33, 19, 19, 10, 5, 1, 1], "incorrect_counts": [1, 0, 5, 5, 10, 9, 10, 8, 8, 13, 13, 14, 10, 21, 15, 16, 21, 11, 2, 1]}, "energy/min": {"edges": [-0.6361721754074097, -0.5738012850284576, -0.5114303946495056, -0.4490595042705536, -0.3866886138916016, -0.32431772351264954, -0.2619468331336975, -0.1995759427547455, -0.13720505237579345, -0.07483416199684145, -0.012463271617889404, 0.049907618761062644, 0.11227850914001469, 0.17464939951896674, 0.23702028989791868, 0.2993911802768707, 0.3617620706558228, 0.4241329610347748, 0.48650385141372676, 0.5488747417926789, 0.6112456321716309], "correct_counts": [10, 45, 106, 187, 224, 203, 140, 82, 53, 42, 45, 31, 38, 17, 25, 23, 16, 10, 3, 7], "incorrect_counts": [0, 2, 1, 5, 8, 11, 10, 7, 8, 9, 14, 8, 8, 18, 23, 13, 18, 17, 8, 5]}, "energy/std": {"edges": [0.09172678671306, 0.10609920482521892, 0.12047162293737784, 0.13484404104953673, 0.14921645916169568, 0.16358887727385457, 0.1779612953860135, 0.1923337134981724, 0.20670613161033133, 0.22107854972249025, 0.23545096783464917, 0.24982338594680809, 0.264195804058967, 0.27856822217112587, 0.2929406402832848, 0.30731305839544376, 0.32168547650760265, 0.33605789461976154, 0.3504303127319205, 0.3648027308440794, 0.37917514895623833], "correct_counts": [11, 25, 52, 107, 109, 155, 166, 148, 140, 133, 93, 55, 50, 25, 15, 12, 5, 3, 1, 2], "incorrect_counts": [0, 6, 6, 12, 19, 19, 20, 28, 17, 23, 17, 9, 5, 5, 3, 0, 2, 1, 0, 1]}, "curv/lmax_mean": {"edges": [1.0000314265489578, 1.0005539747575918, 1.001076522966226, 1.00159907117486, 1.002121619383494, 1.002644167592128, 1.0031667158007622, 1.0036892640093962, 1.0042118122180304, 1.0047343604266643, 1.0052569086352983, 1.0057794568439324, 1.0063020050525664, 1.0068245532612006, 1.0073471014698345, 1.0078696496784687, 1.0083921978871027, 1.0089147460957368, 1.0094372943043708, 1.009959842513005, 1.010482390721639], "correct_counts": [106, 240, 245, 181, 115, 97, 84, 74, 42, 38, 24, 22, 11, 10, 5, 4, 6, 1, 1, 1], "incorrect_counts": [15, 28, 24, 21, 25, 22, 14, 12, 9, 7, 3, 3, 3, 1, 2, 2, 0, 0, 1, 1]}, "curv/lmax_best": {"edges": [0.9999086856842041, 1.0016137421131135, 1.0033187985420227, 1.005023854970932, 1.0067289113998412, 1.0084339678287506, 1.01013902425766, 1.0118440806865692, 1.0135491371154786, 1.0152541935443877, 1.0169592499732971, 1.0186643064022065, 1.0203693628311157, 1.022074419260025, 1.0237794756889342, 1.0254845321178436, 1.027189588546753, 1.0288946449756622, 1.0305997014045716, 1.0323047578334807, 1.0340098142623901], "correct_counts": [815, 192, 101, 73, 40, 24, 16, 15, 5, 8, 5, 3, 1, 1, 3, 1, 1, 0, 1, 2], "incorrect_counts": [114, 36, 14, 7, 9, 2, 2, 2, 2, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0]}, "curv/trace_mean": {"edges": [31.999862353007, 32.003960879643756, 32.008059406280516, 32.012157932917276, 32.016256459554036, 32.020354986190796, 32.024453512827556, 32.028552039464316, 32.032650566101076, 32.036749092737836, 32.040847619374595, 32.044946146011355, 32.049044672648115, 32.053143199284875, 32.05724172592163, 32.06134025255839, 32.06543877919515, 32.06953730583191, 32.07363583246867, 32.07773435910543, 32.08183288574219], "correct_counts": [1, 2, 41, 105, 113, 142, 143, 135, 129, 115, 82, 90, 71, 49, 50, 16, 10, 7, 3, 3], "incorrect_counts": [1, 4, 10, 4, 18, 13, 15, 16, 19, 13, 16, 20, 10, 14, 5, 9, 4, 2, 0, 0]}, "curv/trace_best": {"edges": [31.99859619140625, 32.006431579589844, 32.01426696777344, 32.02210235595703, 32.029937744140625, 32.03777313232422, 32.04560852050781, 32.053443908691406, 32.061279296875, 32.069114685058594, 32.07695007324219, 32.08478546142578, 32.092620849609375, 32.10045623779297, 32.10829162597656, 32.116127014160156, 32.12396240234375, 32.131797790527344, 32.13963317871094, 32.14746856689453, 32.155303955078125], "correct_counts": [38, 192, 243, 218, 177, 139, 87, 80, 37, 35, 10, 14, 13, 9, 7, 2, 2, 1, 1, 2], "incorrect_counts": [9, 27, 33, 30, 22, 19, 14, 9, 7, 9, 5, 2, 3, 1, 0, 3, 0, 0, 0, 0]}, "dynamics/steps": {"edges": [1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7000000000000002, 1.75, 1.8, 1.85, 1.9, 1.9500000000000002, 2.0], "correct_counts": [1307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/monotonic": {"edges": [0.7233333333333333, 0.7286666666666666, 0.734, 0.7393333333333333, 0.7446666666666666, 0.75, 0.7553333333333333, 0.7606666666666666, 0.766, 0.7713333333333333, 0.7766666666666666, 0.782, 0.7873333333333333, 0.7926666666666667, 0.798, 0.8033333333333333, 0.8086666666666668, 0.8140000000000001, 0.8193333333333334, 0.8246666666666668, 0.8300000000000001], "correct_counts": [5, 6, 15, 32, 65, 152, 180, 167, 199, 166, 163, 72, 43, 30, 6, 3, 3, 0, 0, 0], "incorrect_counts": [0, 0, 0, 3, 3, 18, 10, 13, 15, 23, 23, 16, 25, 13, 8, 12, 7, 0, 3, 1]}, "dynamics/drop": {"edges": [64.10532930493355, 78.80553761435051, 93.50574592376749, 108.20595423318446, 122.90616254260144, 137.60637085201842, 152.30657916143537, 167.00678747085234, 181.70699578026932, 196.40720408968627, 211.10741239910325, 225.80762070852023, 240.50782901793718, 255.20803732735416, 269.90824563677114, 284.6084539461881, 299.3086622556051, 314.00887056502205, 328.709078874439, 343.409287183856, 358.10949549327296], "correct_counts": [9, 55, 217, 358, 369, 180, 66, 19, 11, 10, 8, 3, 0, 2, 0, 0, 0, 0, 0, 0], "incorrect_counts": [0, 6, 22, 22, 25, 26, 21, 17, 7, 5, 8, 13, 9, 2, 3, 2, 1, 2, 0, 2]}, "dynamics/residual": {"edges": [1.1507504383722942, 1.166901803513368, 1.1830531686544419, 1.1992045337955157, 1.2153558989365896, 1.2315072640776634, 1.2476586292187373, 1.2638099943598111, 1.279961359500885, 1.2961127246419588, 1.3122640897830329, 1.3284154549241065, 1.3445668200651806, 1.3607181852062544, 1.3768695503473283, 1.3930209154884021, 1.409172280629476, 1.4253236457705498, 1.4414750109116237, 1.4576263760526975, 1.4737777411937714], "correct_counts": [6, 9, 15, 37, 63, 94, 128, 188, 171, 147, 140, 124, 85, 52, 27, 10, 6, 3, 1, 1], "incorrect_counts": [0, 3, 0, 7, 6, 17, 20, 18, 24, 18, 27, 20, 17, 7, 3, 2, 2, 1, 1, 0]}}}, "feature_ablation": {"full": 0.024274182941440476, "drop_basin": 0.04808611569032442, "basin_only": 0.05844366219985415, "drop_energy": 0.031331435374528864, "energy_only": 0.040657703926391564, "drop_curv": 0.02375638180042061, "curv_only": 0.1256930708759337, "drop_dynamics": 0.02454171163749289, "dynamics_only": 0.07986208469806944}, "ece_geometry": 0.04318340190787242, "accuracy_ood": 0.21866666666666668, "aurc_ood": {"geometry": 0.7118811950177575, "rho_basin": 0.7176588499789294, "energy_min": 0.7122394640249904, "energy_mean": 0.7126699633509685, "energy_std": 0.7809376529088858, "msp": 0.653860323850022, "temp_msp": 0.6533021641143179, "entropy": 0.6592791559765256, "softmax_learned": 0.648426119528371, "geom_softmax": 0.7027844744493177}, "ood_ltt": {"alpha": 0.1, "lambda_hat": 0.42350354070874574, "selective_risk": 0.7672155688622755, "coverage": 0.8906666666666667, "risk_within_budget": false}, "ood_validity": {"target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "id_risk": [0.0, 0.03186907838070629, 0.07778608825729244, 0.12750333778371162, 0.12750333778371162, 0.12750333778371162], "ood_risk": [0.0, 0.7385321100917431, 0.7672155688622755, 0.7813333333333333, 0.7813333333333333, 0.7813333333333333], "id_coverage": [0.0, 0.774, 0.8913333333333333, 0.9986666666666667, 0.9986666666666667, 0.9986666666666667], "ood_coverage": [0.0, 0.7266666666666667, 0.8906666666666667, 1.0, 1.0, 1.0]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "70cee2dae378", "timestamp": "2026-07-30T11:18:05.366179+00:00", "git_sha": "67dc7a2", "config_hash": "f5d7b65bb5d0", "config": {"run": {"seed": 2, "task": "graph_planning", "notes": "graph adaptive-halting run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 25, "batch_size": 128, "lr": 0.001, "n_train": 8000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 800, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 800}, "task": {"graph_planning": {"n_nodes": 7, "ood_n_nodes": 10, "edge_prob": 0.4, "max_len": 4}}}, "task": "graph_planning", "split": "halting", "seed": 2, "metrics": {"k_restarts": 12, "steps": 50, "alpha": 0.1, "n_calib": 800, "n_test": 800, "final_train_loss": 0.178430438041687, "tau_hat": 0.8333, "compute_used": 0.15805000000000002, "compute_saved": 0.84195, "halting_risk": 0.085, "risk_within_budget": true, "full_accuracy": 0.68, "halted_accuracy": 0.66875, "accuracy_drop": 0.011250000000000093, "risk_monotone_in_tau": true, "calib_risk_by_tau": [0.4625, 0.4625, 0.4625, 0.4625, 0.38375, 0.38375, 0.29875, 0.125, 0.125, 0.08875, 0.0125, 0.0125], "calib_compute_by_tau": [0.0, 0.0, 0.0, 0.000625, 0.022874999999999833, 0.022874999999999833, 0.0464749999999998, 0.12292500000000013, 0.12292500000000013, 0.16652500000000037, 0.29684999999999984, 0.29684999999999984], "tau_sweep": {"taus": [0.0833, 0.1667, 0.25, 0.3333, 0.4167, 0.5, 0.5833, 0.6667, 0.75, 0.8333, 0.9167, 1.0], "compute_used": [0.0, 0.0, 0.0, 0.0004, 0.022099999999999998, 0.022099999999999998, 0.04724999999999999, 0.11257500000000001, 0.11257500000000001, 0.15805000000000002, 0.29180000000000006, 0.29180000000000006], "accuracy": [0.4525, 0.4525, 0.4525, 0.45875, 0.51625, 0.51625, 0.565, 0.65, 0.65, 0.66875, 0.68375, 0.68375], "disagreement": [0.47375, 0.47375, 0.47375, 0.46875, 0.3575, 0.3575, 0.28875, 0.135, 0.135, 0.085, 0.01375, 0.01375]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "496414a8f2cd", "timestamp": "2026-07-30T11:18:07.813782+00:00", "git_sha": "67dc7a2", "config_hash": "593dfa402e05", "config": {"run": {"seed": 1, "task": "graph_planning", "notes": "E2 graph selective-prediction"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 25, "batch_size": 128, "lr": 0.001, "n_train": 8000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 1500, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 1500}, "task": {"graph_planning": {"n_nodes": 7, "ood_n_nodes": 10, "edge_prob": 0.4, "max_len": 4}}}, "task": "graph_planning", "split": "selective", "seed": 1, "metrics": {"n_fit": 1500, "n_calib": 1500, "n_test": 1500, "k_restarts": 12, "objective": "basin_center", "sampler": "langevin", "feature_set": "base", "accuracy_id": 0.6913333333333334, "base_error": 0.30866666666666664, "final_train_loss": 0.2080264687538147, "feature_names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "aurc": {"geometry": 0.17444436593752588, "rho_basin": 0.25642611361139067, "energy_min": 0.19964686175564186, "energy_mean": 0.19527316950353119, "energy_std": 0.29427140691389053, "msp": 0.1401480752960069, "temp_msp": 0.1386661861515245, "entropy": 0.13897082240953618, "softmax_learned": 0.13937315202394526, "geom_softmax": 0.14038547133382884}, "temperature": 3.1187914236161625, "best_energy_baseline": "energy_mean", "best_baseline": "temp_msp", "delta_aurc_vs_energy_min": [0.025202495818115983, 0.014181046993774545, 0.03686871923665555], "delta_aurc_vs_best_energy": [0.020828803566005305, 0.010827769204687336, 0.03126253972294629], "delta_aurc_vs_best_baseline": [-0.03577817978600137, -0.047061369377102535, -0.024631857225653937], "delta_aurc_geom_adds": [-0.0010123193098835748, -0.0035237183803231006, 0.0015377190771044024], "geometry_wins": true, "geometry_wins_vs_baseline": false, "geometry_adds_over_softmax": false, "risk_coverage": {"geometry": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.03333333333333333, 0.05555555555555555, 0.058333333333333334, 0.06666666666666667, 0.0722222222222222, 0.07142857142857144, 0.07916666666666666, 0.0962962962962963, 0.09333333333333335, 0.09696969696969697, 0.1111111111111111, 0.1076923076923077, 0.12380952380952381, 0.12888888888888886, 0.13125, 0.14313725490196078, 0.14629629629629626, 0.15263157894736842, 0.16, 0.16984126984126982, 0.16818181818181818, 0.16956521739130434, 0.17500000000000007, 0.18, 0.18333333333333332, 0.19012345679012346, 0.19285714285714287, 0.1965517241379312, 0.20222222222222222, 0.2075268817204301, 0.2125, 0.21818181818181817, 0.22156862745098038, 0.22666666666666677, 0.23055555555555554, 0.23513513513513515, 0.2394736842105263, 0.24188034188034188, 0.25, 0.2544715447154471, 0.26031746031746045, 0.26666666666666666, 0.27121212121212124, 0.2777777777777778, 0.2797101449275362, 0.2865248226950356, 0.2930555555555556, 0.3013605442176871, 0.30866666666666664]}, "rho_basin": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.25061025223759165, 0.25061025223759126, 0.25061025223759165, 0.25061025223759187, 0.25061025223759203, 0.2506102522375921, 0.2506102522375922, 0.25061025223759226, 0.25061025223759226, 0.2506102522375923, 0.2506102522375923, 0.25061025223759237, 0.25061025223759237, 0.25061025223759237, 0.2506102522375924, 0.2506102522375924, 0.2506102522375924, 0.25061025223759165, 0.2506102522375909, 0.2506102522375903, 0.25061025223758976, 0.2506102522375892, 0.25061025223758876, 0.2506102522375883, 0.2506102522375879, 0.25061025223758754, 0.2506102522375872, 0.25061025223758693, 0.2506102522375866, 0.2506102522375863, 0.2506102522375861, 0.2506102522375858, 0.2506102522375856, 0.2506102522375854, 0.250610252237586, 0.25061025223758654, 0.2506102522375871, 0.2506102522375876, 0.2506102522375881, 0.25061025223758854, 0.25086440519577363, 0.25830140485312647, 0.26539249754967215, 0.2724913065076973, 0.2813721918639928, 0.2896286231884037, 0.2953804868698466, 0.2993618618618604, 0.3036646916831233, 0.30866666666666537]}, "energy_min": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.08333333333333333, 0.08888888888888889, 0.075, 0.09333333333333334, 0.08333333333333331, 0.09523809523809525, 0.1125, 0.1259259259259259, 0.13333333333333322, 0.1484848484848485, 0.16111111111111112, 0.16666666666666666, 0.1761904761904762, 0.18000000000000013, 0.19166666666666668, 0.19607843137254902, 0.19444444444444442, 0.1912280701754386, 0.19, 0.18888888888888886, 0.1893939393939394, 0.19710144927536233, 0.19305555555555565, 0.2, 0.2141025641025641, 0.21975308641975308, 0.22142857142857142, 0.2229885057471264, 0.22777777777777777, 0.22903225806451613, 0.23125, 0.23737373737373738, 0.24215686274509804, 0.24857142857142855, 0.24629629629629626, 0.25225225225225223, 0.26228070175438595, 0.26837606837606837, 0.26916666666666667, 0.2764227642276422, 0.27698412698412694, 0.2837209302325581, 0.28939393939393937, 0.2911111111111111, 0.29347826086956524, 0.2971631205673758, 0.29861111111111116, 0.30272108843537415, 0.30866666666666664]}, "energy_mean": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.1, 0.06666666666666667, 0.044444444444444446, 0.075, 0.08666666666666667, 0.10555555555555554, 0.10000000000000002, 0.10833333333333334, 0.1111111111111111, 0.10333333333333335, 0.10909090909090909, 0.125, 0.13846153846153847, 0.15476190476190477, 0.16222222222222218, 0.17708333333333334, 0.18235294117647058, 0.187037037037037, 0.18421052631578946, 0.18666666666666668, 0.1857142857142857, 0.18484848484848485, 0.19710144927536233, 0.1972222222222223, 0.20933333333333334, 0.21025641025641026, 0.2111111111111111, 0.21428571428571427, 0.2172413793103448, 0.22333333333333333, 0.22258064516129034, 0.22708333333333333, 0.23232323232323232, 0.24019607843137256, 0.23999999999999996, 0.24444444444444455, 0.24864864864864866, 0.2543859649122807, 0.2606837606837607, 0.2633333333333333, 0.265040650406504, 0.269047619047619, 0.26976744186046514, 0.275, 0.28, 0.286231884057971, 0.29290780141843964, 0.2979166666666666, 0.3006802721088435, 0.30866666666666664]}, "energy_std": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.16666666666666666, 0.2222222222222222, 0.24166666666666667, 0.2733333333333333, 0.29999999999999993, 0.3285714285714286, 0.30833333333333335, 0.3074074074074074, 0.30000000000000004, 0.296969696969697, 0.2972222222222222, 0.29743589743589743, 0.3047619047619048, 0.3044444444444444, 0.30833333333333335, 0.307843137254902, 0.3111111111111112, 0.30526315789473685, 0.3, 0.30476190476190473, 0.31212121212121213, 0.3101449275362319, 0.3097222222222222, 0.30933333333333335, 0.308974358974359, 0.30987654320987656, 0.31547619047619047, 0.31149425287356314, 0.31222222222222223, 0.31290322580645163, 0.31666666666666665, 0.31414141414141417, 0.31470588235294117, 0.31619047619047613, 0.31481481481481477, 0.31261261261261264, 0.3131578947368421, 0.31367521367521367, 0.31333333333333335, 0.3113821138211382, 0.3063492063492063, 0.3077519379844961, 0.30606060606060603, 0.3074074074074074, 0.30579710144927535, 0.30780141843971626, 0.3069444444444444, 0.3054421768707483, 0.30866666666666664]}, "msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.006666666666666667, 0.005555555555555555, 0.009523809523809526, 0.029166666666666667, 0.037037037037037035, 0.04000000000000001, 0.045454545454545456, 0.04722222222222222, 0.05128205128205128, 0.05476190476190476, 0.062222222222222394, 0.08125, 0.08235294117647059, 0.08703703703703702, 0.09298245614035087, 0.1, 0.11428571428571427, 0.11818181818181818, 0.1289855072463768, 0.13472222222222233, 0.14133333333333334, 0.14487179487179488, 0.15185185185185185, 0.15595238095238095, 0.16321839080459769, 0.1711111111111111, 0.17956989247311828, 0.18854166666666666, 0.19696969696969696, 0.2, 0.2066666666666668, 0.212037037037037, 0.21711711711711712, 0.22631578947368422, 0.23504273504273504, 0.24083333333333334, 0.24634146341463425, 0.25555555555555565, 0.26124031007751936, 0.2689393939393939, 0.28074074074074074, 0.28695652173913044, 0.2914893617021276, 0.2965277777777777, 0.3006802721088435, 0.30866666666666664]}, "temp_msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.019047619047619053, 0.025, 0.02962962962962963, 0.030000000000000002, 0.045454545454545456, 0.05, 0.05384615384615385, 0.06428571428571428, 0.06666666666666665, 0.07083333333333333, 0.0784313725490196, 0.08888888888888888, 0.09649122807017543, 0.10333333333333333, 0.11904761904761903, 0.12272727272727273, 0.13478260869565217, 0.13888888888888887, 0.144, 0.14871794871794872, 0.15432098765432098, 0.1630952380952381, 0.1701149425287356, 0.17333333333333334, 0.18064516129032257, 0.1875, 0.1919191919191919, 0.1931372549019608, 0.19999999999999998, 0.21018518518518517, 0.21711711711711712, 0.22631578947368422, 0.23162393162393163, 0.23666666666666666, 0.24065040650406502, 0.2476190476190476, 0.2527131782945736, 0.25984848484848483, 0.2674074074074074, 0.27608695652173915, 0.28439716312056734, 0.29375000000000007, 0.3, 0.30866666666666664]}, "entropy": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.006666666666666667, 0.005555555555555555, 0.009523809523809526, 0.029166666666666667, 0.037037037037037035, 0.04000000000000001, 0.045454545454545456, 0.04722222222222222, 0.05128205128205128, 0.05476190476190476, 0.06444444444444443, 0.07916666666666666, 0.08235294117647059, 0.08703703703703702, 0.09298245614035087, 0.1, 0.11269841269841269, 0.11818181818181818, 0.12753623188405797, 0.13472222222222233, 0.14266666666666666, 0.14615384615384616, 0.15185185185185185, 0.15714285714285714, 0.1597701149425289, 0.1688888888888889, 0.17956989247311828, 0.1875, 0.19595959595959597, 0.1980392156862745, 0.20380952380952394, 0.20833333333333331, 0.2153153153153153, 0.2236842105263158, 0.23247863247863249, 0.23916666666666667, 0.24715447154471543, 0.2523809523809525, 0.2596899224806202, 0.26666666666666666, 0.27185185185185184, 0.2753623188405797, 0.2836879432624115, 0.2923611111111112, 0.3020408163265306, 0.30866666666666664]}, "softmax_learned": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.019047619047619053, 0.025, 0.02962962962962963, 0.030000000000000002, 0.045454545454545456, 0.05, 0.05384615384615385, 0.06428571428571428, 0.06666666666666665, 0.07083333333333333, 0.0803921568627451, 0.09074074074074073, 0.09473684210526316, 0.10666666666666667, 0.11587301587301586, 0.12575757575757576, 0.1318840579710145, 0.13750000000000012, 0.14266666666666666, 0.15, 0.1580246913580247, 0.1619047619047619, 0.16896551724137948, 0.17555555555555555, 0.1860215053763441, 0.190625, 0.19494949494949496, 0.2, 0.20380952380952377, 0.21018518518518517, 0.218018018018018, 0.22543859649122808, 0.23162393162393163, 0.23666666666666666, 0.24227642276422762, 0.24841269841269853, 0.25193798449612403, 0.25984848484848483, 0.2688888888888889, 0.27608695652173915, 0.28581560283687957, 0.29513888888888884, 0.3013605442176871, 0.30866666666666664]}, "geom_softmax": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.008333333333333333, 0.006666666666666667, 0.005555555555555555, 0.009523809523809526, 0.016666666666666666, 0.018518518518518517, 0.036666666666666674, 0.045454545454545456, 0.05, 0.05384615384615385, 0.06428571428571428, 0.07333333333333332, 0.075, 0.08235294117647059, 0.08888888888888888, 0.09473684210526316, 0.10666666666666667, 0.11587301587301586, 0.12121212121212122, 0.13623188405797101, 0.138888888888889, 0.14533333333333334, 0.14743589743589744, 0.15308641975308643, 0.1630952380952381, 0.1701149425287356, 0.17777777777777778, 0.17956989247311828, 0.18645833333333334, 0.19292929292929292, 0.19705882352941176, 0.20190476190476186, 0.21018518518518517, 0.21981981981981982, 0.22719298245614036, 0.2358974358974359, 0.24083333333333334, 0.24715447154471543, 0.25396825396825407, 0.262015503875969, 0.26515151515151514, 0.2733333333333333, 0.27898550724637683, 0.2879432624113475, 0.2944444444444444, 0.3013605442176871, 0.30866666666666664]}}, "ltt": {"alpha": 0.1, "delta": 0.05, "lambda_hat": 0.10002331995594467, "coverage": 0.05533333333333333, "abstain_rate": 0.9446666666666667, "selective_risk": 0.04819277108433735, "selective_accuracy": 0.9518072289156626, "risk_within_budget": true}, "coverage_validity": {"alpha_grid": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "achieved_risk": [0.0, 0.0, 0.04819277108433735, 0.09177215189873418, 0.14555765595463138, 0.257781324820431], "coverage": [0.0, 0.0, 0.05533333333333333, 0.21066666666666667, 0.3526666666666667, 0.8353333333333334]}, "feature_diagnostics": {"names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "auroc": {"basin/rho": 0.6105604512101905, "basin/entropy": 0.3894051831687602, "basin/dispersion": 0.4935923737479979, "energy/mean": 0.32511127171542764, "energy/min": 0.34090696080861266, "energy/std": 0.49929498407726225, "curv/lmax_mean": 0.460030699954804, "curv/lmax_best": 0.4852384036856608, "curv/trace_mean": 0.45787191412343714, "curv/trace_best": 0.44889311458747716, "dynamics/steps": 0.5, "dynamics/monotonic": 0.6295333981767476, "dynamics/drop": 0.6543776594304471, "dynamics/residual": 0.504791400680231}, "hist": {"basin/rho": {"edges": [0.3333333333333333, 0.36666666666666664, 0.4, 0.43333333333333335, 0.4666666666666667, 0.5, 0.5333333333333333, 0.5666666666666667, 0.6000000000000001, 0.6333333333333333, 0.6666666666666667, 0.7000000000000001, 0.7333333333333334, 0.7666666666666667, 0.8, 0.8333333333333335, 0.8666666666666667, 0.9000000000000001, 0.9333333333333333, 0.9666666666666668, 1.0], "correct_counts": [0, 0, 0, 0, 0, 10, 0, 15, 0, 19, 0, 0, 14, 0, 20, 0, 0, 38, 0, 921], "incorrect_counts": [1, 0, 0, 0, 0, 12, 0, 16, 0, 18, 0, 0, 18, 0, 41, 0, 0, 49, 0, 308]}, "basin/entropy": {"edges": [0.0, 0.05493061443340548, 0.10986122886681096, 0.16479184330021646, 0.21972245773362192, 0.2746530721670274, 0.3295836866004329, 0.3845143010338384, 0.43944491546724385, 0.4943755299006493, 0.5493061443340548, 0.6042367587674603, 0.6591673732008658, 0.7140979876342712, 0.7690286020676768, 0.8239592165010822, 0.8788898309344877, 0.9338204453678932, 0.9887510598012986, 1.0436816742347041, 1.0986122886681096], "correct_counts": [921, 0, 0, 0, 0, 38, 0, 0, 20, 0, 13, 18, 22, 1, 0, 1, 2, 1, 0, 0], "incorrect_counts": [308, 0, 0, 0, 0, 49, 0, 0, 41, 0, 17, 18, 20, 1, 0, 0, 6, 1, 1, 1]}, "basin/dispersion": {"edges": [1.6302329015759547, 1.6519526525410289, 1.6736724035061028, 1.6953921544711767, 1.717111905436251, 1.738831656401325, 1.760551407366399, 1.782271158331473, 1.8039909092965472, 1.8257106602616213, 1.8474304112266953, 1.8691501621917692, 1.8908699131568434, 1.9125896641219176, 1.9343094150869915, 1.9560291660520654, 1.9777489170171396, 1.9994686679822138, 2.021188418947288, 2.0429081699123617, 2.064627920877436], "correct_counts": [3, 7, 19, 24, 43, 69, 85, 117, 140, 123, 109, 96, 71, 49, 39, 21, 8, 11, 2, 1], "incorrect_counts": [3, 2, 5, 13, 20, 26, 34, 70, 48, 47, 61, 42, 37, 21, 14, 12, 3, 2, 2, 1]}, "energy/mean": {"edges": [-0.2394584914048513, -0.18958720415830613, -0.13971591691176097, -0.08984462966521581, -0.03997334241867065, 0.00989794482787451, 0.05976923207441967, 0.10964051932096483, 0.15951180656751, 0.20938309381405515, 0.2592543810606003, 0.3091256683071454, 0.3589969555536906, 0.4088682428002358, 0.4587395300467809, 0.508610817293326, 0.5584821045398712, 0.6083533917864165, 0.6582246790329616, 0.7080959662795067, 0.7579672535260519], "correct_counts": [1, 2, 17, 33, 48, 57, 92, 101, 101, 103, 96, 84, 85, 75, 68, 32, 24, 12, 3, 3], "incorrect_counts": [0, 0, 3, 0, 3, 12, 13, 34, 37, 34, 43, 35, 55, 53, 47, 40, 31, 13, 6, 4]}, "energy/min": {"edges": [-0.5504263639450073, -0.4960724890232086, -0.4417186141014099, -0.38736473917961123, -0.3330108642578125, -0.2786569893360138, -0.22430311441421513, -0.16994923949241642, -0.1155953645706177, -0.06124148964881898, -0.006887614727020264, 0.0474662601947784, 0.10182013511657706, 0.15617401003837583, 0.2105278849601745, 0.26488175988197327, 0.31923563480377193, 0.3735895097255706, 0.42794338464736936, 0.48229725956916814, 0.5366511344909668], "correct_counts": [5, 10, 14, 45, 55, 80, 92, 92, 122, 96, 106, 90, 70, 53, 51, 31, 16, 3, 4, 2], "incorrect_counts": [0, 0, 0, 7, 7, 11, 33, 38, 24, 44, 49, 50, 60, 56, 31, 25, 13, 7, 6, 2]}, "energy/std": {"edges": [0.07470156430161684, 0.08989143155875684, 0.10508129881589684, 0.12027116607303683, 0.13546103333017684, 0.15065090058731684, 0.1658407678444568, 0.1810306351015968, 0.1962205023587368, 0.2114103696158768, 0.2266002368730168, 0.2417901041301568, 0.2569799713872968, 0.2721698386444368, 0.2873597059015768, 0.3025495731587168, 0.3177394404158568, 0.3329293076729968, 0.3481191749301368, 0.3633090421872768, 0.3784989094444168], "correct_counts": [6, 12, 28, 49, 74, 114, 143, 136, 129, 114, 85, 68, 39, 21, 5, 2, 7, 3, 1, 1], "incorrect_counts": [1, 0, 7, 21, 47, 48, 58, 69, 65, 51, 29, 26, 17, 9, 5, 5, 1, 1, 1, 2]}, "curv/lmax_mean": {"edges": [1.000376323858897, 1.0019486089547476, 1.0035208940505982, 1.0050931791464488, 1.0066654642422994, 1.00823774933815, 1.0098100344340006, 1.0113823195298512, 1.0129546046257019, 1.0145268897215525, 1.0160991748174033, 1.017671459913254, 1.0192437450091045, 1.0208160301049551, 1.0223883152008058, 1.0239606002966564, 1.025532885392507, 1.0271051704883576, 1.0286774555842082, 1.0302497406800588, 1.0318220257759094], "correct_counts": [105, 233, 210, 179, 108, 77, 40, 38, 17, 14, 6, 6, 0, 2, 1, 0, 0, 0, 0, 1], "incorrect_counts": [26, 89, 110, 82, 57, 35, 24, 16, 7, 5, 7, 1, 1, 3, 0, 0, 0, 0, 0, 0]}, "curv/lmax_best": {"edges": [1.0, 1.0040147721767425, 1.008029544353485, 1.0120443165302277, 1.0160590887069703, 1.0200738608837128, 1.0240886330604553, 1.0281034052371978, 1.0321181774139405, 1.036132949590683, 1.0401477217674255, 1.044162493944168, 1.0481772661209106, 1.0521920382976533, 1.0562068104743958, 1.0602215826511383, 1.0642363548278808, 1.0682511270046233, 1.072265899181366, 1.0762806713581086, 1.080295443534851], "correct_counts": [618, 188, 73, 67, 29, 18, 13, 8, 9, 5, 3, 3, 0, 0, 1, 1, 0, 0, 0, 1], "incorrect_counts": [273, 79, 45, 17, 16, 9, 11, 4, 2, 5, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0]}, "curv/trace_mean": {"edges": [32.01093864440918, 32.016456858317056, 32.02197507222493, 32.02749328613281, 32.03301150004069, 32.03852971394857, 32.04404792785645, 32.04956614176432, 32.0550843556722, 32.06060256958008, 32.06612078348796, 32.07163899739584, 32.077157211303714, 32.08267542521159, 32.08819363911947, 32.093711853027344, 32.09923006693522, 32.104748280843104, 32.11026649475098, 32.11578470865886, 32.121302922566734], "correct_counts": [4, 3, 28, 47, 70, 105, 117, 148, 132, 120, 77, 60, 45, 29, 22, 13, 9, 3, 3, 2], "incorrect_counts": [0, 0, 4, 15, 26, 42, 52, 66, 59, 61, 50, 30, 20, 11, 10, 7, 4, 3, 2, 1]}, "curv/trace_best": {"edges": [32.004268646240234, 32.017548370361325, 32.03082809448242, 32.044107818603514, 32.05738754272461, 32.0706672668457, 32.083946990966794, 32.09722671508789, 32.11050643920898, 32.12378616333008, 32.13706588745117, 32.15034561157226, 32.16362533569336, 32.17690505981445, 32.19018478393555, 32.20346450805664, 32.21674423217773, 32.23002395629883, 32.24330368041992, 32.25658340454102, 32.26986312866211], "correct_counts": [57, 188, 204, 172, 154, 85, 58, 46, 29, 18, 10, 5, 5, 4, 0, 1, 0, 0, 0, 1], "incorrect_counts": [19, 73, 80, 73, 62, 48, 48, 14, 16, 11, 9, 6, 2, 1, 0, 0, 0, 0, 1, 0]}, "dynamics/steps": {"edges": [1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7000000000000002, 1.75, 1.8, 1.85, 1.9, 1.9500000000000002, 2.0], "correct_counts": [1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/monotonic": {"edges": [0.64, 0.6465833333333334, 0.6531666666666667, 0.6597500000000001, 0.6663333333333333, 0.6729166666666667, 0.6795000000000001, 0.6860833333333334, 0.6926666666666668, 0.69925, 0.7058333333333334, 0.7124166666666668, 0.7190000000000001, 0.7255833333333335, 0.7321666666666667, 0.7387500000000001, 0.7453333333333335, 0.7519166666666668, 0.7585000000000002, 0.7650833333333336, 0.7716666666666668], "correct_counts": [0, 1, 6, 33, 38, 72, 88, 131, 131, 143, 112, 110, 69, 46, 27, 15, 9, 1, 3, 2], "incorrect_counts": [1, 4, 8, 23, 37, 38, 59, 79, 63, 52, 40, 32, 11, 6, 4, 4, 1, 0, 1, 0]}, "dynamics/drop": {"edges": [13.908043444156647, 21.40915985889733, 28.91027627363801, 36.41139268837869, 43.912509103119376, 51.413625517860055, 58.914741932600734, 66.41585834734141, 73.9169747620821, 81.41809117682278, 88.91920759156346, 96.42032400630414, 103.92144042104482, 111.4225568357855, 118.92367325052619, 126.42478966526687, 133.92590608000756, 141.42702249474823, 148.92813890948892, 156.4292553242296, 163.93037173897028], "correct_counts": [208, 321, 228, 142, 66, 39, 18, 7, 1, 2, 2, 1, 0, 0, 1, 0, 0, 0, 0, 1], "incorrect_counts": [171, 167, 74, 36, 8, 5, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/residual": {"edges": [1.1329888900121052, 1.1492741659283636, 1.1655594418446222, 1.1818447177608808, 1.1981299936771392, 1.2144152695933976, 1.2307005455096562, 1.2469858214259149, 1.2632710973421732, 1.2795563732584316, 1.2958416491746902, 1.3121269250909489, 1.3284122010072072, 1.3446974769234656, 1.3609827528397243, 1.3772680287559829, 1.3935533046722413, 1.4098385805884996, 1.4261238565047583, 1.4424091324210169, 1.4586944083372753], "correct_counts": [2, 2, 8, 21, 32, 71, 96, 106, 140, 134, 137, 98, 88, 51, 28, 13, 7, 2, 0, 1], "incorrect_counts": [0, 2, 5, 9, 17, 29, 50, 50, 43, 70, 55, 47, 42, 21, 15, 5, 1, 1, 0, 1]}}}, "feature_ablation": {"full": 0.17444436593752588, "drop_basin": 0.19274098660894265, "basin_only": 0.25332924341284363, "drop_energy": 0.18924590060362054, "energy_only": 0.19443145021840322, "drop_curv": 0.17517378968044742, "curv_only": 0.2700948061609721, "drop_dynamics": 0.1729185443228303, "dynamics_only": 0.20883041928582574}, "ece_geometry": 0.026691927047032264, "accuracy_ood": 0.256, "aurc_ood": {"geometry": 0.7974446165188469, "rho_basin": 0.7490819589151309, "energy_min": 0.739655450497817, "energy_mean": 0.7320500499695907, "energy_std": 0.7382724006497628, "msp": 0.8071610870796354, "temp_msp": 0.7964482735224696, "entropy": 0.8055287758926124, "softmax_learned": 0.7951769722870716, "geom_softmax": 0.7840875769994762}, "ood_ltt": {"alpha": 0.1, "lambda_hat": 0.10002331995594467, "selective_risk": 0.8401486988847584, "coverage": 0.17933333333333334, "risk_within_budget": false}, "ood_validity": {"target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "id_risk": [0.0, 0.0, 0.04819277108433735, 0.09177215189873418, 0.14555765595463138, 0.257781324820431], "ood_risk": [0.0, 0.0, 0.8401486988847584, 0.7979094076655052, 0.7672955974842768, 0.7467778620166793], "id_coverage": [0.0, 0.0, 0.05533333333333333, 0.21066666666666667, 0.3526666666666667, 0.8353333333333334], "ood_coverage": [0.0, 0.0, 0.17933333333333334, 0.38266666666666665, 0.53, 0.8793333333333333]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "5bd7ea239e4f", "timestamp": "2026-07-30T11:18:10.314612+00:00", "git_sha": "67dc7a2", "config_hash": "0c62cba9cde6", "config": {"run": {"seed": 3, "task": "arithmetic", "notes": "E1 selective-prediction main run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 7, "batch_size": 128, "lr": 0.001, "n_train": 6000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 1500, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 1500}, "task": {"arithmetic": {"modular": false, "n_operands": 6, "ood_n_operands": 6, "max_operand": 7, "ood_max_operand": 9}}}, "task": "arithmetic", "split": "selective", "seed": 3, "metrics": {"n_fit": 1500, "n_calib": 1500, "n_test": 1500, "k_restarts": 12, "objective": "basin_center", "sampler": "langevin", "feature_set": "base", "accuracy_id": 0.8166666666666667, "base_error": 0.18333333333333332, "final_train_loss": 0.943260669708252, "feature_names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "aurc": {"geometry": 0.05870278336778761, "rho_basin": 0.09557280117969337, "energy_min": 0.23308855480058674, "energy_mean": 0.2321489497773461, "energy_std": 0.18295923460701716, "msp": 0.05696229801237003, "temp_msp": 0.05587579661080262, "entropy": 0.07586100765649918, "softmax_learned": 0.05495199413638246, "geom_softmax": 0.050518887585592184}, "temperature": 0.3191283771478964, "best_energy_baseline": "energy_std", "best_baseline": "temp_msp", "delta_aurc_vs_energy_min": [0.17438577143279912, 0.1451627099062947, 0.20605252294076107], "delta_aurc_vs_best_energy": [0.12425645123922954, 0.1000795529758454, 0.14889441517869045], "delta_aurc_vs_best_baseline": [-0.002826986756984985, -0.013129993995088966, 0.00820655055054469], "delta_aurc_geom_adds": [0.004433106550790274, -6.164051281485425e-05, 0.010993126415850296], "geometry_wins": true, "geometry_wins_vs_baseline": false, "geometry_adds_over_softmax": false, "risk_coverage": {"geometry": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.016666666666666666, 0.011111111111111112, 0.025, 0.02, 0.022222222222222334, 0.03333333333333334, 0.0375, 0.03333333333333333, 0.030000000000000002, 0.030303030303030304, 0.030555555555555555, 0.028205128205128206, 0.02857142857142857, 0.026666666666666665, 0.025, 0.023529411764705882, 0.027777777777777773, 0.028070175438596492, 0.028333333333333332, 0.033333333333333326, 0.031818181818181815, 0.030434782608695653, 0.03472222222222222, 0.03333333333333333, 0.03333333333333333, 0.03333333333333333, 0.03214285714285714, 0.03678160919540229, 0.04, 0.043010752688172046, 0.05, 0.05656565656565657, 0.061764705882352944, 0.06857142857142856, 0.07499999999999998, 0.08288288288288288, 0.09210526315789473, 0.09658119658119659, 0.1025, 0.1089430894308943, 0.11746031746031757, 0.12558139534883722, 0.1340909090909091, 0.14222222222222222, 0.1492753623188406, 0.15886524822695047, 0.1659722222222222, 0.1727891156462585, 0.18333333333333332]}, "rho_basin": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.08056394763343407, 0.08056394763343401, 0.08056394763343389, 0.08056394763343398, 0.08056394763343411, 0.08056394763343418, 0.08056394763343414, 0.08056394763343395, 0.08056394763343383, 0.08056394763343372, 0.08056394763343364, 0.08056394763343357, 0.08056394763343351, 0.08056394763343364, 0.08056394763343382, 0.08056394763343397, 0.08056394763343411, 0.08056394763343423, 0.08056394763343434, 0.08056394763343444, 0.08056394763343454, 0.08056394763343462, 0.08056394763343469, 0.08056394763343476, 0.08056394763343482, 0.08056394763343488, 0.08056394763343493, 0.08056394763343497, 0.08056394763343502, 0.08056394763343507, 0.0805639476334351, 0.08056394763343515, 0.08056394763343518, 0.08431372549019736, 0.08825396825396971, 0.09197530864197688, 0.09549549549549718, 0.09883040935672696, 0.10364611747590659, 0.10983156028368969, 0.11571527417401978, 0.12196428571428705, 0.12901162790697826, 0.13573863636363812, 0.14465408805031627, 0.15422477440525198, 0.16295752832803626, 0.17129629629629786, 0.1791748957647589, 0.18333333333333446]}, "energy_min": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.43333333333333335, 0.4666666666666667, 0.45555555555555555, 0.43333333333333335, 0.41333333333333333, 0.3722222222222222, 0.35714285714285704, 0.35, 0.3296296296296296, 0.31000000000000005, 0.2909090909090909, 0.275, 0.258974358974359, 0.25952380952380955, 0.24666666666666662, 0.24166666666666667, 0.23921568627450981, 0.2259259259259259, 0.22280701754385965, 0.22, 0.21428571428571425, 0.2106060606060606, 0.2028985507246377, 0.1958333333333333, 0.18933333333333333, 0.18717948717948718, 0.18271604938271604, 0.17857142857142858, 0.1758620689655174, 0.17555555555555555, 0.17311827956989248, 0.16979166666666667, 0.16767676767676767, 0.16372549019607843, 0.16190476190476188, 0.16018518518518515, 0.15945945945945947, 0.15789473684210525, 0.15726495726495726, 0.155, 0.15609756097560987, 0.15555555555555553, 0.1573643410852713, 0.15984848484848485, 0.16074074074074074, 0.16231884057971013, 0.1617021276595746, 0.16805555555555565, 0.17482993197278912, 0.18333333333333332]}, "energy_mean": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.43333333333333335, 0.43333333333333335, 0.43333333333333335, 0.39166666666666666, 0.38, 0.3722222222222223, 0.37619047619047624, 0.3458333333333333, 0.32592592592592595, 0.3133333333333334, 0.296969696969697, 0.28888888888888886, 0.2743589743589744, 0.2619047619047619, 0.2533333333333333, 0.24583333333333332, 0.2411764705882353, 0.2333333333333333, 0.22456140350877193, 0.22333333333333333, 0.2126984126984128, 0.20606060606060606, 0.19855072463768117, 0.19444444444444453, 0.19066666666666668, 0.18846153846153846, 0.18518518518518517, 0.18095238095238095, 0.17816091954022986, 0.17555555555555555, 0.17419354838709677, 0.16979166666666667, 0.16767676767676767, 0.16470588235294117, 0.16190476190476188, 0.16018518518518515, 0.16036036036036036, 0.16052631578947368, 0.1606837606837607, 0.1575, 0.15772357723577232, 0.15555555555555553, 0.15271317829457365, 0.15378787878787878, 0.15703703703703703, 0.1608695652173913, 0.16241134751773048, 0.16527777777777786, 0.17551020408163265, 0.18333333333333332]}, "energy_std": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.2, 0.2, 0.2, 0.2, 0.18, 0.16666666666666674, 0.16666666666666669, 0.175, 0.17407407407407408, 0.1766666666666667, 0.17575757575757575, 0.17222222222222222, 0.17435897435897435, 0.1738095238095238, 0.17111111111111107, 0.17291666666666666, 0.16666666666666666, 0.17407407407407421, 0.1824561403508772, 0.18, 0.17777777777777776, 0.17575757575757575, 0.1855072463768116, 0.1833333333333333, 0.184, 0.18846153846153846, 0.18641975308641975, 0.1880952380952381, 0.18620689655172412, 0.18222222222222223, 0.1817204301075269, 0.18125, 0.18282828282828284, 0.17941176470588235, 0.17809523809523806, 0.17870370370370367, 0.1810810810810811, 0.18333333333333332, 0.18461538461538463, 0.18416666666666667, 0.18292682926829265, 0.18095238095238092, 0.17984496124031008, 0.18181818181818182, 0.18074074074074073, 0.17971014492753623, 0.18085106382978722, 0.1819444444444444, 0.18231292517006803, 0.18333333333333332]}, "msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.016666666666666666, 0.011111111111111112, 0.008333333333333333, 0.006666666666666667, 0.005555555555555555, 0.004761904761904763, 0.004166666666666667, 0.003703703703703704, 0.006666666666666668, 0.006060606060606061, 0.005555555555555556, 0.005128205128205128, 0.007142857142857143, 0.008888888888889071, 0.014583333333333334, 0.01764705882352941, 0.02037037037037037, 0.01929824561403509, 0.023333333333333334, 0.025396825396825393, 0.02727272727272727, 0.03333333333333333, 0.0361111111111111, 0.037333333333333336, 0.038461538461538464, 0.043209876543209874, 0.04880952380952381, 0.05057471264367815, 0.05555555555555555, 0.05913978494623656, 0.06354166666666666, 0.06868686868686869, 0.07450980392156863, 0.0790476190476192, 0.0842592592592594, 0.0936936936936937, 0.10263157894736842, 0.10854700854700855, 0.1125, 0.11951219512195134, 0.1238095238095238, 0.12868217054263567, 0.1340909090909091, 0.13925925925925925, 0.14492753623188406, 0.1546099290780143, 0.16388888888888897, 0.17210884353741496, 0.18333333333333332]}, "temp_msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.016666666666666666, 0.011111111111111112, 0.008333333333333333, 0.006666666666666667, 0.005555555555555555, 0.004761904761904763, 0.004166666666666667, 0.007407407407407408, 0.006666666666666668, 0.00909090909090909, 0.011111111111111112, 0.010256410256410256, 0.011904761904761904, 0.01111111111111111, 0.014583333333333334, 0.01568627450980392, 0.018518518518518514, 0.02456140350877193, 0.02666666666666667, 0.025396825396825393, 0.025757575757575757, 0.02608695652173913, 0.027777777777777773, 0.03333333333333333, 0.035897435897435895, 0.037037037037037035, 0.036904761904761905, 0.04482758620689673, 0.051111111111111114, 0.054838709677419356, 0.059375, 0.06565656565656566, 0.06862745098039216, 0.07238095238095237, 0.07870370370370369, 0.08738738738738738, 0.09736842105263158, 0.10341880341880341, 0.10583333333333333, 0.1113821138211382, 0.11904761904761903, 0.12713178294573643, 0.13257575757575757, 0.13925925925925925, 0.14565217391304347, 0.15460992907801416, 0.16180555555555554, 0.17142857142857143, 0.18333333333333332]}, "entropy": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.011111111111111112, 0.008333333333333333, 0.006666666666666667, 0.005555555555555555, 0.004761904761904763, 0.008333333333333333, 0.014814814814814815, 0.01666666666666667, 0.015151515151515152, 0.030555555555555555, 0.03333333333333333, 0.04047619047619048, 0.04666666666666666, 0.04791666666666667, 0.054901960784313725, 0.06296296296296294, 0.07192982456140351, 0.07166666666666667, 0.073015873015873, 0.07727272727272727, 0.08115942028985507, 0.08472222222222221, 0.08533333333333333, 0.0858974358974359, 0.08888888888888889, 0.0880952380952381, 0.09080459770114942, 0.09444444444444444, 0.09462365591397849, 0.09479166666666666, 0.09696969696969697, 0.09607843137254903, 0.09904761904761904, 0.09999999999999999, 0.1036036036036036, 0.10614035087719298, 0.10854700854700855, 0.11333333333333333, 0.11707317073170731, 0.12301587301587312, 0.12713178294573643, 0.1303030303030303, 0.1385185185185185, 0.1463768115942029, 0.15815602836879444, 0.16666666666666674, 0.17551020408163265, 0.18333333333333332]}, "softmax_learned": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.016666666666666666, 0.011111111111111112, 0.008333333333333333, 0.006666666666666667, 0.005555555555555555, 0.004761904761904763, 0.004166666666666667, 0.003703703703703704, 0.006666666666666668, 0.006060606060606061, 0.005555555555555556, 0.007692307692307693, 0.007142857142857143, 0.008888888888888887, 0.0125, 0.013725490196078431, 0.014814814814814814, 0.01929824561403509, 0.018333333333333333, 0.023809523809523805, 0.025757575757575757, 0.03188405797101449, 0.033333333333333326, 0.034666666666666665, 0.03717948717948718, 0.04197530864197531, 0.04404761904761905, 0.04712643678160919, 0.04777777777777778, 0.054838709677419356, 0.058333333333333334, 0.0595959595959596, 0.06568627450980392, 0.07142857142857158, 0.07777777777777777, 0.08828828828828829, 0.09824561403508772, 0.10512820512820513, 0.11166666666666666, 0.11544715447154484, 0.1222222222222222, 0.12868217054263567, 0.1340909090909091, 0.13925925925925925, 0.14492753623188406, 0.1496453900709221, 0.16319444444444453, 0.17210884353741496, 0.18333333333333332]}, "geom_softmax": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004166666666666667, 0.003703703703703704, 0.003333333333333334, 0.0030303030303030303, 0.002777777777777778, 0.002564102564102564, 0.002380952380952381, 0.004444444444444444, 0.00625, 0.01568627450980392, 0.014814814814814814, 0.015789473684210527, 0.018333333333333333, 0.020634920634920763, 0.022727272727272728, 0.02608695652173913, 0.029166666666666664, 0.032, 0.03076923076923077, 0.0345679012345679, 0.04047619047619048, 0.04367816091954022, 0.04666666666666667, 0.05268817204301075, 0.05520833333333333, 0.06060606060606061, 0.06568627450980392, 0.06952380952380968, 0.07870370370370369, 0.08198198198198198, 0.09210526315789473, 0.09658119658119659, 0.10416666666666667, 0.1105691056910569, 0.11666666666666665, 0.12015503875968993, 0.12878787878787878, 0.1385185185185185, 0.1463768115942029, 0.1546099290780143, 0.16388888888888886, 0.17346938775510204, 0.18333333333333332]}}, "ltt": {"alpha": 0.1, "delta": 0.05, "lambda_hat": 0.19510629211616212, "coverage": 0.7346666666666667, "abstain_rate": 0.2653333333333333, "selective_risk": 0.08076225045372051, "selective_accuracy": 0.9192377495462795, "risk_within_budget": true}, "coverage_validity": {"alpha_grid": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "achieved_risk": [0.0, 0.03294117647058824, 0.08076225045372051, 0.13363705391040243, 0.18333333333333332, 0.18333333333333332], "coverage": [0.0, 0.5666666666666667, 0.7346666666666667, 0.878, 1.0, 1.0]}, "feature_diagnostics": {"names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "auroc": {"basin/rho": 0.753239332096475, "basin/entropy": 0.2466538033395176, "basin/dispersion": 0.48666716141001853, "energy/mean": 0.5139591836734694, "energy/min": 0.512973654916512, "energy/std": 0.4924705009276438, "curv/lmax_mean": 0.4856994434137291, "curv/lmax_best": 0.5174768089053803, "curv/trace_mean": 0.48462486085343226, "curv/trace_best": 0.49396066790352505, "dynamics/steps": 0.5, "dynamics/monotonic": 0.3888029684601113, "dynamics/drop": 0.36415881261595545, "dynamics/residual": 0.48698775510204084}, "hist": {"basin/rho": {"edges": [0.5, 0.525, 0.55, 0.575, 0.6, 0.625, 0.65, 0.675, 0.7, 0.725, 0.75, 0.775, 0.8, 0.825, 0.8500000000000001, 0.875, 0.9, 0.925, 0.95, 0.9750000000000001, 1.0], "correct_counts": [19, 0, 0, 38, 0, 0, 22, 0, 0, 0, 46, 0, 0, 61, 0, 0, 126, 0, 0, 913], "incorrect_counts": [12, 0, 0, 49, 0, 0, 31, 0, 0, 0, 34, 0, 0, 33, 0, 0, 36, 0, 0, 80]}, "basin/entropy": {"edges": [0.0, 0.053756966200268666, 0.10751393240053733, 0.161270898600806, 0.21502786480107466, 0.2687848310013433, 0.322541797201612, 0.3762987634018807, 0.43005572960214933, 0.483812695802418, 0.5375696620026866, 0.5913266282029553, 0.645083594403224, 0.6988405606034926, 0.7525975268037614, 0.80635449300403, 0.8601114592042987, 0.9138684254045674, 0.967625391604836, 1.0213823578051047, 1.0751393240053733], "correct_counts": [913, 0, 0, 0, 0, 126, 0, 0, 58, 0, 44, 21, 52, 5, 0, 1, 2, 3, 0, 0], "incorrect_counts": [80, 0, 0, 0, 0, 36, 0, 0, 30, 0, 33, 29, 55, 4, 0, 2, 2, 2, 0, 2]}, "basin/dispersion": {"edges": [1.6335429597808506, 1.6563980282608786, 1.6792530967409065, 1.7021081652209342, 1.7249632337009622, 1.74781830218099, 1.770673370661018, 1.793528439141046, 1.8163835076210737, 1.8392385761011016, 1.8620936445811296, 1.8849487130611575, 1.9078037815411855, 1.9306588500212132, 1.9535139185012411, 1.976368986981269, 1.999224055461297, 2.022079123941325, 2.0449341924213527, 2.067789260901381, 2.0906443293814085], "correct_counts": [4, 17, 24, 49, 72, 120, 146, 160, 154, 138, 136, 83, 51, 36, 13, 11, 7, 3, 0, 1], "incorrect_counts": [2, 5, 6, 11, 12, 20, 36, 41, 31, 32, 28, 18, 14, 9, 5, 4, 1, 0, 0, 0]}, "energy/mean": {"edges": [0.2628147800763448, 0.32859045515457785, 0.394366130232811, 0.4601418053110441, 0.5259174803892772, 0.5916931554675102, 0.6574688305457433, 0.7232445056239765, 0.7890201807022095, 0.8547958557804427, 0.9205715308586757, 0.9863472059369087, 1.052122881015142, 1.117898556093375, 1.183674231171608, 1.2494499062498412, 1.3152255813280742, 1.3810012564063072, 1.4467769314845405, 1.5125526065627735, 1.5783282816410065], "correct_counts": [4, 11, 34, 36, 54, 72, 130, 190, 248, 209, 112, 45, 20, 17, 12, 10, 5, 10, 4, 2], "incorrect_counts": [4, 9, 24, 18, 26, 14, 19, 22, 26, 26, 13, 16, 8, 4, 5, 7, 9, 16, 7, 2]}, "energy/min": {"edges": [-0.1004953682422638, -0.031788365542888636, 0.03691863715648652, 0.10562563985586168, 0.17433264255523684, 0.24303964525461197, 0.31174664795398715, 0.38045365065336234, 0.44916065335273747, 0.5178676560521126, 0.5865746587514877, 0.655281661450863, 0.7239886641502381, 0.7926956668496132, 0.8614026695489885, 0.9301096722483635, 0.9988166749477387, 1.067523677647114, 1.136230680346489, 1.2049376830458642, 1.2736446857452393], "correct_counts": [2, 5, 20, 30, 59, 66, 94, 170, 219, 219, 154, 79, 37, 25, 15, 4, 8, 10, 6, 3], "incorrect_counts": [2, 5, 15, 26, 20, 21, 11, 28, 20, 22, 22, 22, 10, 2, 12, 3, 13, 10, 9, 2]}, "energy/std": {"edges": [0.0727182563720062, 0.08970085308900838, 0.10668344980601056, 0.12366604652301275, 0.14064864324001491, 0.1576312399570171, 0.1746138366740193, 0.19159643339102148, 0.20857903010802364, 0.22556162682502584, 0.24254422354202804, 0.2595268202590302, 0.2765094169760324, 0.29349201369303457, 0.31047461041003677, 0.3274572071270389, 0.3444398038440411, 0.3614224005610433, 0.3784049972780455, 0.39538759399504764, 0.41237019071204983], "correct_counts": [3, 12, 31, 78, 116, 175, 179, 175, 161, 122, 66, 39, 36, 17, 7, 4, 2, 1, 0, 1], "incorrect_counts": [1, 4, 5, 17, 25, 32, 50, 37, 36, 23, 15, 11, 8, 7, 2, 1, 0, 0, 0, 1]}, "curv/lmax_mean": {"edges": [0.9985763331254324, 0.9991151432196299, 0.9996539533138276, 1.0001927634080252, 1.0007315735022226, 1.0012703835964203, 1.001809193690618, 1.0023480037848156, 1.002886813879013, 1.0034256239732107, 1.0039644340674083, 1.0045032441616057, 1.0050420542558034, 1.005580864350001, 1.0061196744441987, 1.0066584845383961, 1.0071972946325938, 1.0077361047267914, 1.008274914820989, 1.0088137249151865, 1.0093525350093842], "correct_counts": [5, 75, 353, 340, 144, 100, 71, 43, 38, 24, 11, 8, 3, 3, 3, 2, 0, 0, 1, 1], "incorrect_counts": [2, 42, 63, 45, 18, 19, 13, 16, 14, 11, 3, 8, 9, 4, 2, 1, 4, 0, 0, 1]}, "curv/lmax_best": {"edges": [0.9949193000793457, 0.9967200636863709, 0.998520827293396, 1.000321590900421, 1.0021223545074462, 1.0039231181144714, 1.0057238817214966, 1.0075246453285218, 1.0093254089355468, 1.011126172542572, 1.0129269361495972, 1.0147276997566224, 1.0165284633636475, 1.0183292269706725, 1.0201299905776977, 1.021930754184723, 1.023731517791748, 1.0255322813987733, 1.0273330450057983, 1.0291338086128234, 1.0309345722198486], "correct_counts": [2, 7, 798, 299, 53, 28, 20, 8, 3, 2, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [0, 5, 168, 50, 14, 12, 7, 5, 6, 1, 2, 2, 0, 0, 1, 0, 0, 1, 0, 1]}, "curv/trace_mean": {"edges": [31.87547731399536, 31.885582343737283, 31.89568737347921, 31.90579240322113, 31.915897432963053, 31.92600246270498, 31.9361074924469, 31.946212522188823, 31.956317551930745, 31.96642258167267, 31.976527611414593, 31.986632641156515, 31.99673767089844, 32.00684270064036, 32.016947730382284, 32.02705276012421, 32.03715778986613, 32.04726281960806, 32.05736784934998, 32.0674728790919, 32.077577908833824], "correct_counts": [1, 4, 3, 10, 7, 8, 13, 5, 9, 18, 31, 112, 275, 293, 177, 112, 112, 28, 6, 1], "incorrect_counts": [1, 7, 7, 14, 9, 5, 1, 4, 6, 9, 12, 14, 26, 39, 16, 30, 32, 25, 14, 4]}, "curv/trace_best": {"edges": [31.76333999633789, 31.782129287719727, 31.800918579101562, 31.8197078704834, 31.838497161865234, 31.85728645324707, 31.876075744628906, 31.894865036010742, 31.913654327392578, 31.932443618774414, 31.95123291015625, 31.970022201538086, 31.988811492919922, 32.00760078430176, 32.026390075683594, 32.04517936706543, 32.063968658447266, 32.0827579498291, 32.10154724121094, 32.12033653259277, 32.13912582397461], "correct_counts": [0, 0, 1, 1, 2, 4, 6, 5, 8, 24, 28, 67, 429, 398, 160, 69, 15, 6, 2, 0], "incorrect_counts": [1, 0, 0, 1, 3, 4, 8, 7, 8, 13, 11, 20, 49, 61, 38, 23, 19, 7, 1, 1]}, "dynamics/steps": {"edges": [1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7000000000000002, 1.75, 1.8, 1.85, 1.9, 1.9500000000000002, 2.0], "correct_counts": [1225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/monotonic": {"edges": [0.7166666666666667, 0.7225, 0.7283333333333334, 0.7341666666666667, 0.74, 0.7458333333333333, 0.7516666666666667, 0.7575000000000001, 0.7633333333333334, 0.7691666666666668, 0.7750000000000001, 0.7808333333333334, 0.7866666666666667, 0.7925000000000001, 0.7983333333333335, 0.8041666666666668, 0.8100000000000002, 0.8158333333333334, 0.8216666666666668, 0.8275000000000001, 0.8333333333333335], "correct_counts": [0, 4, 12, 26, 70, 112, 154, 221, 164, 186, 98, 80, 49, 26, 14, 6, 2, 1, 0, 0], "incorrect_counts": [1, 2, 2, 7, 7, 17, 26, 34, 29, 36, 31, 18, 23, 13, 12, 9, 4, 1, 2, 1]}, "dynamics/drop": {"edges": [61.334716603159904, 75.90191677833597, 90.46911695351203, 105.0363171286881, 119.60351730386415, 134.1707174790402, 148.7379176542163, 163.30511782939234, 177.8723180045684, 192.43951817974448, 207.00671835492054, 221.5739185300966, 236.14111870527267, 250.70831888044873, 265.2755190556248, 279.84271923080087, 294.4099194059769, 308.977119581153, 323.54431975632906, 338.1115199315051, 352.67872010668117], "correct_counts": [9, 83, 206, 341, 278, 152, 75, 26, 21, 10, 7, 4, 3, 4, 1, 4, 0, 0, 0, 1], "incorrect_counts": [4, 13, 31, 49, 51, 30, 23, 11, 10, 10, 9, 9, 6, 2, 5, 3, 3, 2, 3, 1]}, "dynamics/residual": {"edges": [1.1505316942930222, 1.167380820463101, 1.18422994663318, 1.201079072803259, 1.2179281989733377, 1.2347773251434166, 1.2516264513134956, 1.2684755774835745, 1.2853247036536535, 1.3021738298237324, 1.3190229559938111, 1.33587208216389, 1.352721208333969, 1.369570334504048, 1.386419460674127, 1.4032685868442059, 1.4201177130142848, 1.4369668391843635, 1.4538159653544425, 1.4706650915245214, 1.4875142176946003], "correct_counts": [5, 5, 18, 43, 58, 105, 154, 167, 200, 158, 129, 72, 53, 26, 19, 5, 5, 2, 0, 1], "incorrect_counts": [2, 1, 5, 12, 12, 28, 27, 35, 35, 42, 25, 19, 14, 11, 3, 4, 0, 0, 0, 0]}}}, "feature_ablation": {"full": 0.05870278336778761, "drop_basin": 0.1212250149219976, "basin_only": 0.10427090599426178, "drop_energy": 0.06538158737574412, "energy_only": 0.22639015279001842, "drop_curv": 0.06922486202954758, "curv_only": 0.1307398782738768, "drop_dynamics": 0.05576840446247957, "dynamics_only": 0.14639777564824447}, "ece_geometry": 0.06136028513989912, "accuracy_ood": 0.228, "aurc_ood": {"geometry": 0.7893819156935301, "rho_basin": 0.7193987522326036, "energy_min": 0.775468743535553, "energy_mean": 0.7822174952275276, "energy_std": 0.78991149550709, "msp": 0.6564852425996128, "temp_msp": 0.668475764448379, "entropy": 0.645240327250456, "softmax_learned": 0.6585912199386484, "geom_softmax": 0.688751813066032}, "ood_ltt": {"alpha": 0.1, "lambda_hat": 0.19510629211616212, "selective_risk": 0.7557117750439367, "coverage": 0.7586666666666667, "risk_within_budget": false}, "ood_validity": {"target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "id_risk": [0.0, 0.03294117647058824, 0.08076225045372051, 0.13363705391040243, 0.18333333333333332, 0.18333333333333332], "ood_risk": [0.0, 0.7344497607655502, 0.7557117750439367, 0.7721611721611722, 0.772, 0.772], "id_coverage": [0.0, 0.5666666666666667, 0.7346666666666667, 0.878, 1.0, 1.0], "ood_coverage": [0.0, 0.5573333333333333, 0.7586666666666667, 0.91, 1.0, 1.0]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "470896cedd8b", "timestamp": "2026-07-30T11:18:11.814580+00:00", "git_sha": "67dc7a2", "config_hash": "11f62a585e6b", "config": {"run": {"seed": 3, "task": "graph_planning", "notes": "graph adaptive-halting run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 25, "batch_size": 128, "lr": 0.001, "n_train": 8000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 800, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 800}, "task": {"graph_planning": {"n_nodes": 7, "ood_n_nodes": 10, "edge_prob": 0.4, "max_len": 4}}}, "task": "graph_planning", "split": "halting", "seed": 3, "metrics": {"k_restarts": 12, "steps": 50, "alpha": 0.1, "n_calib": 800, "n_test": 800, "final_train_loss": 0.16939383745193481, "tau_hat": 0.8333, "compute_used": 0.16787500000000002, "compute_saved": 0.832125, "halting_risk": 0.09, "risk_within_budget": true, "full_accuracy": 0.695, "halted_accuracy": 0.6725, "accuracy_drop": 0.022499999999999964, "risk_monotone_in_tau": true, "calib_risk_by_tau": [0.445, 0.445, 0.445, 0.4425, 0.36625, 0.36625, 0.3, 0.14625, 0.14625, 0.09, 0.01375, 0.01375], "calib_compute_by_tau": [0.0, 0.0, 0.0, 0.00034999999999999994, 0.020199999999999853, 0.020199999999999853, 0.04342499999999971, 0.11570000000000019, 0.11570000000000019, 0.16180000000000028, 0.28835, 0.28835], "tau_sweep": {"taus": [0.0833, 0.1667, 0.25, 0.3333, 0.4167, 0.5, 0.5833, 0.6667, 0.75, 0.8333, 0.9167, 1.0], "compute_used": [0.0, 0.0, 0.0, 0.00025, 0.021575, 0.021575, 0.04630000000000001, 0.11372500000000002, 0.11372500000000002, 0.16787500000000002, 0.3007, 0.3007], "accuracy": [0.4825, 0.4825, 0.4825, 0.4825, 0.5375, 0.5375, 0.57375, 0.65, 0.65, 0.6725, 0.6975, 0.6975], "disagreement": [0.45625, 0.45625, 0.45625, 0.455, 0.36625, 0.36625, 0.29375, 0.15, 0.15, 0.09, 0.00625, 0.00625]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "52b5abb0b7f5", "timestamp": "2026-07-30T11:18:16.252284+00:00", "git_sha": "67dc7a2", "config_hash": "4a05527e96db", "config": {"run": {"seed": 4, "task": "arithmetic", "notes": "E1 selective-prediction main run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 7, "batch_size": 128, "lr": 0.001, "n_train": 6000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 1500, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 1500}, "task": {"arithmetic": {"modular": false, "n_operands": 6, "ood_n_operands": 6, "max_operand": 7, "ood_max_operand": 9}}}, "task": "arithmetic", "split": "selective", "seed": 4, "metrics": {"n_fit": 1500, "n_calib": 1500, "n_test": 1500, "k_restarts": 12, "objective": "basin_center", "sampler": "langevin", "feature_set": "base", "accuracy_id": 0.8186666666666667, "base_error": 0.18133333333333335, "final_train_loss": 0.8476110100746155, "feature_names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "aurc": {"geometry": 0.06427468985900764, "rho_basin": 0.0938341849888178, "energy_min": 0.16779814857981604, "energy_mean": 0.16802618351251167, "energy_std": 0.18329896021538222, "msp": 0.04726303074136766, "temp_msp": 0.04804395890364349, "entropy": 0.055194322709484635, "softmax_learned": 0.04644594370809505, "geom_softmax": 0.04320914064226967}, "temperature": 0.32684245355705593, "best_energy_baseline": "energy_min", "best_baseline": "msp", "delta_aurc_vs_energy_min": [0.1035234587208084, 0.08007861277177743, 0.12748935525846986], "delta_aurc_vs_best_energy": [0.1035234587208084, 0.08007861277177743, 0.12748935525846986], "delta_aurc_vs_best_baseline": [-0.01701165911763998, -0.029246370318950058, -0.00695285101955189], "delta_aurc_geom_adds": [0.0032368030658253832, 0.0009814347315345426, 0.005577143152198716], "geometry_wins": true, "geometry_wins_vs_baseline": false, "geometry_adds_over_softmax": true, "risk_coverage": {"geometry": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.06666666666666667, 0.03333333333333333, 0.044444444444444446, 0.03333333333333333, 0.04, 0.03888888888888888, 0.038095238095238106, 0.04583333333333333, 0.04814814814814815, 0.04333333333333334, 0.03939393939393939, 0.03888888888888889, 0.041025641025641026, 0.04047619047619048, 0.03777777777777796, 0.041666666666666664, 0.0392156862745098, 0.03888888888888888, 0.03859649122807018, 0.03833333333333333, 0.0365079365079365, 0.03636363636363636, 0.036231884057971016, 0.03888888888888888, 0.03866666666666667, 0.041025641025641026, 0.040740740740740744, 0.039285714285714285, 0.04137931034482758, 0.044444444444444446, 0.04408602150537634, 0.046875, 0.052525252525252523, 0.058823529411764705, 0.06666666666666665, 0.07314814814814813, 0.08018018018018018, 0.08421052631578947, 0.08803418803418804, 0.09583333333333334, 0.1032520325203252, 0.1119047619047619, 0.12170542635658915, 0.1318181818181818, 0.13703703703703704, 0.14130434782608695, 0.14751773049645386, 0.15694444444444455, 0.16870748299319727, 0.18133333333333335]}, "rho_basin": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0789724072312084, 0.07897240723120841, 0.07897240723120832, 0.07897240723120841, 0.07897240723120853, 0.07897240723120862, 0.07897240723120866, 0.07897240723120871, 0.07897240723120874, 0.07897240723120877, 0.0789724072312088, 0.07897240723120881, 0.07897240723120882, 0.07897240723120871, 0.07897240723120849, 0.07897240723120831, 0.07897240723120814, 0.07897240723120799, 0.07897240723120785, 0.07897240723120774, 0.07897240723120763, 0.07897240723120753, 0.07897240723120744, 0.07897240723120737, 0.07897240723120728, 0.07897240723120721, 0.07897240723120715, 0.07897240723120709, 0.07897240723120703, 0.07897240723120698, 0.07897240723120694, 0.07897240723120688, 0.07897240723120684, 0.0789724072312068, 0.07897240723120677, 0.08567985794013051, 0.09224978403060469, 0.09847392453736972, 0.10437887835148014, 0.11010416666666574, 0.11656504065040564, 0.12271825396825312, 0.1285852713178286, 0.13747294372294272, 0.14625081221572334, 0.1526061530638181, 0.15899906329452637, 0.16708595387840589, 0.1745857317285879, 0.18133333333333207]}, "energy_min": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.2, 0.16666666666666666, 0.14444444444444443, 0.14166666666666666, 0.14, 0.13888888888888887, 0.14761904761904746, 0.15416666666666667, 0.15555555555555556, 0.17333333333333337, 0.18787878787878787, 0.18611111111111112, 0.17692307692307693, 0.18095238095238095, 0.18888888888888886, 0.19166666666666668, 0.18823529411764706, 0.187037037037037, 0.18421052631578946, 0.18333333333333332, 0.17936507936507934, 0.17575757575757575, 0.1753623188405797, 0.16944444444444443, 0.17466666666666666, 0.17051282051282052, 0.16790123456790124, 0.16547619047619047, 0.164367816091954, 0.1622222222222222, 0.16129032258064516, 0.16145833333333334, 0.1606060606060606, 0.1588235294117647, 0.15714285714285728, 0.1583333333333333, 0.15855855855855855, 0.1587719298245614, 0.15726495726495726, 0.1575, 0.16016260162601623, 0.1603174603174603, 0.15968992248062017, 0.15984848484848485, 0.16, 0.16376811594202897, 0.16879432624113486, 0.17430555555555552, 0.1761904761904762, 0.18133333333333335]}, "energy_mean": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.23333333333333334, 0.11666666666666667, 0.14444444444444443, 0.125, 0.13333333333333333, 0.1333333333333333, 0.14285714285714288, 0.1625, 0.17037037037037037, 0.1766666666666667, 0.1787878787878788, 0.16944444444444445, 0.16666666666666666, 0.17142857142857143, 0.17777777777777776, 0.18333333333333332, 0.1843137254901961, 0.1907407407407407, 0.18947368421052632, 0.18333333333333332, 0.1857142857142857, 0.18333333333333332, 0.17971014492753623, 0.18055555555555564, 0.18, 0.17435897435897435, 0.16913580246913582, 0.17023809523809524, 0.16666666666666663, 0.16333333333333333, 0.16344086021505377, 0.16354166666666667, 0.1606060606060606, 0.1607843137254902, 0.15809523809523807, 0.15648148148148144, 0.15675675675675677, 0.1570175438596491, 0.1564102564102564, 0.15583333333333332, 0.15528455284552842, 0.1539682539682541, 0.1558139534883721, 0.15833333333333333, 0.1637037037037037, 0.16376811594202897, 0.1652482269503547, 0.17013888888888887, 0.1761904761904762, 0.18133333333333335]}, "energy_std": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.26666666666666666, 0.18333333333333332, 0.17777777777777778, 0.2, 0.17333333333333334, 0.18888888888888886, 0.17619047619047623, 0.17083333333333334, 0.18888888888888888, 0.1866666666666667, 0.18484848484848485, 0.19166666666666668, 0.18974358974358974, 0.19285714285714287, 0.1933333333333333, 0.18958333333333333, 0.1843137254901961, 0.1796296296296296, 0.1824561403508772, 0.17833333333333334, 0.17936507936507934, 0.18484848484848485, 0.1826086956521739, 0.18055555555555552, 0.17866666666666667, 0.18205128205128204, 0.17901234567901234, 0.17857142857142858, 0.17931034482758618, 0.18, 0.17849462365591398, 0.17604166666666668, 0.17575757575757575, 0.17941176470588235, 0.17999999999999997, 0.1851851851851853, 0.1837837837837838, 0.1850877192982456, 0.18461538461538463, 0.1825, 0.18455284552845538, 0.1841269841269841, 0.1813953488372093, 0.18181818181818182, 0.18074074074074073, 0.17898550724637682, 0.17872340425531913, 0.18055555555555564, 0.18095238095238095, 0.18133333333333335]}, "msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.002777777777777778, 0.007692307692307693, 0.007142857142857143, 0.008888888888888887, 0.008333333333333333, 0.00980392156862745, 0.01111111111111111, 0.014035087719298246, 0.013333333333333334, 0.02063492063492063, 0.019696969696969695, 0.020289855072463767, 0.02222222222222222, 0.025333333333333333, 0.029487179487179487, 0.028395061728395062, 0.03214285714285714, 0.03563218390804597, 0.03777777777777778, 0.04086021505376344, 0.04583333333333333, 0.050505050505050504, 0.05392156862745098, 0.060952380952380945, 0.06481481481481495, 0.07387387387387387, 0.08421052631578947, 0.09145299145299145, 0.09916666666666667, 0.10894308943089444, 0.11507936507936506, 0.12558139534883722, 0.13484848484848486, 0.14148148148148149, 0.1463768115942029, 0.15248226950354607, 0.1597222222222223, 0.17006802721088435, 0.18133333333333335]}, "temp_msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.003333333333333334, 0.0030303030303030303, 0.002777777777777778, 0.002564102564102564, 0.002380952380952381, 0.002222222222222222, 0.00625, 0.0058823529411764705, 0.007407407407407407, 0.008771929824561403, 0.013333333333333334, 0.017460317460317457, 0.019696969696969695, 0.01884057971014493, 0.02083333333333333, 0.024, 0.02435897435897436, 0.028395061728395062, 0.02976190476190476, 0.033333333333333326, 0.03666666666666667, 0.03870967741935484, 0.04895833333333333, 0.05454545454545454, 0.05784313725490196, 0.06761904761904777, 0.07314814814814813, 0.07927927927927927, 0.08421052631578947, 0.09487179487179487, 0.10416666666666667, 0.1121951219512195, 0.11904761904761915, 0.1263565891472868, 0.13787878787878788, 0.14444444444444443, 0.1536231884057971, 0.16099290780141842, 0.16666666666666674, 0.1727891156462585, 0.18133333333333335]}, "entropy": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004166666666666667, 0.003703703703703704, 0.003333333333333334, 0.0030303030303030303, 0.005555555555555556, 0.010256410256410256, 0.009523809523809525, 0.013333333333333332, 0.01875, 0.01764705882352941, 0.02037037037037037, 0.02631578947368421, 0.03333333333333333, 0.0365079365079365, 0.03636363636363636, 0.04057971014492753, 0.04166666666666666, 0.044, 0.047435897435897434, 0.04938271604938271, 0.05119047619047619, 0.0528735632183908, 0.05555555555555555, 0.05806451612903226, 0.059375, 0.06363636363636363, 0.06862745098039216, 0.07428571428571444, 0.08148148148148163, 0.08828828828828829, 0.09385964912280702, 0.1, 0.10416666666666667, 0.11056910569105703, 0.11746031746031757, 0.12170542635658915, 0.12727272727272726, 0.13555555555555557, 0.14420289855072463, 0.15602836879432622, 0.1659722222222222, 0.17482993197278912, 0.18133333333333335]}, "softmax_learned": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.007692307692307693, 0.007142857142857143, 0.008888888888888887, 0.008333333333333333, 0.00784313725490196, 0.009259259259259257, 0.012280701754385965, 0.013333333333333334, 0.014285714285714284, 0.019696969696969695, 0.01884057971014493, 0.01944444444444444, 0.025333333333333333, 0.026923076923076925, 0.027160493827160494, 0.02976190476190476, 0.03448275862068965, 0.03666666666666667, 0.03870967741935484, 0.041666666666666664, 0.047474747474747475, 0.050980392156862744, 0.057142857142857134, 0.06388888888888887, 0.07027027027027027, 0.08333333333333333, 0.0905982905982906, 0.09916666666666667, 0.10731707317073169, 0.11825396825396824, 0.1263565891472868, 0.1340909090909091, 0.14074074074074075, 0.1471014492753623, 0.15248226950354607, 0.16041666666666665, 0.16938775510204082, 0.18133333333333335]}, "geom_softmax": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.002380952380952381, 0.006666666666666666, 0.00625, 0.0058823529411764705, 0.007407407407407407, 0.008771929824561403, 0.008333333333333333, 0.009523809523809523, 0.010606060606060607, 0.013043478260869565, 0.013888888888888886, 0.016, 0.016666666666666666, 0.01728395061728395, 0.02261904761904762, 0.022988505747126433, 0.02666666666666667, 0.03118279569892473, 0.034375, 0.04040404040404041, 0.05, 0.056190476190476187, 0.06944444444444443, 0.07567567567567568, 0.08596491228070176, 0.09145299145299145, 0.1025, 0.10813008130081313, 0.11428571428571427, 0.12093023255813953, 0.12727272727272726, 0.1348148148148148, 0.14057971014492754, 0.14751773049645386, 0.15555555555555567, 0.1673469387755102, 0.18133333333333335]}}, "ltt": {"alpha": 0.1, "delta": 0.05, "lambda_hat": 0.2286224610513153, "coverage": 0.7453333333333333, "abstain_rate": 0.25466666666666665, "selective_risk": 0.08050089445438283, "selective_accuracy": 0.9194991055456172, "risk_within_budget": true}, "coverage_validity": {"alpha_grid": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "achieved_risk": [0.0, 0.0, 0.08050089445438283, 0.1198443579766537, 0.180787191460974, 0.180787191460974], "coverage": [0.0, 0.0, 0.7453333333333333, 0.8566666666666667, 0.9993333333333333, 0.9993333333333333]}, "feature_diagnostics": {"names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "auroc": {"basin/rho": 0.7546809134891742, "basin/entropy": 0.24459157645142748, "basin/dispersion": 0.5018112904771029, "energy/mean": 0.44983773232420005, "energy/min": 0.4503197451619084, "energy/std": 0.5016077074152137, "curv/lmax_mean": 0.44767017148879096, "curv/lmax_best": 0.4978010035447404, "curv/trace_mean": 0.5762478444146388, "curv/trace_best": 0.5414815457942135, "dynamics/steps": 0.5, "dynamics/monotonic": 0.4247101935236635, "dynamics/drop": 0.38611324008430736, "dynamics/residual": 0.4858749281471546}, "hist": {"basin/rho": {"edges": [0.4166666666666667, 0.44583333333333336, 0.47500000000000003, 0.5041666666666667, 0.5333333333333333, 0.5625, 0.5916666666666667, 0.6208333333333333, 0.65, 0.6791666666666667, 0.7083333333333333, 0.7375, 0.7666666666666666, 0.7958333333333334, 0.825, 0.8541666666666666, 0.8833333333333333, 0.9125, 0.9416666666666667, 0.9708333333333332, 1.0], "correct_counts": [1, 0, 19, 0, 0, 24, 0, 0, 32, 0, 0, 26, 0, 0, 60, 0, 0, 98, 0, 968], "incorrect_counts": [1, 0, 20, 0, 0, 29, 0, 0, 25, 0, 0, 30, 0, 0, 36, 0, 0, 48, 0, 83]}, "basin/entropy": {"edges": [0.0, 0.05387781635334003, 0.10775563270668007, 0.16163344906002008, 0.21551126541336013, 0.2693890817667002, 0.32326689812004017, 0.3771447144733802, 0.43102253082672026, 0.4849003471800603, 0.5387781635334004, 0.5926559798867403, 0.6465337962400803, 0.7004116125934204, 0.7542894289467604, 0.8081672453001005, 0.8620450616534405, 0.9159228780067805, 0.9698006943601206, 1.0236785107134607, 1.0775563270668007], "correct_counts": [968, 0, 0, 0, 0, 98, 0, 0, 56, 0, 27, 28, 38, 3, 0, 2, 3, 3, 1, 1], "incorrect_counts": [83, 0, 0, 0, 0, 48, 0, 0, 32, 0, 30, 20, 40, 4, 0, 4, 1, 3, 3, 4]}, "basin/dispersion": {"edges": [1.5954158897257813, 1.6171290768517932, 1.6388422639778053, 1.6605554511038172, 1.6822686382298293, 1.7039818253558412, 1.7256950124818533, 1.7474081996078652, 1.7691213867338773, 1.7908345738598892, 1.8125477609859013, 1.8342609481119132, 1.855974135237925, 1.8776873223639372, 1.899400509489949, 1.9211136966159612, 1.942826883741973, 1.9645400708679852, 1.986253257993997, 2.007966445120009, 2.029679632246021], "correct_counts": [2, 3, 4, 17, 21, 42, 76, 99, 132, 162, 172, 143, 128, 92, 50, 34, 32, 5, 11, 3], "incorrect_counts": [1, 2, 2, 2, 7, 11, 18, 14, 40, 28, 30, 37, 29, 17, 9, 11, 9, 2, 2, 1]}, "energy/mean": {"edges": [0.7072046200434366, 0.7702707141637801, 0.8333368082841237, 0.8964029024044672, 0.9594689965248107, 1.0225350906451542, 1.0856011847654976, 1.1486672788858412, 1.2117333730061848, 1.2747994671265284, 1.337865561246872, 1.4009316553672153, 1.4639977494875587, 1.5270638436079023, 1.5901299377282458, 1.6531960318485894, 1.716262125968933, 1.7793282200892766, 1.8423943142096197, 1.9054604083299633, 1.9685265024503071], "correct_counts": [3, 7, 28, 55, 67, 56, 50, 51, 47, 44, 61, 68, 152, 167, 172, 90, 63, 27, 14, 6], "incorrect_counts": [2, 3, 2, 7, 14, 15, 16, 6, 14, 15, 14, 13, 20, 24, 23, 26, 21, 20, 12, 5]}, "energy/min": {"edges": [0.34151938557624817, 0.41106958836317065, 0.48061979115009307, 0.5501699939370155, 0.619720196723938, 0.6892703995108604, 0.7588206022977828, 0.8283708050847053, 0.8979210078716278, 0.9674712106585502, 1.0370214134454727, 1.106571616232395, 1.1761218190193174, 1.24567202180624, 1.3152222245931624, 1.384772427380085, 1.4543226301670074, 1.5238728329539297, 1.5934230357408523, 1.6629732385277747, 1.7325234413146973], "correct_counts": [4, 7, 14, 36, 72, 61, 65, 60, 49, 64, 85, 112, 140, 168, 120, 87, 57, 20, 5, 2], "incorrect_counts": [2, 2, 2, 6, 9, 11, 25, 11, 17, 15, 13, 18, 17, 28, 25, 23, 29, 10, 7, 2]}, "energy/std": {"edges": [0.05760100396148682, 0.07262729619130567, 0.08765358842112453, 0.1026798806509434, 0.11770617288076225, 0.13273246511058112, 0.14775875734039998, 0.16278504957021883, 0.1778113418000377, 0.19283763402985654, 0.2078639262596754, 0.22289021848949425, 0.23791651071931313, 0.25294280294913196, 0.26796909517895084, 0.2829953874087697, 0.29802167963858855, 0.31304797186840744, 0.32807426409822626, 0.34310055632804515, 0.358126848557864], "correct_counts": [1, 1, 5, 16, 47, 70, 130, 148, 166, 165, 142, 111, 89, 55, 34, 20, 9, 9, 7, 3], "incorrect_counts": [0, 1, 0, 7, 7, 19, 28, 32, 36, 34, 38, 25, 16, 6, 12, 3, 5, 2, 1, 0]}, "curv/lmax_mean": {"edges": [0.9986266444126765, 0.9987258836627007, 0.9988251229127247, 0.9989243621627489, 0.9990236014127731, 0.9991228406627972, 0.9992220799128214, 0.9993213191628456, 0.9994205584128697, 0.9995197976628939, 0.9996190369129181, 0.9997182761629422, 0.9998175154129664, 0.9999167546629905, 1.0000159939130147, 1.0001152331630387, 1.000214472413063, 1.0003137116630871, 1.0004129509131112, 1.0005121901631355, 1.0006114294131596], "correct_counts": [4, 4, 12, 17, 40, 61, 83, 170, 202, 224, 175, 145, 55, 20, 5, 7, 1, 0, 1, 2], "incorrect_counts": [0, 0, 2, 4, 6, 9, 19, 27, 41, 47, 55, 42, 16, 2, 1, 0, 0, 1, 0, 0]}, "curv/lmax_best": {"edges": [0.9928669929504395, 0.9932882905006408, 0.9937095880508423, 0.9941308856010437, 0.9945521831512452, 0.9949734807014465, 0.9953947782516479, 0.9958160758018494, 0.9962373733520508, 0.9966586709022522, 0.9970799684524536, 0.997501266002655, 0.9979225635528565, 0.9983438611030578, 0.9987651586532593, 0.9991864562034607, 0.9996077537536621, 1.0000290513038634, 1.000450348854065, 1.0008716464042664, 1.0012929439544678], "correct_counts": [1, 0, 0, 0, 3, 0, 1, 3, 3, 8, 12, 17, 24, 46, 87, 216, 787, 16, 1, 3], "incorrect_counts": [0, 0, 0, 0, 1, 1, 0, 1, 0, 2, 1, 3, 7, 7, 19, 52, 177, 1, 0, 0]}, "curv/trace_mean": {"edges": [31.90089813868205, 31.906382902463278, 31.911867666244508, 31.917352430025737, 31.922837193806966, 31.928321957588196, 31.933806721369425, 31.939291485150655, 31.944776248931884, 31.950261012713113, 31.955745776494346, 31.961230540275576, 31.966715304056805, 31.972200067838035, 31.977684831619264, 31.983169595400494, 31.988654359181723, 31.994139122962952, 31.999623886744182, 32.00510865052541, 32.01059341430664], "correct_counts": [1, 0, 3, 9, 12, 18, 28, 35, 60, 119, 205, 224, 185, 111, 75, 58, 48, 26, 7, 4], "incorrect_counts": [2, 3, 6, 9, 10, 17, 11, 14, 24, 20, 26, 26, 24, 24, 16, 20, 11, 8, 0, 1]}, "curv/trace_best": {"edges": [31.815509796142578, 31.82562713623047, 31.83574447631836, 31.84586181640625, 31.85597915649414, 31.86609649658203, 31.876213836669923, 31.886331176757814, 31.8964485168457, 31.906565856933593, 31.916683197021484, 31.926800537109376, 31.936917877197267, 31.947035217285155, 31.957152557373046, 31.967269897460938, 31.97738723754883, 31.98750457763672, 31.997621917724608, 32.0077392578125, 32.01785659790039], "correct_counts": [2, 0, 0, 0, 2, 2, 5, 7, 10, 27, 39, 61, 89, 141, 208, 232, 221, 134, 40, 8], "incorrect_counts": [1, 0, 0, 0, 1, 6, 4, 6, 3, 3, 18, 17, 23, 23, 46, 36, 38, 37, 8, 2]}, "dynamics/steps": {"edges": [1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7000000000000002, 1.75, 1.8, 1.85, 1.9, 1.9500000000000002, 2.0], "correct_counts": [1228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/monotonic": {"edges": [0.7133333333333334, 0.7192500000000001, 0.7251666666666667, 0.7310833333333334, 0.737, 0.7429166666666667, 0.7488333333333334, 0.75475, 0.7606666666666667, 0.7665833333333334, 0.7725, 0.7784166666666666, 0.7843333333333333, 0.79025, 0.7961666666666667, 0.8020833333333334, 0.808, 0.8139166666666666, 0.8198333333333333, 0.82575, 0.8316666666666667], "correct_counts": [1, 0, 1, 16, 23, 81, 100, 182, 139, 225, 208, 103, 97, 28, 14, 7, 0, 2, 1, 0], "incorrect_counts": [0, 0, 1, 5, 12, 15, 21, 26, 25, 34, 30, 27, 29, 18, 14, 4, 6, 1, 2, 2]}, "dynamics/drop": {"edges": [67.27662276228268, 83.98704353620607, 100.69746431012948, 117.40788508405288, 134.11830585797628, 150.82872663189966, 167.53914740582306, 184.24956817974646, 200.95998895366984, 217.67040972759327, 234.38083050151664, 251.09125127544002, 267.80167204936345, 284.5120928232868, 301.2225135972102, 317.9329343711336, 334.643355145057, 351.3537759189804, 368.0641966929038, 384.7746174668272, 401.4850382407506], "correct_counts": [32, 125, 299, 388, 250, 73, 29, 10, 8, 4, 4, 1, 3, 2, 0, 0, 0, 0, 0, 0], "incorrect_counts": [7, 27, 53, 44, 42, 26, 18, 14, 10, 9, 6, 5, 5, 2, 1, 0, 0, 1, 1, 1]}, "dynamics/residual": {"edges": [1.1324152251084645, 1.1493771066268283, 1.1663389881451924, 1.1833008696635563, 1.2002627511819204, 1.2172246327002842, 1.2341865142186481, 1.2511483957370122, 1.268110277255376, 1.2850721587737401, 1.302034040292104, 1.318995921810468, 1.335957803328832, 1.3529196848471958, 1.36988156636556, 1.3868434478839238, 1.4038053294022879, 1.4207672109206517, 1.4377290924390156, 1.4546909739573797, 1.4716528554757435], "correct_counts": [0, 2, 3, 12, 27, 70, 95, 129, 174, 178, 180, 122, 103, 65, 37, 16, 11, 3, 0, 1], "incorrect_counts": [1, 0, 3, 3, 4, 7, 21, 40, 32, 37, 33, 34, 27, 15, 8, 3, 4, 0, 0, 0]}}}, "feature_ablation": {"full": 0.06427468985900764, "drop_basin": 0.14673850433647762, "basin_only": 0.09760604064778008, "drop_energy": 0.06786327554908943, "energy_only": 0.16911957473150377, "drop_curv": 0.06430618294867559, "curv_only": 0.16700293935048874, "drop_dynamics": 0.08141400025100816, "dynamics_only": 0.15248745127459712}, "ece_geometry": 0.04329479322788476, "accuracy_ood": 0.20533333333333334, "aurc_ood": {"geometry": 0.8356043455050088, "rho_basin": 0.7448845820671743, "energy_min": 0.8118924901529568, "energy_mean": 0.8124673916344924, "energy_std": 0.7923987846566023, "msp": 0.6753445881260701, "temp_msp": 0.6744023348611631, "entropy": 0.680160911605275, "softmax_learned": 0.6733183754314852, "geom_softmax": 0.7053885590469356}, "ood_ltt": {"alpha": 0.1, "lambda_hat": 0.2286224610513153, "selective_risk": 0.7856517935258093, "coverage": 0.762, "risk_within_budget": false}, "ood_validity": {"target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "id_risk": [0.0, 0.0, 0.08050089445438283, 0.1198443579766537, 0.180787191460974, 0.180787191460974], "ood_risk": [0.0, 0.0, 0.7856517935258093, 0.7973568281938326, 0.7945296864576384, 0.7945296864576384], "id_coverage": [0.0, 0.0, 0.7453333333333333, 0.8566666666666667, 0.9993333333333333, 0.9993333333333333], "ood_coverage": [0.0, 0.0, 0.762, 0.908, 0.9993333333333333, 0.9993333333333333]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "9202e8561acd", "timestamp": "2026-07-30T11:18:17.733920+00:00", "git_sha": "67dc7a2", "config_hash": "882cd7965583", "config": {"run": {"seed": 2, "task": "graph_planning", "notes": "E2 graph selective-prediction"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 25, "batch_size": 128, "lr": 0.001, "n_train": 8000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 1500, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 1500}, "task": {"graph_planning": {"n_nodes": 7, "ood_n_nodes": 10, "edge_prob": 0.4, "max_len": 4}}}, "task": "graph_planning", "split": "selective", "seed": 2, "metrics": {"n_fit": 1500, "n_calib": 1500, "n_test": 1500, "k_restarts": 12, "objective": "basin_center", "sampler": "langevin", "feature_set": "base", "accuracy_id": 0.676, "base_error": 0.324, "final_train_loss": 0.178430438041687, "feature_names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "aurc": {"geometry": 0.171055380110865, "rho_basin": 0.27572696927818435, "energy_min": 0.36405839474901913, "energy_mean": 0.36247574254442705, "energy_std": 0.31446505154630827, "msp": 0.1492232493426811, "temp_msp": 0.1400399776291398, "entropy": 0.14691654031820944, "softmax_learned": 0.13872939213414487, "geom_softmax": 0.13922822696499798}, "temperature": 3.0741721331379974, "best_energy_baseline": "energy_std", "best_baseline": "temp_msp", "delta_aurc_vs_energy_min": [0.19300301463815414, 0.16374700571598869, 0.2228419667503917], "delta_aurc_vs_best_energy": [0.14340967143544328, 0.11666052233185831, 0.17035195020051339], "delta_aurc_vs_best_baseline": [-0.031015402481725185, -0.041904924234315785, -0.02069570943951534], "delta_aurc_geom_adds": [-0.0004988348308531121, -0.004262736925938849, 0.002823621624244659], "geometry_wins": true, "geometry_wins_vs_baseline": false, "geometry_adds_over_softmax": false, "risk_coverage": {"geometry": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.03333333333333333, 0.03333333333333333, 0.05, 0.06, 0.0611111111111111, 0.052380952380952396, 0.058333333333333334, 0.05925925925925926, 0.07333333333333335, 0.07575757575757576, 0.08055555555555556, 0.08717948717948718, 0.1, 0.11777777777777775, 0.12083333333333333, 0.13333333333333333, 0.13888888888888903, 0.15263157894736842, 0.15666666666666668, 0.16825396825396824, 0.16515151515151516, 0.1753623188405797, 0.18055555555555552, 0.18133333333333335, 0.18461538461538463, 0.18518518518518517, 0.19166666666666668, 0.19885057471264383, 0.20555555555555555, 0.2129032258064516, 0.2125, 0.21616161616161617, 0.22254901960784312, 0.22761904761904772, 0.2296296296296296, 0.23603603603603604, 0.24473684210526317, 0.2504273504273504, 0.25583333333333336, 0.26178861788617896, 0.2682539682539682, 0.27674418604651163, 0.2825757575757576, 0.2911111111111111, 0.2920289855072464, 0.298581560283688, 0.30555555555555564, 0.3149659863945578, 0.324]}, "rho_basin": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.2713365539452495, 0.27133655394524997, 0.2713365539452501, 0.2713365539452501, 0.2713365539452494, 0.27133655394524897, 0.2713365539452487, 0.27133655394524847, 0.2713365539452483, 0.27133655394524814, 0.271336553945248, 0.2713365539452479, 0.2713365539452478, 0.27133655394524775, 0.2713365539452477, 0.27133655394524764, 0.2713365539452476, 0.2713365539452475, 0.2713365539452475, 0.27133655394524747, 0.2713365539452474, 0.2713365539452474, 0.27133655394524736, 0.27133655394524736, 0.27133655394524736, 0.2713365539452473, 0.2713365539452473, 0.2713365539452473, 0.27133655394524725, 0.27133655394524725, 0.27133655394524725, 0.2713365539452477, 0.2713365539452485, 0.2713365539452493, 0.2713365539452501, 0.27133655394525075, 0.2713365539452514, 0.271336553945252, 0.27133655394525263, 0.2713365539452532, 0.27133655394525374, 0.27440476190476587, 0.27932816537468025, 0.2850505050505078, 0.2945185185185211, 0.30036231884058223, 0.30460992907801665, 0.30988247863248064, 0.3159183673469404, 0.32400000000000173]}, "energy_min": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.3, 0.38333333333333336, 0.3888888888888889, 0.35, 0.36666666666666664, 0.3722222222222223, 0.3809523809523808, 0.38333333333333336, 0.3888888888888889, 0.3999999999999999, 0.4, 0.39166666666666666, 0.382051282051282, 0.38571428571428573, 0.38222222222222235, 0.38125, 0.3843137254901961, 0.3833333333333333, 0.3824561403508772, 0.38166666666666665, 0.3841269841269841, 0.38181818181818183, 0.3739130434782609, 0.36805555555555564, 0.37066666666666664, 0.36923076923076925, 0.36419753086419754, 0.35714285714285715, 0.3551724137931034, 0.3511111111111111, 0.3473118279569892, 0.34479166666666666, 0.34545454545454546, 0.34215686274509804, 0.3447619047619049, 0.3462962962962962, 0.3441441441441441, 0.34385964912280703, 0.33931623931623933, 0.3458333333333333, 0.3430894308943089, 0.33968253968253964, 0.3372093023255814, 0.33636363636363636, 0.337037037037037, 0.33695652173913043, 0.33333333333333326, 0.32986111111111105, 0.32789115646258504, 0.324]}, "energy_mean": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.36666666666666664, 0.4, 0.3888888888888889, 0.4, 0.38, 0.3944444444444445, 0.3857142857142858, 0.3875, 0.3962962962962963, 0.4033333333333334, 0.4121212121212121, 0.4, 0.3871794871794872, 0.38571428571428573, 0.38888888888888884, 0.38333333333333336, 0.3764705882352941, 0.36851851851851847, 0.36666666666666664, 0.37, 0.37301587301587297, 0.36515151515151517, 0.36231884057971014, 0.3555555555555555, 0.36, 0.3641025641025641, 0.36419753086419754, 0.3595238095238095, 0.3597701149425287, 0.3566666666666667, 0.3602150537634409, 0.3572916666666667, 0.3525252525252525, 0.34901960784313724, 0.3419047619047618, 0.3398148148148149, 0.3396396396396396, 0.33596491228070174, 0.33589743589743587, 0.3325, 0.33333333333333326, 0.3333333333333334, 0.33565891472868215, 0.3340909090909091, 0.3340740740740741, 0.33260869565217394, 0.3319148936170212, 0.32986111111111105, 0.3272108843537415, 0.324]}, "energy_std": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.23333333333333334, 0.38333333333333336, 0.32222222222222224, 0.31666666666666665, 0.3333333333333333, 0.3222222222222222, 0.31428571428571433, 0.325, 0.32222222222222224, 0.33000000000000007, 0.3242424242424242, 0.3138888888888889, 0.3128205128205128, 0.3119047619047619, 0.3088888888888888, 0.31875, 0.3196078431372549, 0.31666666666666676, 0.32105263157894737, 0.32, 0.32222222222222235, 0.32575757575757575, 0.32608695652173914, 0.31944444444444436, 0.316, 0.3128205128205128, 0.3135802469135803, 0.31547619047619047, 0.32183908045977005, 0.3211111111111111, 0.3150537634408602, 0.315625, 0.3151515151515151, 0.3196078431372549, 0.3133333333333333, 0.31481481481481494, 0.3162162162162162, 0.31666666666666665, 0.3162393162393162, 0.315, 0.31219512195121946, 0.311904761904762, 0.31317829457364343, 0.3151515151515151, 0.3148148148148148, 0.31956521739130433, 0.3205673758865248, 0.3208333333333333, 0.3224489795918367, 0.324]}, "msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.011111111111111112, 0.025, 0.02666666666666667, 0.027777777777777887, 0.038095238095238106, 0.04583333333333333, 0.044444444444444446, 0.04333333333333334, 0.048484848484848485, 0.05277777777777778, 0.05641025641025641, 0.05952380952380952, 0.06888888888888887, 0.075, 0.0784313725490196, 0.08333333333333347, 0.09298245614035087, 0.09833333333333333, 0.10793650793650805, 0.11212121212121212, 0.12318840579710146, 0.12916666666666665, 0.13866666666666666, 0.14871794871794872, 0.16049382716049382, 0.16666666666666666, 0.17931034482758637, 0.18444444444444444, 0.19462365591397848, 0.20416666666666666, 0.2111111111111111, 0.21568627450980393, 0.2257142857142858, 0.2296296296296296, 0.23603603603603604, 0.24298245614035088, 0.24871794871794872, 0.25166666666666665, 0.2560975609756097, 0.26269841269841265, 0.27364341085271315, 0.2825757575757576, 0.29185185185185186, 0.29927536231884055, 0.30283687943262405, 0.3131944444444445, 0.31700680272108844, 0.324]}, "temp_msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.02666666666666667, 0.027777777777777773, 0.028571428571428577, 0.03333333333333333, 0.044444444444444446, 0.046666666666666676, 0.048484848484848485, 0.05, 0.05128205128205128, 0.05238095238095238, 0.05777777777777776, 0.06666666666666667, 0.07254901960784314, 0.07962962962962962, 0.0824561403508772, 0.09, 0.09523809523809536, 0.10909090909090909, 0.11159420289855072, 0.12083333333333332, 0.128, 0.1358974358974359, 0.14320987654320988, 0.15119047619047618, 0.16206896551724134, 0.1688888888888889, 0.17526881720430107, 0.184375, 0.19090909090909092, 0.2019607843137255, 0.21047619047619062, 0.21666666666666679, 0.22342342342342342, 0.23333333333333334, 0.24017094017094018, 0.24666666666666667, 0.24959349593495947, 0.25555555555555565, 0.26434108527131783, 0.271969696969697, 0.28074074074074074, 0.2898550724637681, 0.29787234042553196, 0.3083333333333334, 0.3142857142857143, 0.324]}, "entropy": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.011111111111111112, 0.025, 0.02666666666666667, 0.033333333333333326, 0.038095238095238106, 0.04583333333333333, 0.044444444444444446, 0.04333333333333334, 0.045454545454545456, 0.05277777777777778, 0.05641025641025641, 0.05952380952380952, 0.06888888888888887, 0.07291666666666667, 0.0784313725490196, 0.08333333333333331, 0.09298245614035087, 0.09833333333333333, 0.10476190476190489, 0.11060606060606061, 0.12028985507246377, 0.12777777777777774, 0.13733333333333334, 0.14743589743589744, 0.15679012345679014, 0.16547619047619047, 0.17241379310344845, 0.18333333333333332, 0.1935483870967742, 0.2, 0.2080808080808081, 0.21666666666666667, 0.21999999999999997, 0.22500000000000012, 0.23333333333333334, 0.23684210526315788, 0.2452991452991453, 0.2475, 0.25447154471544725, 0.26349206349206344, 0.2689922480620155, 0.2765151515151515, 0.2822222222222222, 0.2898550724637681, 0.2971631205673759, 0.30763888888888896, 0.3149659863945578, 0.324]}, "softmax_learned": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.02666666666666667, 0.027777777777777773, 0.028571428571428577, 0.03333333333333333, 0.044444444444444446, 0.046666666666666676, 0.048484848484848485, 0.05, 0.04871794871794872, 0.05238095238095238, 0.05777777777777776, 0.0625, 0.07254901960784314, 0.08148148148148147, 0.0824561403508772, 0.09166666666666666, 0.09999999999999999, 0.10303030303030303, 0.11159420289855072, 0.1222222222222222, 0.12533333333333332, 0.13076923076923078, 0.13703703703703704, 0.14523809523809525, 0.15632183908045974, 0.16444444444444445, 0.17096774193548386, 0.17916666666666667, 0.19292929292929292, 0.2019607843137255, 0.21047619047619062, 0.21111111111111108, 0.21711711711711712, 0.2236842105263158, 0.23162393162393163, 0.24, 0.24878048780487802, 0.25714285714285723, 0.26744186046511625, 0.2765151515151515, 0.2822222222222222, 0.28840579710144926, 0.30000000000000004, 0.30555555555555564, 0.31564625850340133, 0.324]}, "geom_softmax": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.016666666666666666, 0.011111111111111112, 0.016666666666666666, 0.02, 0.02222222222222222, 0.03333333333333334, 0.0375, 0.04814814814814815, 0.05000000000000001, 0.048484848484848485, 0.04722222222222222, 0.05128205128205128, 0.05714285714285714, 0.059999999999999984, 0.06875, 0.07058823529411765, 0.07037037037037051, 0.07894736842105263, 0.08666666666666667, 0.09206349206349206, 0.10151515151515152, 0.10579710144927536, 0.11805555555555565, 0.12933333333333333, 0.1371794871794872, 0.14814814814814814, 0.15, 0.1586206896551726, 0.1688888888888889, 0.16989247311827957, 0.178125, 0.18282828282828284, 0.19215686274509805, 0.20380952380952377, 0.20925925925925937, 0.21621621621621623, 0.22105263157894736, 0.22735042735042735, 0.24, 0.2512195121951219, 0.25555555555555565, 0.2651162790697674, 0.2727272727272727, 0.2814814814814815, 0.28840579710144926, 0.29574468085106387, 0.30555555555555564, 0.3142857142857143, 0.324]}}, "ltt": {"alpha": 0.1, "delta": 0.05, "lambda_hat": null, "coverage": 0.0, "abstain_rate": 1.0, "selective_risk": 0.0, "selective_accuracy": null, "risk_within_budget": true}, "coverage_validity": {"alpha_grid": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "achieved_risk": [0.0, 0.0, 0.0, 0.08443271767810026, 0.17467248908296942, 0.29086892488954347], "coverage": [0.0, 0.0, 0.0, 0.25266666666666665, 0.458, 0.9053333333333333]}, "feature_diagnostics": {"names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "auroc": {"basin/rho": 0.6013891932695351, "basin/entropy": 0.3986422593972451, "basin/dispersion": 0.5095047929805765, "energy/mean": 0.5601111192279283, "energy/min": 0.5643785358885074, "energy/std": 0.4847708216654086, "curv/lmax_mean": 0.41285074796470805, "curv/lmax_best": 0.4264839571107377, "curv/trace_mean": 0.4132007857079082, "curv/trace_best": 0.4264727964870415, "dynamics/steps": 0.5, "dynamics/monotonic": 0.6341983831300071, "dynamics/drop": 0.6880605676901973, "dynamics/residual": 0.5087742794295501}, "hist": {"basin/rho": {"edges": [0.4166666666666667, 0.44583333333333336, 0.47500000000000003, 0.5041666666666667, 0.5333333333333333, 0.5625, 0.5916666666666667, 0.6208333333333333, 0.65, 0.6791666666666667, 0.7083333333333333, 0.7375, 0.7666666666666666, 0.7958333333333334, 0.825, 0.8541666666666666, 0.8833333333333333, 0.9125, 0.9416666666666667, 0.9708333333333332, 1.0], "correct_counts": [0, 0, 4, 0, 0, 11, 0, 0, 17, 0, 0, 27, 0, 0, 13, 0, 0, 37, 0, 905], "incorrect_counts": [3, 0, 11, 0, 0, 19, 0, 0, 22, 0, 0, 27, 0, 0, 32, 0, 0, 35, 0, 337]}, "basin/entropy": {"edges": [0.0, 0.05387781635334005, 0.1077556327066801, 0.16163344906002014, 0.2155112654133602, 0.26938908176670023, 0.3232668981200403, 0.3771447144733803, 0.4310225308267204, 0.4849003471800604, 0.5387781635334005, 0.5926559798867406, 0.6465337962400806, 0.7004116125934206, 0.7542894289467607, 0.8081672453001008, 0.8620450616534407, 0.9159228780067807, 0.9698006943601208, 1.023678510713461, 1.077556327066801], "correct_counts": [905, 0, 0, 0, 0, 37, 0, 0, 13, 0, 26, 16, 13, 1, 0, 1, 1, 0, 1, 0], "incorrect_counts": [337, 0, 0, 0, 0, 35, 0, 0, 30, 0, 29, 22, 23, 0, 0, 0, 0, 6, 1, 3]}, "basin/dispersion": {"edges": [1.5494269388744626, 1.5753932117990324, 1.6013594847236021, 1.6273257576481717, 1.6532920305727414, 1.6792583034973112, 1.705224576421881, 1.7311908493464507, 1.7571571222710203, 1.78312339519559, 1.8090896681201598, 1.8350559410447296, 1.8610222139692993, 1.8869884868938689, 1.9129547598184387, 1.9389210327430084, 1.9648873056675782, 1.990853578592148, 2.0168198515167175, 2.0427861244412875, 2.068752397365857], "correct_counts": [1, 0, 1, 5, 9, 20, 41, 60, 100, 132, 118, 143, 138, 96, 78, 44, 16, 7, 3, 2], "incorrect_counts": [0, 0, 0, 3, 4, 10, 19, 30, 52, 59, 67, 66, 66, 53, 25, 11, 12, 6, 2, 1]}, "energy/mean": {"edges": [0.35141998281081516, 0.40477818859120207, 0.45813639437158904, 0.511494600151976, 0.5648528059323629, 0.6182110117127497, 0.6715692174931367, 0.7249274232735237, 0.7782856290539106, 0.8316438348342975, 0.8850020406146843, 0.9383602463950713, 0.9917184521754583, 1.0450766579558453, 1.0984348637362322, 1.151793069516619, 1.205151275297006, 1.258509481077393, 1.3118676868577799, 1.3652258926381666, 1.4185840984185536], "correct_counts": [1, 5, 7, 15, 31, 47, 60, 110, 134, 136, 153, 124, 76, 67, 24, 14, 6, 3, 1, 0], "incorrect_counts": [1, 2, 3, 12, 22, 29, 42, 64, 65, 66, 54, 51, 41, 23, 7, 2, 1, 0, 0, 1]}, "energy/min": {"edges": [0.005636274814605713, 0.06192324459552765, 0.11821021437644959, 0.17449718415737153, 0.23078415393829346, 0.2870711237192154, 0.34335809350013735, 0.39964506328105925, 0.4559320330619812, 0.5122190028429031, 0.5685059726238251, 0.624792942404747, 0.681079912185669, 0.737366881966591, 0.7936538517475128, 0.8499408215284348, 0.9062277913093567, 0.9625147610902787, 1.0188017308712005, 1.0750887006521226, 1.1313756704330444], "correct_counts": [0, 4, 12, 17, 26, 45, 70, 99, 99, 133, 142, 135, 100, 66, 30, 18, 10, 4, 3, 1], "incorrect_counts": [3, 1, 4, 14, 15, 27, 50, 56, 59, 65, 48, 70, 33, 28, 5, 5, 2, 0, 0, 1]}, "energy/std": {"edges": [0.08805419068698145, 0.1022881234272592, 0.11652205616753694, 0.1307559889078147, 0.14498992164809243, 0.15922385438837017, 0.17345778712864793, 0.1876917198689257, 0.20192565260920342, 0.21615958534948115, 0.23039351808975891, 0.24462745083003667, 0.2588613835703144, 0.27309531631059214, 0.2873292490508699, 0.30156318179114766, 0.3157971145314254, 0.3300310472717031, 0.34426498001198086, 0.35849891275225865, 0.3727328454925364], "correct_counts": [3, 20, 34, 71, 87, 101, 112, 135, 119, 103, 97, 48, 35, 18, 14, 7, 3, 3, 2, 2], "incorrect_counts": [1, 6, 20, 34, 42, 43, 58, 54, 56, 47, 43, 25, 24, 14, 7, 5, 4, 1, 1, 1]}, "curv/lmax_mean": {"edges": [0.9987722436587015, 0.9993831271926562, 0.9999940107266108, 1.0006048942605654, 1.00121577779452, 1.0018266613284748, 1.0024375448624292, 1.0030484283963839, 1.0036593119303385, 1.0042701954642932, 1.0048810789982479, 1.0054919625322025, 1.006102846066157, 1.0067137296001116, 1.0073246131340663, 1.007935496668021, 1.0085463802019756, 1.0091572637359303, 1.0097681472698847, 1.0103790308038394, 1.010989914337794], "correct_counts": [25, 409, 306, 100, 76, 38, 17, 22, 9, 4, 2, 2, 2, 0, 1, 0, 0, 1, 0, 0], "incorrect_counts": [7, 163, 122, 58, 52, 28, 18, 13, 1, 6, 5, 4, 3, 2, 1, 0, 0, 0, 1, 2]}, "curv/lmax_best": {"edges": [0.994626522064209, 0.996795529127121, 0.998964536190033, 1.001133543252945, 1.003302550315857, 1.005471557378769, 1.0076405644416808, 1.009809571504593, 1.0119785785675048, 1.014147585630417, 1.0163165926933289, 1.0184855997562408, 1.0206546068191529, 1.0228236138820648, 1.024992620944977, 1.0271616280078888, 1.0293306350708007, 1.0314996421337128, 1.0336686491966247, 1.0358376562595368, 1.0380066633224487], "correct_counts": [2, 32, 858, 74, 12, 17, 9, 6, 1, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0], "incorrect_counts": [0, 8, 380, 49, 20, 10, 8, 4, 4, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1]}, "curv/trace_mean": {"edges": [31.937888781229656, 31.94588975906372, 31.953890736897787, 31.961891714731852, 31.96989269256592, 31.977893670399986, 31.98589464823405, 31.993895626068117, 32.00189660390218, 32.00989758173625, 32.01789855957031, 32.02589953740438, 32.03390051523844, 32.04190149307251, 32.04990247090658, 32.057903448740646, 32.06590442657471, 32.07390540440878, 32.08190638224284, 32.08990736007691, 32.09790833791097], "correct_counts": [2, 2, 7, 13, 44, 105, 162, 171, 165, 124, 86, 62, 42, 14, 7, 6, 0, 1, 1, 0], "incorrect_counts": [1, 0, 0, 2, 19, 37, 62, 71, 62, 58, 56, 53, 25, 21, 6, 8, 2, 2, 0, 1]}, "curv/trace_best": {"edges": [31.915437698364258, 31.92645502090454, 31.937472343444824, 31.948489665985107, 31.95950698852539, 31.970524311065674, 31.981541633605957, 31.99255895614624, 32.00357627868652, 32.01459360122681, 32.02561092376709, 32.03662824630737, 32.047645568847656, 32.05866289138794, 32.06968021392822, 32.080697536468506, 32.09171485900879, 32.10273218154907, 32.113749504089355, 32.12476682662964, 32.13578414916992], "correct_counts": [1, 3, 4, 15, 24, 72, 187, 272, 219, 83, 64, 32, 11, 14, 5, 3, 1, 2, 2, 0], "incorrect_counts": [0, 0, 1, 4, 8, 28, 72, 113, 98, 67, 31, 25, 20, 7, 2, 2, 4, 0, 0, 4]}, "dynamics/steps": {"edges": [1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7000000000000002, 1.75, 1.8, 1.85, 1.9, 1.9500000000000002, 2.0], "correct_counts": [1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/monotonic": {"edges": [0.6266666666666666, 0.6335833333333333, 0.6405, 0.6474166666666666, 0.6543333333333333, 0.6612499999999999, 0.6681666666666666, 0.6750833333333333, 0.6819999999999999, 0.6889166666666666, 0.6958333333333333, 0.70275, 0.7096666666666667, 0.7165833333333333, 0.7235, 0.7304166666666667, 0.7373333333333333, 0.74425, 0.7511666666666666, 0.7580833333333333, 0.765], "correct_counts": [0, 1, 0, 2, 11, 27, 70, 66, 104, 139, 106, 151, 103, 104, 56, 41, 17, 8, 5, 3], "incorrect_counts": [1, 0, 2, 1, 9, 19, 51, 53, 70, 86, 70, 49, 38, 21, 6, 7, 3, 0, 0, 0]}, "dynamics/drop": {"edges": [13.154254739483198, 18.10311585428814, 23.051976969093083, 28.000838083898028, 32.94969919870297, 37.898560313507915, 42.847421428312856, 47.796282543117805, 52.745143657922746, 57.69400477272769, 62.64286588753263, 67.59172700233758, 72.54058811714252, 77.48944923194746, 82.4383103467524, 87.38717146155734, 92.33603257636229, 97.28489369116723, 102.23375480597217, 107.18261592077711, 112.13147703558207], "correct_counts": [30, 195, 219, 177, 127, 90, 75, 41, 23, 14, 11, 6, 2, 0, 0, 1, 2, 0, 0, 1], "incorrect_counts": [47, 166, 126, 86, 28, 17, 9, 4, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/residual": {"edges": [1.1386153648296993, 1.1545183179279168, 1.1704212710261346, 1.1863242241243521, 1.2022271772225699, 1.2181301303207874, 1.2340330834190052, 1.2499360365172227, 1.2658389896154405, 1.281741942713658, 1.2976448958118758, 1.3135478489100934, 1.329450802008311, 1.3453537551065287, 1.3612567082047462, 1.377159661302964, 1.3930626144011815, 1.4089655674993993, 1.4248685205976168, 1.4407714736958344, 1.4566744267940521], "correct_counts": [3, 6, 12, 16, 39, 62, 84, 127, 128, 130, 105, 107, 82, 46, 33, 17, 8, 3, 4, 2], "incorrect_counts": [0, 5, 7, 9, 16, 35, 39, 59, 65, 55, 57, 52, 34, 24, 17, 5, 5, 2, 0, 0]}}}, "feature_ablation": {"full": 0.171055380110865, "drop_basin": 0.1829945994753527, "basin_only": 0.26530562690580933, "drop_energy": 0.17457640922091583, "energy_only": 0.2742022787633486, "drop_curv": 0.17754820658338663, "curv_only": 0.2744174989631136, "drop_dynamics": 0.23819230827142496, "dynamics_only": 0.19387207186931513}, "ece_geometry": 0.02548324869690796, "accuracy_ood": 0.3373333333333333, "aurc_ood": {"geometry": 0.6989345686143652, "rho_basin": 0.6359528685037144, "energy_min": 0.7674459887692104, "energy_mean": 0.7665803824715032, "energy_std": 0.6744965485414989, "msp": 0.6434757978241759, "temp_msp": 0.6232780631954414, "entropy": 0.6410680306163067, "softmax_learned": 0.6172112155416478, "geom_softmax": 0.6429259653235739}, "ood_ltt": {"alpha": 0.1, "lambda_hat": null, "selective_risk": 0.0, "coverage": 0.0, "risk_within_budget": true}, "ood_validity": {"target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "id_risk": [0.0, 0.0, 0.0, 0.08443271767810026, 0.17467248908296942, 0.29086892488954347], "ood_risk": [0.0, 0.0, 0.0, 0.7046332046332047, 0.6484284051222352, 0.6544011544011544], "id_coverage": [0.0, 0.0, 0.0, 0.25266666666666665, 0.458, 0.9053333333333333], "ood_coverage": [0.0, 0.0, 0.0, 0.3453333333333333, 0.5726666666666667, 0.924]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "ed919980f35e", "timestamp": "2026-07-30T11:18:18.349654+00:00", "git_sha": "67dc7a2", "config_hash": "9345e2bfd6b6", "config": {"run": {"seed": 4, "task": "graph_planning", "notes": "graph adaptive-halting run"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 25, "batch_size": 128, "lr": 0.001, "n_train": 8000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 800, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 800}, "task": {"graph_planning": {"n_nodes": 7, "ood_n_nodes": 10, "edge_prob": 0.4, "max_len": 4}}}, "task": "graph_planning", "split": "halting", "seed": 4, "metrics": {"k_restarts": 12, "steps": 50, "alpha": 0.1, "n_calib": 800, "n_test": 800, "final_train_loss": 0.21295076608657837, "tau_hat": 0.8333, "compute_used": 0.16995000000000002, "compute_saved": 0.83005, "halting_risk": 0.0625, "risk_within_budget": true, "full_accuracy": 0.7125, "halted_accuracy": 0.69875, "accuracy_drop": 0.01375000000000004, "risk_monotone_in_tau": true, "calib_risk_by_tau": [0.47375, 0.47375, 0.47375, 0.47375, 0.37875, 0.37875, 0.305, 0.1525, 0.1525, 0.0925, 0.00375, 0.00375], "calib_compute_by_tau": [0.0, 0.0, 0.0, 0.0002, 0.02342499999999982, 0.02342499999999982, 0.04669999999999976, 0.12322499999999997, 0.12322499999999997, 0.16990000000000008, 0.3011249999999997, 0.3011249999999997], "tau_sweep": {"taus": [0.0833, 0.1667, 0.25, 0.3333, 0.4167, 0.5, 0.5833, 0.6667, 0.75, 0.8333, 0.9167, 1.0], "compute_used": [0.0, 0.0, 0.0, 0.0006000000000000001, 0.02255, 0.02255, 0.04832500000000001, 0.12350000000000001, 0.12350000000000001, 0.16995000000000002, 0.28452500000000003, 0.28452500000000003], "accuracy": [0.50875, 0.50875, 0.50875, 0.50875, 0.575, 0.575, 0.60875, 0.68125, 0.68125, 0.69875, 0.715, 0.715], "disagreement": [0.46, 0.46, 0.46, 0.4575, 0.365, 0.365, 0.29625, 0.12625, 0.12625, 0.0625, 0.0075, 0.0075]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "8bc3880fccd3", "timestamp": "2026-07-30T11:18:25.395740+00:00", "git_sha": "67dc7a2", "config_hash": "e71a6b480861", "config": {"run": {"seed": 3, "task": "graph_planning", "notes": "E2 graph selective-prediction"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 25, "batch_size": 128, "lr": 0.001, "n_train": 8000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 1500, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 1500}, "task": {"graph_planning": {"n_nodes": 7, "ood_n_nodes": 10, "edge_prob": 0.4, "max_len": 4}}}, "task": "graph_planning", "split": "selective", "seed": 3, "metrics": {"n_fit": 1500, "n_calib": 1500, "n_test": 1500, "k_restarts": 12, "objective": "basin_center", "sampler": "langevin", "feature_set": "base", "accuracy_id": 0.6993333333333334, "base_error": 0.3006666666666667, "final_train_loss": 0.16939383745193481, "feature_names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "aurc": {"geometry": 0.1692573776986855, "rho_basin": 0.24609437563326478, "energy_min": 0.3844670869106948, "energy_mean": 0.38671775365041733, "energy_std": 0.3004012009214803, "msp": 0.13953947812056736, "temp_msp": 0.1394850202783934, "entropy": 0.1389181572354234, "softmax_learned": 0.14027532142085386, "geom_softmax": 0.14356994986612653}, "temperature": 3.4489855894245665, "best_energy_baseline": "energy_std", "best_baseline": "entropy", "delta_aurc_vs_energy_min": [0.2152097092120093, 0.17912220172475968, 0.24781429220048076], "delta_aurc_vs_best_energy": [0.1311438232227948, 0.10332126037643946, 0.1586253487976973], "delta_aurc_vs_best_baseline": [-0.030339220463262123, -0.04263629548652781, -0.01865656419709403], "delta_aurc_geom_adds": [-0.0032946284452726737, -0.007710391733187864, 0.00034702386219870264], "geometry_wins": true, "geometry_wins_vs_baseline": false, "geometry_adds_over_softmax": false, "risk_coverage": {"geometry": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.05, 0.044444444444444446, 0.041666666666666664, 0.04, 0.055555555555555546, 0.06666666666666649, 0.08333333333333333, 0.09259259259259259, 0.09666666666666668, 0.10606060606060606, 0.1111111111111111, 0.12051282051282051, 0.12142857142857143, 0.12888888888888886, 0.13125, 0.13529411764705881, 0.1444444444444444, 0.15087719298245614, 0.15333333333333332, 0.15555555555555567, 0.1621212121212121, 0.16521739130434782, 0.1708333333333333, 0.172, 0.1782051282051282, 0.18271604938271604, 0.18452380952380953, 0.18965517241379307, 0.19333333333333333, 0.1989247311827957, 0.20416666666666666, 0.2111111111111111, 0.21568627450980393, 0.2200000000000001, 0.22129629629629627, 0.22612612612612612, 0.2307017543859649, 0.23675213675213674, 0.24083333333333334, 0.24471544715447152, 0.2452380952380952, 0.25193798449612403, 0.26212121212121214, 0.26814814814814814, 0.27608695652173915, 0.28297872340425545, 0.2881944444444445, 0.2925170068027211, 0.3006666666666667]}, "rho_basin": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.24052718286655675, 0.24052718286655714, 0.2405271828665573, 0.24052718286655736, 0.24052718286655742, 0.24052718286655744, 0.24052718286655747, 0.2405271828665575, 0.2405271828665575, 0.24052718286655753, 0.24052718286655753, 0.24052718286655753, 0.24052718286655753, 0.24052718286655755, 0.24052718286655755, 0.24052718286655755, 0.24052718286655755, 0.24052718286655736, 0.2405271828665566, 0.24052718286655594, 0.24052718286655536, 0.2405271828665548, 0.2405271828665543, 0.24052718286655386, 0.24052718286655345, 0.24052718286655306, 0.2405271828665527, 0.24052718286655236, 0.24052718286655206, 0.24052718286655178, 0.2405271828665515, 0.24052718286655125, 0.240527182866551, 0.24052718286655078, 0.24052718286655056, 0.24052718286655075, 0.24052718286655134, 0.24052718286655186, 0.2405271828665524, 0.24052718286655286, 0.2436781609195365, 0.24937055281882495, 0.254798182304193, 0.2613517992424205, 0.26839120370370007, 0.27427536231883703, 0.2794175343292557, 0.28733747044916935, 0.2942050894431817, 0.3006666666666636]}, "energy_min": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.5666666666666667, 0.55, 0.5222222222222223, 0.525, 0.49333333333333335, 0.4555555555555556, 0.4619047619047618, 0.45416666666666666, 0.4444444444444444, 0.43000000000000005, 0.41515151515151516, 0.4222222222222222, 0.4153846153846154, 0.4023809523809524, 0.4044444444444444, 0.38958333333333334, 0.4, 0.39999999999999997, 0.39473684210526316, 0.39166666666666666, 0.38888888888888884, 0.3893939393939394, 0.3826086956521739, 0.37638888888888883, 0.376, 0.3769230769230769, 0.37777777777777777, 0.37142857142857144, 0.36896551724137927, 0.36666666666666664, 0.36451612903225805, 0.359375, 0.3565656565656566, 0.35, 0.3447619047619049, 0.3444444444444444, 0.3441441441441441, 0.3368421052631579, 0.3299145299145299, 0.32416666666666666, 0.32032520325203245, 0.31746031746031755, 0.3147286821705426, 0.31136363636363634, 0.30666666666666664, 0.30434782608695654, 0.3056737588652483, 0.3055555555555555, 0.30272108843537415, 0.3006666666666667]}, "energy_mean": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.6, 0.5, 0.5111111111111111, 0.5416666666666666, 0.5, 0.4833333333333333, 0.46190476190476204, 0.4375, 0.4148148148148148, 0.4133333333333334, 0.4121212121212121, 0.40555555555555556, 0.4230769230769231, 0.42857142857142855, 0.4288888888888888, 0.4166666666666667, 0.40784313725490196, 0.4074074074074075, 0.39824561403508774, 0.39, 0.3857142857142857, 0.3803030303030303, 0.37681159420289856, 0.3777777777777778, 0.37466666666666665, 0.36923076923076925, 0.36666666666666664, 0.36666666666666664, 0.36436781609195396, 0.3622222222222222, 0.36236559139784946, 0.36041666666666666, 0.35454545454545455, 0.35, 0.3428571428571428, 0.33703703703703697, 0.3324324324324324, 0.3298245614035088, 0.32735042735042735, 0.32666666666666666, 0.32520325203252026, 0.31904761904761897, 0.31705426356589145, 0.31363636363636366, 0.3148148148148148, 0.3101449275362319, 0.3085106382978723, 0.3055555555555555, 0.3034013605442177, 0.3006666666666667]}, "energy_std": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.3, 0.25, 0.26666666666666666, 0.30833333333333335, 0.32, 0.3277777777777777, 0.31428571428571417, 0.3125, 0.3074074074074074, 0.29666666666666675, 0.29393939393939394, 0.2833333333333333, 0.28717948717948716, 0.2976190476190476, 0.30222222222222217, 0.29791666666666666, 0.2980392156862745, 0.29444444444444456, 0.3017543859649123, 0.295, 0.29841269841269835, 0.3015151515151515, 0.3072463768115942, 0.3083333333333334, 0.30133333333333334, 0.2987179487179487, 0.29506172839506173, 0.2976190476190476, 0.29540229885057484, 0.29444444444444445, 0.2913978494623656, 0.296875, 0.29494949494949496, 0.296078431372549, 0.298095238095238, 0.2981481481481481, 0.3, 0.3017543859649123, 0.30085470085470084, 0.3, 0.30162601626016255, 0.3007936507936507, 0.3023255813953488, 0.29924242424242425, 0.3, 0.30144927536231886, 0.30212765957446813, 0.30347222222222214, 0.3034013605442177, 0.3006666666666667]}, "msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.016666666666666666, 0.022222222222222223, 0.025, 0.02, 0.033333333333333444, 0.042857142857142864, 0.0375, 0.037037037037037035, 0.04333333333333334, 0.051515151515151514, 0.05, 0.05384615384615385, 0.06190476190476191, 0.06444444444444443, 0.075, 0.07647058823529412, 0.08518518518518532, 0.08771929824561403, 0.09666666666666666, 0.10000000000000013, 0.10757575757575757, 0.12173913043478261, 0.1236111111111111, 0.128, 0.13974358974358975, 0.15185185185185185, 0.1595238095238095, 0.16781609195402297, 0.17444444444444446, 0.17956989247311828, 0.184375, 0.1898989898989899, 0.19509803921568628, 0.20190476190476203, 0.20740740740740737, 0.21081081081081082, 0.21929824561403508, 0.22393162393162394, 0.23166666666666666, 0.23739837398373995, 0.24365079365079376, 0.2527131782945736, 0.25757575757575757, 0.26666666666666666, 0.2746376811594203, 0.28014184397163117, 0.28750000000000003, 0.2945578231292517, 0.3006666666666667]}, "temp_msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.016666666666666666, 0.03333333333333333, 0.025, 0.02, 0.03888888888888888, 0.042857142857142864, 0.0375, 0.044444444444444446, 0.05000000000000001, 0.051515151515151514, 0.05277777777777778, 0.05897435897435897, 0.06666666666666667, 0.0711111111111111, 0.07916666666666666, 0.0803921568627451, 0.08888888888888888, 0.0912280701754386, 0.1, 0.10793650793650793, 0.11666666666666667, 0.12753623188405797, 0.13611111111111107, 0.14266666666666666, 0.14871794871794872, 0.1506172839506173, 0.14761904761904762, 0.15172413793103465, 0.16, 0.16774193548387098, 0.17395833333333333, 0.18181818181818182, 0.19019607843137254, 0.1933333333333333, 0.19814814814814827, 0.2036036036036036, 0.21403508771929824, 0.2230769230769231, 0.23, 0.23577235772357735, 0.24365079365079376, 0.2503875968992248, 0.25681818181818183, 0.26666666666666666, 0.27318840579710146, 0.2794326241134753, 0.28750000000000003, 0.2945578231292517, 0.3006666666666667]}, "entropy": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.016666666666666666, 0.022222222222222223, 0.025, 0.02, 0.033333333333333444, 0.042857142857142864, 0.0375, 0.037037037037037035, 0.04333333333333334, 0.051515151515151514, 0.05, 0.05384615384615385, 0.06190476190476191, 0.06444444444444443, 0.075, 0.0784313725490196, 0.08518518518518517, 0.08947368421052632, 0.09666666666666666, 0.10158730158730157, 0.11060606060606061, 0.12173913043478261, 0.1236111111111111, 0.13066666666666665, 0.1371794871794872, 0.15185185185185185, 0.1595238095238095, 0.1701149425287356, 0.17333333333333334, 0.17849462365591398, 0.18125, 0.18686868686868688, 0.19607843137254902, 0.19904761904761903, 0.2046296296296296, 0.21081081081081082, 0.2149122807017544, 0.2222222222222222, 0.22833333333333333, 0.23495934959349604, 0.24285714285714283, 0.25116279069767444, 0.25681818181818183, 0.26222222222222225, 0.27246376811594203, 0.2780141843971631, 0.2881944444444445, 0.2945578231292517, 0.3006666666666667]}, "softmax_learned": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.016666666666666666, 0.03333333333333333, 0.025, 0.02, 0.03888888888888888, 0.042857142857142864, 0.0375, 0.044444444444444446, 0.05000000000000001, 0.051515151515151514, 0.05277777777777778, 0.05641025641025641, 0.06666666666666667, 0.0711111111111111, 0.07916666666666666, 0.0803921568627451, 0.09259259259259257, 0.0912280701754386, 0.1, 0.11111111111111122, 0.11666666666666667, 0.1289855072463768, 0.13750000000000012, 0.13866666666666666, 0.14871794871794872, 0.14814814814814814, 0.1523809523809524, 0.15632183908045993, 0.16, 0.16344086021505377, 0.17291666666666666, 0.17777777777777778, 0.18529411764705883, 0.19428571428571442, 0.20092592592592604, 0.21171171171171171, 0.21842105263157896, 0.2264957264957265, 0.23166666666666666, 0.2373983739837398, 0.24920634920634932, 0.2558139534883721, 0.2628787878787879, 0.2688888888888889, 0.27318840579710146, 0.28085106382978736, 0.2895833333333334, 0.2965986394557823, 0.3006666666666667]}, "geom_softmax": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.03333333333333333, 0.03333333333333333, 0.044444444444444446, 0.041666666666666664, 0.04666666666666667, 0.04444444444444444, 0.04761904761904762, 0.04583333333333333, 0.05555555555555555, 0.05000000000000001, 0.051515151515151514, 0.05555555555555555, 0.0641025641025641, 0.06904761904761905, 0.0711111111111111, 0.07708333333333334, 0.07647058823529412, 0.08333333333333347, 0.09649122807017543, 0.10333333333333333, 0.10952380952380951, 0.11818181818181818, 0.12753623188405797, 0.13888888888888887, 0.14933333333333335, 0.15, 0.15679012345679014, 0.15714285714285714, 0.1597701149425287, 0.16111111111111112, 0.17419354838709677, 0.18229166666666666, 0.18686868686868688, 0.19215686274509805, 0.19809523809523824, 0.20833333333333331, 0.2126126126126126, 0.21666666666666667, 0.2205128205128205, 0.2275, 0.2373983739837398, 0.24285714285714297, 0.2503875968992248, 0.2606060606060606, 0.2688888888888889, 0.2746376811594203, 0.28297872340425545, 0.29027777777777786, 0.29523809523809524, 0.3006666666666667]}}, "ltt": {"alpha": 0.1, "delta": 0.05, "lambda_hat": null, "coverage": 0.0, "abstain_rate": 1.0, "selective_risk": 0.0, "selective_accuracy": null, "risk_within_budget": true}, "coverage_validity": {"alpha_grid": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "achieved_risk": [0.0, 0.0, 0.0, 0.04, 0.1474820143884892, 0.24862313139260425], "coverage": [0.0, 0.0, 0.0, 0.08333333333333333, 0.37066666666666664, 0.8473333333333334]}, "feature_diagnostics": {"names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "auroc": {"basin/rho": 0.6182722855047252, "basin/entropy": 0.3819771337500185, "basin/dispersion": 0.5171306639836483, "energy/mean": 0.6203141414376272, "energy/min": 0.6201915455327532, "energy/std": 0.4974624761413573, "curv/lmax_mean": 0.40259966730007885, "curv/lmax_best": 0.42140334264075807, "curv/trace_mean": 0.3792493748665713, "curv/trace_best": 0.3965438523438012, "dynamics/steps": 0.5, "dynamics/monotonic": 0.6171382733846404, "dynamics/drop": 0.658876894688004, "dynamics/residual": 0.4726389191268635}, "hist": {"basin/rho": {"edges": [0.4166666666666667, 0.44583333333333336, 0.47500000000000003, 0.5041666666666667, 0.5333333333333333, 0.5625, 0.5916666666666667, 0.6208333333333333, 0.65, 0.6791666666666667, 0.7083333333333333, 0.7375, 0.7666666666666666, 0.7958333333333334, 0.825, 0.8541666666666666, 0.8833333333333333, 0.9125, 0.9416666666666667, 0.9708333333333332, 1.0], "correct_counts": [0, 0, 7, 0, 0, 11, 0, 0, 16, 0, 0, 21, 0, 0, 27, 0, 0, 45, 0, 922], "incorrect_counts": [2, 0, 10, 0, 0, 16, 0, 0, 31, 0, 0, 21, 0, 0, 37, 0, 0, 42, 0, 292]}, "basin/entropy": {"edges": [0.0, 0.05387781635334005, 0.1077556327066801, 0.16163344906002014, 0.2155112654133602, 0.26938908176670023, 0.3232668981200403, 0.3771447144733803, 0.4310225308267204, 0.4849003471800604, 0.5387781635334005, 0.5926559798867406, 0.6465337962400806, 0.7004116125934206, 0.7542894289467607, 0.8081672453001008, 0.8620450616534407, 0.9159228780067807, 0.9698006943601208, 1.023678510713461, 1.077556327066801], "correct_counts": [922, 0, 0, 0, 0, 45, 0, 0, 26, 0, 20, 14, 18, 2, 0, 2, 0, 0, 0, 0], "incorrect_counts": [292, 0, 0, 0, 0, 42, 0, 0, 37, 0, 21, 28, 21, 0, 0, 3, 0, 3, 2, 2]}, "basin/dispersion": {"edges": [1.6360123866978022, 1.6586197833827276, 1.6812271800676528, 1.7038345767525782, 1.7264419734375036, 1.749049370122429, 1.7716567668073542, 1.7942641634922796, 1.816871560177205, 1.8394789568621301, 1.8620863535470555, 1.884693750231981, 1.907301146916906, 1.9299085436018315, 1.952515940286757, 1.975123336971682, 1.9977307336566075, 2.020338130341533, 2.042945527026458, 2.0655529237113837, 2.088160320396309], "correct_counts": [6, 20, 22, 37, 64, 93, 127, 135, 130, 132, 92, 72, 54, 31, 12, 12, 8, 0, 1, 1], "incorrect_counts": [6, 2, 14, 24, 28, 35, 62, 54, 59, 52, 38, 36, 17, 9, 10, 3, 2, 0, 0, 0]}, "energy/mean": {"edges": [0.3838568776845932, 0.4383304126560688, 0.4928039476275444, 0.54727748259902, 0.6017510175704956, 0.6562245525419712, 0.7106980875134468, 0.7651716224849223, 0.819645157456398, 0.8741186924278737, 0.9285922273993492, 0.9830657623708248, 1.0375392973423003, 1.092012832313776, 1.1464863672852514, 1.2009599022567272, 1.2554334372282028, 1.3099069721996783, 1.3643805071711541, 1.4188540421426297, 1.4733275771141052], "correct_counts": [2, 0, 1, 4, 18, 35, 82, 85, 88, 112, 96, 98, 124, 85, 91, 75, 28, 14, 8, 3], "incorrect_counts": [0, 1, 1, 10, 16, 40, 40, 61, 47, 41, 46, 43, 25, 29, 23, 15, 8, 4, 1, 0]}, "energy/min": {"edges": [0.027186661958694458, 0.08629510253667831, 0.14540354311466216, 0.204511983692646, 0.26362042427062987, 0.32272886484861374, 0.38183730542659755, 0.4409457460045814, 0.5000541865825653, 0.5591626271605491, 0.618271067738533, 0.6773795083165168, 0.7364879488945006, 0.7955963894724846, 0.8547048300504684, 0.9138132706284523, 0.9729217112064361, 1.03203015178442, 1.0911385923624037, 1.1502470329403878, 1.2093554735183716], "correct_counts": [2, 1, 3, 4, 10, 29, 65, 87, 108, 97, 108, 132, 100, 108, 80, 51, 38, 20, 5, 1], "incorrect_counts": [0, 2, 3, 3, 16, 34, 39, 45, 64, 55, 50, 43, 30, 13, 17, 19, 12, 4, 2, 0]}, "energy/std": {"edges": [0.07382017921196296, 0.09055781346466538, 0.10729544771736779, 0.12403308197007021, 0.14077071622277262, 0.15750835047547504, 0.17424598472817746, 0.19098361898087987, 0.2077212532335823, 0.2244588874862847, 0.24119652173898712, 0.25793415599168956, 0.2746717902443919, 0.2914094244970944, 0.30814705874979675, 0.3248846930024992, 0.3416223272552016, 0.35835996150790406, 0.3750975957606064, 0.3918352300133089, 0.40857286426601125], "correct_counts": [1, 15, 27, 55, 114, 137, 142, 168, 125, 101, 66, 37, 24, 18, 9, 6, 1, 1, 0, 2], "incorrect_counts": [1, 7, 6, 31, 44, 60, 67, 55, 63, 45, 29, 22, 12, 5, 3, 1, 0, 0, 0, 0]}, "curv/lmax_mean": {"edges": [0.9988120545943578, 0.9993241287767888, 0.9998362029592196, 1.0003482771416505, 1.0008603513240815, 1.0013724255065124, 1.0018844996889433, 1.0023965738713743, 1.002908648053805, 1.003420722236236, 1.0039327964186668, 1.0044448706010978, 1.0049569447835287, 1.0054690189659596, 1.0059810931483906, 1.0064931673308215, 1.0070052415132524, 1.0075173156956831, 1.008029389878114, 1.008541464060545, 1.009053538242976], "correct_counts": [74, 546, 194, 97, 68, 34, 12, 9, 8, 3, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1], "incorrect_counts": [20, 160, 119, 73, 36, 18, 8, 8, 3, 3, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0]}, "curv/lmax_best": {"edges": [0.9947617650032043, 0.9963587850332261, 0.9979558050632477, 0.9995528250932694, 1.001149845123291, 1.0027468651533127, 1.0043438851833344, 1.005940905213356, 1.0075379252433776, 1.0091349452733993, 1.010731965303421, 1.0123289853334427, 1.0139260053634644, 1.015523025393486, 1.0171200454235076, 1.0187170654535294, 1.020314085483551, 1.0219111055135728, 1.0235081255435943, 1.025105145573616, 1.0267021656036377], "correct_counts": [5, 26, 176, 761, 49, 14, 9, 6, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1], "incorrect_counts": [0, 6, 66, 325, 30, 11, 6, 4, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0]}, "curv/trace_mean": {"edges": [31.904047807057697, 31.912987621625263, 31.92192743619283, 31.930867250760397, 31.93980706532796, 31.948746879895527, 31.957686694463092, 31.96662650903066, 31.975566323598226, 31.98450613816579, 31.99344595273336, 32.00238576730092, 32.01132558186849, 32.02026539643606, 32.02920521100362, 32.03814502557119, 32.04708484013875, 32.05602465470632, 32.06496446927389, 32.07390428384145, 32.08284409840902], "correct_counts": [3, 7, 33, 45, 64, 85, 106, 128, 85, 99, 93, 93, 107, 63, 22, 15, 0, 0, 0, 1], "incorrect_counts": [0, 1, 4, 9, 13, 25, 22, 40, 37, 54, 55, 54, 65, 44, 16, 10, 1, 1, 0, 0]}, "curv/trace_best": {"edges": [31.818470001220703, 31.8322847366333, 31.8460994720459, 31.859914207458495, 31.873728942871093, 31.88754367828369, 31.90135841369629, 31.915173149108888, 31.928987884521483, 31.94280261993408, 31.95661735534668, 31.970432090759278, 31.984246826171876, 31.99806156158447, 32.01187629699707, 32.02569103240967, 32.03950576782226, 32.053320503234865, 32.06713523864746, 32.08094997406006, 32.094764709472656], "correct_counts": [1, 0, 0, 4, 3, 11, 18, 37, 62, 91, 120, 152, 183, 189, 104, 40, 23, 5, 4, 2], "incorrect_counts": [0, 0, 0, 0, 1, 2, 2, 11, 13, 22, 34, 64, 79, 101, 73, 30, 12, 5, 0, 2]}, "dynamics/steps": {"edges": [1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7000000000000002, 1.75, 1.8, 1.85, 1.9, 1.9500000000000002, 2.0], "correct_counts": [1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/monotonic": {"edges": [0.6383333333333333, 0.64425, 0.6501666666666667, 0.6560833333333334, 0.662, 0.6679166666666667, 0.6738333333333333, 0.67975, 0.6856666666666666, 0.6915833333333333, 0.6975, 0.7034166666666667, 0.7093333333333334, 0.71525, 0.7211666666666667, 0.7270833333333333, 0.733, 0.7389166666666667, 0.7448333333333333, 0.75075, 0.7566666666666667], "correct_counts": [1, 3, 7, 17, 31, 48, 58, 112, 80, 141, 139, 94, 114, 59, 54, 36, 29, 17, 5, 4], "incorrect_counts": [1, 4, 5, 13, 15, 41, 39, 54, 53, 60, 62, 37, 27, 20, 7, 7, 4, 0, 2, 0]}, "dynamics/drop": {"edges": [13.959155107537905, 18.96183481787642, 23.96451452821493, 28.967194238553446, 33.96987394889196, 38.97255365923047, 43.97523336956898, 48.977913079907495, 53.98059279024601, 58.98327250058452, 63.98595221092303, 68.98863192126154, 73.99131163160007, 78.99399134193858, 83.9966710522771, 88.9993507626156, 94.00203047295412, 99.00471018329263, 104.00738989363114, 109.01006960396965, 114.01274931430817], "correct_counts": [79, 187, 237, 186, 131, 96, 46, 37, 21, 12, 6, 5, 1, 4, 0, 0, 0, 0, 0, 1], "incorrect_counts": [55, 154, 109, 67, 37, 17, 6, 3, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/residual": {"edges": [1.157560517390569, 1.1732419848442077, 1.1889234522978465, 1.2046049197514852, 1.220286387205124, 1.2359678546587625, 1.2516493221124012, 1.26733078956604, 1.2830122570196787, 1.2986937244733174, 1.3143751919269562, 1.330056659380595, 1.3457381268342337, 1.3614195942878724, 1.3771010617415111, 1.3927825291951499, 1.4084639966487886, 1.4241454641024274, 1.439826931556066, 1.4555083990097046, 1.4711898664633434], "correct_counts": [3, 8, 18, 25, 60, 82, 109, 133, 138, 137, 115, 81, 48, 43, 25, 11, 6, 6, 0, 1], "incorrect_counts": [1, 3, 6, 13, 16, 30, 48, 44, 73, 63, 52, 36, 28, 22, 6, 4, 2, 3, 0, 1]}}}, "feature_ablation": {"full": 0.1692573776986855, "drop_basin": 0.18552230550261628, "basin_only": 0.2468147564151358, "drop_energy": 0.16719687851591933, "energy_only": 0.2326850496812418, "drop_curv": 0.16787121777273564, "curv_only": 0.21755660344088715, "drop_dynamics": 0.20149229331418345, "dynamics_only": 0.1919227889302135}, "ece_geometry": 0.03336447508959324, "accuracy_ood": 0.3913333333333333, "aurc_ood": {"geometry": 0.5517668728716215, "rho_basin": 0.5982679701853475, "energy_min": 0.7015110277968405, "energy_mean": 0.6965205335334852, "energy_std": 0.5928168612920839, "msp": 0.5847105053531803, "temp_msp": 0.5671211036886489, "entropy": 0.5831236345427478, "softmax_learned": 0.5653040354208368, "geom_softmax": 0.5762981866215301}, "ood_ltt": {"alpha": 0.1, "lambda_hat": null, "selective_risk": 0.0, "coverage": 0.0, "risk_within_budget": true}, "ood_validity": {"target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "id_risk": [0.0, 0.0, 0.0, 0.04, 0.1474820143884892, 0.24862313139260425], "ood_risk": [0.0, 0.0, 0.0, 0.508130081300813, 0.5472636815920398, 0.5952569169960474], "id_coverage": [0.0, 0.0, 0.0, 0.08333333333333333, 0.37066666666666664, 0.8473333333333334], "ood_coverage": [0.0, 0.0, 0.0, 0.164, 0.402, 0.8433333333333334]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} +{"run_id": "fadafc6f2be1", "timestamp": "2026-07-30T11:18:32.832325+00:00", "git_sha": "67dc7a2", "config_hash": "68757964fe31", "config": {"run": {"seed": 4, "task": "graph_planning", "notes": "E2 graph selective-prediction"}, "model": {"latent_dim": 32, "hidden_dim": 128, "context_dim": 64}, "inference": {"k_restarts": 12, "steps": 50, "step_size": 0.1, "temperature": 0.05, "init_scale": 1.0, "grad_tol": 0.001, "sampler": "langevin", "anneal_levels": 10, "anneal_steps_per_level": 8, "anneal_step_max": 0.2, "anneal_step_min": 0.01}, "train": {"epochs": 25, "batch_size": 128, "lr": 0.001, "n_train": 8000, "n_neg": 4, "neg_noise": 0.5, "objective": "basin_center", "ired_noise_min": 0.1, "ired_noise_max": 1.5, "ired_ridge": 0.05, "ired_decode_weight": 1.0, "ired_stat_weight": 1.0}, "eval": {"n_eval": 1500, "richer_geometry": false}, "conformal": {"alpha": 0.1, "delta": 0.05, "n_calib": 1500}, "task": {"graph_planning": {"n_nodes": 7, "ood_n_nodes": 10, "edge_prob": 0.4, "max_len": 4}}}, "task": "graph_planning", "split": "selective", "seed": 4, "metrics": {"n_fit": 1500, "n_calib": 1500, "n_test": 1500, "k_restarts": 12, "objective": "basin_center", "sampler": "langevin", "feature_set": "base", "accuracy_id": 0.7233333333333334, "base_error": 0.27666666666666667, "final_train_loss": 0.21295076608657837, "feature_names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "aurc": {"geometry": 0.13283935829490207, "rho_basin": 0.2203375977159469, "energy_min": 0.22303149688717805, "energy_mean": 0.2229615383788109, "energy_std": 0.2828103710565153, "msp": 0.11262886470219956, "temp_msp": 0.10870154503937372, "entropy": 0.11070403088349459, "softmax_learned": 0.10852657318026214, "geom_softmax": 0.11090419152467276}, "temperature": 2.696299147690236, "best_energy_baseline": "energy_mean", "best_baseline": "temp_msp", "delta_aurc_vs_energy_min": [0.09019213859227598, 0.06793479403590773, 0.11328673363081866], "delta_aurc_vs_best_energy": [0.09012218008390882, 0.06776140718139022, 0.11351940825686613], "delta_aurc_vs_best_baseline": [-0.024137813255528343, -0.03391303611106098, -0.015078370648686666], "delta_aurc_geom_adds": [-0.002377618344410623, -0.00461672765694182, -0.00014286112737259382], "geometry_wins": true, "geometry_wins_vs_baseline": false, "geometry_adds_over_softmax": false, "risk_coverage": {"geometry": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.011111111111111112, 0.03333333333333333, 0.04666666666666667, 0.0611111111111111, 0.06190476190476192, 0.058333333333333334, 0.05925925925925926, 0.06333333333333335, 0.06060606060606061, 0.06388888888888888, 0.0641025641025641, 0.07142857142857142, 0.07333333333333332, 0.08333333333333333, 0.08627450980392157, 0.09259259259259257, 0.09473684210526316, 0.1, 0.09999999999999999, 0.10606060606060606, 0.1144927536231884, 0.12499999999999999, 0.132, 0.1371794871794872, 0.13950617283950617, 0.14285714285714285, 0.14942528735632182, 0.15777777777777777, 0.16451612903225807, 0.16770833333333332, 0.1696969696969697, 0.17745098039215687, 0.182857142857143, 0.18333333333333346, 0.1873873873873874, 0.19210526315789472, 0.19743589743589743, 0.20833333333333334, 0.21382113821138207, 0.22142857142857153, 0.22713178294573644, 0.23712121212121212, 0.2437037037037037, 0.2492753623188406, 0.2553191489361703, 0.26111111111111107, 0.2673469387755102, 0.27666666666666667]}, "rho_basin": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.21457489878542518, 0.21457489878542532, 0.21457489878542535, 0.21457489878542538, 0.21457489878542538, 0.2145748987854248, 0.21457489878542438, 0.21457489878542407, 0.21457489878542382, 0.21457489878542366, 0.21457489878542416, 0.21457489878542454, 0.2145748987854249, 0.21457489878542518, 0.21457489878542543, 0.21457489878542566, 0.21457489878542588, 0.21457489878542604, 0.21457489878542618, 0.21457489878542632, 0.21457489878542643, 0.21457489878542657, 0.21457489878542665, 0.21457489878542677, 0.21457489878542685, 0.21457489878542693, 0.21457489878542702, 0.2145748987854271, 0.21457489878542715, 0.2145748987854272, 0.21457489878542726, 0.21457489878542732, 0.21457489878542738, 0.21457489878542743, 0.21457489878542746, 0.21457489878542751, 0.21457489878542754, 0.2145748987854276, 0.21457489878542763, 0.21457489878542765, 0.21457489878542768, 0.22157872157872383, 0.22962497381102207, 0.2366573902288203, 0.24228269085412094, 0.2488657844990567, 0.2561229314420826, 0.26513888888889064, 0.27217068645640213, 0.2766666666666679]}, "energy_min": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.16666666666666666, 0.15, 0.15555555555555556, 0.15833333333333333, 0.18666666666666668, 0.2000000000000001, 0.19999999999999984, 0.20416666666666666, 0.21481481481481482, 0.21000000000000002, 0.20606060606060606, 0.19722222222222222, 0.2, 0.1976190476190476, 0.19777777777777775, 0.20208333333333334, 0.20588235294117646, 0.20185185185185184, 0.19649122807017544, 0.20666666666666667, 0.20793650793650792, 0.21212121212121213, 0.2217391304347826, 0.2208333333333333, 0.21733333333333332, 0.21666666666666667, 0.21728395061728395, 0.21785714285714286, 0.21494252873563216, 0.21444444444444444, 0.21505376344086022, 0.21979166666666666, 0.22525252525252526, 0.23039215686274508, 0.23523809523809536, 0.2425925925925927, 0.24684684684684685, 0.24912280701754386, 0.24957264957264957, 0.25916666666666666, 0.26260162601626025, 0.2666666666666668, 0.27054263565891473, 0.271969696969697, 0.2785185185185185, 0.2811594202898551, 0.28014184397163117, 0.27986111111111106, 0.2789115646258503, 0.27666666666666667]}, "energy_mean": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.23333333333333334, 0.3, 0.23333333333333334, 0.21666666666666667, 0.19333333333333333, 0.17777777777777776, 0.18095238095238098, 0.17916666666666667, 0.17777777777777778, 0.17333333333333337, 0.16363636363636364, 0.17777777777777778, 0.17692307692307693, 0.1880952380952381, 0.19777777777777775, 0.2, 0.20588235294117646, 0.20185185185185184, 0.2, 0.205, 0.20793650793650792, 0.20909090909090908, 0.21304347826086956, 0.21111111111111108, 0.21333333333333335, 0.2153846153846154, 0.21481481481481482, 0.21785714285714286, 0.21954022988505742, 0.22111111111111112, 0.221505376344086, 0.22395833333333334, 0.22626262626262628, 0.22941176470588234, 0.23428571428571426, 0.23703703703703702, 0.24234234234234234, 0.24736842105263157, 0.25213675213675213, 0.2575, 0.26178861788617896, 0.2682539682539682, 0.2689922480620155, 0.27424242424242423, 0.2777777777777778, 0.27753623188405796, 0.2773049645390072, 0.2770833333333333, 0.2755102040816326, 0.27666666666666667]}, "energy_std": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.16666666666666666, 0.25, 0.28888888888888886, 0.2916666666666667, 0.29333333333333333, 0.30000000000000004, 0.2952380952380951, 0.2916666666666667, 0.2962962962962963, 0.2933333333333332, 0.296969696969697, 0.2972222222222222, 0.2923076923076923, 0.29523809523809524, 0.2955555555555555, 0.28958333333333336, 0.28627450980392155, 0.28703703703703715, 0.28421052631578947, 0.285, 0.28095238095238106, 0.2772727272727273, 0.2768115942028985, 0.2791666666666666, 0.2773333333333333, 0.2794871794871795, 0.2814814814814815, 0.2785714285714286, 0.2770114942528737, 0.2788888888888889, 0.2849462365591398, 0.28125, 0.2797979797979798, 0.28431372549019607, 0.2828571428571428, 0.2814814814814816, 0.2828828828828829, 0.2850877192982456, 0.28034188034188035, 0.2833333333333333, 0.2821138211382113, 0.28333333333333344, 0.2837209302325581, 0.2803030303030303, 0.2837037037037037, 0.2826086956521739, 0.2808510638297872, 0.27847222222222234, 0.2782312925170068, 0.27666666666666667]}, "msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.013333333333333334, 0.016666666666666663, 0.028571428571428577, 0.03333333333333333, 0.03333333333333333, 0.04000000000000001, 0.03939393939393939, 0.03888888888888889, 0.038461538461538464, 0.04523809523809524, 0.048888888888888885, 0.05, 0.049019607843137254, 0.0537037037037037, 0.06140350877192982, 0.06833333333333333, 0.07619047619047618, 0.0803030303030303, 0.08695652173913043, 0.09583333333333333, 0.10133333333333333, 0.1064102564102564, 0.11481481481481481, 0.12142857142857143, 0.1287356321839082, 0.1288888888888889, 0.13225806451612904, 0.13854166666666667, 0.14242424242424243, 0.14901960784313725, 0.15904761904761902, 0.1666666666666668, 0.17477477477477477, 0.18070175438596492, 0.18888888888888888, 0.19833333333333333, 0.20569105691056908, 0.21428571428571438, 0.2248062015503876, 0.23484848484848486, 0.24074074074074073, 0.24710144927536232, 0.2553191489361703, 0.2625000000000001, 0.2687074829931973, 0.27666666666666667]}, "temp_msp": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.013333333333333334, 0.01666666666666678, 0.028571428571428577, 0.03333333333333333, 0.02962962962962963, 0.03333333333333334, 0.030303030303030304, 0.03333333333333333, 0.03333333333333333, 0.0380952380952381, 0.04444444444444444, 0.04375, 0.054901960784313725, 0.0611111111111111, 0.06140350877192982, 0.065, 0.07142857142857141, 0.07575757575757576, 0.08550724637681159, 0.09305555555555554, 0.10133333333333333, 0.1064102564102564, 0.10987654320987654, 0.1130952380952381, 0.11954022988505764, 0.12111111111111111, 0.12580645161290321, 0.13125, 0.13535353535353536, 0.14313725490196078, 0.15238095238095237, 0.15925925925925924, 0.17117117117117117, 0.17982456140350878, 0.18974358974358974, 0.1925, 0.19756097560975622, 0.20555555555555566, 0.21627906976744185, 0.22272727272727272, 0.22814814814814816, 0.23840579710144927, 0.2482269503546099, 0.26111111111111124, 0.2693877551020408, 0.27666666666666667]}, "entropy": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.013333333333333334, 0.016666666666666663, 0.028571428571428577, 0.03333333333333333, 0.03333333333333333, 0.04000000000000001, 0.03939393939393939, 0.03888888888888889, 0.038461538461538464, 0.04523809523809524, 0.048888888888888885, 0.04791666666666667, 0.050980392156862744, 0.05185185185185184, 0.05964912280701754, 0.07, 0.07301587301587315, 0.0803030303030303, 0.08550724637681159, 0.09444444444444455, 0.10266666666666667, 0.1064102564102564, 0.11358024691358025, 0.12142857142857143, 0.12413793103448273, 0.12777777777777777, 0.12795698924731183, 0.13229166666666667, 0.1414141414141414, 0.14705882352941177, 0.15523809523809537, 0.16388888888888903, 0.17117117117117117, 0.17982456140350878, 0.18632478632478633, 0.19583333333333333, 0.20325203252032517, 0.21111111111111108, 0.21627906976744185, 0.225, 0.23037037037037036, 0.2391304347826087, 0.24822695035461, 0.25972222222222235, 0.2687074829931973, 0.27666666666666667]}, "softmax_learned": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.0, 0.013333333333333334, 0.01666666666666678, 0.028571428571428577, 0.03333333333333333, 0.02962962962962963, 0.03333333333333334, 0.030303030303030304, 0.03333333333333333, 0.03333333333333333, 0.0380952380952381, 0.04444444444444444, 0.04583333333333333, 0.052941176470588235, 0.0611111111111111, 0.06315789473684211, 0.065, 0.07142857142857141, 0.07575757575757576, 0.08695652173913043, 0.09444444444444443, 0.1, 0.10384615384615385, 0.1111111111111111, 0.11071428571428571, 0.11379310344827584, 0.12111111111111111, 0.12903225806451613, 0.13333333333333333, 0.13737373737373737, 0.14411764705882352, 0.15428571428571441, 0.16388888888888903, 0.17027027027027028, 0.17543859649122806, 0.1811965811965812, 0.19166666666666668, 0.20000000000000012, 0.20555555555555566, 0.2131782945736434, 0.22045454545454546, 0.22666666666666666, 0.23768115942028986, 0.2503546099290781, 0.25972222222222235, 0.2673469387755102, 0.27666666666666667]}, "geom_softmax": {"coverage": [0.02, 0.04, 0.06, 0.08, 0.1, 0.12000000000000001, 0.13999999999999999, 0.16, 0.18, 0.19999999999999998, 0.22, 0.24, 0.26, 0.28, 0.30000000000000004, 0.32, 0.34, 0.36000000000000004, 0.38, 0.4, 0.42000000000000004, 0.44, 0.46, 0.48000000000000004, 0.5, 0.52, 0.54, 0.56, 0.5800000000000001, 0.6, 0.62, 0.64, 0.66, 0.68, 0.7000000000000001, 0.7200000000000001, 0.74, 0.76, 0.78, 0.8, 0.8200000000000001, 0.8400000000000001, 0.86, 0.88, 0.9, 0.92, 0.9400000000000001, 0.9600000000000001, 0.98, 1.0], "risk": [0.0, 0.0, 0.0, 0.008333333333333333, 0.02, 0.02222222222222222, 0.038095238095238106, 0.03333333333333333, 0.02962962962962963, 0.026666666666666672, 0.030303030303030304, 0.03333333333333333, 0.03333333333333333, 0.0380952380952381, 0.04222222222222222, 0.05, 0.056862745098039215, 0.0611111111111111, 0.06315789473684211, 0.06333333333333334, 0.07777777777777777, 0.08181818181818182, 0.08405797101449275, 0.09444444444444443, 0.09733333333333333, 0.10384615384615385, 0.10740740740740741, 0.11666666666666667, 0.11839080459770113, 0.12, 0.12688172043010754, 0.134375, 0.14343434343434344, 0.15196078431372548, 0.15619047619047632, 0.16851851851851865, 0.17747747747747747, 0.18421052631578946, 0.18803418803418803, 0.195, 0.2032520325203253, 0.21428571428571438, 0.22170542635658916, 0.22424242424242424, 0.23037037037037036, 0.24130434782608695, 0.2503546099290781, 0.2590277777777779, 0.2687074829931973, 0.27666666666666667]}}, "ltt": {"alpha": 0.1, "delta": 0.05, "lambda_hat": null, "coverage": 0.0, "abstain_rate": 1.0, "selective_risk": 0.0, "selective_accuracy": null, "risk_within_budget": true}, "coverage_validity": {"alpha_grid": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "achieved_risk": [0.0, 0.0, 0.0, 0.09854604200323101, 0.1673511293634497, 0.2629043358568479], "coverage": [0.0, 0.0, 0.0, 0.4126666666666667, 0.6493333333333333, 0.9686666666666667]}, "feature_diagnostics": {"names": ["basin/rho", "basin/entropy", "basin/dispersion", "energy/mean", "energy/min", "energy/std", "curv/lmax_mean", "curv/lmax_best", "curv/trace_mean", "curv/trace_best", "dynamics/steps", "dynamics/monotonic", "dynamics/drop", "dynamics/residual"], "auroc": {"basin/rho": 0.6281605685414469, "basin/entropy": 0.3710199322636167, "basin/dispersion": 0.518520903892066, "energy/mean": 0.40323580034423406, "energy/min": 0.4087679751263117, "energy/std": 0.5148586974626617, "curv/lmax_mean": 0.565753150852257, "curv/lmax_best": 0.5177969018932874, "curv/trace_mean": 0.6267170062739437, "curv/trace_best": 0.6067603131419688, "dynamics/steps": 0.5, "dynamics/monotonic": 0.6530875576036866, "dynamics/drop": 0.6858031203153628, "dynamics/residual": 0.515407251124313}, "hist": {"basin/rho": {"edges": [0.3333333333333333, 0.36666666666666664, 0.4, 0.43333333333333335, 0.4666666666666667, 0.5, 0.5333333333333333, 0.5666666666666667, 0.6000000000000001, 0.6333333333333333, 0.6666666666666667, 0.7000000000000001, 0.7333333333333334, 0.7666666666666667, 0.8, 0.8333333333333335, 0.8666666666666667, 0.9000000000000001, 0.9333333333333333, 0.9666666666666668, 1.0], "correct_counts": [1, 0, 1, 0, 0, 8, 0, 14, 0, 14, 0, 0, 20, 0, 25, 0, 0, 32, 0, 970], "incorrect_counts": [0, 0, 0, 0, 0, 8, 0, 19, 0, 31, 0, 0, 26, 0, 24, 0, 0, 42, 0, 265]}, "basin/entropy": {"edges": [0.0, 0.06430286684371896, 0.1286057336874379, 0.19290860053115688, 0.2572114673748758, 0.32151433421859477, 0.38581720106231376, 0.4501200679060327, 0.5144229347497516, 0.5787258015934706, 0.6430286684371895, 0.7073315352809085, 0.7716344021246275, 0.8359372689683464, 0.9002401358120654, 0.9645430026557843, 1.0288458694995033, 1.0931487363432222, 1.1574516031869413, 1.2217544700306602, 1.286057336874379], "correct_counts": [970, 0, 0, 0, 32, 0, 0, 23, 22, 14, 22, 0, 0, 0, 0, 0, 1, 0, 0, 1], "incorrect_counts": [265, 0, 0, 0, 42, 0, 0, 22, 23, 28, 22, 5, 2, 2, 2, 1, 1, 0, 0, 0]}, "basin/dispersion": {"edges": [1.596239525312261, 1.6174650761676754, 1.63869062702309, 1.6599161778785045, 1.681141728733919, 1.7023672795893336, 1.723592830444748, 1.7448183813001625, 1.766043932155577, 1.7872694830109916, 1.808495033866406, 1.8297205847218205, 1.8509461355772352, 1.8721716864326496, 1.893397237288064, 1.9146227881434785, 1.9358483389988932, 1.9570738898543076, 1.978299440709722, 1.9995249915651367, 2.020750542420551], "correct_counts": [2, 3, 9, 15, 23, 41, 58, 97, 122, 144, 143, 151, 93, 75, 34, 31, 25, 11, 4, 4], "incorrect_counts": [3, 1, 4, 6, 10, 23, 30, 38, 50, 46, 53, 37, 32, 28, 23, 17, 7, 2, 2, 3]}, "energy/mean": {"edges": [0.7065413296222687, 0.7633304566144943, 0.82011958360672, 0.8769087105989456, 0.9336978375911713, 0.9904869645833969, 1.0472760915756225, 1.1040652185678481, 1.160854345560074, 1.2176434725522993, 1.2744325995445251, 1.3312217265367507, 1.3880108535289764, 1.4447999805212022, 1.5015891075134276, 1.5583782345056534, 1.615167361497879, 1.6719564884901046, 1.7287456154823302, 1.785534742474556, 1.8423238694667816], "correct_counts": [20, 74, 153, 170, 154, 92, 69, 54, 36, 38, 27, 31, 32, 27, 39, 26, 25, 10, 4, 4], "incorrect_counts": [3, 23, 26, 55, 45, 32, 28, 22, 20, 30, 24, 27, 21, 22, 11, 10, 8, 3, 5, 0]}, "energy/min": {"edges": [0.33243313431739807, 0.39178149551153185, 0.4511298567056656, 0.5104782178997993, 0.5698265790939331, 0.6291749402880669, 0.6885233014822006, 0.7478716626763344, 0.8072200238704681, 0.8665683850646019, 0.9259167462587357, 0.9852651074528694, 1.0446134686470032, 1.103961829841137, 1.1633101910352708, 1.2226585522294044, 1.2820069134235381, 1.341355274617672, 1.4007036358118057, 1.4600519970059396, 1.5194003582000732], "correct_counts": [10, 39, 87, 128, 161, 124, 119, 69, 56, 36, 42, 32, 28, 35, 24, 38, 20, 16, 16, 5], "incorrect_counts": [4, 3, 27, 35, 38, 48, 28, 24, 32, 32, 19, 30, 27, 19, 21, 10, 9, 5, 2, 2]}, "energy/std": {"edges": [0.0686106895390655, 0.08331282795302085, 0.09801496636697621, 0.11271710478093155, 0.1274192431948869, 0.14212138160884225, 0.1568235200227976, 0.17152565843675294, 0.18622779685070828, 0.20092993526466363, 0.21563207367861897, 0.23033421209257432, 0.24503635050652972, 0.25973848892048507, 0.2744406273344404, 0.28914276574839576, 0.3038449041623511, 0.31854704257630645, 0.3332491809902618, 0.34795131940421714, 0.3626534578181725], "correct_counts": [2, 1, 21, 20, 52, 85, 126, 148, 145, 127, 113, 81, 52, 50, 24, 11, 8, 12, 6, 1], "incorrect_counts": [1, 0, 4, 10, 22, 40, 50, 52, 51, 58, 40, 36, 24, 9, 6, 5, 6, 1, 0, 0]}, "curv/lmax_mean": {"edges": [0.9988646656274796, 0.9989987172186374, 0.9991327688097954, 0.9992668204009533, 0.9994008719921113, 0.9995349235832691, 0.999668975174427, 0.999803026765585, 0.9999370783567428, 1.0000711299479008, 1.0002051815390587, 1.0003392331302166, 1.0004732847213744, 1.0006073363125325, 1.0007413879036904, 1.0008754394948483, 1.0010094910860061, 1.001143542677164, 1.001277594268322, 1.00141164585948, 1.0015456974506378], "correct_counts": [2, 6, 14, 38, 85, 156, 260, 277, 129, 58, 33, 11, 10, 4, 1, 0, 0, 0, 0, 1], "incorrect_counts": [1, 3, 5, 14, 39, 80, 117, 100, 27, 11, 10, 2, 2, 3, 0, 0, 0, 0, 0, 1]}, "curv/lmax_best": {"edges": [0.9917203187942505, 0.9926185190677643, 0.993516719341278, 0.9944149196147919, 0.9953131198883056, 0.9962113201618195, 0.9971095204353333, 0.998007720708847, 0.9989059209823609, 0.9998041212558746, 1.0007023215293884, 1.0016005218029023, 1.002498722076416, 1.0033969223499297, 1.0042951226234436, 1.0051933228969574, 1.0060915231704712, 1.006989723443985, 1.0078879237174987, 1.0087861239910125, 1.0096843242645264], "correct_counts": [2, 0, 0, 1, 3, 5, 6, 44, 279, 726, 15, 2, 0, 0, 1, 0, 0, 0, 0, 1], "incorrect_counts": [0, 0, 0, 0, 1, 2, 1, 17, 112, 273, 6, 0, 3, 0, 0, 0, 0, 0, 0, 0]}, "curv/trace_mean": {"edges": [31.799927234649658, 31.81066793600718, 31.821408637364705, 31.83214933872223, 31.84289004007975, 31.85363074143728, 31.864371442794802, 31.875112144152325, 31.88585284550985, 31.896593546867372, 31.907334248224895, 31.91807494958242, 31.928815650939942, 31.939556352297465, 31.95029705365499, 31.961037755012512, 31.97177845637004, 31.982519157727562, 31.993259859085086, 32.004000560442606, 32.01474126180013], "correct_counts": [1, 0, 2, 4, 24, 15, 36, 40, 48, 41, 49, 36, 35, 35, 34, 51, 95, 234, 256, 49], "incorrect_counts": [1, 0, 0, 6, 6, 16, 18, 29, 34, 23, 23, 30, 11, 18, 23, 24, 37, 53, 54, 9]}, "curv/trace_best": {"edges": [31.509716033935547, 31.5363130569458, 31.562910079956055, 31.58950710296631, 31.616104125976562, 31.642701148986816, 31.66929817199707, 31.695895195007324, 31.722492218017578, 31.749089241027832, 31.775686264038086, 31.80228328704834, 31.828880310058594, 31.855477333068848, 31.8820743560791, 31.908671379089355, 31.93526840209961, 31.961865425109863, 31.988462448120117, 32.01505947113037, 32.041656494140625], "correct_counts": [0, 0, 0, 0, 0, 1, 1, 3, 6, 12, 14, 11, 30, 29, 60, 81, 113, 245, 460, 19], "incorrect_counts": [1, 0, 0, 0, 0, 0, 0, 0, 1, 5, 4, 11, 18, 29, 31, 44, 73, 90, 100, 8]}, "dynamics/steps": {"edges": [1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5, 1.55, 1.6, 1.65, 1.7000000000000002, 1.75, 1.8, 1.85, 1.9, 1.9500000000000002, 2.0], "correct_counts": [1085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "incorrect_counts": [415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/monotonic": {"edges": [0.6349999999999999, 0.6424166666666666, 0.6498333333333333, 0.6572499999999999, 0.6646666666666666, 0.6720833333333334, 0.6795, 0.6869166666666666, 0.6943333333333334, 0.7017500000000001, 0.7091666666666667, 0.7165833333333333, 0.7240000000000001, 0.7314166666666668, 0.7388333333333335, 0.7462500000000001, 0.7536666666666668, 0.7610833333333336, 0.7685000000000002, 0.7759166666666668, 0.7833333333333335], "correct_counts": [2, 1, 10, 26, 66, 85, 136, 130, 185, 113, 121, 110, 51, 23, 14, 4, 5, 2, 0, 1], "incorrect_counts": [0, 3, 13, 28, 34, 39, 87, 60, 73, 38, 14, 18, 7, 1, 0, 0, 0, 0, 0, 0]}, "dynamics/drop": {"edges": [14.541723385453224, 20.501742911587158, 26.461762437721095, 32.42178196385503, 38.381801489988966, 44.3418210161229, 50.30184054225684, 56.261860068390774, 62.22187959452471, 68.18189912065864, 74.14191864679258, 80.10193817292651, 86.06195769906046, 92.02197722519439, 97.98199675132832, 103.94201627746226, 109.90203580359619, 115.86205532973013, 121.82207485586406, 127.782094381998, 133.74211390813193], "correct_counts": [123, 310, 256, 163, 111, 52, 33, 12, 7, 6, 6, 2, 2, 1, 0, 0, 0, 0, 0, 1], "incorrect_counts": [107, 178, 75, 29, 18, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "dynamics/residual": {"edges": [1.1456641207138698, 1.162844986965259, 1.180025853216648, 1.1972067194680374, 1.2143875857194266, 1.2315684519708157, 1.248749318222205, 1.2659301844735942, 1.2831110507249832, 1.3002919169763725, 1.3174727832277617, 1.3346536494791508, 1.35183451573054, 1.3690153819819293, 1.3861962482333183, 1.4033771144847076, 1.4205579807360968, 1.4377388469874859, 1.4549197132388751, 1.4721005794902644, 1.4892814457416534], "correct_counts": [1, 2, 5, 8, 34, 72, 93, 117, 158, 165, 146, 97, 86, 51, 28, 12, 9, 0, 0, 1], "incorrect_counts": [1, 0, 5, 5, 17, 30, 33, 51, 49, 66, 50, 46, 31, 16, 9, 2, 1, 3, 0, 0]}}}, "feature_ablation": {"full": 0.13283935829490207, "drop_basin": 0.14997742438206202, "basin_only": 0.23296177845321314, "drop_energy": 0.13721472001078186, "energy_only": 0.22409660098635795, "drop_curv": 0.13097753105946705, "curv_only": 0.2233216171978382, "drop_dynamics": 0.18117722135638095, "dynamics_only": 0.1620572670509188}, "ece_geometry": 0.0349920061619123, "accuracy_ood": 0.30666666666666664, "aurc_ood": {"geometry": 0.7404444300750076, "rho_basin": 0.6925842472273304, "energy_min": 0.6068299457403026, "energy_mean": 0.6018766592700887, "energy_std": 0.6989784553631774, "msp": 0.7052802742194463, "temp_msp": 0.697350347427352, "entropy": 0.7029716683577881, "softmax_learned": 0.6957988290761644, "geom_softmax": 0.7041596483562519}, "ood_ltt": {"alpha": 0.1, "lambda_hat": null, "selective_risk": 0.0, "coverage": 0.0, "risk_within_budget": true}, "ood_validity": {"target": [0.02, 0.05, 0.1, 0.15, 0.2, 0.3], "id_risk": [0.0, 0.0, 0.0, 0.09854604200323101, 0.1673511293634497, 0.2629043358568479], "ood_risk": [0.0, 0.0, 0.0, 0.7001114827201784, 0.6923751095530236, 0.6935593220338983], "id_coverage": [0.0, 0.0, 0.0, 0.4126666666666667, 0.6493333333333333, 0.9686666666666667], "ood_coverage": [0.0, 0.0, 0.0, 0.598, 0.7606666666666667, 0.9833333333333333]}}, "env": {"python": "3.12.13", "platform": "macOS-26.5.2-arm64-arm-64bit", "jax": "0.11.0", "jax_backend": "cpu"}, "artifact_paths": []} diff --git a/src/edc/plotting.py b/src/edc/plotting.py index 516dbe2..2a91003 100644 --- a/src/edc/plotting.py +++ b/src/edc/plotting.py @@ -45,26 +45,34 @@ def use_style() -> None: def _selective_row(rows: list[dict[str, Any]]) -> dict[str, Any]: - """Metrics of the headline ``split == 'selective'`` run for F2/F3 (raises if none). + """Metrics of the headline ``split == 'selective'`` run for F2/F3/F5/F6 (raises if none). - Prefers the full-fold run — largest ``n_test`` (E1), ties broken by ledger order — so the - single-run figures reflect the authoritative experiment, not whatever reduced-fold sweep cell - happened to land last in the ledger. + These are the arithmetic E1 figures by design (their captions name arithmetic), so we prefer + the arithmetic full-fold run — largest ``n_test``, ties broken by ledger order. Preferring + arithmetic matters since Phase 4l added full-fold (``n_test=1500``) graph rows that would + otherwise tie and flip the figures to graph. Falls back to the global run if no arithmetic row. """ sel = [r for r in rows if r.get("split") == "selective"] if not sel: raise ValueError("no split='selective' ledger row; run experiments/run_experiment.py first") + arith = [r for r in sel if r.get("task") == "arithmetic"] + pool = arith or sel # largest n_test wins; ties broken toward the latest ledger row (freshest full-fold E1). - headline = max(enumerate(sel), key=lambda iv: (iv[1]["metrics"].get("n_test", 0), iv[0]))[1] + headline = max(enumerate(pool), key=lambda iv: (iv[1]["metrics"].get("n_test", 0), iv[0]))[1] return headline["metrics"] def _halting_row(rows: list[dict[str, Any]]) -> dict[str, Any]: - """Metrics of the latest ``split == 'halting'`` run for F4 (raises if none).""" + """Metrics of the headline ``split == 'halting'`` run for F4 (raises if none). + + F4 is the arithmetic halting Pareto by design, so prefer the latest arithmetic halting row; + Phase 4l added graph halting rows (appended later) that would otherwise flip F4 to graph. + """ sel = [r for r in rows if r.get("split") == "halting"] if not sel: raise ValueError("no split='halting' ledger row; run experiments/run_halting.py first") - return sel[-1]["metrics"] + arith = [r for r in sel if r.get("task") == "arithmetic"] + return (arith or sel)[-1]["metrics"] def risk_coverage_curve(rows: list[dict[str, Any]], out_path: str) -> str: # F2 diff --git a/tests/test_aggregate.py b/tests/test_aggregate.py index 8f3e6c7..054d703 100644 --- a/tests/test_aggregate.py +++ b/tests/test_aggregate.py @@ -7,7 +7,7 @@ def _row(k, seed, delta, lo, geo=0.08, best=0.14, task="arithmetic", baseline_delta=None, - geom_adds=None, feature_set=None): + geom_adds=None, feature_set=None, ood=None): m = { "k_restarts": k, "accuracy_id": 0.8, "base_error": 0.2, "best_energy_baseline": "energy_min", @@ -26,6 +26,12 @@ def _row(k, seed, delta, lo, geo=0.08, best=0.14, task="arithmetic", baseline_de if geom_adds is not None: # Phase 4j field ga, glo = geom_adds m["delta_aurc_geom_adds"] = [ga, glo, ga + 0.02] + if ood is not None: # Phase 4l fields (include_ood rows only) + acc_ood, risk_ood, cov_ood, within = ood + m["accuracy_ood"] = acc_ood + m["ltt"]["selective_risk"] = 0.07 # ID selective risk (already present) + m["ood_ltt"] = {"selective_risk": risk_ood, "coverage": cov_ood, + "risk_within_budget": within} fs_tag = "" if feature_set is not None: # Phase 4k field (older rows omit it) m["feature_set"] = feature_set @@ -116,6 +122,66 @@ def test_feature_set_filter_and_fallback(): assert hr["delta_aurc_geom_adds"][0] > hb["delta_aurc_geom_adds"][0] +def test_ood_aggregation_and_fallback(): + # Phase 4l: rows run with include_ood carry accuracy_ood + ood_ltt; aggregate_cell summarises + # them and counts seeds whose OOD risk stays within budget. + rows = [_row(12, s, 0.09, 0.07, baseline_delta=(0.05, 0.03), + ood=(0.4, 0.5, 0.8, False)) for s in range(5)] + a = aggregate.aggregate_cell(rows) + assert a["has_ood"] is True + assert a["accuracy_ood"][0] == 0.4 + assert a["ood_selective_risk"][0] == 0.5 # OOD risk >> alpha => guarantee breaks + assert a["ltt_selective_risk"][0] == 0.07 # ID risk stays low + assert a["seeds_ood_within_budget"] == 0 + + # one seed happens to stay within budget on OOD + mixed = [_row(12, s, 0.09, 0.07, ood=(0.4, 0.09 if s == 0 else 0.5, 0.8, s == 0)) + for s in range(5)] + assert aggregate.aggregate_cell(mixed)["seeds_ood_within_budget"] == 1 + + # pre-4l rows (no OOD fields) fall back cleanly: has_ood False, NaNs, no crash + old = aggregate.aggregate_cell([_row(12, s, 0.09, 0.07) for s in range(3)]) + assert old["has_ood"] is False + import math + assert math.isnan(old["accuracy_ood"][0]) + + +def _hrow(seed, compute_saved, halting_risk, within=True, monotone=True, tau=0.9, + task="arithmetic"): + m = { + "k_restarts": 12, "alpha": 0.1, "tau_hat": tau, + "compute_saved": compute_saved, "compute_used": 1.0 - compute_saved, + "halting_risk": halting_risk, "risk_within_budget": within, + "risk_monotone_in_tau": monotone, "full_accuracy": 0.8, "accuracy_drop": 0.0, + } + return { + "run_id": f"h{seed}", "seed": seed, "split": "halting", "task": task, + "config_hash": f"{task}_halt_{seed}", "git_sha": "abc1234", + "env": {"python": "3.12", "jax": "0.11", "jax_backend": "cpu"}, + "config": {"inference": {"k_restarts": 12}}, "metrics": m, + } + + +def test_halting_aggregation(): + # split="halting" rows are excluded from selective_rows and aggregated separately (Phase 4l). + hrows = [_hrow(s, 0.5 + 0.01 * s, 0.02) for s in range(5)] + selrows = [_row(12, s, 0.09, 0.07) for s in range(3)] + mixed = hrows + selrows + assert aggregate.halting_rows(mixed) == hrows # halting split filter + assert len(aggregate.selective_rows(mixed)) == 3 # halting rows excluded + + a = aggregate.aggregate_halting_cell(hrows) + assert a["n_seeds"] == 5 and a["task"] == "arithmetic" + assert abs(a["compute_saved"][0] - 0.52) < 1e-9 + assert a["halting_risk"][0] == 0.02 + assert a["seeds_within_budget"] == 5 and a["seeds_risk_monotone"] == 5 + + # a seed over budget is counted honestly; a None tau_hat (infeasible) doesn't crash the mean + over = [_hrow(0, 0.3, 0.2, within=False, tau=None), *[_hrow(s, 0.5, 0.02) for s in range(1, 4)]] + ao = aggregate.aggregate_halting_cell(over) + assert ao["seeds_within_budget"] == 3 and ao["n_tau_feasible"] == 3 + + def test_feature_ablation_aggregates(): rows = [_row(12, s, 0.09, 0.07, geo=0.08, baseline_delta=(0.05, 0.03)) for s in range(4)] fa = aggregate.aggregate_cell(rows)["feature_ablation"] diff --git a/tests/test_make_tables.py b/tests/test_make_tables.py index 9b364e3..f606d52 100644 --- a/tests/test_make_tables.py +++ b/tests/test_make_tables.py @@ -6,7 +6,7 @@ import aggregate import make_tables -from test_aggregate import _row +from test_aggregate import _hrow, _row def _rows(): @@ -47,5 +47,25 @@ def test_tables_well_formed(): assert t3.count(r"\\") >= len(rows) +def test_t6_halting_and_t7_ood(): + # T6 from split="halting" rows; T7 from include_ood selective rows (Phase 4l). + hrows = [_hrow(s, 0.55, 0.02, task="arithmetic") for s in range(5)] + t6 = make_tables._t6(hrows) + assert r"\label{tab:halting}" in t6 + assert t6.split(r"\midrule")[1].split(r"\bottomrule")[0].count(r"\\") == 1 # one task row + assert "arithmetic" in t6 + + ood_rows = [_row(12, s, 0.09, 0.07, baseline_delta=(0.05, 0.03), + ood=(0.4, 0.5, 0.8, False)) for s in range(5)] + t7 = make_tables._t7(ood_rows) + assert r"\label{tab:oodstress}" in t7 + assert "arithmetic" in t7 and r"\toprule" in t7 + + # both are gated: absent data -> empty body + assert make_tables._t6([]).split(r"\midrule")[1].split(r"\bottomrule")[0].strip() == "" + no_ood = [_row(12, s, 0.09, 0.07, baseline_delta=(0.05, 0.03)) for s in range(3)] + assert make_tables._t7(no_ood).split(r"\midrule")[1].split(r"\bottomrule")[0].strip() == "" + + def test_delta_pm_formatting(): assert make_tables._pm((0.0736, 0.012)) == r"$0.074 \pm 0.012$" diff --git a/tests/test_run_halting_sweep.py b/tests/test_run_halting_sweep.py new file mode 100644 index 0000000..2937266 --- /dev/null +++ b/tests/test_run_halting_sweep.py @@ -0,0 +1,41 @@ +"""run_halting_sweep expands a seed grid and appends split='halting' rows via evaluate_halting. + +Offline + CPU-only and FAST: ``evaluate_halting`` and the ledger ``append`` are monkeypatched, so +no reasoner is trained and the real ledger is never written — we only check the runner's plumbing +(grid expansion, one halting row per cell, correct split/seed). +""" + +import run_halting_sweep + + +def _fake_metrics(): + return {"tau_hat": 0.9, "compute_saved": 0.5, "halting_risk": 0.02, + "risk_within_budget": True, "alpha": 0.1} + + +def test_run_halting_sweep_appends_one_halting_row_per_cell(tmp_path, monkeypatch): + sweep = tmp_path / "smoke_halting.toml" + sweep.write_text( + '[sweep]\nbase = "configs/smoke.toml"\n[sweep.grid]\n"run.seed" = [0, 1, 2]\n') + + calls = {"eval": 0} + captured = [] + + def fake_eval(cfg, task): + calls["eval"] += 1 + return _fake_metrics() + + def fake_append(record): + row = record.to_row() if hasattr(record, "to_row") else record + captured.append(row) + return row + + monkeypatch.setattr(run_halting_sweep, "evaluate_halting", fake_eval) + monkeypatch.setattr(run_halting_sweep, "append", fake_append) + + rc = run_halting_sweep.main([str(sweep)]) + assert rc == 0 + assert calls["eval"] == 3 # one evaluate_halting per seed cell + assert len(captured) == 3 + assert all(r["split"] == "halting" for r in captured) + assert sorted(r["seed"] for r in captured) == [0, 1, 2] diff --git a/tests/test_run_sweep.py b/tests/test_run_sweep.py index 36f198d..276ae6e 100644 --- a/tests/test_run_sweep.py +++ b/tests/test_run_sweep.py @@ -35,7 +35,17 @@ def test_cell_config_applies_override_and_cell(): def test_load_sweep_reads_real_config(): - base, override, cells = run_sweep.load_sweep("configs/sweeps/k_restarts.toml") + base, override, cells, opts = run_sweep.load_sweep("configs/sweeps/k_restarts.toml") assert len(cells) == 25 # 5 K x 5 seeds assert override["eval.n_eval"] == 600 assert base["task"]["arithmetic"]["n_operands"] == 6 # base experiment config resolved + assert opts["include_ood"] is False # default: OOD skipped in sweeps + + +def test_load_sweep_include_ood_flag(): + # Phase 4l: a sweep can opt the OOD fold back on for multi-seed OOD / certification runs. + base, override, cells, opts = run_sweep.load_sweep("configs/sweeps/graph_cert_seeds.toml") + assert opts["include_ood"] is True + assert len(cells) == 5 # 5 seeds + assert base["conformal"]["n_calib"] == 1500 # full folds (no reduced-fold override) + assert "conformal.n_calib" not in override