report pipeline: build_report.py (self-contained HTML), eval --runs-dir, plot fix, halting per-input steps

8 files changed, 354 insertions, 6 deletions

diff --git a/.gitignore b/.gitignore
index f9bf0eb..a0b7284 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,4 @@ __pycache__/
 .venv/
 runs/
 .pytest_cache/
+report/
diff --git a/runs-take2/rnn/seed0/best.pt b/runs-take2/rnn/seed0/best.pt
new file mode 100644
index 0000000..3dc0b21
--- /dev/null
+++ b/runs-take2/rnn/seed0/best.pt
diff --git a/runs-take2/rnn/seed0/config.json b/runs-take2/rnn/seed0/config.json
new file mode 100644
index 0000000..5433dd4
--- /dev/null
+++ b/runs-take2/rnn/seed0/config.json
@@ -0,0 +1,30 @@
+{
+  "vocab_mode": "digits",
+  "range_start": 2,
+  "range_end": 100,
+  "holdout_frac": 0.3,
+  "seed": 0,
+  "model": "rnn",
+  "d_model": 128,
+  "max_steps": 20,
+  "halting": true,
+  "halt_penalty": 0.01,
+  "halt_warmup_steps": 1000,
+  "halt_ramp_end_steps": 5000,
+  "min_steps": 2,
+  "n_layers": 2,
+  "n_heads": 4,
+  "lr": 0.001,
+  "weight_decay": 1.0,
+  "max_train_steps": 200000,
+  "eval_every": 200,
+  "early_stop_em": 1.0,
+  "early_stop_patience": 5,
+  "batch_size": 32,
+  "max_out_len": 6,
+  "log_n_examples": 10,
+  "out_dir": "runs-take2",
+  "vocab": 11,
+  "eos_id": 10,
+  "pad_id": 11
+}
\ No newline at end of file
diff --git a/runs-take2/rnn/seed0/metrics.csv b/runs-take2/rnn/seed0/metrics.csv
new file mode 100644
index 0000000..9269796
--- /dev/null
+++ b/runs-take2/rnn/seed0/metrics.csv
@@ -0,0 +1,23 @@
+step,train_loss,train_token_acc,train_em,val_token_acc,val_em,mean_halt_steps,log_examples,param_count

+200,0.4903171956539154,0.8701923076923077,0.6666666666666666,0.5340909090909091,0.1,4.234026908874512,3:3~5;4:7~5;8:9~11;9:1~11;10:ok;12:33~13;13:11~17;15:33~17;18:31~19;21:33~23,36620

+400,0.17726276814937592,0.9855769230769231,0.9565217391304348,0.5681818181818182,0.2,6.274046421051025,3:3~5;4:7~5;8:9~11;9:1~11;10:ok;12:33~13;13:11~17;15:37~17;18:31~19;21:ok,36620

+600,0.5476387143135071,0.9278846153846154,0.782608695652174,0.5909090909090909,0.2,7.251992225646973,3:3~5;4:7~5;8:9~11;9:1~11;10:1~11;12:33~13;13:11~17;15:37~17;18:31~19;21:ok,36620

+800,0.09844685345888138,1.0,1.0,0.5568181818181818,0.16666666666666666,7.7551164627075195,3:3~5;4:7~5;8:9~11;9:1~11;10:ok;12:23~13;13:31~17;15:37~17;18:31~19;21:ok,36620

+1000,0.10235479474067688,1.0,1.0,0.5909090909090909,0.13333333333333333,10.119675636291504,3:7~5;4:7~5;8:9~11;9:1~11;10:ok;12:23~13;13:11~17;15:73~17;18:31~19;21:ok,36620

+1200,0.26242315769195557,0.9855769230769231,0.9565217391304348,0.6022727272727273,0.2,6.777245998382568,3:3~5;4:7~5;8:9~11;9:1~11;10:ok;12:23~13;13:13~17;15:37~17;18:31~19;21:ok,36620

+1400,0.29470914602279663,0.8990384615384616,0.7681159420289855,0.5340909090909091,0.16666666666666666,8.125110626220703,3:7~5;4:7~5;8:9~11;9:ok;10:1~11;12:23~13;13:11~17;15:23~17;18:23~19;21:ok,36620

+1600,0.14815756678581238,0.9903846153846154,0.9710144927536232,0.6022727272727273,0.2,8.417304992675781,3:3~5;4:7~5;8:1~11;9:1~11;10:ok;12:23~13;13:13~17;15:79~17;18:21~19;21:ok,36620

+1800,0.14240387082099915,0.9951923076923077,0.9855072463768116,0.6136363636363636,0.23333333333333334,10.154568672180176,3:7~5;4:7~5;8:9~11;9:1~11;10:ok;12:23~13;13:11~17;15:77~17;18:31~19;21:ok,36620

+2000,0.34200283885002136,0.9182692307692307,0.7681159420289855,0.6363636363636364,0.3,7.1683454513549805,3:7~5;4:7~5;8:9~11;9:1~11;10:ok;12:23~13;13:ok;15:77~17;18:23~19;21:ok,36620

+2200,0.1432158350944519,0.9855769230769231,0.9565217391304348,0.6931818181818182,0.3333333333333333,10.690964698791504,3:7~5;4:7~5;8:1~11;9:1~11;10:ok;12:23~13;13:11~17;15:97~17;18:29~19;21:ok,36620

+2400,0.35249558091163635,0.9326923076923077,0.855072463768116,0.5795454545454546,0.16666666666666666,8.192712783813477,3:3~5;4:7~5;8:1~11;9:1~11;10:ok;12:23~13;13:11~17;15:59~17;18:21~19;21:ok,36620

+2600,0.23677189648151398,0.9807692307692307,0.9420289855072463,0.6022727272727273,0.16666666666666666,9.858160018920898,3:3~5;4:7~5;8:1~11;9:1~11;10:ok;12:23~13;13:11~17;15:77~17;18:23~19;21:ok,36620

+2800,0.34069204330444336,0.9423076923076923,0.8260869565217391,0.5568181818181818,0.13333333333333333,6.819947242736816,3:3~5;4:7~5;8:9~11;9:1~11;10:ok;12:29~13;13:11~17;15:59~17;18:83~19;21:29~23,36620

+3000,0.24851882457733154,0.9230769230769231,0.8115942028985508,0.7045454545454546,0.36666666666666664,6.370993614196777,3:3~5;4:7~5;8:9~11;9:7~11;10:ok;12:23~13;13:37~17;15:71~17;18:33~19;21:ok,36620

+3200,0.38633477687835693,0.9519230769230769,0.8840579710144928,0.6136363636363636,0.26666666666666666,6.982702255249023,3:3~5;4:7~5;8:7~11;9:1~11;10:ok;12:23~13;13:31~17;15:79~17;18:83~19;21:ok,36620

+3400,0.22584262490272522,1.0,1.0,0.6477272727272727,0.23333333333333334,8.568931579589844,3:7~5;4:7~5;8:1~11;9:1~11;10:ok;12:23~13;13:37~17;15:79~17;18:23~19;21:ok,36620

+3600,0.6283265948295593,0.8894230769230769,0.7391304347826086,0.5681818181818182,0.2,8.007800102233887,3:3~5;4:7~5;8:7~11;9:3~11;10:ok;12:33~13;13:11~17;15:73~17;18:83~19;21:ok,36620

+3800,0.3623741865158081,0.9423076923076923,0.855072463768116,0.6818181818181818,0.3333333333333333,7.691666603088379,3:3~5;4:7~5;8:1~11;9:3~11;10:ok;12:23~13;13:31~17;15:97~17;18:29~19;21:ok,36620

+4000,0.2987810969352722,0.9807692307692307,0.9420289855072463,0.6022727272727273,0.2,7.724488735198975,3:3~5;4:7~5;8:3~11;9:1~11;10:ok;12:23~13;13:11~17;15:79~17;18:33~19;21:ok,36620

+4200,0.2570030093193054,0.9375,0.8405797101449275,0.5909090909090909,0.2,8.140658378601074,3:3~5;4:7~5;8:8~11;9:1~11;10:ok;12:23~13;13:37~17;15:97~17;18:83~19;21:29~23,36620

+4400,1.5922589302062988,0.6153846153846154,0.2463768115942029,0.4772727272727273,0.06666666666666667,5.81683349609375,3:7~5;4:7~5;8:-1~11;9:1~11;10:71~11;12:33~13;13:79~17;15:73~17;18:89~19;21:33~23,36620

diff --git a/runs-take2/rnn/seed0/run_meta.json b/runs-take2/rnn/seed0/run_meta.json
new file mode 100644
index 0000000..587b392
--- /dev/null
+++ b/runs-take2/rnn/seed0/run_meta.json
@@ -0,0 +1,14 @@
+{
+  "python": "3.11.15",
+  "torch": "2.13.0+cpu",
+  "numpy": "2.4.6",
+  "device": "cpu",
+  "torch_threads": 2,
+  "cmd": [
+    "/home/jayrup/Projects/prime-grokking/src/train.py",
+    "rnn",
+    "0",
+    "--out_dir",
+    "runs-take2"
+  ]
+}
\ No newline at end of file
diff --git a/scripts/build_report.py b/scripts/build_report.py
new file mode 100644
index 0000000..4fdeb07
--- /dev/null
+++ b/scripts/build_report.py
@@ -0,0 +1,271 @@
+#!/usr/bin/env python
+"""Build the self-contained HTML experiment report: report/index.html
+
+Reads results.json + curves.png from both model run dirs, renders extra figures
+(probe errors, halting scatter) inline, embeds everything as base64 — one file,
+viewable on a phone over the tailnet.
+
+Usage: .venv/bin/python -m scripts.build_report [--runs-dir DIR]
+"""
+import argparse
+import base64
+import io
+import json
+import os
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+from src.eval import FLAGGED_SIEVE_PREDS
+
+CODE_LEGEND = {
+    "O1": "Sharp transition — grokking-like",
+    "O2": "Memorization won (train 100%, val low)",
+    "O3": "Gradual rise — smooth heuristic, NOT grokking by our definition",
+    "O4": "Train never saturated — setup/optimization failure, no interpretation",
+    "O-PARTIAL": "Train saturated, val ended mid-range — partial in-range generalization",
+    "H1": "Halt gate collapsed to the min_steps floor",
+    "H2": "Halt gate pinned at K — never learned to halt",
+    "H3": "Intermediate steps, positive gap correlation — learned compute budget",
+    "H4": "Intermediate steps, no gap correlation — noisy/unused",
+    "P1": "Errors = predictions of 121/143/169/187/209 — learned {2,3,5,7} sieve",
+    "P2": "Scattered errors — memorization / non-transferable heuristics",
+    "P3": "Probe accuracy ≥85% — surprising, needs seed verification",
+    "P4": "Fails trivial evens/5-multiples — pure memorization",
+}
+
+INTERP = {
+    "O1": "The tied cell + heavy weight decay found a structured in-range solution; memorization was repelled. NOT yet evidence of the full sieve — may be skip-evens/5s + divisibility heuristics.",
+    "O2": "The lookup table is the lower-norm solution under these hyperparameters — consistent with the spec's 'memorization is a stronger attractor'.",
+    "O3": "A 'fast' generalizing solution exists that gradient descent finds directly — still publishable, but not grokking.",
+    "O4": "No scientific interpretation until the training setup is fixed.",
+    "O-PARTIAL": "The model learned part of the mapping but never completed the in-range transition within budget.",
+    "H1": "ACT failed — read architecture conclusions with a collapsed gate; the fixed-K ablation becomes the informative run.",
+    "H2": "Never learned to halt — penalty too weak.",
+    "H3": "Evidence of a learned computation budget: more compute steps for larger gaps.",
+    "H4": "The halt signal is not being used meaningfully.",
+    "P1": "Strongest positive result available at this scale: definitive evidence of a learned sieve with the training-range divisor set.",
+    "P2": "No evidence of a divisibility-based algorithm transferring out-of-range.",
+    "P3": "Richer algorithm than the {2,3,5,7} sieve — treat with suspicion, verify across seeds.",
+    "P4": "The in-range solution didn't even transfer the trivial heuristics — strong memorization evidence.",
+}
+
+
+def b64(png_bytes: bytes) -> str:
+    return "data:image/png;base64," + base64.b64encode(png_bytes).decode()
+
+
+def img_b64(path: str) -> str:
+    with open(path, "rb") as fh:
+        return b64(fh.read())
+
+
+def probe_fig(result: dict, model: str) -> str:
+    errors = result.get("errors", [])
+    flagged = [e for e in errors if e["pred"] in FLAGGED_SIEVE_PREDS]
+    fig, ax = plt.subplots(figsize=(10, 3.2))
+    xs, ys, cs, ls = [], [], [], []
+    for e in errors:
+        xs.append(e["n"]); ys.append(1.0)
+        cs.append("tab:red" if e["pred"] in FLAGGED_SIEVE_PREDS else "tab:blue")
+        ls.append(f"n={e['n']}\npred {e['pred']}\ntarget {e['target']}")
+    ax.scatter(xs, ys, c=cs, s=28, zorder=3)
+    ax.set_xlim(100, 201)
+    ax.set_ylim(0.6, 1.4)
+    ax.set_yticks([1.0]); ax.set_yticklabels(["wrong"])
+    ax.set_xlabel("input n")
+    ax.set_title(f"{model}: probe errors (red = predicted a no-small-divisor composite "
+                 f"{sorted(FLAGGED_SIEVE_PREDS)})")
+    for x, y, l in zip(xs, ys, ls):
+        ax.annotate(l, (x, y), textcoords="offset points", xytext=(0, 10), fontsize=6,
+                    ha="center", rotation=90, va="bottom")
+    plt.tight_layout()
+    buf = io.BytesIO(); plt.savefig(buf, dpi=110); plt.close(fig)
+    return b64(buf.getvalue())
+
+
+def halting_fig(result: dict, cfg: dict) -> str:
+    gaps = result.get("gaps", [])
+    steps = result.get("steps_by_n", [])
+    n = list(range(cfg["range_start"], cfg["range_end"] + 1))
+    fig, ax = plt.subplots(figsize=(8, 3))
+    ax.scatter(gaps, steps, s=14)
+    ax.set_xlabel("gap to next prime")
+    ax.set_ylabel("mean halt steps")
+    ax.set_title(f"RNN halting vs gap-to-next-prime (rho={result.get('corr_gap', 0):.3f})")
+    plt.tight_layout()
+    buf = io.BytesIO(); plt.savefig(buf, dpi=110); plt.close(fig)
+    return b64(buf.getvalue())
+
+
+def meta_table(run_dir: str) -> str:
+    rows = []
+    try:
+        with open(os.path.join(run_dir, "run_meta.json")) as fh:
+            m = json.load(fh)
+        for k, v in m.items():
+            rows.append(f"<tr><td>{k}</td><td>{v}</td></tr>")
+    except FileNotFoundError:
+        rows.append("<tr><td colspan=2>run_meta.json missing</td></tr>")
+    return "".join(rows)
+
+
+def code_badge(code: str) -> str:
+    color = {"O1": "#2e7d32", "O2": "#c62828", "O3": "#f9a825", "O4": "#757575",
+             "O-PARTIAL": "#f9a825", "H1": "#c62828", "H2": "#c62828", "H3": "#2e7d32",
+             "H4": "#757575", "P1": "#2e7d32", "P2": "#c62828", "P3": "#f9a825",
+             "P4": "#c62828"}.get(code, "#757575")
+    return (f'<span class="badge" style="background:{color}">{code}</span> '
+            f'<span class="dim">{CODE_LEGEND.get(code, "")}</span>')
+
+
+def model_section(model: str, runs_dir: str) -> str:
+    rd = os.path.join(runs_dir, model, "seed0")
+    with open(os.path.join(rd, "results.json")) as fh:
+        res = json.load(fh)
+    with open(os.path.join(rd, "config.json")) as fh:
+        cfg = json.load(fh)
+    sig = res["signature"]
+    final = res["final"]
+    probe = final.get("probe", {})
+    hlt = final.get("halting")
+    curves = img_b64(os.path.join(rd, "curves.png"))
+    parts = [f"<h2>{model} — seed 0</h2>"]
+    parts.append(f'<p><b>Signature:</b> {code_badge(sig["code"])}</p>')
+    if hlt:
+        parts.append(f'<p><b>Halting:</b> {code_badge(hlt["code"])} '
+                     f'(mean {hlt["mean_steps"]:.2f} steps, ρ(gap, steps) = {hlt["corr_gap"]:.3f})</p>')
+    parts.append(f'<p><b>Probe [101,200]:</b> {code_badge(probe.get("code", "?"))} '
+                 f'— accuracy {probe.get("acc", float("nan")):.3f} '
+                 f'({probe.get("correct", 0)}/{probe.get("total", 0)}), '
+                 f'{len(probe.get("errors", []))} errors</p>')
+    parts.append(f'<p><b>Val exact-match</b> (last.pt, unselected): {final["val_exact_match"]:.3f} · '
+                 f'(best.pt, val-selected): {res["val_selected"]["val_exact_match"]:.3f} '
+                 f'<span class="dim">— best.pt is selection-holed by design</span></p>')
+    parts.append(f'<p><b>Params:</b> {final["params"]:,} · vocab={cfg["vocab"]} · '
+                 f'wd={cfg["weight_decay"]} · lr={cfg["lr"]} · K={cfg["max_steps"]} '
+                 f'· halting={cfg["halting"]}</p>')
+    parts.append(f'<img class="fig" src="{curves}" alt="curves">')
+    if probe.get("errors"):
+        parts.append(f'<img class="fig" src="{probe_fig(probe, model)}" alt="probe errors">')
+    if hlt and "gaps" in hlt:
+        parts.append(f'<img class="fig" src="{halting_fig(hlt, cfg)}" alt="halting">')
+    parts.append(f"<pre class='dim'>{json.dumps({'signature': sig, 'probe': {k: v for k, v in probe.items() if k != 'errors'}, 'probe_errors': probe.get('errors', []), 'halting': {k: v for k, v in (hlt or {}).items() if k not in ('gaps', 'steps_by_n')}}, indent=2)}</pre>")
+    return "".join(parts)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--runs-dir", default="runs-take2")
+    ap.add_argument("--out", default="report/index.html")
+    a = ap.parse_args()
+
+    rnn_sec = model_section("rnn", a.runs_dir)
+    tf_sec = model_section("transformer", a.runs_dir)
+    meta_rnn = meta_table(os.path.join(a.runs_dir, "rnn", "seed0"))
+    meta_tf = meta_table(os.path.join(a.runs_dir, "transformer", "seed0"))
+
+    html = f"""<!DOCTYPE html>
+<html lang="en"><head><meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>prime-grokking — Experiment 1</title>
+<style>
+:root {{ color-scheme: dark; }}
+body {{ font-family: -apple-system, system-ui, sans-serif; margin: 0 auto; max-width: 860px;
+       padding: 16px; background: #0f1115; color: #d7dbe0; line-height: 1.55; }}
+h1 {{ font-size: 1.5rem; }} h2 {{ font-size: 1.2rem; border-bottom: 1px solid #2a2f38; padding-bottom: 4px; }}
+a {{ color: #7fb3ff; }}
+.dim {{ color: #8b929c; font-size: .85rem; }}
+.badge {{ display: inline-block; padding: 1px 8px; border-radius: 10px; color: #fff; font-weight: 600; font-size: .8rem; }}
+.fig {{ width: 100%; height: auto; margin: 8px 0; border-radius: 8px; }}
+table {{ border-collapse: collapse; width: 100%; font-size: .85rem; }}
+td, th {{ border: 1px solid #2a2f38; padding: 4px 8px; text-align: left; }}
+pre {{ overflow-x: auto; background: #15181d; padding: 10px; border-radius: 8px; font-size: .72rem; }}
+blockquote {{ border-left: 3px solid #3a4150; margin-left: 0; padding-left: 12px; color: #a9b0ba; }}
+</style></head><body>
+<h1>prime-grokking — Experiment 1 (seed 0)</h1>
+<p class="dim">Can a minimal architecture <i>grok</i> the next-prime function? Pre-registered experiment ·
+2026-08-14 · runs on voidlaptop CPU · repo: ssh://meru/~/projects/prime-grokking.git</p>
+
+<h2>Why this exists</h2>
+<p>Grokking — the delayed jump from memorization to generalization — is one of the cleanest windows
+into the <b>memorization → generalization transition</b> (Q3 of the research agenda). The canonical
+grokking task is modular addition, which has smooth group structure (a low-complexity solution:
+rotate on a circle). <b>Next-prime strips that away:</b> it looks smooth but is structurally a
+combinatorial search (skip candidates, test divisibility, stop at the first hit). The hypothesis,
+from the spec: a <b>weight-tied</b> recurrent cell should be able to represent the algorithmic
+solution (one divisibility operation reused K times) with <i>fewer</i> effective parameters than a
+lookup table — so under heavy weight decay, the algorithm should beat memorization. If it can't
+happen here, in this maximally simple setting, that's evidence architectures need something
+fundamentally different to cross the gap.</p>
+
+<h2>What we did</h2>
+<ul>
+<li><b>Task:</b> n → next prime, n ∈ [2,100], 30% random holdout (seed 0), tokens = decimal digits + EOS.</li>
+<li><b>Arm A (main):</b> weight-tied RNN — ONE 2-layer cell reused for input read-in, K=20 compute
+steps, and output decoding; ACT learned halting gate with penalty ramp.</li>
+<li><b>Arm B (baseline):</b> GPT-style transformer, d_model=128, 2 layers, 4 heads. Fixed d_model;
+param counts logged, NOT matched (weight sharing is the studied variable).</li>
+<li><b>Pre-registered:</b> the full outcome→interpretation matrix (codes O1–O4, H1–H4, P1–P4) was
+committed to the research repo BEFORE any runs. Lock chain:
+<code>00c696d → b3fe898 → 3fe3935 → 3015189</code>.</li>
+<li><b>Multi-model review:</b> Gemini 3.6 Flash design review (removed the un-tied GRU decoder that
+would have masked the weight-tying claim; added λ ramp) and an OpenAI Codex hostile code review
+that caught two BLOCKERs before launch (batch-layout dependence of representations; misclassified
+prereg codes) plus the off-by-one in the halting floor and a probe-diagnostic semantics error
+(errors are flagged by <i>prediction</i>, not input — set extended to include 209=11·19).</li>
+<li><b>Verification:</b> 34 tests green, including a classifier regression suite with ground-truth
+stub models (a pure {{2,3,5,7}} sieve must classify P1 — it does).</li>
+</ul>
+
+<h2>Results</h2>
+{rnn_sec}
+{tf_sec}
+
+<h2>Interpretation (locked before the runs)</h2>
+<p>Every code below has a pre-registered meaning; the report only matches measurements against it.</p>
+<table><tr><th>Code</th><th>Meaning (from preregistration.md)</th></tr>
+{"".join(f"<tr><td><b>{k}</b></td><td>{v}</td></tr>" for k, v in INTERP.items())}
+</table>
+<blockquote><b>Seed-0 caveat:</b> everything here is one seed. Seeds {{1,2}} are required before any
+claim. Val EM on best.pt is selection-holed (early stopping uses it); probe + halting use the
+unselected last checkpoint. Training-range caveat: for n ≤ 100 the sieve only needs divisors
+{{2,3,5,7}} — in-range generalization does not imply the general algorithm.</blockquote>
+
+<h2>Next steps (jayrup's call)</h2>
+<ul>
+<li>Seeds 1, 2 (required before any claim)</li>
+<li>wd sweep {{0.01, 0.1, 0.3, 1.0, 3.0}}</li>
+<li>Ablations: halting=False (fixed-K), vocab_mode=integers, range [2,200]/[2,1000]</li>
+<li>If O2: smaller train fractions (40%/50%) to sharpen the memorization attractor</li>
+</ul>
+
+<h2>Environment</h2>
+<table><tr><th>rnn</th><th></th></tr>{meta_rnn}
+<tr><th>transformer</th><th></th></tr>{meta_tf}</table>
+
+<h2>Reproduce</h2>
+<pre>git clone ssh://meru/~/projects/prime-grokking.git && cd prime-grokking
+python3 -m venv .venv
+.venv/bin/pip install torch --index-url https://download.pytorch.org/whl/cpu
+.venv/bin/pip install numpy matplotlib pytest
+.venv/bin/pytest tests/ -q
+scripts/run_experiment.sh rnn 0
+scripts/run_experiment.sh transformer 0
+.venv/bin/python -m src.eval rnn 0; .venv/bin/python -m src.eval transformer 0
+.venv/bin/python -m scripts.plot rnn 0; .venv/bin/python -m scripts.plot transformer 0</pre>
+
+<p class="dim">Generated by scripts/build_report.py · design docs: design/experiment-spec.md,
+design/preregistration.md, design/reviews/ · honest-limitations policy: results are compared
+against the pre-registered matrix verbatim; nothing is interpreted post-hoc.</p>
+</body></html>"""
+
+    os.makedirs(os.path.dirname(a.out), exist_ok=True)
+    with open(a.out, "w") as fh:
+        fh.write(html)
+    print(f"saved {a.out} ({len(html)//1024} KB)")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/plot.py b/scripts/plot.py
index 14536e5..6a71c3b 100644
--- a/scripts/plot.py
+++ b/scripts/plot.py
@@ -1,4 +1,5 @@
-"""Plot train/val curves from a metrics.csv. Usage: python -m scripts.plot <model> <seed>"""
+"""Plot train/val curves from a metrics.csv. Usage: python -m scripts.plot <model> <seed> [--runs-dir DIR]"""
+import argparse
 import csv
 import sys
 
@@ -8,8 +9,14 @@ import matplotlib.pyplot as plt
 
 
 def main() -> None:
-    model, seed = sys.argv[1], sys.argv[2]
-    path = f"runs/{model}/seed{seed}/metrics.csv"
+    ap = argparse.ArgumentParser()
+    ap.add_argument("model")
+    ap.add_argument("seed")
+    ap.add_argument("--runs-dir", default="runs")
+    a = ap.parse_args()
+    model, seed = a.model, a.seed
+    path = f"{a.runs_dir}/{model}/seed{seed}/metrics.csv"
+    out = f"{a.runs_dir}/{model}/seed{seed}/curves.png"
     with open(path) as fh:
         rows = list(csv.DictReader(fh))
     steps = [int(r["step"]) for r in rows]
@@ -40,7 +47,6 @@ def main() -> None:
     axes[1, 1].set_xlabel("step")
     fig.suptitle(f"prime-grokking — {model} seed {seed}")
     plt.tight_layout()
-    out = f"runs/{model}/seed{seed}/curves.png"
     plt.savefig(out, dpi=110)
     print(f"saved {out}")
 
diff --git a/src/eval.py b/src/eval.py
index 0e769ac..7dcd289 100644
--- a/src/eval.py
+++ b/src/eval.py
@@ -132,7 +132,8 @@ def halting_report(model, cfg: Config) -> dict:
     else:
         code = "H4"          # intermediate but no positive gap correlation — noisy/unused
     return {"code": code, "mean_steps": mean, "corr_gap": rho,
-            "min_steps": cfg.min_steps, "max_steps": cfg.max_steps}
+            "min_steps": cfg.min_steps, "max_steps": cfg.max_steps,
+            "gaps": gaps, "steps_by_n": steps}
 
 
 def _load(out_dir: str, ckpt: str, cfg: Config):
@@ -146,8 +147,10 @@ def main() -> None:
     ap = argparse.ArgumentParser(description="prime-grokking eval")
     ap.add_argument("model")
     ap.add_argument("seed")
+    ap.add_argument("--ckpt", default="last.pt")
+    ap.add_argument("--runs-dir", default="runs")
     a = ap.parse_args()
-    out_dir = os.path.join("runs", a.model, f"seed{a.seed}")
+    out_dir = os.path.join(a.runs_dir, a.model, f"seed{a.seed}")
     cfg = Config.load(os.path.join(out_dir, "config.json"))
 
     _, val_in = get_splits(cfg)