From 39b8218a5fe660200a833c03b36afbde6c119467 Mon Sep 17 00:00:00 2001 From: Void Agent Date: Fri, 14 Aug 2026 13:27:54 +0100 Subject: eval: P1 diagnostic prediction-based (flagged set +209), classifier regression suite (10 tests, stub ground truth); 34 tests green --- design/preregistration.md | 21 ++++++ src/eval.py | 17 +++-- tests/test_data.py | 7 +- tests/test_eval_classification.py | 135 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 173 insertions(+), 7 deletions(-) create mode 100644 tests/test_eval_classification.py diff --git a/design/preregistration.md b/design/preregistration.md index 30f731f..91e5036 100644 --- a/design/preregistration.md +++ b/design/preregistration.md @@ -144,3 +144,24 @@ pre-launch and pre-run. selection-holed) and `last.pt` (unselected); probe + halting analyses use `last.pt`. 6. **Dead config removed:** `halt_eps` (was never read). 7. **`run_meta.json`:** python/torch/numpy versions, device, thread count recorded per run. + +--- + +## Addendum 3 (2026-08-14, pre-launch — P1 diagnostic corrected) + +Trigger: ad-hoc verification of `src/eval.py` against ground-truth stub models (promoted to +`tests/test_eval_classification.py`). Pre-launch; P1 operationalization corrected before any +results were seen. + +1. **P1 concerns PREDICTIONS, not inputs.** A model that learned only the {2,3,5,7} sieve errs + on inputs n = 113–120, 139–142, 167–168, 181–186, 199–200 — cases where the first candidate + with no divisor ≤ 7 is composite. The signature is "predicted next prime" ∈ the flagged set, + NOT "input n" ∈ the flagged set. Addendum 2's phrasing ("≥3 of {121,143,169,187} wrong") was + ambiguous and the first implementation checked inputs — corrected. +2. **Set extended to {121, 143, 169, 187, 209}.** The original four are the no-small-factor + composites ≤ 200; the probe's candidate window is actually [102, 211] (targets of + n ∈ [101, 200] reach 211), and 209 = 11×19 is likewise mispredicted (n = 199, 200 → pred 209, + target 211). +3. **P1 thresholds (replace Addendum 2's):** ≥ 3 errors total, ≥ 3 DISTINCT flagged predictions, + ≥ 80% of error predictions flagged. A pure {2,3,5,7} sieve produces exactly 22 errors, 100% + flagged (5 distinct values). P2–P4 unchanged. diff --git a/src/eval.py b/src/eval.py index fdc2dec..0e769ac 100644 --- a/src/eval.py +++ b/src/eval.py @@ -18,7 +18,11 @@ from src.data import build_examples, decode_tokens, encode_int, get_splits, next from src.model_api import build_model, greedy_decode from src.train import evaluate -FLAGGED_COMPOSITES = {121, 143, 169, 187} # need divisors 11, 13 — beyond the {2,3,5,7} sieve +# Composites with no prime factor <= 7 inside the probe's candidate window [102, 211]. +# A model that learned only the {2,3,5,7} sieve predicts THESE as "next primes" (errors on +# n = 113..120, 139..142, 167..168, 181..186, 199..200 — 22 errors total). Includes 209 = 11*19, +# which the original {121,143,169,187} set (composites <= 200) missed — see Addendum 3. +FLAGGED_SIEVE_PREDS = {121, 143, 169, 187, 209} def probe_report(model, cfg: Config, lo: int = 101, hi: int = 200) -> dict: @@ -43,19 +47,22 @@ def probe_report(model, cfg: Config, lo: int = 101, hi: int = 200) -> dict: easy_wrong += 1 total = hi - lo + 1 acc = correct / total - flagged = [e for e in errors if e["n"] in FLAGGED_COMPOSITES] - # classification per prereg + Addendum 2 operationalization; P4 checked first + flagged = [e for e in errors if e["pred"] in FLAGGED_SIEVE_PREDS] + distinct_flagged = len({e["pred"] for e in flagged}) + flagged_frac = len(flagged) / len(errors) if errors else 0.0 + # classification per prereg + Addendum 3 operationalization; P4 checked first if easy_total and easy_wrong / easy_total > 0.5: code = "P4" # fails trivial evens/5-multiples -> pure memorization elif acc >= 0.85: code = "P3" # surprising success beyond expectation - elif len(flagged) >= 3 and len(errors) <= 6 and all(e["n"] in FLAGGED_COMPOSITES for e in errors): - code = "P1" # errors concentrated on composites needing divisors 11,13 -> learned sieve + elif len(errors) >= 3 and distinct_flagged >= 3 and flagged_frac >= 0.8: + code = "P1" # errors = sieves predicting no-small-factor composites -> learned {2,3,5,7} sieve else: code = "P2" # scattered errors -> memorization / non-transferable heuristics return { "code": code, "acc": acc, "correct": correct, "total": total, "errors": errors, "flagged_errors": flagged, + "flagged_pred_fraction": flagged_frac, "distinct_flagged_preds": distinct_flagged, "easy_total": easy_total, "easy_wrong": easy_wrong, "easy_err_rate": (easy_wrong / easy_total) if easy_total else None, } diff --git a/tests/test_data.py b/tests/test_data.py index cd4e620..9f651fb 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -75,7 +75,10 @@ def test_make_batch_shapes_and_padding(): def test_flagged_composites_are_composite(): - # sanity: the diagnostic set in eval.py really is composite and needs divisors > 7 - for n in (121, 143, 169, 187): + # sanity: the diagnostic set really is composite, has no divisors <= 7, and covers + # the probe's candidate window [102, 211] (209 = 11*19 included — Addendum 3) + from src.eval import FLAGGED_SIEVE_PREDS + assert FLAGGED_SIEVE_PREDS == {121, 143, 169, 187, 209} + for n in FLAGGED_SIEVE_PREDS: assert any(n % d == 0 for d in range(2, int(n ** 0.5) + 1)) assert all(n % d != 0 for d in (2, 3, 5, 7)) diff --git a/tests/test_eval_classification.py b/tests/test_eval_classification.py new file mode 100644 index 0000000..3f9994d --- /dev/null +++ b/tests/test_eval_classification.py @@ -0,0 +1,135 @@ +"""Classification correctness for src/eval.py — the preregistered O/H/P codes. + +Ground truth is synthetic: stub models with KNOWN behavior feed the classifier, +and synthetic metrics CSVs feed grokking_signature. (Original: ad-hoc verifier +/tmp/hermes-verify-eval.py, promoted to a permanent regression suite.) +""" +import csv +import os +import tempfile + +import torch + +from src.config import Config +from src.data import sieve_primes +from src.eval import FLAGGED_SIEVE_PREDS, grokking_signature, halting_report, probe_report + +DIGITS_CFG = Config(vocab_mode="digits") +EOS = DIGITS_CFG.eos_id +PAD = DIGITS_CFG.pad_id + + +def _decode_row(row) -> int: + toks = [int(t) for t in row.tolist() if int(t) != PAD] + return int("".join(map(str, toks))) if toks else -1 + + +class StubModel(torch.nn.Module): + """One-hot logits forcing greedy_decode to emit exactly pred_fn(n).""" + + def __init__(self, pred_fn): + super().__init__() + self.pred_fn = pred_fn + + def forward(self, x, y_in): + B, T_out = x.shape[0], y_in.shape[1] + logits = torch.full((B, T_out, 11), -100.0) + for i in range(B): + n = _decode_row(x[i]) + toks = [int(d) for d in str(self.pred_fn(n))] + [EOS] + for t in range(T_out): + tok = toks[t] if t < len(toks) else EOS + logits[i, t, tok] = 100.0 + return {"logits": logits, "halt_steps": None} + + +def _sieve35(n): + c = n + 1 + while any(c % d == 0 for d in (2, 3, 5, 7)): + c += 1 + return c + + +def _perfect(n): + return next(p for p in sieve_primes(500) if p > n) + + +def _easy_only(n): + return _perfect(n) if (n % 2 == 0 or n % 5 == 0) else 199 + + +def test_probe_sieve35_classified_p1(): + """A pure {2,3,5,7} sieve must classify P1: errors are predictions of 121/143/169/187/209.""" + r = probe_report(StubModel(_sieve35), DIGITS_CFG) + preds = sorted({e["pred"] for e in r["errors"]}) + assert preds == sorted(FLAGGED_SIEVE_PREDS), r["errors"] + assert len(r["errors"]) == 22 # n in 113..120, 139..142, 167..168, 181..186, 199..200 + assert r["code"] == "P1", r + + +def test_probe_perfect_classified_p3(): + r = probe_report(StubModel(_perfect), DIGITS_CFG) + assert r["acc"] == 1.0 and r["code"] == "P3", r + + +def test_probe_easy_only_classified_p2(): + r = probe_report(StubModel(_easy_only), DIGITS_CFG) + assert r["code"] == "P2", r + + +def test_probe_always_wrong_classified_p4(): + r = probe_report(StubModel(lambda n: n), DIGITS_CFG) + assert r["code"] == "P4", r + + +def _write_csv(path, train_ems, val_ems): + with open(path, "w", newline="") as fh: + w = csv.writer(fh) + w.writerow(["step", "train_loss", "train_token_acc", "train_em", "val_token_acc", + "val_em", "mean_halt_steps", "log_examples", "param_count"]) + for i, (t, v) in enumerate(zip(train_ems, val_ems)): + w.writerow([i * 200, 0.1, 1.0, t, 1.0, v, 10.0, "", 1000]) + + +def _sig_of(tmp, tr, va): + p = os.path.join(tmp, "m.csv") + _write_csv(p, tr, va) + return grokking_signature(p) + + +def test_signature_o1_sharp_transition(tmp_path): + tr = [0.99] * 20 + va = [0.1] * 15 + [0.2, 0.95, 0.99, 1.0, 1.0] + r = _sig_of(str(tmp_path), tr, va) + assert r["code"] == "O1", r + + +def test_signature_o2_memorization(tmp_path): + assert _sig_of(str(tmp_path), [1.0] * 20, [0.1] * 20)["code"] == "O2" + + +def test_signature_o3_gradual(tmp_path): + tr = [1.0] * 20 + va = [0.3, 0.45, 0.6, 0.75, 0.9, 0.95] + [1.0] * 14 + assert _sig_of(str(tmp_path), tr, va)["code"] == "O3" + + +def test_signature_o4_train_fails(tmp_path): + assert _sig_of(str(tmp_path), [0.6] * 20, [0.2] * 20)["code"] == "O4" + + +def test_signature_opartial(tmp_path): + assert _sig_of(str(tmp_path), [1.0] * 20, [0.6] * 20)["code"] == "O-PARTIAL" + + +def test_halting_h1_collapse_and_h2_pin(): + from src.model_api import build_model + cfg = Config(model="rnn") + m = build_model(cfg) + with torch.no_grad(): + m.halt_head.bias.fill_(50.0) + assert halting_report(m, cfg)["code"] == "H1" + m2 = build_model(cfg) + with torch.no_grad(): + m2.halt_head.bias.fill_(-50.0) + assert halting_report(m2, cfg)["code"] == "H2" -- cgit v1.2.3