From 535bb1dcf727e2273b956553c0491ec80ad8ec98 Mon Sep 17 00:00:00 2001 From: Void Agent Date: Mon, 17 Aug 2026 15:26:16 +0100 Subject: fix D2: integers-mode probe is N/A by construction (OOV probe inputs), functional regression test; D2 scored (int tokens collapse task: O2, 0-13% vs digits 17-87%); NOTES + report Phase 4 updated; 42 tests --- src/eval.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'src/eval.py') diff --git a/src/eval.py b/src/eval.py index c964d38..e93f95d 100644 --- a/src/eval.py +++ b/src/eval.py @@ -184,7 +184,14 @@ def main() -> None: else "final checkpoint, no val selection"), } if ckpt == "last.pt": - entry["probe"] = probe_report(model, cfg) + if cfg.vocab_mode == "integers": + entry["probe"] = { + "code": "N/A", + "note": "atomic integer tokens make out-of-range inputs out-of-vocabulary by construction; " + "D2 is scored on in-range val EM only (Addendum 5)", + } + else: + entry["probe"] = probe_report(model, cfg) if cfg.model == "rnn": entry["halting"] = halting_report(model, cfg) entry["per_example"] = [{"n": "".join(map(str, x)), "target": t, "pred": p, "ok": ok} -- cgit v1.2.3