fix D2: integers-mode probe is N/A by construction (OOV probe inputs), functional regression test; D2 scored (int tokens collapse task: O2, 0-13% vs digits 17-87%); NOTES + report Phase 4 updated; 42 tests

1 files changed, 46 insertions, 0 deletions

diff --git a/tests/test_phase3_flags.py b/tests/test_phase3_flags.py
index 4ba8be1..063822e 100644
--- a/tests/test_phase3_flags.py
+++ b/tests/test_phase3_flags.py
@@ -57,6 +57,52 @@ def test_train_frac_subsamples_train_only():
     assert set(tr) <= set(tr_full)
 
 
+def test_integers_mode_eval_does_not_crash_probe(tmp_path):
+    """Regression: integers-mode eval crashed in the probe (out-of-vocab inputs >= 104).
+    Train a tiny integers model, then run the REAL eval main against it — must not crash,
+    and the probe must be N/A (Addendum 5: D2 is scored on in-range val EM only)."""
+    import json
+    import os
+    import subprocess
+    import sys
+    import torch
+    from src.data import build_examples, get_splits, make_batch
+    from src.model_api import build_model
+
+    cfg = Config(vocab_mode="integers")
+    m = build_model(cfg)
+    train_in, val_in = get_splits(cfg)
+    train_ex = build_examples(train_in, cfg)
+    opt = torch.optim.AdamW(m.parameters(), lr=1e-3)
+    for _ in range(10):
+        ex = train_ex[:16]
+        b = make_batch(ex, cfg)
+        out = m(b["x"], b["y_in"])
+        loss = torch.nn.functional.cross_entropy(
+            out["logits"].reshape(-1, cfg.vocab), b["y"].clamp(max=cfg.vocab - 1).reshape(-1), reduction="none")
+        mask = b["y_mask"].float().reshape(-1)
+        loss = (loss * mask).sum() / mask.sum()
+        opt.zero_grad(); loss.backward(); opt.step()
+
+    od = tmp_path / "rnn" / "seed0"
+    os.makedirs(od, exist_ok=True)
+    cfg.save(str(od / "config.json"))
+    torch.save(m.state_dict(), str(od / "last.pt"))
+    torch.save(m.state_dict(), str(od / "best.pt"))
+    import csv
+    with open(str(od / "metrics.csv"), "w", newline="") as fh:
+        w = csv.writer(fh)
+        w.writerow(["step", "train_loss", "train_token_acc", "train_em", "val_token_acc",
+                    "val_em", "mean_halt_steps", "log_examples", "param_count"])
+        w.writerow([200, 0.1, 1.0, 1.0, 1.0, 0.5, 3.0, "", 1000])
+
+    r = subprocess.run([sys.executable, "-m", "src.eval", "rnn", "0", "--runs-dir", str(tmp_path)],
+                       capture_output=True, text=True, cwd=os.path.join(os.path.dirname(__file__), ".."))
+    assert r.returncode == 0, r.stderr[-600:]
+    res = json.load(open(str(od / "results.json")))
+    assert res["final"]["probe"]["code"] == "N/A"
+
+
 def test_new_flags_parse():
     a = parse_args(["rnn", "0", "--task_mode", "is_prime", "--train_frac", "0.4", "--lr_decay", "True"])
     assert a.task_mode == "is_prime" and a.train_frac == 0.4 and a.lr_decay is True