From 535bb1dcf727e2273b956553c0491ec80ad8ec98 Mon Sep 17 00:00:00 2001 From: Void Agent Date: Mon, 17 Aug 2026 15:26:16 +0100 Subject: fix D2: integers-mode probe is N/A by construction (OOV probe inputs), functional regression test; D2 scored (int tokens collapse task: O2, 0-13% vs digits 17-87%); NOTES + report Phase 4 updated; 42 tests --- tests/test_phase3_flags.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) (limited to 'tests') diff --git a/tests/test_phase3_flags.py b/tests/test_phase3_flags.py index 4ba8be1..063822e 100644 --- a/tests/test_phase3_flags.py +++ b/tests/test_phase3_flags.py @@ -57,6 +57,52 @@ def test_train_frac_subsamples_train_only(): assert set(tr) <= set(tr_full) +def test_integers_mode_eval_does_not_crash_probe(tmp_path): + """Regression: integers-mode eval crashed in the probe (out-of-vocab inputs >= 104). + Train a tiny integers model, then run the REAL eval main against it — must not crash, + and the probe must be N/A (Addendum 5: D2 is scored on in-range val EM only).""" + import json + import os + import subprocess + import sys + import torch + from src.data import build_examples, get_splits, make_batch + from src.model_api import build_model + + cfg = Config(vocab_mode="integers") + m = build_model(cfg) + train_in, val_in = get_splits(cfg) + train_ex = build_examples(train_in, cfg) + opt = torch.optim.AdamW(m.parameters(), lr=1e-3) + for _ in range(10): + ex = train_ex[:16] + b = make_batch(ex, cfg) + out = m(b["x"], b["y_in"]) + loss = torch.nn.functional.cross_entropy( + out["logits"].reshape(-1, cfg.vocab), b["y"].clamp(max=cfg.vocab - 1).reshape(-1), reduction="none") + mask = b["y_mask"].float().reshape(-1) + loss = (loss * mask).sum() / mask.sum() + opt.zero_grad(); loss.backward(); opt.step() + + od = tmp_path / "rnn" / "seed0" + os.makedirs(od, exist_ok=True) + cfg.save(str(od / "config.json")) + torch.save(m.state_dict(), str(od / "last.pt")) + torch.save(m.state_dict(), str(od / "best.pt")) + import csv + with open(str(od / "metrics.csv"), "w", newline="") as fh: + w = csv.writer(fh) + w.writerow(["step", "train_loss", "train_token_acc", "train_em", "val_token_acc", + "val_em", "mean_halt_steps", "log_examples", "param_count"]) + w.writerow([200, 0.1, 1.0, 1.0, 1.0, 0.5, 3.0, "", 1000]) + + r = subprocess.run([sys.executable, "-m", "src.eval", "rnn", "0", "--runs-dir", str(tmp_path)], + capture_output=True, text=True, cwd=os.path.join(os.path.dirname(__file__), "..")) + assert r.returncode == 0, r.stderr[-600:] + res = json.load(open(str(od / "results.json"))) + assert res["final"]["probe"]["code"] == "N/A" + + def test_new_flags_parse(): a = parse_args(["rnn", "0", "--task_mode", "is_prime", "--train_frac", "0.4", "--lr_decay", "True"]) assert a.task_mode == "is_prime" and a.train_frac == 0.4 and a.lr_decay is True -- cgit v1.2.3