diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/eval.py | 9 |
1 files changed, 8 insertions, 1 deletions
diff --git a/src/eval.py b/src/eval.py index c964d38..e93f95d 100644 --- a/src/eval.py +++ b/src/eval.py @@ -184,7 +184,14 @@ def main() -> None: else "final checkpoint, no val selection"), } if ckpt == "last.pt": - entry["probe"] = probe_report(model, cfg) + if cfg.vocab_mode == "integers": + entry["probe"] = { + "code": "N/A", + "note": "atomic integer tokens make out-of-range inputs out-of-vocabulary by construction; " + "D2 is scored on in-range val EM only (Addendum 5)", + } + else: + entry["probe"] = probe_report(model, cfg) if cfg.model == "rnn": entry["halting"] = halting_report(model, cfg) entry["per_example"] = [{"n": "".join(map(str, x)), "target": t, "pred": p, "ok": ok} |