From 39b8218a5fe660200a833c03b36afbde6c119467 Mon Sep 17 00:00:00 2001 From: Void Agent Date: Fri, 14 Aug 2026 13:27:54 +0100 Subject: eval: P1 diagnostic prediction-based (flagged set +209), classifier regression suite (10 tests, stub ground truth); 34 tests green --- src/eval.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/eval.py b/src/eval.py index fdc2dec..0e769ac 100644 --- a/src/eval.py +++ b/src/eval.py @@ -18,7 +18,11 @@ from src.data import build_examples, decode_tokens, encode_int, get_splits, next from src.model_api import build_model, greedy_decode from src.train import evaluate -FLAGGED_COMPOSITES = {121, 143, 169, 187} # need divisors 11, 13 — beyond the {2,3,5,7} sieve +# Composites with no prime factor <= 7 inside the probe's candidate window [102, 211]. +# A model that learned only the {2,3,5,7} sieve predicts THESE as "next primes" (errors on +# n = 113..120, 139..142, 167..168, 181..186, 199..200 — 22 errors total). Includes 209 = 11*19, +# which the original {121,143,169,187} set (composites <= 200) missed — see Addendum 3. +FLAGGED_SIEVE_PREDS = {121, 143, 169, 187, 209} def probe_report(model, cfg: Config, lo: int = 101, hi: int = 200) -> dict: @@ -43,19 +47,22 @@ def probe_report(model, cfg: Config, lo: int = 101, hi: int = 200) -> dict: easy_wrong += 1 total = hi - lo + 1 acc = correct / total - flagged = [e for e in errors if e["n"] in FLAGGED_COMPOSITES] - # classification per prereg + Addendum 2 operationalization; P4 checked first + flagged = [e for e in errors if e["pred"] in FLAGGED_SIEVE_PREDS] + distinct_flagged = len({e["pred"] for e in flagged}) + flagged_frac = len(flagged) / len(errors) if errors else 0.0 + # classification per prereg + Addendum 3 operationalization; P4 checked first if easy_total and easy_wrong / easy_total > 0.5: code = "P4" # fails trivial evens/5-multiples -> pure memorization elif acc >= 0.85: code = "P3" # surprising success beyond expectation - elif len(flagged) >= 3 and len(errors) <= 6 and all(e["n"] in FLAGGED_COMPOSITES for e in errors): - code = "P1" # errors concentrated on composites needing divisors 11,13 -> learned sieve + elif len(errors) >= 3 and distinct_flagged >= 3 and flagged_frac >= 0.8: + code = "P1" # errors = sieves predicting no-small-factor composites -> learned {2,3,5,7} sieve else: code = "P2" # scattered errors -> memorization / non-transferable heuristics return { "code": code, "acc": acc, "correct": correct, "total": total, "errors": errors, "flagged_errors": flagged, + "flagged_pred_fraction": flagged_frac, "distinct_flagged_preds": distinct_flagged, "easy_total": easy_total, "easy_wrong": easy_wrong, "easy_err_rate": (easy_wrong / easy_total) if easy_total else None, } -- cgit v1.2.3