eval: P1 diagnostic prediction-based (flagged set +209), classifier regression suite (10 tests, stub ground truth); 34 tests green

1 files changed, 12 insertions, 5 deletions

diff --git a/src/eval.py b/src/eval.py
index fdc2dec..0e769ac 100644
--- a/src/eval.py
+++ b/src/eval.py
@@ -18,7 +18,11 @@ from src.data import build_examples, decode_tokens, encode_int, get_splits, next
 from src.model_api import build_model, greedy_decode
 from src.train import evaluate
 
-FLAGGED_COMPOSITES = {121, 143, 169, 187}   # need divisors 11, 13 — beyond the {2,3,5,7} sieve
+# Composites with no prime factor <= 7 inside the probe's candidate window [102, 211].
+# A model that learned only the {2,3,5,7} sieve predicts THESE as "next primes" (errors on
+# n = 113..120, 139..142, 167..168, 181..186, 199..200 — 22 errors total). Includes 209 = 11*19,
+# which the original {121,143,169,187} set (composites <= 200) missed — see Addendum 3.
+FLAGGED_SIEVE_PREDS = {121, 143, 169, 187, 209}
 
 
 def probe_report(model, cfg: Config, lo: int = 101, hi: int = 200) -> dict:
@@ -43,19 +47,22 @@ def probe_report(model, cfg: Config, lo: int = 101, hi: int = 200) -> dict:
                 easy_wrong += 1
     total = hi - lo + 1
     acc = correct / total
-    flagged = [e for e in errors if e["n"] in FLAGGED_COMPOSITES]
-    # classification per prereg + Addendum 2 operationalization; P4 checked first
+    flagged = [e for e in errors if e["pred"] in FLAGGED_SIEVE_PREDS]
+    distinct_flagged = len({e["pred"] for e in flagged})
+    flagged_frac = len(flagged) / len(errors) if errors else 0.0
+    # classification per prereg + Addendum 3 operationalization; P4 checked first
     if easy_total and easy_wrong / easy_total > 0.5:
         code = "P4"      # fails trivial evens/5-multiples -> pure memorization
     elif acc >= 0.85:
         code = "P3"      # surprising success beyond expectation
-    elif len(flagged) >= 3 and len(errors) <= 6 and all(e["n"] in FLAGGED_COMPOSITES for e in errors):
-        code = "P1"      # errors concentrated on composites needing divisors 11,13 -> learned sieve
+    elif len(errors) >= 3 and distinct_flagged >= 3 and flagged_frac >= 0.8:
+        code = "P1"      # errors = sieves predicting no-small-factor composites -> learned {2,3,5,7} sieve
     else:
         code = "P2"      # scattered errors -> memorization / non-transferable heuristics
     return {
         "code": code, "acc": acc, "correct": correct, "total": total,
         "errors": errors, "flagged_errors": flagged,
+        "flagged_pred_fraction": flagged_frac, "distinct_flagged_preds": distinct_flagged,
         "easy_total": easy_total, "easy_wrong": easy_wrong,
         "easy_err_rate": (easy_wrong / easy_total) if easy_total else None,
     }