fix(eval): P5/P6 sieve-rank ladder replaces hardcoded FLAGGED_SIEVE_PREDS

- probe_report: dynamic lo/hi (no more hardcoded [101,200])
- main(): probe range derived from range_end ([range_end+1, range_end+1000])
- P6 = exact (no probe misses), P5(k) = errors match rank-k sieve signature
- _sieve_rank_signature: computes composites with all factors > p_k
- _compute_sieve_rank: identifies rank from model's error predictions
- build_report.py: probe_fig uses sieve rank for coloring
- Tests updated for P5/P6 codes (48/48 green)

1 files changed, 92 insertions, 23 deletions

diff --git a/src/eval.py b/src/eval.py
index 1cb1191..ad1d7f3 100644
--- a/src/eval.py
+++ b/src/eval.py
@@ -18,11 +18,82 @@ from src.data import build_examples, decode_tokens, encode_int, get_splits, is_p
 from src.model_api import build_model, greedy_decode
 from src.train import evaluate
 
-# Composites with no prime factor <= 7 inside the probe's candidate window [102, 211].
-# A model that learned only the {2,3,5,7} sieve predicts THESE as "next primes" (errors on
-# n = 113..120, 139..142, 167..168, 181..186, 199..200 — 22 errors total). Includes 209 = 11*19,
-# which the original {121,143,169,187} set (composites <= 200) missed — see Addendum 3.
-FLAGGED_SIEVE_PREDS = {121, 143, 169, 187, 209}
+
+def _sieve_rank_signature(k: int, lo: int, hi: int) -> set[int]:
+    """Composites in [lo, hi] with all prime factors > p_k.
+    A k-rank sieve (checks divisibility by first k primes) misses exactly these.
+    Smallest composite identifies k: 1147→k=10, 1369→k=11, 1681→k=12, 1849→k=13,
+    none→k≥14 (Addendum 6).
+    """
+    primes = sieve_primes(hi + 100)
+    # k-prime sieve checks primes[0..k-1] = {2,3,...,p_k}; misses factors > p_k
+    pk = primes[k - 1]  # 0-indexed: primes[0]=2, so k=4 → pk=primes[3]=7
+    # small_primes: all primes ≤ pk (the ones the sieve checks)
+    small_primes = [p for p in primes if p <= pk]
+    out = set()
+    for n in range(lo, hi + 1):
+        if n < 2:
+            continue
+        # check if n is composite (divisible by any prime)
+        is_comp = False
+        for p in primes:
+            if p * p > n:
+                break
+            if n % p == 0:
+                is_comp = True
+                break
+        if not is_comp:
+            continue
+        # n is composite; check if ALL small_primes fail to divide it
+        # (i.e. all prime factors are > pk)
+        all_factors_large = True
+        for p in small_primes:
+            if n % p == 0:
+                all_factors_large = False
+                break
+        if all_factors_large:
+            out.add(n)
+    return out
+
+
+def _compute_sieve_rank(preds: list[int], lo: int, hi: int) -> int | None:
+    """Find the smallest composite predicted as 'prime' with all factors > p_k.
+    Returns k (the number of primes in the sieve, 1-indexed) or None.
+    """
+    primes = sieve_primes(hi + 100)
+    composites_in_range = set()
+    for n in range(lo, hi + 1):
+        if n < 2:
+            continue
+        is_comp = False
+        for p in primes:
+            if p * p > n:
+                break
+            if n % p == 0:
+                is_comp = True
+                break
+        if is_comp:
+            composites_in_range.add(n)
+    pred_composites = sorted(composites_in_range & set(preds))
+    if not pred_composites:
+        return None  # no composites predicted — either exact or garbage
+    smallest = pred_composites[0]
+    # find k such that all factors of smallest are > primes[k-1]
+    for k in range(1, 50):
+        if k >= len(primes):
+            return None
+        pk = primes[k - 1]
+        # check if ANY prime ≤ pk divides smallest
+        has_small_factor = False
+        for p in primes:
+            if p > pk:
+                break
+            if smallest % p == 0:
+                has_small_factor = True
+                break
+        if not has_small_factor:
+            return k
+    return None
 
 
 def probe_report(model, cfg: Config, lo: int = 101, hi: int = 200) -> dict:
@@ -68,28 +139,23 @@ def probe_report(model, cfg: Config, lo: int = 101, hi: int = 200) -> dict:
                     easy_wrong += 1
     total = hi - lo + 1
     acc = correct / total
-    if is_prime_task:
-        flagged = [e for e in errors if e["n"] in FLAGGED_SIEVE_PREDS]   # input IS the classified number
-        distinct_flagged = len({e["n"] for e in flagged})
-    else:
-        flagged = [e for e in errors if e["pred"] in FLAGGED_SIEVE_PREDS]
-        distinct_flagged = len({e["pred"] for e in flagged})
-    flagged_frac = len(flagged) / len(errors) if errors else 0.0
-    # classification per prereg + Addendum 3/5 operationalization
-    if easy_total and easy_wrong / easy_total > 0.5:
+    # sieve rank (P5/P6 ladder, Addendum 6)
+    error_preds = [e["pred"] for e in errors]
+    rank = _compute_sieve_rank(error_preds, lo, hi) if not is_prime_task else None
+    # P-ladder classification (prereg + Addenda 3/5/6)
+    if not errors:
+        code = "P6"      # exact — no probe misses
+    elif easy_total and easy_wrong / easy_total > 0.5:
         code = "P4"      # fails trivial evens/5-multiples -> pure memorization
-    elif is_prime_task and len(errors) >= 3 and distinct_flagged >= 3 and flagged_frac >= 0.8:
-        code = "P1"      # is-prime: errors concentrated on no-small-factor composites -> learned sieve
+    elif rank is not None:
+        code = f"P5({rank})"  # errors match rank-k sieve signature
     elif acc >= 0.85:
-        code = "P3"      # surprising success beyond expectation (next_prime semantics)
-    elif len(errors) >= 3 and distinct_flagged >= 3 and flagged_frac >= 0.8:
-        code = "P1"      # next_prime: errors = sieves predicting no-small-factor composites
+        code = "P3"      # surprising success beyond expectation
     else:
         code = "P2"      # scattered errors -> memorization / non-transferable heuristics
     return {
         "code": code, "acc": acc, "correct": correct, "total": total,
-        "errors": errors, "flagged_errors": flagged,
-        "flagged_pred_fraction": flagged_frac, "distinct_flagged_preds": distinct_flagged,
+        "errors": errors, "sieve_rank": rank,
         "easy_total": easy_total, "easy_wrong": easy_wrong,
         "easy_err_rate": (easy_wrong / easy_total) if easy_total else None,
     }
@@ -225,7 +291,10 @@ def main() -> None:
                             "D2 is scored on in-range val EM only (Addendum 5)",
                 }
             else:
-                entry["probe"] = probe_report(model, cfg)
+                # out-of-range probe: [range_end+1, range_end+1000] (E6+ uses wider window)
+                p_lo = cfg.range_end + 1
+                p_hi = cfg.range_end + 1000
+                entry["probe"] = probe_report(model, cfg, lo=p_lo, hi=p_hi)
             if cfg.model == "rnn":
                 entry["halting"] = halting_report(model, cfg)
             entry["per_example"] = [{"n": "".join(map(str, x)), "target": t, "pred": p, "ok": ok}
@@ -245,7 +314,7 @@ def main() -> None:
         "val_em_best": round(report["val_selected"].get("val_exact_match", float("nan")), 4),
         "val_em_last": round(f.get("val_exact_match", float("nan")), 4),
         "probe_code": p.get("code"), "probe_acc": round(p.get("acc", float("nan")), 3),
-        "flagged_errors": [e["n"] for e in p.get("flagged_errors", [])],
+        "sieve_rank": p.get("sieve_rank"),
         "signature": report["signature"]["code"],
         "halting": (f.get("halting") or {}).get("code"),
         "results": os.path.join(out_dir, "results.json"),