summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--scripts/build_report.py21
-rw-r--r--src/eval.py115
-rw-r--r--tests/test_data.py11
-rw-r--r--tests/test_eval_classification.py16
4 files changed, 122 insertions, 41 deletions
diff --git a/scripts/build_report.py b/scripts/build_report.py
index 4b74686..90849a2 100644
--- a/scripts/build_report.py
+++ b/scripts/build_report.py
@@ -17,7 +17,7 @@ import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
-from src.eval import FLAGGED_SIEVE_PREDS
+from src.eval import _sieve_rank_signature
CODE_LEGEND = {
"O1": "Sharp transition — grokking-like",
@@ -63,20 +63,29 @@ def img_b64(path: str) -> str:
def probe_fig(result: dict, model: str) -> str:
errors = result.get("errors", [])
- flagged = [e for e in errors if e["pred"] in FLAGGED_SIEVE_PREDS]
+ if not errors:
+ return ""
+ probe = result.get("probe", {})
+ rank = probe.get("sieve_rank")
+ lo = min(e["n"] for e in errors) - 5 if errors else 100
+ hi = max(e["n"] for e in errors) + 5 if errors else 200
+ flagged = set()
+ if rank is not None:
+ flagged = _sieve_rank_signature(rank, lo, hi)
fig, ax = plt.subplots(figsize=(10, 3.2))
xs, ys, cs, ls = [], [], [], []
for e in errors:
xs.append(e["n"]); ys.append(1.0)
- cs.append("tab:red" if e["pred"] in FLAGGED_SIEVE_PREDS else "tab:blue")
+ cs.append("tab:red" if e["pred"] in flagged else "tab:blue")
ls.append(f"n={e['n']}\npred {e['pred']}\ntarget {e['target']}")
ax.scatter(xs, ys, c=cs, s=28, zorder=3)
- ax.set_xlim(100, 201)
+ ax.set_xlim(lo - 1, hi + 1)
ax.set_ylim(0.6, 1.4)
ax.set_yticks([1.0]); ax.set_yticklabels(["wrong"])
ax.set_xlabel("input n")
- ax.set_title(f"{model}: probe errors (red = predicted a no-small-divisor composite "
- f"{sorted(FLAGGED_SIEVE_PREDS)})")
+ rank_str = f" rank-{rank}" if rank is not None else ""
+ ax.set_title(f"{model}: probe errors (red = sieve{rank_str} signature "
+ f"{sorted(flagged) if flagged else 'none'})")
for x, y, l in zip(xs, ys, ls):
ax.annotate(l, (x, y), textcoords="offset points", xytext=(0, 10), fontsize=6,
ha="center", rotation=90, va="bottom")
diff --git a/src/eval.py b/src/eval.py
index 1cb1191..ad1d7f3 100644
--- a/src/eval.py
+++ b/src/eval.py
@@ -18,11 +18,82 @@ from src.data import build_examples, decode_tokens, encode_int, get_splits, is_p
from src.model_api import build_model, greedy_decode
from src.train import evaluate
-# Composites with no prime factor <= 7 inside the probe's candidate window [102, 211].
-# A model that learned only the {2,3,5,7} sieve predicts THESE as "next primes" (errors on
-# n = 113..120, 139..142, 167..168, 181..186, 199..200 — 22 errors total). Includes 209 = 11*19,
-# which the original {121,143,169,187} set (composites <= 200) missed — see Addendum 3.
-FLAGGED_SIEVE_PREDS = {121, 143, 169, 187, 209}
+
+def _sieve_rank_signature(k: int, lo: int, hi: int) -> set[int]:
+ """Composites in [lo, hi] with all prime factors > p_k.
+ A k-rank sieve (checks divisibility by first k primes) misses exactly these.
+ Smallest composite identifies k: 1147→k=10, 1369→k=11, 1681→k=12, 1849→k=13,
+ none→k≥14 (Addendum 6).
+ """
+ primes = sieve_primes(hi + 100)
+ # k-prime sieve checks primes[0..k-1] = {2,3,...,p_k}; misses factors > p_k
+ pk = primes[k - 1] # 0-indexed: primes[0]=2, so k=4 → pk=primes[3]=7
+ # small_primes: all primes ≤ pk (the ones the sieve checks)
+ small_primes = [p for p in primes if p <= pk]
+ out = set()
+ for n in range(lo, hi + 1):
+ if n < 2:
+ continue
+ # check if n is composite (divisible by any prime)
+ is_comp = False
+ for p in primes:
+ if p * p > n:
+ break
+ if n % p == 0:
+ is_comp = True
+ break
+ if not is_comp:
+ continue
+ # n is composite; check if ALL small_primes fail to divide it
+ # (i.e. all prime factors are > pk)
+ all_factors_large = True
+ for p in small_primes:
+ if n % p == 0:
+ all_factors_large = False
+ break
+ if all_factors_large:
+ out.add(n)
+ return out
+
+
+def _compute_sieve_rank(preds: list[int], lo: int, hi: int) -> int | None:
+ """Find the smallest composite predicted as 'prime' with all factors > p_k.
+ Returns k (the number of primes in the sieve, 1-indexed) or None.
+ """
+ primes = sieve_primes(hi + 100)
+ composites_in_range = set()
+ for n in range(lo, hi + 1):
+ if n < 2:
+ continue
+ is_comp = False
+ for p in primes:
+ if p * p > n:
+ break
+ if n % p == 0:
+ is_comp = True
+ break
+ if is_comp:
+ composites_in_range.add(n)
+ pred_composites = sorted(composites_in_range & set(preds))
+ if not pred_composites:
+ return None # no composites predicted — either exact or garbage
+ smallest = pred_composites[0]
+ # find k such that all factors of smallest are > primes[k-1]
+ for k in range(1, 50):
+ if k >= len(primes):
+ return None
+ pk = primes[k - 1]
+ # check if ANY prime ≤ pk divides smallest
+ has_small_factor = False
+ for p in primes:
+ if p > pk:
+ break
+ if smallest % p == 0:
+ has_small_factor = True
+ break
+ if not has_small_factor:
+ return k
+ return None
def probe_report(model, cfg: Config, lo: int = 101, hi: int = 200) -> dict:
@@ -68,28 +139,23 @@ def probe_report(model, cfg: Config, lo: int = 101, hi: int = 200) -> dict:
easy_wrong += 1
total = hi - lo + 1
acc = correct / total
- if is_prime_task:
- flagged = [e for e in errors if e["n"] in FLAGGED_SIEVE_PREDS] # input IS the classified number
- distinct_flagged = len({e["n"] for e in flagged})
- else:
- flagged = [e for e in errors if e["pred"] in FLAGGED_SIEVE_PREDS]
- distinct_flagged = len({e["pred"] for e in flagged})
- flagged_frac = len(flagged) / len(errors) if errors else 0.0
- # classification per prereg + Addendum 3/5 operationalization
- if easy_total and easy_wrong / easy_total > 0.5:
+ # sieve rank (P5/P6 ladder, Addendum 6)
+ error_preds = [e["pred"] for e in errors]
+ rank = _compute_sieve_rank(error_preds, lo, hi) if not is_prime_task else None
+ # P-ladder classification (prereg + Addenda 3/5/6)
+ if not errors:
+ code = "P6" # exact — no probe misses
+ elif easy_total and easy_wrong / easy_total > 0.5:
code = "P4" # fails trivial evens/5-multiples -> pure memorization
- elif is_prime_task and len(errors) >= 3 and distinct_flagged >= 3 and flagged_frac >= 0.8:
- code = "P1" # is-prime: errors concentrated on no-small-factor composites -> learned sieve
+ elif rank is not None:
+ code = f"P5({rank})" # errors match rank-k sieve signature
elif acc >= 0.85:
- code = "P3" # surprising success beyond expectation (next_prime semantics)
- elif len(errors) >= 3 and distinct_flagged >= 3 and flagged_frac >= 0.8:
- code = "P1" # next_prime: errors = sieves predicting no-small-factor composites
+ code = "P3" # surprising success beyond expectation
else:
code = "P2" # scattered errors -> memorization / non-transferable heuristics
return {
"code": code, "acc": acc, "correct": correct, "total": total,
- "errors": errors, "flagged_errors": flagged,
- "flagged_pred_fraction": flagged_frac, "distinct_flagged_preds": distinct_flagged,
+ "errors": errors, "sieve_rank": rank,
"easy_total": easy_total, "easy_wrong": easy_wrong,
"easy_err_rate": (easy_wrong / easy_total) if easy_total else None,
}
@@ -225,7 +291,10 @@ def main() -> None:
"D2 is scored on in-range val EM only (Addendum 5)",
}
else:
- entry["probe"] = probe_report(model, cfg)
+ # out-of-range probe: [range_end+1, range_end+1000] (E6+ uses wider window)
+ p_lo = cfg.range_end + 1
+ p_hi = cfg.range_end + 1000
+ entry["probe"] = probe_report(model, cfg, lo=p_lo, hi=p_hi)
if cfg.model == "rnn":
entry["halting"] = halting_report(model, cfg)
entry["per_example"] = [{"n": "".join(map(str, x)), "target": t, "pred": p, "ok": ok}
@@ -245,7 +314,7 @@ def main() -> None:
"val_em_best": round(report["val_selected"].get("val_exact_match", float("nan")), 4),
"val_em_last": round(f.get("val_exact_match", float("nan")), 4),
"probe_code": p.get("code"), "probe_acc": round(p.get("acc", float("nan")), 3),
- "flagged_errors": [e["n"] for e in p.get("flagged_errors", [])],
+ "sieve_rank": p.get("sieve_rank"),
"signature": report["signature"]["code"],
"halting": (f.get("halting") or {}).get("code"),
"results": os.path.join(out_dir, "results.json"),
diff --git a/tests/test_data.py b/tests/test_data.py
index 9f651fb..2fb131e 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -75,10 +75,11 @@ def test_make_batch_shapes_and_padding():
def test_flagged_composites_are_composite():
- # sanity: the diagnostic set really is composite, has no divisors <= 7, and covers
- # the probe's candidate window [102, 211] (209 = 11*19 included — Addendum 3)
- from src.eval import FLAGGED_SIEVE_PREDS
- assert FLAGGED_SIEVE_PREDS == {121, 143, 169, 187, 209}
- for n in FLAGGED_SIEVE_PREDS:
+ # sanity: the rank-4 sieve signature set is composite, has no divisors <= 7,
+ # and covers the probe's candidate window [102, 211] (Addendum 3/6)
+ from src.eval import _sieve_rank_signature
+ sig = _sieve_rank_signature(4, 101, 200)
+ assert sig == {121, 143, 169, 187, 209}, f"unexpected rank-4 signature: {sig}"
+ for n in sig:
assert any(n % d == 0 for d in range(2, int(n ** 0.5) + 1))
assert all(n % d != 0 for d in (2, 3, 5, 7))
diff --git a/tests/test_eval_classification.py b/tests/test_eval_classification.py
index 7114166..2cc217b 100644
--- a/tests/test_eval_classification.py
+++ b/tests/test_eval_classification.py
@@ -12,7 +12,7 @@ import torch
from src.config import Config
from src.data import sieve_primes
-from src.eval import FLAGGED_SIEVE_PREDS, grokking_signature, halting_report, probe_report
+from src.eval import _sieve_rank_signature, grokking_signature, halting_report, probe_report
DIGITS_CFG = Config(vocab_mode="digits")
EOS = DIGITS_CFG.eos_id
@@ -58,18 +58,20 @@ def _easy_only(n):
return _perfect(n) if (n % 2 == 0 or n % 5 == 0) else 199
-def test_probe_sieve35_classified_p1():
- """A pure {2,3,5,7} sieve must classify P1: errors are predictions of 121/143/169/187/209."""
+def test_probe_sieve35_classified_p5():
+ """A pure {2,3,5,7} sieve must classify P5(4): errors match the rank-4 sieve signature."""
r = probe_report(StubModel(_sieve35), DIGITS_CFG)
preds = sorted({e["pred"] for e in r["errors"]})
- assert preds == sorted(FLAGGED_SIEVE_PREDS), r["errors"]
+ expected = sorted(_sieve_rank_signature(4, 101, 200))
+ assert preds == expected, r["errors"]
assert len(r["errors"]) == 22 # n in 113..120, 139..142, 167..168, 181..186, 199..200
- assert r["code"] == "P1", r
+ assert r["code"] == "P5(4)", r
+ assert r["sieve_rank"] == 4
-def test_probe_perfect_classified_p3():
+def test_probe_perfect_classified_p6():
r = probe_report(StubModel(_perfect), DIGITS_CFG)
- assert r["acc"] == 1.0 and r["code"] == "P3", r
+ assert r["acc"] == 1.0 and r["code"] == "P6", r
def test_probe_easy_only_classified_p2():
generated by cgit v1.2.3 (git 2.46.0) at 2026-08-31 19:16:28 +0000