summaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
authorVoid Agent <void@jayrup.hermes>2026-08-20 16:55:49 +0100
committerVoid Agent <void@jayrup.hermes>2026-08-20 16:55:49 +0100
commitf8995751de925571d4eec669ab7ab8b1c3d76cb9 (patch)
treeb75eac4be20fd2285ca9a3d4fd55d479f8de5d6d /tests
parente0fec42563680396f51a07805187bd0a8539ea15 (diff)
fix(eval): P5/P6 sieve-rank ladder replaces hardcoded FLAGGED_SIEVE_PREDS
- probe_report: dynamic lo/hi (no more hardcoded [101,200]) - main(): probe range derived from range_end ([range_end+1, range_end+1000]) - P6 = exact (no probe misses), P5(k) = errors match rank-k sieve signature - _sieve_rank_signature: computes composites with all factors > p_k - _compute_sieve_rank: identifies rank from model's error predictions - build_report.py: probe_fig uses sieve rank for coloring - Tests updated for P5/P6 codes (48/48 green)
Diffstat (limited to 'tests')
-rw-r--r--tests/test_data.py11
-rw-r--r--tests/test_eval_classification.py16
2 files changed, 15 insertions, 12 deletions
diff --git a/tests/test_data.py b/tests/test_data.py
index 9f651fb..2fb131e 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -75,10 +75,11 @@ def test_make_batch_shapes_and_padding():
def test_flagged_composites_are_composite():
- # sanity: the diagnostic set really is composite, has no divisors <= 7, and covers
- # the probe's candidate window [102, 211] (209 = 11*19 included — Addendum 3)
- from src.eval import FLAGGED_SIEVE_PREDS
- assert FLAGGED_SIEVE_PREDS == {121, 143, 169, 187, 209}
- for n in FLAGGED_SIEVE_PREDS:
+ # sanity: the rank-4 sieve signature set is composite, has no divisors <= 7,
+ # and covers the probe's candidate window [102, 211] (Addendum 3/6)
+ from src.eval import _sieve_rank_signature
+ sig = _sieve_rank_signature(4, 101, 200)
+ assert sig == {121, 143, 169, 187, 209}, f"unexpected rank-4 signature: {sig}"
+ for n in sig:
assert any(n % d == 0 for d in range(2, int(n ** 0.5) + 1))
assert all(n % d != 0 for d in (2, 3, 5, 7))
diff --git a/tests/test_eval_classification.py b/tests/test_eval_classification.py
index 7114166..2cc217b 100644
--- a/tests/test_eval_classification.py
+++ b/tests/test_eval_classification.py
@@ -12,7 +12,7 @@ import torch
from src.config import Config
from src.data import sieve_primes
-from src.eval import FLAGGED_SIEVE_PREDS, grokking_signature, halting_report, probe_report
+from src.eval import _sieve_rank_signature, grokking_signature, halting_report, probe_report
DIGITS_CFG = Config(vocab_mode="digits")
EOS = DIGITS_CFG.eos_id
@@ -58,18 +58,20 @@ def _easy_only(n):
return _perfect(n) if (n % 2 == 0 or n % 5 == 0) else 199
-def test_probe_sieve35_classified_p1():
- """A pure {2,3,5,7} sieve must classify P1: errors are predictions of 121/143/169/187/209."""
+def test_probe_sieve35_classified_p5():
+ """A pure {2,3,5,7} sieve must classify P5(4): errors match the rank-4 sieve signature."""
r = probe_report(StubModel(_sieve35), DIGITS_CFG)
preds = sorted({e["pred"] for e in r["errors"]})
- assert preds == sorted(FLAGGED_SIEVE_PREDS), r["errors"]
+ expected = sorted(_sieve_rank_signature(4, 101, 200))
+ assert preds == expected, r["errors"]
assert len(r["errors"]) == 22 # n in 113..120, 139..142, 167..168, 181..186, 199..200
- assert r["code"] == "P1", r
+ assert r["code"] == "P5(4)", r
+ assert r["sieve_rank"] == 4
-def test_probe_perfect_classified_p3():
+def test_probe_perfect_classified_p6():
r = probe_report(StubModel(_perfect), DIGITS_CFG)
- assert r["acc"] == 1.0 and r["code"] == "P3", r
+ assert r["acc"] == 1.0 and r["code"] == "P6", r
def test_probe_easy_only_classified_p2():