speedup: batched eval (46x, sieve-stub verified), run_sweep orchestrator (2-way parallel, idempotent, summaries), E1 jobs; 36 tests

1 files changed, 929 insertions, 0 deletions

diff --git a/runs-take2/rnn/seed0/results.json b/runs-take2/rnn/seed0/results.json
new file mode 100644
index 0000000..3e8e532
--- /dev/null
+++ b/runs-take2/rnn/seed0/results.json
@@ -0,0 +1,929 @@
+{
+  "model": "rnn",
+  "seed": 0,
+  "params": null,
+  "val_selected": {
+    "ckpt": "best.pt",
+    "params": 36620,
+    "val_token_acc": 0.7045454545454546,
+    "val_exact_match": 0.36666666666666664,
+    "note": "val-selected checkpoint (early stop / best-on-val per prereg \u2014 val EM here is selection-holed, not an untouched test estimate"
+  },
+  "final": {
+    "ckpt": "last.pt",
+    "params": 36620,
+    "val_token_acc": 0.5909090909090909,
+    "val_exact_match": 0.16666666666666666,
+    "note": "final checkpoint, no val selection",
+    "probe": {
+      "code": "P4",
+      "acc": 0.01,
+      "correct": 1,
+      "total": 100,
+      "errors": [
+        {
+          "n": 101,
+          "target": 103,
+          "pred": 101
+        },
+        {
+          "n": 102,
+          "target": 103,
+          "pred": 199
+        },
+        {
+          "n": 103,
+          "target": 107,
+          "pred": 101
+        },
+        {
+          "n": 104,
+          "target": 107,
+          "pred": 119
+        },
+        {
+          "n": 105,
+          "target": 107,
+          "pred": 199
+        },
+        {
+          "n": 106,
+          "target": 107,
+          "pred": 101
+        },
+        {
+          "n": 107,
+          "target": 109,
+          "pred": 199
+        },
+        {
+          "n": 109,
+          "target": 113,
+          "pred": 899
+        },
+        {
+          "n": 110,
+          "target": 113,
+          "pred": 101
+        },
+        {
+          "n": 111,
+          "target": 113,
+          "pred": 1
+        },
+        {
+          "n": 112,
+          "target": 113,
+          "pred": 99
+        },
+        {
+          "n": 113,
+          "target": 127,
+          "pred": 10
+        },
+        {
+          "n": 114,
+          "target": 127,
+          "pred": 19
+        },
+        {
+          "n": 115,
+          "target": 127,
+          "pred": 99
+        },
+        {
+          "n": 116,
+          "target": 127,
+          "pred": 19
+        },
+        {
+          "n": 117,
+          "target": 127,
+          "pred": 19
+        },
+        {
+          "n": 118,
+          "target": 127,
+          "pred": 1
+        },
+        {
+          "n": 119,
+          "target": 127,
+          "pred": 899
+        },
+        {
+          "n": 120,
+          "target": 127,
+          "pred": 199
+        },
+        {
+          "n": 121,
+          "target": 127,
+          "pred": 999
+        },
+        {
+          "n": 122,
+          "target": 127,
+          "pred": 299
+        },
+        {
+          "n": 123,
+          "target": 127,
+          "pred": 299
+        },
+        {
+          "n": 124,
+          "target": 127,
+          "pred": 299
+        },
+        {
+          "n": 125,
+          "target": 127,
+          "pred": 999
+        },
+        {
+          "n": 126,
+          "target": 127,
+          "pred": 997
+        },
+        {
+          "n": 127,
+          "target": 131,
+          "pred": 999
+        },
+        {
+          "n": 128,
+          "target": 131,
+          "pred": 999
+        },
+        {
+          "n": 129,
+          "target": 131,
+          "pred": 299
+        },
+        {
+          "n": 130,
+          "target": 131,
+          "pred": 101
+        },
+        {
+          "n": 131,
+          "target": 137,
+          "pred": 10
+        },
+        {
+          "n": 132,
+          "target": 137,
+          "pred": 299
+        },
+        {
+          "n": 133,
+          "target": 137,
+          "pred": 899
+        },
+        {
+          "n": 134,
+          "target": 137,
+          "pred": 899
+        },
+        {
+          "n": 135,
+          "target": 137,
+          "pred": 899
+        },
+        {
+          "n": 136,
+          "target": 137,
+          "pred": 19
+        },
+        {
+          "n": 137,
+          "target": 139,
+          "pred": 899
+        },
+        {
+          "n": 138,
+          "target": 139,
+          "pred": 899
+        },
+        {
+          "n": 139,
+          "target": 149,
+          "pred": 899
+        },
+        {
+          "n": 140,
+          "target": 149,
+          "pred": 119
+        },
+        {
+          "n": 141,
+          "target": 149,
+          "pred": 19
+        },
+        {
+          "n": 142,
+          "target": 149,
+          "pred": 899
+        },
+        {
+          "n": 143,
+          "target": 149,
+          "pred": 899
+        },
+        {
+          "n": 144,
+          "target": 149,
+          "pred": 899
+        },
+        {
+          "n": 145,
+          "target": 149,
+          "pred": 899
+        },
+        {
+          "n": 146,
+          "target": 149,
+          "pred": 199
+        },
+        {
+          "n": 147,
+          "target": 149,
+          "pred": 899
+        },
+        {
+          "n": 148,
+          "target": 149,
+          "pred": 899
+        },
+        {
+          "n": 149,
+          "target": 151,
+          "pred": 899
+        },
+        {
+          "n": 150,
+          "target": 151,
+          "pred": 893
+        },
+        {
+          "n": 151,
+          "target": 157,
+          "pred": 99
+        },
+        {
+          "n": 152,
+          "target": 157,
+          "pred": 999
+        },
+        {
+          "n": 153,
+          "target": 157,
+          "pred": 899
+        },
+        {
+          "n": 154,
+          "target": 157,
+          "pred": 899
+        },
+        {
+          "n": 155,
+          "target": 157,
+          "pred": 899
+        },
+        {
+          "n": 156,
+          "target": 157,
+          "pred": 99
+        },
+        {
+          "n": 157,
+          "target": 163,
+          "pred": 999
+        },
+        {
+          "n": 158,
+          "target": 163,
+          "pred": 899
+        },
+        {
+          "n": 159,
+          "target": 163,
+          "pred": 899
+        },
+        {
+          "n": 160,
+          "target": 163,
+          "pred": 101
+        },
+        {
+          "n": 161,
+          "target": 163,
+          "pred": 19
+        },
+        {
+          "n": 162,
+          "target": 163,
+          "pred": 997
+        },
+        {
+          "n": 163,
+          "target": 167,
+          "pred": 199
+        },
+        {
+          "n": 164,
+          "target": 167,
+          "pred": 199
+        },
+        {
+          "n": 165,
+          "target": 167,
+          "pred": 99
+        },
+        {
+          "n": 166,
+          "target": 167,
+          "pred": 199
+        },
+        {
+          "n": 167,
+          "target": 173,
+          "pred": 99
+        },
+        {
+          "n": 168,
+          "target": 173,
+          "pred": 99
+        },
+        {
+          "n": 169,
+          "target": 173,
+          "pred": 899
+        },
+        {
+          "n": 170,
+          "target": 173,
+          "pred": 199
+        },
+        {
+          "n": 171,
+          "target": 173,
+          "pred": 99
+        },
+        {
+          "n": 172,
+          "target": 173,
+          "pred": 999
+        },
+        {
+          "n": 173,
+          "target": 179,
+          "pred": 99
+        },
+        {
+          "n": 174,
+          "target": 179,
+          "pred": 99
+        },
+        {
+          "n": 175,
+          "target": 179,
+          "pred": 999
+        },
+        {
+          "n": 176,
+          "target": 179,
+          "pred": 99
+        },
+        {
+          "n": 177,
+          "target": 179,
+          "pred": 999
+        },
+        {
+          "n": 178,
+          "target": 179,
+          "pred": 999
+        },
+        {
+          "n": 179,
+          "target": 181,
+          "pred": 899
+        },
+        {
+          "n": 180,
+          "target": 181,
+          "pred": 893
+        },
+        {
+          "n": 181,
+          "target": 191,
+          "pred": 9
+        },
+        {
+          "n": 182,
+          "target": 191,
+          "pred": 999
+        },
+        {
+          "n": 183,
+          "target": 191,
+          "pred": 899
+        },
+        {
+          "n": 184,
+          "target": 191,
+          "pred": 899
+        },
+        {
+          "n": 185,
+          "target": 191,
+          "pred": 899
+        },
+        {
+          "n": 186,
+          "target": 191,
+          "pred": 999
+        },
+        {
+          "n": 187,
+          "target": 191,
+          "pred": 999
+        },
+        {
+          "n": 188,
+          "target": 191,
+          "pred": 899
+        },
+        {
+          "n": 189,
+          "target": 191,
+          "pred": 899
+        },
+        {
+          "n": 190,
+          "target": 191,
+          "pred": 899
+        },
+        {
+          "n": 191,
+          "target": 193,
+          "pred": 899
+        },
+        {
+          "n": 192,
+          "target": 193,
+          "pred": 299
+        },
+        {
+          "n": 193,
+          "target": 197,
+          "pred": 899
+        },
+        {
+          "n": 194,
+          "target": 197,
+          "pred": 899
+        },
+        {
+          "n": 195,
+          "target": 197,
+          "pred": 899
+        },
+        {
+          "n": 196,
+          "target": 197,
+          "pred": 899
+        },
+        {
+          "n": 197,
+          "target": 199,
+          "pred": 899
+        },
+        {
+          "n": 198,
+          "target": 199,
+          "pred": 899
+        },
+        {
+          "n": 199,
+          "target": 211,
+          "pred": 899
+        },
+        {
+          "n": 200,
+          "target": 211,
+          "pred": 199
+        }
+      ],
+      "flagged_errors": [],
+      "flagged_pred_fraction": 0.0,
+      "distinct_flagged_preds": 0,
+      "easy_total": 60,
+      "easy_wrong": 59,
+      "easy_err_rate": 0.9833333333333333
+    },
+    "halting": {
+      "code": "H4",
+      "mean_steps": 3.123047472250582,
+      "corr_gap": 0.2214679834807299,
+      "min_steps": 2,
+      "max_steps": 20,
+      "gaps": [
+        1,
+        2,
+        1,
+        2,
+        1,
+        4,
+        3,
+        2,
+        1,
+        2,
+        1,
+        4,
+        3,
+        2,
+        1,
+        2,
+        1,
+        4,
+        3,
+        2,
+        1,
+        6,
+        5,
+        4,
+        3,
+        2,
+        1,
+        2,
+        1,
+        6,
+        5,
+        4,
+        3,
+        2,
+        1,
+        4,
+        3,
+        2,
+        1,
+        2,
+        1,
+        4,
+        3,
+        2,
+        1,
+        6,
+        5,
+        4,
+        3,
+        2,
+        1,
+        6,
+        5,
+        4,
+        3,
+        2,
+        1,
+        2,
+        1,
+        6,
+        5,
+        4,
+        3,
+        2,
+        1,
+        4,
+        3,
+        2,
+        1,
+        2,
+        1,
+        6,
+        5,
+        4,
+        3,
+        2,
+        1,
+        4,
+        3,
+        2,
+        1,
+        6,
+        5,
+        4,
+        3,
+        2,
+        1,
+        8,
+        7,
+        6,
+        5,
+        4,
+        3,
+        2,
+        1,
+        4,
+        3,
+        2,
+        1
+      ],
+      "steps_by_n": [
+        3.014195680618286,
+        3.135343313217163,
+        3.090144634246826,
+        3.1713850498199463,
+        2.900202989578247,
+        3.0106613636016846,
+        3.2104332447052,
+        3.325666666030884,
+        2.9300780296325684,
+        2.890700578689575,
+        2.9069631099700928,
+        2.9783923625946045,
+        2.9744765758514404,
+        3.008406162261963,
+        2.8595876693725586,
+        2.9244143962860107,
+        3.0445709228515625,
+        3.0422301292419434,
+        2.99932861328125,
+        2.9410030841827393,
+        3.0114541053771973,
+        3.107120990753174,
+        3.0779876708984375,
+        3.136584520339966,
+        2.921391010284424,
+        3.0053720474243164,
+        3.1654398441314697,
+        3.233349561691284,
+        3.1324732303619385,
+        3.0467514991760254,
+        3.1338253021240234,
+        3.2645983695983887,
+        3.2252910137176514,
+        3.281956672668457,
+        3.031636953353882,
+        3.1119115352630615,
+        3.2983484268188477,
+        3.3865015506744385,
+        3.0938613414764404,
+        3.022742986679077,
+        3.0877559185028076,
+        3.202849864959717,
+        3.175102472305298,
+        3.2327229976654053,
+        3.0024092197418213,
+        3.0831265449523926,
+        3.2592060565948486,
+        3.3100247383117676,
+        3.1473474502563477,
+        3.071608066558838,
+        3.163006544113159,
+        3.2788093090057373,
+        3.24808406829834,
+        3.296496868133545,
+        3.0659124851226807,
+        3.1418488025665283,
+        3.3155531883239746,
+        3.416996717453003,
+        2.898960828781128,
+        2.858566999435425,
+        2.882908344268799,
+        2.96085524559021,
+        2.9518916606903076,
+        2.9972848892211914,
+        2.83113169670105,
+        2.901516914367676,
+        3.0369534492492676,
+        3.0480051040649414,
+        2.9847333431243896,
+        2.941075325012207,
+        2.98722767829895,
+        3.0637786388397217,
+        3.0518579483032227,
+        3.094906806945801,
+        2.921523094177246,
+        2.9897327423095703,
+        3.1205570697784424,
+        3.164365768432617,
+        3.1843652725219727,
+        3.113823175430298,
+        3.196122884750366,
+        3.2959306240081787,
+        3.2752671241760254,
+        3.3164381980895996,
+        3.111222743988037,
+        3.1724257469177246,
+        3.317817449569702,
+        3.412120819091797,
+        3.267571210861206,
+        3.1642038822174072,
+        3.3226613998413086,
+        3.455141305923462,
+        3.400142192840576,
+        3.4917120933532715,
+        3.1787307262420654,
+        3.2746527194976807,
+        3.4824793338775635,
+        3.616750955581665,
+        2.89864182472229
+      ]
+    },
+    "per_example": [
+      {
+        "n": "3",
+        "target": 5,
+        "pred": 9,
+        "ok": false
+      },
+      {
+        "n": "4",
+        "target": 5,
+        "pred": 7,
+        "ok": false
+      },
+      {
+        "n": "8",
+        "target": 11,
+        "pred": 7,
+        "ok": false
+      },
+      {
+        "n": "9",
+        "target": 11,
+        "pred": 1,
+        "ok": false
+      },
+      {
+        "n": "10",
+        "target": 11,
+        "pred": 11,
+        "ok": true
+      },
+      {
+        "n": "12",
+        "target": 13,
+        "pred": 23,
+        "ok": false
+      },
+      {
+        "n": "13",
+        "target": 17,
+        "pred": 41,
+        "ok": false
+      },
+      {
+        "n": "15",
+        "target": 17,
+        "pred": 37,
+        "ok": false
+      },
+      {
+        "n": "18",
+        "target": 19,
+        "pred": 33,
+        "ok": false
+      },
+      {
+        "n": "21",
+        "target": 23,
+        "pred": 23,
+        "ok": true
+      },
+      {
+        "n": "24",
+        "target": 29,
+        "pred": 23,
+        "ok": false
+      },
+      {
+        "n": "25",
+        "target": 29,
+        "pred": 23,
+        "ok": false
+      },
+      {
+        "n": "26",
+        "target": 29,
+        "pred": 67,
+        "ok": false
+      },
+      {
+        "n": "31",
+        "target": 37,
+        "pred": 41,
+        "ok": false
+      },
+      {
+        "n": "32",
+        "target": 37,
+        "pred": 29,
+        "ok": false
+      },
+      {
+        "n": "39",
+        "target": 41,
+        "pred": 97,
+        "ok": false
+      },
+      {
+        "n": "43",
+        "target": 47,
+        "pred": 37,
+        "ok": false
+      },
+      {
+        "n": "49",
+        "target": 53,
+        "pred": 79,
+        "ok": false
+      },
+      {
+        "n": "50",
+        "target": 53,
+        "pred": 41,
+        "ok": false
+      },
+      {
+        "n": "54",
+        "target": 59,
+        "pred": 53,
+        "ok": false
+      },
+      {
+        "n": "56",
+        "target": 59,
+        "pred": 57,
+        "ok": false
+      },
+      {
+        "n": "61",
+        "target": 67,
+        "pred": 67,
+        "ok": true
+      },
+      {
+        "n": "65",
+        "target": 67,
+        "pred": 67,
+        "ok": true
+      },
+      {
+        "n": "68",
+        "target": 71,
+        "pred": 79,
+        "ok": false
+      },
+      {
+        "n": "78",
+        "target": 79,
+        "pred": 89,
+        "ok": false
+      },
+      {
+        "n": "82",
+        "target": 83,
+        "pred": 29,
+        "ok": false
+      },
+      {
+        "n": "84",
+        "target": 89,
+        "pred": 59,
+        "ok": false
+      },
+      {
+        "n": "88",
+        "target": 89,
+        "pred": 89,
+        "ok": true
+      },
+      {
+        "n": "92",
+        "target": 97,
+        "pred": 31,
+        "ok": false
+      },
+      {
+        "n": "95",
+        "target": 97,
+        "pred": 61,
+        "ok": false
+      }
+    ]
+  },
+  "signature": {
+    "code": "O4",
+    "sat_start_eval": null,
+    "val_hi_eval_idx": null,
+    "transition_width_evals": null,
+    "n_evals": 1000,
+    "final_train_em": 0.7971014492753623,
+    "final_val_em": 0.16666666666666666,
+    "max_train_em": 1.0
+  }
+}
\ No newline at end of file