{
  "model": "rnn",
  "seed": 0,
  "params": null,
  "val_selected": {
    "ckpt": "best.pt",
    "params": 36620,
    "val_token_acc": 0.7045454545454546,
    "val_exact_match": 0.36666666666666664,
    "note": "val-selected checkpoint (early stop / best-on-val per prereg \u2014 val EM here is selection-holed, not an untouched test estimate"
  },
  "final": {
    "ckpt": "last.pt",
    "params": 36620,
    "val_token_acc": 0.5909090909090909,
    "val_exact_match": 0.16666666666666666,
    "note": "final checkpoint, no val selection",
    "probe": {
      "code": "P4",
      "acc": 0.01,
      "correct": 1,
      "total": 100,
      "errors": [
        {
          "n": 101,
          "target": 103,
          "pred": 101
        },
        {
          "n": 102,
          "target": 103,
          "pred": 199
        },
        {
          "n": 103,
          "target": 107,
          "pred": 101
        },
        {
          "n": 104,
          "target": 107,
          "pred": 119
        },
        {
          "n": 105,
          "target": 107,
          "pred": 199
        },
        {
          "n": 106,
          "target": 107,
          "pred": 101
        },
        {
          "n": 107,
          "target": 109,
          "pred": 199
        },
        {
          "n": 109,
          "target": 113,
          "pred": 899
        },
        {
          "n": 110,
          "target": 113,
          "pred": 101
        },
        {
          "n": 111,
          "target": 113,
          "pred": 1
        },
        {
          "n": 112,
          "target": 113,
          "pred": 99
        },
        {
          "n": 113,
          "target": 127,
          "pred": 10
        },
        {
          "n": 114,
          "target": 127,
          "pred": 19
        },
        {
          "n": 115,
          "target": 127,
          "pred": 99
        },
        {
          "n": 116,
          "target": 127,
          "pred": 19
        },
        {
          "n": 117,
          "target": 127,
          "pred": 19
        },
        {
          "n": 118,
          "target": 127,
          "pred": 1
        },
        {
          "n": 119,
          "target": 127,
          "pred": 899
        },
        {
          "n": 120,
          "target": 127,
          "pred": 199
        },
        {
          "n": 121,
          "target": 127,
          "pred": 999
        },
        {
          "n": 122,
          "target": 127,
          "pred": 299
        },
        {
          "n": 123,
          "target": 127,
          "pred": 299
        },
        {
          "n": 124,
          "target": 127,
          "pred": 299
        },
        {
          "n": 125,
          "target": 127,
          "pred": 999
        },
        {
          "n": 126,
          "target": 127,
          "pred": 997
        },
        {
          "n": 127,
          "target": 131,
          "pred": 999
        },
        {
          "n": 128,
          "target": 131,
          "pred": 999
        },
        {
          "n": 129,
          "target": 131,
          "pred": 299
        },
        {
          "n": 130,
          "target": 131,
          "pred": 101
        },
        {
          "n": 131,
          "target": 137,
          "pred": 10
        },
        {
          "n": 132,
          "target": 137,
          "pred": 299
        },
        {
          "n": 133,
          "target": 137,
          "pred": 899
        },
        {
          "n": 134,
          "target": 137,
          "pred": 899
        },
        {
          "n": 135,
          "target": 137,
          "pred": 899
        },
        {
          "n": 136,
          "target": 137,
          "pred": 19
        },
        {
          "n": 137,
          "target": 139,
          "pred": 899
        },
        {
          "n": 138,
          "target": 139,
          "pred": 899
        },
        {
          "n": 139,
          "target": 149,
          "pred": 899
        },
        {
          "n": 140,
          "target": 149,
          "pred": 119
        },
        {
          "n": 141,
          "target": 149,
          "pred": 19
        },
        {
          "n": 142,
          "target": 149,
          "pred": 899
        },
        {
          "n": 143,
          "target": 149,
          "pred": 899
        },
        {
          "n": 144,
          "target": 149,
          "pred": 899
        },
        {
          "n": 145,
          "target": 149,
          "pred": 899
        },
        {
          "n": 146,
          "target": 149,
          "pred": 199
        },
        {
          "n": 147,
          "target": 149,
          "pred": 899
        },
        {
          "n": 148,
          "target": 149,
          "pred": 899
        },
        {
          "n": 149,
          "target": 151,
          "pred": 899
        },
        {
          "n": 150,
          "target": 151,
          "pred": 893
        },
        {
          "n": 151,
          "target": 157,
          "pred": 99
        },
        {
          "n": 152,
          "target": 157,
          "pred": 999
        },
        {
          "n": 153,
          "target": 157,
          "pred": 899
        },
        {
          "n": 154,
          "target": 157,
          "pred": 899
        },
        {
          "n": 155,
          "target": 157,
          "pred": 899
        },
        {
          "n": 156,
          "target": 157,
          "pred": 99
        },
        {
          "n": 157,
          "target": 163,
          "pred": 999
        },
        {
          "n": 158,
          "target": 163,
          "pred": 899
        },
        {
          "n": 159,
          "target": 163,
          "pred": 899
        },
        {
          "n": 160,
          "target": 163,
          "pred": 101
        },
        {
          "n": 161,
          "target": 163,
          "pred": 19
        },
        {
          "n": 162,
          "target": 163,
          "pred": 997
        },
        {
          "n": 163,
          "target": 167,
          "pred": 199
        },
        {
          "n": 164,
          "target": 167,
          "pred": 199
        },
        {
          "n": 165,
          "target": 167,
          "pred": 99
        },
        {
          "n": 166,
          "target": 167,
          "pred": 199
        },
        {
          "n": 167,
          "target": 173,
          "pred": 99
        },
        {
          "n": 168,
          "target": 173,
          "pred": 99
        },
        {
          "n": 169,
          "target": 173,
          "pred": 899
        },
        {
          "n": 170,
          "target": 173,
          "pred": 199
        },
        {
          "n": 171,
          "target": 173,
          "pred": 99
        },
        {
          "n": 172,
          "target": 173,
          "pred": 999
        },
        {
          "n": 173,
          "target": 179,
          "pred": 99
        },
        {
          "n": 174,
          "target": 179,
          "pred": 99
        },
        {
          "n": 175,
          "target": 179,
          "pred": 999
        },
        {
          "n": 176,
          "target": 179,
          "pred": 99
        },
        {
          "n": 177,
          "target": 179,
          "pred": 999
        },
        {
          "n": 178,
          "target": 179,
          "pred": 999
        },
        {
          "n": 179,
          "target": 181,
          "pred": 899
        },
        {
          "n": 180,
          "target": 181,
          "pred": 893
        },
        {
          "n": 181,
          "target": 191,
          "pred": 9
        },
        {
          "n": 182,
          "target": 191,
          "pred": 999
        },
        {
          "n": 183,
          "target": 191,
          "pred": 899
        },
        {
          "n": 184,
          "target": 191,
          "pred": 899
        },
        {
          "n": 185,
          "target": 191,
          "pred": 899
        },
        {
          "n": 186,
          "target": 191,
          "pred": 999
        },
        {
          "n": 187,
          "target": 191,
          "pred": 999
        },
        {
          "n": 188,
          "target": 191,
          "pred": 899
        },
        {
          "n": 189,
          "target": 191,
          "pred": 899
        },
        {
          "n": 190,
          "target": 191,
          "pred": 899
        },
        {
          "n": 191,
          "target": 193,
          "pred": 899
        },
        {
          "n": 192,
          "target": 193,
          "pred": 299
        },
        {
          "n": 193,
          "target": 197,
          "pred": 899
        },
        {
          "n": 194,
          "target": 197,
          "pred": 899
        },
        {
          "n": 195,
          "target": 197,
          "pred": 899
        },
        {
          "n": 196,
          "target": 197,
          "pred": 899
        },
        {
          "n": 197,
          "target": 199,
          "pred": 899
        },
        {
          "n": 198,
          "target": 199,
          "pred": 899
        },
        {
          "n": 199,
          "target": 211,
          "pred": 899
        },
        {
          "n": 200,
          "target": 211,
          "pred": 199
        }
      ],
      "flagged_errors": [],
      "flagged_pred_fraction": 0.0,
      "distinct_flagged_preds": 0,
      "easy_total": 60,
      "easy_wrong": 59,
      "easy_err_rate": 0.9833333333333333
    },
    "halting": {
      "code": "H4",
      "mean_steps": 3.123047472250582,
      "corr_gap": 0.2214679834807299,
      "min_steps": 2,
      "max_steps": 20,
      "gaps": [
        1,
        2,
        1,
        2,
        1,
        4,
        3,
        2,
        1,
        2,
        1,
        4,
        3,
        2,
        1,
        2,
        1,
        4,
        3,
        2,
        1,
        6,
        5,
        4,
        3,
        2,
        1,
        2,
        1,
        6,
        5,
        4,
        3,
        2,
        1,
        4,
        3,
        2,
        1,
        2,
        1,
        4,
        3,
        2,
        1,
        6,
        5,
        4,
        3,
        2,
        1,
        6,
        5,
        4,
        3,
        2,
        1,
        2,
        1,
        6,
        5,
        4,
        3,
        2,
        1,
        4,
        3,
        2,
        1,
        2,
        1,
        6,
        5,
        4,
        3,
        2,
        1,
        4,
        3,
        2,
        1,
        6,
        5,
        4,
        3,
        2,
        1,
        8,
        7,
        6,
        5,
        4,
        3,
        2,
        1,
        4,
        3,
        2,
        1
      ],
      "steps_by_n": [
        3.014195680618286,
        3.135343313217163,
        3.090144634246826,
        3.1713850498199463,
        2.900202989578247,
        3.0106613636016846,
        3.2104332447052,
        3.325666666030884,
        2.9300780296325684,
        2.890700578689575,
        2.9069631099700928,
        2.9783923625946045,
        2.9744765758514404,
        3.008406162261963,
        2.8595876693725586,
        2.9244143962860107,
        3.0445709228515625,
        3.0422301292419434,
        2.99932861328125,
        2.9410030841827393,
        3.0114541053771973,
        3.107120990753174,
        3.0779876708984375,
        3.136584520339966,
        2.921391010284424,
        3.0053720474243164,
        3.1654398441314697,
        3.233349561691284,
        3.1324732303619385,
        3.0467514991760254,
        3.1338253021240234,
        3.2645983695983887,
        3.2252910137176514,
        3.281956672668457,
        3.031636953353882,
        3.1119115352630615,
        3.2983484268188477,
        3.3865015506744385,
        3.0938613414764404,
        3.022742986679077,
        3.0877559185028076,
        3.202849864959717,
        3.175102472305298,
        3.2327229976654053,
        3.0024092197418213,
        3.0831265449523926,
        3.2592060565948486,
        3.3100247383117676,
        3.1473474502563477,
        3.071608066558838,
        3.163006544113159,
        3.2788093090057373,
        3.24808406829834,
        3.296496868133545,
        3.0659124851226807,
        3.1418488025665283,
        3.3155531883239746,
        3.416996717453003,
        2.898960828781128,
        2.858566999435425,
        2.882908344268799,
        2.96085524559021,
        2.9518916606903076,
        2.9972848892211914,
        2.83113169670105,
        2.901516914367676,
        3.0369534492492676,
        3.0480051040649414,
        2.9847333431243896,
        2.941075325012207,
        2.98722767829895,
        3.0637786388397217,
        3.0518579483032227,
        3.094906806945801,
        2.921523094177246,
        2.9897327423095703,
        3.1205570697784424,
        3.164365768432617,
        3.1843652725219727,
        3.113823175430298,
        3.196122884750366,
        3.2959306240081787,
        3.2752671241760254,
        3.3164381980895996,
        3.111222743988037,
        3.1724257469177246,
        3.317817449569702,
        3.412120819091797,
        3.267571210861206,
        3.1642038822174072,
        3.3226613998413086,
        3.455141305923462,
        3.400142192840576,
        3.4917120933532715,
        3.1787307262420654,
        3.2746527194976807,
        3.4824793338775635,
        3.616750955581665,
        2.89864182472229
      ]
    },
    "per_example": [
      {
        "n": "3",
        "target": 5,
        "pred": 9,
        "ok": false
      },
      {
        "n": "4",
        "target": 5,
        "pred": 7,
        "ok": false
      },
      {
        "n": "8",
        "target": 11,
        "pred": 7,
        "ok": false
      },
      {
        "n": "9",
        "target": 11,
        "pred": 1,
        "ok": false
      },
      {
        "n": "10",
        "target": 11,
        "pred": 11,
        "ok": true
      },
      {
        "n": "12",
        "target": 13,
        "pred": 23,
        "ok": false
      },
      {
        "n": "13",
        "target": 17,
        "pred": 41,
        "ok": false
      },
      {
        "n": "15",
        "target": 17,
        "pred": 37,
        "ok": false
      },
      {
        "n": "18",
        "target": 19,
        "pred": 33,
        "ok": false
      },
      {
        "n": "21",
        "target": 23,
        "pred": 23,
        "ok": true
      },
      {
        "n": "24",
        "target": 29,
        "pred": 23,
        "ok": false
      },
      {
        "n": "25",
        "target": 29,
        "pred": 23,
        "ok": false
      },
      {
        "n": "26",
        "target": 29,
        "pred": 67,
        "ok": false
      },
      {
        "n": "31",
        "target": 37,
        "pred": 41,
        "ok": false
      },
      {
        "n": "32",
        "target": 37,
        "pred": 29,
        "ok": false
      },
      {
        "n": "39",
        "target": 41,
        "pred": 97,
        "ok": false
      },
      {
        "n": "43",
        "target": 47,
        "pred": 37,
        "ok": false
      },
      {
        "n": "49",
        "target": 53,
        "pred": 79,
        "ok": false
      },
      {
        "n": "50",
        "target": 53,
        "pred": 41,
        "ok": false
      },
      {
        "n": "54",
        "target": 59,
        "pred": 53,
        "ok": false
      },
      {
        "n": "56",
        "target": 59,
        "pred": 57,
        "ok": false
      },
      {
        "n": "61",
        "target": 67,
        "pred": 67,
        "ok": true
      },
      {
        "n": "65",
        "target": 67,
        "pred": 67,
        "ok": true
      },
      {
        "n": "68",
        "target": 71,
        "pred": 79,
        "ok": false
      },
      {
        "n": "78",
        "target": 79,
        "pred": 89,
        "ok": false
      },
      {
        "n": "82",
        "target": 83,
        "pred": 29,
        "ok": false
      },
      {
        "n": "84",
        "target": 89,
        "pred": 59,
        "ok": false
      },
      {
        "n": "88",
        "target": 89,
        "pred": 89,
        "ok": true
      },
      {
        "n": "92",
        "target": 97,
        "pred": 31,
        "ok": false
      },
      {
        "n": "95",
        "target": 97,
        "pred": 61,
        "ok": false
      }
    ]
  },
  "signature": {
    "code": "O4",
    "sat_start_eval": null,
    "val_hi_eval_idx": null,
    "transition_width_evals": null,
    "n_evals": 1000,
    "final_train_em": 0.7971014492753623,
    "final_val_em": 0.16666666666666666,
    "max_train_em": 1.0
  }
}