{
  "model": "transformer",
  "seed": 0,
  "params": null,
  "val_selected": {
    "ckpt": "best.pt",
    "params": 407947,
    "val_token_acc": 0.9545454545454546,
    "val_exact_match": 0.8666666666666667,
    "note": "val-selected checkpoint (early stop / best-on-val per prereg \u2014 val EM here is selection-holed, not an untouched test estimate"
  },
  "final": {
    "ckpt": "last.pt",
    "params": 407947,
    "val_token_acc": 0.8863636363636364,
    "val_exact_match": 0.7,
    "note": "final checkpoint, no val selection",
    "probe": {
      "code": "P4",
      "acc": 0.0,
      "correct": 0,
      "total": 100,
      "errors": [
        {
          "n": 101,
          "target": 103,
          "pred": 101
        },
        {
          "n": 102,
          "target": 103,
          "pred": 101
        },
        {
          "n": 103,
          "target": 107,
          "pred": 101
        },
        {
          "n": 104,
          "target": 107,
          "pred": 101
        },
        {
          "n": 105,
          "target": 107,
          "pred": 171
        },
        {
          "n": 106,
          "target": 107,
          "pred": 101
        },
        {
          "n": 107,
          "target": 109,
          "pred": 101
        },
        {
          "n": 108,
          "target": 109,
          "pred": 101
        },
        {
          "n": 109,
          "target": 113,
          "pred": 101
        },
        {
          "n": 110,
          "target": 113,
          "pred": 101
        },
        {
          "n": 111,
          "target": 113,
          "pred": 101
        },
        {
          "n": 112,
          "target": 113,
          "pred": 101
        },
        {
          "n": 113,
          "target": 127,
          "pred": 101
        },
        {
          "n": 114,
          "target": 127,
          "pred": 171
        },
        {
          "n": 115,
          "target": 127,
          "pred": 171
        },
        {
          "n": 116,
          "target": 127,
          "pred": 171
        },
        {
          "n": 117,
          "target": 127,
          "pred": 101
        },
        {
          "n": 118,
          "target": 127,
          "pred": 101
        },
        {
          "n": 119,
          "target": 127,
          "pred": 20
        },
        {
          "n": 120,
          "target": 127,
          "pred": 20
        },
        {
          "n": 121,
          "target": 127,
          "pred": 101
        },
        {
          "n": 122,
          "target": 127,
          "pred": 20
        },
        {
          "n": 123,
          "target": 127,
          "pred": 20
        },
        {
          "n": 124,
          "target": 127,
          "pred": 101
        },
        {
          "n": 125,
          "target": 127,
          "pred": 101
        },
        {
          "n": 126,
          "target": 127,
          "pred": 20
        },
        {
          "n": 127,
          "target": 131,
          "pred": 20
        },
        {
          "n": 128,
          "target": 131,
          "pred": 20
        },
        {
          "n": 129,
          "target": 131,
          "pred": 20
        },
        {
          "n": 130,
          "target": 131,
          "pred": 90
        },
        {
          "n": 131,
          "target": 137,
          "pred": 97
        },
        {
          "n": 132,
          "target": 137,
          "pred": 37
        },
        {
          "n": 133,
          "target": 137,
          "pred": 97
        },
        {
          "n": 134,
          "target": 137,
          "pred": 97
        },
        {
          "n": 135,
          "target": 137,
          "pred": 37
        },
        {
          "n": 136,
          "target": 137,
          "pred": 97
        },
        {
          "n": 137,
          "target": 139,
          "pred": 101
        },
        {
          "n": 138,
          "target": 139,
          "pred": 101
        },
        {
          "n": 139,
          "target": 149,
          "pred": 101
        },
        {
          "n": 140,
          "target": 149,
          "pred": 101
        },
        {
          "n": 141,
          "target": 149,
          "pred": 101
        },
        {
          "n": 142,
          "target": 149,
          "pred": 101
        },
        {
          "n": 143,
          "target": 149,
          "pred": 171
        },
        {
          "n": 144,
          "target": 149,
          "pred": 171
        },
        {
          "n": 145,
          "target": 149,
          "pred": 171
        },
        {
          "n": 146,
          "target": 149,
          "pred": 171
        },
        {
          "n": 147,
          "target": 149,
          "pred": 50
        },
        {
          "n": 148,
          "target": 149,
          "pred": 50
        },
        {
          "n": 149,
          "target": 151,
          "pred": 60
        },
        {
          "n": 150,
          "target": 151,
          "pred": 50
        },
        {
          "n": 151,
          "target": 157,
          "pred": 50
        },
        {
          "n": 152,
          "target": 157,
          "pred": 50
        },
        {
          "n": 153,
          "target": 157,
          "pred": 50
        },
        {
          "n": 154,
          "target": 157,
          "pred": 50
        },
        {
          "n": 155,
          "target": 157,
          "pred": 171
        },
        {
          "n": 156,
          "target": 157,
          "pred": 50
        },
        {
          "n": 157,
          "target": 163,
          "pred": 50
        },
        {
          "n": 158,
          "target": 163,
          "pred": 50
        },
        {
          "n": 159,
          "target": 163,
          "pred": 60
        },
        {
          "n": 160,
          "target": 163,
          "pred": 101
        },
        {
          "n": 161,
          "target": 163,
          "pred": 101
        },
        {
          "n": 162,
          "target": 163,
          "pred": 171
        },
        {
          "n": 163,
          "target": 167,
          "pred": 97
        },
        {
          "n": 164,
          "target": 167,
          "pred": 97
        },
        {
          "n": 165,
          "target": 167,
          "pred": 97
        },
        {
          "n": 166,
          "target": 167,
          "pred": 97
        },
        {
          "n": 167,
          "target": 173,
          "pred": 101
        },
        {
          "n": 168,
          "target": 173,
          "pred": 101
        },
        {
          "n": 169,
          "target": 173,
          "pred": 50
        },
        {
          "n": 170,
          "target": 173,
          "pred": 70
        },
        {
          "n": 171,
          "target": 173,
          "pred": 70
        },
        {
          "n": 172,
          "target": 173,
          "pred": 70
        },
        {
          "n": 173,
          "target": 179,
          "pred": 79
        },
        {
          "n": 174,
          "target": 179,
          "pred": 77
        },
        {
          "n": 175,
          "target": 179,
          "pred": 77
        },
        {
          "n": 176,
          "target": 179,
          "pred": 70
        },
        {
          "n": 177,
          "target": 179,
          "pred": 101
        },
        {
          "n": 178,
          "target": 179,
          "pred": 70
        },
        {
          "n": 179,
          "target": 181,
          "pred": 80
        },
        {
          "n": 180,
          "target": 181,
          "pred": 80
        },
        {
          "n": 181,
          "target": 191,
          "pred": 80
        },
        {
          "n": 182,
          "target": 191,
          "pred": 80
        },
        {
          "n": 183,
          "target": 191,
          "pred": 80
        },
        {
          "n": 184,
          "target": 191,
          "pred": 87
        },
        {
          "n": 185,
          "target": 191,
          "pred": 87
        },
        {
          "n": 186,
          "target": 191,
          "pred": 87
        },
        {
          "n": 187,
          "target": 191,
          "pred": 101
        },
        {
          "n": 188,
          "target": 191,
          "pred": 101
        },
        {
          "n": 189,
          "target": 191,
          "pred": 87
        },
        {
          "n": 190,
          "target": 191,
          "pred": 97
        },
        {
          "n": 191,
          "target": 193,
          "pred": 97
        },
        {
          "n": 192,
          "target": 193,
          "pred": 97
        },
        {
          "n": 193,
          "target": 197,
          "pred": 97
        },
        {
          "n": 194,
          "target": 197,
          "pred": 97
        },
        {
          "n": 195,
          "target": 197,
          "pred": 97
        },
        {
          "n": 196,
          "target": 197,
          "pred": 97
        },
        {
          "n": 197,
          "target": 199,
          "pred": 101
        },
        {
          "n": 198,
          "target": 199,
          "pred": 101
        },
        {
          "n": 199,
          "target": 211,
          "pred": 101
        },
        {
          "n": 200,
          "target": 211,
          "pred": 101
        }
      ],
      "flagged_errors": [],
      "flagged_pred_fraction": 0.0,
      "distinct_flagged_preds": 0,
      "easy_total": 60,
      "easy_wrong": 60,
      "easy_err_rate": 1.0
    },
    "per_example": [
      {
        "n": "3",
        "target": 5,
        "pred": 3,
        "ok": false
      },
      {
        "n": "4",
        "target": 5,
        "pred": 7,
        "ok": false
      },
      {
        "n": "8",
        "target": 11,
        "pred": 1,
        "ok": false
      },
      {
        "n": "9",
        "target": 11,
        "pred": 31,
        "ok": false
      },
      {
        "n": "10",
        "target": 11,
        "pred": 13,
        "ok": false
      },
      {
        "n": "12",
        "target": 13,
        "pred": 13,
        "ok": true
      },
      {
        "n": "13",
        "target": 17,
        "pred": 17,
        "ok": true
      },
      {
        "n": "15",
        "target": 17,
        "pred": 17,
        "ok": true
      },
      {
        "n": "18",
        "target": 19,
        "pred": 19,
        "ok": true
      },
      {
        "n": "21",
        "target": 23,
        "pred": 23,
        "ok": true
      },
      {
        "n": "24",
        "target": 29,
        "pred": 29,
        "ok": true
      },
      {
        "n": "25",
        "target": 29,
        "pred": 29,
        "ok": true
      },
      {
        "n": "26",
        "target": 29,
        "pred": 29,
        "ok": true
      },
      {
        "n": "31",
        "target": 37,
        "pred": 31,
        "ok": false
      },
      {
        "n": "32",
        "target": 37,
        "pred": 37,
        "ok": true
      },
      {
        "n": "39",
        "target": 41,
        "pred": 31,
        "ok": false
      },
      {
        "n": "43",
        "target": 47,
        "pred": 47,
        "ok": true
      },
      {
        "n": "49",
        "target": 53,
        "pred": 61,
        "ok": false
      },
      {
        "n": "50",
        "target": 53,
        "pred": 53,
        "ok": true
      },
      {
        "n": "54",
        "target": 59,
        "pred": 59,
        "ok": true
      },
      {
        "n": "56",
        "target": 59,
        "pred": 59,
        "ok": true
      },
      {
        "n": "61",
        "target": 67,
        "pred": 61,
        "ok": false
      },
      {
        "n": "65",
        "target": 67,
        "pred": 67,
        "ok": true
      },
      {
        "n": "68",
        "target": 71,
        "pred": 71,
        "ok": true
      },
      {
        "n": "78",
        "target": 79,
        "pred": 79,
        "ok": true
      },
      {
        "n": "82",
        "target": 83,
        "pred": 83,
        "ok": true
      },
      {
        "n": "84",
        "target": 89,
        "pred": 89,
        "ok": true
      },
      {
        "n": "88",
        "target": 89,
        "pred": 89,
        "ok": true
      },
      {
        "n": "92",
        "target": 97,
        "pred": 97,
        "ok": true
      },
      {
        "n": "95",
        "target": 97,
        "pred": 97,
        "ok": true
      }
    ]
  },
  "signature": {
    "code": "O-PARTIAL",
    "sat_start_eval": 0,
    "val_hi_eval_idx": null,
    "transition_width_evals": null,
    "n_evals": 1000,
    "final_train_em": 1.0,
    "final_val_em": 0.7,
    "max_train_em": 1.0
  }
}