From 66f99ee30087a5f28ad852e581a0334c7f556091 Mon Sep 17 00:00:00 2001 From: Void Agent Date: Wed, 29 Jul 2026 17:42:47 +0100 Subject: Initial project setup: J-lens implementation for nanoGPT - Core J-lens computation (batched per-layer, per-token gradient method) - Project README with background and experiment plan - Sync script for meru Docker container deployment - Upstream nanoGPT model code copied to src/ Architecture: Computes d(log_p(token))/d(residual_stream) averaged over corpus contexts, replicating Anthropic's Jacobian Lens technique. --- src/train_shakespeare_char.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/train_shakespeare_char.py (limited to 'src/train_shakespeare_char.py') diff --git a/src/train_shakespeare_char.py b/src/train_shakespeare_char.py new file mode 100644 index 0000000..41c81df --- /dev/null +++ b/src/train_shakespeare_char.py @@ -0,0 +1,37 @@ +# train a miniature character-level shakespeare model +# good for debugging and playing on macbooks and such + +out_dir = 'out-shakespeare-char' +eval_interval = 250 # keep frequent because we'll overfit +eval_iters = 200 +log_interval = 10 # don't print too too often + +# we expect to overfit on this small dataset, so only save when val improves +always_save_checkpoint = False + +wandb_log = False # override via command line if you like +wandb_project = 'shakespeare-char' +wandb_run_name = 'mini-gpt' + +dataset = 'shakespeare_char' +gradient_accumulation_steps = 1 +batch_size = 64 +block_size = 256 # context of up to 256 previous characters + +# baby GPT model :) +n_layer = 6 +n_head = 6 +n_embd = 384 +dropout = 0.2 + +learning_rate = 1e-3 # with baby networks can afford to go a bit higher +max_iters = 5000 +lr_decay_iters = 5000 # make equal to max_iters usually +min_lr = 1e-4 # learning_rate / 10 usually +beta2 = 0.99 # make a bit bigger because number of tokens per iter is small + +warmup_iters = 100 # not super necessary potentially + +# on macbook also add +# device = 'cpu' # run on cpu only +# compile = False # do not torch compile the model -- cgit v1.2.3