From 66f99ee30087a5f28ad852e581a0334c7f556091 Mon Sep 17 00:00:00 2001 From: Void Agent Date: Wed, 29 Jul 2026 17:42:47 +0100 Subject: Initial project setup: J-lens implementation for nanoGPT - Core J-lens computation (batched per-layer, per-token gradient method) - Project README with background and experiment plan - Sync script for meru Docker container deployment - Upstream nanoGPT model code copied to src/ Architecture: Computes d(log_p(token))/d(residual_stream) averaged over corpus contexts, replicating Anthropic's Jacobian Lens technique. --- src/config/finetune_shakespeare.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/config/finetune_shakespeare.py (limited to 'src/config/finetune_shakespeare.py') diff --git a/src/config/finetune_shakespeare.py b/src/config/finetune_shakespeare.py new file mode 100644 index 0000000..148a4c4 --- /dev/null +++ b/src/config/finetune_shakespeare.py @@ -0,0 +1,25 @@ +import time + +out_dir = 'out-shakespeare' +eval_interval = 5 +eval_iters = 40 +wandb_log = False # feel free to turn on +wandb_project = 'shakespeare' +wandb_run_name = 'ft-' + str(time.time()) + +dataset = 'shakespeare' +init_from = 'gpt2-xl' # this is the largest GPT-2 model + +# only save checkpoints if the validation loss improves +always_save_checkpoint = False + +# the number of examples per iter: +# 1 batch_size * 32 grad_accum * 1024 tokens = 32,768 tokens/iter +# shakespeare has 301,966 tokens, so 1 epoch ~= 9.2 iters +batch_size = 1 +gradient_accumulation_steps = 32 +max_iters = 20 + +# finetune at constant LR +learning_rate = 3e-5 +decay_lr = False -- cgit v1.2.3