From db4ec5bb3839bc5cc50d82e427848595d14b3070 Mon Sep 17 00:00:00 2001 From: Void Agent Date: Wed, 29 Jul 2026 17:47:48 +0100 Subject: Restructure: nanoGPT at root, custom code in src/ - Move model.py, train.py, configurator.py to root for nanoGPT compatibility - data/ and config/ directories at root with Shakespeare dataset prep scripts - src/jlens.py updated to import model from project root - Cleaned up stale src/config/ and duplicate src/ files - Fixed .gitignore: exclude out-shakespeare-char/ instead of raw data dirs --- config/train_gpt2.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 config/train_gpt2.py (limited to 'config/train_gpt2.py') diff --git a/config/train_gpt2.py b/config/train_gpt2.py new file mode 100644 index 0000000..8f19273 --- /dev/null +++ b/config/train_gpt2.py @@ -0,0 +1,25 @@ +# config for training GPT-2 (124M) down to very nice loss of ~2.85 on 1 node of 8X A100 40GB +# launch as the following (e.g. in a screen session) and wait ~5 days: +# $ torchrun --standalone --nproc_per_node=8 train.py config/train_gpt2.py + +wandb_log = True +wandb_project = 'owt' +wandb_run_name='gpt2-124M' + +# these make the total batch size be ~0.5M +# 12 batch size * 1024 block size * 5 gradaccum * 8 GPUs = 491,520 +batch_size = 12 +block_size = 1024 +gradient_accumulation_steps = 5 * 8 + +# this makes total number of tokens be 300B +max_iters = 600000 +lr_decay_iters = 600000 + +# eval stuff +eval_interval = 1000 +eval_iters = 200 +log_interval = 10 + +# weight decay +weight_decay = 1e-1 -- cgit v1.2.3