diff options
Diffstat (limited to 'src/train_gpt2.py')
| -rw-r--r-- | src/train_gpt2.py | 25 |
1 files changed, 0 insertions, 25 deletions
diff --git a/src/train_gpt2.py b/src/train_gpt2.py deleted file mode 100644 index 8f19273..0000000 --- a/src/train_gpt2.py +++ /dev/null @@ -1,25 +0,0 @@ -# config for training GPT-2 (124M) down to very nice loss of ~2.85 on 1 node of 8X A100 40GB -# launch as the following (e.g. in a screen session) and wait ~5 days: -# $ torchrun --standalone --nproc_per_node=8 train.py config/train_gpt2.py - -wandb_log = True -wandb_project = 'owt' -wandb_run_name='gpt2-124M' - -# these make the total batch size be ~0.5M -# 12 batch size * 1024 block size * 5 gradaccum * 8 GPUs = 491,520 -batch_size = 12 -block_size = 1024 -gradient_accumulation_steps = 5 * 8 - -# this makes total number of tokens be 300B -max_iters = 600000 -lr_decay_iters = 600000 - -# eval stuff -eval_interval = 1000 -eval_iters = 200 -log_interval = 10 - -# weight decay -weight_decay = 1e-1 |