diff options
| -rw-r--r-- | src/dim_starvation.py | 2 | ||||
| -rw-r--r-- | src/freq_ablation.py | 2 | ||||
| -rwxr-xr-x | sync_and_run.sh | 7 |
3 files changed, 9 insertions, 2 deletions
diff --git a/src/dim_starvation.py b/src/dim_starvation.py index edc00be..2cb9d12 100644 --- a/src/dim_starvation.py +++ b/src/dim_starvation.py @@ -50,7 +50,7 @@ def train_model(out_dir, d_model, data_dir=DATA_DIR): n_params = sum(p.numel() for p in model.parameters()) print(f" d_model={d_model}, n_head={n_head}, params={n_params/1e6:.2f}M") - optimizer = model.configure_optimizers(weight_decay=0.1, lr=1e-3, + optimizer = model.configure_optimizers(weight_decay=0.1, learning_rate=1e-3, betas=(0.9, 0.99), device_type='cuda') os.makedirs(out_dir, exist_ok=True) best_val = 1e9 diff --git a/src/freq_ablation.py b/src/freq_ablation.py index c011140..1227ae6 100644 --- a/src/freq_ablation.py +++ b/src/freq_ablation.py @@ -33,7 +33,7 @@ def train(data_dir, out_dir, max_iters=5000): model = GPT(config).to(device) print(f" Params: {sum(p.numel() for p in model.parameters())/1e6:.2f}M") - optimizer = model.configure_optimizers(weight_decay=0.1, lr=1e-3, + optimizer = model.configure_optimizers(weight_decay=0.1, learning_rate=1e-3, betas=(0.9, 0.99), device_type='cuda') os.makedirs(out_dir, exist_ok=True) best_val = 1e9 diff --git a/sync_and_run.sh b/sync_and_run.sh index b716f32..a809331 100755 --- a/sync_and_run.sh +++ b/sync_and_run.sh @@ -16,6 +16,13 @@ rsync -avz --delete \ --exclude='__pycache__' \ --exclude='*.pyc' \ --exclude='outputs/*' \ + --exclude='data/*/train.bin' \ + --exclude='data/*/val.bin' \ + --exclude='data/*/meta.pkl' \ + --exclude='data/freq_experiment/' \ + --exclude='out-shakespeare-char/' \ + --exclude='out-freq-*/' \ + --exclude='out-starved-*/' \ --exclude='data/*' \ "${LOCAL_PROJECT}/" \ "${MACHINE}:${REMOTE_CODE}/" |