diff options
| -rw-r--r-- | config/train_shakespeare_char.py | 4 | ||||
| -rw-r--r-- | model.py | 3 |
2 files changed, 6 insertions, 1 deletions
diff --git a/config/train_shakespeare_char.py b/config/train_shakespeare_char.py index 41c81df..ec4e5b5 100644 --- a/config/train_shakespeare_char.py +++ b/config/train_shakespeare_char.py @@ -35,3 +35,7 @@ warmup_iters = 100 # not super necessary potentially # on macbook also add # device = 'cpu' # run on cpu only # compile = False # do not torch compile the model + +# Maxwell GPU fix (K2200 — no bf16 support) +dtype = 'float32' # bfloat16 requires compute 8.0+; Maxwell is 5.0 +flash = False # disable SDPA — uses bf16 internally on CUDA 11.8 @@ -42,7 +42,8 @@ class CausalSelfAttention(nn.Module): self.n_embd = config.n_embd self.dropout = config.dropout # flash attention make GPU go brrrrr but support is only in PyTorch >= 2.0 - self.flash = hasattr(torch.nn.functional, 'scaled_dot_product_attention') + # Maxwell GPUs (compute 5.0) don't support bf16 ops used by SDPA internals + self.flash = hasattr(torch.nn.functional, 'scaled_dot_product_attention') and getattr(config, 'flash', True) if not self.flash: print("WARNING: using slow attention. Flash Attention requires PyTorch >= 2.0") # causal mask to ensure that attention is only applied to the left in the input sequence |