updated hypr for my gpu

2026-01-07 12:59:44 -05:00
parent 6f037c4a9a
commit 121640bab6
1 changed files with 7 additions and 7 deletions
--- a/train.py
+++ b/train.py
@@ -10,16 +10,16 @@ import data
 import log

 hypr = {
-    "embed_size":  256,
-    "n_heads":     4,
-    "n_blocks":    4,
+    "embed_size":  512,
+    "n_heads":     8,
+    "n_blocks":    6,
    "block_size":  256,
    "batch_size":  16,
-    "starting_lr": 3e-4,
-    "minimum_lr":  3e-5,
+    "starting_lr": 6e-4,
+    "minimum_lr":  6e-5,
    "warmup":      1_000,
-    "steps":       5_000,
-    "encoding":    "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "steps":       20_000,
+    "encoding":    "gpt2",
    "dataset":     "HuggingFaceTB/smollm-corpus",
    "subset":      "cosmopedia-v2",
 }