k/llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated hypr for my gpu

Browse Source
This commit is contained in:
k
2026-01-07 12:59:44 -05:00
parent 6f037c4a9a
commit 121640bab6
1 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
train.py
View File

@@ -10,16 +10,16 @@ import data
import log import log
hypr = { hypr = {
"embed_size": 256, "embed_size": 512,
"n_heads": 4, "n_heads": 8,
"n_blocks": 4, "n_blocks": 6,
"block_size": 256, "block_size": 256,
"batch_size": 16, "batch_size": 16,
"starting_lr": 3e-4, "starting_lr": 6e-4,
"minimum_lr": 3e-5, "minimum_lr": 6e-5,
"warmup": 1_000, "warmup": 1_000,
"steps": 5_000, "steps": 20_000,
"encoding": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", "encoding": "gpt2",
"dataset": "HuggingFaceTB/smollm-corpus", "dataset": "HuggingFaceTB/smollm-corpus",
"subset": "cosmopedia-v2", "subset": "cosmopedia-v2",
} }