Upload folder using huggingface_hub

README.md

@@ -0,0 +1,99 @@
+# nanochat training report
+
+Generated: 2025-11-02 23:26:37
+
+## Environment
+
+### Git Information
+- Branch: main
+- Commit: 45f875f (dirty)
+- Message: update with checkpoint save(test)
+
+### Hardware
+- Platform: Linux
+- CPUs: 128 cores (256 logical)
+- Memory: 160.0 GB
+- GPUs: 1x NVIDIA A100-SXM4-40GB
+- GPU Memory: 39.5 GB total
+- CUDA Version: 12.8
+- Hourly Rate: $1.79/hour
+
+### Software
+- Python: 3.10.19
+- PyTorch: 2.8.0+cu128
+
+
+### Bloat
+- Characters: 419,982
+- Lines: 10,409
+- Files: 52
+- Tokens (approx): 104,995
+- Dependencies (uv.lock lines): 2,220
+
+Run started: 2025-11-02 23:26:38
+
+---
+
+## Midtraining
+timestamp: 2025-11-03 12:34:20
+
+- run: dummy
+- device_type: 
+- dtype: bfloat16
+- num_iterations: -1
+- max_seq_len: 2048
+- device_batch_size: 1
+- unembedding_lr: 0.0040
+- embedding_lr: 0.2000
+- matrix_lr: 0.0200
+- init_lr_frac: 1.0000
+- weight_decay: 0.0000
+- eval_every: 150
+- eval_tokens: 10,485,760
+- total_batch_size: 524,288
+- checkpoint_every: 100
+- dry_run: 0
+- Number of iterations: 813
+- DDP world size: 1
+- Minimum validation bpb: 0.3972
+
+
+## Chat SFT
+timestamp: 2025-11-04 02:59:21
+
+- run: dummy
+- source: mid
+- device_type: 
+- dtype: bfloat16
+- device_batch_size: 1
+- num_epochs: 1
+- num_iterations: -1
+- target_examples_per_step: 32
+- unembedding_lr: 0.0040
+- embedding_lr: 0.2000
+- matrix_lr: 0.0200
+- weight_decay: 0.0000
+- init_lr_frac: 0.0200
+- eval_every: 100
+- eval_steps: 100
+- eval_metrics_every: 200
+- eval_metrics_max_problems: 1024
+- checkpoint_every: 50
+- Training rows: 22,439
+- Number of iterations: 701
+- Training loss: 1.3129
+- Validation loss: nan
+
+
+## Summary
+
+- Characters: 419,982
+- Lines: 10,409
+- Files: 52
+- Tokens (approx): 104,995
+- Dependencies (uv.lock lines): 2,220
+
+| Metric          | BASE     | MID      | SFT      | RL       |
+|-----------------|----------|----------|----------|----------|
+
+Total wall clock time: 27h32m

base/meta_000002.json

@@ -0,0 +1,39 @@
+{
+  "step": 2,
+  "val_bpb": 2.6584760219879953,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 65536,
+    "n_layer": 32,
+    "n_head": 16,
+    "n_kv_head": 16,
+    "n_embd": 2048
+  },
+  "user_config": {
+    "run": "dummy",
+    "device_type": "",
+    "depth": 32,
+    "max_seq_len": 2048,
+    "num_iterations": 2,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 20,
+    "device_batch_size": 1,
+    "total_batch_size": 32768,
+    "embedding_lr": 0.2,
+    "unembedding_lr": 0.004,
+    "weight_decay": 0.0,
+    "matrix_lr": 0.02,
+    "grad_clip": 1.0,
+    "warmup_ratio": 0.0,
+    "warmdown_ratio": 0.2,
+    "final_lr_frac": 0.0,
+    "eval_every": 999999,
+    "eval_tokens": 2048,
+    "core_metric_every": 999999,
+    "core_metric_max_per_task": 500,
+    "sample_every": 999999,
+    "model_tag": ""
+  },
+  "device_batch_size": 1,
+  "max_seq_len": 2048
+}

base/meta_008000.json

@@ -0,0 +1,39 @@
+{
+  "step": 8000,
+  "val_bpb": 0.7965255783929484,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 65536,
+    "n_layer": 32,
+    "n_head": 16,
+    "n_kv_head": 16,
+    "n_embd": 2048
+  },
+  "user_config": {
+    "run": "dummy",
+    "device_type": "",
+    "depth": 32,
+    "max_seq_len": 2048,
+    "num_iterations": 8000,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 20,
+    "device_batch_size": 1,
+    "total_batch_size": 262144,
+    "embedding_lr": 0.2,
+    "unembedding_lr": 0.004,
+    "weight_decay": 0.0,
+    "matrix_lr": 0.02,
+    "grad_clip": 1.0,
+    "warmup_ratio": 0.0,
+    "warmdown_ratio": 0.2,
+    "final_lr_frac": 0.0,
+    "eval_every": 250,
+    "eval_tokens": 4096,
+    "core_metric_every": 250,
+    "core_metric_max_per_task": 12,
+    "sample_every": 250,
+    "model_tag": ""
+  },
+  "device_batch_size": 1,
+  "max_seq_len": 2048
+}

base/model_000002.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:938a4e3a951b3fb3d315b2b129c5f2c8d45f76f52a9ce760ad1366e3bdcb1dda
+size 7247837167

base/model_008000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd8aa4ec0454f1bdecc63bc6b67a2bc193b872950fee2639357b2120f9d37d79
+size 7247837167

base/optim_000002.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:486c914c8ba0746661f298eac6284023c4d21fc719fca590a272928699de7680
+size 8204121436

base/optim_008000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96ec9cfbf26b54b70b9893041d6379bdc30a58bc09949801de9567e883eabba4
+size 8204121436

mid_train/meta_000813.json

@@ -0,0 +1,30 @@
+{
+  "step": 813,
+  "val_bpb": 0.39722378266856306,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 65536,
+    "n_layer": 32,
+    "n_head": 16,
+    "n_kv_head": 16,
+    "n_embd": 2048
+  },
+  "user_config": {
+    "run": "dummy",
+    "device_type": "",
+    "dtype": "bfloat16",
+    "num_iterations": -1,
+    "max_seq_len": 2048,
+    "device_batch_size": 1,
+    "unembedding_lr": 0.004,
+    "embedding_lr": 0.2,
+    "matrix_lr": 0.02,
+    "init_lr_frac": 1.0,
+    "weight_decay": 0.0,
+    "eval_every": 150,
+    "eval_tokens": 10485760,
+    "total_batch_size": 524288,
+    "checkpoint_every": 100,
+    "dry_run": 0
+  }
+}

mid_train/model_000813.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18e475a962f44163d56a40818d98db57407ffbc38f8224a0ad1e1b9e80efcb38
+size 7247837167

mid_train/optim_000813.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:828fa45b4318459d7202323953559362d4eefebf519728d2f60b4ead7e5426c2
+size 8204121436

report/chat-sft.md

@@ -0,0 +1,26 @@
+## Chat SFT
+timestamp: 2025-11-04 02:59:21
+
+- run: dummy
+- source: mid
+- device_type: 
+- dtype: bfloat16
+- device_batch_size: 1
+- num_epochs: 1
+- num_iterations: -1
+- target_examples_per_step: 32
+- unembedding_lr: 0.0040
+- embedding_lr: 0.2000
+- matrix_lr: 0.0200
+- weight_decay: 0.0000
+- init_lr_frac: 0.0200
+- eval_every: 100
+- eval_steps: 100
+- eval_metrics_every: 200
+- eval_metrics_max_problems: 1024
+- checkpoint_every: 50
+- Training rows: 22,439
+- Number of iterations: 701
+- Training loss: 1.3129
+- Validation loss: nan
+

report/header.md

@@ -0,0 +1,36 @@
+# nanochat training report
+
+Generated: 2025-11-02 23:26:37
+
+## Environment
+
+### Git Information
+- Branch: main
+- Commit: 45f875f (dirty)
+- Message: update with checkpoint save(test)
+
+### Hardware
+- Platform: Linux
+- CPUs: 128 cores (256 logical)
+- Memory: 160.0 GB
+- GPUs: 1x NVIDIA A100-SXM4-40GB
+- GPU Memory: 39.5 GB total
+- CUDA Version: 12.8
+- Hourly Rate: $1.79/hour
+
+### Software
+- Python: 3.10.19
+- PyTorch: 2.8.0+cu128
+
+
+### Bloat
+- Characters: 419,982
+- Lines: 10,409
+- Files: 52
+- Tokens (approx): 104,995
+- Dependencies (uv.lock lines): 2,220
+
+Run started: 2025-11-02 23:26:38
+
+---
+

report/midtraining.md

@@ -0,0 +1,23 @@
+## Midtraining
+timestamp: 2025-11-03 12:34:20
+
+- run: dummy
+- device_type: 
+- dtype: bfloat16
+- num_iterations: -1
+- max_seq_len: 2048
+- device_batch_size: 1
+- unembedding_lr: 0.0040
+- embedding_lr: 0.2000
+- matrix_lr: 0.0200
+- init_lr_frac: 1.0000
+- weight_decay: 0.0000
+- eval_every: 150
+- eval_tokens: 10,485,760
+- total_batch_size: 524,288
+- checkpoint_every: 100
+- dry_run: 0
+- Number of iterations: 813
+- DDP world size: 1
+- Minimum validation bpb: 0.3972
+

report/report.md

@@ -0,0 +1,99 @@
+# nanochat training report
+
+Generated: 2025-11-02 23:26:37
+
+## Environment
+
+### Git Information
+- Branch: main
+- Commit: 45f875f (dirty)
+- Message: update with checkpoint save(test)
+
+### Hardware
+- Platform: Linux
+- CPUs: 128 cores (256 logical)
+- Memory: 160.0 GB
+- GPUs: 1x NVIDIA A100-SXM4-40GB
+- GPU Memory: 39.5 GB total
+- CUDA Version: 12.8
+- Hourly Rate: $1.79/hour
+
+### Software
+- Python: 3.10.19
+- PyTorch: 2.8.0+cu128
+
+
+### Bloat
+- Characters: 419,982
+- Lines: 10,409
+- Files: 52
+- Tokens (approx): 104,995
+- Dependencies (uv.lock lines): 2,220
+
+Run started: 2025-11-02 23:26:38
+
+---
+
+## Midtraining
+timestamp: 2025-11-03 12:34:20
+
+- run: dummy
+- device_type: 
+- dtype: bfloat16
+- num_iterations: -1
+- max_seq_len: 2048
+- device_batch_size: 1
+- unembedding_lr: 0.0040
+- embedding_lr: 0.2000
+- matrix_lr: 0.0200
+- init_lr_frac: 1.0000
+- weight_decay: 0.0000
+- eval_every: 150
+- eval_tokens: 10,485,760
+- total_batch_size: 524,288
+- checkpoint_every: 100
+- dry_run: 0
+- Number of iterations: 813
+- DDP world size: 1
+- Minimum validation bpb: 0.3972
+
+
+## Chat SFT
+timestamp: 2025-11-04 02:59:21
+
+- run: dummy
+- source: mid
+- device_type: 
+- dtype: bfloat16
+- device_batch_size: 1
+- num_epochs: 1
+- num_iterations: -1
+- target_examples_per_step: 32
+- unembedding_lr: 0.0040
+- embedding_lr: 0.2000
+- matrix_lr: 0.0200
+- weight_decay: 0.0000
+- init_lr_frac: 0.0200
+- eval_every: 100
+- eval_steps: 100
+- eval_metrics_every: 200
+- eval_metrics_max_problems: 1024
+- checkpoint_every: 50
+- Training rows: 22,439
+- Number of iterations: 701
+- Training loss: 1.3129
+- Validation loss: nan
+
+
+## Summary
+
+- Characters: 419,982
+- Lines: 10,409
+- Files: 52
+- Tokens (approx): 104,995
+- Dependencies (uv.lock lines): 2,220
+
+| Metric          | BASE     | MID      | SFT      | RL       |
+|-----------------|----------|----------|----------|----------|
+
+Total wall clock time: 27h32m

sft/meta_000700.json

@@ -0,0 +1,14 @@
+{
+  "step": 700,
+  "val_loss": NaN,
+  "mmlu_acc": 0.34375,
+  "arc_easy_acc": 0.4091796875,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 65536,
+    "n_layer": 32,
+    "n_head": 16,
+    "n_kv_head": 16,
+    "n_embd": 2048
+  }
+}

sft/model_000700.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e927a960386a949b702669bcdb66bf7398b0ca9852617a3a3ca8ae431c70a18
+size 7247837167

sft/optim_000700.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4360b4112f587ab59d637843634c8f3335c6fa3ae1c94e6e9b55fd0470bbabed
+size 8204121436

tokenizer/token_bytes.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae39c27aae519d14071efc95f9a558ba0b7ede47e7d83ad4f198422b44c5f70e
+size 263721

tokenizer/tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c060565a46fe83b49d99005acba796f2a630daa7970eb49f7513b89f9fb40e0
+size 846208