diff --git a/.research/events.jsonl b/.research/events.jsonl index 9832e4e..4b4184e 100644 --- a/.research/events.jsonl +++ b/.research/events.jsonl @@ -9,5 +9,6 @@ {"id": "1f544207d4084d928f2bad7d2d298be7", "ts": "2026-10-16T18:50:23+09:00", "task": "NPUTRAIN-19", "result": "8B full FT RNGD 1장 accum 4: 13.13 s/step(2048 tok) = 156 tok/s, accum 1(49.6 tok/s)의 3.1배. 전송 고정비 ~8.6 s는 그대로", "next": "A100 8B full FT 대조(e11 GPU 비면); 카드 2장 레이어 분할(NPUTRAIN-18)", "state": "active", "done": false, "blocker": "", "metrics": {"step_time_s": "13.13"}, "params": {"model": "Llama-3.1-8B-Instruct", "mode": "full", "seq_len": "512", "accum": "4", "device": "rngd", "node": "e11", "offload": "pipeline4"}, "artifacts": [{"path": "experiments/data/pipe8b4-rngd.jsonl", "size": 20585, "sha256": "0426259b65b231a8"}], "experiments": ["experiments/2026-10-16-full8b-offload-step.md"], "labels": ["benchmark", "fine-tuning", "rngd"], "review": false, "todos": [], "note": "", "agent": "claude", "host": "control-node", "branch": "main", "commit": "5de85aa-dirty"} {"id": "9f702fdb114e4b25b23508bb2bab2d14", "ts": "2026-10-21T15:37:39+09:00", "task": "#34 resident device AdamW + host-throughput step model", "result": "compile --adamw-only 150 s (was >2.5 h); 2 RNGD x 8 resident layers 4.57 s/step vs 1-card offload 5.22 s (+14% tok/s, steps 3-7, intermittent OOM #40); --update-workers 8 -8%; host-throughput model within 15% on 6 configs", "next": "fix resident OOM (#40), then 200-step run", "state": "active", "done": false, "blocker": "", "metrics": {"tok_s": "448"}, "params": {"seq_len": "512", "accum": "4", "resident_layers": "8", "cards": "2", "sdk": "furiosa-torch-2026.2.0"}, "artifacts": [], "experiments": ["experiments/2026-10-21-resident-adamw.md"], "labels": ["benchmark"], "review": true, "todos": [], "note": "", "agent": "claude", "host": "control-node", "branch": "perf/34-resident-adamw-compile", "commit": "8539644-dirty"} {"id": "f97382c821f344d08c88e8afc0c290ca", "ts": "2026-10-21T20:20:40+09:00", "task": "#33 Qwen2 gradient check on RNGD", "result": "npu-lora check passes (worst 3.1%, was 44%); 0 negative-loss steps in 400 (was 11); loss curve matches A100", "next": "Llama-3.1-8B check with the new head; Qwen3", "state": "active", "done": false, "blocker": "", "metrics": {"step_time_s": "2.74"}, "params": {}, "artifacts": [], "experiments": ["experiments/2026-10-21-qwen2-grad-precision.md"], "labels": ["bug", "experiment"], "review": true, "todos": [], "note": "", "agent": "claude", "host": "DESKTOP-R49IQ68", "branch": "fix/33-qwen-lora-grad", "commit": "a7e206a"} +{"id": "aa4e47d3f9354d9ca18b563fd512c24c", "ts": "2026-10-21T21:10:59+09:00", "task": "#40 resident AdamW DRAM_SHARED OOM", "result": "cause: furiosa-torch 2026.2 process-global DRAM_SHARED flag raced by layer-worker uploads (not memory); fix: one lock for graph calls + device allocations; 8/8 resident runs 25/25 steps no OOM; 2 RNGD x 8 resident 4.27 s/step (480 tok/s) vs 2-card offload 5.04 s, model -1.1%", "next": "200-step 2-card resident run (#40 done-when); 1-card resident 4 timing at low host load; report the runtime flag to Furiosa", "state": "active", "done": false, "blocker": "", "metrics": {"tok_s": "479.6", "step_s": "4.27"}, "params": {"cards": "2", "resident_layers": "8", "host_load": "46-81", "sdk": "furiosa-torch-2026.2.0"}, "artifacts": [], "experiments": ["experiments/2026-10-21-dram-shared-race.md"], "labels": ["bug", "benchmark"], "review": true, "todos": [], "note": "", "agent": "claude", "host": "control-node", "branch": "fix/40-dram-shared-race", "commit": "0e58322-dirty"} {"id": "b958b783d2b041f2ba43e072eb284abc", "ts": "2026-10-21T21:57:00+09:00", "task": "#35 MoE hot experts batched", "result": "batched hot experts 13.0 s/step vs host-only 13.5 (alternating steps, Qwen3-30B-A3B, RNGD); per-expert main 42.4 vs 17.2", "next": "more rows per call (longer S), hot fraction 0.5, drop the recomputed forward", "state": "active", "done": false, "blocker": "", "metrics": {"step_time_s": "13.0"}, "params": {}, "artifacts": [], "experiments": ["experiments/2026-10-21-moe-batched-hot-experts.md"], "labels": ["experiment", "benchmark"], "review": false, "todos": [], "note": "", "agent": "claude", "host": "DESKTOP-R49IQ68", "branch": "perf/35-batch-hot-experts", "commit": "9d5e498"} {"id": "c37c232f20a045e0aa3315cc1c786106", "ts": "2026-10-21T22:27:58+09:00", "task": "#35 MoE hot experts batched", "result": "hot+keep_graph 5.19 s/step vs host-only+keep 8.00 (paired median -2.66 s, 6/6 rounds) and vs main host-only 9.21 s; Qwen3-30B-A3B S128 RNGD", "next": "", "state": "idle", "done": true, "blocker": "", "metrics": {"step_time_s": "5.19"}, "params": {}, "artifacts": [], "experiments": ["experiments/2026-10-21-moe-batched-hot-experts.md"], "labels": ["experiment", "benchmark"], "review": false, "todos": [], "note": "", "agent": "claude", "host": "DESKTOP-R49IQ68", "branch": "perf/35-hot-experts-faster", "commit": "1fee9f2-dirty"} diff --git a/STATUS.md b/STATUS.md index a7fd770..0bf7315 100644 --- a/STATUS.md +++ b/STATUS.md @@ -1,12 +1,12 @@ # STATUS — RNGD NPU 파인튜닝 학습 -- **State**: idle -- **Updated**: 2026-10-21 22:27+09:00 -- **Agent**: claude@DESKTOP-R49IQ68 -- **Task**: #35 MoE hot experts batched (done) -- **Last result**: hot+keep_graph 5.19 s/step vs host-only+keep 8.00 (paired median -2.66 s, 6/6 rounds) and vs main host-only 9.21 s; Qwen3-30B-A3B S128 RNGD -- **Next**: - +- **State**: active +- **Updated**: 2026-10-21 21:10+09:00 +- **Agent**: claude@control-node +- **Task**: #40 resident AdamW DRAM_SHARED OOM +- **Last result**: cause: furiosa-torch 2026.2 process-global DRAM_SHARED flag raced by layer-worker uploads (not memory); fix: one lock for graph calls + device allocations; 8/8 resident runs 25/25 steps no OOM; 2 RNGD x 8 resident 4.27 s/step (480 tok/s) vs 2-card offload 5.04 s, model -1.1% +- **Next**: 200-step 2-card resident run (#40 done-when); 1-card resident 4 timing at low host load; report the runtime flag to Furiosa - **Blockers**: none @@ -57,6 +57,15 @@ - Experiments: experiments/2026-10-21-moe-batched-hot-experts.md - By: claude@DESKTOP-R49IQ68 · perf/35-batch-hot-experts@9d5e498 +### 2026-10-21 21:10+09:00 — #40 resident AdamW DRAM_SHARED OOM + +- Result: cause: furiosa-torch 2026.2 process-global DRAM_SHARED flag raced by layer-worker uploads (not memory); fix: one lock for graph calls + device allocations; 8/8 resident runs 25/25 steps no OOM; 2 RNGD x 8 resident 4.27 s/step (480 tok/s) vs 2-card offload 5.04 s, model -1.1% +- Next: 200-step 2-card resident run (#40 done-when); 1-card resident 4 timing at low host load; report the runtime flag to Furiosa +- Metrics: tok_s=479.6, step_s=4.27 +- Params: cards=2, resident_layers=8, host_load=46-81, sdk=furiosa-torch-2026.2.0 +- Experiments: experiments/2026-10-21-dram-shared-race.md +- By: claude@control-node · fix/40-dram-shared-race@0e58322-dirty + ### 2026-10-21 20:20+09:00 — #33 Qwen2 gradient check on RNGD - Result: npu-lora check passes (worst 3.1%, was 44%); 0 negative-loss steps in 400 (was 11); loss curve matches A100 diff --git a/docs/compiler-workarounds.md b/docs/compiler-workarounds.md index 5461f18..f624a4e 100644 --- a/docs/compiler-workarounds.md +++ b/docs/compiler-workarounds.md @@ -45,3 +45,20 @@ apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu libc6-dev-arm64 `npu-lora job` 과 `lmd-top --plan train` 이 만드는 Job 은 이 설치를 먼저 한다. 컴파일러는 또 `~/.cache/furiosa/compiler` 를 스스로 만들지 않으므로 백엔드가 미리 만든다. + +## 런타임: 여러 스레드에서 장치 메모리 할당 (#40) + +`torch.OutOfMemoryError: failed to allocate … bytes on DRAM_SHARED` 가 전체 파인튜닝 레이어 작업 스레드의 업로드에서 +간헐적으로 났다(`--resident-layers` 실행, 3–13 step 뒤). 원인은 메모리 부족이 아니다. + +- furiosa-torch 2026.2 는 장치 메모리 종류(DRAM / DRAM_SHARED)를 프로세스 전체에 하나뿐인 플래그로 둔다 + (`furiosa.native_torch.memory._set_dram_shared_mode` / `_set_memory_type`). 한 스레드에서 켜면 다른 스레드도 켜진 값을 본다. +- 컴파일된 그래프 호출은 매번 이 플래그를 잠깐 DRAM_SHARED 로 바꾼다(x·2 그래프 300번을 부르는 동안 다른 스레드가 플래그를 800만 번 읽어 1978번 DRAM_SHARED 를 봄). +- 그 순간 다른 스레드가 장치 텐서를 만들면 DRAM_SHARED 에 잡히는데, 칩 1개 카드에서 그 영역은 0 바이트라 바로 실패한다. + 보통 할당과 그래프 출력은 모두 DRAM(47.5 GiB 한 영역)에 잡힌다. +- 메인 스레드가 그래프를 도는 동안 작업 스레드가 올리는 시간이 길수록 잘 겹친다. 상주 실행은 메인 스레드가 상주 AdamW + 그래프를 돌리는 바로 그때 작업 스레드들이 업로드해서 몇 step 안에 걸렸고, 오프로드만 하는 실행은 덜 겹쳐서 운 좋게 피했다. + +대응: `backends.DEVICE_ALLOC_LOCK` — 그래프 호출(`Serialized`)과 장치 할당(`FuriosaBackend.to_device` 의 `torch.empty`)이 +같은 잠금을 잡는다. 복사(`copy_`)는 잠금 밖이라 업로드 속도는 같다(117 MB bf16: `t.to` 6.05 GB/s, `empty`+`copy_` 6.01 GB/s). +재현: `experiments/data/dram-shared/race_repro.py`(잠금 없이 그래프 6번 만에 OOM, 잠금으로 120 s·그래프 58 799번 동안 없음). diff --git a/docs/design.md b/docs/design.md index 2fb456d..8a438d5 100644 --- a/docs/design.md +++ b/docs/design.md @@ -92,24 +92,29 @@ T_wait = max(0, N_off · (t_a + t_u) / W − T_res − N_res,first · t_fwd) forward waits for nearly all offloaded layers. Resident layers sit at the front of each card's range, so the next forward runs through them (`t_fwd` each) while the host jobs finish. -Check on RNGD, Llama-3.1-8B, S=512, accum 4 (host terms from the one-card offload run of the same host; -records: [resident AdamW](../experiments/2026-10-21-resident-adamw.md), [two cards](../experiments/2026-10-21-two-card-split.md)): +Check on RNGD, Llama-3.1-8B, S=512, accum 4 (host terms from an offload run on the same host at the same load; +records: [resident AdamW](../experiments/2026-10-21-resident-adamw.md), [two cards](../experiments/2026-10-21-two-card-split.md), +[DRAM_SHARED race](../experiments/2026-10-21-dram-shared-race.md)): | host, configuration | predicted | measured | error | | --- | ---: | ---: | ---: | | 96 cores, 1 card, offload, W=8 (per-layer terms of the W=4 run) | 4.53 s | 4.82 s | −6 % | -| 96 cores, 1 card, 4 resident layers (steps 6–12; stopped at 12, #40) | 5.10 s | 5.12 s | −0.4 % | -| 96 cores, 2 cards, 8 resident layers per card (steps 3–7; the run stopped at 7, #40) | 4.33 s | 4.57 s | −5 % | +| 96 cores, 1 card, 4 resident layers (steps 6–12, uncontended; that run stopped at 12, #40) | 5.10 s | 5.12 s | −0.4 % | +| 96 cores at load ~100, 1 card, 4 resident layers (steps 6–25; terms of the 1-card offload run right before) | 10.82 s | 9.72 s | +11 % | +| 96 cores, 2 cards, 8 resident layers per card (steps 6–25; terms of the 2-card offload run back to back) | 4.22 s | 4.27 s | −1 % | | 24 cores, 2 cards, offload | 8.33 s | 9.26 s | −10 % | The link-bandwidth model predicted the last row as 5.6 s (−40 %). What the model says: once half of the layers are resident, `T_wait` is a few hundred ms and `T_dev` -(3.6 s on two cards) is the step. More resident layers only help until `T_wait` reaches zero; after that the +(3.5 s on two cards) is the step: 2 cards × 8 resident layers run at 4.27 s/step, 15 % faster than 2-card offload (5.04 s). More resident layers only help until `T_wait` reaches zero; after that the device compute path, not the optimizer, is the next limit. A card holds about 9 resident 8B layers next to its share of bf16 weights and gradients, so all 32 layers resident needs 4 cards. -Resident layers currently hit an intermittent device out-of-memory error after 3–13 steps -([#40](https://github.com/ai-computing/npu-train/issues/40)). +Resident runs used to stop with an intermittent `OutOfMemoryError ... on DRAM_SHARED` after 3–13 steps +([#40](https://github.com/ai-computing/npu-train/issues/40)). The cause was not memory: furiosa-torch 2026.2 keeps the +device memory type in one process-wide flag that every graph call briefly sets to DRAM_SHARED (0 bytes on a single-chip +card), so a layer worker's upload at that moment failed. Graph calls and device allocations now share one lock +(`backends.DEVICE_ALLOC_LOCK`; see [compiler workarounds](compiler-workarounds.md)). ## Job layout on Kubernetes diff --git a/experiments/2026-10-21-dram-shared-race.md b/experiments/2026-10-21-dram-shared-race.md new file mode 100644 index 0000000..f6e5215 --- /dev/null +++ b/experiments/2026-10-21-dram-shared-race.md @@ -0,0 +1,119 @@ +# 상주 AdamW 실행의 DRAM_SHARED OOM: 원인(프로세스 전역 메모리 종류 플래그 경합)과 수정 + +- **Date**: 2026-10-21 +- **Task**: #40 (원인·수정), #34 (상주 실행 25 step 측정) +- **Host / device**: npu-node-1 RNGD(카드 3장 중 pod 가 배정받은 1–2장), 노드 CPU 96코어, `furiosaai/furiosa-llm:latest` + (furiosa-torch 2026.2.0, torch 2.10.0+cpu, transformers 5.1.0), 드라이버·펌웨어 2026.4.0. + 실행마다 직전에 `/proc/*/fd` 에서 `/dev/rngd` 를 연 프로세스가 없음을 확인했다(`runall.log`; 한 번은 다른 사용자의 프로세스가 + npu3 를 8분 열고 있어서 닫힐 때까지 기다렸다). npu-node-2 는 쓰지 않았다. +- **Commit**: 브랜치 `fix/40-dram-shared-race`(main `a07aefc` 위). pod 에는 작업 트리의 `npu_lora/` 를 ConfigMap 으로 올렸다. +- **Host load**: §1 의 학습 실행은 다른 사용자의 CPU 작업으로 load 가 350–650(96코어)인 동안 돌아서 step 시간은 신뢰성 확인용이다. + §2 는 load 가 내려간 뒤(2장 27–81, 1장 27–111) 쟀다. load 는 `data/dram-shared/load-npu-node-1.txt`(60 s 간격). + +## Question + +1. `failed to allocate … bytes on DRAM_SHARED` 는 무엇이 차서 나는가? DRAM_SHARED 는 DRAM 과 다른 영역인가, 크기는? +2. 고칠 수 있는가? 고친 뒤 상주 실행(2장 × 상주 8, 1장 × 상주 4)이 25 step 을 3번 연속 끝내는가? + +## Setup + +가설을 싼 것부터 시험했다. + +1. **영역 크기** (`data/dram-shared/pool_probe.py`, 카드 1장): 64 MB bf16 업로드를 계속 쌓으면 47.44 GiB 에서 + `… on DRAM` 으로 멈춘다. 그래프 출력(x·2, 64 MB)을 쌓아도 47.38 GiB 에서 `DRAM`. 업로드 8 GiB 를 잡고 그래프 출력을 쌓으면 + 39.38 GiB 에서 멈춘다 — 업로드와 그래프 출력은 같은 DRAM 한 영역(47.5 GiB, `capacity: 51002736640`)에 잡힌다. + DRAM_SHARED 에는 아무것도 잡히지 않는다. 실패 메시지의 `capacity: 0` 이 DRAM_SHARED 영역 크기다(칩 1개 카드). +2. **누가 DRAM_SHARED 를 쓰는가**: `furiosa.native_torch.memory` 에 `_set_dram_shared_mode()` / `_set_memory_type()` 이 있고, + `furiosa.torch.utils.with_dram_shared` 가 이것으로 할당 종류를 잠깐 바꾼다. 이 플래그가 스레드별인지 시험했다 + (`data/dram-shared/race_repro.py`). + - `flag`: 다른 스레드에서 켠 DRAM_SHARED 모드를 메인 스레드도 본다(`True`). 그 상태에서 메인 스레드의 8 MB 업로드는 + `OutOfMemoryError: failed to allocate 8388608 bytes on DRAM_SHARED` — #40 과 같은 오류다. **플래그는 프로세스 전역이다.** + - `watch`: 메인 스레드가 x·2 그래프를 300번 부르는 동안 다른 스레드가 플래그를 7 994 000번 읽어 1 978번 DRAM_SHARED 를 봤다 + (상주 AdamW 그래프 30번: 1 763 563번 중 72번). **모든 그래프 호출이 플래그를 잠깐 DRAM_SHARED 로 바꾼다.** + - `race`: 메인 스레드는 그래프, 4 스레드는 [1024, 4096] bf16 업로드를 반복 → 그래프 6번 만에 DRAM_SHARED OOM. + 상주 AdamW 그래프 [14336, 4096] 와 같이 돌려도 3 s 안에 같은 OOM. +3. **결론 — 원인**: 학습 루프는 메인 스레드에서 그래프를, 레이어 작업 스레드(`--update-workers`, 기본 4)에서 bf16 가중치 업로드를 + 돌린다. 업로드의 장치 할당이 다른 스레드의 그래프 호출 순간과 겹치면 DRAM_SHARED(0 바이트)에 잡혀 실패한다. 메모리 부족·누수·조각화가 + 아니다. 상주 실행은 step 끝에 메인 스레드가 상주 AdamW 그래프(카드당 상주 층 × 7개)를 연달아 부르는 바로 그때 작업 스레드들이 + 업로드를 시작해서 겹침이 많았고, 오프로드만 하는 실행은 메인 스레드가 대부분 업로드 future 를 기다리느라 겹침이 적어 피했다 + (같은 경합이 확률만 낮을 뿐 있다). 상주 층 수·step 수와 상관없이 "몇 step 뒤 간헐적"인 것도 이것으로 설명된다. +4. 기각한 가설: 업로드 스테이징 영역 고갈(업로드는 DRAM 에 직접 잡히고, 한 스레드에서 400회 반복해도 늘지 않았다 — 10-21 기록), + 조각화(DRAM 은 47.4 GiB 까지 채워지고 실패는 다른 영역), 참조로 살아 있는 버퍼(실패 영역이 비어 있다). + +**수정** (`npu_lora/backends.py`): `DEVICE_ALLOC_LOCK` 하나를 그래프 호출(`Serialized` 래퍼, `FuriosaBackend.compile` 이 돌려줌)과 +장치 할당(`FuriosaBackend.to_device` 의 `torch.empty(device=…)`)이 함께 잡는다. 복사(`copy_`)는 잠금 밖이다. +업로드 속도는 같다(117 MB bf16, 중앙값: `t.to(dev)` 19.4 ms = 6.05 GB/s, `empty` + `copy_` 19.5 ms = 6.01 GB/s, 값 동일). +`race_repro.py fixed`: 같은 경합을 120 s 동안 그래프 58 799번·업로드 60 725번 돌려 오류 0. 상주 AdamW 그래프로도 120 s, 그래프 +35 557번·업로드 55 075번, 업로드 검사(500번마다 내려 비교) 불일치 0. + +## Command + +```bash +# 재현 (카드 1장 pod, PYTHONPATH=npu-train) +python3 experiments/data/dram-shared/race_repro.py flag|watch|race 60|fixed 120 +python3 experiments/data/dram-shared/pool_probe.py upload|graph|"mix 8"|graphfirst +# 학습 (10-21 resident-adamw 기록과 같은 레시피) +python3 -m npu_lora.cli train --full --model meta-llama/Llama-3.1-8B-Instruct --data /mnt/store/datasets/quality/nsmc_mix_train.jsonl \ + --cache-dir /cache/furiosa --seq-len 512 --steps 25 --accum 4 --lr 1e-5 --seed 0 --run-name ntc- --out /tmp/out \ + --backend furiosa:auto,furiosa:auto --resident-layers 8 # 1장: --backend furiosa:auto --resident-layers 4 +``` + +## Results + +### 1. 신뢰성 — 고친 코드로 연속 실행 (16:26–17:40 KST, host load 350–650) + +| 실행 | 카드 | 상주/카드 | step | OOM | step(6–25 중앙값) | tok/s | 대기 | 상주 갱신 | load1 중앙값 | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| r2-res8-a | 2 | 8 | 25/25 | 0 | 8.13 s | 252 | 0.39 | 0.70 | 501 | +| r2-res8-b | 2 | 8 | 25/25 | 0 | 9.31 s | 220 | 0.49 | 0.68 | 384 | +| r2-res8-c | 2 | 8 | 25/25 | 0 | 9.47 s | 216 | 0.56 | 0.71 | 352 | +| r1-res4-a | 1 | 4 | 25/25 | 0 | 9.61 s | 213 | 2.05 | 0.19 | 384 | +| r1-res4-b | 1 | 4 | 25/25 | 0 | 9.75 s | 210 | 2.13 | 0.15 | 429 | +| r1-res4-c | 1 | 4 | 25/25 | 0 | 12.27 s | 167 | 2.59 | 0.27 | 656 | + +- 고치기 전 같은 구성은 2장 × 상주 8 이 5번 중 4번(3, 7, 7, 13 step), 1장 × 상주 4 가 1번 중 1번(12 step) OOM 으로 멈췄다 + (10-21 resident-adamw 기록 §4). 고친 뒤 6번 모두 25 step 을 끝냈다. +- loss 는 고치기 전 상주 실행과 비트 단위로 같다(2장 상주 8: 1.7157, 1.5649, 1.0858, 1.1035, 1.3369; 1장 상주 4: 1.7157, 1.5645, + 1.0821, 1.1063, 1.3412). 같은 구성 세 번도 25 step 모두 같다 — 잠금은 결과를 바꾸지 않는다. +- step 시간은 load 에 따라 8.1–12.3 s 로 흔들린다(host AdamW 합 7.9–11.7 s). 성능 비교와 #34 의 모델 검증은 아래 §2 에서 낮은 load 로 한다. + +### 2. 낮은 load 측정 (#34) + +load 가 내려가기를 기다려(`timing.log`: 실행 직전 1분 load < 150, 카드 2장이 쿠버네티스에서 비어 있고 열린 카드는 모두 k8s 배정분) 한 Job 에서 +2장 상주 8 과 2장 오프로드를 이어서 돌렸고, 이어서 1장 오프로드와 1장 상주 4 를 돌렸다. 6–25 step 중앙값. + +| 실행 | step 구간(KST) | load1 | step | tok/s | 대기 | D2H 합 | AdamW 합 | H2D 합 | 상주 갱신 | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:| +| 2장 상주 8 (`ntc-t2-res8`) | 19:38–19:40 | 46–81 | **4.27 s** | 480 | 0.24 | 1.19 | 4.01 | 1.28 | 0.48 | +| 2장 오프로드 (`ntc-t2-off`) | 19:43–19:46 | 27–71 | **5.04 s** | 407 | 1.56 | 2.41 | 4.10 | 2.48 | — | +| 1장 오프로드 (`ntc-t1-off`) | 19:51–19:55 | 71–107 | 10.66 s | 192 | 2.47 | 4.08 | 9.30 | 3.13 | — | +| 1장 상주 4 (`ntc-t1-res4`) | 20:02–20:06 | 27–111 | 9.72 s | 211 | 1.62 | 2.80 | 8.65 | 2.80 | 0.22 | + +- **2장 + 카드당 상주 8: 4.27 s/step, 480 tok/s — 같은 시각 2장 오프로드(5.04 s)보다 −15 %(+18 % tok/s).** 25 step 전 구간 값이고, + OOM 으로 5 step 만 쟀던 10-21 값(4.57 s, steps 3–7)을 대신한다. 오프로드 대기가 1.56 → 0.24 s 로 줄었다. +- 1장 실행 두 개는 load 가 다시 100 근처(96코어)로 올라간 동안 돌았다. host AdamW 합이 9.3 s 로 경합 없을 때(4.18 s, 10-21 기록)의 두 배를 + 넘고, 장치 경로(step − 대기)도 8.2 s 로 경합 없을 때(3.7 s)의 두 배다 — 메인 스레드도 CPU 를 기다린다. 그래서 1장 숫자는 경합 조건의 값이다. + load < 60 에서 1장 두 실행을 다시 하려고 20:10–21:02 동안 10분 간격으로 6번 확인했지만 load 가 84–126 이라 그만뒀다. +- loss 는 상주 실행끼리, 오프로드 실행끼리 §1·10-21 기록과 비트 단위로 같다. + +**step 모델 확인** (docs/design.md 「Optimizer placement: step-time model」). 호스트 항은 같은 시각·같은 카드 구성의 오프로드 실행에서 잰다: +`T_dev` = step − 대기, `t_a` = AdamW 합 / 32, `t_u` = H2D 합 / 32, W = 4, 장치 AdamW 28 ms/층, `t_fwd` = T_dev / (32 × 4 × 3). + +| 구성 | T_dev | t_a + t_u | T_res | T_wait | 예측 | 측정 | 오차 | +|---|---:|---:|---:|---:|---:|---:|---:| +| 2장 상주 8 (2장 오프로드 항, load ~50–70) | 3.47 | 0.128 + 0.077 | 0.45 | 0.30 | **4.22 s** | 4.27 s | **−1.1 %** | +| 1장 상주 4 (1장 오프로드 항, load ~100) | 8.18 | 0.291 + 0.098 | 0.11 | 2.52 | 10.82 s | 9.72 s | +11.3 % | + +두 구성 모두 15 % 안이다. 1장 줄은 두 실행 사이에 load 가 흔들려(27–111) 오차가 더 크다. 측정한 장치 AdamW 는 2장 29.8 ms/층, 1장(경합) +54.5 ms/층이다 — 메인 스레드에서 도는 그래프 호출도 host 경합의 영향을 받는다. + +Raw: `experiments/data/dram-shared/*.jsonl`(step 기록; `t2.jsonl` 에 `ntc-t2-res8`·`ntc-t2-off` 두 실행), `runall.log`·`timing.log`(실행 전 카드 점유·load 확인과 결과), +`load-npu-node-1.txt`(host load), `race_repro.py`, `pool_probe.py`. + +## Conclusion + +#40 의 OOM 은 메모리 부족이 아니라 furiosa-torch 2026.2 의 프로세스 전역 메모리 종류 플래그를 그래프 호출과 다른 스레드의 장치 할당이 +경합해서 났다. 그래프 호출과 장치 할당을 한 잠금으로 묶어 고쳤고(업로드 속도 변화 없음), 고친 뒤 상주 실행 6번(2장 × 8 세 번, +1장 × 4 세 번, 낮은 load 에서 두 번 더)이 모두 25 step 을 끝냈다. +낮은 load 에서 2장 × 상주 8 은 4.27 s/step(480 tok/s)으로 같은 시각 2장 오프로드(5.04 s)보다 15 % 빠르고, host 처리량 step 모델이 +−1.1 % 로 맞췄다. 1장 × 상주 4 는 load ~100 에서만 쟀다(9.72 s, 모델 +11 %). 런타임 쪽 문제(플래그가 스레드별이 아님)는 재현 스크립트와 함께 Furiosa 에 알릴 몫이다. diff --git a/experiments/2026-10-21-resident-adamw.md b/experiments/2026-10-21-resident-adamw.md index 0adc7bd..878d001 100644 --- a/experiments/2026-10-21-resident-adamw.md +++ b/experiments/2026-10-21-resident-adamw.md @@ -111,7 +111,8 @@ bf16 업로드(`to_device`)다. - `furiosa-smi status` 는 실행 내내 0.00/47.50 GiB 로 보여서 밖에서 메모리를 볼 수 없었다. - 상주 갱신을 오프로드 업로드보다 먼저 돌리는 순서(조각화 가설)도 시험했지만 결과가 섞여(2장 상주 8: 13, 완주; 2장 상주 6: 7) 되돌렸다. `r2-res8-try3`, `r2-res8-mem`, `r2-res6`, `r1-res3` 이 그 변형이고, 나머지와 PR 코드는 오프로드 업로드와 상주 갱신이 겹치는 원래 순서다. - 원인은 모른다 — #40 에서 따로 본다. + 원인은 모른다 — #40 에서 따로 본다. (후속: 메모리 부족이 아니라 프로세스 전역 메모리 종류 플래그 경합이었다 — + [2026-10-21-dram-shared-race.md](2026-10-21-dram-shared-race.md).) ### 5. 경합 host 에서 같은 조건 비교 (15:00–15:25 KST) diff --git a/experiments/data/dram-shared/load-npu-node-1.txt b/experiments/data/dram-shared/load-npu-node-1.txt new file mode 100644 index 0000000..9e6d86e --- /dev/null +++ b/experiments/data/dram-shared/load-npu-node-1.txt @@ -0,0 +1,243 @@ +1792566418 353.38 364.32 313.74 320/7047 8 +1792566478 337.39 357.66 314.61 307/7020 10 +1792566538 410.19 373.78 322.79 473/6969 12 +1792566598 428.18 385.63 330.06 470/6789 14 +1792566658 447.39 399.05 338.15 426/7139 16 +1792566718 464.66 412.14 346.42 498/7124 18 +1792566778 479.73 425.86 355.25 465/7244 20 +1792566838 472.20 434.34 362.63 369/7253 22 +1792566898 469.44 439.51 368.84 432/7219 24 +1792566958 475.61 448.02 376.26 519/7191 26 +1792567018 485.73 455.66 383.36 453/7107 28 +1792567078 484.02 459.79 389.26 511/7137 30 +1792567138 479.36 463.25 394.89 458/7332 32 +1792567198 474.14 464.31 399.52 469/7027 34 +1792567258 471.54 465.43 403.96 445/7081 36 +1792567318 463.18 463.82 407.24 468/6715 38 +1792567378 468.67 465.49 411.37 467/6720 40 +1792567438 461.56 464.16 414.31 471/6785 42 +1792567498 451.50 461.23 416.45 476/6679 44 +1792567558 467.69 463.89 420.16 403/6892 46 +1792567618 447.29 459.57 421.46 557/6943 48 +1792567678 450.13 457.84 423.24 417/7262 50 +1792567738 452.53 457.11 425.15 453/7336 52 +1792567798 471.37 461.00 428.45 474/7261 54 +1792567858 500.31 470.81 433.84 470/11914 56 +1792567918 501.40 476.70 438.19 515/10842 58 +1792567978 505.74 480.87 441.95 554/11847 60 +1792568038 487.43 481.30 444.60 498/12462 62 +1792568098 501.68 486.82 448.81 523/7495 64 +1792568158 489.68 486.87 451.26 468/6939 66 +1792568218 413.55 467.57 446.97 376/6641 68 +1792568278 378.47 446.91 441.12 447/7718 70 +1792568338 375.86 434.56 437.28 265/7773 72 +1792568398 359.96 418.59 431.57 428/8058 74 +1792568458 392.80 417.55 430.41 290/11851 76 +1792568518 375.36 409.69 427.00 333/11499 78 +1792568578 356.92 398.00 421.89 243/12101 80 +1792568638 399.54 400.33 421.05 548/11966 82 +1792568698 425.49 408.16 422.47 375/7669 84 +1792568758 406.30 407.15 421.32 294/7592 86 +1792568818 372.20 396.95 416.92 329/7340 88 +1792568878 382.95 394.17 414.64 423/7435 90 +1792568938 348.96 383.21 409.64 332/7196 92 +1792568998 336.14 372.03 404.08 333/7367 94 +1792569058 339.62 367.17 400.44 241/7379 96 +1792569118 317.67 355.60 394.37 362/7518 98 +1792569178 321.30 349.06 389.65 331/6762 100 +1792569238 351.57 353.93 388.83 462/10244 102 +1792569298 349.79 352.85 386.27 298/10699 104 +1792569358 356.98 354.48 384.74 415/11439 106 +1792569418 352.89 354.13 382.74 418/10511 108 +1792569478 385.02 362.62 383.84 407/7021 110 +1792569538 389.26 368.48 384.55 412/6914 112 +1792569598 311.43 349.77 377.11 286/6632 114 +1792569658 251.55 326.09 367.29 357/6567 116 +1792569718 279.68 321.45 363.16 317/6664 118 +1792569778 335.66 328.54 362.90 348/6878 120 +1792569838 376.46 342.38 365.57 374/6716 122 +1792569898 358.94 344.60 364.94 213/7581 124 +1792569958 329.46 338.84 361.68 274/6528 126 +1792570018 298.08 327.44 356.29 377/6725 128 +1792570078 341.15 335.05 357.13 304/6699 130 +1792570138 387.80 348.98 360.50 407/6754 132 +1792570198 394.04 356.78 362.40 430/6728 134 +1792570258 401.27 364.34 364.56 329/6713 136 +1792570318 367.75 361.28 363.49 426/6685 138 +1792570378 360.54 360.08 362.92 371/6645 140 +1792570438 355.35 358.17 362.06 378/6651 142 +1792570498 357.17 358.18 361.82 377/6664 144 +1792570558 384.25 365.99 364.29 333/10830 146 +1792570618 365.45 364.71 364.00 414/10581 148 +1792570678 383.61 369.51 365.67 454/10253 150 +1792570738 419.42 381.66 370.04 407/9989 152 +1792570798 413.12 386.90 372.59 426/6718 154 +1792570858 414.47 391.28 374.95 433/6712 156 +1792570918 388.41 389.31 375.36 259/6460 158 +1792570978 352.24 378.08 372.34 389/6778 160 +1792571038 379.59 380.70 373.58 441/6778 162 +1792571098 420.32 392.04 377.91 416/8115 164 +1792571158 426.04 398.65 381.05 398/11158 166 +1792571218 439.04 407.22 385.09 429/11270 168 +1792571278 430.39 410.26 387.50 461/10929 170 +1792571338 428.93 413.23 389.93 440/9519 172 +1792571398 426.19 415.73 392.28 434/6714 174 +1792571458 507.17 442.12 402.88 480/7316 176 +1792571518 464.19 442.12 405.38 420/7025 178 +1792571578 438.87 439.62 406.86 494/6879 180 +1792571638 429.32 436.64 407.89 460/6790 182 +1792571698 464.12 444.74 412.45 562/10651 184 +1792571758 588.24 480.62 426.60 642/10126 186 +1792571818 656.08 521.84 444.27 623/10961 188 +1792571878 803.34 581.28 469.13 1089/11952 190 +1792571938 925.69 655.04 501.40 899/12321 192 +1792571999 983.64 720.18 533.35 825/8718 194 +1792572059 981.18 768.03 561.52 847/8358 196 +1792572119 987.21 808.67 588.41 1076/8577 198 +1792572179 1052.19 863.74 621.35 887/8361 200 +1792572239 824.59 836.51 627.40 699/8374 202 +1792572299 763.13 815.83 633.31 736/8584 204 +1792572359 757.75 804.77 640.94 704/8557 206 +1792572419 695.75 779.91 642.74 782/8332 208 +1792572479 665.78 754.36 642.45 825/8162 210 +1792572539 679.10 741.39 644.96 773/8584 212 +1792572599 700.20 740.31 650.87 585/8082 214 +1792572659 660.29 721.21 649.88 674/11059 216 +1792572719 718.39 726.67 656.22 848/10777 218 +1792572779 756.66 738.48 664.82 632/7982 220 +1792572839 637.40 707.67 658.84 529/7768 222 +1792572899 563.15 672.70 649.82 564/7550 224 +1792572959 550.08 649.90 643.48 597/7681 226 +1792573019 594.44 643.60 641.66 675/7680 228 +1792573079 613.84 640.40 640.70 692/7755 230 +1792573139 590.25 630.89 637.53 565/7827 232 +1792573199 596.17 623.89 634.62 622/7705 234 +1792573259 631.43 631.53 636.71 498/7592 236 +1792573319 536.07 604.87 627.27 485/7520 238 +1792573379 515.24 586.47 619.54 516/7654 240 +1792573439 545.37 582.66 616.19 458/7641 242 +1792573499 518.91 569.29 609.56 461/7240 244 +1792573559 497.31 554.59 602.02 501/7262 246 +1792573619 487.51 542.12 594.79 397/7401 248 +1792573679 467.99 528.00 586.73 425/7232 250 +1792573739 436.62 507.07 575.79 349/7026 252 +1792573799 414.74 487.61 564.78 395/6892 254 +1792573859 380.83 465.27 552.31 366/6880 256 +1792573919 357.36 444.70 539.87 168/6867 258 +1792573979 341.52 422.98 526.37 327/6708 260 +1792574039 354.25 411.70 516.01 204/6649 262 +1792574099 318.22 392.47 502.95 345/6515 264 +1792574159 345.43 386.49 493.95 279/6694 266 +1792574219 313.28 369.95 481.55 320/7159 268 +1792574279 326.80 362.62 471.99 340/7124 270 +1792574339 333.01 357.24 463.25 378/7364 272 +1792574399 353.86 359.65 457.49 453/7108 274 +1792574459 396.57 371.73 455.57 302/7104 276 +1792574519 336.56 359.38 446.11 339/6302 278 +1792574579 285.41 342.29 434.90 293/6641 280 +1792574639 285.13 332.59 425.80 222/6622 282 +1792574700 269.50 319.69 415.55 209/6608 284 +1792574760 270.44 309.68 406.04 406/6750 286 +1792574820 323.00 317.98 402.88 369/7101 288 +1792574880 331.08 321.52 398.81 299/7168 290 +1792574940 294.34 313.19 391.14 337/6909 292 +1792575000 267.23 302.29 382.54 273/7032 294 +1792575060 275.20 298.69 376.30 357/7118 296 +1792575120 307.16 302.21 372.56 256/6844 298 +1792575180 310.92 303.99 368.75 188/6855 300 +1792575240 266.35 294.09 361.39 184/6605 302 +1792575300 243.08 282.22 353.09 192/6376 304 +1792575360 257.41 279.66 347.79 147/6639 306 +1792575420 256.58 273.78 341.41 310/6175 308 +1792575480 243.97 268.69 335.53 293/6191 310 +1792575540 235.16 261.68 328.93 250/6184 312 +1792575600 211.06 251.05 321.14 194/6190 314 +1792575660 196.68 239.61 312.82 193/6384 316 +1792575720 187.63 230.61 305.21 100/5517 318 +1792575780 146.48 210.69 293.67 120/6201 320 +1792575840 210.09 217.43 290.79 163/6199 322 +1792575900 174.95 206.90 282.63 104/6055 324 +1792575960 167.40 197.11 274.42 261/6607 326 +1792576020 214.42 204.68 272.15 213/5878 328 +1792576080 187.11 199.10 266.03 148/6108 330 +1792576140 189.02 196.59 260.93 192/6198 332 +1792576200 232.79 208.23 260.93 264/6207 334 +1792576260 246.26 216.65 260.53 258/6167 336 +1792576320 252.58 223.70 260.20 196/6159 338 +1792576380 253.66 228.74 259.61 248/6150 340 +1792576440 258.43 235.00 259.85 213/6136 342 +1792576500 249.97 236.20 258.67 265/6144 344 +1792576560 250.80 239.02 258.24 249/6149 346 +1792576620 243.08 240.11 257.47 177/5521 348 +1792576680 210.98 231.41 253.39 165/5653 350 +1792576740 210.80 227.55 250.68 228/6273 352 +1792576800 212.24 224.86 248.31 177/6790 354 +1792576860 176.63 212.38 242.56 126/5882 356 +1792576920 160.35 201.33 236.87 103/6156 358 +1792576980 146.84 190.73 231.04 101/5443 360 +1792577040 128.30 177.31 223.90 134/5548 362 +1792577100 167.43 179.41 221.69 192/5549 364 +1792577160 181.33 181.31 219.69 153/5550 366 +1792577220 184.67 182.49 217.71 193/5538 368 +1792577280 185.72 183.34 215.81 111/5526 370 +1792577340 176.71 181.03 212.97 167/5521 372 +1792577400 178.40 180.45 210.75 187/5526 374 +1792577460 174.62 179.29 208.47 162/5522 376 +1792577520 172.50 178.18 206.28 188/5448 378 +1792577580 182.58 179.96 205.13 182/5449 380 +1792577640 173.26 176.61 202.33 182/5490 382 +1792577700 201.65 183.90 203.21 223/5775 384 +1792577760 154.65 174.81 198.95 102/7456 386 +1792577820 141.12 165.87 194.27 186/5393 388 +1792577880 163.98 168.04 193.24 183/5397 390 +1792577940 173.91 169.88 192.28 191/5389 392 +1792578000 180.40 172.38 191.73 194/5355 394 +1792578060 184.47 174.55 191.24 139/5474 396 +1792578120 180.05 175.75 190.64 185/5668 398 +1792578180 180.07 176.73 190.05 223/10110 400 +1792578240 155.49 170.19 186.97 177/10060 402 +1792578300 143.00 165.95 184.59 101/10164 404 +1792578360 128.02 157.16 180.38 148/10034 406 +1792578420 107.10 147.59 175.73 100/13543 408 +1792578480 59.83 127.61 167.16 5/20159 410 +1792578540 73.38 119.72 161.98 30/18174 412 +1792578600 82.27 112.61 156.82 101/10252 414 +1792578660 91.00 109.24 152.87 100/10065 416 +1792578720 97.44 107.93 149.70 100/10828 418 +1792578780 97.81 105.99 146.41 99/10814 420 +1792578840 99.06 104.90 143.50 112/9865 422 +1792578900 102.06 104.62 140.98 98/10229 424 +1792578960 103.00 104.45 138.65 103/10153 426 +1792579020 94.16 102.27 135.79 7/15109 428 +1792579080 80.73 97.79 132.19 103/11019 430 +1792579140 71.30 92.99 128.44 41/25770 432 +1792579200 45.53 82.60 122.70 11/23851 434 +1792579260 43.48 74.75 117.46 5/19292 436 +1792579320 38.30 67.99 112.48 7/19897 438 +1792579380 32.65 61.26 107.40 5/19677 440 +1792579440 70.90 63.94 105.25 11/17927 442 +1792579500 52.63 60.68 101.58 66/23254 444 +1792579560 27.28 51.84 95.97 123/22983 446 +1792579620 53.96 53.75 93.79 101/10988 448 +1792579680 68.13 57.41 92.51 104/14633 450 +1792579740 59.54 56.88 90.14 104/5577 452 +1792579800 38.65 52.01 86.42 18/5340 454 +1792579860 71.15 58.14 86.33 157/5340 456 +1792579920 99.85 69.03 88.33 101/8003 458 +1792579980 94.64 73.14 88.54 100/9571 460 +1792580040 100.85 79.13 89.65 102/7585 462 +1792580100 107.31 85.25 91.12 100/9192 464 +1792580160 149.66 100.91 96.10 97/5606 466 +1792580220 118.83 100.92 96.40 98/5393 468 +1792580280 99.56 99.04 96.07 97/5371 470 +1792580340 92.51 97.74 95.84 3/5417 472 +1792580400 78.79 93.74 94.63 5/5406 474 +1792580460 31.53 77.42 88.96 52/5403 476 +1792580520 27.25 67.30 84.73 63/5378 478 +1792580580 94.76 78.50 87.44 101/6803 480 +1792580640 97.59 82.52 88.28 100/8220 482 +1792580700 96.68 84.57 88.60 100/9198 484 +1792580760 110.98 90.45 90.34 110/9512 486 +1792580820 102.86 91.86 90.83 97/5552 488 +1792580880 101.28 93.54 91.49 104/5372 490 +1792580940 114.42 97.73 93.00 149/5485 492 diff --git a/experiments/data/dram-shared/pool_probe.py b/experiments/data/dram-shared/pool_probe.py new file mode 100644 index 0000000..1df90d4 --- /dev/null +++ b/experiments/data/dram-shared/pool_probe.py @@ -0,0 +1,133 @@ +"""#40 probe: what lives in DRAM vs DRAM_SHARED on RNGD, and how big each pool is. + +usage: PYTHONPATH= python3 pool_probe.py + info furiosa-torch memory API / env strings + upload keep uploading 64 MB bf16 tensors (to_device) until OOM + graph keep graph outputs (x*2, [4096,4096] fp32 = 64 MB) until OOM + mix N hold N GiB of uploads, then fill with graph outputs (one pool or two?) + graphfirst fill with graph outputs, then upload +""" +import glob +import os +import re +import subprocess +import sys +import time + +import torch + +mode = sys.argv[1] +BF = torch.bfloat16 + + +def say(*a): + print(*a, flush=True) + + +if mode == "info": + import furiosa.torch as ft + say("furiosa.torch attrs:", [x for x in dir(ft) if re.search("mem|alloc|stat|cache|pool|dram", x, re.I)]) + for name in ("furiosa", "rngd"): + m = getattr(torch, name, None) + if m is not None: + say(f"torch.{name}:", [x for x in dir(m) if re.search("mem|alloc|stat|cache|pool", x, re.I)]) + base = os.path.dirname(ft.__file__) + say("pkg dir", base) + sos = [p for p in glob.glob(os.path.join(os.path.dirname(base), "**", "*.so*"), recursive=True) + if "furiosa" in p] + for p in sos: + say("so", p, os.path.getsize(p)) + pat = re.compile(rb"[ -~]{4,}") + seen = set() + for p in sos: + data = open(p, "rb").read() + for s in pat.findall(data): + t = s.decode() + if re.search(r"DRAM_SHARED|dram.?shared|shared.?dram|SharedDram|DramShared", t, re.I) or \ + re.search(r"^(FURIOSA|RNGD|NPU|FURIOSA_TORCH|NPU_)[A-Z0-9_]{3,}$", t): + if t not in seen and len(t) < 300: + seen.add(t) + say("str", os.path.basename(p), t) + sys.exit(0) + +from npu_lora import backends, graph as G # noqa: E402 + +be = backends.make("furiosa:auto", cache_dir="/cache/furiosa") +dev = be.dev +say("device", dev, be.describe()) +for k in ("memory_allocated", "memory_reserved", "memory_stats", "mem_get_info"): + for m in (getattr(torch, "furiosa", None), getattr(torch, "rngd", None)): + f = getattr(m, k, None) if m is not None else None + if f: + try: + say("api", k, f()) + except Exception as e: # noqa: BLE001 + say("api", k, "error", repr(e)[:200]) +MB = 1 << 20 +keep = [] + + +def fill_uploads(limit_gb=None, shape=(4096, 8192)): + t = torch.zeros(shape, dtype=BF) + n = 0 + t0 = time.time() + try: + while limit_gb is None or n * t.numel() * 2 < limit_gb * (1 << 30): + keep.append(be.to_device(t)) + n += 1 + if n % 50 == 0: + say(f"upload {n} x {t.numel() * 2 // MB} MB = {n * t.numel() * 2 / (1 << 30):.2f} GiB {time.time() - t0:.0f}s") + except Exception as e: # noqa: BLE001 + say(f"upload OOM after {n} x {t.numel() * 2 // MB} MB = {n * t.numel() * 2 / (1 << 30):.2f} GiB: {repr(e)[:200]}") + return n, True + return n, False + + +def graph_fn(): + smp = [torch.zeros(4096, 4096)] + return G.compile_parts(be, "probe-x2-4096", G.build(lambda x: (x * 2.0,), smp), smp, max_mm=0, log=say) + + +def fill_graph(fn, x): + outs, n = [], 0 + try: + while True: + out = fn(x) + outs.append(out[0] if isinstance(out, (tuple, list)) else out) + n += 1 + if n % 50 == 0: + say(f"graph {n} x 64 MB = {n / 16:.2f} GiB") + except Exception as e: # noqa: BLE001 + say(f"graph OOM after {n} x 64 MB = {n / 16:.2f} GiB: {repr(e)[:200]}") + return outs + + +if mode == "upload": + fill_uploads() +elif mode == "graph": + fn = graph_fn() + x = be.to_device(torch.ones(4096, 4096)) + fill_graph(fn, x) +elif mode == "mix": + # DRAM_SHARED first, then graph outputs: are they the same pool? + fn = graph_fn() + x = be.to_device(torch.ones(4096, 4096)) + lim = float(sys.argv[2]) if len(sys.argv) > 2 else 8 + n, oom = fill_uploads(lim) + say(f"holding {n} uploads ({lim} GiB)") + outs = fill_graph(fn, x) + say("graph outputs kept", len(outs)) + # free graph outputs, try uploads again + outs = None + n2, _ = fill_uploads() + say("uploads after freeing graph outputs", n2) +elif mode == "graphfirst": + fn = graph_fn() + x = be.to_device(torch.ones(4096, 4096)) + outs = fill_graph(fn, x) + say("holding graph outputs", len(outs)) + n2, _ = fill_uploads() + outs = outs[: len(outs) // 2] + say("freed half the graph outputs") + n3, _ = fill_uploads() +say("DONE", mode) diff --git a/experiments/data/dram-shared/r1-res4-a.jsonl b/experiments/data/dram-shared/r1-res4-a.jsonl new file mode 100644 index 0000000..1a47ca2 --- /dev/null +++ b/experiments/data/dram-shared/r1-res4-a.jsonl @@ -0,0 +1,27 @@ +{"ts": 1792570082.469, "run": "ntc-r1-res4-a", "event": "start", "examples": 8000} +{"ts": 1792570536.283, "run": "ntc-r1-res4-a", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 17.614, "tokens_per_s": 116.3, "wait_weights_s": 0.001, "norm_s": 0.207, "prev_d2h_s": 0.0, "prev_adamw_s": 0.138, "prev_upload_s": 0.0, "resident_update_s": 0.139} +{"ts": 1792570552.17, "run": "ntc-r1-res4-a", "event": "step", "step": 2, "loss": 1.5645148754119873, "grad_norm": 45.859045193746276, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 15.884, "tokens_per_s": 128.9, "wait_weights_s": 8.071, "norm_s": 0.285, "prev_d2h_s": 15.507, "prev_adamw_s": 19.787, "prev_upload_s": 3.482, "resident_update_s": 0.22} +{"ts": 1792570561.253, "run": "ntc-r1-res4-a", "event": "step", "step": 3, "loss": 1.0821138471364975, "grad_norm": 80.97597200661254, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 9.079, "tokens_per_s": 225.6, "wait_weights_s": 1.516, "norm_s": 0.171, "prev_d2h_s": 2.722, "prev_adamw_s": 8.009, "prev_upload_s": 2.488, "resident_update_s": 0.104} +{"ts": 1792570570.599, "run": "ntc-r1-res4-a", "event": "step", "step": 4, "loss": 1.1063324678689241, "grad_norm": 20.461211574277034, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 9.343, "tokens_per_s": 219.2, "wait_weights_s": 1.809, "norm_s": 0.282, "prev_d2h_s": 2.852, "prev_adamw_s": 8.37, "prev_upload_s": 2.608, "resident_update_s": 0.213} +{"ts": 1792570579.444, "run": "ntc-r1-res4-a", "event": "step", "step": 5, "loss": 1.3412490785121918, "grad_norm": 13.312377605287216, "lr": 5e-06, "tokens": 2048, "seconds": 8.84, "tokens_per_s": 231.7, "wait_weights_s": 1.804, "norm_s": 0.267, "prev_d2h_s": 2.841, "prev_adamw_s": 8.192, "prev_upload_s": 2.649, "resident_update_s": 0.19} +{"ts": 1792570589.211, "run": "ntc-r1-res4-a", "event": "step", "step": 6, "loss": 1.6975219249725342, "grad_norm": 16.00722035826498, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 9.763, "tokens_per_s": 209.8, "wait_weights_s": 2.047, "norm_s": 0.275, "prev_d2h_s": 3.297, "prev_adamw_s": 8.792, "prev_upload_s": 2.888, "resident_update_s": 0.215} +{"ts": 1792570598.598, "run": "ntc-r1-res4-a", "event": "step", "step": 7, "loss": 1.7448283731937408, "grad_norm": 16.504575148400374, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 9.383, "tokens_per_s": 218.3, "wait_weights_s": 2.144, "norm_s": 0.3, "prev_d2h_s": 3.031, "prev_adamw_s": 9.273, "prev_upload_s": 2.69, "resident_update_s": 0.222} +{"ts": 1792570608.169, "run": "ntc-r1-res4-a", "event": "step", "step": 8, "loss": 1.1026900857686996, "grad_norm": 12.248325808936602, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 9.568, "tokens_per_s": 214.0, "wait_weights_s": 2.377, "norm_s": 0.211, "prev_d2h_s": 3.298, "prev_adamw_s": 9.366, "prev_upload_s": 2.623, "resident_update_s": 0.141} +{"ts": 1792570617.827, "run": "ntc-r1-res4-a", "event": "step", "step": 9, "loss": 0.8907594978809357, "grad_norm": 11.50092219153214, "lr": 9e-06, "tokens": 2048, "seconds": 9.651, "tokens_per_s": 212.2, "wait_weights_s": 2.451, "norm_s": 0.24, "prev_d2h_s": 5.166, "prev_adamw_s": 8.435, "prev_upload_s": 2.776, "resident_update_s": 0.171} +{"ts": 1792570627.543, "run": "ntc-r1-res4-a", "event": "step", "step": 10, "loss": 1.1144881709478796, "grad_norm": 9.905762855896496, "lr": 1e-05, "tokens": 2048, "seconds": 9.713, "tokens_per_s": 210.9, "wait_weights_s": 2.342, "norm_s": 0.2, "prev_d2h_s": 4.422, "prev_adamw_s": 9.055, "prev_upload_s": 2.736, "resident_update_s": 0.141} +{"ts": 1792570637.419, "run": "ntc-r1-res4-a", "event": "step", "step": 11, "loss": 1.1876610815525055, "grad_norm": 10.395274096641511, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 9.873, "tokens_per_s": 207.4, "wait_weights_s": 2.172, "norm_s": 0.251, "prev_d2h_s": 3.846, "prev_adamw_s": 9.021, "prev_upload_s": 2.577, "resident_update_s": 0.189} +{"ts": 1792570647.88, "run": "ntc-r1-res4-a", "event": "step", "step": 12, "loss": 1.1581125259399414, "grad_norm": 15.290384430447183, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 10.455, "tokens_per_s": 195.9, "wait_weights_s": 2.342, "norm_s": 0.315, "prev_d2h_s": 5.03, "prev_adamw_s": 7.749, "prev_upload_s": 2.812, "resident_update_s": 0.248} +{"ts": 1792570657.708, "run": "ntc-r1-res4-a", "event": "step", "step": 13, "loss": 1.3791157603263855, "grad_norm": 9.747475614234334, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 9.82, "tokens_per_s": 208.6, "wait_weights_s": 1.947, "norm_s": 0.265, "prev_d2h_s": 3.569, "prev_adamw_s": 9.61, "prev_upload_s": 2.652, "resident_update_s": 0.186} +{"ts": 1792570667.778, "run": "ntc-r1-res4-a", "event": "step", "step": 14, "loss": 1.1589331775903702, "grad_norm": 32.01337868395624, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 10.064, "tokens_per_s": 203.5, "wait_weights_s": 2.149, "norm_s": 0.299, "prev_d2h_s": 3.927, "prev_adamw_s": 9.174, "prev_upload_s": 2.613, "resident_update_s": 0.243} +{"ts": 1792570677.696, "run": "ntc-r1-res4-a", "event": "step", "step": 15, "loss": 1.0737810730934143, "grad_norm": 8.799479131650747, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 9.912, "tokens_per_s": 206.6, "wait_weights_s": 2.169, "norm_s": 0.256, "prev_d2h_s": 4.469, "prev_adamw_s": 8.327, "prev_upload_s": 2.806, "resident_update_s": 0.173} +{"ts": 1792570688.054, "run": "ntc-r1-res4-a", "event": "step", "step": 16, "loss": 1.280925452709198, "grad_norm": 9.947090175365714, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 10.349, "tokens_per_s": 197.9, "wait_weights_s": 2.053, "norm_s": 0.304, "prev_d2h_s": 3.99, "prev_adamw_s": 8.796, "prev_upload_s": 2.834, "resident_update_s": 0.236} +{"ts": 1792570697.195, "run": "ntc-r1-res4-a", "event": "step", "step": 17, "loss": 1.150767520070076, "grad_norm": 13.82387576598276, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 9.138, "tokens_per_s": 224.1, "wait_weights_s": 1.886, "norm_s": 0.285, "prev_d2h_s": 3.341, "prev_adamw_s": 9.39, "prev_upload_s": 2.544, "resident_update_s": 0.213} +{"ts": 1792570706.748, "run": "ntc-r1-res4-a", "event": "step", "step": 18, "loss": 1.4344564378261566, "grad_norm": 9.048538449354128, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 9.551, "tokens_per_s": 214.4, "wait_weights_s": 2.123, "norm_s": 0.277, "prev_d2h_s": 5.595, "prev_adamw_s": 7.389, "prev_upload_s": 2.853, "resident_update_s": 0.203} +{"ts": 1792570715.982, "run": "ntc-r1-res4-a", "event": "step", "step": 19, "loss": 0.8110396191477776, "grad_norm": 8.728112205652602, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 9.226, "tokens_per_s": 222.0, "wait_weights_s": 1.939, "norm_s": 0.24, "prev_d2h_s": 3.785, "prev_adamw_s": 8.429, "prev_upload_s": 2.617, "resident_update_s": 0.169} +{"ts": 1792570725.454, "run": "ntc-r1-res4-a", "event": "step", "step": 20, "loss": 1.5605111122131348, "grad_norm": 8.477922126542014, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 9.467, "tokens_per_s": 216.3, "wait_weights_s": 1.8, "norm_s": 0.226, "prev_d2h_s": 3.356, "prev_adamw_s": 7.713, "prev_upload_s": 2.721, "resident_update_s": 0.136} +{"ts": 1792570734.941, "run": "ntc-r1-res4-a", "event": "step", "step": 21, "loss": 1.2799706757068634, "grad_norm": 11.843185668255165, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 9.484, "tokens_per_s": 215.9, "wait_weights_s": 2.052, "norm_s": 0.231, "prev_d2h_s": 3.134, "prev_adamw_s": 9.408, "prev_upload_s": 2.795, "resident_update_s": 0.173} +{"ts": 1792570743.344, "run": "ntc-r1-res4-a", "event": "step", "step": 22, "loss": 1.3606796711683273, "grad_norm": 7.80455122264179, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 8.399, "tokens_per_s": 243.8, "wait_weights_s": 1.744, "norm_s": 0.232, "prev_d2h_s": 2.669, "prev_adamw_s": 8.349, "prev_upload_s": 2.681, "resident_update_s": 0.182} +{"ts": 1792570752.352, "run": "ntc-r1-res4-a", "event": "step", "step": 23, "loss": 0.896641917526722, "grad_norm": 7.997609483077358, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 9.004, "tokens_per_s": 227.5, "wait_weights_s": 1.619, "norm_s": 0.347, "prev_d2h_s": 2.525, "prev_adamw_s": 7.921, "prev_upload_s": 2.633, "resident_update_s": 0.28} +{"ts": 1792570762.1, "run": "ntc-r1-res4-a", "event": "step", "step": 24, "loss": 0.8824600055813789, "grad_norm": 10.408052662151386, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 9.743, "tokens_per_s": 210.2, "wait_weights_s": 1.608, "norm_s": 0.271, "prev_d2h_s": 2.874, "prev_adamw_s": 7.961, "prev_upload_s": 2.723, "resident_update_s": 0.21} +{"ts": 1792570770.121, "run": "ntc-r1-res4-a", "event": "step", "step": 25, "loss": 1.3030380047857761, "grad_norm": 12.350533050629535, "lr": 0.0, "tokens": 2048, "seconds": 8.015, "tokens_per_s": 255.5, "wait_weights_s": 1.556, "norm_s": 0.186, "prev_d2h_s": 2.622, "prev_adamw_s": 7.156, "prev_upload_s": 2.422, "resident_update_s": 0.121} +{"ts": 1792570797.818, "run": "ntc-r1-res4-a", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8110396191477776, "mode": "full"} diff --git a/experiments/data/dram-shared/r1-res4-b.jsonl b/experiments/data/dram-shared/r1-res4-b.jsonl new file mode 100644 index 0000000..23e8e23 --- /dev/null +++ b/experiments/data/dram-shared/r1-res4-b.jsonl @@ -0,0 +1,27 @@ +{"ts": 1792570902.35, "run": "ntc-r1-res4-b", "event": "start", "examples": 8000} +{"ts": 1792571107.602, "run": "ntc-r1-res4-b", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 17.295, "tokens_per_s": 118.4, "wait_weights_s": 0.001, "norm_s": 0.234, "prev_d2h_s": 0.0, "prev_adamw_s": 0.165, "prev_upload_s": 0.0, "resident_update_s": 0.165} +{"ts": 1792571123.498, "run": "ntc-r1-res4-b", "event": "step", "step": 2, "loss": 1.5645148754119873, "grad_norm": 45.859045193746276, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 15.894, "tokens_per_s": 128.9, "wait_weights_s": 8.104, "norm_s": 0.247, "prev_d2h_s": 16.366, "prev_adamw_s": 20.173, "prev_upload_s": 3.483, "resident_update_s": 0.179} +{"ts": 1792571132.583, "run": "ntc-r1-res4-b", "event": "step", "step": 3, "loss": 1.0821138471364975, "grad_norm": 80.97597200661254, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 9.082, "tokens_per_s": 225.5, "wait_weights_s": 1.575, "norm_s": 0.187, "prev_d2h_s": 2.798, "prev_adamw_s": 7.935, "prev_upload_s": 2.657, "resident_update_s": 0.123} +{"ts": 1792571141.932, "run": "ntc-r1-res4-b", "event": "step", "step": 4, "loss": 1.1063324678689241, "grad_norm": 20.461211574277034, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 9.346, "tokens_per_s": 219.1, "wait_weights_s": 1.849, "norm_s": 0.261, "prev_d2h_s": 2.875, "prev_adamw_s": 8.613, "prev_upload_s": 2.577, "resident_update_s": 0.192} +{"ts": 1792571151.083, "run": "ntc-r1-res4-b", "event": "step", "step": 5, "loss": 1.3412490785121918, "grad_norm": 13.312377605287216, "lr": 5e-06, "tokens": 2048, "seconds": 9.147, "tokens_per_s": 223.9, "wait_weights_s": 1.932, "norm_s": 0.191, "prev_d2h_s": 2.943, "prev_adamw_s": 9.086, "prev_upload_s": 2.615, "resident_update_s": 0.139} +{"ts": 1792571161.562, "run": "ntc-r1-res4-b", "event": "step", "step": 6, "loss": 1.6975219249725342, "grad_norm": 16.00722035826498, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 10.476, "tokens_per_s": 195.5, "wait_weights_s": 2.153, "norm_s": 0.347, "prev_d2h_s": 3.287, "prev_adamw_s": 8.776, "prev_upload_s": 2.827, "resident_update_s": 0.278} +{"ts": 1792571171.336, "run": "ntc-r1-res4-b", "event": "step", "step": 7, "loss": 1.7448283731937408, "grad_norm": 16.504575148400374, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 9.771, "tokens_per_s": 209.6, "wait_weights_s": 2.076, "norm_s": 0.226, "prev_d2h_s": 2.866, "prev_adamw_s": 10.139, "prev_upload_s": 2.908, "resident_update_s": 0.121} +{"ts": 1792571180.757, "run": "ntc-r1-res4-b", "event": "step", "step": 8, "loss": 1.1026900857686996, "grad_norm": 12.248325808936602, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 9.417, "tokens_per_s": 217.5, "wait_weights_s": 2.042, "norm_s": 0.275, "prev_d2h_s": 2.909, "prev_adamw_s": 9.86, "prev_upload_s": 2.585, "resident_update_s": 0.202} +{"ts": 1792571190.303, "run": "ntc-r1-res4-b", "event": "step", "step": 9, "loss": 0.8907594978809357, "grad_norm": 11.50092219153214, "lr": 9e-06, "tokens": 2048, "seconds": 9.543, "tokens_per_s": 214.6, "wait_weights_s": 1.694, "norm_s": 0.193, "prev_d2h_s": 3.185, "prev_adamw_s": 7.964, "prev_upload_s": 2.61, "resident_update_s": 0.122} +{"ts": 1792571199.696, "run": "ntc-r1-res4-b", "event": "step", "step": 10, "loss": 1.1144881709478796, "grad_norm": 9.905762855896496, "lr": 1e-05, "tokens": 2048, "seconds": 9.39, "tokens_per_s": 218.1, "wait_weights_s": 2.073, "norm_s": 0.247, "prev_d2h_s": 3.087, "prev_adamw_s": 10.036, "prev_upload_s": 2.892, "resident_update_s": 0.191} +{"ts": 1792571209.394, "run": "ntc-r1-res4-b", "event": "step", "step": 11, "loss": 1.1876610815525055, "grad_norm": 10.395274096641511, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 9.695, "tokens_per_s": 211.2, "wait_weights_s": 2.214, "norm_s": 0.186, "prev_d2h_s": 2.958, "prev_adamw_s": 9.27, "prev_upload_s": 2.794, "resident_update_s": 0.102} +{"ts": 1792571219.383, "run": "ntc-r1-res4-b", "event": "step", "step": 12, "loss": 1.1581125259399414, "grad_norm": 15.290384430447183, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 9.987, "tokens_per_s": 205.1, "wait_weights_s": 2.35, "norm_s": 0.327, "prev_d2h_s": 4.221, "prev_adamw_s": 9.3, "prev_upload_s": 2.794, "resident_update_s": 0.257} +{"ts": 1792571228.997, "run": "ntc-r1-res4-b", "event": "step", "step": 13, "loss": 1.3791157603263855, "grad_norm": 9.747475614234334, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 9.609, "tokens_per_s": 213.1, "wait_weights_s": 1.88, "norm_s": 0.181, "prev_d2h_s": 2.924, "prev_adamw_s": 9.21, "prev_upload_s": 2.718, "resident_update_s": 0.113} +{"ts": 1792571238.626, "run": "ntc-r1-res4-b", "event": "step", "step": 14, "loss": 1.1589331775903702, "grad_norm": 32.01337868395624, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 9.626, "tokens_per_s": 212.8, "wait_weights_s": 2.107, "norm_s": 0.239, "prev_d2h_s": 4.066, "prev_adamw_s": 8.506, "prev_upload_s": 2.776, "resident_update_s": 0.182} +{"ts": 1792571248.451, "run": "ntc-r1-res4-b", "event": "step", "step": 15, "loss": 1.0737810730934143, "grad_norm": 8.799479131650747, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 9.821, "tokens_per_s": 208.5, "wait_weights_s": 2.138, "norm_s": 0.175, "prev_d2h_s": 3.398, "prev_adamw_s": 9.189, "prev_upload_s": 2.636, "resident_update_s": 0.115} +{"ts": 1792571258.243, "run": "ntc-r1-res4-b", "event": "step", "step": 16, "loss": 1.280925452709198, "grad_norm": 9.947090175365714, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 9.79, "tokens_per_s": 209.2, "wait_weights_s": 2.401, "norm_s": 0.196, "prev_d2h_s": 4.823, "prev_adamw_s": 9.839, "prev_upload_s": 2.619, "resident_update_s": 0.134} +{"ts": 1792571268.578, "run": "ntc-r1-res4-b", "event": "step", "step": 17, "loss": 1.150767520070076, "grad_norm": 13.82387576598276, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 10.331, "tokens_per_s": 198.2, "wait_weights_s": 2.258, "norm_s": 0.295, "prev_d2h_s": 4.55, "prev_adamw_s": 8.862, "prev_upload_s": 2.894, "resident_update_s": 0.223} +{"ts": 1792571278.924, "run": "ntc-r1-res4-b", "event": "step", "step": 18, "loss": 1.4344564378261566, "grad_norm": 9.048538449354128, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 10.341, "tokens_per_s": 198.0, "wait_weights_s": 2.3, "norm_s": 0.19, "prev_d2h_s": 3.734, "prev_adamw_s": 9.762, "prev_upload_s": 2.914, "resident_update_s": 0.137} +{"ts": 1792571287.62, "run": "ntc-r1-res4-b", "event": "step", "step": 19, "loss": 0.8110396191477776, "grad_norm": 8.728112205652602, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 8.69, "tokens_per_s": 235.7, "wait_weights_s": 2.299, "norm_s": 0.287, "prev_d2h_s": 4.509, "prev_adamw_s": 8.356, "prev_upload_s": 2.576, "resident_update_s": 0.203} +{"ts": 1792571296.953, "run": "ntc-r1-res4-b", "event": "step", "step": 20, "loss": 1.5605111122131348, "grad_norm": 8.477922126542014, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 9.33, "tokens_per_s": 219.5, "wait_weights_s": 1.871, "norm_s": 0.27, "prev_d2h_s": 3.03, "prev_adamw_s": 9.087, "prev_upload_s": 2.744, "resident_update_s": 0.21} +{"ts": 1792571307.371, "run": "ntc-r1-res4-b", "event": "step", "step": 21, "loss": 1.2799706757068634, "grad_norm": 11.843185668255165, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 10.416, "tokens_per_s": 196.6, "wait_weights_s": 2.122, "norm_s": 0.247, "prev_d2h_s": 3.843, "prev_adamw_s": 9.219, "prev_upload_s": 2.775, "resident_update_s": 0.182} +{"ts": 1792571317.228, "run": "ntc-r1-res4-b", "event": "step", "step": 22, "loss": 1.3606796711683273, "grad_norm": 7.80455122264179, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 9.851, "tokens_per_s": 207.9, "wait_weights_s": 2.172, "norm_s": 0.219, "prev_d2h_s": 4.619, "prev_adamw_s": 8.663, "prev_upload_s": 2.826, "resident_update_s": 0.151} +{"ts": 1792571327.157, "run": "ntc-r1-res4-b", "event": "step", "step": 23, "loss": 0.896641917526722, "grad_norm": 7.997609483077358, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 9.928, "tokens_per_s": 206.3, "wait_weights_s": 2.133, "norm_s": 0.208, "prev_d2h_s": 3.564, "prev_adamw_s": 8.377, "prev_upload_s": 2.71, "resident_update_s": 0.148} +{"ts": 1792571336.886, "run": "ntc-r1-res4-b", "event": "step", "step": 24, "loss": 0.8824600055813789, "grad_norm": 10.408052662151386, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 9.726, "tokens_per_s": 210.6, "wait_weights_s": 1.679, "norm_s": 0.217, "prev_d2h_s": 2.817, "prev_adamw_s": 7.921, "prev_upload_s": 2.632, "resident_update_s": 0.157} +{"ts": 1792571346.283, "run": "ntc-r1-res4-b", "event": "step", "step": 25, "loss": 1.3030380047857761, "grad_norm": 12.350533050629535, "lr": 0.0, "tokens": 2048, "seconds": 9.395, "tokens_per_s": 218.0, "wait_weights_s": 1.786, "norm_s": 0.167, "prev_d2h_s": 3.039, "prev_adamw_s": 8.591, "prev_upload_s": 2.667, "resident_update_s": 0.106} +{"ts": 1792571369.372, "run": "ntc-r1-res4-b", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8110396191477776, "mode": "full"} diff --git a/experiments/data/dram-shared/r1-res4-c.jsonl b/experiments/data/dram-shared/r1-res4-c.jsonl new file mode 100644 index 0000000..87b8046 --- /dev/null +++ b/experiments/data/dram-shared/r1-res4-c.jsonl @@ -0,0 +1,27 @@ +{"ts": 1792571481.801, "run": "ntc-r1-res4-c", "event": "start", "examples": 8000} +{"ts": 1792571682.036, "run": "ntc-r1-res4-c", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 17.331, "tokens_per_s": 118.2, "wait_weights_s": 0.001, "norm_s": 0.383, "prev_d2h_s": 0.0, "prev_adamw_s": 0.312, "prev_upload_s": 0.0, "resident_update_s": 0.312} +{"ts": 1792571697.164, "run": "ntc-r1-res4-c", "event": "step", "step": 2, "loss": 1.5645148754119873, "grad_norm": 45.859045193746276, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 15.126, "tokens_per_s": 135.4, "wait_weights_s": 8.021, "norm_s": 0.287, "prev_d2h_s": 16.529, "prev_adamw_s": 21.397, "prev_upload_s": 3.427, "resident_update_s": 0.222} +{"ts": 1792571707.041, "run": "ntc-r1-res4-c", "event": "step", "step": 3, "loss": 1.0821138471364975, "grad_norm": 80.97597200661254, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 9.87, "tokens_per_s": 207.5, "wait_weights_s": 1.784, "norm_s": 0.349, "prev_d2h_s": 2.996, "prev_adamw_s": 8.139, "prev_upload_s": 2.643, "resident_update_s": 0.274} +{"ts": 1792571717.345, "run": "ntc-r1-res4-c", "event": "step", "step": 4, "loss": 1.1063324678689241, "grad_norm": 20.461211574277034, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 10.3, "tokens_per_s": 198.8, "wait_weights_s": 1.797, "norm_s": 0.347, "prev_d2h_s": 3.078, "prev_adamw_s": 9.659, "prev_upload_s": 2.63, "resident_update_s": 0.273} +{"ts": 1792571729.321, "run": "ntc-r1-res4-c", "event": "step", "step": 5, "loss": 1.3412490785121918, "grad_norm": 13.312377605287216, "lr": 5e-06, "tokens": 2048, "seconds": 11.971, "tokens_per_s": 171.1, "wait_weights_s": 2.413, "norm_s": 0.343, "prev_d2h_s": 3.969, "prev_adamw_s": 10.026, "prev_upload_s": 3.345, "resident_update_s": 0.245} +{"ts": 1792571742.654, "run": "ntc-r1-res4-c", "event": "step", "step": 6, "loss": 1.6975219249725342, "grad_norm": 16.00722035826498, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 13.33, "tokens_per_s": 153.6, "wait_weights_s": 2.257, "norm_s": 0.333, "prev_d2h_s": 3.733, "prev_adamw_s": 11.43, "prev_upload_s": 2.765, "resident_update_s": 0.263} +{"ts": 1792571754.799, "run": "ntc-r1-res4-c", "event": "step", "step": 7, "loss": 1.7448283731937408, "grad_norm": 16.504575148400374, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 12.142, "tokens_per_s": 168.7, "wait_weights_s": 2.43, "norm_s": 0.236, "prev_d2h_s": 3.795, "prev_adamw_s": 12.732, "prev_upload_s": 3.039, "resident_update_s": 0.174} +{"ts": 1792571767.669, "run": "ntc-r1-res4-c", "event": "step", "step": 8, "loss": 1.1026900857686996, "grad_norm": 12.248325808936602, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 12.864, "tokens_per_s": 159.2, "wait_weights_s": 2.66, "norm_s": 0.397, "prev_d2h_s": 4.618, "prev_adamw_s": 13.068, "prev_upload_s": 3.347, "resident_update_s": 0.311} +{"ts": 1792571780.154, "run": "ntc-r1-res4-c", "event": "step", "step": 9, "loss": 0.8907594978809357, "grad_norm": 11.50092219153214, "lr": 9e-06, "tokens": 2048, "seconds": 12.48, "tokens_per_s": 164.1, "wait_weights_s": 2.348, "norm_s": 0.442, "prev_d2h_s": 3.504, "prev_adamw_s": 12.836, "prev_upload_s": 2.95, "resident_update_s": 0.35} +{"ts": 1792571792.822, "run": "ntc-r1-res4-c", "event": "step", "step": 10, "loss": 1.1144881709478796, "grad_norm": 9.905762855896496, "lr": 1e-05, "tokens": 2048, "seconds": 12.664, "tokens_per_s": 161.7, "wait_weights_s": 2.609, "norm_s": 0.33, "prev_d2h_s": 5.206, "prev_adamw_s": 11.145, "prev_upload_s": 3.127, "resident_update_s": 0.257} +{"ts": 1792571805.382, "run": "ntc-r1-res4-c", "event": "step", "step": 11, "loss": 1.1876610815525055, "grad_norm": 10.395274096641511, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 12.556, "tokens_per_s": 163.1, "wait_weights_s": 2.608, "norm_s": 0.327, "prev_d2h_s": 5.018, "prev_adamw_s": 11.153, "prev_upload_s": 2.992, "resident_update_s": 0.246} +{"ts": 1792571817.499, "run": "ntc-r1-res4-c", "event": "step", "step": 12, "loss": 1.1581125259399414, "grad_norm": 15.290384430447183, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 12.114, "tokens_per_s": 169.1, "wait_weights_s": 2.502, "norm_s": 0.359, "prev_d2h_s": 4.297, "prev_adamw_s": 11.812, "prev_upload_s": 3.137, "resident_update_s": 0.268} +{"ts": 1792571830.296, "run": "ntc-r1-res4-c", "event": "step", "step": 13, "loss": 1.3791157603263855, "grad_norm": 9.747475614234334, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 12.791, "tokens_per_s": 160.1, "wait_weights_s": 2.782, "norm_s": 0.567, "prev_d2h_s": 5.288, "prev_adamw_s": 12.444, "prev_upload_s": 3.384, "resident_update_s": 0.487} +{"ts": 1792571842.404, "run": "ntc-r1-res4-c", "event": "step", "step": 14, "loss": 1.1589331775903702, "grad_norm": 32.01337868395624, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 12.105, "tokens_per_s": 169.2, "wait_weights_s": 3.084, "norm_s": 0.31, "prev_d2h_s": 5.339, "prev_adamw_s": 13.11, "prev_upload_s": 3.502, "resident_update_s": 0.254} +{"ts": 1792571854.347, "run": "ntc-r1-res4-c", "event": "step", "step": 15, "loss": 1.0737810730934143, "grad_norm": 8.799479131650747, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 11.941, "tokens_per_s": 171.5, "wait_weights_s": 2.777, "norm_s": 0.465, "prev_d2h_s": 4.502, "prev_adamw_s": 11.675, "prev_upload_s": 3.081, "resident_update_s": 0.372} +{"ts": 1792571866.927, "run": "ntc-r1-res4-c", "event": "step", "step": 16, "loss": 1.280925452709198, "grad_norm": 9.947090175365714, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 12.565, "tokens_per_s": 163.0, "wait_weights_s": 2.741, "norm_s": 0.375, "prev_d2h_s": 5.518, "prev_adamw_s": 11.474, "prev_upload_s": 3.436, "resident_update_s": 0.283} +{"ts": 1792571879.12, "run": "ntc-r1-res4-c", "event": "step", "step": 17, "loss": 1.150767520070076, "grad_norm": 13.82387576598276, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 12.189, "tokens_per_s": 168.0, "wait_weights_s": 2.814, "norm_s": 0.305, "prev_d2h_s": 5.036, "prev_adamw_s": 12.47, "prev_upload_s": 3.348, "resident_update_s": 0.234} +{"ts": 1792571891.954, "run": "ntc-r1-res4-c", "event": "step", "step": 18, "loss": 1.4344564378261566, "grad_norm": 9.048538449354128, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 12.831, "tokens_per_s": 159.6, "wait_weights_s": 2.633, "norm_s": 0.36, "prev_d2h_s": 4.392, "prev_adamw_s": 12.965, "prev_upload_s": 2.886, "resident_update_s": 0.276} +{"ts": 1792571904.317, "run": "ntc-r1-res4-c", "event": "step", "step": 19, "loss": 0.8110396191477776, "grad_norm": 8.728112205652602, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 12.36, "tokens_per_s": 165.7, "wait_weights_s": 2.617, "norm_s": 0.323, "prev_d2h_s": 5.904, "prev_adamw_s": 10.369, "prev_upload_s": 3.365, "resident_update_s": 0.243} +{"ts": 1792571916.171, "run": "ntc-r1-res4-c", "event": "step", "step": 20, "loss": 1.5605111122131348, "grad_norm": 8.477922126542014, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 11.851, "tokens_per_s": 172.8, "wait_weights_s": 2.363, "norm_s": 0.307, "prev_d2h_s": 4.09, "prev_adamw_s": 10.562, "prev_upload_s": 3.051, "resident_update_s": 0.238} +{"ts": 1792571927.512, "run": "ntc-r1-res4-c", "event": "step", "step": 21, "loss": 1.2799706757068634, "grad_norm": 11.843185668255165, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 11.335, "tokens_per_s": 180.7, "wait_weights_s": 2.158, "norm_s": 0.374, "prev_d2h_s": 3.586, "prev_adamw_s": 11.526, "prev_upload_s": 2.861, "resident_update_s": 0.294} +{"ts": 1792571938.93, "run": "ntc-r1-res4-c", "event": "step", "step": 22, "loss": 1.3606796711683273, "grad_norm": 7.80455122264179, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 11.415, "tokens_per_s": 179.4, "wait_weights_s": 2.356, "norm_s": 0.307, "prev_d2h_s": 3.781, "prev_adamw_s": 10.988, "prev_upload_s": 3.128, "resident_update_s": 0.23} +{"ts": 1792571951.288, "run": "ntc-r1-res4-c", "event": "step", "step": 23, "loss": 0.896641917526722, "grad_norm": 7.997609483077358, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 12.346, "tokens_per_s": 165.9, "wait_weights_s": 2.243, "norm_s": 0.392, "prev_d2h_s": 3.862, "prev_adamw_s": 10.267, "prev_upload_s": 3.075, "resident_update_s": 0.299} +{"ts": 1792571963.263, "run": "ntc-r1-res4-c", "event": "step", "step": 24, "loss": 0.8824600055813789, "grad_norm": 10.408052662151386, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 11.971, "tokens_per_s": 171.1, "wait_weights_s": 2.23, "norm_s": 0.361, "prev_d2h_s": 3.61, "prev_adamw_s": 11.794, "prev_upload_s": 3.072, "resident_update_s": 0.298} +{"ts": 1792571975.074, "run": "ntc-r1-res4-c", "event": "step", "step": 25, "loss": 1.3030380047857761, "grad_norm": 12.350533050629535, "lr": 0.0, "tokens": 2048, "seconds": 11.806, "tokens_per_s": 173.5, "wait_weights_s": 2.562, "norm_s": 0.365, "prev_d2h_s": 4.158, "prev_adamw_s": 11.867, "prev_upload_s": 3.374, "resident_update_s": 0.265} +{"ts": 1792572002.114, "run": "ntc-r1-res4-c", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8110396191477776, "mode": "full"} diff --git a/experiments/data/dram-shared/r2-res8-a.jsonl b/experiments/data/dram-shared/r2-res8-a.jsonl new file mode 100644 index 0000000..c1cf694 --- /dev/null +++ b/experiments/data/dram-shared/r2-res8-a.jsonl @@ -0,0 +1,27 @@ +{"ts": 1792567619.878, "run": "ntc-r2-res8-a", "event": "start", "examples": 8000} +{"ts": 1792567849.335, "run": "ntc-r2-res8-a", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 18.381, "tokens_per_s": 111.4, "wait_weights_s": 0.001, "norm_s": 0.817, "prev_d2h_s": 0.0, "prev_adamw_s": 0.733, "prev_upload_s": 0.0, "resident_update_s": 0.74, "devices": 2} +{"ts": 1792567860.746, "run": "ntc-r2-res8-a", "event": "step", "step": 2, "loss": 1.564907282590866, "grad_norm": 45.87919322328091, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 11.406, "tokens_per_s": 179.6, "wait_weights_s": 3.006, "norm_s": 0.836, "prev_d2h_s": 8.263, "prev_adamw_s": 12.985, "prev_upload_s": 2.113, "resident_update_s": 0.78, "devices": 2} +{"ts": 1792567868.694, "run": "ntc-r2-res8-a", "event": "step", "step": 3, "loss": 1.0858428925275803, "grad_norm": 83.79944461492215, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 7.935, "tokens_per_s": 258.1, "wait_weights_s": 0.169, "norm_s": 0.666, "prev_d2h_s": 1.718, "prev_adamw_s": 7.045, "prev_upload_s": 1.878, "resident_update_s": 0.603, "devices": 2} +{"ts": 1792567877.184, "run": "ntc-r2-res8-a", "event": "step", "step": 4, "loss": 1.1035496043041348, "grad_norm": 20.500255990601204, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 8.483, "tokens_per_s": 241.4, "wait_weights_s": 0.453, "norm_s": 0.738, "prev_d2h_s": 1.624, "prev_adamw_s": 7.978, "prev_upload_s": 1.577, "resident_update_s": 0.679, "devices": 2} +{"ts": 1792567884.842, "run": "ntc-r2-res8-a", "event": "step", "step": 5, "loss": 1.336936205625534, "grad_norm": 13.448047566857905, "lr": 5e-06, "tokens": 2048, "seconds": 7.654, "tokens_per_s": 267.6, "wait_weights_s": 0.446, "norm_s": 0.831, "prev_d2h_s": 1.845, "prev_adamw_s": 8.66, "prev_upload_s": 1.762, "resident_update_s": 0.778, "devices": 2} +{"ts": 1792567892.844, "run": "ntc-r2-res8-a", "event": "step", "step": 6, "loss": 1.6996679306030273, "grad_norm": 16.169723071831996, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 7.999, "tokens_per_s": 256.0, "wait_weights_s": 0.196, "norm_s": 0.747, "prev_d2h_s": 1.627, "prev_adamw_s": 7.226, "prev_upload_s": 1.672, "resident_update_s": 0.687, "devices": 2} +{"ts": 1792567900.501, "run": "ntc-r2-res8-a", "event": "step", "step": 7, "loss": 1.744823396205902, "grad_norm": 16.281523243942537, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 7.653, "tokens_per_s": 267.6, "wait_weights_s": 0.397, "norm_s": 0.774, "prev_d2h_s": 1.64, "prev_adamw_s": 7.592, "prev_upload_s": 1.682, "resident_update_s": 0.707, "devices": 2} +{"ts": 1792567908.691, "run": "ntc-r2-res8-a", "event": "step", "step": 8, "loss": 1.1050789058208466, "grad_norm": 12.173483887642671, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 8.185, "tokens_per_s": 250.2, "wait_weights_s": 0.509, "norm_s": 0.77, "prev_d2h_s": 1.995, "prev_adamw_s": 7.174, "prev_upload_s": 1.707, "resident_update_s": 0.669, "devices": 2} +{"ts": 1792567917.449, "run": "ntc-r2-res8-a", "event": "step", "step": 9, "loss": 0.8817741200327873, "grad_norm": 11.427005350223444, "lr": 9e-06, "tokens": 2048, "seconds": 8.756, "tokens_per_s": 233.9, "wait_weights_s": 0.599, "norm_s": 1.029, "prev_d2h_s": 2.341, "prev_adamw_s": 8.099, "prev_upload_s": 1.804, "resident_update_s": 0.962, "devices": 2} +{"ts": 1792567925.287, "run": "ntc-r2-res8-a", "event": "step", "step": 10, "loss": 1.1135812494903803, "grad_norm": 9.879150640935412, "lr": 1e-05, "tokens": 2048, "seconds": 7.835, "tokens_per_s": 261.4, "wait_weights_s": 0.142, "norm_s": 0.755, "prev_d2h_s": 1.801, "prev_adamw_s": 8.666, "prev_upload_s": 1.848, "resident_update_s": 0.675, "devices": 2} +{"ts": 1792567933.956, "run": "ntc-r2-res8-a", "event": "step", "step": 11, "loss": 1.1879100799560547, "grad_norm": 10.501148626361774, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 8.666, "tokens_per_s": 236.3, "wait_weights_s": 0.629, "norm_s": 0.8, "prev_d2h_s": 2.215, "prev_adamw_s": 7.836, "prev_upload_s": 1.687, "resident_update_s": 0.734, "devices": 2} +{"ts": 1792567942.336, "run": "ntc-r2-res8-a", "event": "step", "step": 12, "loss": 1.152946263551712, "grad_norm": 15.871865774465505, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 8.377, "tokens_per_s": 244.5, "wait_weights_s": 0.458, "norm_s": 0.752, "prev_d2h_s": 2.113, "prev_adamw_s": 8.284, "prev_upload_s": 1.911, "resident_update_s": 0.675, "devices": 2} +{"ts": 1792567950.093, "run": "ntc-r2-res8-a", "event": "step", "step": 13, "loss": 1.3831146657466888, "grad_norm": 9.823916977098824, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 7.753, "tokens_per_s": 264.2, "wait_weights_s": 0.344, "norm_s": 0.712, "prev_d2h_s": 1.8, "prev_adamw_s": 7.821, "prev_upload_s": 1.718, "resident_update_s": 0.663, "devices": 2} +{"ts": 1792567957.734, "run": "ntc-r2-res8-a", "event": "step", "step": 14, "loss": 1.1788776516914368, "grad_norm": 25.40595380374666, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 7.639, "tokens_per_s": 268.1, "wait_weights_s": 0.523, "norm_s": 0.58, "prev_d2h_s": 1.823, "prev_adamw_s": 7.125, "prev_upload_s": 1.587, "resident_update_s": 0.517, "devices": 2} +{"ts": 1792567966.154, "run": "ntc-r2-res8-a", "event": "step", "step": 15, "loss": 1.0667250454425812, "grad_norm": 9.45781636499448, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 8.411, "tokens_per_s": 243.5, "wait_weights_s": 0.682, "norm_s": 0.957, "prev_d2h_s": 2.167, "prev_adamw_s": 7.587, "prev_upload_s": 1.615, "resident_update_s": 0.873, "devices": 2} +{"ts": 1792567974.019, "run": "ntc-r2-res8-a", "event": "step", "step": 16, "loss": 1.2814564108848572, "grad_norm": 9.82564657798959, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 7.858, "tokens_per_s": 260.6, "wait_weights_s": 0.184, "norm_s": 0.789, "prev_d2h_s": 1.687, "prev_adamw_s": 7.728, "prev_upload_s": 1.754, "resident_update_s": 0.724, "devices": 2} +{"ts": 1792567982.103, "run": "ntc-r2-res8-a", "event": "step", "step": 17, "loss": 1.1525242775678635, "grad_norm": 12.536942518503388, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 8.081, "tokens_per_s": 253.4, "wait_weights_s": 0.356, "norm_s": 0.918, "prev_d2h_s": 1.824, "prev_adamw_s": 7.932, "prev_upload_s": 1.734, "resident_update_s": 0.852, "devices": 2} +{"ts": 1792567990.395, "run": "ntc-r2-res8-a", "event": "step", "step": 18, "loss": 1.4375255107879639, "grad_norm": 8.443515455504027, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 8.288, "tokens_per_s": 247.1, "wait_weights_s": 0.322, "norm_s": 0.783, "prev_d2h_s": 3.091, "prev_adamw_s": 7.642, "prev_upload_s": 1.828, "resident_update_s": 0.699, "devices": 2} +{"ts": 1792567998.735, "run": "ntc-r2-res8-a", "event": "step", "step": 19, "loss": 0.8065920546650887, "grad_norm": 8.344874941911247, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 8.337, "tokens_per_s": 245.7, "wait_weights_s": 0.407, "norm_s": 0.851, "prev_d2h_s": 1.948, "prev_adamw_s": 8.06, "prev_upload_s": 1.624, "resident_update_s": 0.78, "devices": 2} +{"ts": 1792568007.251, "run": "ntc-r2-res8-a", "event": "step", "step": 20, "loss": 1.5583547055721283, "grad_norm": 8.224272573940292, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 8.507, "tokens_per_s": 240.7, "wait_weights_s": 0.344, "norm_s": 0.998, "prev_d2h_s": 2.036, "prev_adamw_s": 8.308, "prev_upload_s": 1.779, "resident_update_s": 0.93, "devices": 2} +{"ts": 1792568014.983, "run": "ntc-r2-res8-a", "event": "step", "step": 21, "loss": 1.2753856480121613, "grad_norm": 11.440582162042189, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 7.728, "tokens_per_s": 265.0, "wait_weights_s": 0.279, "norm_s": 0.618, "prev_d2h_s": 1.838, "prev_adamw_s": 8.249, "prev_upload_s": 1.783, "resident_update_s": 0.545, "devices": 2} +{"ts": 1792568023.228, "run": "ntc-r2-res8-a", "event": "step", "step": 22, "loss": 1.357256829738617, "grad_norm": 7.770549618130633, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 8.243, "tokens_per_s": 248.5, "wait_weights_s": 0.48, "norm_s": 0.805, "prev_d2h_s": 1.758, "prev_adamw_s": 8.182, "prev_upload_s": 1.673, "resident_update_s": 0.729, "devices": 2} +{"ts": 1792568031.199, "run": "ntc-r2-res8-a", "event": "step", "step": 23, "loss": 0.8956179767847061, "grad_norm": 7.827806084439568, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 7.968, "tokens_per_s": 257.0, "wait_weights_s": 0.384, "norm_s": 0.772, "prev_d2h_s": 1.945, "prev_adamw_s": 8.121, "prev_upload_s": 1.743, "resident_update_s": 0.699, "devices": 2} +{"ts": 1792568038.828, "run": "ntc-r2-res8-a", "event": "step", "step": 24, "loss": 0.8916763514280319, "grad_norm": 10.107818403310903, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 7.626, "tokens_per_s": 268.5, "wait_weights_s": 0.451, "norm_s": 0.727, "prev_d2h_s": 1.864, "prev_adamw_s": 7.916, "prev_upload_s": 1.807, "resident_update_s": 0.675, "devices": 2} +{"ts": 1792568047.263, "run": "ntc-r2-res8-a", "event": "step", "step": 25, "loss": 1.2988715544342995, "grad_norm": 12.731063043562827, "lr": 0.0, "tokens": 2048, "seconds": 8.432, "tokens_per_s": 242.9, "wait_weights_s": 0.299, "norm_s": 0.801, "prev_d2h_s": 1.83, "prev_adamw_s": 7.096, "prev_upload_s": 1.701, "resident_update_s": 0.739, "devices": 2} +{"ts": 1792568078.019, "run": "ntc-r2-res8-a", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8065920546650887, "mode": "full"} diff --git a/experiments/data/dram-shared/r2-res8-b.jsonl b/experiments/data/dram-shared/r2-res8-b.jsonl new file mode 100644 index 0000000..9bf27c1 --- /dev/null +++ b/experiments/data/dram-shared/r2-res8-b.jsonl @@ -0,0 +1,27 @@ +{"ts": 1792568202.621, "run": "ntc-r2-res8-b", "event": "start", "examples": 8000} +{"ts": 1792568432.26, "run": "ntc-r2-res8-b", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 16.906, "tokens_per_s": 121.1, "wait_weights_s": 0.001, "norm_s": 0.633, "prev_d2h_s": 0.0, "prev_adamw_s": 0.563, "prev_upload_s": 0.0, "resident_update_s": 0.564, "devices": 2} +{"ts": 1792568444.584, "run": "ntc-r2-res8-b", "event": "step", "step": 2, "loss": 1.564907282590866, "grad_norm": 45.87919322328091, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 12.32, "tokens_per_s": 166.2, "wait_weights_s": 3.432, "norm_s": 0.821, "prev_d2h_s": 8.725, "prev_adamw_s": 12.676, "prev_upload_s": 1.796, "resident_update_s": 0.76, "devices": 2} +{"ts": 1792568453.058, "run": "ntc-r2-res8-b", "event": "step", "step": 3, "loss": 1.0858428925275803, "grad_norm": 83.79944461492215, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 8.471, "tokens_per_s": 241.8, "wait_weights_s": 0.116, "norm_s": 0.636, "prev_d2h_s": 1.593, "prev_adamw_s": 7.744, "prev_upload_s": 1.513, "resident_update_s": 0.579, "devices": 2} +{"ts": 1792568461.797, "run": "ntc-r2-res8-b", "event": "step", "step": 4, "loss": 1.1035496043041348, "grad_norm": 20.500255990601204, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 8.735, "tokens_per_s": 234.5, "wait_weights_s": 0.376, "norm_s": 0.752, "prev_d2h_s": 1.609, "prev_adamw_s": 8.471, "prev_upload_s": 1.512, "resident_update_s": 0.694, "devices": 2} +{"ts": 1792568470.598, "run": "ntc-r2-res8-b", "event": "step", "step": 5, "loss": 1.336936205625534, "grad_norm": 13.448047566857905, "lr": 5e-06, "tokens": 2048, "seconds": 8.798, "tokens_per_s": 232.8, "wait_weights_s": 0.301, "norm_s": 0.829, "prev_d2h_s": 1.712, "prev_adamw_s": 8.171, "prev_upload_s": 1.525, "resident_update_s": 0.764, "devices": 2} +{"ts": 1792568479.265, "run": "ntc-r2-res8-b", "event": "step", "step": 6, "loss": 1.6996679306030273, "grad_norm": 16.169723071831996, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 8.664, "tokens_per_s": 236.4, "wait_weights_s": 0.303, "norm_s": 0.803, "prev_d2h_s": 1.565, "prev_adamw_s": 8.058, "prev_upload_s": 1.567, "resident_update_s": 0.674, "devices": 2} +{"ts": 1792568488.251, "run": "ntc-r2-res8-b", "event": "step", "step": 7, "loss": 1.744823396205902, "grad_norm": 16.281523243942537, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 8.983, "tokens_per_s": 228.0, "wait_weights_s": 0.788, "norm_s": 0.779, "prev_d2h_s": 2.191, "prev_adamw_s": 9.107, "prev_upload_s": 1.762, "resident_update_s": 0.709, "devices": 2} +{"ts": 1792568498.715, "run": "ntc-r2-res8-b", "event": "step", "step": 8, "loss": 1.1050789058208466, "grad_norm": 12.173483887642671, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 10.46, "tokens_per_s": 195.8, "wait_weights_s": 0.661, "norm_s": 0.942, "prev_d2h_s": 2.609, "prev_adamw_s": 8.25, "prev_upload_s": 1.667, "resident_update_s": 0.869, "devices": 2} +{"ts": 1792568508.924, "run": "ntc-r2-res8-b", "event": "step", "step": 9, "loss": 0.8817741200327873, "grad_norm": 11.427005350223444, "lr": 9e-06, "tokens": 2048, "seconds": 10.206, "tokens_per_s": 200.7, "wait_weights_s": 0.586, "norm_s": 0.817, "prev_d2h_s": 2.305, "prev_adamw_s": 9.453, "prev_upload_s": 1.671, "resident_update_s": 0.748, "devices": 2} +{"ts": 1792568517.86, "run": "ntc-r2-res8-b", "event": "step", "step": 10, "loss": 1.1135812494903803, "grad_norm": 9.879150640935412, "lr": 1e-05, "tokens": 2048, "seconds": 8.934, "tokens_per_s": 229.2, "wait_weights_s": 0.458, "norm_s": 0.678, "prev_d2h_s": 2.165, "prev_adamw_s": 8.502, "prev_upload_s": 1.812, "resident_update_s": 0.596, "devices": 2} +{"ts": 1792568527.363, "run": "ntc-r2-res8-b", "event": "step", "step": 11, "loss": 1.1879100799560547, "grad_norm": 10.501148626361774, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 9.501, "tokens_per_s": 215.6, "wait_weights_s": 0.46, "norm_s": 0.921, "prev_d2h_s": 2.656, "prev_adamw_s": 8.495, "prev_upload_s": 1.638, "resident_update_s": 0.805, "devices": 2} +{"ts": 1792568536.731, "run": "ntc-r2-res8-b", "event": "step", "step": 12, "loss": 1.152946263551712, "grad_norm": 15.871865774465505, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 9.365, "tokens_per_s": 218.7, "wait_weights_s": 0.423, "norm_s": 0.823, "prev_d2h_s": 1.82, "prev_adamw_s": 9.469, "prev_upload_s": 1.564, "resident_update_s": 0.755, "devices": 2} +{"ts": 1792568545.729, "run": "ntc-r2-res8-b", "event": "step", "step": 13, "loss": 1.3831146657466888, "grad_norm": 9.823916977098824, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 8.995, "tokens_per_s": 227.7, "wait_weights_s": 0.353, "norm_s": 0.632, "prev_d2h_s": 2.266, "prev_adamw_s": 8.526, "prev_upload_s": 1.757, "resident_update_s": 0.564, "devices": 2} +{"ts": 1792568555.403, "run": "ntc-r2-res8-b", "event": "step", "step": 14, "loss": 1.1788776516914368, "grad_norm": 25.40595380374666, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 9.672, "tokens_per_s": 211.7, "wait_weights_s": 0.833, "norm_s": 0.905, "prev_d2h_s": 3.162, "prev_adamw_s": 7.961, "prev_upload_s": 1.618, "resident_update_s": 0.826, "devices": 2} +{"ts": 1792568564.573, "run": "ntc-r2-res8-b", "event": "step", "step": 15, "loss": 1.0667250454425812, "grad_norm": 9.45781636499448, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 9.166, "tokens_per_s": 223.4, "wait_weights_s": 0.547, "norm_s": 0.887, "prev_d2h_s": 1.988, "prev_adamw_s": 9.637, "prev_upload_s": 1.69, "resident_update_s": 0.814, "devices": 2} +{"ts": 1792568573.223, "run": "ntc-r2-res8-b", "event": "step", "step": 16, "loss": 1.2814564108848572, "grad_norm": 9.82564657798959, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 8.648, "tokens_per_s": 236.8, "wait_weights_s": 0.449, "norm_s": 0.632, "prev_d2h_s": 3.632, "prev_adamw_s": 8.456, "prev_upload_s": 2.018, "resident_update_s": 0.548, "devices": 2} +{"ts": 1792568582.318, "run": "ntc-r2-res8-b", "event": "step", "step": 17, "loss": 1.1525242775678635, "grad_norm": 12.536942518503388, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 9.09, "tokens_per_s": 225.3, "wait_weights_s": 0.55, "norm_s": 0.741, "prev_d2h_s": 2.5, "prev_adamw_s": 7.503, "prev_upload_s": 1.685, "resident_update_s": 0.675, "devices": 2} +{"ts": 1792568591.045, "run": "ntc-r2-res8-b", "event": "step", "step": 18, "loss": 1.4375255107879639, "grad_norm": 8.443515455504027, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 8.724, "tokens_per_s": 234.8, "wait_weights_s": 0.528, "norm_s": 0.7, "prev_d2h_s": 2.058, "prev_adamw_s": 8.645, "prev_upload_s": 1.662, "resident_update_s": 0.627, "devices": 2} +{"ts": 1792568600.169, "run": "ntc-r2-res8-b", "event": "step", "step": 19, "loss": 0.8065920546650887, "grad_norm": 8.344874941911247, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 9.12, "tokens_per_s": 224.6, "wait_weights_s": 0.529, "norm_s": 0.762, "prev_d2h_s": 1.941, "prev_adamw_s": 8.376, "prev_upload_s": 1.683, "resident_update_s": 0.657, "devices": 2} +{"ts": 1792568609.692, "run": "ntc-r2-res8-b", "event": "step", "step": 20, "loss": 1.5583547055721283, "grad_norm": 8.224272573940292, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 9.519, "tokens_per_s": 215.2, "wait_weights_s": 0.561, "norm_s": 0.843, "prev_d2h_s": 2.065, "prev_adamw_s": 8.349, "prev_upload_s": 1.774, "resident_update_s": 0.751, "devices": 2} +{"ts": 1792568619.914, "run": "ntc-r2-res8-b", "event": "step", "step": 21, "loss": 1.2753856480121613, "grad_norm": 11.440582162042189, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 10.218, "tokens_per_s": 200.4, "wait_weights_s": 0.351, "norm_s": 0.785, "prev_d2h_s": 2.114, "prev_adamw_s": 8.307, "prev_upload_s": 1.78, "resident_update_s": 0.689, "devices": 2} +{"ts": 1792568629.49, "run": "ntc-r2-res8-b", "event": "step", "step": 22, "loss": 1.357256829738617, "grad_norm": 7.770549618130633, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 9.573, "tokens_per_s": 213.9, "wait_weights_s": 0.441, "norm_s": 0.695, "prev_d2h_s": 2.012, "prev_adamw_s": 8.84, "prev_upload_s": 1.852, "resident_update_s": 0.621, "devices": 2} +{"ts": 1792568639.102, "run": "ntc-r2-res8-b", "event": "step", "step": 23, "loss": 0.8956179767847061, "grad_norm": 7.827806084439568, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 9.603, "tokens_per_s": 213.3, "wait_weights_s": 0.777, "norm_s": 0.68, "prev_d2h_s": 2.086, "prev_adamw_s": 9.232, "prev_upload_s": 2.006, "resident_update_s": 0.617, "devices": 2} +{"ts": 1792568649.098, "run": "ntc-r2-res8-b", "event": "step", "step": 24, "loss": 0.8916763514280319, "grad_norm": 10.107818403310903, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 9.99, "tokens_per_s": 205.0, "wait_weights_s": 0.419, "norm_s": 0.86, "prev_d2h_s": 1.98, "prev_adamw_s": 8.415, "prev_upload_s": 1.689, "resident_update_s": 0.787, "devices": 2} +{"ts": 1792568658.369, "run": "ntc-r2-res8-b", "event": "step", "step": 25, "loss": 1.2988715544342995, "grad_norm": 12.731063043562827, "lr": 0.0, "tokens": 2048, "seconds": 9.257, "tokens_per_s": 221.2, "wait_weights_s": 0.233, "norm_s": 0.724, "prev_d2h_s": 1.85, "prev_adamw_s": 8.178, "prev_upload_s": 1.82, "resident_update_s": 0.625, "devices": 2} +{"ts": 1792568686.575, "run": "ntc-r2-res8-b", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8065920546650887, "mode": "full"} diff --git a/experiments/data/dram-shared/r2-res8-c.jsonl b/experiments/data/dram-shared/r2-res8-c.jsonl new file mode 100644 index 0000000..3138dc8 --- /dev/null +++ b/experiments/data/dram-shared/r2-res8-c.jsonl @@ -0,0 +1,27 @@ +{"ts": 1792568868.024, "run": "ntc-r2-res8-c", "event": "start", "examples": 8000} +{"ts": 1792569213.661, "run": "ntc-r2-res8-c", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 19.133, "tokens_per_s": 107.0, "wait_weights_s": 0.001, "norm_s": 0.902, "prev_d2h_s": 0.0, "prev_adamw_s": 0.816, "prev_upload_s": 0.0, "resident_update_s": 0.818, "devices": 2} +{"ts": 1792569227.299, "run": "ntc-r2-res8-c", "event": "step", "step": 2, "loss": 1.564907282590866, "grad_norm": 45.87919322328091, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 13.633, "tokens_per_s": 150.2, "wait_weights_s": 3.229, "norm_s": 0.897, "prev_d2h_s": 9.301, "prev_adamw_s": 14.408, "prev_upload_s": 2.483, "resident_update_s": 0.842, "devices": 2} +{"ts": 1792569236.482, "run": "ntc-r2-res8-c", "event": "step", "step": 3, "loss": 1.0858428925275803, "grad_norm": 83.79944461492215, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 9.181, "tokens_per_s": 223.1, "wait_weights_s": 0.273, "norm_s": 0.865, "prev_d2h_s": 2.149, "prev_adamw_s": 9.013, "prev_upload_s": 1.997, "resident_update_s": 0.791, "devices": 2} +{"ts": 1792569245.444, "run": "ntc-r2-res8-c", "event": "step", "step": 4, "loss": 1.1035496043041348, "grad_norm": 20.500255990601204, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 8.957, "tokens_per_s": 228.7, "wait_weights_s": 0.393, "norm_s": 0.842, "prev_d2h_s": 2.147, "prev_adamw_s": 8.184, "prev_upload_s": 2.079, "resident_update_s": 0.761, "devices": 2} +{"ts": 1792569255.176, "run": "ntc-r2-res8-c", "event": "step", "step": 5, "loss": 1.336936205625534, "grad_norm": 13.448047566857905, "lr": 5e-06, "tokens": 2048, "seconds": 9.73, "tokens_per_s": 210.5, "wait_weights_s": 0.505, "norm_s": 0.918, "prev_d2h_s": 2.324, "prev_adamw_s": 8.249, "prev_upload_s": 2.147, "resident_update_s": 0.841, "devices": 2} +{"ts": 1792569264.684, "run": "ntc-r2-res8-c", "event": "step", "step": 6, "loss": 1.6996679306030273, "grad_norm": 16.169723071831996, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 9.503, "tokens_per_s": 215.5, "wait_weights_s": 0.332, "norm_s": 0.902, "prev_d2h_s": 2.16, "prev_adamw_s": 8.13, "prev_upload_s": 2.043, "resident_update_s": 0.827, "devices": 2} +{"ts": 1792569274.109, "run": "ntc-r2-res8-c", "event": "step", "step": 7, "loss": 1.744823396205902, "grad_norm": 16.281523243942537, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 9.422, "tokens_per_s": 217.4, "wait_weights_s": 0.632, "norm_s": 0.786, "prev_d2h_s": 2.315, "prev_adamw_s": 9.513, "prev_upload_s": 2.172, "resident_update_s": 0.708, "devices": 2} +{"ts": 1792569283.457, "run": "ntc-r2-res8-c", "event": "step", "step": 8, "loss": 1.1050789058208466, "grad_norm": 12.173483887642671, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 9.341, "tokens_per_s": 219.2, "wait_weights_s": 0.546, "norm_s": 0.776, "prev_d2h_s": 2.173, "prev_adamw_s": 8.416, "prev_upload_s": 2.086, "resident_update_s": 0.684, "devices": 2} +{"ts": 1792569293.13, "run": "ntc-r2-res8-c", "event": "step", "step": 9, "loss": 0.8817741200327873, "grad_norm": 11.427005350223444, "lr": 9e-06, "tokens": 2048, "seconds": 9.67, "tokens_per_s": 211.8, "wait_weights_s": 0.56, "norm_s": 0.737, "prev_d2h_s": 2.228, "prev_adamw_s": 8.58, "prev_upload_s": 2.061, "resident_update_s": 0.654, "devices": 2} +{"ts": 1792569302.536, "run": "ntc-r2-res8-c", "event": "step", "step": 10, "loss": 1.1135812494903803, "grad_norm": 9.879150640935412, "lr": 1e-05, "tokens": 2048, "seconds": 9.404, "tokens_per_s": 217.8, "wait_weights_s": 0.523, "norm_s": 0.729, "prev_d2h_s": 2.195, "prev_adamw_s": 8.542, "prev_upload_s": 1.992, "resident_update_s": 0.659, "devices": 2} +{"ts": 1792569311.825, "run": "ntc-r2-res8-c", "event": "step", "step": 11, "loss": 1.1879100799560547, "grad_norm": 10.501148626361774, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 9.286, "tokens_per_s": 220.6, "wait_weights_s": 0.642, "norm_s": 0.845, "prev_d2h_s": 2.268, "prev_adamw_s": 9.115, "prev_upload_s": 2.1, "resident_update_s": 0.771, "devices": 2} +{"ts": 1792569321.395, "run": "ntc-r2-res8-c", "event": "step", "step": 12, "loss": 1.152946263551712, "grad_norm": 15.871865774465505, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 9.562, "tokens_per_s": 214.2, "wait_weights_s": 0.458, "norm_s": 0.86, "prev_d2h_s": 2.219, "prev_adamw_s": 8.223, "prev_upload_s": 2.063, "resident_update_s": 0.748, "devices": 2} +{"ts": 1792569330.783, "run": "ntc-r2-res8-c", "event": "step", "step": 13, "loss": 1.3831146657466888, "grad_norm": 9.823916977098824, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 9.383, "tokens_per_s": 218.3, "wait_weights_s": 0.489, "norm_s": 0.764, "prev_d2h_s": 2.149, "prev_adamw_s": 8.72, "prev_upload_s": 2.08, "resident_update_s": 0.688, "devices": 2} +{"ts": 1792569340.551, "run": "ntc-r2-res8-c", "event": "step", "step": 14, "loss": 1.1788776516914368, "grad_norm": 25.40595380374666, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 9.764, "tokens_per_s": 209.8, "wait_weights_s": 0.678, "norm_s": 0.845, "prev_d2h_s": 2.776, "prev_adamw_s": 7.939, "prev_upload_s": 2.207, "resident_update_s": 0.761, "devices": 2} +{"ts": 1792569350.275, "run": "ntc-r2-res8-c", "event": "step", "step": 15, "loss": 1.0667250454425812, "grad_norm": 9.45781636499448, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 9.719, "tokens_per_s": 210.7, "wait_weights_s": 0.559, "norm_s": 0.823, "prev_d2h_s": 2.435, "prev_adamw_s": 9.066, "prev_upload_s": 1.915, "resident_update_s": 0.761, "devices": 2} +{"ts": 1792569359.484, "run": "ntc-r2-res8-c", "event": "step", "step": 16, "loss": 1.2814564108848572, "grad_norm": 9.82564657798959, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 9.206, "tokens_per_s": 222.5, "wait_weights_s": 0.824, "norm_s": 0.669, "prev_d2h_s": 3.044, "prev_adamw_s": 8.851, "prev_upload_s": 2.259, "resident_update_s": 0.577, "devices": 2} +{"ts": 1792569369.86, "run": "ntc-r2-res8-c", "event": "step", "step": 17, "loss": 1.1525242775678635, "grad_norm": 12.536942518503388, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 10.372, "tokens_per_s": 197.5, "wait_weights_s": 1.032, "norm_s": 1.032, "prev_d2h_s": 3.217, "prev_adamw_s": 8.961, "prev_upload_s": 1.982, "resident_update_s": 0.943, "devices": 2} +{"ts": 1792569379.5, "run": "ntc-r2-res8-c", "event": "step", "step": 18, "loss": 1.4375255107879639, "grad_norm": 8.443515455504027, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 9.638, "tokens_per_s": 212.5, "wait_weights_s": 0.655, "norm_s": 0.621, "prev_d2h_s": 2.57, "prev_adamw_s": 9.984, "prev_upload_s": 2.254, "resident_update_s": 0.541, "devices": 2} +{"ts": 1792569389.741, "run": "ntc-r2-res8-c", "event": "step", "step": 19, "loss": 0.8065920546650887, "grad_norm": 8.344874941911247, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 10.239, "tokens_per_s": 200.0, "wait_weights_s": 1.04, "norm_s": 0.906, "prev_d2h_s": 3.36, "prev_adamw_s": 9.305, "prev_upload_s": 1.908, "resident_update_s": 0.814, "devices": 2} +{"ts": 1792569399.189, "run": "ntc-r2-res8-c", "event": "step", "step": 20, "loss": 1.5583547055721283, "grad_norm": 8.224272573940292, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 9.445, "tokens_per_s": 216.8, "wait_weights_s": 0.646, "norm_s": 0.763, "prev_d2h_s": 2.351, "prev_adamw_s": 9.07, "prev_upload_s": 2.103, "resident_update_s": 0.691, "devices": 2} +{"ts": 1792569408.959, "run": "ntc-r2-res8-c", "event": "step", "step": 21, "loss": 1.2753856480121613, "grad_norm": 11.440582162042189, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 9.767, "tokens_per_s": 209.7, "wait_weights_s": 0.574, "norm_s": 0.903, "prev_d2h_s": 3.556, "prev_adamw_s": 7.739, "prev_upload_s": 2.212, "resident_update_s": 0.827, "devices": 2} +{"ts": 1792569418.76, "run": "ntc-r2-res8-c", "event": "step", "step": 22, "loss": 1.357256829738617, "grad_norm": 7.770549618130633, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 9.797, "tokens_per_s": 209.0, "wait_weights_s": 0.458, "norm_s": 0.945, "prev_d2h_s": 2.843, "prev_adamw_s": 8.717, "prev_upload_s": 2.072, "resident_update_s": 0.876, "devices": 2} +{"ts": 1792569427.516, "run": "ntc-r2-res8-c", "event": "step", "step": 23, "loss": 0.8956179767847061, "grad_norm": 7.827806084439568, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 8.753, "tokens_per_s": 234.0, "wait_weights_s": 0.429, "norm_s": 0.748, "prev_d2h_s": 2.509, "prev_adamw_s": 9.033, "prev_upload_s": 2.036, "resident_update_s": 0.68, "devices": 2} +{"ts": 1792569436.175, "run": "ntc-r2-res8-c", "event": "step", "step": 24, "loss": 0.8916763514280319, "grad_norm": 10.107818403310903, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 8.653, "tokens_per_s": 236.7, "wait_weights_s": 0.567, "norm_s": 0.733, "prev_d2h_s": 2.151, "prev_adamw_s": 8.286, "prev_upload_s": 2.008, "resident_update_s": 0.687, "devices": 2} +{"ts": 1792569444.678, "run": "ntc-r2-res8-c", "event": "step", "step": 25, "loss": 1.2988715544342995, "grad_norm": 12.731063043562827, "lr": 0.0, "tokens": 2048, "seconds": 8.497, "tokens_per_s": 241.0, "wait_weights_s": 0.476, "norm_s": 0.806, "prev_d2h_s": 2.123, "prev_adamw_s": 7.511, "prev_upload_s": 2.026, "resident_update_s": 0.702, "devices": 2} +{"ts": 1792569476.506, "run": "ntc-r2-res8-c", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8065920546650887, "mode": "full"} diff --git a/experiments/data/dram-shared/race_repro.py b/experiments/data/dram-shared/race_repro.py new file mode 100644 index 0000000..63840f4 --- /dev/null +++ b/experiments/data/dram-shared/race_repro.py @@ -0,0 +1,93 @@ +"""#40 repro: furiosa-torch 2026.2's device memory type is process-global, so an upload from one thread +while another thread runs a compiled graph can land in DRAM_SHARED (0 bytes on a single-chip RNGD) and fail. + + PYTHONPATH= python3 race_repro.py flag # mode set in a thread is seen by the main thread + PYTHONPATH= python3 race_repro.py watch # graph calls flip the mode for a moment + PYTHONPATH= python3 race_repro.py race # OOM on DRAM_SHARED within seconds + PYTHONPATH= python3 race_repro.py fixed # npu_lora.backends (lock): no OOM in 120 s + +Uses one card (furiosa:auto) and a trivial compiled graph (x * 2, [4096, 4096] fp32). +""" +import sys +import threading +import time + +import torch + +from npu_lora import backends, graph as G + +mode = sys.argv[1] +secs = float(sys.argv[2]) if len(sys.argv) > 2 else 120 +be = backends.make("furiosa:auto") +from furiosa.native_torch import memory as M # noqa: E402 (after furiosa.torch) + +if mode == "flag": + ev, go = threading.Event(), threading.Event() + + def other(): + prev = M._set_dram_shared_mode() + ev.set() + go.wait() + M._set_memory_type(prev) + + t = threading.Thread(target=other) + t.start() + ev.wait() + print("main thread sees dram_shared mode set by another thread:", M._is_dram_shared_mode()) + try: + torch.zeros(1024, 4096, dtype=torch.bfloat16).to(be.dev) + print("upload ok") + except Exception as e: # noqa: BLE001 + print("upload failed:", repr(e)) + go.set() + t.join() + sys.exit(0) + +smp = [torch.zeros(4096, 4096)] +fn = G.compile_parts(be, "repro-x2-4096", G.build(lambda x: (x * 2.0,), smp), smp, max_mm=0, log=print) +x = be.to_device(torch.ones(4096, 4096)) +if mode != "fixed": + fn = fn.fn if isinstance(fn, backends.Serialized) else fn # call the module without the lock +stop, err, n_up = [False], [], [0] + +if mode == "watch": + hits, polls = [0], [0] + + def poll(): + while not stop[0]: + polls[0] += 1 + hits[0] += M._is_dram_shared_mode() + + t = threading.Thread(target=poll) + t.start() + for _ in range(300): + fn(x) + stop[0] = True + t.join() + print(f"{hits[0]} of {polls[0]} polls saw dram_shared mode during 300 graph calls") + sys.exit(0) + +host = torch.zeros(1024, 4096, dtype=torch.bfloat16) +up = be.to_device if mode == "fixed" else (lambda t: t.to(be.dev)) + + +def uploader(): + try: + while not stop[0]: + up(host) + n_up[0] += 1 + except Exception as e: # noqa: BLE001 + err.append(repr(e)[:200]) + + +ths = [threading.Thread(target=uploader) for _ in range(4)] +for t in ths: + t.start() +t0, calls = time.time(), 0 +while time.time() - t0 < secs and not err: + fn(x) + calls += 1 +stop[0] = True +for t in ths: + t.join() +print(f"{mode}: {calls} graph calls, {n_up[0]} uploads in {time.time() - t0:.1f} s, errors: {err[:2]}") diff --git a/experiments/data/dram-shared/runall.log b/experiments/data/dram-shared/runall.log new file mode 100644 index 0000000..b88bdfe --- /dev/null +++ b/experiments/data/dram-shared/runall.log @@ -0,0 +1,19 @@ +16:26:10 r2-res8-a before: load 467.69 463.89 420.16 +16:35:36 r2-res8-a status=1 steps=25 oom=0 +16:35:56 r2-res8-b before: load 496.44 488.07 451.06 +16:45:42 r2-res8-b status=1 steps=25 oom=0 +16:46:04 r2-res8-c before: load 408.27 407.57 421.61 +16:58:32 r2-res8-c status=1 steps=25 oom=0 +16:58:49 r1-res4-a before: load 400.13 368.77 384.98 holder 373900 python3 rngd/npu3 +17:00:08 r1-res4-a before: load 312.68 350.66 377.54 holder 373900 python3 rngd/npu3 +17:01:28 r1-res4-a before: load 266.99 325.89 366.56 holder 373900 python3 rngd/npu3 +17:03:00 r1-res4-a before: load 308.59 323.37 362.22 holder 373900 python3 rngd/npu3 +17:04:30 r1-res4-a before: load 378.35 343.34 365.75 holder 373900 python3 rngd/npu3 +17:05:57 r1-res4-a before: load 339.76 341.14 363.02 holder 373900 python3 rngd/npu3 +17:07:17 r1-res4-a before: load 304.88 328.36 356.44 +17:20:35 r1-res4-a status=1 steps=25 oom=0 +17:20:55 r1-res4-b before: load 409.68 389.27 374.05 +17:30:00 r1-res4-b status=1 steps=25 oom=0 +17:30:35 r1-res4-c before: load 424.25 415.70 392.52 +17:40:31 r1-res4-c status=1 steps=25 oom=0 +17:40:31 ALLDONE diff --git a/experiments/data/dram-shared/t1-off.jsonl b/experiments/data/dram-shared/t1-off.jsonl new file mode 100644 index 0000000..ea5b8a8 --- /dev/null +++ b/experiments/data/dram-shared/t1-off.jsonl @@ -0,0 +1,27 @@ +{"ts": 1792579688.95, "run": "ntc-t1-off", "event": "start", "examples": 8000} +{"ts": 1792579879.156, "run": "ntc-t1-off", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 16.474, "tokens_per_s": 124.3, "wait_weights_s": 0.001, "norm_s": 0.077, "prev_d2h_s": 0.0, "prev_adamw_s": 0.0, "prev_upload_s": 0.0} +{"ts": 1792579897.802, "run": "ntc-t1-off", "event": "step", "step": 2, "loss": 1.5625602304935455, "grad_norm": 45.83976931927431, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 18.638, "tokens_per_s": 109.9, "wait_weights_s": 10.076, "norm_s": 0.079, "prev_d2h_s": 18.748, "prev_adamw_s": 21.989, "prev_upload_s": 4.123} +{"ts": 1792579908.004, "run": "ntc-t1-off", "event": "step", "step": 3, "loss": 1.0878866240382195, "grad_norm": 83.62784643840048, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 10.196, "tokens_per_s": 200.9, "wait_weights_s": 1.936, "norm_s": 0.078, "prev_d2h_s": 3.121, "prev_adamw_s": 8.758, "prev_upload_s": 2.889} +{"ts": 1792579918.555, "run": "ntc-t1-off", "event": "step", "step": 4, "loss": 1.105992036871612, "grad_norm": 20.339171194820047, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 10.54, "tokens_per_s": 194.3, "wait_weights_s": 2.187, "norm_s": 0.06, "prev_d2h_s": 3.348, "prev_adamw_s": 8.743, "prev_upload_s": 3.03} +{"ts": 1792579928.94, "run": "ntc-t1-off", "event": "step", "step": 5, "loss": 1.3360941708087921, "grad_norm": 13.385490951262877, "lr": 5e-06, "tokens": 2048, "seconds": 10.38, "tokens_per_s": 197.3, "wait_weights_s": 2.167, "norm_s": 0.088, "prev_d2h_s": 3.127, "prev_adamw_s": 9.003, "prev_upload_s": 3.072} +{"ts": 1792579939.679, "run": "ntc-t1-off", "event": "step", "step": 6, "loss": 1.706906497478485, "grad_norm": 16.335248507856008, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 10.736, "tokens_per_s": 190.8, "wait_weights_s": 2.084, "norm_s": 0.088, "prev_d2h_s": 3.067, "prev_adamw_s": 8.431, "prev_upload_s": 2.928} +{"ts": 1792579949.957, "run": "ntc-t1-off", "event": "step", "step": 7, "loss": 1.7494316697120667, "grad_norm": 16.13586935662257, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 10.273, "tokens_per_s": 199.4, "wait_weights_s": 2.072, "norm_s": 0.081, "prev_d2h_s": 2.877, "prev_adamw_s": 9.563, "prev_upload_s": 2.947} +{"ts": 1792579960.355, "run": "ntc-t1-off", "event": "step", "step": 8, "loss": 1.1023952513933182, "grad_norm": 12.115460431474599, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 10.394, "tokens_per_s": 197.0, "wait_weights_s": 2.256, "norm_s": 0.07, "prev_d2h_s": 3.092, "prev_adamw_s": 9.603, "prev_upload_s": 2.791} +{"ts": 1792579970.822, "run": "ntc-t1-off", "event": "step", "step": 9, "loss": 0.8882534727454185, "grad_norm": 11.315825465381323, "lr": 9e-06, "tokens": 2048, "seconds": 10.463, "tokens_per_s": 195.7, "wait_weights_s": 2.46, "norm_s": 0.08, "prev_d2h_s": 3.643, "prev_adamw_s": 9.518, "prev_upload_s": 3.075} +{"ts": 1792579981.603, "run": "ntc-t1-off", "event": "step", "step": 10, "loss": 1.1120138672413304, "grad_norm": 9.943527357181816, "lr": 1e-05, "tokens": 2048, "seconds": 10.776, "tokens_per_s": 190.1, "wait_weights_s": 2.636, "norm_s": 0.076, "prev_d2h_s": 4.12, "prev_adamw_s": 9.245, "prev_upload_s": 3.248} +{"ts": 1792579992.33, "run": "ntc-t1-off", "event": "step", "step": 11, "loss": 1.1846136450767517, "grad_norm": 10.343349532186359, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 10.721, "tokens_per_s": 191.0, "wait_weights_s": 2.584, "norm_s": 0.102, "prev_d2h_s": 4.108, "prev_adamw_s": 9.138, "prev_upload_s": 3.184} +{"ts": 1792580003.586, "run": "ntc-t1-off", "event": "step", "step": 12, "loss": 1.149821549654007, "grad_norm": 15.667075483844068, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 11.25, "tokens_per_s": 182.0, "wait_weights_s": 2.637, "norm_s": 0.094, "prev_d2h_s": 4.353, "prev_adamw_s": 10.198, "prev_upload_s": 3.111} +{"ts": 1792580014.52, "run": "ntc-t1-off", "event": "step", "step": 13, "loss": 1.3779243230819702, "grad_norm": 9.736832260042876, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 10.929, "tokens_per_s": 187.4, "wait_weights_s": 2.563, "norm_s": 0.091, "prev_d2h_s": 4.513, "prev_adamw_s": 9.902, "prev_upload_s": 3.232} +{"ts": 1792580025.174, "run": "ntc-t1-off", "event": "step", "step": 14, "loss": 1.1682318598031998, "grad_norm": 28.767873129222597, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 10.65, "tokens_per_s": 192.3, "wait_weights_s": 2.473, "norm_s": 0.089, "prev_d2h_s": 3.569, "prev_adamw_s": 9.936, "prev_upload_s": 3.021} +{"ts": 1792580035.799, "run": "ntc-t1-off", "event": "step", "step": 15, "loss": 1.0693852007389069, "grad_norm": 9.241371995864162, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 10.621, "tokens_per_s": 192.8, "wait_weights_s": 2.661, "norm_s": 0.066, "prev_d2h_s": 5.043, "prev_adamw_s": 8.757, "prev_upload_s": 3.181} +{"ts": 1792580046.377, "run": "ntc-t1-off", "event": "step", "step": 16, "loss": 1.2823715209960938, "grad_norm": 10.081775683695783, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 10.572, "tokens_per_s": 193.7, "wait_weights_s": 2.67, "norm_s": 0.085, "prev_d2h_s": 4.31, "prev_adamw_s": 9.357, "prev_upload_s": 3.213} +{"ts": 1792580057.041, "run": "ntc-t1-off", "event": "step", "step": 17, "loss": 1.1385412216186523, "grad_norm": 13.139631625067793, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 10.66, "tokens_per_s": 192.1, "wait_weights_s": 2.213, "norm_s": 0.11, "prev_d2h_s": 3.602, "prev_adamw_s": 8.179, "prev_upload_s": 2.712} +{"ts": 1792580068.06, "run": "ntc-t1-off", "event": "step", "step": 18, "loss": 1.4344590604305267, "grad_norm": 9.256871006458512, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 11.014, "tokens_per_s": 185.9, "wait_weights_s": 2.475, "norm_s": 0.065, "prev_d2h_s": 4.051, "prev_adamw_s": 9.524, "prev_upload_s": 3.261} +{"ts": 1792580078.651, "run": "ntc-t1-off", "event": "step", "step": 19, "loss": 0.8083997741341591, "grad_norm": 8.44625464614702, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 10.587, "tokens_per_s": 193.4, "wait_weights_s": 2.605, "norm_s": 0.082, "prev_d2h_s": 4.254, "prev_adamw_s": 9.693, "prev_upload_s": 3.028} +{"ts": 1792580089.844, "run": "ntc-t1-off", "event": "step", "step": 20, "loss": 1.5581613630056381, "grad_norm": 8.408779834946662, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 11.19, "tokens_per_s": 183.0, "wait_weights_s": 2.85, "norm_s": 0.077, "prev_d2h_s": 5.504, "prev_adamw_s": 8.121, "prev_upload_s": 3.199} +{"ts": 1792580101.171, "run": "ntc-t1-off", "event": "step", "step": 21, "loss": 1.272824227809906, "grad_norm": 11.512978955760328, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 11.318, "tokens_per_s": 181.0, "wait_weights_s": 2.649, "norm_s": 0.073, "prev_d2h_s": 4.155, "prev_adamw_s": 9.421, "prev_upload_s": 3.214} +{"ts": 1792580111.917, "run": "ntc-t1-off", "event": "step", "step": 22, "loss": 1.3625558465719223, "grad_norm": 7.841532302364816, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 10.741, "tokens_per_s": 190.7, "wait_weights_s": 2.44, "norm_s": 0.083, "prev_d2h_s": 5.817, "prev_adamw_s": 8.453, "prev_upload_s": 3.229} +{"ts": 1792580122.479, "run": "ntc-t1-off", "event": "step", "step": 23, "loss": 0.9013836085796356, "grad_norm": 7.894083656715112, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 10.558, "tokens_per_s": 194.0, "wait_weights_s": 2.206, "norm_s": 0.095, "prev_d2h_s": 3.348, "prev_adamw_s": 8.56, "prev_upload_s": 3.098} +{"ts": 1792580132.859, "run": "ntc-t1-off", "event": "step", "step": 24, "loss": 0.8878279626369476, "grad_norm": 10.04998445436565, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 10.372, "tokens_per_s": 197.5, "wait_weights_s": 2.249, "norm_s": 0.085, "prev_d2h_s": 3.344, "prev_adamw_s": 9.058, "prev_upload_s": 3.146} +{"ts": 1792580142.564, "run": "ntc-t1-off", "event": "step", "step": 25, "loss": 1.3050681483000517, "grad_norm": 12.53712000739592, "lr": 0.0, "tokens": 2048, "seconds": 9.7, "tokens_per_s": 211.1, "wait_weights_s": 2.239, "norm_s": 0.091, "prev_d2h_s": 3.265, "prev_adamw_s": 9.146, "prev_upload_s": 3.101} +{"ts": 1792580166.274, "run": "ntc-t1-off", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8083997741341591, "mode": "full"} diff --git a/experiments/data/dram-shared/t1-res4.jsonl b/experiments/data/dram-shared/t1-res4.jsonl new file mode 100644 index 0000000..a4e7c65 --- /dev/null +++ b/experiments/data/dram-shared/t1-res4.jsonl @@ -0,0 +1,27 @@ +{"ts": 1792580278.2, "run": "ntc-t1-res4", "event": "start", "examples": 8000} +{"ts": 1792580571.576, "run": "ntc-t1-res4", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 19.486, "tokens_per_s": 105.1, "wait_weights_s": 0.001, "norm_s": 0.301, "prev_d2h_s": 0.0, "prev_adamw_s": 0.22, "prev_upload_s": 0.0, "resident_update_s": 0.22} +{"ts": 1792580586.934, "run": "ntc-t1-res4", "event": "step", "step": 2, "loss": 1.5645148754119873, "grad_norm": 45.859045193746276, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 15.355, "tokens_per_s": 133.4, "wait_weights_s": 6.444, "norm_s": 0.274, "prev_d2h_s": 13.361, "prev_adamw_s": 18.189, "prev_upload_s": 3.788, "resident_update_s": 0.196} +{"ts": 1792580597.411, "run": "ntc-t1-res4", "event": "step", "step": 3, "loss": 1.0821138471364975, "grad_norm": 80.97597200661254, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 10.472, "tokens_per_s": 195.6, "wait_weights_s": 2.077, "norm_s": 0.27, "prev_d2h_s": 3.023, "prev_adamw_s": 9.436, "prev_upload_s": 2.892, "resident_update_s": 0.218} +{"ts": 1792580607.141, "run": "ntc-t1-res4", "event": "step", "step": 4, "loss": 1.1063324678689241, "grad_norm": 20.461211574277034, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 9.724, "tokens_per_s": 210.6, "wait_weights_s": 1.592, "norm_s": 0.171, "prev_d2h_s": 2.765, "prev_adamw_s": 8.388, "prev_upload_s": 2.781, "resident_update_s": 0.145} +{"ts": 1792580616.807, "run": "ntc-t1-res4", "event": "step", "step": 5, "loss": 1.3412490785121918, "grad_norm": 13.312377605287216, "lr": 5e-06, "tokens": 2048, "seconds": 9.664, "tokens_per_s": 211.9, "wait_weights_s": 1.603, "norm_s": 0.29, "prev_d2h_s": 2.7, "prev_adamw_s": 7.677, "prev_upload_s": 2.594, "resident_update_s": 0.237} +{"ts": 1792580626.753, "run": "ntc-t1-res4", "event": "step", "step": 6, "loss": 1.6975219249725342, "grad_norm": 16.00722035826498, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 9.942, "tokens_per_s": 206.0, "wait_weights_s": 1.617, "norm_s": 0.282, "prev_d2h_s": 2.775, "prev_adamw_s": 8.819, "prev_upload_s": 2.871, "resident_update_s": 0.227} +{"ts": 1792580636.531, "run": "ntc-t1-res4", "event": "step", "step": 7, "loss": 1.7448283731937408, "grad_norm": 16.504575148400374, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 9.776, "tokens_per_s": 209.5, "wait_weights_s": 1.542, "norm_s": 0.231, "prev_d2h_s": 2.832, "prev_adamw_s": 8.633, "prev_upload_s": 2.825, "resident_update_s": 0.165} +{"ts": 1792580645.917, "run": "ntc-t1-res4", "event": "step", "step": 8, "loss": 1.1026900857686996, "grad_norm": 12.248325808936602, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 9.381, "tokens_per_s": 218.3, "wait_weights_s": 1.733, "norm_s": 0.286, "prev_d2h_s": 2.902, "prev_adamw_s": 8.641, "prev_upload_s": 2.88, "resident_update_s": 0.224} +{"ts": 1792580655.757, "run": "ntc-t1-res4", "event": "step", "step": 9, "loss": 0.8907594978809357, "grad_norm": 11.50092219153214, "lr": 9e-06, "tokens": 2048, "seconds": 9.837, "tokens_per_s": 208.2, "wait_weights_s": 1.618, "norm_s": 0.277, "prev_d2h_s": 2.767, "prev_adamw_s": 8.229, "prev_upload_s": 2.975, "resident_update_s": 0.214} +{"ts": 1792580665.48, "run": "ntc-t1-res4", "event": "step", "step": 10, "loss": 1.1144881709478796, "grad_norm": 9.905762855896496, "lr": 1e-05, "tokens": 2048, "seconds": 9.721, "tokens_per_s": 210.7, "wait_weights_s": 1.644, "norm_s": 0.257, "prev_d2h_s": 2.797, "prev_adamw_s": 8.854, "prev_upload_s": 2.776, "resident_update_s": 0.188} +{"ts": 1792580675.049, "run": "ntc-t1-res4", "event": "step", "step": 11, "loss": 1.1876610815525055, "grad_norm": 10.395274096641511, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 9.566, "tokens_per_s": 214.1, "wait_weights_s": 1.657, "norm_s": 0.255, "prev_d2h_s": 2.8, "prev_adamw_s": 8.749, "prev_upload_s": 2.898, "resident_update_s": 0.192} +{"ts": 1792580684.159, "run": "ntc-t1-res4", "event": "step", "step": 12, "loss": 1.1581125259399414, "grad_norm": 15.290384430447183, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 9.107, "tokens_per_s": 224.9, "wait_weights_s": 1.672, "norm_s": 0.294, "prev_d2h_s": 2.897, "prev_adamw_s": 8.652, "prev_upload_s": 2.878, "resident_update_s": 0.22} +{"ts": 1792580693.9, "run": "ntc-t1-res4", "event": "step", "step": 13, "loss": 1.3791157603263855, "grad_norm": 9.747475614234334, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 9.736, "tokens_per_s": 210.4, "wait_weights_s": 1.433, "norm_s": 0.313, "prev_d2h_s": 2.604, "prev_adamw_s": 8.111, "prev_upload_s": 2.812, "resident_update_s": 0.222} +{"ts": 1792580703.523, "run": "ntc-t1-res4", "event": "step", "step": 14, "loss": 1.1589331775903702, "grad_norm": 32.01337868395624, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 9.621, "tokens_per_s": 212.9, "wait_weights_s": 1.445, "norm_s": 0.251, "prev_d2h_s": 2.563, "prev_adamw_s": 8.515, "prev_upload_s": 2.615, "resident_update_s": 0.194} +{"ts": 1792580713.028, "run": "ntc-t1-res4", "event": "step", "step": 15, "loss": 1.0737810730934143, "grad_norm": 8.799479131650747, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 9.502, "tokens_per_s": 215.5, "wait_weights_s": 1.377, "norm_s": 0.29, "prev_d2h_s": 2.442, "prev_adamw_s": 7.432, "prev_upload_s": 2.51, "resident_update_s": 0.215} +{"ts": 1792580722.779, "run": "ntc-t1-res4", "event": "step", "step": 16, "loss": 1.280925452709198, "grad_norm": 9.947090175365714, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 9.744, "tokens_per_s": 210.2, "wait_weights_s": 1.437, "norm_s": 0.295, "prev_d2h_s": 2.499, "prev_adamw_s": 8.747, "prev_upload_s": 2.532, "resident_update_s": 0.235} +{"ts": 1792580732.847, "run": "ntc-t1-res4", "event": "step", "step": 17, "loss": 1.150767520070076, "grad_norm": 13.82387576598276, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 10.065, "tokens_per_s": 203.5, "wait_weights_s": 1.626, "norm_s": 0.276, "prev_d2h_s": 2.902, "prev_adamw_s": 8.874, "prev_upload_s": 2.893, "resident_update_s": 0.193} +{"ts": 1792580742.543, "run": "ntc-t1-res4", "event": "step", "step": 18, "loss": 1.4344564378261566, "grad_norm": 9.048538449354128, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 9.691, "tokens_per_s": 211.3, "wait_weights_s": 1.648, "norm_s": 0.299, "prev_d2h_s": 2.82, "prev_adamw_s": 9.25, "prev_upload_s": 2.791, "resident_update_s": 0.236} +{"ts": 1792580752.553, "run": "ntc-t1-res4", "event": "step", "step": 19, "loss": 0.8110396191477776, "grad_norm": 8.728112205652602, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 10.006, "tokens_per_s": 204.7, "wait_weights_s": 1.812, "norm_s": 0.307, "prev_d2h_s": 3.214, "prev_adamw_s": 8.846, "prev_upload_s": 2.754, "resident_update_s": 0.237} +{"ts": 1792580762.065, "run": "ntc-t1-res4", "event": "step", "step": 20, "loss": 1.5605111122131348, "grad_norm": 8.477922126542014, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 9.509, "tokens_per_s": 215.4, "wait_weights_s": 1.528, "norm_s": 0.248, "prev_d2h_s": 2.954, "prev_adamw_s": 8.313, "prev_upload_s": 2.669, "resident_update_s": 0.176} +{"ts": 1792580771.871, "run": "ntc-t1-res4", "event": "step", "step": 21, "loss": 1.2799706757068634, "grad_norm": 11.843185668255165, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 9.804, "tokens_per_s": 208.9, "wait_weights_s": 1.918, "norm_s": 0.242, "prev_d2h_s": 3.453, "prev_adamw_s": 8.024, "prev_upload_s": 2.859, "resident_update_s": 0.217} +{"ts": 1792580781.274, "run": "ntc-t1-res4", "event": "step", "step": 22, "loss": 1.3606796711683273, "grad_norm": 7.80455122264179, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 9.401, "tokens_per_s": 217.9, "wait_weights_s": 1.449, "norm_s": 0.322, "prev_d2h_s": 2.6, "prev_adamw_s": 7.655, "prev_upload_s": 2.525, "resident_update_s": 0.252} +{"ts": 1792580790.446, "run": "ntc-t1-res4", "event": "step", "step": 23, "loss": 0.896641917526722, "grad_norm": 7.997609483077358, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 9.169, "tokens_per_s": 223.4, "wait_weights_s": 1.506, "norm_s": 0.33, "prev_d2h_s": 2.591, "prev_adamw_s": 8.738, "prev_upload_s": 2.65, "resident_update_s": 0.242} +{"ts": 1792580800.163, "run": "ntc-t1-res4", "event": "step", "step": 24, "loss": 0.8824600055813789, "grad_norm": 10.408052662151386, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 9.715, "tokens_per_s": 210.8, "wait_weights_s": 1.606, "norm_s": 0.207, "prev_d2h_s": 3.94, "prev_adamw_s": 7.695, "prev_upload_s": 2.802, "resident_update_s": 0.144} +{"ts": 1792580810.052, "run": "ntc-t1-res4", "event": "step", "step": 25, "loss": 1.3030380047857761, "grad_norm": 12.350533050629535, "lr": 0.0, "tokens": 2048, "seconds": 9.886, "tokens_per_s": 207.2, "wait_weights_s": 1.787, "norm_s": 0.293, "prev_d2h_s": 2.778, "prev_adamw_s": 9.18, "prev_upload_s": 2.776, "resident_update_s": 0.232} +{"ts": 1792580838.02, "run": "ntc-t1-res4", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8110396191477776, "mode": "full"} diff --git a/experiments/data/dram-shared/t2.jsonl b/experiments/data/dram-shared/t2.jsonl new file mode 100644 index 0000000..7ff04b7 --- /dev/null +++ b/experiments/data/dram-shared/t2.jsonl @@ -0,0 +1,54 @@ +{"ts": 1792578894.061, "run": "ntc-t2-res8", "event": "start", "examples": 8000} +{"ts": 1792579125.36, "run": "ntc-t2-res8", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 14.712, "tokens_per_s": 139.2, "wait_weights_s": 0.0, "norm_s": 0.532, "prev_d2h_s": 0.0, "prev_adamw_s": 0.497, "prev_upload_s": 0.0, "resident_update_s": 0.5, "devices": 2} +{"ts": 1792579131.744, "run": "ntc-t2-res8", "event": "step", "step": 2, "loss": 1.564907282590866, "grad_norm": 45.87919322328091, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 6.381, "tokens_per_s": 320.9, "wait_weights_s": 1.763, "norm_s": 0.566, "prev_d2h_s": 5.342, "prev_adamw_s": 7.022, "prev_upload_s": 1.292, "resident_update_s": 0.537, "devices": 2} +{"ts": 1792579135.858, "run": "ntc-t2-res8", "event": "step", "step": 3, "loss": 1.0858428925275803, "grad_norm": 83.79944461492215, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 4.112, "tokens_per_s": 498.0, "wait_weights_s": 0.046, "norm_s": 0.507, "prev_d2h_s": 1.119, "prev_adamw_s": 3.827, "prev_upload_s": 1.208, "resident_update_s": 0.477, "devices": 2} +{"ts": 1792579140.241, "run": "ntc-t2-res8", "event": "step", "step": 4, "loss": 1.1035496043041348, "grad_norm": 20.500255990601204, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 4.38, "tokens_per_s": 467.5, "wait_weights_s": 0.204, "norm_s": 0.529, "prev_d2h_s": 1.159, "prev_adamw_s": 4.026, "prev_upload_s": 1.232, "resident_update_s": 0.501, "devices": 2} +{"ts": 1792579144.396, "run": "ntc-t2-res8", "event": "step", "step": 5, "loss": 1.336936205625534, "grad_norm": 13.448047566857905, "lr": 5e-06, "tokens": 2048, "seconds": 4.153, "tokens_per_s": 493.1, "wait_weights_s": 0.167, "norm_s": 0.483, "prev_d2h_s": 1.157, "prev_adamw_s": 3.861, "prev_upload_s": 1.165, "resident_update_s": 0.456, "devices": 2} +{"ts": 1792579148.785, "run": "ntc-t2-res8", "event": "step", "step": 6, "loss": 1.6996679306030273, "grad_norm": 16.169723071831996, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 4.379, "tokens_per_s": 467.7, "wait_weights_s": 0.24, "norm_s": 0.519, "prev_d2h_s": 1.169, "prev_adamw_s": 3.936, "prev_upload_s": 1.301, "resident_update_s": 0.488, "devices": 2} +{"ts": 1792579153.064, "run": "ntc-t2-res8", "event": "step", "step": 7, "loss": 1.744823396205902, "grad_norm": 16.281523243942537, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 4.273, "tokens_per_s": 479.3, "wait_weights_s": 0.169, "norm_s": 0.552, "prev_d2h_s": 1.182, "prev_adamw_s": 3.932, "prev_upload_s": 1.275, "resident_update_s": 0.524, "devices": 2} +{"ts": 1792579157.255, "run": "ntc-t2-res8", "event": "step", "step": 8, "loss": 1.1050789058208466, "grad_norm": 12.173483887642671, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 4.185, "tokens_per_s": 489.4, "wait_weights_s": 0.186, "norm_s": 0.502, "prev_d2h_s": 1.158, "prev_adamw_s": 4.094, "prev_upload_s": 1.203, "resident_update_s": 0.469, "devices": 2} +{"ts": 1792579161.426, "run": "ntc-t2-res8", "event": "step", "step": 9, "loss": 0.8817741200327873, "grad_norm": 11.427005350223444, "lr": 9e-06, "tokens": 2048, "seconds": 4.168, "tokens_per_s": 491.3, "wait_weights_s": 0.219, "norm_s": 0.474, "prev_d2h_s": 1.211, "prev_adamw_s": 4.072, "prev_upload_s": 1.297, "resident_update_s": 0.446, "devices": 2} +{"ts": 1792579165.666, "run": "ntc-t2-res8", "event": "step", "step": 10, "loss": 1.1135812494903803, "grad_norm": 9.879150640935412, "lr": 1e-05, "tokens": 2048, "seconds": 4.237, "tokens_per_s": 483.4, "wait_weights_s": 0.201, "norm_s": 0.515, "prev_d2h_s": 1.168, "prev_adamw_s": 3.979, "prev_upload_s": 1.28, "resident_update_s": 0.484, "devices": 2} +{"ts": 1792579169.985, "run": "ntc-t2-res8", "event": "step", "step": 11, "loss": 1.1879100799560547, "grad_norm": 10.501148626361774, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 4.313, "tokens_per_s": 474.9, "wait_weights_s": 0.154, "norm_s": 0.536, "prev_d2h_s": 1.164, "prev_adamw_s": 3.871, "prev_upload_s": 1.26, "resident_update_s": 0.504, "devices": 2} +{"ts": 1792579174.184, "run": "ntc-t2-res8", "event": "step", "step": 12, "loss": 1.152946263551712, "grad_norm": 15.871865774465505, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 4.191, "tokens_per_s": 488.6, "wait_weights_s": 0.272, "norm_s": 0.515, "prev_d2h_s": 1.168, "prev_adamw_s": 4.152, "prev_upload_s": 1.228, "resident_update_s": 0.479, "devices": 2} +{"ts": 1792579178.6, "run": "ntc-t2-res8", "event": "step", "step": 13, "loss": 1.3831146657466888, "grad_norm": 9.823916977098824, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 4.41, "tokens_per_s": 464.4, "wait_weights_s": 0.229, "norm_s": 0.531, "prev_d2h_s": 1.202, "prev_adamw_s": 4.298, "prev_upload_s": 1.261, "resident_update_s": 0.495, "devices": 2} +{"ts": 1792579183.102, "run": "ntc-t2-res8", "event": "step", "step": 14, "loss": 1.1788776516914368, "grad_norm": 25.40595380374666, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 4.496, "tokens_per_s": 455.6, "wait_weights_s": 0.445, "norm_s": 0.545, "prev_d2h_s": 1.619, "prev_adamw_s": 4.079, "prev_upload_s": 1.261, "resident_update_s": 0.515, "devices": 2} +{"ts": 1792579187.347, "run": "ntc-t2-res8", "event": "step", "step": 15, "loss": 1.0667250454425812, "grad_norm": 9.45781636499448, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 4.238, "tokens_per_s": 483.3, "wait_weights_s": 0.219, "norm_s": 0.506, "prev_d2h_s": 1.361, "prev_adamw_s": 3.902, "prev_upload_s": 1.372, "resident_update_s": 0.476, "devices": 2} +{"ts": 1792579191.618, "run": "ntc-t2-res8", "event": "step", "step": 16, "loss": 1.2814564108848572, "grad_norm": 9.82564657798959, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 4.267, "tokens_per_s": 479.9, "wait_weights_s": 0.25, "norm_s": 0.491, "prev_d2h_s": 1.211, "prev_adamw_s": 3.96, "prev_upload_s": 1.259, "resident_update_s": 0.455, "devices": 2} +{"ts": 1792579195.794, "run": "ntc-t2-res8", "event": "step", "step": 17, "loss": 1.1525242775678635, "grad_norm": 12.536942518503388, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 4.169, "tokens_per_s": 491.2, "wait_weights_s": 0.23, "norm_s": 0.507, "prev_d2h_s": 1.156, "prev_adamw_s": 4.03, "prev_upload_s": 1.319, "resident_update_s": 0.48, "devices": 2} +{"ts": 1792579200.047, "run": "ntc-t2-res8", "event": "step", "step": 18, "loss": 1.4375255107879639, "grad_norm": 8.443515455504027, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 4.25, "tokens_per_s": 481.9, "wait_weights_s": 0.226, "norm_s": 0.48, "prev_d2h_s": 1.178, "prev_adamw_s": 4.047, "prev_upload_s": 1.33, "resident_update_s": 0.446, "devices": 2} +{"ts": 1792579204.263, "run": "ntc-t2-res8", "event": "step", "step": 19, "loss": 0.8065920546650887, "grad_norm": 8.344874941911247, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 4.214, "tokens_per_s": 486.0, "wait_weights_s": 0.271, "norm_s": 0.497, "prev_d2h_s": 1.145, "prev_adamw_s": 4.151, "prev_upload_s": 1.251, "resident_update_s": 0.468, "devices": 2} +{"ts": 1792579208.558, "run": "ntc-t2-res8", "event": "step", "step": 20, "loss": 1.5583547055721283, "grad_norm": 8.224272573940292, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 4.29, "tokens_per_s": 477.3, "wait_weights_s": 0.327, "norm_s": 0.498, "prev_d2h_s": 1.477, "prev_adamw_s": 3.802, "prev_upload_s": 1.327, "resident_update_s": 0.473, "devices": 2} +{"ts": 1792579213.015, "run": "ntc-t2-res8", "event": "step", "step": 21, "loss": 1.2753856480121613, "grad_norm": 11.440582162042189, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 4.454, "tokens_per_s": 459.9, "wait_weights_s": 0.284, "norm_s": 0.437, "prev_d2h_s": 1.283, "prev_adamw_s": 3.855, "prev_upload_s": 1.366, "resident_update_s": 0.409, "devices": 2} +{"ts": 1792579217.359, "run": "ntc-t2-res8", "event": "step", "step": 22, "loss": 1.357256829738617, "grad_norm": 7.770549618130633, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 4.341, "tokens_per_s": 471.7, "wait_weights_s": 0.302, "norm_s": 0.525, "prev_d2h_s": 1.444, "prev_adamw_s": 3.749, "prev_upload_s": 1.286, "resident_update_s": 0.496, "devices": 2} +{"ts": 1792579221.589, "run": "ntc-t2-res8", "event": "step", "step": 23, "loss": 0.8956179767847061, "grad_norm": 7.827806084439568, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 4.228, "tokens_per_s": 484.4, "wait_weights_s": 0.225, "norm_s": 0.476, "prev_d2h_s": 1.148, "prev_adamw_s": 3.985, "prev_upload_s": 1.327, "resident_update_s": 0.446, "devices": 2} +{"ts": 1792579225.985, "run": "ntc-t2-res8", "event": "step", "step": 24, "loss": 0.8916763514280319, "grad_norm": 10.107818403310903, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 4.394, "tokens_per_s": 466.1, "wait_weights_s": 0.299, "norm_s": 0.479, "prev_d2h_s": 1.242, "prev_adamw_s": 4.102, "prev_upload_s": 1.243, "resident_update_s": 0.443, "devices": 2} +{"ts": 1792579230.38, "run": "ntc-t2-res8", "event": "step", "step": 25, "loss": 1.2988715544342995, "grad_norm": 12.731063043562827, "lr": 0.0, "tokens": 2048, "seconds": 4.392, "tokens_per_s": 466.3, "wait_weights_s": 0.289, "norm_s": 0.523, "prev_d2h_s": 1.189, "prev_adamw_s": 4.315, "prev_upload_s": 1.371, "resident_update_s": 0.486, "devices": 2} +{"ts": 1792579259.483, "run": "ntc-t2-res8", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8065920546650887, "mode": "full"} +{"ts": 1792579274.323, "run": "ntc-t2-off", "event": "start", "examples": 8000} +{"ts": 1792579452.649, "run": "ntc-t2-off", "event": "step", "step": 1, "loss": 1.7157380282878876, "grad_norm": 63.1584758920002, "lr": 1.0000000000000002e-06, "tokens": 2048, "seconds": 12.784, "tokens_per_s": 160.2, "wait_weights_s": 0.0, "norm_s": 0.036, "prev_d2h_s": 0.0, "prev_adamw_s": 0.0, "prev_upload_s": 0.0, "devices": 2} +{"ts": 1792579462.835, "run": "ntc-t2-off", "event": "step", "step": 2, "loss": 1.5625602304935455, "grad_norm": 45.83976931927431, "lr": 2.0000000000000003e-06, "tokens": 2048, "seconds": 10.181, "tokens_per_s": 201.2, "wait_weights_s": 6.083, "norm_s": 0.032, "prev_d2h_s": 11.791, "prev_adamw_s": 13.737, "prev_upload_s": 2.784, "devices": 2} +{"ts": 1792579468.008, "run": "ntc-t2-off", "event": "step", "step": 3, "loss": 1.0878866240382195, "grad_norm": 83.62784643840048, "lr": 3.0000000000000005e-06, "tokens": 2048, "seconds": 5.169, "tokens_per_s": 396.2, "wait_weights_s": 1.674, "norm_s": 0.031, "prev_d2h_s": 2.31, "prev_adamw_s": 4.755, "prev_upload_s": 2.429, "devices": 2} +{"ts": 1792579473.177, "run": "ntc-t2-off", "event": "step", "step": 4, "loss": 1.105992036871612, "grad_norm": 20.339171194820047, "lr": 4.000000000000001e-06, "tokens": 2048, "seconds": 5.166, "tokens_per_s": 396.5, "wait_weights_s": 1.545, "norm_s": 0.033, "prev_d2h_s": 2.352, "prev_adamw_s": 3.94, "prev_upload_s": 2.527, "devices": 2} +{"ts": 1792579478.275, "run": "ntc-t2-off", "event": "step", "step": 5, "loss": 1.3360941708087921, "grad_norm": 13.385490951262877, "lr": 5e-06, "tokens": 2048, "seconds": 5.094, "tokens_per_s": 402.0, "wait_weights_s": 1.558, "norm_s": 0.034, "prev_d2h_s": 2.412, "prev_adamw_s": 4.176, "prev_upload_s": 2.477, "devices": 2} +{"ts": 1792579483.337, "run": "ntc-t2-off", "event": "step", "step": 6, "loss": 1.706906497478485, "grad_norm": 16.335248507856008, "lr": 6.000000000000001e-06, "tokens": 2048, "seconds": 5.059, "tokens_per_s": 404.8, "wait_weights_s": 1.559, "norm_s": 0.03, "prev_d2h_s": 2.408, "prev_adamw_s": 4.124, "prev_upload_s": 2.473, "devices": 2} +{"ts": 1792579488.367, "run": "ntc-t2-off", "event": "step", "step": 7, "loss": 1.7494316697120667, "grad_norm": 16.13586935662257, "lr": 7.000000000000001e-06, "tokens": 2048, "seconds": 5.027, "tokens_per_s": 407.4, "wait_weights_s": 1.578, "norm_s": 0.031, "prev_d2h_s": 2.411, "prev_adamw_s": 4.201, "prev_upload_s": 2.51, "devices": 2} +{"ts": 1792579493.399, "run": "ntc-t2-off", "event": "step", "step": 8, "loss": 1.1023952513933182, "grad_norm": 12.115460431474599, "lr": 8.000000000000001e-06, "tokens": 2048, "seconds": 5.029, "tokens_per_s": 407.3, "wait_weights_s": 1.545, "norm_s": 0.028, "prev_d2h_s": 2.434, "prev_adamw_s": 4.062, "prev_upload_s": 2.461, "devices": 2} +{"ts": 1792579498.465, "run": "ntc-t2-off", "event": "step", "step": 9, "loss": 0.8882534727454185, "grad_norm": 11.315825465381323, "lr": 9e-06, "tokens": 2048, "seconds": 5.063, "tokens_per_s": 404.5, "wait_weights_s": 1.538, "norm_s": 0.028, "prev_d2h_s": 2.403, "prev_adamw_s": 4.088, "prev_upload_s": 2.406, "devices": 2} +{"ts": 1792579503.415, "run": "ntc-t2-off", "event": "step", "step": 10, "loss": 1.1120138672413304, "grad_norm": 9.943527357181816, "lr": 1e-05, "tokens": 2048, "seconds": 4.946, "tokens_per_s": 414.1, "wait_weights_s": 1.54, "norm_s": 0.032, "prev_d2h_s": 2.409, "prev_adamw_s": 4.033, "prev_upload_s": 2.485, "devices": 2} +{"ts": 1792579508.507, "run": "ntc-t2-off", "event": "step", "step": 11, "loss": 1.1846136450767517, "grad_norm": 10.343349532186359, "lr": 9.890738003669029e-06, "tokens": 2048, "seconds": 5.09, "tokens_per_s": 402.4, "wait_weights_s": 1.565, "norm_s": 0.027, "prev_d2h_s": 2.341, "prev_adamw_s": 4.104, "prev_upload_s": 2.527, "devices": 2} +{"ts": 1792579513.486, "run": "ntc-t2-off", "event": "step", "step": 12, "loss": 1.149821549654007, "grad_norm": 15.667075483844068, "lr": 9.567727288213005e-06, "tokens": 2048, "seconds": 4.975, "tokens_per_s": 411.7, "wait_weights_s": 1.554, "norm_s": 0.031, "prev_d2h_s": 2.369, "prev_adamw_s": 3.989, "prev_upload_s": 2.497, "devices": 2} +{"ts": 1792579518.606, "run": "ntc-t2-off", "event": "step", "step": 13, "loss": 1.3779243230819702, "grad_norm": 9.736832260042876, "lr": 9.045084971874738e-06, "tokens": 2048, "seconds": 5.116, "tokens_per_s": 400.3, "wait_weights_s": 1.565, "norm_s": 0.031, "prev_d2h_s": 2.424, "prev_adamw_s": 4.31, "prev_upload_s": 2.471, "devices": 2} +{"ts": 1792579523.791, "run": "ntc-t2-off", "event": "step", "step": 14, "loss": 1.1682318598031998, "grad_norm": 28.767873129222597, "lr": 8.345653031794292e-06, "tokens": 2048, "seconds": 5.181, "tokens_per_s": 395.3, "wait_weights_s": 1.675, "norm_s": 0.03, "prev_d2h_s": 2.367, "prev_adamw_s": 4.603, "prev_upload_s": 2.429, "devices": 2} +{"ts": 1792579528.871, "run": "ntc-t2-off", "event": "step", "step": 15, "loss": 1.0693852007389069, "grad_norm": 9.241371995864162, "lr": 7.500000000000001e-06, "tokens": 2048, "seconds": 5.076, "tokens_per_s": 403.4, "wait_weights_s": 1.616, "norm_s": 0.031, "prev_d2h_s": 2.396, "prev_adamw_s": 4.379, "prev_upload_s": 2.445, "devices": 2} +{"ts": 1792579533.909, "run": "ntc-t2-off", "event": "step", "step": 16, "loss": 1.2823715209960938, "grad_norm": 10.081775683695783, "lr": 6.545084971874738e-06, "tokens": 2048, "seconds": 5.035, "tokens_per_s": 406.8, "wait_weights_s": 1.552, "norm_s": 0.026, "prev_d2h_s": 2.573, "prev_adamw_s": 3.876, "prev_upload_s": 2.54, "devices": 2} +{"ts": 1792579538.95, "run": "ntc-t2-off", "event": "step", "step": 17, "loss": 1.1385412216186523, "grad_norm": 13.139631625067793, "lr": 5.522642316338268e-06, "tokens": 2048, "seconds": 5.036, "tokens_per_s": 406.6, "wait_weights_s": 1.531, "norm_s": 0.032, "prev_d2h_s": 2.457, "prev_adamw_s": 3.983, "prev_upload_s": 2.481, "devices": 2} +{"ts": 1792579543.987, "run": "ntc-t2-off", "event": "step", "step": 18, "loss": 1.4344590604305267, "grad_norm": 9.256871006458512, "lr": 4.477357683661734e-06, "tokens": 2048, "seconds": 5.034, "tokens_per_s": 406.9, "wait_weights_s": 1.583, "norm_s": 0.031, "prev_d2h_s": 2.409, "prev_adamw_s": 4.282, "prev_upload_s": 2.421, "devices": 2} +{"ts": 1792579549.007, "run": "ntc-t2-off", "event": "step", "step": 19, "loss": 0.8083997741341591, "grad_norm": 8.44625464614702, "lr": 3.4549150281252635e-06, "tokens": 2048, "seconds": 5.017, "tokens_per_s": 408.2, "wait_weights_s": 1.569, "norm_s": 0.034, "prev_d2h_s": 2.487, "prev_adamw_s": 4.016, "prev_upload_s": 2.483, "devices": 2} +{"ts": 1792579553.998, "run": "ntc-t2-off", "event": "step", "step": 20, "loss": 1.5581613630056381, "grad_norm": 8.408779834946662, "lr": 2.5000000000000015e-06, "tokens": 2048, "seconds": 4.987, "tokens_per_s": 410.6, "wait_weights_s": 1.537, "norm_s": 0.034, "prev_d2h_s": 2.389, "prev_adamw_s": 4.055, "prev_upload_s": 2.483, "devices": 2} +{"ts": 1792579558.973, "run": "ntc-t2-off", "event": "step", "step": 21, "loss": 1.272824227809906, "grad_norm": 11.512978955760328, "lr": 1.6543469682057105e-06, "tokens": 2048, "seconds": 4.972, "tokens_per_s": 411.9, "wait_weights_s": 1.563, "norm_s": 0.028, "prev_d2h_s": 2.461, "prev_adamw_s": 4.054, "prev_upload_s": 2.397, "devices": 2} +{"ts": 1792579563.98, "run": "ntc-t2-off", "event": "step", "step": 22, "loss": 1.3625558465719223, "grad_norm": 7.841532302364816, "lr": 9.549150281252633e-07, "tokens": 2048, "seconds": 5.003, "tokens_per_s": 409.3, "wait_weights_s": 1.534, "norm_s": 0.031, "prev_d2h_s": 2.417, "prev_adamw_s": 3.991, "prev_upload_s": 2.454, "devices": 2} +{"ts": 1792579569.106, "run": "ntc-t2-off", "event": "step", "step": 23, "loss": 0.9013836085796356, "grad_norm": 7.894083656715112, "lr": 4.322727117869951e-07, "tokens": 2048, "seconds": 5.122, "tokens_per_s": 399.8, "wait_weights_s": 1.622, "norm_s": 0.027, "prev_d2h_s": 2.391, "prev_adamw_s": 4.354, "prev_upload_s": 2.454, "devices": 2} +{"ts": 1792579574.311, "run": "ntc-t2-off", "event": "step", "step": 24, "loss": 0.8878279626369476, "grad_norm": 10.04998445436565, "lr": 1.0926199633097156e-07, "tokens": 2048, "seconds": 5.202, "tokens_per_s": 393.7, "wait_weights_s": 1.635, "norm_s": 0.031, "prev_d2h_s": 2.441, "prev_adamw_s": 4.187, "prev_upload_s": 2.503, "devices": 2} +{"ts": 1792579579.533, "run": "ntc-t2-off", "event": "step", "step": 25, "loss": 1.3050681483000517, "grad_norm": 12.53712000739592, "lr": 0.0, "tokens": 2048, "seconds": 5.219, "tokens_per_s": 392.4, "wait_weights_s": 1.632, "norm_s": 0.032, "prev_d2h_s": 2.525, "prev_adamw_s": 4.215, "prev_upload_s": 2.534, "devices": 2} +{"ts": 1792579612.576, "run": "ntc-t2-off", "event": "done", "out": "/tmp/out", "steps": 25, "best_loss": 0.8083997741341591, "mode": "full"} diff --git a/experiments/data/dram-shared/timing.log b/experiments/data/dram-shared/timing.log new file mode 100644 index 0000000..4489874 --- /dev/null +++ b/experiments/data/dram-shared/timing.log @@ -0,0 +1,16 @@ +17:43:29 gate2 try 1: load 1002.77 764.73 558.23 k8s-rngd=0 +17:59:18 gate2 try 2: load 617.71 638.62 640.10 holder 1326202 python3 rngd/npu0 k8s-rngd=1 +18:14:35 gate2 try 3: load 347.18 405.32 511.12 holder 1397758 python3 rngd/npu1 k8s-rngd=2 +18:29:53 gate2 try 4: load 280.21 307.06 385.79 holder 1443611 python3 rngd/npu1 k8s-rngd=1 +18:45:08 gate2 try 5: load 180.00 208.42 283.52 holder 1505534 python3 rngd/npu0 holder 1537819 python3 rngd/npu1 k8s-rngd=2 +19:00:27 gate2 try 6: load 207.98 223.32 247.43 holder 1505534 python3 rngd/npu0 holder 1608053 python3 rngd/npu1 k8s-rngd=2 +19:15:39 gate2 try 7: load 184.38 181.61 201.84 holder 1505532 python3 rngd/npu0 k8s-rngd=1 +19:30:54 gate2 try 8: load 87.70 109.58 153.69 holder 1709461 python3 rngd/npu0 k8s-rngd=1 +--- restart (gate fix) 19:33:57 +19:34:14 gate2 try 1: load 99.06 104.90 143.50 holder 1709461 python3 rngd/npu0 k8s-rngd=1 +19:47:12 t2 status=1 steps=50 oom=0 load=load 49.62 52.90 92.88 holder 1709461 python3 rngd/npu0 +19:47:33 gate1 try 1: load 42.98 51.37 91.95 holder 1709461 python3 rngd/npu0 k8s-rngd=1 +19:56:39 t1-off status=1 steps=25 oom=0 load=load 124.24 100.97 96.34 holder 447163 python3 rngd/npu0 +19:57:09 gate1 try 1: load 118.83 100.92 96.40 holder 447163 python3 rngd/npu0 k8s-rngd=1 +20:07:56 t1-res4 status=1 steps=25 oom=0 load=load 101.28 93.54 91.49 holder 447163 python3 rngd/npu0 +20:08:12 TIMINGDONE diff --git a/npu_lora/backends.py b/npu_lora/backends.py index ad86c48..9a20936 100644 --- a/npu_lora/backends.py +++ b/npu_lora/backends.py @@ -6,6 +6,7 @@ import hashlib import importlib.metadata as md import os +import threading import time import torch @@ -98,6 +99,28 @@ def describe(self): return {"backend": self.name, "device": torch.cuda.get_device_name(self.dev)} +# furiosa-torch 2026.2 keeps the device memory type (DRAM / DRAM_SHARED) in one process-wide flag, and every +# compiled-graph call sets it to DRAM_SHARED for a moment. A device allocation from another thread in that moment +# lands in DRAM_SHARED, which has 0 bytes on a single-chip card: `OutOfMemoryError ... on DRAM_SHARED` (#40). +# The full-FT layer workers upload while the main thread runs graphs, so graph calls and device allocations share +# this lock. Only the allocation is under it; the host-to-device copy runs outside. +DEVICE_ALLOC_LOCK = threading.Lock() + + +class Serialized: + """A compiled module whose calls hold `lock` (other attributes pass through).""" + + def __init__(self, fn, lock=DEVICE_ALLOC_LOCK): + self.fn, self.lock = fn, lock + + def __call__(self, *args): + with self.lock: + return self.fn(*args) + + def __getattr__(self, name): + return getattr(self.fn, name) + + class FuriosaBackend(Backend): """FuriosaAI RNGD. furiosa-torch 의 CompileModule 로 그래프를 컴파일한다.""" name = "furiosa" @@ -189,13 +212,23 @@ def compile(self, key, gm, sample): self._save(m, path) except Exception as e: print(f"[furiosa] could not cache {key}: {e!r}", flush=True) - if not self.compile_only: + if self.compile_only: + print(f"[furiosa] {key}: {mode} {time.time() - t0:.0f}s", flush=True) + return m + with DEVICE_ALLOC_LOCK: m.to(self.dev) print(f"[furiosa] {key}: {mode} {time.time() - t0:.0f}s", flush=True) - return m + return Serialized(m) def to_device(self, t): - return t if self.compile_only else t.to(self.dev) + if self.compile_only: + return t + if t.device.type != "cpu": + return t.to(self.dev) + # allocate under the lock (see DEVICE_ALLOC_LOCK), copy outside it: same speed as t.to(dev), 6.0 GB/s + with DEVICE_ALLOC_LOCK: + out = torch.empty(t.shape, dtype=t.dtype, device=self.dev) + return out.copy_(t) def to_host(self, t, out=None): # 미리 잡아 둔 버퍼로 받으면 19.9 GB/s, t.cpu() 는 매번 새 메모리를 잡느라 3.0 GB/s (RNGD) diff --git a/tests/test_cpu.py b/tests/test_cpu.py index dc94203..74b9dba 100644 --- a/tests/test_cpu.py +++ b/tests/test_cpu.py @@ -187,6 +187,30 @@ def test_visible_chip_picks_the_nth_assigned_card(): os.environ.update(env) +def test_serialized_graph_call_holds_the_device_alloc_lock(): + """#40: a compiled graph call must not overlap a device allocation from a layer worker thread""" + import threading + from npu_lora import backends + + class Mod: + tag = "edf" + + def __call__(self, x): + assert backends.DEVICE_ALLOC_LOCK.locked() + return x + 1 + + m = backends.Serialized(Mod()) + assert m(1) == 2 and m.tag == "edf" and not backends.DEVICE_ALLOC_LOCK.locked() + order = [] + with backends.DEVICE_ALLOC_LOCK: # a worker is allocating: the call waits for it + t = threading.Thread(target=lambda: order.append(m(0))) + t.start() + t.join(0.2) + order.append("alloc done") + t.join() + assert order == ["alloc done", 1] + + def test_rope_mm_equals_rope(): x = torch.randn(4, 16, 32) cos, sin = torch.randn(16, 32), torch.randn(16, 32)