From e6a8a9653cf871489b3214b8fee0dc0c52dbaef6 Mon Sep 17 00:00:00 2001 From: gspark Date: Sat, 24 Oct 2026 00:19:10 +0900 Subject: [PATCH] test: validate live pending-prefill faults and explicit worker replacement (#244) --- .../examples/pd/serving_matrix/COVERAGE.md | 6 +- .../examples/pd/serving_matrix/README.md | 7 + .../examples/pd/serving_matrix/STATUS.md | 2 + .../pd/serving_matrix/check_live_http.py | 128 +++- .../evidence244/environment-failure.json | 9 + .../serving_matrix/evidence244/events.jsonl | 3 + .../evidence244/live244-gemma-faults.json | 507 +++++++++++++++ .../evidence244/live244-koni-faults.json | 564 +++++++++++++++++ .../evidence244/live244-qwen06b-faults.json | 597 ++++++++++++++++++ .../evidence244/live244-qwen4b-faults.json | 507 +++++++++++++++ .../evidence244/live244-summary.json | 52 ++ .../2026-10-24-live-pd-fault244.md | 65 ++ .../pd/serving_matrix/observer_factory.py | 37 ++ .../serving_matrix/test_observer_factory.py | 39 ++ 14 files changed, 2503 insertions(+), 20 deletions(-) create mode 100644 opt_prime/examples/pd/serving_matrix/evidence244/environment-failure.json create mode 100644 opt_prime/examples/pd/serving_matrix/evidence244/events.jsonl create mode 100644 opt_prime/examples/pd/serving_matrix/evidence244/live244-gemma-faults.json create mode 100644 opt_prime/examples/pd/serving_matrix/evidence244/live244-koni-faults.json create mode 100644 opt_prime/examples/pd/serving_matrix/evidence244/live244-qwen06b-faults.json create mode 100644 opt_prime/examples/pd/serving_matrix/evidence244/live244-qwen4b-faults.json create mode 100644 opt_prime/examples/pd/serving_matrix/evidence244/live244-summary.json create mode 100644 opt_prime/examples/pd/serving_matrix/experiments/2026-10-24-live-pd-fault244.md create mode 100644 opt_prime/examples/pd/serving_matrix/observer_factory.py create mode 100644 opt_prime/examples/pd/serving_matrix/test_observer_factory.py diff --git a/opt_prime/examples/pd/serving_matrix/COVERAGE.md b/opt_prime/examples/pd/serving_matrix/COVERAGE.md index 9eb3bc2b..744931f6 100644 --- a/opt_prime/examples/pd/serving_matrix/COVERAGE.md +++ b/opt_prime/examples/pd/serving_matrix/COVERAGE.md @@ -41,4 +41,8 @@ Four models,48producer runs,31/32 EOS comparisons at6026–6428input tokens. Bot All four short-input paths pass14normal/concurrent/reuse responses each:56total,64remote prefills including cancellation requests,0local fallbacks. Requests trigger actual RNGD subprocess computation; no KV replay. Each model passes explicit cancellation, socket disconnection and subsequent reuse. Gemma/Qwen usek3; static B1 KONI usesk1 with#236. SIGPIPE-safe HTTP is#214/#215. These are B1 hardware requests with queued client concurrency, not hardware batch4. Native parity includes shared wrong answers. -Width2048 live checks now pass for all four models:56responses and64remote prefills. Short+medium total112responses/128prefills, with no local fallbacks. Width8192 live checks also pass all four models:56responses/64remote prefills at6026–6428input tokens. Combined short+medium+long:168responses/192remote prefills,0fallback. Pending-prefill hardware cancellation/worker-failure recovery, cross-host RNGD-to-GPU live PD, hybrid prefix/host restore, applicable hardware batch>1, genuine producer/transfer overlap and matched native-vs-PD latency/throughput remain unverified. GPU-to-GPU live HTTP is tested separately under#240: default BF16 concurrent35/48; explicit uncached/chunk256 producer controls#241 recover48/48concurrent and12/12native/PD serial outputs. Gemma numerical#233 remains unresolved. Overall goal remains active. +Width2048 live checks now pass for all four models:56responses and64remote prefills. Short+medium total112responses/128prefills, with no local fallbacks. Width8192 live checks also pass all four models:56responses/64remote prefills at6026–6428input tokens. Combined short+medium+long:168responses/192remote prefills,0fallback. Readiness during terminal producer failure/automatic service restart, cross-host RNGD-to-GPU live PD, hybrid prefix/host restore, applicable hardware batch>1, genuine producer/transfer overlap and matched native-vs-PD latency/throughput remain unverified. GPU-to-GPU live HTTP is tested separately under#240: default BF16 concurrent35/48; explicit uncached/chunk256 producer controls#241 recover48/48concurrent and12/12native/PD serial outputs. Gemma numerical#233 remains unresolved. Overall goal remains active. + +## Pending-prefill hardware faults (#244) + +All four models pass width2048 native-pending cancellation499, owned worker termination with two queued/in-flight500responses plus subsequent500, shared-memory unlink and explicit replacement parity.64/64normal outputs,88remote admission attempts including failures,0fallback. Error/cleanup0.4125–2.2284s excludes replacement initialization. Native-entry markers plus unfinished Futures confirm fault timing; no artificial gate/replay. CPU adapters50passed+5subtests. This is explicit replacement at an idle boundary, not automatic production restart; readiness while terminally unavailable remains to audit. See live-pd-fault244 record/evidence244. diff --git a/opt_prime/examples/pd/serving_matrix/README.md b/opt_prime/examples/pd/serving_matrix/README.md index be922bfe..f2108aca 100644 --- a/opt_prime/examples/pd/serving_matrix/README.md +++ b/opt_prime/examples/pd/serving_matrix/README.md @@ -178,3 +178,10 @@ Short-input producer/full-logit parity and real shared-memory requests are verif `check_live_http.py` runs actual RNGD prefill and ATOM HTTP decode from token fixtures (it does not load saved KV). Pass `--batching` with the reviewed ATOM scheduler, `--http-module` with the #214 SIGPIPE-fixed server, the model/artifact cache, `--worker-python`, factory/width and fixture/baseline JSON. It checks two serial passes, four queued clients, cancel/disconnect and reuse. Factories: `prefill:Prefill` for short Qwen3, `long_width_variant:make_prefill` for the explicit experimental long Qwen variant, `gemma_prefill:Prefill`, and `koni_prefill:Prefill` for partitioned KONI widths divisible by512. KONI uses `--kv-blocks 1` with #236; other measured artifacts use3. Explicit head_dim is preserved; Llama's missing field is inferred from hidden size/attention heads. All four short-input live HTTP paths pass56normal/reuse response checks and64remote prefills including cancellations. Native-output parity does not imply answer quality. Width2048 live validation also passes all four models; width8192 live queue is pending; hardware batch remains1 and no matched performance benefit is established. See live-http234 record. + +### Native pending-prefill fault checks + +`check_live_http.py --prefill-markers "$MARKERS" --engine-factory observer_factory:make_prefill` +adds cancellation and owned-worker termination while a native call is pending, followed by explicit replacement at an idle scheduler boundary. Set `PREFILL_OBSERVER_FACTORY` to the validated underlying adapter. `--pd-module` selects a trusted local parent worker implementation (the hardware run used #239); `--http-module` and `--batching` retain their existing roles. + +The observer writes atomic start/done records without delaying native execution or replaying KV. Successful normal outputs, cancelled requests and failed remote admissions are counted separately. The mode is a destructive test of **its own child process**, not a production auto-restart mechanism. Exact four-model commands and retained failure evidence are in `experiments/2026-10-24-live-pd-fault244.md` and `evidence244/`. diff --git a/opt_prime/examples/pd/serving_matrix/STATUS.md b/opt_prime/examples/pd/serving_matrix/STATUS.md index cc1c7e9e..48f69475 100644 --- a/opt_prime/examples/pd/serving_matrix/STATUS.md +++ b/opt_prime/examples/pd/serving_matrix/STATUS.md @@ -67,3 +67,5 @@ All four models:56/56 normal/concurrent/reuse responses equal native ATOM,64RNGD Medium live HTTP:1526–1643tokens,56/56 exact outputs and64remote/0local; short+medium112responses/128remote. Same cancellation/disconnect reuse checks pass. Long live queue active; full goal remains open. See live-http-mid234. Long live HTTP follow-up: all four width8192 routes pass56/56responses and64remote/0local prefills, cancellation/disconnect reuse included. Short+medium+long168/168responses/192remote. All queue stages exited0. HardwareB1, nativeATOM equality only; no quality, performance or overlap claim. See live-http-long234 record/evidence234. + +Four-model pending-prefill fault checks#244 pass:64normal output checks/88remote attempts, cancellation499, three worker-death500responses per model, slab unlink and explicit replacement reuse.50adapter tests+5subtests pass. Initial environment ABI failure preserved. Readiness, automatic restart, matched serving performance and true overlap remain distinct work. diff --git a/opt_prime/examples/pd/serving_matrix/check_live_http.py b/opt_prime/examples/pd/serving_matrix/check_live_http.py index ffecd42d..0b5d3c7d 100644 --- a/opt_prime/examples/pd/serving_matrix/check_live_http.py +++ b/opt_prime/examples/pd/serving_matrix/check_live_http.py @@ -7,6 +7,7 @@ import sys import threading import time +import urllib.error import urllib.request from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -24,8 +25,10 @@ p.add_argument("--http-module", required=True) p.add_argument("--baseline", required=True) p.add_argument("--output", required=True) +p.add_argument("--pd-module", help="trusted local subprocess worker implementation") +p.add_argument("--prefill-markers", help="enable pending-prefill fault checks using observer_factory") a = p.parse_args() -for name, path in [ +for name, path in ([ ("opt_prime.accel.pd.pd_batching", a.pd_module) ] if a.pd_module else []) + [ ("opt_prime.accel.platforms.atom.batching", a.batching), ("opt_prime.accel.serve.llmd", a.http_module), ]: @@ -61,27 +64,31 @@ worker = None owner = thread = server = http = None rows = [] +faults = {} try: model = a.model engine.start(model) c = engine.model.config.get_text_config() - worker = SubprocessPrefillWorker( - a.worker_python, - [ - "--model", - model, - "--width", - str(a.width), - "--engine-factory", - a.engine_factory, - ], - slots=1, - layers=c.num_hidden_layers, - kv_heads=c.num_key_value_heads, - head_dim=getattr(c, "head_dim", None) or c.hidden_size // c.num_attention_heads, - max_context=a.width, - env=dict(os.environ), - ) + def new_worker(): + return SubprocessPrefillWorker( + a.worker_python, + [ + "--model", + model, + "--width", + str(a.width), + "--engine-factory", + a.engine_factory, + ], + slots=1, + layers=c.num_hidden_layers, + kv_heads=c.num_key_value_heads, + head_dim=getattr(c, "head_dim", None) or c.hidden_size // c.num_attention_heads, + max_context=a.width, + env=dict(os.environ, **({"PREFILL_OBSERVER_DIR": a.prefill_markers} if a.prefill_markers else {})), + ) + + worker = new_worker() tok = AutoTokenizer.from_pretrained(model, local_files_only=True) b = engine.batcher(prefix_cache=False, prefill_worker=worker, local_prefill=False) service = BatchedService( @@ -164,9 +171,88 @@ def complete(r, rid): time.sleep(0.01) assert mode not in b.by_id complete(cases[0], "after-" + mode) + extra_prefills = 0 + if a.prefill_markers: + from multiprocessing import shared_memory + + def wait_for(predicate, description, timeout=30): + end = time.monotonic() + timeout + while time.monotonic() < end: + value = predicate() + if value: + return value + time.sleep(0.002) + raise TimeoutError(description) + + def started(rid): + marker = Path(a.prefill_markers) / f"{worker._proc.pid}.json" + if not marker.exists(): + return None + event = json.loads(marker.read_text()) + with b._lock: + r = b.by_id.get(rid) + pending = r is not None and r.pending is not None and not r.pending.done() + return event if pending and event["phase"] == "start" else None + + def outcome(rid): + body = {"model": model, "prompt": cases[-1]["ids"], "max_tokens": 64} + try: + with request("/v1/completions", body, rid) as response: + return {"status": response.status, "body": json.load(response)} + except urllib.error.HTTPError as error: + return {"status": error.code, "body": json.loads(error.read())} + + with ThreadPoolExecutor(max_workers=2) as clients: + pending = clients.submit(outcome, "cancel-during-prefill") + event = wait_for(lambda: started("cancel-during-prefill"), "native prefill entry") + with request("/v1/cancel", {}, "cancel-during-prefill") as cancelled: + assert cancelled.status == 202 + cancelled = pending.result(timeout=30) + assert cancelled["status"] == 499, cancelled + wait_for(lambda: not b.by_id, "cancelled lane drain") + complete(cases[0], "after-pending-cancel") + faults["pending_cancel"] = {"native_entry": event, "outcome": cancelled, "reuse": True} + extra_prefills += 1 + + failed = clients.submit(outcome, "kill-during-prefill") + event = wait_for(lambda: started("kill-during-prefill"), "native prefill entry before kill") + queued = clients.submit(outcome, "queued-at-kill") + wait_for(lambda: "queued-at-kill" in b.by_id, "second request admission") + assert started("kill-during-prefill"), "native call completed before termination" + before = time.perf_counter() + worker._proc.kill() # only this experiment's owned child + results = [failed.result(timeout=30), queued.result(timeout=30)] + assert all(r["status"] == 500 for r in results), results + wait_for(lambda: not b.by_id, "failed request drain") + dead = outcome("submit-after-worker-death") + assert dead["status"] == 500, dead + wait_for(lambda: not b.by_id, "dead-worker rejection drain") + assert not b.poisoned and sorted(b.free) == [0] + name = worker._shm.name + worker.close() + try: + leaked = shared_memory.SharedMemory(name=name) + except FileNotFoundError: + pass + else: + leaked.close() + raise AssertionError("dead worker slab still exists") + faults["worker_death"] = {"native_entry": event, "outcomes": results, + "later_request": dead, "failure_s": time.perf_counter() - before, + "slab_unlinked": True} + extra_prefills += 3 + worker = new_worker() + # Explicit replacement at a verified idle boundary, not automatic retry/restart. + assert not b.by_id and sorted(b.free) == [0] + b.worker = worker + complete(cases[0], "after-worker-replacement") + faults["worker_death"]["replacement_reuse"] = True + assert b.stats_pd["local"] == 0 and sorted(b.free) == [0] - assert b.stats_pd["remote"] == len(rows) + 2 + assert b.stats_pd["remote"] == len(rows) + 2 + extra_prefills result = { + "passed": True, + "faults": faults, "worker": "live_rngd_subprocess_prefill", "rows": rows, "stats_pd": b.stats_pd, @@ -180,6 +266,10 @@ def complete(r, rid): ), flush=True, ) +except Exception as error: + Path(a.output).write_text(json.dumps({"passed": False, "error": repr(error), "rows": rows, + "faults": faults}, ensure_ascii=False, indent=2) + "\n") + raise finally: if owner: owner.submit("stop") diff --git a/opt_prime/examples/pd/serving_matrix/evidence244/environment-failure.json b/opt_prime/examples/pd/serving_matrix/evidence244/environment-failure.json new file mode 100644 index 00000000..89fc7477 --- /dev/null +++ b/opt_prime/examples/pd/serving_matrix/evidence244/environment-failure.json @@ -0,0 +1,9 @@ +{ + "passed": false, + "phase": "before engine initialization", + "cause": "Container-global Torch selected instead of established Torch2.11.0+cpu; ATOM extension ABI undefined symbol", + "initial_exit": 1, + "fix": "Select existing package base with process-local PYTHONUSERBASE; no installation or HOME modification", + "verified_torch": "2.11.0+cpu", + "verified_transformers": "5.8.1" +} diff --git a/opt_prime/examples/pd/serving_matrix/evidence244/events.jsonl b/opt_prime/examples/pd/serving_matrix/evidence244/events.jsonl new file mode 100644 index 00000000..e5bf012b --- /dev/null +++ b/opt_prime/examples/pd/serving_matrix/evidence244/events.jsonl @@ -0,0 +1,3 @@ +{"id": "11a8f7bf084d4e33b17ea4cb18fd82e4", "ts": "2026-10-24T00:05:18+09:00", "task": "Live pending-prefill fault recovery (aicomp #244)", "result": "Prepared native-call observer and live HTTP cancellation/termination/replacement checks; launched owned RNGD+ATOM Qwen0.6B width2048 run35484 using subprocess369c8ff. No fault success claimed yet.", "next": "Monitor same process and verify pending native entry, bounded errors, slab cleanup and replacement output parity; preserve failure evidence.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"width": "2048", "hardware_batch": "1"}, "artifacts": [], "experiments": ["experiments/2026-10-24-live-pd-fault244.md"], "labels": ["experiment", "rngd"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} +{"id": "9c5a3488784d400d8f179c382b8e0e29", "ts": "2026-10-24T00:10:47+09:00", "task": "Live pending-prefill fault recovery (aicomp #244)", "result": "Qwen0.6B real RNGD->ATOM pending cancel499/reuse pass; owned worker kill yields3bounded500responses,slab unlinked and replacement parity pass(0.4125s failure+cleanup).16successful outputs/22remote attempts. Environment failure retained;3model queue57029 active.", "next": "Collect remaining Qwen4B/Gemma/KONI fault results and publish instrumentation/evidence; no automatic-restart claim.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"width": "2048", "hardware_batch": "1"}, "artifacts": [{"path": "analysis/serving-matrix-201/live244-qwen06b-faults.json", "size": 13654, "sha256": "8bd4bfe2eac28439"}], "experiments": ["experiments/2026-10-24-live-pd-fault244.md"], "labels": ["experiment", "rngd"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} +{"id": "f4346de8bd5c4af084f29e6f65d979ac", "ts": "2026-10-24T00:17:06+09:00", "task": "Live pending-prefill fault recovery (aicomp #244)", "result": "All4models pass real RNGD->ATOM pending-cancel499,worker-death3x500,slab unlink and explicit replacement parity.64/64normal outputs,88remote attempts/0fallback. Error+cleanup0.4125–2.2284s excluding restart.50adapter tests+5subtests pass; all jobs terminal.", "next": "Publish fault instrumentation/evidence; audit dead-producer readiness, then matched performance and compute-transfer overlap.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"width": "2048", "hardware_batch": "1"}, "artifacts": [{"path": "analysis/serving-matrix-201/live244-summary.json", "size": 1418, "sha256": "81c8de87ffdba800"}], "experiments": ["experiments/2026-10-24-live-pd-fault244.md"], "labels": ["experiment", "rngd"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} diff --git a/opt_prime/examples/pd/serving_matrix/evidence244/live244-gemma-faults.json b/opt_prime/examples/pd/serving_matrix/evidence244/live244-gemma-faults.json new file mode 100644 index 00000000..8a4adbf0 --- /dev/null +++ b/opt_prime/examples/pd/serving_matrix/evidence244/live244-gemma-faults.json @@ -0,0 +1,507 @@ +{ + "passed": true, + "faults": { + "pending_cancel": { + "native_entry": { + "sequence": 17, + "phase": "start", + "tokens": 1628, + "time": 743246.128749334 + }, + "outcome": { + "status": 499, + "body": { + "error": "request cancelled", + "retryable": false + } + }, + "reuse": true + }, + "worker_death": { + "native_entry": { + "sequence": 19, + "phase": "start", + "tokens": 1628, + "time": 743250.099539859 + }, + "outcomes": [ + { + "status": 500, + "body": { + "error": "engine failed; owner retained", + "retryable": false + } + }, + { + "status": 500, + "body": { + "error": "engine failed; owner retained", + "retryable": false + } + } + ], + "later_request": { + "status": 500, + "body": { + "error": "engine failed; owner retained", + "retryable": false + } + }, + "failure_s": 0.5187927359947935, + "slab_unlinked": true, + "replacement_reuse": true + } + }, + "worker": "live_rngd_subprocess_prefill", + "rows": [ + { + "case": "capital-100", + "request": "normal-0-0", + "response": { + "id": "normal-0-0", + "object": "text_completion", + "created": 1792768240, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 50429, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1623, + "completion_tokens": 2, + "total_tokens": 1625 + } + }, + "latency_s": 2.0564947499660775 + }, + { + "case": "arithmetic-100", + "request": "normal-0-1", + "response": { + "id": "normal-0-1", + "object": "text_completion", + "created": 1792768243, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "40\n", + "token_ids": [ + 236812, + 236771, + 107, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1624, + "completion_tokens": 4, + "total_tokens": 1628 + } + }, + "latency_s": 2.0516654840903357 + }, + { + "case": "korean-100", + "request": "normal-0-2", + "response": { + "id": "normal-0-2", + "object": "text_completion", + "created": 1792768245, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "서울", + "token_ids": [ + 98665, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1624, + "completion_tokens": 2, + "total_tokens": 1626 + } + }, + "latency_s": 2.014097706065513 + }, + { + "case": "retrieval-100", + "request": "normal-0-3", + "response": { + "id": "normal-0-3", + "object": "text_completion", + "created": 1792768247, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "MAPLE", + "token_ids": [ + 28487, + 1618, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1628, + "completion_tokens": 3, + "total_tokens": 1631 + } + }, + "latency_s": 2.0312483090674505 + }, + { + "case": "capital-100", + "request": "normal-1-0", + "response": { + "id": "normal-1-0", + "object": "text_completion", + "created": 1792768249, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 50429, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1623, + "completion_tokens": 2, + "total_tokens": 1625 + } + }, + "latency_s": 1.9728056750027463 + }, + { + "case": "arithmetic-100", + "request": "normal-1-1", + "response": { + "id": "normal-1-1", + "object": "text_completion", + "created": 1792768251, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "40\n", + "token_ids": [ + 236812, + 236771, + 107, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1624, + "completion_tokens": 4, + "total_tokens": 1628 + } + }, + "latency_s": 2.025119759957306 + }, + { + "case": "korean-100", + "request": "normal-1-2", + "response": { + "id": "normal-1-2", + "object": "text_completion", + "created": 1792768253, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "서울", + "token_ids": [ + 98665, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1624, + "completion_tokens": 2, + "total_tokens": 1626 + } + }, + "latency_s": 2.0022191760363057 + }, + { + "case": "retrieval-100", + "request": "normal-1-3", + "response": { + "id": "normal-1-3", + "object": "text_completion", + "created": 1792768255, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "MAPLE", + "token_ids": [ + 28487, + 1618, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1628, + "completion_tokens": 3, + "total_tokens": 1631 + } + }, + "latency_s": 2.027281573973596 + }, + { + "case": "capital-100", + "request": "concurrent-0", + "response": { + "id": "concurrent-0", + "object": "text_completion", + "created": 1792768257, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 50429, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1623, + "completion_tokens": 2, + "total_tokens": 1625 + } + }, + "latency_s": 2.0133031060686335 + }, + { + "case": "korean-100", + "request": "concurrent-2", + "response": { + "id": "concurrent-2", + "object": "text_completion", + "created": 1792768259, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "서울", + "token_ids": [ + 98665, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1624, + "completion_tokens": 2, + "total_tokens": 1626 + } + }, + "latency_s": 4.028183768969029 + }, + { + "case": "arithmetic-100", + "request": "concurrent-1", + "response": { + "id": "concurrent-1", + "object": "text_completion", + "created": 1792768261, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "40\n", + "token_ids": [ + 236812, + 236771, + 107, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1624, + "completion_tokens": 4, + "total_tokens": 1628 + } + }, + "latency_s": 6.058910413994454 + }, + { + "case": "retrieval-100", + "request": "concurrent-3", + "response": { + "id": "concurrent-3", + "object": "text_completion", + "created": 1792768263, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "MAPLE", + "token_ids": [ + 28487, + 1618, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1628, + "completion_tokens": 3, + "total_tokens": 1631 + } + }, + "latency_s": 8.087063654093072 + }, + { + "case": "capital-100", + "request": "after-cancel", + "response": { + "id": "after-cancel", + "object": "text_completion", + "created": 1792768267, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 50429, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1623, + "completion_tokens": 2, + "total_tokens": 1625 + } + }, + "latency_s": 2.017120509990491 + }, + { + "case": "capital-100", + "request": "after-disconnect", + "response": { + "id": "after-disconnect", + "object": "text_completion", + "created": 1792768271, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 50429, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1623, + "completion_tokens": 2, + "total_tokens": 1625 + } + }, + "latency_s": 2.0050113649340346 + }, + { + "case": "capital-100", + "request": "after-pending-cancel", + "response": { + "id": "after-pending-cancel", + "object": "text_completion", + "created": 1792768275, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 50429, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1623, + "completion_tokens": 2, + "total_tokens": 1625 + } + }, + "latency_s": 3.9181563950842246 + }, + { + "case": "capital-100", + "request": "after-worker-replacement", + "response": { + "id": "after-worker-replacement", + "object": "text_completion", + "created": 1792768283, + "model": "unsloth/gemma-3-1b-it", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 50429, + 106 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1623, + "completion_tokens": 2, + "total_tokens": 1625 + } + }, + "latency_s": 2.0114431550027803 + } + ], + "stats_pd": { + "remote": 22, + "local": 0 + }, + "free": [ + 0 + ], + "cancellation_reuse": true +} diff --git a/opt_prime/examples/pd/serving_matrix/evidence244/live244-koni-faults.json b/opt_prime/examples/pd/serving_matrix/evidence244/live244-koni-faults.json new file mode 100644 index 00000000..1c85a9f3 --- /dev/null +++ b/opt_prime/examples/pd/serving_matrix/evidence244/live244-koni-faults.json @@ -0,0 +1,564 @@ +{ + "passed": true, + "faults": { + "pending_cancel": { + "native_entry": { + "sequence": 17, + "phase": "start", + "tokens": 1643, + "time": 743376.441240277 + }, + "outcome": { + "status": 499, + "body": { + "error": "request cancelled", + "retryable": false + } + }, + "reuse": true + }, + "worker_death": { + "native_entry": { + "sequence": 19, + "phase": "start", + "tokens": 1643, + "time": 743378.830774283 + }, + "outcomes": [ + { + "status": 500, + "body": { + "error": "engine failed; owner retained", + "retryable": false + } + }, + { + "status": 500, + "body": { + "error": "engine failed; owner retained", + "retryable": false + } + } + ], + "later_request": { + "status": 500, + "body": { + "error": "engine failed; owner retained", + "retryable": false + } + }, + "failure_s": 2.2283839259762317, + "slab_unlinked": true, + "replacement_reuse": true + } + }, + "worker": "live_rngd_subprocess_prefill", + "rows": [ + { + "case": "capital-100", + "request": "normal-0-0", + "response": { + "id": "normal-0-0", + "object": "text_completion", + "created": 1792768380, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "The capital of France is Paris.", + "token_ids": [ + 791, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1638, + "completion_tokens": 8, + "total_tokens": 1646 + } + }, + "latency_s": 2.4833756249863654 + }, + { + "case": "arithmetic-100", + "request": "normal-0-1", + "response": { + "id": "normal-0-1", + "object": "text_completion", + "created": 1792768381, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "42", + "token_ids": [ + 2983, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1637, + "completion_tokens": 2, + "total_tokens": 1639 + } + }, + "latency_s": 1.3839052859693766 + }, + { + "case": "korean-100", + "request": "normal-0-2", + "response": { + "id": "normal-0-2", + "object": "text_completion", + "created": 1792768383, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "대한민국의 수도는 서울입니다.", + "token_ids": [ + 113760, + 108709, + 21028, + 116992, + 16969, + 106010, + 80052, + 13, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1639, + "completion_tokens": 9, + "total_tokens": 1648 + } + }, + "latency_s": 1.8056919019436464 + }, + { + "case": "retrieval-100", + "request": "normal-0-3", + "response": { + "id": "normal-0-3", + "object": "text_completion", + "created": 1792768384, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "MAPLE", + "token_ids": [ + 18082, + 877, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1643, + "completion_tokens": 3, + "total_tokens": 1646 + } + }, + "latency_s": 1.362900058971718 + }, + { + "case": "capital-100", + "request": "normal-1-0", + "response": { + "id": "normal-1-0", + "object": "text_completion", + "created": 1792768386, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "The capital of France is Paris.", + "token_ids": [ + 791, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1638, + "completion_tokens": 8, + "total_tokens": 1646 + } + }, + "latency_s": 1.5326326549984515 + }, + { + "case": "arithmetic-100", + "request": "normal-1-1", + "response": { + "id": "normal-1-1", + "object": "text_completion", + "created": 1792768387, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "42", + "token_ids": [ + 2983, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1637, + "completion_tokens": 2, + "total_tokens": 1639 + } + }, + "latency_s": 1.2049848400056362 + }, + { + "case": "korean-100", + "request": "normal-1-2", + "response": { + "id": "normal-1-2", + "object": "text_completion", + "created": 1792768389, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "대한민국의 수도는 서울입니다.", + "token_ids": [ + 113760, + 108709, + 21028, + 116992, + 16969, + 106010, + 80052, + 13, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1639, + "completion_tokens": 9, + "total_tokens": 1648 + } + }, + "latency_s": 1.6321332439547405 + }, + { + "case": "retrieval-100", + "request": "normal-1-3", + "response": { + "id": "normal-1-3", + "object": "text_completion", + "created": 1792768390, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "MAPLE", + "token_ids": [ + 18082, + 877, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1643, + "completion_tokens": 3, + "total_tokens": 1646 + } + }, + "latency_s": 1.2548069320619106 + }, + { + "case": "capital-100", + "request": "concurrent-0", + "response": { + "id": "concurrent-0", + "object": "text_completion", + "created": 1792768392, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "The capital of France is Paris.", + "token_ids": [ + 791, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1638, + "completion_tokens": 8, + "total_tokens": 1646 + } + }, + "latency_s": 1.5685934909852222 + }, + { + "case": "korean-100", + "request": "concurrent-2", + "response": { + "id": "concurrent-2", + "object": "text_completion", + "created": 1792768393, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "대한민국의 수도는 서울입니다.", + "token_ids": [ + 113760, + 108709, + 21028, + 116992, + 16969, + 106010, + 80052, + 13, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1639, + "completion_tokens": 9, + "total_tokens": 1648 + } + }, + "latency_s": 3.137992416974157 + }, + { + "case": "retrieval-100", + "request": "concurrent-3", + "response": { + "id": "concurrent-3", + "object": "text_completion", + "created": 1792768394, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "MAPLE", + "token_ids": [ + 18082, + 877, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1643, + "completion_tokens": 3, + "total_tokens": 1646 + } + }, + "latency_s": 4.311101814964786 + }, + { + "case": "arithmetic-100", + "request": "concurrent-1", + "response": { + "id": "concurrent-1", + "object": "text_completion", + "created": 1792768395, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "42", + "token_ids": [ + 2983, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1637, + "completion_tokens": 2, + "total_tokens": 1639 + } + }, + "latency_s": 5.399370403960347 + }, + { + "case": "capital-100", + "request": "after-cancel", + "response": { + "id": "after-cancel", + "object": "text_completion", + "created": 1792768398, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "The capital of France is Paris.", + "token_ids": [ + 791, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1638, + "completion_tokens": 8, + "total_tokens": 1646 + } + }, + "latency_s": 1.573151625925675 + }, + { + "case": "capital-100", + "request": "after-disconnect", + "response": { + "id": "after-disconnect", + "object": "text_completion", + "created": 1792768401, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "The capital of France is Paris.", + "token_ids": [ + 791, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1638, + "completion_tokens": 8, + "total_tokens": 1646 + } + }, + "latency_s": 1.574550086981617 + }, + { + "case": "capital-100", + "request": "after-pending-cancel", + "response": { + "id": "after-pending-cancel", + "object": "text_completion", + "created": 1792768403, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "The capital of France is Paris.", + "token_ids": [ + 791, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1638, + "completion_tokens": 8, + "total_tokens": 1646 + } + }, + "latency_s": 2.335373285925016 + }, + { + "case": "capital-100", + "request": "after-worker-replacement", + "response": { + "id": "after-worker-replacement", + "object": "text_completion", + "created": 1792768478, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "The capital of France is Paris.", + "token_ids": [ + 791, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1638, + "completion_tokens": 8, + "total_tokens": 1646 + } + }, + "latency_s": 2.3037637390661985 + } + ], + "stats_pd": { + "remote": 22, + "local": 0 + }, + "free": [ + 0 + ], + "cancellation_reuse": true +} diff --git a/opt_prime/examples/pd/serving_matrix/evidence244/live244-qwen06b-faults.json b/opt_prime/examples/pd/serving_matrix/evidence244/live244-qwen06b-faults.json new file mode 100644 index 00000000..ddd90df8 --- /dev/null +++ b/opt_prime/examples/pd/serving_matrix/evidence244/live244-qwen06b-faults.json @@ -0,0 +1,597 @@ +{ + "passed": true, + "faults": { + "pending_cancel": { + "native_entry": { + "sequence": 17, + "phase": "start", + "tokens": 1531, + "time": 742994.278980963 + }, + "outcome": { + "status": 499, + "body": { + "error": "request cancelled", + "retryable": false + } + }, + "reuse": true + }, + "worker_death": { + "native_entry": { + "sequence": 19, + "phase": "start", + "tokens": 1531, + "time": 742995.110346686 + }, + "outcomes": [ + { + "status": 500, + "body": { + "error": "engine failed; owner retained", + "retryable": false + } + }, + { + "status": 500, + "body": { + "error": "engine failed; owner retained", + "retryable": false + } + } + ], + "later_request": { + "status": 500, + "body": { + "error": "engine failed; owner retained", + "retryable": false + } + }, + "failure_s": 0.41250347695313394, + "slab_unlinked": true, + "replacement_reuse": true + } + }, + "worker": "live_rngd_subprocess_prefill", + "rows": [ + { + "case": "capital-100", + "request": "normal-0-0", + "response": { + "id": "normal-0-0", + "object": "text_completion", + "created": 1792768011, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 0.7184591679833829 + }, + { + "case": "arithmetic-100", + "request": "normal-0-1", + "response": { + "id": "normal-0-1", + "object": "text_completion", + "created": 1792768011, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "17 + 25 = 42\n\nReply with only the integer: 42.", + "token_ids": [ + 16, + 22, + 488, + 220, + 17, + 20, + 284, + 220, + 19, + 17, + 271, + 20841, + 448, + 1172, + 279, + 7546, + 25, + 220, + 19, + 17, + 13, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1527, + "completion_tokens": 22, + "total_tokens": 1549 + } + }, + "latency_s": 0.6511491970159113 + }, + { + "case": "korean-100", + "request": "normal-0-2", + "response": { + "id": "normal-0-2", + "object": "text_completion", + "created": 1792768012, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "대한민국의 수도는 **인천**입니다.", + "token_ids": [ + 66845, + 23573, + 125496, + 124785, + 20401, + 134013, + 16560, + 3070, + 31328, + 129034, + 334, + 78952, + 13, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1531, + "completion_tokens": 14, + "total_tokens": 1545 + } + }, + "latency_s": 0.5921240509487689 + }, + { + "case": "retrieval-100", + "request": "normal-0-3", + "response": { + "id": "normal-0-3", + "object": "text_completion", + "created": 1792768012, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "MAPLE", + "token_ids": [ + 17617, + 867, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1531, + "completion_tokens": 3, + "total_tokens": 1534 + } + }, + "latency_s": 0.48913957504555583 + }, + { + "case": "capital-100", + "request": "normal-1-0", + "response": { + "id": "normal-1-0", + "object": "text_completion", + "created": 1792768013, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 0.5172563960077241 + }, + { + "case": "arithmetic-100", + "request": "normal-1-1", + "response": { + "id": "normal-1-1", + "object": "text_completion", + "created": 1792768013, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "17 + 25 = 42\n\nReply with only the integer: 42.", + "token_ids": [ + 16, + 22, + 488, + 220, + 17, + 20, + 284, + 220, + 19, + 17, + 271, + 20841, + 448, + 1172, + 279, + 7546, + 25, + 220, + 19, + 17, + 13, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1527, + "completion_tokens": 22, + "total_tokens": 1549 + } + }, + "latency_s": 0.6564970829058439 + }, + { + "case": "korean-100", + "request": "normal-1-2", + "response": { + "id": "normal-1-2", + "object": "text_completion", + "created": 1792768014, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "대한민국의 수도는 **인천**입니다.", + "token_ids": [ + 66845, + 23573, + 125496, + 124785, + 20401, + 134013, + 16560, + 3070, + 31328, + 129034, + 334, + 78952, + 13, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1531, + "completion_tokens": 14, + "total_tokens": 1545 + } + }, + "latency_s": 0.6137128290720284 + }, + { + "case": "retrieval-100", + "request": "normal-1-3", + "response": { + "id": "normal-1-3", + "object": "text_completion", + "created": 1792768015, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "MAPLE", + "token_ids": [ + 17617, + 867, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1531, + "completion_tokens": 3, + "total_tokens": 1534 + } + }, + "latency_s": 0.4792871120152995 + }, + { + "case": "capital-100", + "request": "concurrent-0", + "response": { + "id": "concurrent-0", + "object": "text_completion", + "created": 1792768015, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 0.50218248704914 + }, + { + "case": "korean-100", + "request": "concurrent-2", + "response": { + "id": "concurrent-2", + "object": "text_completion", + "created": 1792768016, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "대한민국의 수도는 **인천**입니다.", + "token_ids": [ + 66845, + 23573, + 125496, + 124785, + 20401, + 134013, + 16560, + 3070, + 31328, + 129034, + 334, + 78952, + 13, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1531, + "completion_tokens": 14, + "total_tokens": 1545 + } + }, + "latency_s": 1.0984293079236522 + }, + { + "case": "arithmetic-100", + "request": "concurrent-1", + "response": { + "id": "concurrent-1", + "object": "text_completion", + "created": 1792768016, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "17 + 25 = 42\n\nReply with only the integer: 42.", + "token_ids": [ + 16, + 22, + 488, + 220, + 17, + 20, + 284, + 220, + 19, + 17, + 271, + 20841, + 448, + 1172, + 279, + 7546, + 25, + 220, + 19, + 17, + 13, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1527, + "completion_tokens": 22, + "total_tokens": 1549 + } + }, + "latency_s": 1.7432945950422436 + }, + { + "case": "retrieval-100", + "request": "concurrent-3", + "response": { + "id": "concurrent-3", + "object": "text_completion", + "created": 1792768017, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "MAPLE", + "token_ids": [ + 17617, + 867, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1531, + "completion_tokens": 3, + "total_tokens": 1534 + } + }, + "latency_s": 2.2537973639555275 + }, + { + "case": "capital-100", + "request": "after-cancel", + "response": { + "id": "after-cancel", + "object": "text_completion", + "created": 1792768018, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 0.5118080760585144 + }, + { + "case": "capital-100", + "request": "after-disconnect", + "response": { + "id": "after-disconnect", + "object": "text_completion", + "created": 1792768019, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 0.5619687719736248 + }, + { + "case": "capital-100", + "request": "after-pending-cancel", + "response": { + "id": "after-pending-cancel", + "object": "text_completion", + "created": 1792768020, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 0.778010827023536 + }, + { + "case": "capital-100", + "request": "after-worker-replacement", + "response": { + "id": "after-worker-replacement", + "object": "text_completion", + "created": 1792768026, + "model": "Qwen/Qwen3-0.6B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 0.5777030759491026 + } + ], + "stats_pd": { + "remote": 22, + "local": 0 + }, + "free": [ + 0 + ], + "cancellation_reuse": true +} diff --git a/opt_prime/examples/pd/serving_matrix/evidence244/live244-qwen4b-faults.json b/opt_prime/examples/pd/serving_matrix/evidence244/live244-qwen4b-faults.json new file mode 100644 index 00000000..49f662af --- /dev/null +++ b/opt_prime/examples/pd/serving_matrix/evidence244/live244-qwen4b-faults.json @@ -0,0 +1,507 @@ +{ + "passed": true, + "faults": { + "pending_cancel": { + "native_entry": { + "sequence": 17, + "phase": "start", + "tokens": 1531, + "time": 743170.661368717 + }, + "outcome": { + "status": 499, + "body": { + "error": "request cancelled", + "retryable": false + } + }, + "reuse": true + }, + "worker_death": { + "native_entry": { + "sequence": 19, + "phase": "start", + "tokens": 1531, + "time": 743172.1345372 + }, + "outcomes": [ + { + "status": 500, + "body": { + "error": "engine failed; owner retained", + "retryable": false + } + }, + { + "status": 500, + "body": { + "error": "engine failed; owner retained", + "retryable": false + } + } + ], + "later_request": { + "status": 500, + "body": { + "error": "engine failed; owner retained", + "retryable": false + } + }, + "failure_s": 1.3533421650063246, + "slab_unlinked": true, + "replacement_reuse": true + } + }, + "worker": "live_rngd_subprocess_prefill", + "rows": [ + { + "case": "capital-100", + "request": "normal-0-0", + "response": { + "id": "normal-0-0", + "object": "text_completion", + "created": 1792768182, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 1.1837089440086856 + }, + { + "case": "arithmetic-100", + "request": "normal-0-1", + "response": { + "id": "normal-0-1", + "object": "text_completion", + "created": 1792768182, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "42", + "token_ids": [ + 19, + 17, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1527, + "completion_tokens": 3, + "total_tokens": 1530 + } + }, + "latency_s": 0.8890542679000646 + }, + { + "case": "korean-100", + "request": "normal-0-2", + "response": { + "id": "normal-0-2", + "object": "text_completion", + "created": 1792768183, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "서울", + "token_ids": [ + 26698, + 126893, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1531, + "completion_tokens": 3, + "total_tokens": 1534 + } + }, + "latency_s": 0.9133233150932938 + }, + { + "case": "retrieval-100", + "request": "normal-0-3", + "response": { + "id": "normal-0-3", + "object": "text_completion", + "created": 1792768184, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "MAPLE", + "token_ids": [ + 17617, + 867, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1531, + "completion_tokens": 3, + "total_tokens": 1534 + } + }, + "latency_s": 0.922397986985743 + }, + { + "case": "capital-100", + "request": "normal-1-0", + "response": { + "id": "normal-1-0", + "object": "text_completion", + "created": 1792768185, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 0.8858926560496911 + }, + { + "case": "arithmetic-100", + "request": "normal-1-1", + "response": { + "id": "normal-1-1", + "object": "text_completion", + "created": 1792768186, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "42", + "token_ids": [ + 19, + 17, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1527, + "completion_tokens": 3, + "total_tokens": 1530 + } + }, + "latency_s": 0.9197248310083523 + }, + { + "case": "korean-100", + "request": "normal-1-2", + "response": { + "id": "normal-1-2", + "object": "text_completion", + "created": 1792768187, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "서울", + "token_ids": [ + 26698, + 126893, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1531, + "completion_tokens": 3, + "total_tokens": 1534 + } + }, + "latency_s": 0.9242150940699503 + }, + { + "case": "retrieval-100", + "request": "normal-1-3", + "response": { + "id": "normal-1-3", + "object": "text_completion", + "created": 1792768188, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "MAPLE", + "token_ids": [ + 17617, + 867, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1531, + "completion_tokens": 3, + "total_tokens": 1534 + } + }, + "latency_s": 0.9364806730300188 + }, + { + "case": "capital-100", + "request": "concurrent-0", + "response": { + "id": "concurrent-0", + "object": "text_completion", + "created": 1792768189, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 0.8895177990198135 + }, + { + "case": "korean-100", + "request": "concurrent-2", + "response": { + "id": "concurrent-2", + "object": "text_completion", + "created": 1792768190, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "서울", + "token_ids": [ + 26698, + 126893, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1531, + "completion_tokens": 3, + "total_tokens": 1534 + } + }, + "latency_s": 1.8021970769623294 + }, + { + "case": "arithmetic-100", + "request": "concurrent-1", + "response": { + "id": "concurrent-1", + "object": "text_completion", + "created": 1792768191, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "42", + "token_ids": [ + 19, + 17, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1527, + "completion_tokens": 3, + "total_tokens": 1530 + } + }, + "latency_s": 2.7133366559864953 + }, + { + "case": "retrieval-100", + "request": "concurrent-3", + "response": { + "id": "concurrent-3", + "object": "text_completion", + "created": 1792768192, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "MAPLE", + "token_ids": [ + 17617, + 867, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1531, + "completion_tokens": 3, + "total_tokens": 1534 + } + }, + "latency_s": 3.643704427056946 + }, + { + "case": "capital-100", + "request": "after-cancel", + "response": { + "id": "after-cancel", + "object": "text_completion", + "created": 1792768193, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 0.8845278210937977 + }, + { + "case": "capital-100", + "request": "after-disconnect", + "response": { + "id": "after-disconnect", + "object": "text_completion", + "created": 1792768195, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 0.8707106240326539 + }, + { + "case": "capital-100", + "request": "after-pending-cancel", + "response": { + "id": "after-pending-cancel", + "object": "text_completion", + "created": 1792768197, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 1.4200673369923607 + }, + { + "case": "capital-100", + "request": "after-worker-replacement", + "response": { + "id": "after-worker-replacement", + "object": "text_completion", + "created": 1792768220, + "model": "Qwen/Qwen3-4B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 59604, + 151645 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1526, + "completion_tokens": 2, + "total_tokens": 1528 + } + }, + "latency_s": 0.9736899510025978 + } + ], + "stats_pd": { + "remote": 22, + "local": 0 + }, + "free": [ + 0 + ], + "cancellation_reuse": true +} diff --git a/opt_prime/examples/pd/serving_matrix/evidence244/live244-summary.json b/opt_prime/examples/pd/serving_matrix/evidence244/live244-summary.json new file mode 100644 index 00000000..af4773e0 --- /dev/null +++ b/opt_prime/examples/pd/serving_matrix/evidence244/live244-summary.json @@ -0,0 +1,52 @@ +{ + "models": [ + { + "model": "qwen06b", + "normal_output_checks": 16, + "remote_attempts": 22, + "local_fallbacks": 0, + "pending_cancel_status": 499, + "failed_requests": 3, + "failure_cleanup_s": 0.41250347695313394, + "slab_unlinked": true, + "replacement_reuse": true + }, + { + "model": "qwen4b", + "normal_output_checks": 16, + "remote_attempts": 22, + "local_fallbacks": 0, + "pending_cancel_status": 499, + "failed_requests": 3, + "failure_cleanup_s": 1.3533421650063246, + "slab_unlinked": true, + "replacement_reuse": true + }, + { + "model": "gemma", + "normal_output_checks": 16, + "remote_attempts": 22, + "local_fallbacks": 0, + "pending_cancel_status": 499, + "failed_requests": 3, + "failure_cleanup_s": 0.5187927359947935, + "slab_unlinked": true, + "replacement_reuse": true + }, + { + "model": "koni", + "normal_output_checks": 16, + "remote_attempts": 22, + "local_fallbacks": 0, + "pending_cancel_status": 499, + "failed_requests": 3, + "failure_cleanup_s": 2.2283839259762317, + "slab_unlinked": true, + "replacement_reuse": true + } + ], + "normal_output_checks": 64, + "remote_attempts": 88, + "local_fallbacks": 0, + "scope": "native call pending cancellation and owned worker death; explicit replacement, not automatic restart" +} diff --git a/opt_prime/examples/pd/serving_matrix/experiments/2026-10-24-live-pd-fault244.md b/opt_prime/examples/pd/serving_matrix/experiments/2026-10-24-live-pd-fault244.md new file mode 100644 index 00000000..76c9db50 --- /dev/null +++ b/opt_prime/examples/pd/serving_matrix/experiments/2026-10-24-live-pd-fault244.md @@ -0,0 +1,65 @@ +# Pending-prefill cancellation and worker replacement on live RNGD to ATOM + +- **Date**: 2026-10-24 +- **Task**: aicomp #244 +- **Host / device**: existing owned paired RNGD+ATOM allocation, one device each; ATOM B1/context9216/k3 +- **Commit**: adapter04dab7c plus exp/244-live-pd-faults instrumentation; subprocess worker369c8ff, ATOM schedulerdf02223, HTTP9e442cd + +## Question + +Can a real pending RNGD prefill be cancelled without corrupting the next ATOM request? Does terminating its owned worker fail queued requests promptly and allow explicit healthy replacement? + +## Setup + +Four models, width2048, four medium semantic cases1526–1643tokens. Qwen3-0.6B runs first, followed by Qwen3-4B,Gemma3-1B,KONI8B with their validated underlying factories; KONI usesk1, Gemma corrected artifactroot. Reuse the existing artifact and input/native-output fixtures. Test-only observer_factory marks native prefill entry and completion atomically; it does not delay execution, gate results or replay saved KV. Fault injection requires both an entry marker and an unfinished request Future. The instrumented parent imports the already CPU-tested SubprocessPrefillWorker candidate process-locally; shared SDK installations are unchanged. + +The standard14output checks and16remote requests run first. Then cancel during native prefill, verify reuse, terminate only this harness's worker while a second HTTP request is queued, assert both fail and a later request fails, close/unlink the dead slab, explicitly construct a replacement worker at an idle boundary and verify native-output equality. This is not automatic service restart or replay. No device-wide reset or other workload is touched. + +## Command + +Inside the owned paired allocation, with the existing ATOM bootstrap/environment: + +```bash +bash run-live-http244.sh --batching atom_batching236.py --http-module llmd214.py \ + --pd-module pd_batching238.py --cache-dir atom-artifacts --kv-blocks 3 \ + --model Qwen/Qwen3-0.6B --fixtures semantic227-qwen06b-fixtures.json \ + --cases capital-100 arithmetic-100 korean-100 retrieval-100 \ + --worker-python "$SCRATCH/worker_python234.sh" --engine-factory observer_factory:make_prefill \ + --width 2048 --baseline semantic227-qwen06b-atom.jsonl \ + --prefill-markers "$SCRATCH/fault244-markers" --output live244-qwen06b-faults.json +``` + +The launch script selects check_live_http244.py, a snapshot of the checked-out validator. Worker/ATOM/HTTP module snapshots are pinned to the commits above. Producer SDK and decoder versions match live-http-long234. + +## Results + +Initial environment failure: session35484 exited1 before engine initialization because the interpreter selected container-global Torch instead of the established user-site Torch2.11.0+cpu. The ATOM extension reported an undefined symbol. Preserve live244-qwen06b-faults-env-failed.log. Selecting the established package base through PYTHONUSERBASE for this process restored Torch2.11.0+cpu/Transformers5.8.1 without changing any installation or HOME. + +Qwen3-0.6B rerun75749 exited0:16/16 successful outputs match nativeATOM,22remote admission attempts/0local. Native-entry sequence17 at1531tokens was cancelled withHTTP499 and reuse passed. Sequence19 was terminated while a second request queued; both returned500, a later request also returned500, all lanes drained and the dead slab was unlinked. Failure/cleanup time0.4125s. Explicit replacement worker then produced matching output. Counter22includes failed remote admissions, not22successful native computations. + +Three-model queue57029 completed with all stages exit0. All four models passed: + +| Model | Successful output matches | Remote attempts | Error/cleanup seconds | Replacement parity | +|---|---:|---:|---:|---| +| Qwen3-0.6B |16/16|22|0.4125|pass| +| Qwen3-4B |16/16|22|1.3533|pass| +| Gemma3-1B |16/16|22|0.5188|pass| +| KONI8B |16/16|22|2.2284|pass| + +Every model has one native-pending cancellation499, two failures500 at worker death and one subsequent500, slab unlink verified, then explicit replacement/reuse. Aggregate64output checks/88remote admission attempts/0local fallback. Error/cleanup time excludes replacement initialization. Marker snapshots prove native call entry and unfinished Future at injection, not a hardware utilization trace. All child benchmark processes exited. CPU serving_matrix plus worker adapter suite50passed and5subtests; Ruff/diff checks pass. + +## Conclusion + +The four-model hardware lifecycle matrix is complete; failed environment run and native-entry evidence are preserved. Readiness while a producer is terminally unavailable and automatic service restart were not asserted by this harness. Full goal remains active, including matched performance, compute/copy overlap, broader cache conditions and Gemma#233. + +## Other model commands + +Use the same command with `PREFILL_OBSERVER_FACTORY` selecting the underlying adapter and these substitutions; all usewidth2048 and the four *-100 cases: + +| Tag | Model | Underlying factory | Artifact directory | KV blocks | +|---|---|---|---|---:| +| qwen4b | Qwen/Qwen3-4B | long_width_variant:make_prefill | atom-artifacts |3| +| gemma | unsloth/gemma-3-1b-it | gemma_prefill:Prefill | atom-artifacts-global225 |3| +| koni | KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024 | koni_prefill:Prefill | atom-artifacts |1| + +Fixtures are semantic227--fixtures.json except KONI uses semantic227-koni-fixtures-eos.json; baseline semantic227--atom.jsonl; output live244--faults.json. Exact local queue script:/tmp/run_live244_queue.sh. Public JSON/summary accompany this record under evidence244. diff --git a/opt_prime/examples/pd/serving_matrix/observer_factory.py b/opt_prime/examples/pd/serving_matrix/observer_factory.py new file mode 100644 index 00000000..30d47b46 --- /dev/null +++ b/opt_prime/examples/pd/serving_matrix/observer_factory.py @@ -0,0 +1,37 @@ +"""Test-only native-call entry/completion markers; no execution gate or KV replay.""" +import importlib +import json +import os +import time +from pathlib import Path + + +def make_prefill(config, weights, width, chip=0): + module, name = os.environ.get( + "PREFILL_OBSERVER_FACTORY", "long_width_variant:make_prefill" + ).split(":") + engine = getattr(importlib.import_module(module), name)(config, weights, width, chip) + directory = Path(os.environ["PREFILL_OBSERVER_DIR"]) + directory.mkdir(parents=True, exist_ok=True) + marker = directory / f"{os.getpid()}.json" + original = engine.prefill + sequence = 0 + + def prefill(tokens): + nonlocal sequence + sequence += 1 + + def record(phase): + temporary = marker.with_suffix(".tmp") + temporary.write_text(json.dumps({"sequence": sequence, "phase": phase, + "tokens": len(tokens), "time": time.monotonic()})) + temporary.replace(marker) + + record("start") + try: + return original(tokens) + finally: + record("done") + + engine.prefill = prefill + return engine diff --git a/opt_prime/examples/pd/serving_matrix/test_observer_factory.py b/opt_prime/examples/pd/serving_matrix/test_observer_factory.py new file mode 100644 index 00000000..861a833e --- /dev/null +++ b/opt_prime/examples/pd/serving_matrix/test_observer_factory.py @@ -0,0 +1,39 @@ +"""Fault-test instrumentation must preserve results/errors and mark call boundaries.""" +import json +import os +import sys +from types import SimpleNamespace + +import pytest +from observer_factory import make_prefill + + +@pytest.mark.parametrize("fails", [False, True]) +def test_observer_preserves_native_call_and_error(monkeypatch, tmp_path, fails): + marker = tmp_path / f"{os.getpid()}.json" + result = object() + error = RuntimeError("native call failed") + tokens = [1, 2, 3] + + def prefill(received): + assert received is tokens + event = json.loads(marker.read_text()) + assert event["phase"] == "start" and event["tokens"] == 3 + if fails: + raise error + return result + + engine = SimpleNamespace(prefill=prefill) + monkeypatch.setitem(sys.modules, "observer_test_engine", SimpleNamespace(make=lambda *a: engine)) + monkeypatch.setenv("PREFILL_OBSERVER_FACTORY", "observer_test_engine:make") + monkeypatch.setenv("PREFILL_OBSERVER_DIR", str(tmp_path)) + wrapped = make_prefill({}, {}, 2048) + for sequence in (1, 2): + if fails: + with pytest.raises(RuntimeError) as caught: + wrapped.prefill(tokens) + assert caught.value is error + else: + assert wrapped.prefill(tokens) is result + event = json.loads(marker.read_text()) + assert event["phase"] == "done" and event["sequence"] == sequence