diff --git a/.gitignore b/.gitignore index e30fe21..35948d1 100644 --- a/.gitignore +++ b/.gitignore @@ -228,3 +228,5 @@ results/ # are listed in SHA256SUMS and must be committed. Without this exception the # *.log rule above drops them silently. Other logs stay ignored. !docs/benchmarks/*/raw/**/server.log +# Real-model verification evidence: the pytest log is listed in SHA256SUMS. +!docs/benchmarks/whisper-t4/raw/pytest.log diff --git a/README.md b/README.md index 24ed124..35d10dc 100644 --- a/README.md +++ b/README.md @@ -222,9 +222,10 @@ The repository contains the following benchmark evidence, with checksums recorde | Phase 1 | 5 | | Phase 3 | 60 | | Phase 4 | 105 | -| **Total** | **170** | +| Real-model verification (T4) | 3 | +| **Total** | **173** | -All three expected manifests were present, and the referenced files matched their recorded checksums. +All four expected manifests were present, and the referenced files matched their recorded checksums. Verify the evidence with: @@ -236,15 +237,16 @@ The verifier checks the manifests that exist. A complete repository audit should ### Test validation -The recorded test validation covered: +The offline suite (fake and gated backends) runs in CI on every pull request: -- Python 3.11 and Python 3.13. -- All four combinations of the two experimental scheduling flags. -- Eight configurations in total. +- Linux: Python 3.11 and 3.13 × all four combinations of the two experimental scheduling flags (eight configurations). +- macOS: Python 3.13, default flags. -**Result: 209 tests passed per configuration, with one real-model test deselected.** +**Result: 269 tests passed per configuration, with the 8 real-model tests deselected.** -The recorded validation included baseline compatibility checks with scheduling flags disabled, deterministic inference-order traces, and WebSocket protocol tests. +It includes baseline compatibility checks with scheduling flags disabled, deterministic inference-order traces, WebSocket protocol tests, server-shutdown reliability, and the real-model recorder's validation rules. + +**Real-model tests: 8 of 8 passed on an NVIDIA T4** (faster-whisper `base.en`, CUDA FP16, commit `096a034`). The run covered the model adapter, server and WebSocket levels. Evidence: [docs/benchmarks/whisper-t4](docs/benchmarks/whisper-t4/README.md). ### Reproducing the analyses @@ -259,6 +261,7 @@ Detailed procedures and evidence are available in: - [Phase 1 T4 Pilot](docs/benchmarks/phase1-t4/README.md) - [Phase 3 Scheduling Results](docs/benchmarks/scheduling-t4/README.md) - [Phase 4 Results and Evidence](docs/benchmarks/phase4-t4/README.md) +- [Real-Model Verification Evidence](docs/benchmarks/whisper-t4/README.md) - [Colab GPU Workflow](benchmarks/COLAB.md) - [Measurement Definitions](docs/measurements.md) @@ -319,13 +322,15 @@ python clients/stream_wav.py --synthetic 3 The fake adapter emits synthetic transcript tokens and is useful for exercising the WebSocket lifecycle without GPU access. -### Real-model smoke test +### Real-model tests ```bash -pytest -m whisper +python -m pytest -q -m whisper ``` -This test requires access to the actual Whisper model weights and may download them on first use. It was excluded from the recorded validation because model downloads were unavailable in that environment. A successful real-model smoke test should be confirmed separately before claiming end-to-end model validation. +Eight tests run the real `base.en` model at three levels: the adapter (device and compute type in effect, transcripts of reference speech, error wrapping), the server (`/health` after loading from settings, clean shutdown), and a real WebSocket stream of reference speech. They need the model weights (downloaded on first use) and, for the speech tests, the 5-utterance reference set; `ECHORELAY_TEST_DEVICE=cuda` selects the GPU. + +**Verified on an NVIDIA T4:** all 8 passed with CUDA FP16 (corpus WER 0.0531, streamed-utterance WER 0.1154). That is one configuration; CPU, other GPUs and other model sizes are not verified. See [Real-model verification](docs/whisper-verification.md) for what each test claims, the Colab procedure (including a required CUDA library-path fix), and the verification log. ### Docker @@ -546,6 +551,7 @@ These limitations are part of the engineering record and inform the next researc | [Scheduling Design](docs/scheduling-design.md) | Scheduler mechanics, invariants, and race conditions | | [Measurement Definitions](docs/measurements.md) | Metrics, timing boundaries, and analysis methodology | | [GPU Workflow](benchmarks/COLAB.md) | Colab experiment execution | +| [Real-Model Verification](docs/whisper-verification.md) | Real Whisper test suite, T4 result, troubleshooting | | [Protocol](docs/protocol.md) | WebSocket messages and error behavior | --- diff --git a/benchmarks/COLAB.md b/benchmarks/COLAB.md index aedb14e..6dd69c3 100644 --- a/benchmarks/COLAB.md +++ b/benchmarks/COLAB.md @@ -1,7 +1,8 @@ # Running the GPU checks and benchmarks on Google Colab -Nothing in this directory has been run on a GPU yet. Run the stages **in -order** and stop at the first failure: the smoke stages exist so that a +These stages produced the committed T4 evidence in `docs/benchmarks/`: the +Phase 1–4 benchmarks, and the real-model verification in stage 2b. Run the +stages **in order** and stop at the first failure: the smoke stages exist so that a broken CUDA setup or model path is found in two minutes, not halfway through a benchmark. @@ -27,7 +28,7 @@ Runtime → Change runtime type → **T4 GPU** (or better). ### Public repository ```bash -!git clone -b feat/streaming-mvp https://github.com/hydrangeas20/echorelay.git +!git clone https://github.com/hydrangeas20/echorelay.git ``` ### Private repository (no credentials in the notebook, URL, or `.git/config`) @@ -58,8 +59,7 @@ env = { "GIT_CONFIG_VALUE_0": f"AUTHORIZATION: basic {auth}", } subprocess.run( - ["git", "clone", "-b", "feat/streaming-mvp", - "https://github.com/hydrangeas20/echorelay.git"], + ["git", "clone", "https://github.com/hydrangeas20/echorelay.git"], env=env, check=True, ) del token, auth, env @@ -103,11 +103,37 @@ If loading fails with a `libcudnn` / `libcublas` error: faster-whisper ≥1.1 `nvidia-cublas-cu12 nvidia-cudnn-cu12==9.*` and set `LD_LIBRARY_PATH` as described in the faster-whisper README, then rerun the smoke test. +Observed on Colab (October 2026): `Library libcublas.so.12 is not found or +cannot be loaded`, because the image exposes CUDA 13's `libcublas.so.13` +while the pip-installed `libcublas.so.12` sat outside `LD_LIBRARY_PATH` +(`/usr/lib64-nvidia`). Prepending its directory, and cuDNN 9's if present, +fixed it. The exact steps are in +[docs/whisper-verification.md](../docs/whisper-verification.md#troubleshooting-incident-colab-cuda-libraries). + `prepare_audio` downloads Mini LibriSpeech dev-clean-2 (~126 MB) from OpenSLR and writes the first 50 utterances of 3–20 s (sorted by id) with references. Keep `eval_audio/` unchanged across all runs; every result file records each file's SHA-256. +### 2b. Real-model test suite (`pytest -m whisper`) + +The adapter, server and WebSocket tests against the real model, with a +record of the environment and results. Needs the 5-utterance reference set +(not the 50-file `eval_audio`): + +Use the two cells in [docs/whisper-verification.md](../docs/whisper-verification.md#how-to-run). +They prepare `eval_audio5`, fix the cuBLAS/cuDNN library path, and run +`python -m benchmarks.whisper_verification --device cuda` **with Colab's +default `PYTHONPATH=/env/python` removed**. The recorder rejects any +`PYTHONPATH` or `PYTEST_ADDOPTS` override as an evidence-integrity violation, +so a plain `!python -m …` in Colab reports `NOT COMPLETE` even when all tests +pass. + +Pass criterion: the last line starts with `COMPLETE: 8 passed, 0 failed, 0 +errors, 0 skipped` and the exit code is 0. Otherwise the recorder prints +`NOT COMPLETE` and each problem. The verified run is in +[docs/benchmarks/whisper-t4](../docs/benchmarks/whisper-t4/README.md). + ## 3. Server smoke test (one real stream) ```bash diff --git a/benchmarks/whisper_verification.py b/benchmarks/whisper_verification.py new file mode 100644 index 0000000..98e664b --- /dev/null +++ b/benchmarks/whisper_verification.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Run the real-model test suite (``pytest -m whisper``) and record what ran. + + python -m benchmarks.prepare_audio --out eval_audio5 --num 5 + python -m benchmarks.whisper_verification --audio-dir eval_audio5 --device cuda + +Writes to ``--out`` (default ``results/whisper_verification``): + +* ``record.json`` UTC time, git commit, GPU/driver (``nvidia-smi``), Python and + package versions, model, device, compute type, the exact + command, per-test outcomes, the runtime identity the tests + observed, and every reason the run is not complete. +* ``pytest.log`` the complete pytest output. +* ``junit.xml`` per-test results, including recorded properties. + +``complete`` is true only if all of these hold (``problems`` lists each one +that does not): + +* exactly the expected real-model tests ran (``EXPECTED_TESTS``), all passed, + and none failed, errored or was skipped; +* the identity the loaded runtime reported during the run matches the + request: faster-whisper's ``WhisperModel`` over CTranslate2's ``Whisper``, + the requested device and compute type, the ``base.en`` weights snapshot, + and (for CUDA) a visible CUDA device and the expected GPU in ``nvidia-smi``. + The requested settings in the record are not evidence of any of this; +* no ``PYTEST_ADDOPTS`` or ``PYTHONPATH`` override (they could inject plugins + or substitute modules), and the checkout had no local edits. + +Exit status: pytest's if non-zero, otherwise 0 if complete, else 3. +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +import xml.etree.ElementTree as ET +from datetime import datetime, timezone +from pathlib import Path + +from .common import environment, write_json + +REPO = Path(__file__).resolve().parents[1] +MODEL = "base.en" # what tests/test_whisper.py loads +EXPECTED_TESTS = frozenset({ + "test_adapter_loads_with_requested_device_and_compute_type", + "test_adapter_decodes_silence_and_empty_audio", + "test_adapter_transcribes_real_speech", + "test_adapter_final_decode_is_deterministic", + "test_adapter_wraps_decode_errors", + "test_server_loads_real_model_from_settings", + "test_websocket_streams_real_speech", + "test_server_shutdown_releases_inference_workers", +}) +IDENTITY_TEST = "test_adapter_loads_with_requested_device_and_compute_type" +WHISPER_CLASS = "faster_whisper.transcribe.WhisperModel" +CT2_CLASS = "ctranslate2._ext.Whisper" +OVERRIDE_VARS = ("PYTEST_ADDOPTS", "PYTHONPATH") + + +def _nvidia_smi() -> str | None: + if not shutil.which("nvidia-smi"): + return None + q = subprocess.run(["nvidia-smi"], capture_output=True, text=True, timeout=30, check=False) + return q.stdout if q.returncode == 0 else f"nvidia-smi exited {q.returncode}: {q.stderr}" + + +def _junit_results(path: Path) -> dict: + root = ET.parse(path).getroot() + suites = [root] if root.tag == "testsuite" else root.findall("testsuite") + tests = [] + for case in (c for s in suites for c in s.iter("testcase")): + outcome, message = "passed", None + for tag, name in (("failure", "failed"), ("error", "error"), ("skipped", "skipped")): + el = case.find(tag) + if el is not None: + outcome, message = name, (el.get("message") or "")[:500] + break + entry = {"name": case.get("name"), "classname": case.get("classname"), + "outcome": outcome, "time_s": float(case.get("time", 0))} + if message is not None: + entry["message"] = message + props = {p.get("name"): p.get("value") for p in case.iter("property")} + if props: + entry["properties"] = props + tests.append(entry) + counts = {k: sum(t["outcome"] == k for t in tests) for k in ("passed", "failed", "error", "skipped")} + return {"collected": len(tests), **counts, "tests": tests} + + +def validate(*, results: dict, device: str, compute_type: str, expect_gpu: str | None, + gpus: list[str], overrides: dict, git: dict | None, exit_code: int) -> list[str]: + """Every reason the run does not verify the requested configuration.""" + problems: list[str] = [] + if exit_code != 0: + problems.append(f"pytest exited {exit_code}") + if results.get("junit_missing"): + return [*problems, "no JUnit results (pytest did not run the suite)"] + + tests = results["tests"] + names = [t["name"] for t in tests] + if len(names) != len(set(names)): + problems.append(f"duplicate test names: {sorted(names)}") + if set(names) != EXPECTED_TESTS: + missing, extra = EXPECTED_TESTS - set(names), set(names) - EXPECTED_TESTS + problems.append(f"ran the wrong test set (missing {sorted(missing)}, unexpected {sorted(extra)})") + for t in tests: + if t["outcome"] != "passed": + problems.append(f"{t['name']}: {t['outcome']}" + (f" ({t['message']})" if t.get("message") else "")) + if results["passed"] != len(EXPECTED_TESTS): + problems.append(f"{results['passed']} passed, expected exactly {len(EXPECTED_TESTS)}") + + seen = next((t.get("properties", {}) for t in tests if t["name"] == IDENTITY_TEST), {}) + + def check(key: str, ok, want: str) -> None: + value = seen.get(key) + if value is None: + problems.append(f"runtime did not report {key}") + elif not ok(value): + problems.append(f"{key} = {value!r}, expected {want}") + + check("observed_whisper_class", lambda v: v == WHISPER_CLASS, WHISPER_CLASS) + check("observed_ct2_class", lambda v: v == CT2_CLASS, CT2_CLASS) + check("observed_ct2_device", lambda v: v == device, repr(device)) + if device == "cuda": + check("observed_ct2_compute_type", lambda v: v == compute_type, repr(compute_type)) + check("observed_cuda_devices", lambda v: v.isdigit() and int(v) >= 1, ">= 1") + else: + check("observed_ct2_compute_type", lambda v: v.startswith(compute_type), f"{compute_type}*") + snapshot = f"models--Systran--faster-whisper-{MODEL}" + check("observed_model_path", lambda v: snapshot in Path(v).parts, f"a path inside {snapshot}/") + check("observed_ct2_is_multilingual", lambda v: v == str(not MODEL.endswith(".en")), + str(not MODEL.endswith(".en"))) + check("observed_model_bin_sha256", lambda v: len(v) == 64, "a SHA-256 of model.bin") + + if device == "cuda" and expect_gpu and not any(expect_gpu in g for g in gpus): + problems.append(f"nvidia-smi lists {gpus or 'no GPU'}, expected one containing {expect_gpu!r}") + for k, v in overrides.items(): + if v: + problems.append(f"{k} is set ({v!r}); it can inject plugins or substitute modules") + if not git or git.get("dirty") is not False: + problems.append(f"checkout is not a clean commit: {git}") + return problems + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--audio-dir", required=True, help="output of prepare_audio --num 5") + p.add_argument("--device", default="cuda", choices=["cuda", "cpu"]) + p.add_argument("--compute-type", default=None, help="default: float16 on cuda, int8 on cpu") + p.add_argument("--expect-gpu", default="T4", + help="substring nvidia-smi must report for --device cuda ('' to skip the check)") + p.add_argument("--out", default="results/whisper_verification") + a = p.parse_args() + + compute_type = a.compute_type or ("float16" if a.device == "cuda" else "int8") + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + junit = out / "junit.xml" + junit.unlink(missing_ok=True) # never read a previous run's results + # junit_family=xunit1: the format that carries record_property values. + cmd = [sys.executable, "-m", "pytest", "-m", "whisper", "-rA", "-s", "-p", "no:cacheprovider", + "-o", "junit_family=xunit1", f"--junitxml={junit}", "tests/test_whisper.py"] + env = {**os.environ, "ECHORELAY_TEST_DEVICE": a.device, + "ECHORELAY_TEST_COMPUTE_TYPE": compute_type, + "ECHORELAY_TEST_AUDIO_DIR": str(Path(a.audio_dir).resolve())} + overrides = {k: os.environ.get(k) for k in OVERRIDE_VARS} + + started = datetime.now(timezone.utc) + print("$", " ".join(cmd), flush=True) + proc = subprocess.run(cmd, cwd=REPO, env=env, capture_output=True, text=True, check=False) + finished = datetime.now(timezone.utc) + (out / "pytest.log").write_text(proc.stdout + proc.stderr) + print(proc.stdout + proc.stderr) + + results = _junit_results(junit) if junit.exists() else {"collected": 0, "junit_missing": True} + env_info = environment() + problems = validate(results=results, device=a.device, compute_type=compute_type, + expect_gpu=a.expect_gpu or None, gpus=env_info.get("gpus", []), + overrides=overrides, git=env_info.get("git"), exit_code=proc.returncode) + complete = not problems + write_json(out / "record.json", { + "kind": "real_model_verification", + "started_utc": started.isoformat(), "finished_utc": finished.isoformat(), + "command": " ".join(["python", *cmd[1:]]), + "requested": {"model": MODEL, "device": a.device, "compute_type": compute_type, + "expect_gpu": a.expect_gpu or None, + "note": "requested settings; the observed identity below is the evidence"}, + "env": {k: env[k] for k in ("ECHORELAY_TEST_DEVICE", "ECHORELAY_TEST_COMPUTE_TYPE", + "ECHORELAY_TEST_AUDIO_DIR")}, + "pytest_overrides": {**overrides, "ECHORELAY_MODEL_DIR": os.environ.get("ECHORELAY_MODEL_DIR")}, + "observed": next((t.get("properties", {}) for t in results.get("tests", []) + if t["name"] == IDENTITY_TEST), {}), + "environment": env_info, + "nvidia_smi": _nvidia_smi(), + "pytest_exit_code": proc.returncode, + "results": results, + "expected_tests": sorted(EXPECTED_TESTS), + "complete": complete, + "problems": problems, + }) + if complete: + print(f"COMPLETE: {results['passed']} passed, 0 failed, 0 errors, 0 skipped; " + f"observed {a.device}/{compute_type}; wrote {out}/") + else: + print(f"NOT COMPLETE ({len(problems)} problem(s)); wrote {out}/") + for prob in problems: + print(f" - {prob}") + sys.exit(proc.returncode if proc.returncode else (0 if complete else 3)) + + +if __name__ == "__main__": + main() diff --git a/docs/benchmarks/whisper-t4/README.md b/docs/benchmarks/whisper-t4/README.md new file mode 100644 index 0000000..cdc4a2e --- /dev/null +++ b/docs/benchmarks/whisper-t4/README.md @@ -0,0 +1,85 @@ +# Real-model verification on an NVIDIA T4 + +The real-model test suite (`tests/test_whisper.py`, `pytest -m whisper`) run +against faster-whisper `base.en` with CUDA FP16 on a Colab Tesla T4, recorded +by `benchmarks/whisper_verification.py`. Suite design, claims and limits: +[docs/whisper-verification.md](../../whisper-verification.md). + +## Result + +| | | +|---|---| +| Run (UTC) | 2026-10-25 04:06:41 → 04:06:52 | +| Commit | `096a0347b620e4e5a8696cd6288379c3a3ccae77` (clean checkout) | +| GPU / driver | Tesla T4, 15360 MiB / 580.82.07 (`nvidia-smi` reports CUDA 13.0) | +| Python / platform | 3.13.16 / Linux-6.6.122+-x86_64-with-glibc2.39 | +| faster-whisper / CTranslate2 | 1.2.1 / 4.8.3 | +| Observed runtime | `faster_whisper.transcribe.WhisperModel` over `ctranslate2._ext.Whisper`; device `cuda`, compute type `float16`; 1 CUDA device; English-only | +| Weights | `models--Systran--faster-whisper-base.en`, snapshot `3d3d5dee26484f91867d81cb899cfcf72b96be6c`; `model.bin` SHA-256 `2a166925…e93e2ef` | +| Tests | **8 passed, 0 failed, 0 errors, 0 skipped** (exactly the 8 expected tests); pytest exit 0 | +| Recorder verdict | `complete: true`, `problems: []`; no `PYTEST_ADDOPTS` / `PYTHONPATH` override | +| Corpus WER, final settings (5 files) | 0.0531 (bound 0.12) | +| Corpus WER, partial settings | 0.0531 | +| WebSocket stream WER (1272-135031-0000) | 0.1154 (bound 0.25); 10 partials, then one final | + +WER uses the v2 normalizer (the default since Phase 2, see +[measurements](../../measurements.md#7-wer)). The bounds are regression +guards set before this run and were not changed after it. + +## Files + +`raw/` holds the recorder's output **unmodified**, as returned from the Colab +session (`whisper_verification_final.zip`, SHA-256 `1a2bbf4e…bd544c8`). Hashes +are in `raw/SHA256SUMS`, checked by `python -m benchmarks.verify_evidence`. + +| File | Contents | +|---|---| +| `raw/record.json` | Environment, `nvidia-smi`, requested settings, observed identity, per-test outcomes, verdict | +| `raw/pytest.log` | Complete pytest output, including all five transcripts | +| `raw/junit.xml` | Per-test results and recorded properties | + +The only host-specific values are Colab paths (`/content`, `/root/.cache`) and +the JUnit `hostname` (an ephemeral container ID). No credentials or personal +data are present. Model weights and audio are not committed. + +## Independent checks on the evidence + +Done on the files above, without trusting the recorder's own verdict: + +* `junit.xml` re-parsed with the recorder's parser reproduces + `record.json`'s `results` exactly; the validator re-run on those raw inputs + reports no problems. +* Every JUnit test `line` matches the `def` of that test in + `tests/test_whisper.py` at the recorded commit, so the suite that ran is the + committed one. `tests/test_whisper.py` and `benchmarks/whisper_verification.py` + are unchanged since that commit. +* `pytest.log` agrees (`8 passed`, rootdir `/content/echorelay`), and its + timestamps fall inside the recorded run window. +* The commit exists on `test/whisper-t4-verification` (PR #9). + +## Comparison with recorded T4 evidence + +* **Transcripts:** all five final transcripts are word for word identical to + the FP16 hypotheses recorded in Phase 1 + (`../phase1-t4/raw/precision_t4.json`). That run used the same GPU, driver and + library versions. +* **WER:** 0.0531 is exactly the v2 corpus WER committed for the same five + files in Phase 3 (for example `../scheduling-t4/raw/04_r1_both/c4.json`). + Phase 1 reported 0.0708 for the same transcripts because it used the v1 + normalizer. The difference is scoring, not decoding. +* The streamed utterance's transcript and WER (0.1154) match Phase 1's + `gpu_smoke.json`. + +## Run conditions not captured by the record + +These come from the Colab session (the operator's account), not from the +files above. See [troubleshooting](../../whisper-verification.md#troubleshooting-incident-colab-cuda-libraries): + +* `LD_LIBRARY_PATH` was extended to the pip-installed cuBLAS 12 (and cuDNN 9, + if present) directories before the run. The recorder does not record + `LD_LIBRARY_PATH`. +* The recorder was started with Colab's default `PYTHONPATH=/env/python` + removed from its environment, so the record shows no override. +* pytest auto-loaded the plugins installed on the Colab image (`typeguard`, + `anyio`, `langsmith`, `platformdirs`, besides `asyncio`); see `pytest.log`. + None is injected by an override, and none is used by the suite. diff --git a/docs/benchmarks/whisper-t4/raw/SHA256SUMS b/docs/benchmarks/whisper-t4/raw/SHA256SUMS new file mode 100644 index 0000000..ce2b207 --- /dev/null +++ b/docs/benchmarks/whisper-t4/raw/SHA256SUMS @@ -0,0 +1,3 @@ +9e6529111890322974db90aa6ae76c0ce2cb5af8ffec5fda8d442b1bb170a770 junit.xml +79d6683b7521f90bafda7491672b020f8fae25cfbf14ec5c8c85a2daa2d478a3 pytest.log +b88d8f5086af15f1c4e9672c12331da5d55fec8863cf77915b1a7c45da4c019a record.json diff --git a/docs/benchmarks/whisper-t4/raw/junit.xml b/docs/benchmarks/whisper-t4/raw/junit.xml new file mode 100644 index 0000000..ff1eb9f --- /dev/null +++ b/docs/benchmarks/whisper-t4/raw/junit.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/benchmarks/whisper-t4/raw/pytest.log b/docs/benchmarks/whisper-t4/raw/pytest.log new file mode 100644 index 0000000..24cd112 --- /dev/null +++ b/docs/benchmarks/whisper-t4/raw/pytest.log @@ -0,0 +1,31 @@ +============================= test session starts ============================== +platform linux -- Python 3.13.16, pytest-8.4.2, pluggy-1.7.0 +rootdir: /content/echorelay +configfile: pyproject.toml +plugins: asyncio-1.4.0, typeguard-4.6.0, anyio-4.15.1, langsmith-0.14.11, platformdirs-4.13.5 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function +collected 8 items + +tests/test_whisper.py .. +cuda/float16 final corpus WER = 0.0531 over 5 files + 1272-135031-0000: Because you are sleeping instead of conquering, the lovely rose princess has become a fiddle without a bow, while poor Shaggy sits there, accuing dove. + 1272-135031-0001: "'He has gone, and gone for good,' answered Polychrome, who had managed to squeeze into the room beside the dragon, and had witnessed the occurrences with much interest. + 1272-135031-0002: I have remained a prisoner only because I wish to be one. And with this, he stepped forward and burst the stout chains as easily as if they had been threads. + 1272-135031-0003: The little girl had been asleep, but she heard the raps and opened the door. + 1272-135031-0004: The king has flooded disgrace and your friends are asking for you. +.... +websocket: 10 partials; final WER 0.1154: Because you are sleeping instead of conquering, the lovely rose princess has become a fiddle without a bow, while poor Shaggy sits there, accuing dove. +.. + +==================================== PASSES ==================================== +------ generated xml file: /content/whisper_verification_final/junit.xml ------- +=========================== short test summary info ============================ +PASSED tests/test_whisper.py::test_adapter_loads_with_requested_device_and_compute_type +PASSED tests/test_whisper.py::test_adapter_decodes_silence_and_empty_audio +PASSED tests/test_whisper.py::test_adapter_transcribes_real_speech +PASSED tests/test_whisper.py::test_adapter_final_decode_is_deterministic +PASSED tests/test_whisper.py::test_adapter_wraps_decode_errors +PASSED tests/test_whisper.py::test_server_loads_real_model_from_settings +PASSED tests/test_whisper.py::test_websocket_streams_real_speech +PASSED tests/test_whisper.py::test_server_shutdown_releases_inference_workers +============================== 8 passed in 9.72s =============================== diff --git a/docs/benchmarks/whisper-t4/raw/record.json b/docs/benchmarks/whisper-t4/raw/record.json new file mode 100644 index 0000000..20e8e64 --- /dev/null +++ b/docs/benchmarks/whisper-t4/raw/record.json @@ -0,0 +1,146 @@ +{ + "kind": "real_model_verification", + "started_utc": "2026-10-25T04:06:41.498807+00:00", + "finished_utc": "2026-10-25T04:06:52.965638+00:00", + "command": "python -m pytest -m whisper -rA -s -p no:cacheprovider -o junit_family=xunit1 --junitxml=/content/whisper_verification_final/junit.xml tests/test_whisper.py", + "requested": { + "model": "base.en", + "device": "cuda", + "compute_type": "float16", + "expect_gpu": "T4", + "note": "requested settings; the observed identity below is the evidence" + }, + "env": { + "ECHORELAY_TEST_DEVICE": "cuda", + "ECHORELAY_TEST_COMPUTE_TYPE": "float16", + "ECHORELAY_TEST_AUDIO_DIR": "/content/eval_audio5" + }, + "pytest_overrides": { + "PYTEST_ADDOPTS": null, + "PYTHONPATH": null, + "ECHORELAY_MODEL_DIR": null + }, + "observed": { + "observed_whisper_class": "faster_whisper.transcribe.WhisperModel", + "observed_ct2_class": "ctranslate2._ext.Whisper", + "observed_ct2_device": "cuda", + "observed_ct2_compute_type": "float16", + "observed_ct2_is_multilingual": "False", + "observed_cuda_devices": "1", + "observed_model_path": "/root/.cache/huggingface/hub/models--Systran--faster-whisper-base.en/snapshots/3d3d5dee26484f91867d81cb899cfcf72b96be6c", + "observed_model_bin_sha256": "2a166925539a16005f14ff328359f9b9adb9dc4fb631bb3b227526862e93e2ef" + }, + "environment": { + "python": "3.13.16", + "platform": "Linux-6.6.122+-x86_64-with-glibc2.39", + "cpu": "x86_64", + "packages": { + "faster-whisper": "1.2.1", + "ctranslate2": "4.8.3", + "numpy": "2.1.3", + "fastapi": "0.141.1", + "uvicorn": "0.54.0", + "websockets": "15.0.1", + "jiwer": "4.0.0" + }, + "gpus": [ + "Tesla T4, 580.82.07, 15360 MiB" + ], + "ctranslate2_cuda_devices": 1, + "git": { + "commit": "096a0347b620e4e5a8696cd6288379c3a3ccae77", + "dirty": false + } + }, + "nvidia_smi": "Sun Oct 25 04:06:53 2026 \n+-----------------------------------------------------------------------------------------+\n| NVIDIA-SMI 580.82.07 Driver Version: 580.82.07 CUDA Version: 13.0 |\n+-----------------------------------------+------------------------+----------------------+\n| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n| | | MIG M. |\n|=========================================+========================+======================|\n| 0 Tesla T4 Off | 00000000:00:04.0 Off | 0 |\n| N/A 53C P0 36W / 70W | 3MiB / 15360MiB | 1% Default |\n| | | N/A |\n+-----------------------------------------+------------------------+----------------------+\n\n+-----------------------------------------------------------------------------------------+\n| Processes: |\n| GPU GI CI PID Type Process name GPU Memory |\n| ID ID Usage |\n|=========================================================================================|\n| No running processes found |\n+-----------------------------------------------------------------------------------------+\n", + "pytest_exit_code": 0, + "results": { + "collected": 8, + "passed": 8, + "failed": 0, + "error": 0, + "skipped": 0, + "tests": [ + { + "name": "test_adapter_loads_with_requested_device_and_compute_type", + "classname": "tests.test_whisper", + "outcome": "passed", + "time_s": 1.379, + "properties": { + "observed_whisper_class": "faster_whisper.transcribe.WhisperModel", + "observed_ct2_class": "ctranslate2._ext.Whisper", + "observed_ct2_device": "cuda", + "observed_ct2_compute_type": "float16", + "observed_ct2_is_multilingual": "False", + "observed_cuda_devices": "1", + "observed_model_path": "/root/.cache/huggingface/hub/models--Systran--faster-whisper-base.en/snapshots/3d3d5dee26484f91867d81cb899cfcf72b96be6c", + "observed_model_bin_sha256": "2a166925539a16005f14ff328359f9b9adb9dc4fb631bb3b227526862e93e2ef" + } + }, + { + "name": "test_adapter_decodes_silence_and_empty_audio", + "classname": "tests.test_whisper", + "outcome": "passed", + "time_s": 0.345 + }, + { + "name": "test_adapter_transcribes_real_speech", + "classname": "tests.test_whisper", + "outcome": "passed", + "time_s": 0.9, + "properties": { + "corpus_wer_final": "0.05309734513274336", + "corpus_wer_partial_settings": "0.05309734513274336" + } + }, + { + "name": "test_adapter_final_decode_is_deterministic", + "classname": "tests.test_whisper", + "outcome": "passed", + "time_s": 0.235 + }, + { + "name": "test_adapter_wraps_decode_errors", + "classname": "tests.test_whisper", + "outcome": "passed", + "time_s": 0.001 + }, + { + "name": "test_server_loads_real_model_from_settings", + "classname": "tests.test_whisper", + "outcome": "passed", + "time_s": 0.577, + "properties": { + "reported_health_model": "{\"backend\": \"whisper\", \"name\": \"base.en\", \"device\": \"cuda\", \"compute_type\": \"float16\"}" + } + }, + { + "name": "test_websocket_streams_real_speech", + "classname": "tests.test_whisper", + "outcome": "passed", + "time_s": 5.815, + "properties": { + "websocket_wer": "0.11538461538461539" + } + }, + { + "name": "test_server_shutdown_releases_inference_workers", + "classname": "tests.test_whisper", + "outcome": "passed", + "time_s": 0.359 + } + ] + }, + "expected_tests": [ + "test_adapter_decodes_silence_and_empty_audio", + "test_adapter_final_decode_is_deterministic", + "test_adapter_loads_with_requested_device_and_compute_type", + "test_adapter_transcribes_real_speech", + "test_adapter_wraps_decode_errors", + "test_server_loads_real_model_from_settings", + "test_server_shutdown_releases_inference_workers", + "test_websocket_streams_real_speech" + ], + "complete": true, + "problems": [] +} diff --git a/docs/whisper-verification.md b/docs/whisper-verification.md new file mode 100644 index 0000000..be9469a --- /dev/null +++ b/docs/whisper-verification.md @@ -0,0 +1,234 @@ +# Real-model verification (Whisper `base.en`) + +> **Status: verified on an NVIDIA T4** (2026-10-25, commit `096a034`). All 8 +> real-model tests passed with faster-whisper `base.en`, CUDA FP16, and the +> recorder reported `complete: true` with no problems. The evidence is +> committed unmodified in [`docs/benchmarks/whisper-t4/`](benchmarks/whisper-t4/README.md). +> One verified configuration only; see [Limitations](#limitations). + +## Why this exists + +The offline suite (`pytest -m "not whisper"`, run in CI) uses fake and gated +backends. Before this change, the only real-model test was a CPU `int8` +smoke test that decoded silence and checked that the result was a string. It +did not check CUDA, FP16, transcript correctness, error handling, the server, +or the WebSocket path, and it had not been confirmed as executed. + +## What the suite claims, by level + +`tests/test_whisper.py` (marker `whisper`, deselected by default). Each level +claims only what it exercises. + +| Level | Test | Asserts | +|---|---|---| +| Adapter | `test_adapter_loads_with_requested_device_and_compute_type` | The device and compute type **CTranslate2 reports in effect** match the request (on CUDA: exactly `float16`, so a silent fallback fails). | +| Adapter | `test_adapter_decodes_silence_and_empty_audio` | Silence decodes to a string with partial and final settings; empty audio returns `""`. | +| Adapter | `test_adapter_transcribes_real_speech` | All 5 reference utterances decode to non-empty text with final and partial settings; final corpus WER ≤ 0.12. | +| Adapter | `test_adapter_final_decode_is_deterministic` | Two final decodes of the same audio give identical text. | +| Adapter | `test_adapter_wraps_decode_errors` | A failure inside faster-whisper (non-numeric input) is raised as `InferenceError`. | +| Server | `test_server_loads_real_model_from_settings` | A real uvicorn server whose lifespan builds the model from `Settings` reports `whisper` / `base.en` / device / compute type on `/health`. | +| WebSocket | `test_websocket_streams_real_speech` | A real WebSocket client streams a 10.9 s utterance in 100 ms frames at 2× real time: `ready` first, ≥1 non-empty provisional partial with consecutive `seq`, exactly one `final`, no `error`, close code 1000. The final text **equals** a direct adapter decode of the same audio (the server path does not alter the audio), and its WER ≤ 0.25. | +| Server | `test_server_shutdown_releases_inference_workers` | After a decode, stopping the server stops the thread, the inference worker threads exit, and the port stops accepting connections. | + +**Fixtures.** The speech tests need the 5-utterance set from +`python -m benchmarks.prepare_audio --num 5` (Mini LibriSpeech dev-clean-2, +first 5 utterances of 3–20 s). They check each file's PCM SHA-256 against the +recorded Phase 1 precision run (`docs/benchmarks/phase1-t4/raw/precision_t4.json`), +so the WER is computed on exactly the audio used by the recorded T4 runs. + +**Thresholds are regression guards, not measurements.** The bounds (0.12 +corpus, 0.25 for the streamed utterance) were set before the T4 run and were +not changed after it; a broken decode path (wrong sample rate, scaling or +truncation) lands far above them. Recorded T4 FP16 values on the same audio, +under the **v2** normalizer the tests use: corpus WER 0.0531 (6 edits / 113 +words, Phase 3 evidence). For the streamed utterance, 0.1154 (3 / 26) was +recorded in `gpu_smoke.json` under v1. The T4 run scored the identical +transcript 0.1154 under v2, so for this utterance the two normalizers agree. + +*Correction.* The suite was written citing Phase 1's 0.0708 (8 / 113) as the +recorded corpus value. That figure was scored with the **v1** normalizer on +the same transcripts; the comparable v2 value is 0.0531. The comment above +`MAX_CORPUS_WER` in `tests/test_whisper.py` still cites 0.0708. It is left +unchanged so that the test file stays byte-identical to the verified commit. +The bound itself is unaffected. + +**Not covered.** GPU memory release after shutdown (only worker threads and +the port are checked), concurrency under load (covered by the benchmarks, +not this suite), other model sizes, `int8_float16`, and GPUs other than the T4. + +## How to run + +On a Colab **T4 GPU** runtime. These are the steps of the verified run, +including the library-path fix it needed (see +[troubleshooting](#troubleshooting-incident-colab-cuda-libraries)). + +Cell 1: clone, install and fetch the reference audio. + +```bash +!nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv +%cd /content +!rm -rf echorelay && git clone -b test/whisper-t4-verification https://github.com/hydrangeas20/echorelay.git +%cd /content/echorelay +!git log -1 --format='%H %s' +!pip install -q -e ".[dev]" soundfile +!python -m benchmarks.prepare_audio --out /content/eval_audio5 --num 5 +``` + +Cell 2: expose cuBLAS 12 (and cuDNN 9, if present), run the recorder without +Colab's default `PYTHONPATH`, and download the evidence. + +```python +import glob, os, subprocess + +cublas = glob.glob("/usr/local/lib/python*/dist-packages/nvidia/cublas/lib/libcublas.so.12") +assert cublas, "cuBLAS 12 library not found" +dirs = [os.path.dirname(cublas[0])] +cudnn = glob.glob("/usr/local/lib/python*/dist-packages/nvidia/cudnn/lib/libcudnn.so.9") +if cudnn: + dirs.append(os.path.dirname(cudnn[0])) +existing = os.environ.get("LD_LIBRARY_PATH", "") +os.environ["LD_LIBRARY_PATH"] = ":".join(dirs + ([existing] if existing else [])) + +env = os.environ.copy() +env.pop("PYTHONPATH", None) # Colab sets /env/python; the recorder rejects any PYTHONPATH +result = subprocess.run( + ["python", "-m", "benchmarks.whisper_verification", "--audio-dir", "/content/eval_audio5", + "--device", "cuda", "--out", "/content/whisper_verification"], + cwd="/content/echorelay", env=env) +print("exit code:", result.returncode) + +subprocess.run(["zip", "-qr", "whisper_verification.zip", "whisper_verification"], cwd="/content", check=True) +from google.colab import files +files.download("/content/whisper_verification.zip") +``` + +It passes if the recorder's last line starts with `COMPLETE: 8 passed, 0 +failed, 0 errors, 0 skipped` and the exit code is 0. + +Once the PR is merged, clone `main` instead of the branch. +`benchmarks/whisper_verification.py` runs `python -m pytest -m whisper -rA -s` +on `tests/test_whisper.py` with `ECHORELAY_TEST_DEVICE=cuda` and +`ECHORELAY_TEST_COMPUTE_TYPE=float16`, then writes `record.json` (UTC times, +git commit and dirty flag, `nvidia-smi` output, GPU and driver, Python and +package versions, the requested model, device and compute type, the exact +command, any `PYTEST_ADDOPTS` / `PYTHONPATH` overrides, per-test outcomes with +recorded WER, and the runtime identity observed during the run), `pytest.log` +and `junit.xml`. + +**What `complete: true` requires.** The requested settings in the record are +not treated as evidence. `complete` is true only if `problems` is empty, i.e.: + +* exactly the 8 expected tests ran (by name), all passed, none failed, errored + or was skipped, and pytest exited 0; +* the identity test recorded what the loaded runtime reported, and it matches + the request: the object is faster-whisper's `WhisperModel` over + CTranslate2's `Whisper` (a substituted model class fails), the CTranslate2 + device and compute type (exactly `float16` on CUDA), at least one CUDA + device, the `models--Systran--faster-whisper-base.en` weights snapshot that + faster-whisper resolved (with the SHA-256 of its `model.bin`), and an + English-only model; +* on CUDA, `nvidia-smi` lists a GPU containing `T4` (`--expect-gpu`); +* `PYTEST_ADDOPTS` and `PYTHONPATH` are unset, so no plugin or module was + injected, and the checkout had no local edits. + +`tests/test_whisper_verification.py` (offline, in CI) checks each of these +rules. The `model.bin` hash is recorded for traceability; there is no +committed reference hash to compare it against. + +The suite also runs on CPU (`int8`) with `--device cpu`. A CPU run verifies +the adapter, server and WebSocket paths with the real model, but not CUDA or +FP16. + +## Verification log + +| UTC date | Commit | GPU / driver | Python | faster-whisper / CTranslate2 | Device / compute | Result | Evidence | +|---|---|---|---|---|---|---|---| +| 2026-10-25 | `096a034` | Tesla T4 / 580.82.07 | 3.13.16 | 1.2.1 / 4.8.3 | cuda / float16 (observed) | **8 passed**, 0 failed / errors / skips; `complete: true`. Corpus WER 0.0531, stream WER 0.1154 | [whisper-t4](benchmarks/whisper-t4/README.md) | + +The evidence README lists the checks made on these files independently of +the recorder's verdict. They include re-deriving the results from +`junit.xml`, re-running the validator, and matching each JUnit test line to +the committed source. It also compares the transcripts with earlier T4 +evidence: all five are word for word identical to Phase 1's FP16 +hypotheses. + +## Troubleshooting incident: Colab CUDA libraries + +*Source: the operator's account of the Colab session. The failed attempts' +output is not in the repository or in the evidence ZIP. Only the final, +successful run's files are committed.* + +1. **First attempt: failed.** Loading the model on CUDA failed with: + + ``` + Library libcublas.so.12 is not found or cannot be loaded + ``` + + The Colab environment had: + + ``` + PYTHONPATH: /env/python + LD_LIBRARY_PATH: /usr/lib64-nvidia + ``` + + The system CUDA installation exposed `libcublas.so.13`. CTranslate2 4.8.3 + needs the CUDA 12 library, which was present as a pip-installed package at + `/usr/local/lib/python3.13/dist-packages/nvidia/cublas/lib/libcublas.so.12` + but was not on the loader's search path. + +2. **Fix:** that directory was prepended to `LD_LIBRARY_PATH` in the kernel, + so child processes inherit it. If `libcudnn.so.9` was present, its + directory was added too. Cell 2 above does the same thing. + +3. **Second attempt: tests passed, recorder rejected it.** With the library + path fixed, all 8 real-model tests passed. The recorder still reported + `NOT COMPLETE`, because Colab's default `PYTHONPATH=/env/python` violates + its evidence-integrity rule (any `PYTHONPATH` could substitute modules). + This was the rule working as designed, and it was not relaxed. + +4. **Final run: complete.** The recorder was run with `PYTHONPATH` removed + from its environment (Cell 2). It exited 0 and reported `complete: true`. + Those are the committed files. + +The record does not capture `LD_LIBRARY_PATH`, so the library-path fix +appears only in this account, not in the evidence. + +## Limitations + +* **One configuration.** `base.en`, CUDA FP16, one Colab T4, one run. + Not verified: CPU `int8`, `int8_float16`, other model sizes or GPUs, and + other driver or library versions. +* **Not covered by the suite:** GPU memory release after shutdown (only + worker threads and the port are checked), and behaviour under concurrent + load (that is the benchmarks' job). +* **Single-run timings.** Test durations in the evidence are a side effect, + not measurements. +* **`model.bin` hash.** Recorded for traceability. No independent reference + hash is committed to compare it against. +* **Not recorded by the recorder:** `LD_LIBRARY_PATH`, and the pytest plugins + auto-loaded from the environment (they appear only in `pytest.log`). +* **Not in CI.** CI runs the offline suite, including the recorder's + validation rules (`tests/test_whisper_verification.py`), but not the + real-model tests. Re-verification is manual (Colab). + +## Before the GPU run + +These checks came before the T4 run. They were made without a GPU or model +weights, because Hugging Face was not reachable from the environment used to +write this suite: + +* The suite is collected by `pytest -m whisper` (8 tests) and deselected by + the default run. +* With faster-whisper's `WhisperModel` replaced by a stub, all 8 tests pass, + and they fail as intended when the stub returns wrong transcripts (both + speech tests), when CUDA is requested with no GPU (the model fixture fails; + it does not skip), and when the audio is not the recorded set (hash check). + Without `ECHORELAY_TEST_AUDIO_DIR`, the 3 speech tests skip with a stated + reason; the recorder reports `complete: false`. +* With the stub, all 8 tests pass but the recorder reports `complete: false` + and names why: the stub's model and CTranslate2 classes, the injected + `PYTEST_ADDOPTS` / `PYTHONPATH`, and the uncommitted checkout. + +None of this is evidence about the real model. It showed only that the tests +and the recorder behave as described; the T4 run above is the real-model +evidence. diff --git a/tests/test_observability.py b/tests/test_observability.py index b6ef13d..7c1bfae 100644 --- a/tests/test_observability.py +++ b/tests/test_observability.py @@ -11,6 +11,7 @@ import numpy as np import pytest from conftest import chunks, collect, pcm16 +from gated import GatedTranscriber, until from echorelay.inference import FakeTranscriber, InferenceRunner from echorelay.metrics import Distribution, Metrics, monitor_event_loop_lag @@ -56,18 +57,31 @@ async def test_runner_separates_queue_wait_from_decode_time(): async def test_runner_counts_timeouts_while_queued(): - runner = InferenceRunner(FakeTranscriber(delay_s=0.5), max_concurrency=1, timeout_s=0.2) + # The test decides when the running decode finishes (no fixed sleep): a + # timed-out decode keeps its slot until its worker thread returns, and the + # release then happens on the loop, so its timing depends on the machine. + gated = GatedTranscriber() + runner = InferenceRunner(gated, max_concurrency=1, timeout_s=0.2) audio = np.zeros(1600, dtype=np.float32) - results = await asyncio.gather( - runner.transcribe(audio, final=False), runner.transcribe(audio, final=False), - return_exceptions=True, - ) - assert all(type(r).__name__ == "InferenceTimeout" for r in results) - assert runner.metrics.counter("inference.partial.timeouts") == 2 - assert runner.metrics.counter("inference.partial.timeouts_while_queued") == 1 - await asyncio.sleep(0.4) - assert runner.in_flight == 0 and runner.waiting == 0 - runner.shutdown() + try: + results = await asyncio.gather( + runner.transcribe(audio, final=False), runner.transcribe(audio, final=False), + return_exceptions=True, + ) + assert all(type(r).__name__ == "InferenceTimeout" for r in results) + assert runner.metrics.counter("inference.partial.timeouts") == 2 + assert runner.metrics.counter("inference.partial.timeouts_while_queued") == 1 + # Both callers gave up; the first decode is still running and holds the slot. + assert gated.started == ["P1600"] + assert runner.in_flight == 1 and runner.waiting == 0 + gated.release_all() + await until(lambda: runner.in_flight == 0, what="timed-out decode to release its slot") + assert runner.in_flight == 0 and runner.waiting == 0 + assert runner.scheduler.free == 1 # slot returned, not leaked to the timed-out waiter + assert gated.started == ["P1600"] # the queued request never ran + finally: + gated.release_all() + runner.shutdown() async def test_event_loop_lag_monitor_detects_a_blocked_loop(): diff --git a/tests/test_units.py b/tests/test_units.py index 2ed662e..beac97d 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -6,6 +6,7 @@ import numpy as np import pytest +from gated import until from echorelay.audio import AudioBuffer, AudioLimitExceeded, InvalidAudioFrame from echorelay.inference import FakeTranscriber, InferenceError, InferenceRunner, InferenceTimeout @@ -49,17 +50,20 @@ async def test_runner_releases_slot_only_when_thread_finishes(): fake = FakeTranscriber(delay_s=0.3) runner = InferenceRunner(fake, max_concurrency=1, timeout_s=0.1) audio = np.zeros(16000, dtype=np.float32) - with pytest.raises(InferenceTimeout): - await runner.transcribe(audio, final=True) - assert runner.in_flight == 1 # thread is still running; slot still held - await asyncio.sleep(0.4) - assert runner.in_flight == 0 - runner.timeout_s = 2.0 - fake.delay_s = 0.0 - result = await runner.transcribe(audio, final=True) - assert result.text == "w1 w2" and result.inference_s >= 0 - assert fake.max_active == 1 - runner.shutdown() + try: + with pytest.raises(InferenceTimeout): + await runner.transcribe(audio, final=True) + assert runner.in_flight == 1 # thread is still running; slot still held + # The release runs on the loop after the worker thread returns; when + # depends on the machine, so wait for it (bounded) rather than sleeping. + await until(lambda: runner.in_flight == 0, what="timed-out decode to release its slot") + runner.timeout_s = 2.0 + fake.delay_s = 0.0 + result = await runner.transcribe(audio, final=True) + assert result.text == "w1 w2" and result.inference_s >= 0 + assert fake.max_active == 1 + finally: + runner.shutdown() async def test_runner_wraps_unexpected_exceptions(): diff --git a/tests/test_whisper.py b/tests/test_whisper.py index 3bfec22..6989c11 100644 --- a/tests/test_whisper.py +++ b/tests/test_whisper.py @@ -1,26 +1,287 @@ -"""Real-model smoke test. Deselected by default; run with ``pytest -m whisper``. +"""Real-model verification. Deselected by default; run with ``pytest -m whisper``. -Requires network access to Hugging Face (or a pre-populated -``ECHORELAY_MODEL_DIR``) to load the ``base.en`` weights. +Three levels, each claiming only what it exercises: + +* adapter ``WhisperTranscriber`` on its own: load, device and compute type + actually in effect, decoding real speech, error wrapping. +* server the FastAPI app building and loading the model from ``Settings`` + and reporting it on ``/health``. +* websocket a real uvicorn server and a real WebSocket client streaming + real speech: partials, one final, clean close, then shutdown. + +Configuration (environment variables): + +``ECHORELAY_TEST_DEVICE`` ``cpu`` (default) or ``cuda``. With ``cuda``, + a missing GPU is a failure, not a skip. +``ECHORELAY_TEST_COMPUTE_TYPE`` default ``int8`` on CPU, ``float16`` on CUDA. +``ECHORELAY_TEST_AUDIO_DIR`` the 5-utterance set from + ``python -m benchmarks.prepare_audio --num 5``. + Speech tests skip without it (reported by -rs). +``ECHORELAY_MODEL_DIR`` optional weights cache. + +The speech tests require exactly the five Mini LibriSpeech dev-clean-2 files +of the recorded Phase 1 precision run (checked by PCM SHA-256), so their WER +is comparable to the recorded FP16 value (0.0708). The WER bound below is a +regression guard, not a measurement. """ from __future__ import annotations +import asyncio +import json +import os +import socket +import threading +import time +from pathlib import Path + +import httpx import numpy as np import pytest +from echorelay.config import Settings +from echorelay.inference.base import InferenceError from echorelay.inference.whisper import WhisperTranscriber pytestmark = pytest.mark.whisper +REPO = Path(__file__).resolve().parents[1] +RECORDED = REPO / "docs/benchmarks/phase1-t4/raw/precision_t4.json" +MODEL = "base.en" +DEVICE = os.getenv("ECHORELAY_TEST_DEVICE", "cpu") +COMPUTE_TYPE = os.getenv("ECHORELAY_TEST_COMPUTE_TYPE", "float16" if DEVICE == "cuda" else "int8") +MODEL_DIR = os.getenv("ECHORELAY_MODEL_DIR") +AUDIO_DIR = os.getenv("ECHORELAY_TEST_AUDIO_DIR") +# Recorded FP16 corpus WER on these five files is 0.0708 (8 edits / 113 words). +# 0.12 allows ~5 extra word errors for library/driver drift; a broken decode +# path (wrong sample rate, wrong scaling, truncated audio) lands far above it. +MAX_CORPUS_WER = 0.12 +# Recorded FP16 WER of the streamed utterance (1272-135031-0000) is 0.1154 +# (3 edits / 26 words, docs/benchmarks/phase1-t4/raw/gpu_smoke.json). +MAX_STREAM_WER = 0.25 + + +# ---------------------------------------------------------------- fixtures +@pytest.fixture(scope="module") +def model() -> WhisperTranscriber: + if DEVICE == "cuda": + import ctranslate2 + + assert ctranslate2.get_cuda_device_count() >= 1, ( + "ECHORELAY_TEST_DEVICE=cuda but CTranslate2 sees no CUDA device") + m = WhisperTranscriber(MODEL, DEVICE, COMPUTE_TYPE, model_dir=MODEL_DIR) + m.load() + return m + + +@pytest.fixture(scope="module") +def utterances(): + if not AUDIO_DIR: + pytest.skip("set ECHORELAY_TEST_AUDIO_DIR (python -m benchmarks.prepare_audio --num 5)") + from benchmarks.common import load_audio_dir + + utts = load_audio_dir(AUDIO_DIR) + recorded = json.loads(RECORDED.read_text())["audio"]["files"] + assert [(u.name, u.sha256) for u in utts] == [(f["name"], f["sha256"]) for f in recorded], ( + "audio set differs from the recorded Phase 1 set; WER would not be comparable") + assert all(u.reference for u in utts), "every utterance needs its .txt reference" + return utts + -def test_whisper_base_en_loads_and_decodes(): - import os +def _wer(refs: list[str], hyps: list[str]) -> float: + from benchmarks.common import corpus_wer - model = WhisperTranscriber("base.en", "cpu", "int8", model_dir=os.getenv("ECHORELAY_MODEL_DIR")) - model.load() + return corpus_wer(refs, hyps) + + +# ------------------------------------------------------------- adapter level +def _class_name(obj) -> str: + return f"{type(obj).__module__}.{type(obj).__qualname__}" + + +def _observed(model: WhisperTranscriber) -> dict[str, str]: + """What the loaded runtime reports about itself, independent of the + settings passed in. Recorded as JUnit properties for the verifier.""" + import hashlib + + import ctranslate2 + from faster_whisper.utils import download_model + + ct2 = model._model.model # faster_whisper.WhisperModel -> ctranslate2.models.Whisper + # Same resolution WhisperModel performed at load time (download_root=MODEL_DIR); + # local_files_only, so this reads the cached snapshot and never downloads. + path = Path(download_model(MODEL, cache_dir=MODEL_DIR, local_files_only=True)) + digest = hashlib.sha256() + with open(path / "model.bin", "rb") as f: + for block in iter(lambda: f.read(1 << 20), b""): + digest.update(block) + return { + "observed_whisper_class": _class_name(model._model), + "observed_ct2_class": _class_name(ct2), + "observed_ct2_device": ct2.device, + "observed_ct2_compute_type": ct2.compute_type, + "observed_ct2_is_multilingual": str(ct2.is_multilingual), + "observed_cuda_devices": str(ctranslate2.get_cuda_device_count()), + "observed_model_path": str(path), + "observed_model_bin_sha256": digest.hexdigest(), + } + + +def test_adapter_loads_with_requested_device_and_compute_type(model, record_property): + """The device and compute type CTranslate2 actually uses, not only the + values passed in.""" + for key, value in _observed(model).items(): # recorded before asserting + record_property(key, value) + assert model.info.backend == "whisper" and model.info.model == MODEL + assert (model.info.device, model.info.compute_type) == (DEVICE, COMPUTE_TYPE) + ct2 = model._model.model # faster_whisper.WhisperModel -> ctranslate2.models.Whisper + assert ct2.device == DEVICE + if DEVICE == "cuda": + assert ct2.compute_type == COMPUTE_TYPE # no silent fallback (e.g. to float32) + else: + assert ct2.compute_type.startswith(COMPUTE_TYPE) # CPU may report int8_float32 + + +def test_adapter_decodes_silence_and_empty_audio(model): silence = np.zeros(16000 * 2, dtype=np.float32) for final in (False, True): - result = model.transcribe(silence, final=final) - assert isinstance(result.text, str) + assert isinstance(model.transcribe(silence, final=final).text, str) assert model.transcribe(np.zeros(0, dtype=np.float32), final=True).text == "" + + +def test_adapter_transcribes_real_speech(model, utterances, record_property): + finals = [model.transcribe(u.float32(), final=True).text for u in utterances] + partial_settings = [model.transcribe(u.float32(), final=False).text for u in utterances] + assert all(t.strip() for t in finals), finals + assert all(t.strip() for t in partial_settings), partial_settings + wer = _wer([u.reference for u in utterances], finals) + record_property("corpus_wer_final", wer) + record_property("corpus_wer_partial_settings", + _wer([u.reference for u in utterances], partial_settings)) + print(f"\n{DEVICE}/{COMPUTE_TYPE} final corpus WER = {wer:.4f} over {len(utterances)} files") + for u, t in zip(utterances, finals): + print(f" {u.name}: {t}") + assert wer <= MAX_CORPUS_WER + + +def test_adapter_final_decode_is_deterministic(model, utterances): + audio = utterances[0].float32() + assert model.transcribe(audio, final=True).text == model.transcribe(audio, final=True).text + + +def test_adapter_wraps_decode_errors(model): + """A failure inside faster-whisper surfaces as InferenceError, which the + session maps to an ``inference_failed`` error event.""" + bad = np.array(["not audio"] * 16000, dtype=object) + with pytest.raises(InferenceError, match="faster-whisper decode failed"): + model.transcribe(bad, final=True) + + +# -------------------------------------------------------------- server level +def _settings(**overrides) -> Settings: + base = {"backend": "whisper", "model_name": MODEL, "device": DEVICE, "compute_type": COMPUTE_TYPE, + "model_dir": MODEL_DIR, "idle_timeout_s": 30.0, "inference_timeout_s": 60.0} + base.update(overrides) + return Settings(**base) + + +def _infer_threads() -> list[threading.Thread]: + return [t for t in threading.enumerate() if t.name.startswith("infer")] + + +@pytest.fixture(scope="module") +def live_server(): + """A real uvicorn server whose lifespan builds and loads the real model + from Settings. Yields its address; stopped (and checked) at teardown.""" + import uvicorn + + from echorelay.app import create_app + + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + app = create_app(_settings()) + server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="warning")) + thread = threading.Thread(target=server.run, daemon=True) + thread.start() + deadline = time.monotonic() + 600 # first load may download the weights + while not server.started: + assert thread.is_alive(), "server exited during start-up (model load failed?)" + assert time.monotonic() < deadline, "server did not start" + time.sleep(0.05) + yield {"addr": f"127.0.0.1:{port}", "server": server, "thread": thread} + server.should_exit = True + thread.join(timeout=30) + + +def test_server_loads_real_model_from_settings(live_server, record_property): + body = httpx.get(f"http://{live_server['addr']}/health", timeout=10).json() + record_property("reported_health_model", json.dumps(body.get("model"))) # self-reported config + assert body["status"] == "ok" + assert body["model"]["backend"] == "whisper" + assert body["model"]["name"] == MODEL + assert body["model"]["device"] == DEVICE + assert body["model"]["compute_type"] == COMPUTE_TYPE + + +# ----------------------------------------------------------- websocket level +async def _stream(addr: str, pcm16: bytes, *, frame_s: float = 0.1, speed: float = 2.0): + """Send PCM in frame_s frames at `speed` x real time, then end; collect events.""" + import websockets + + step = int(frame_s * 16000) * 2 + events: list[dict] = [] + async with websockets.connect(f"ws://{addr}/ws/transcribe", max_size=None) as ws: + + async def receive(): + async for msg in ws: + events.append(json.loads(msg)) + + reader = asyncio.create_task(receive()) + for i in range(0, len(pcm16), step): + await ws.send(pcm16[i:i + step]) + await asyncio.sleep(frame_s / speed) + await ws.send(json.dumps({"type": "end"})) + await asyncio.wait_for(reader, timeout=120) + return events, ws.close_code + + +def test_websocket_streams_real_speech(live_server, model, utterances, record_property): + utt = utterances[0] # 10.9 s + events, close_code = asyncio.run(_stream(live_server["addr"], utt.pcm16)) + kinds = [e["type"] for e in events] + assert kinds[0] == "ready" and kinds[-1] == "final", kinds + assert "error" not in kinds, events + assert close_code == 1000 + partials = [e for e in events if e["type"] == "partial"] + finals = [e for e in events if e["type"] == "final"] + assert len(finals) == 1 + assert partials and all(p["provisional"] and p["text"].strip() for p in partials) + assert [p["seq"] for p in partials] == list(range(1, len(partials) + 1)) + text = finals[0]["text"] + # Same audio, same model and decode settings as a direct adapter call: + # the server path must not alter the audio (framing, scaling, truncation). + assert text == model.transcribe(utt.float32(), final=True).text + wer = _wer([utt.reference], [text]) + record_property("websocket_wer", wer) + print(f"\nwebsocket: {len(partials)} partials; final WER {wer:.4f}: {text}") + assert wer <= MAX_STREAM_WER + + +def test_server_shutdown_releases_inference_workers(live_server): + """Runs last in this module (pytest keeps file order): stops the server + and checks that the lifespan shut the inference pool down.""" + silence = np.zeros(16000, dtype=" dict: + outcomes = outcomes or {} + tests = [] + for name in sorted(names): + t = {"name": name, "outcome": outcomes.get(name, "passed")} + if name == IDENTITY_TEST: + t["properties"] = dict(OBSERVED_CUDA if observed is None else observed) + tests.append(t) + counts = {k: sum(t["outcome"] == k for t in tests) for k in ("passed", "failed", "error", "skipped")} + return {"collected": len(tests), **counts, "tests": tests} + + +def _validate(results=None, **kw) -> list[str]: + args = dict(results=results or _results(), device="cuda", compute_type="float16", + expect_gpu="T4", gpus=["Tesla T4, 580.82.07, 15360 MiB"], + overrides={"PYTEST_ADDOPTS": None, "PYTHONPATH": None}, + git={"commit": "f" * 40, "dirty": False}, exit_code=0) + args.update(kw) + return validate(**args) + + +def test_complete_run_has_no_problems(): + assert _validate() == [] + + +def test_cpu_int8_may_report_int8_float32(): + observed = {**OBSERVED_CUDA, "observed_ct2_device": "cpu", + "observed_ct2_compute_type": "int8_float32", "observed_cuda_devices": "0"} + assert _validate(_results(observed), device="cpu", compute_type="int8", gpus=[]) == [] + + +@pytest.mark.parametrize("outcome", ["failed", "error", "skipped"]) +def test_any_non_passing_test_is_incomplete(outcome): + problems = _validate(_results(outcomes={"test_adapter_transcribes_real_speech": outcome})) + assert any(f"test_adapter_transcribes_real_speech: {outcome}" in p for p in problems) + assert any("expected exactly 8" in p for p in problems) + + +def test_missing_or_extra_tests_are_incomplete(): + fewer = EXPECTED_TESTS - {"test_websocket_streams_real_speech"} + assert any("wrong test set" in p for p in _validate(_results(names=fewer))) + more = EXPECTED_TESTS | {"test_something_else"} + assert any("wrong test set" in p for p in _validate(_results(names=more))) + + +@pytest.mark.parametrize(("key", "value", "match"), [ + ("observed_ct2_device", "cpu", "observed_ct2_device"), + ("observed_ct2_compute_type", "float32", "observed_ct2_compute_type"), # silent fallback + ("observed_ct2_compute_type", "int8_float16", "observed_ct2_compute_type"), + ("observed_cuda_devices", "0", "observed_cuda_devices"), + ("observed_whisper_class", "stub_plugin.StubModel", "observed_whisper_class"), + ("observed_ct2_class", "stub_plugin._CT2", "observed_ct2_class"), + ("observed_model_path", "/cache/models--Systran--faster-whisper-base/snapshots/x", + "observed_model_path"), + ("observed_model_path", "/cache/models--Systran--faster-whisper-small.en/snapshots/x", + "observed_model_path"), + ("observed_ct2_is_multilingual", "True", "observed_ct2_is_multilingual"), +]) +def test_observed_identity_must_match_the_request(key, value, match): + observed = {**OBSERVED_CUDA, key: value} + assert any(match in p for p in _validate(_results(observed))) + + +@pytest.mark.parametrize("key", sorted(OBSERVED_CUDA)) +def test_unreported_identity_is_incomplete(key): + observed = {k: v for k, v in OBSERVED_CUDA.items() if k != key} + assert any(f"did not report {key}" in p for p in _validate(_results(observed))) + + +def test_requested_settings_alone_are_not_evidence(): + """A run whose identity test recorded nothing cannot be complete, whatever + was requested.""" + problems = _validate(_results(observed={})) + assert sum("did not report" in p for p in problems) == len(OBSERVED_CUDA) + + +def test_wrong_or_missing_gpu_is_incomplete(): + assert any("expected one containing 'T4'" in p + for p in _validate(gpus=["NVIDIA A100-SXM4-40GB, 550.0, 40960 MiB"])) + assert any("no GPU" in p for p in _validate(gpus=[])) + assert _validate(gpus=[], expect_gpu=None) == [] # check explicitly disabled + + +@pytest.mark.parametrize("var", ["PYTEST_ADDOPTS", "PYTHONPATH"]) +def test_plugin_or_path_overrides_are_incomplete(var): + overrides = {"PYTEST_ADDOPTS": None, "PYTHONPATH": None, var: "-p stub_plugin"} + assert any(f"{var} is set" in p for p in _validate(overrides=overrides)) + + +@pytest.mark.parametrize("git", [{"commit": "f" * 40, "dirty": True}, None, {"commit": None}]) +def test_dirty_or_unknown_checkout_is_incomplete(git): + assert any("not a clean commit" in p for p in _validate(git=git)) + + +def test_pytest_failure_or_missing_junit_is_incomplete(): + assert any("pytest exited 1" in p for p in _validate(exit_code=1)) + problems = _validate({"collected": 0, "junit_missing": True}, exit_code=4) + assert problems == ["pytest exited 4", "no JUnit results (pytest did not run the suite)"] + + +def test_duplicate_test_names_are_incomplete(): + results = _results() + results["tests"].append(copy.deepcopy(results["tests"][0])) + assert any("duplicate" in p for p in _validate(results))