diff --git a/.github/workflows/fuzz-nightly.yml b/.github/workflows/fuzz-nightly.yml new file mode 100644 index 0000000..64325e7 --- /dev/null +++ b/.github/workflows/fuzz-nightly.yml @@ -0,0 +1,38 @@ +name: fuzz-nightly + +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + inputs: + examples: + description: "number of random programs" + default: "5000" + +jobs: + fuzz: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: actions/setup-node@v4 + with: + node-version: "20" + - run: pip install -e ".[dev]" && pip install -e targets/tiny-gpu --no-deps + # Fresh programs every night (FUZZ_RANDOM=1); Hypothesis prints the seed to reproduce a failure. + - run: pytest tests/test_fuzz.py -x + env: + FUZZ_EXAMPLES: ${{ github.event.inputs.examples || '5000' }} + FUZZ_RANDOM: "1" + - name: Upload the shrunk failing program + if: failure() + uses: actions/upload-artifact@v4 + with: + name: shrunk-failing-program + path: tests/regressions/_last_failure.tiny + if-no-files-found: warn diff --git a/.github/workflows/py.yml b/.github/workflows/py.yml index 9cce37d..c25302f 100644 --- a/.github/workflows/py.yml +++ b/.github/workflows/py.yml @@ -24,3 +24,4 @@ jobs: # The contract tests run first so a submodule bump fails loudly before anything else. - run: pytest tests/test_targets_contract.py - run: pytest + # 100 fixed-seed fuzz examples ride along in the full run above; the big run is fuzz-nightly. diff --git a/.gitignore b/.gitignore index dc1c4a7..fa18521 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ __pycache__/ node_modules/ web/dist/ web/.astro/ + +# fuzzer output (rename and commit as a regression once understood) +tests/regressions/_last_failure.tiny diff --git a/CONTEXT.md b/CONTEXT.md index 0d7b967..3831d5f 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -24,7 +24,10 @@ faked; host-side work is always labelled as host work. - Phase 5 (GPU kernel optimizations): DONE. 557 tests green. `-O0/-O1/-O2`: licm, strength-reduce, kernel-dce | + shared-tile, unroll, fold. Measured table in `docs/ARCHITECTURE.md` (generated by `scripts/opt_table.py`, drift-tested). `-O2` has strictly fewer global loads than `-O0` on every matmul example (exact formula tested). Cost model is now an abstract interpretation. -- Next: Phase 6 (differential fuzzing + hardening; nightly 5000-example job; publishable CLI). +- Phase 6 (differential fuzzing + hardening): DONE. 611 tests green. `tests/test_fuzz.py` runs random programs through gpu -O0/1/2 and tpu -O0/1 on + the real chips against the source oracle; two independent 5000-program runs were green; 13-program corner-case corpus in + `tests/regressions/`; `fuzz-nightly.yml`; `docs/VERIFICATION.md`. The CLI compiler is now publishable on its own. +- Next: Phase 7 (Compiler Explorer web app: Pyodide worker, pass explorer, TPU schedule view, GPU kernel view, head-to-head). ## Fixed decisions (do not re-litigate; change only via docs/ARCHITECTURE.md) | Decision | Choice | diff --git a/README.md b/README.md index 4db2ae4..ac2d9b6 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ One tiny tensor language, two real chips: `C = relu(A @ B)` compiles to [tiny-gpu](https://github.com/deaneeth/tiny-gpu) assembly or to a [tiny-tpu](https://github.com/deaneeth/tiny-tpu) tile schedule, and runs on the real simulator / RTL. -**Status: Phase 5.** The frontend, optimizing pass pipeline and **both backends** work. One program, +**Status: Phase 6.** The frontend, optimizing pass pipeline and **both backends** work. One program, two chips, bit-exact against numpy on each: ```bash @@ -25,3 +25,5 @@ pytest ``` Requires Python 3.12+ and Node 20+ (the tiny-tpu WASM is loaded from the pinned submodule). + +Everything is checked against a numpy oracle on the real chips; see [docs/VERIFICATION.md](docs/VERIFICATION.md). diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md new file mode 100644 index 0000000..f68ca31 --- /dev/null +++ b/docs/VERIFICATION.md @@ -0,0 +1,39 @@ +# Verification + +Every claim the compiler makes is checked against something independent. Layers, from the source out: + +| Layer | What | Where | +|---|---|---| +| Oracle | numpy int64 evaluation of the *source* program; any value outside i32 is an error (tiny-gpu raises on overflow) | `tinyc/ir/ref.py` | +| Interpreters | independent scalar interpreters for Graph IR, the TPU command list, and Kernel IR (with exact tiny-gpu counts, including barriers) | `ir/interp.py`, `tpu/interp.py`, `gpu/interp.py` | +| Per-pass validation | `--verify-each` re-runs the program after *every* pass and compares with the oracle | `passes/manager.py`, `tpu/__init__.py`, `gpu/__init__.py` | +| Real targets | emitted code runs on the unmodified tiny-gpu simulator and the Verilated tiny-tpu RTL | `runtime/` | +| Contract tests | every fact about the targets the design depends on is measured and pinned | `tests/test_targets_contract.py` | +| Properties | Hypothesis: semantics preserved, pipeline idempotent, range-check sound, estimates bound measurements | `tests/**/test_*properties.py`, `tests/passes/test_pipeline.py` | +| Differential fuzzing | random programs through *every* backend and level on the real chips | `tests/test_fuzz.py`, `tests/fuzz_support.py` | +| Regression corpus | corner cases and past bugs, replayed everywhere | `tests/regressions/*.tiny` | +| Goldens & pins | emitted IR/code/diagnostics; cost numbers per example and level | `tests/golden/`, `tests/perf_baseline.json` | +| Docs that quote numbers | the measured optimization table must equal a fresh measurement | `tests/gpu/test_opt_table.py` | + +## Differential fuzzing +`check_program` compiles one program for `gpu -O0/-O1/-O2` and `tpu -O0/-O1`, runs each on the real chip, +and requires bit-exact agreement with the oracle, R5-R7 untouched, estimate bounds containing the +measurement (loads/stores exact), and the TPU cost model equal to the measured cycles. A program a +backend legitimately rejects (a chained matmul on the TPU, an overflow that cannot be ruled out) is +skipped for that backend only. + +```bash +pytest tests/test_fuzz.py # 100 examples, fixed seed (part of CI) +FUZZ_EXAMPLES=5000 FUZZ_RANDOM=1 pytest tests/test_fuzz.py # the nightly run: fresh programs +``` +On failure Hypothesis shrinks the program and it is left in `tests/regressions/_last_failure.tiny` +(the nightly workflow uploads it). Understand the bug, fix it, rename the file to describe the bug and +commit it: `tests/test_regressions.py` then replays it on every backend forever. + +## Bugs the verification found while building +Each is covered by a regression test. +- The simplification passes were not idempotent (CSE broke canonical operand order); fixed by running + them to a fixpoint. `tests/regressions/duplicate_subexpressions.tiny` and `test_manager.py`. +- A pass-generated instruction had no source position (strength reduction's pointer init). +- Fusion was silently untested because the first random generator never produced a fusable pattern. +- Early tests pinned cycle counts the simulator does not produce; measuring first fixed the model. diff --git a/tests/fuzz_support.py b/tests/fuzz_support.py new file mode 100644 index 0000000..1f67f68 --- /dev/null +++ b/tests/fuzz_support.py @@ -0,0 +1,82 @@ +"""One differential check, used by the fuzzer and the regression corpus. + +A program is compiled for every backend and optimization level that can run it, executed on the +real chip/simulator, and every result must equal the numpy oracle computed from the *source* +program (before any pass). Two backends therefore agree with each other by transitivity. +""" + +from __future__ import annotations + +import numpy as np + +from runtime.gpu_runner import run_launch +from runtime.tpu_runner import run_ttpu +from tinyc.driver import build_pipeline, compile_gpu, compile_tpu +from tinyc.frontend import compile_to_graph +from tinyc.ir import ref +from tinyc.ir.errors import PassError, RangeError +from tinyc.passes import PassContext, PassManager + +GPU_LEVELS = (0, 1, 2) +TPU_LEVELS = (0, 1) + + +def check_program(source: str, seed: int = 1) -> dict[str, str]: + """Raises AssertionError (naming target and level) on any disagreement. + + Returns {backend-and-level: "ok" | "rejected: why"} so callers can see what actually ran. + """ + func = compile_to_graph(source, "fuzz.tiny") + inputs = ref.random_inputs(func, np.random.default_rng(seed)) + try: + want = ref.evaluate(func, inputs) + except RangeError: + return {"source": "overflows i32 on these inputs"} + + ran: dict[str, str] = {} + for opt in GPU_LEVELS: + key = f"gpu-O{opt}" + try: + comp = compile_gpu(source, "fuzz.tiny", opt_level=opt) + except PassError as e: + ran[key] = f"rejected: {e.message}" + continue + res = run_launch(comp.launch, inputs) + assert res.reserved_intact, f"{key}: a kernel clobbered R5-R7" + for name, w in zip(comp.launch["outputs"], want): + assert np.array_equal( + res.outputs[name], w + ), f"{key}: output %{name} differs from the oracle" + est = comp.program.estimate + assert est["instructions"]["min"] <= res.instructions <= est["instructions"]["max"], key + assert est["cycles"]["min"] <= res.cycles <= est["cycles"]["max"], key + assert (res.op_counts["LD"], res.op_counts["ST"]) == (est["loads"], est["stores"]), key + ran[key] = "ok" + for opt in TPU_LEVELS: + key = f"tpu-O{opt}" + try: + comp = compile_tpu(source, "fuzz.tiny", opt_level=opt) + except PassError as e: + ran[key] = f"rejected: {e.message}" + continue + res = run_ttpu(comp.program, inputs) + for name, w in zip(comp.schedule.outputs, want): + assert np.array_equal( + res.outputs[name], w + ), f"{key}: output %{name} differs from the oracle" + est = comp.program["estimate"] + assert res.total_cycles == est["rtl_cycles"], f"{key}: cost model != measurement" + assert res.host_ops == est["host_ops"], key + ran[key] = "ok" + return ran + + +def pipeline_is_idempotent(source: str) -> bool: + """Running the graph pipeline on its own output changes nothing.""" + func = compile_to_graph(source, "fuzz.tiny") + try: + PassManager(build_pipeline(None, 1)).run(func, PassContext()) + except PassError: + return True # rejected by range-check: nothing to compare + again = PassManager(build_pipeline(None, 1)).run(func, PassContext()) + return not any(r.changed for r in again.records) diff --git a/tests/regressions/duplicate_subexpressions.tiny b/tests/regressions/duplicate_subexpressions.tiny new file mode 100644 index 0000000..4cd65d8 --- /dev/null +++ b/tests/regressions/duplicate_subexpressions.tiny @@ -0,0 +1,10 @@ +# CSE and canonicalisation must agree about operand order +input x0 : i8[1] +input x1 : i8[2] +v0 = relu(x0) +v1 = relu(x1) +v2 = x0 + x0 +v3 = relu(x0) +v4 = relu(x0) +v5 = v2 + v3 +output v5 diff --git a/tests/regressions/k_is_one.tiny b/tests/regressions/k_is_one.tiny new file mode 100644 index 0000000..3d709a8 --- /dev/null +++ b/tests/regressions/k_is_one.tiny @@ -0,0 +1,5 @@ +# reduction of length 1: a one-iteration loop, tiling of a single word +input A : i8[3, 1] +input B : i8[1, 3] +C = relu(A @ B) +output C diff --git a/tests/regressions/long_reduction.tiny b/tests/regressions/long_reduction.tiny new file mode 100644 index 0000000..28a1e22 --- /dev/null +++ b/tests/regressions/long_reduction.tiny @@ -0,0 +1,5 @@ +# K = 20: the loop is not unrolled but A is still tiled (guarded copy rounds) +input A : i8[2, 20] +input B : i8[20, 3] +C = relu(A @ B) +output C diff --git a/tests/regressions/many_threads.tiny b/tests/regressions/many_threads.tiny new file mode 100644 index 0000000..54c1c04 --- /dev/null +++ b/tests/regressions/many_threads.tiny @@ -0,0 +1,4 @@ +# 900 threads in one launch +input A : i8[30, 30] +B = relu(A) +output B diff --git a/tests/regressions/n_is_one.tiny b/tests/regressions/n_is_one.tiny new file mode 100644 index 0000000..3c29208 --- /dev/null +++ b/tests/regressions/n_is_one.tiny @@ -0,0 +1,5 @@ +# one output column: one-thread blocks (no shared tiling possible) +input A : i8[4, 5] +input B : i8[5, 1] +C = A @ B +output C diff --git a/tests/regressions/one_by_one.tiny b/tests/regressions/one_by_one.tiny new file mode 100644 index 0000000..cc05808 --- /dev/null +++ b/tests/regressions/one_by_one.tiny @@ -0,0 +1,4 @@ +input A : i8[1, 1] +input B : i8[1, 1] +C = relu(A @ B + A @ B) +output C diff --git a/tests/regressions/output_is_const.tiny b/tests/regressions/output_is_const.tiny new file mode 100644 index 0000000..475d6e3 --- /dev/null +++ b/tests/regressions/output_is_const.tiny @@ -0,0 +1,4 @@ +const W : i8[2, 3] = [[1, -2, 3], [0, 0, 0]] +input X : i8[3, 2] +Y = W @ X +output Y, W diff --git a/tests/regressions/partial_tiles.tiny b/tests/regressions/partial_tiles.tiny new file mode 100644 index 0000000..8a3c98b --- /dev/null +++ b/tests/regressions/partial_tiles.tiny @@ -0,0 +1,6 @@ +# 5x6x7: every tile dimension is partial on the TPU; guarded cooperative copy on the GPU +input A : i8[5, 6] +input B : i8[6, 7] +input b : i32[7] +C = relu(A @ B + b) +output C diff --git a/tests/regressions/relu_of_input.tiny b/tests/regressions/relu_of_input.tiny new file mode 100644 index 0000000..6015c68 --- /dev/null +++ b/tests/regressions/relu_of_input.tiny @@ -0,0 +1,3 @@ +input A : i8[3, 3] +R = relu(A) +output R, A diff --git a/tests/regressions/three_chained_elementwise.tiny b/tests/regressions/three_chained_elementwise.tiny new file mode 100644 index 0000000..f033ee7 --- /dev/null +++ b/tests/regressions/three_chained_elementwise.tiny @@ -0,0 +1,8 @@ +input A : i8[4, 4] +input B : i8[4, 4] +input c : i32[4] +M = A @ B +N = relu(M + c) +P = N + N +Q = relu(P) +output Q, M diff --git a/tests/regressions/two_matmuls_shared_operand.tiny b/tests/regressions/two_matmuls_shared_operand.tiny new file mode 100644 index 0000000..d9282f7 --- /dev/null +++ b/tests/regressions/two_matmuls_shared_operand.tiny @@ -0,0 +1,6 @@ +input A : i8[4, 4] +input B : i8[4, 4] +X = A @ B +Y = B @ A +Z = X + Y +output Z diff --git a/tests/regressions/vector_ops.tiny b/tests/regressions/vector_ops.tiny new file mode 100644 index 0000000..280464a --- /dev/null +++ b/tests/regressions/vector_ops.tiny @@ -0,0 +1,5 @@ +# rank-1 values: one block of N threads +input a : i8[7] +input b : i32[7] +C = relu(a + b) +output C diff --git a/tests/regressions/zero_tiles.tiny b/tests/regressions/zero_tiles.tiny new file mode 100644 index 0000000..c15140f --- /dev/null +++ b/tests/regressions/zero_tiles.tiny @@ -0,0 +1,6 @@ +# constant zero blocks: skipped passes, standalone epilogue +input X : i8[8, 4] +input b : i32[4] +const Z : i8[4, 8] = [[0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0]] +Y = relu(Z @ X + b) +output Y diff --git a/tests/test_fuzz.py b/tests/test_fuzz.py new file mode 100644 index 0000000..c907e7d --- /dev/null +++ b/tests/test_fuzz.py @@ -0,0 +1,41 @@ +"""Differential fuzzing: random legal programs, every backend and level, real chips. + + pytest tests/test_fuzz.py # 100 examples, fixed seed (CI) + FUZZ_EXAMPLES=5000 FUZZ_RANDOM=1 pytest tests/test_fuzz.py # nightly: fresh programs + +A failing program is written, shrunk by Hypothesis, to tests/regressions/_last_failure.tiny +(overwritten on each failing run, so the last one is the minimal one). Rename it and commit it as a +regression once the bug is understood. +""" + +import os +from pathlib import Path + +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from tests.fuzz_support import check_program, pipeline_is_idempotent +from tests.gen import programs, tpu_programs + +EXAMPLES = int(os.environ.get("FUZZ_EXAMPLES", "100")) +RANDOM = os.environ.get("FUZZ_RANDOM") == "1" +LAST_FAILURE = Path(__file__).parent / "regressions" / "_last_failure.tiny" + +FUZZ = settings( + max_examples=EXAMPLES, + deadline=None, + derandomize=not RANDOM, + database=None, + suppress_health_check=list(HealthCheck), +) + + +@FUZZ +@given(st.one_of(tpu_programs(), programs()), st.integers(0, 2**32 - 1)) +def test_every_backend_and_level_agrees_with_the_oracle(source, seed): + try: + check_program(source, seed) + assert pipeline_is_idempotent(source) + except BaseException: + LAST_FAILURE.write_text(source) + raise diff --git a/tests/test_regressions.py b/tests/test_regressions.py new file mode 100644 index 0000000..45f163e --- /dev/null +++ b/tests/test_regressions.py @@ -0,0 +1,26 @@ +"""Every program in tests/regressions/ runs on every backend and level. Bugs found by the fuzzer +(or by hand) are added here as `.tiny` files, so they stay fixed.""" + +from pathlib import Path + +import pytest + +from tests.fuzz_support import check_program, pipeline_is_idempotent + +CASES = sorted(Path(__file__).parent.glob("regressions/*.tiny")) + + +def test_the_corpus_is_not_empty(): + assert len(CASES) >= 10 + + +@pytest.mark.parametrize("path", CASES, ids=lambda p: p.stem) +@pytest.mark.parametrize("seed", [1, 2, 3]) +def test_regression_program_agrees_with_the_oracle_everywhere(path, seed): + ran = check_program(path.read_text(), seed) + assert "ok" in ran.values(), f"nothing ran for {path.name}: {ran}" + + +@pytest.mark.parametrize("path", CASES, ids=lambda p: p.stem) +def test_regression_program_pipeline_is_idempotent(path): + assert pipeline_is_idempotent(path.read_text())