From 34bed3954eae3d626c3af72e3db0a43f014c68f7 Mon Sep 17 00:00:00 2001 From: xywei Date: Sun, 4 Oct 2026 10:16:47 -0500 Subject: [PATCH] ci: surface non-gating Poisson tolerance failures --- .../20261004-poisson-ci-diagnostics.md | 29 ++++++++++ docs/poisson-benchmarks.md | 8 +++ scripts/run_poisson_benchmarks.py | 13 +++++ tests/evals/test_poisson_benchmark_cli.py | 54 +++++++++++++++++++ 4 files changed, 104 insertions(+) create mode 100644 docs/exec-plans/completed/20261004-poisson-ci-diagnostics.md create mode 100644 tests/evals/test_poisson_benchmark_cli.py diff --git a/docs/exec-plans/completed/20261004-poisson-ci-diagnostics.md b/docs/exec-plans/completed/20261004-poisson-ci-diagnostics.md new file mode 100644 index 0000000..376280c --- /dev/null +++ b/docs/exec-plans/completed/20261004-poisson-ci-diagnostics.md @@ -0,0 +1,29 @@ +# Make non-gating Poisson failures visible + +## Objective + +Expose failed numerical acceptance in CI without changing benchmark tolerances +or the intentionally non-gating smoke and scheduled workflows. + +## Scope and acceptance + +- Add a warning for failed runs using `--allow-fail`. +- Preserve strict exit status and JSON evidence. +- Test failing/passing, strict/non-gating, and local/Actions paths. +- Document the distinction between command success and numerical acceptance. +- Run the repository quality gate before handoff. + +## Progress + +- [x] Reproduce both quick backends reporting failed tolerances with exit 0. +- [x] Add explicit diagnostics and documentation. +- [x] Test all eight CLI outcomes; `make dev` passed on Python 3.12. +- [x] Verify repository review settings before publication. + +## Outcome + +`make dev` passed: formatting, Ruff, mypy, architecture, docs, the full 385-case +collection (one optional skip), and all 16 cut-panel evaluations. Eight new CLI +cases cover strict/non-gating failures and successes on local and Actions paths. +A real quick jplus run also emitted the expected Actions warning. Numerical +tolerances and non-gating workflow policy are unchanged. diff --git a/docs/poisson-benchmarks.md b/docs/poisson-benchmarks.md index 5dde49d..fa3b0cc 100644 --- a/docs/poisson-benchmarks.md +++ b/docs/poisson-benchmarks.md @@ -23,6 +23,14 @@ Options: - `--profile quick|dense` - `--backend-mode jplus|folded` - `--manifest-path ` for machine-readable output +- `--allow-fail` to report tolerance failures without a failing exit status + +Both the PR smoke workflow and the scheduled dense workflow use `--allow-fail`. +Their green status confirms that the benchmark command completed, not that its +numerical tolerances passed. A tolerance failure emits a GitHub Actions warning +(or a stderr warning locally); inspect `overall_passed` and the scheduled JSON +manifest's `passed` field for numerical acceptance. Without `--allow-fail`, failed +tolerances return exit status 1. ## What Is Measured diff --git a/scripts/run_poisson_benchmarks.py b/scripts/run_poisson_benchmarks.py index 4485bd9..e277dc7 100644 --- a/scripts/run_poisson_benchmarks.py +++ b/scripts/run_poisson_benchmarks.py @@ -5,7 +5,9 @@ import argparse import json +import os from pathlib import Path +import sys from cutkit.evals import run_poisson_benchmarks @@ -71,6 +73,17 @@ def main() -> int: ) print(f"wrote manifest: {args.manifest_path}") + if not result.passed and args.allow_fail: + message = ( + f"Poisson benchmark tolerances failed ({result.profile}, " + f"{result.planar.backend_mode}); --allow-fail keeps this run " + "non-gating. A successful command is not numerical acceptance." + ) + if os.environ.get("GITHUB_ACTIONS") == "true": + print(f"::warning::{message}") + else: + print(f"WARNING: {message}", file=sys.stderr) + if result.passed or args.allow_fail: return 0 return 1 diff --git a/tests/evals/test_poisson_benchmark_cli.py b/tests/evals/test_poisson_benchmark_cli.py new file mode 100644 index 0000000..01eb5f3 --- /dev/null +++ b/tests/evals/test_poisson_benchmark_cli.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import sys + +import pytest + +from cutkit.evals.poisson_benchmarks import ( + PlanarPoissonBenchmarkResult, + PoissonBenchmarkResult, + VolumePoissonBenchmarkResult, +) + + +@pytest.mark.parametrize("passed", [False, True]) +@pytest.mark.parametrize("allow_fail", [False, True]) +@pytest.mark.parametrize("github_actions", [False, True]) +def test_cli_reports_numerical_failure_without_changing_exit_contract( + passed: bool, + allow_fail: bool, + github_actions: bool, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + script = Path(__file__).resolve().parents[2] / "scripts/run_poisson_benchmarks.py" + spec = importlib.util.spec_from_file_location("poisson_benchmark_cli", script) + assert spec is not None and spec.loader is not None + cli = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cli) + result = PoissonBenchmarkResult( + profile="quick", + planar=PlanarPoissonBenchmarkResult("jplus", 1.0, (), 0.001, 0.01, passed), + volume=VolumePoissonBenchmarkResult("jplus", 5, 1.0, (), 0.001, 0.01, passed), + passed=passed, + ) + monkeypatch.setattr(cli, "run_poisson_benchmarks", lambda **kwargs: result) + manifest = tmp_path / "result.json" + args = [str(script), "--manifest-path", str(manifest)] + if allow_fail: + args.append("--allow-fail") + monkeypatch.setattr(sys, "argv", args) + monkeypatch.setenv("GITHUB_ACTIONS", "true" if github_actions else "false") + + assert cli.main() == (0 if passed or allow_fail else 1) + captured = capsys.readouterr() + assert json.loads(manifest.read_text())["passed"] is passed + assert f"overall_passed = {passed}" in captured.out + should_warn = not passed and allow_fail + assert ("::warning::" in captured.out) is (should_warn and github_actions) + assert ("WARNING:" in captured.err) is (should_warn and not github_actions) + assert ("not numerical acceptance" in captured.out + captured.err) is should_warn