Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions docs/exec-plans/completed/20261004-poisson-ci-diagnostics.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Make non-gating Poisson failures visible

## Objective

Expose failed numerical acceptance in CI without changing benchmark tolerances
or the intentionally non-gating smoke and scheduled workflows.

## Scope and acceptance

- Add a warning for failed runs using `--allow-fail`.
- Preserve strict exit status and JSON evidence.
- Test failing/passing, strict/non-gating, and local/Actions paths.
- Document the distinction between command success and numerical acceptance.
- Run the repository quality gate before handoff.

## Progress

- [x] Reproduce both quick backends reporting failed tolerances with exit 0.
- [x] Add explicit diagnostics and documentation.
- [x] Test all eight CLI outcomes; `make dev` passed on Python 3.12.
- [x] Verify repository review settings before publication.

## Outcome

`make dev` passed: formatting, Ruff, mypy, architecture, docs, the full 385-case
collection (one optional skip), and all 16 cut-panel evaluations. Eight new CLI
cases cover strict/non-gating failures and successes on local and Actions paths.
A real quick jplus run also emitted the expected Actions warning. Numerical
tolerances and non-gating workflow policy are unchanged.
8 changes: 8 additions & 0 deletions docs/poisson-benchmarks.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,14 @@ Options:
- `--profile quick|dense`
- `--backend-mode jplus|folded`
- `--manifest-path <path>` for machine-readable output
- `--allow-fail` to report tolerance failures without a failing exit status

Both the PR smoke workflow and the scheduled dense workflow use `--allow-fail`.
Their green status confirms that the benchmark command completed, not that its
numerical tolerances passed. A tolerance failure emits a GitHub Actions warning
(or a stderr warning locally); inspect `overall_passed` and the scheduled JSON
manifest's `passed` field for numerical acceptance. Without `--allow-fail`, failed
tolerances return exit status 1.

## What Is Measured

Expand Down
13 changes: 13 additions & 0 deletions scripts/run_poisson_benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@

import argparse
import json
import os
from pathlib import Path
import sys

from cutkit.evals import run_poisson_benchmarks

Expand Down Expand Up @@ -71,6 +73,17 @@ def main() -> int:
)
print(f"wrote manifest: {args.manifest_path}")

if not result.passed and args.allow_fail:
message = (
f"Poisson benchmark tolerances failed ({result.profile}, "
f"{result.planar.backend_mode}); --allow-fail keeps this run "
"non-gating. A successful command is not numerical acceptance."
)
if os.environ.get("GITHUB_ACTIONS") == "true":
print(f"::warning::{message}")
else:
print(f"WARNING: {message}", file=sys.stderr)

if result.passed or args.allow_fail:
return 0
return 1
Expand Down
54 changes: 54 additions & 0 deletions tests/evals/test_poisson_benchmark_cli.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
from __future__ import annotations

import importlib.util
import json
from pathlib import Path
import sys

import pytest

from cutkit.evals.poisson_benchmarks import (
PlanarPoissonBenchmarkResult,
PoissonBenchmarkResult,
VolumePoissonBenchmarkResult,
)


@pytest.mark.parametrize("passed", [False, True])
@pytest.mark.parametrize("allow_fail", [False, True])
@pytest.mark.parametrize("github_actions", [False, True])
def test_cli_reports_numerical_failure_without_changing_exit_contract(
passed: bool,
allow_fail: bool,
github_actions: bool,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
capsys: pytest.CaptureFixture[str],
) -> None:
script = Path(__file__).resolve().parents[2] / "scripts/run_poisson_benchmarks.py"
spec = importlib.util.spec_from_file_location("poisson_benchmark_cli", script)
assert spec is not None and spec.loader is not None
cli = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cli)
result = PoissonBenchmarkResult(
profile="quick",
planar=PlanarPoissonBenchmarkResult("jplus", 1.0, (), 0.001, 0.01, passed),
volume=VolumePoissonBenchmarkResult("jplus", 5, 1.0, (), 0.001, 0.01, passed),
passed=passed,
)
monkeypatch.setattr(cli, "run_poisson_benchmarks", lambda **kwargs: result)
manifest = tmp_path / "result.json"
args = [str(script), "--manifest-path", str(manifest)]
if allow_fail:
args.append("--allow-fail")
monkeypatch.setattr(sys, "argv", args)
monkeypatch.setenv("GITHUB_ACTIONS", "true" if github_actions else "false")

assert cli.main() == (0 if passed or allow_fail else 1)
captured = capsys.readouterr()
assert json.loads(manifest.read_text())["passed"] is passed
assert f"overall_passed = {passed}" in captured.out
should_warn = not passed and allow_fail
assert ("::warning::" in captured.out) is (should_warn and github_actions)
assert ("WARNING:" in captured.err) is (should_warn and not github_actions)
assert ("not numerical acceptance" in captured.out + captured.err) is should_warn
Loading