Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ TOOLS ?= $(WORK)/tools
REPRO_CONFIGS ?= pymupdf4llm,pdfluent-fast
REPRO_SETS ?= parsebench

.PHONY: help test bench smoke report compare datasets datasets-plan images cost fixtures goldens port tools repro validate public-check odl-verify odl-run
.PHONY: axes-combine axes-gate help test bench smoke report compare datasets datasets-plan images cost fixtures goldens port tools repro validate public-check odl-verify odl-run

help:
@echo "make repro the whole public subset from a clean checkout: pinned tools,"
Expand Down Expand Up @@ -101,6 +101,16 @@ odl-run:
$(PY) -m bench.odl run --fetch --clone $(ODL_CLONE) --recipe "$(RECIPE)" --baseline "$(BASELINE)" \
--out results/odl-$$(date -u +%Y%m%dT%H%M%SZ)

# The 19 holdout cells: `make axes-combine RUNS='results/<a> results/<b>' OUT=axis_means.json`,
# `make axes-gate BASELINE=<file> CANDIDATE=<file>` (exit 1 on a drop, 3 when a value is missing).
OUT ?= axis_means.json
CANDIDATE ?=
axes-combine:
$(PY) -m bench.axes combine $(RUNS) --out $(OUT)

axes-gate:
$(PY) -m bench.axes gate --baseline "$(BASELINE)" --candidate "$(CANDIDATE)" --tolerance 0.005

public-check:
$(PY) scripts/check_public_surface.py

Expand Down
156 changes: 156 additions & 0 deletions bench/axes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
# Copyright (c) 2026 Innovation Trigger B.V.
#
# PDFluent is available under two licences, at your option: the GNU AGPLv3, or
# the PDFluent Commercial Licence. See the LICENSE file in this repository --
# that file travels with the copy you received, which a URL does not.
"""The 19 dataset-by-metric cells and a gate over them.

python -m bench.axes combine <run dir>... --out axis_means.json
python -m bench.axes gate --baseline <file> --candidate <file> [--tolerance 0.005]

The 19 cells are `report.AXES_BY_SET` for FinTabNet (3), PubTables-1M (3) and
DocLayNet (8), plus the five `report.PARSEBENCH` headlines. `combine` writes
one mean per cell, per configuration and split (`<config>|<split>`), from the
per-set quality files of the run directories; for a configuration present in
several directories the first directory wins, as in `bench report`. A
configuration with no value in any cell is left out.

`gate` compares the holdout half only. Exit codes: 0 no cell dropped by more
than the tolerance; 1 a cell or the 19-cell mean dropped (each is named);
2 unreadable input; 3 a value is missing, so nothing can be said, and
`SKIPPED (not a pass): <reason>` is printed. A drop wins over a skip.

This file holds no scores. The files it reads and writes are measurements and
stay with the run that made them.
"""
from __future__ import annotations

import argparse
import json
import pathlib
import sys

from bench import report

SETS = ("fintabnet", "pubtables-1m", "doclaynet", "parsebench")
HOLDOUT = "holdout"
SKIP_EXIT = 3


def cells() -> list[tuple[str, str, str]]:
"""(set, key in `report.quality_numbers`, label in the means file), 19 of them."""
out = []
for set_name in SETS:
if set_name == "parsebench":
out += [(set_name, f"{g}.{m}", f"{report.SET_TITLE[set_name]} {label}")
for g, m, label in report.PARSEBENCH]
else:
out += [(set_name, axis, f"{report.SET_TITLE[set_name]} {label}")
for axis, label in report.AXES_BY_SET[set_name]]
return out


def labels() -> list[str]:
return [label for _s, _k, label in cells()]


def combine(dirs: list[pathlib.Path]) -> dict:
_runs, quality, _speed = report._load_runs(dirs)
wanted = cells()
out: dict[str, dict] = {}
for (_kind, set_name, split, config), (_run_id, run_file) in sorted(quality.items()):
if set_name not in SETS:
continue
numbers = report.quality_numbers(run_file, set_name)
row = out.setdefault(f"{config}|{split}", {label: None for _s, _k, label in wanted})
for s, key, label in wanted:
if s == set_name:
row[label] = (numbers.get(key) or {}).get("mean")
return {k: v for k, v in sorted(out.items()) if any(x is not None for x in v.values())}


def mean(row: dict) -> float | None:
values = [row.get(label) for label in labels()]
if any(v is None for v in values):
return None
return sum(values) / len(values)


def gate(baseline: dict, candidate: dict, tolerance: float,
configs: list[str] | None = None) -> tuple[int, list[str], list[str]]:
"""(exit code, failures, skips). Only `<config>|holdout` rows count."""
failures: list[str] = []
skips: list[str] = []
names = configs or sorted(k.split("|")[0] for k in candidate if k.endswith("|" + HOLDOUT))
if not names:
skips.append(f"the candidate has no {HOLDOUT} rows")
for name in names:
key = f"{name}|{HOLDOUT}"
base, cand = baseline.get(key), candidate.get(key)
if base is None or cand is None:
skips.append(f"{key} is missing from the {'baseline' if base is None else 'candidate'}")
continue
for label in labels():
b, c = base.get(label), cand.get(label)
if b is None or c is None:
skips.append(f"{key} {label}: no value in the {'baseline' if b is None else 'candidate'}")
elif b - c - tolerance > 1e-12:
failures.append(f"{key} {label}: {b:.4f} -> {c:.4f} (drop {b - c:.4f}, "
f"tolerance {tolerance})")
mb, mc = mean(base), mean(cand)
if mb is not None and mc is not None and mb - mc - tolerance > 1e-12:
failures.append(f"{key} mean of 19: {mb:.4f} -> {mc:.4f} (drop {mb - mc:.4f}, "
f"tolerance {tolerance})")
return (1 if failures else SKIP_EXIT if skips else 0), failures, skips


def _fatal(why: str):
print(f"[axes] FATAL: {why}", file=sys.stderr)
raise SystemExit(2)


def _read(path: str) -> dict:
try:
data = json.loads(pathlib.Path(path).read_text())
except (OSError, ValueError) as exc:
_fatal(f"cannot read {path}: {exc}")
if not isinstance(data, dict):
_fatal(f"{path} is not a JSON object")
return data


def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(prog="python -m bench.axes", description=__doc__.split("\n")[0])
sub = ap.add_subparsers(dest="command", required=True)
co = sub.add_parser("combine", help="write axis_means.json from run directories")
co.add_argument("runs", nargs="+", type=pathlib.Path)
co.add_argument("--out", type=pathlib.Path, required=True)
ga = sub.add_parser("gate", help="fail when a holdout cell or the mean drops")
ga.add_argument("--baseline", required=True)
ga.add_argument("--candidate", required=True)
ga.add_argument("--tolerance", type=float, default=0.005)
ga.add_argument("--config", action="append",
help="gate only this configuration (repeatable); default: every one in the candidate")
args = ap.parse_args(argv)
if args.command == "combine":
missing = [d for d in args.runs if not (d / "run.json").is_file()]
if missing:
_fatal(f"no run.json in {missing[0]}")
data = combine(args.runs)
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(json.dumps(data, indent=1) + "\n")
print(f"[axes] {len(data)} rows written to {args.out}")
return 0
code, failures, skips = gate(_read(args.baseline), _read(args.candidate),
args.tolerance, args.config)
for line in failures:
print(f"[axes] FAIL {line}", file=sys.stderr)
for line in skips:
print(f"SKIPPED (not a pass): {line}", file=sys.stderr)
if code == 0:
print("[axes] ok: no holdout cell and no mean dropped by more than the tolerance")
return code


if __name__ == "__main__":
sys.exit(main())
203 changes: 203 additions & 0 deletions tests/test_axes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
# Copyright (c) 2026 Innovation Trigger B.V.
#
# PDFluent is available under two licences, at your option: the GNU AGPLv3, or
# the PDFluent Commercial Licence. See the LICENSE file in this repository --
# that file travels with the copy you received, which a URL does not.
"""bench.axes: combine from synthetic run directories, the gate's thresholds, its bite, its skips."""
import copy
import json
import pathlib
import subprocess
import sys

import pytest

from bench import axes, report
from skipping import skip

ROOT = pathlib.Path(__file__).resolve().parents[1]
AXIS_KEY = {"text": "similarity", "text_geometric": "similarity", "words": "f1", "headings": "f1",
"lists": "f1", "tables": "f1", "teds": "teds", "order": "score",
"order_geometric": "score"}


def _quality_file(set_name: str, values: dict) -> dict:
"""A per-set quality file whose cells equal `values` (axis key -> number)."""
if set_name == "parsebench":
groups: dict = {}
for key, v in values.items():
group, metric = key.split(".")
groups.setdefault(group, {"examples": 4, "metrics": {}})["metrics"][metric] = v
row = {"pages": 4, "groups": groups}
else:
doc = {"name": "d-1", "document": "d"}
for axis, v in values.items():
doc[axis] = {"scored": True, AXIS_KEY[axis]: v}
row = {"pages": 1, "documents": [doc]}
return {"bench": {"tool": "t"}, "tools": {"t": row}}


def _write_run(root: pathlib.Path, name: str, table: dict) -> pathlib.Path:
"""table: {(config, split): {label: number}} -> a run directory."""
d = root / name
d.mkdir(parents=True)
(d / "run.json").write_text(json.dumps({"run_id": name, "machine": {"gpus": [], "spec": {}}}))
for (config, split), row in table.items():
for set_name in axes.SETS:
values = {k: row[label] for s, k, label in axes.cells() if s == set_name and row.get(label) is not None}
if not values:
continue
path = d / "quality" / set_name / split / f"{config}.json"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(_quality_file(set_name, values)))
return d


def _row(base: float) -> dict:
return {label: round(base + 0.01 * i, 6) for i, label in enumerate(axes.labels())}


def test_there_are_nineteen_cells_in_four_sets():
assert len(axes.cells()) == 19 and len(set(axes.labels())) == 19
per_set = {s: sum(1 for c in axes.cells() if c[0] == s) for s in axes.SETS}
assert per_set == {"fintabnet": 3, "pubtables-1m": 3, "doclaynet": 8, "parsebench": 5}


def test_combine_reproduces_the_inputs_for_every_config_and_split(tmp_path):
table = {("eng-a", "holdout"): _row(0.30), ("eng-a", "tuning"): _row(0.35),
("eng-b", "holdout"): _row(0.40)}
run = _write_run(tmp_path, "run-1", table)
got = axes.combine([run])
assert got == {f"{c}|{s}": row for (c, s), row in table.items()}
assert axes.mean(got["eng-a|holdout"]) == pytest.approx(sum(table[("eng-a", "holdout")].values()) / 19)


def test_combine_first_directory_wins_and_two_runs_merge(tmp_path):
one = _write_run(tmp_path, "one", {("eng-a", "holdout"): _row(0.30)})
two = _write_run(tmp_path, "two", {("eng-a", "holdout"): _row(0.90), ("eng-b", "holdout"): _row(0.40)})
got = axes.combine([one, two])
assert got["eng-a|holdout"] == _row(0.30) and got["eng-b|holdout"] == _row(0.40)


def test_combine_leaves_out_an_unmeasured_config_and_keeps_holes_as_null(tmp_path):
row = _row(0.5)
row["ParseBench chart"] = None
run = _write_run(tmp_path, "run", {("eng-a", "holdout"): row})
(run / "quality" / "doclaynet" / "holdout" / "empty.json").write_text(
json.dumps({"bench": {"tool": "t"}, "tools": {"t": {"pages": 0, "documents": []}}}))
got = axes.combine([run])
assert "empty|holdout" not in got
assert got["eng-a|holdout"]["ParseBench chart"] is None
assert axes.mean(got["eng-a|holdout"]) is None


def test_combine_cli_writes_the_file(tmp_path):
run = _write_run(tmp_path, "run", {("eng-a", "holdout"): _row(0.5)})
out = tmp_path / "o" / "axis_means.json"
assert axes.main(["combine", str(run), "--out", str(out)]) == 0
assert json.loads(out.read_text()) == {"eng-a|holdout": _row(0.5)}


def _pair(**override):
base = {"eng-a|holdout": _row(0.5), "eng-a|tuning": _row(0.5)}
cand = copy.deepcopy(base)
cand["eng-a|holdout"].update(override)
return base, cand


def test_gate_passes_identical_and_improved():
base, cand = _pair()
assert axes.gate(base, cand, 0.005)[0] == 0
better = {k: dict(v) for k, v in base.items()}
better["eng-a|holdout"] = {k: v + 0.1 for k, v in better["eng-a|holdout"].items()}
assert axes.gate(base, better, 0.005)[0] == 0


def test_gate_tolerance_boundary_and_cell_name():
label = "DocLayNet lists F1"
base, cand = _pair()
cand["eng-a|holdout"][label] = base["eng-a|holdout"][label] - 0.005
assert axes.gate(base, cand, 0.005)[0] == 0 # exactly the tolerance passes
cand["eng-a|holdout"][label] -= 0.001
code, failures, _ = axes.gate(base, cand, 0.005)
assert code == 1 and any(label in f and "eng-a|holdout" in f for f in failures)


def test_gate_fails_on_the_mean_when_no_single_cell_crosses():
base, cand = _pair()
cand["eng-a|holdout"] = {k: v - 0.004 for k, v in cand["eng-a|holdout"].items()}
code, failures, _ = axes.gate(base, cand, 0.005)
assert code == 0
cand["eng-a|holdout"] = {k: v - 0.0045 for k, v in base["eng-a|holdout"].items()}
assert axes.gate(base, cand, 0.0044)[0] == 1
cand["eng-a|holdout"] = {k: v - 0.004 for k, v in base["eng-a|holdout"].items()}
code, failures, _ = axes.gate(base, cand, 0.0035)
assert code == 1 and any("mean of 19" in f for f in failures)


def test_gate_ignores_the_tuning_half():
base, cand = _pair()
cand["eng-a|tuning"] = {k: 0.0 for k in cand["eng-a|tuning"]}
assert axes.gate(base, cand, 0.005)[0] == 0


def test_mutation_the_gate_bites_every_one_of_the_nineteen_cells():
for label in axes.labels():
base, cand = _pair(**{label: 0.0})
code, failures, _ = axes.gate(base, cand, 0.005)
assert code == 1 and any(label in f for f in failures), label


def test_mutation_a_gate_without_the_comparison_would_pass_a_collapse():
# The same collapse must flip to a pass if the comparison is removed: proves the
# failing assertions above rest on the comparison and not on a side effect.
base, cand = _pair(**{"FinTabNet TEDS": 0.0})
assert axes.gate(base, cand, 0.005)[0] == 1
assert axes.gate(base, cand, 1.0)[0] == 0


def test_gate_skips_when_a_value_or_a_row_is_missing():
base, cand = _pair(**{"ParseBench chart": None})
code, failures, skips = axes.gate(base, cand, 0.005)
assert code == axes.SKIP_EXIT == 3 and not failures and any("ParseBench chart" in s for s in skips)
base, cand = _pair()
del base["eng-a|holdout"]
assert axes.gate(base, cand, 0.005)[0] == 3
assert axes.gate({}, {}, 0.005)[0] == 3


def test_a_drop_wins_over_a_skip():
base, cand = _pair(**{"ParseBench chart": None, "FinTabNet TEDS": 0.0})
assert axes.gate(base, cand, 0.005)[0] == 1


def _cli(*args):
return subprocess.run([sys.executable, "-m", "bench.axes", *args], cwd=ROOT,
capture_output=True, text=True)


def test_cli_exit_codes_and_skip_line(tmp_path):
base, cand = _pair()
b, c = tmp_path / "b.json", tmp_path / "c.json"
b.write_text(json.dumps(base))
c.write_text(json.dumps(cand))
assert _cli("gate", "--baseline", str(b), "--candidate", str(c)).returncode == 0
cand["eng-a|holdout"]["FinTabNet TEDS"] = 0.0
c.write_text(json.dumps(cand))
res = _cli("gate", "--baseline", str(b), "--candidate", str(c))
assert res.returncode == 1 and "FinTabNet TEDS" in res.stderr
cand["eng-a|holdout"]["FinTabNet TEDS"] = None
c.write_text(json.dumps(cand))
res = _cli("gate", "--baseline", str(b), "--candidate", str(c))
assert res.returncode == 3 and "SKIPPED (not a pass): " in res.stderr
assert _cli("gate", "--baseline", str(tmp_path / "none"), "--candidate", str(c)).returncode == 2


def test_the_committed_round_regenerates_byte_for_byte():
"""The 1.2.0 means file is reproducible from the run directories it names."""
results = ROOT / "results"
dirs = [results / "bench-n2-standard-8-20261021T085113Z", results / "bench-c3-standard-8-20261017T163121Z"]
target = results / "round-1.2.0-20261021" / "axis_means.json"
if not target.is_file() or not all((d / "run.json").is_file() for d in dirs):
skip("round 1.2.0 run directories or axis_means.json are not in this checkout")
assert axes.combine(dirs) == json.loads(target.read_text())
Loading