status ● {{ c.verdict|upper }}
@@ -205,7 +222,7 @@
{% for section, items in gaps.items() %}
{% for c in items %}
{{ section|lower if loop.first else "" }}{{ c.verdict|replace("_", " ") }}{{ c.claim }}{% if c.note %} · {{ c.note }}{% endif %}
-{% for d in claim_disclosures(c) if d.key in ("sources", "unjudged_refs", "claim_numbering") %}
+{% for d in claim_disclosures(c) if d.key in ("sources", "unjudged_refs", "claim_numbering", "claim_pairing") %}
{{ '⚠ ' if d.level == 'warn' else '' }}{{ d.short }}
{% endfor %}
{% for j in c.judgements %}
diff --git a/src/papertrace/wizard.py b/src/papertrace/wizard.py
index 874d342..28e89b7 100644
--- a/src/papertrace/wizard.py
+++ b/src/papertrace/wizard.py
@@ -25,9 +25,10 @@
from rich.prompt import Confirm, Prompt
from . import config
+from .ask import ASK_ATTEMPTS, claude_available
from .brand import BANNER
-from .check import ASK_ATTEMPTS, claude_available
from .ingest import _docling_available as docling_available
+from .ingest import resolve_backend
from .models import _LABEL_GROUP, _expand_label_group, is_references_heading
from .refs import DOI_RE
@@ -212,6 +213,13 @@ def workload(pdf: Path) -> dict:
# one extraction call, then one per cited source. Sources are judged in
# groups, so this is an upper bound on the judging calls, not a promise.
cited_source_calls = sum(len(g) for g in groups)
+ # `run_wizard` always requests `backend="auto"`, which `resolve_backend`
+ # turns into docling when it is importable and pymupdf otherwise. On a
+ # pymupdf run the flat second reading is skipped as identical to the first
+ # (`cli._needs_flat_reading`'s own predicate) — so the reference-list call
+ # is never attempted either, and the estimate must not promise a call this
+ # run's own backend will not produce.
+ llm_call = 1 if resolve_backend("auto") != "pymupdf" else 0
return {
"pages": pages,
"places": len(groups),
@@ -220,11 +228,22 @@ def workload(pdf: Path) -> dict:
# how many sources each citation place cites, in reading order — what
# `apply_limits` cuts to the first N places when the claims are capped
"group_sizes": [len(g) for g in groups],
- "model_calls": 1 + cited_source_calls,
- # the retry is real spend: one extraction plus, per judging call, up to
- # ASK_ATTEMPTS attempts. Derived from check.py rather than a local
- # multiplier, so the estimate cannot drift from the policy.
- "model_calls_max": 1 + ASK_ATTEMPTS * cited_source_calls,
+ # the reference-list reading, 0 or 1. Kept as its own key rather than
+ # only folded into the two totals because `apply_limits` has to add it
+ # back: a limit cuts claims and sources, and this call reads the
+ # bibliography, so no limit can remove it.
+ "reflist_calls": llm_call,
+ # one reference-list reading (when the backend leaves a second text to
+ # check it against), one extraction call, then one per cited source.
+ # The reference-list call gets no retry — `reflist.propose` asks once
+ # and reports what it got — so it adds exactly `llm_call` to the base
+ # figure and to the ceiling, on top of the retried figure below rather
+ # than inside it.
+ "model_calls": 1 + llm_call + cited_source_calls,
+ # the retry is real spend: ASK_ATTEMPTS attempts for extraction, and
+ # ASK_ATTEMPTS attempts per judging call. Derived from ask.py rather
+ # than a local multiplier, so the estimate cannot drift from the policy.
+ "model_calls_max": llm_call + ASK_ATTEMPTS * (1 + cited_source_calls),
"style_unrecognised": len(groups) == 0,
}
@@ -236,6 +255,11 @@ def apply_limits(w: dict, *, max_claims: int | None, max_sources: int | None) ->
most N sources obtained means at most N judging calls, one per document.
Neither figure may exceed the unlimited one, and the retry ceiling is
derived the way `workload` derives it, so the two cannot drift.
+
+ The reference-list reading survives every limit: it reads the bibliography,
+ not the claims, so `--max-claims` and `--max-sources` cannot remove it.
+ Dropping it here would advertise a ceiling the run then exceeds, which is
+ the one thing a cost estimate may never do.
"""
sizes = list(w.get("group_sizes") or [])
if max_claims is not None:
@@ -243,7 +267,14 @@ def apply_limits(w: dict, *, max_claims: int | None, max_sources: int | None) ->
judging = sum(sizes)
if max_sources is not None:
judging = min(judging, max_sources)
- return {**w, "model_calls": 1 + judging, "model_calls_max": 1 + ASK_ATTEMPTS * judging}
+ reflist = w.get("reflist_calls") or 0
+ return {
+ **w,
+ "model_calls": 1 + reflist + judging,
+ # same shape as `workload`: the reference-list call is not retried, so
+ # it adds to the ceiling rather than being multiplied into it
+ "model_calls_max": reflist + ASK_ATTEMPTS * (1 + judging),
+ }
def parse_limit(raw: str) -> int | None:
@@ -272,7 +303,7 @@ def supplement_workload(provided_dir: Path | None, supplement: list[Path]) -> in
Counted from the folder rather than from the manifest because this runs
before `refs` does. The alternative is to state a price that leaves the
- supplements out, and check.py's own comment on ASK_ATTEMPTS is the rule
+ supplements out, and ask.py's own comment on ASK_ATTEMPTS is the rule
here: a cost ceiling that gets exceeded is a false promise about money.
"""
from .refs import _SUPPLEMENT_RE
@@ -355,6 +386,47 @@ def _ask_paper() -> Path:
return path
+def _ask_case(paper: Path) -> Path:
+ """Where the audit is written — the one path answer that creates, not finds.
+
+ Routed through `clean_path` like every other path prompt. It was not, and a
+ quoted answer — what Finder's drag-and-drop produces for any path holding a
+ space — became a *relative* name starting with a literal quote, so the case
+ landed under the cwd while every printed line named an absolute folder that
+ did not exist. The other prompts were immune only because they check
+ `.exists()`; nothing can contradict a folder that is about to be created.
+
+ `default_case` already refused the working directory as a case location for
+ this reason ("appears wherever the user happened to be standing"), so a
+ relative answer here is worth a word rather than a silent accept: it is
+ resolved and the resolution is printed. A relative answer is legitimate —
+ `-c demo_case` is in the README — but it is also what a mangled path
+ degrades into, and the resolved line is the one place the difference shows
+ before any money is spent.
+ """
+ while True:
+ raw = Prompt.ask(
+ "\n[bold]Where should I keep this audit?[/bold]\n folder",
+ default=_suggest_case(paper),
+ )
+ case = clean_path(raw)
+ if case.is_absolute():
+ return case
+ if case == Path("."):
+ # `default_case` refuses the working directory outright, and an
+ # answer of whitespace arrives here as `.` — the one relative answer
+ # that names no folder of its own
+ console.print(
+ " [red]That is the folder you are standing in.[/red] An audit needs "
+ "its own,\n so it cannot be mistaken for the rest of the directory."
+ )
+ continue
+ # soft_wrap: a path broken across two lines is a path a reader skims
+ # past, which is the failure this print exists to prevent
+ console.print(f" [dim]relative — writing the audit to[/dim] {case.resolve()}", soft_wrap=True)
+ return case.resolve()
+
+
def _ask_sources(case: Path) -> Path | None:
"""The folder of reference PDFs the user already holds.
@@ -539,10 +611,7 @@ def _n(count: int, noun: str) -> str:
"'coverage not audited' rather than zero gaps.[/dim]"
)
- case = Path(
- Prompt.ask("\n[bold]Where should I keep this audit?[/bold]\n folder",
- default=_suggest_case(paper))
- )
+ case = _ask_case(paper)
provided = _ask_sources(case)
supplement = _ask_supplements()
doi, with_scout = _ask_doi(paper)
@@ -621,7 +690,8 @@ def _n(count: int, noun: str) -> str:
run_cmd(
manuscript=paper, case=case, provided=provided, email=email, model=None,
png=png, backend="auto", with_scout=with_scout, doi=doi, formats=formats,
- supplement=supplement, max_claims=max_claims, max_sources=max_sources,
+ supplement=supplement, llm_refs=True,
+ max_claims=max_claims, max_sources=max_sources,
)
diff --git a/tests/test_ask.py b/tests/test_ask.py
new file mode 100644
index 0000000..9a3e552
--- /dev/null
+++ b/tests/test_ask.py
@@ -0,0 +1,166 @@
+"""The seam, and the two things about it that are now mechanically checked.
+
+`_ask` is the only place this codebase shells out to a model. That used to be a
+convention stated in CLAUDE.md; since `refs` also needs a model reading of the
+bibliography, it is a rule with a test behind it.
+"""
+
+import json
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+
+import pytest # noqa: E402
+
+from papertrace import ask as ask_mod # noqa: E402
+from papertrace import check as check_mod # noqa: E402
+
+
+class _Completed:
+ def __init__(self, model: str | None):
+ self.returncode = 0
+ payload: dict = {"result": "ok"}
+ if model:
+ payload["model"] = model
+ self.stdout = json.dumps(payload)
+ self.stderr = ""
+
+
+# Every way a python file can start another process, not just the one `ask.py`
+# happens to use. Matching `subprocess.run(` alone would have let a future file
+# shell out via `Popen`, `check_output` or `os.system` and still pass a test
+# whose name promises it cannot. `subprocess.TimeoutExpired:` carries no
+# parenthesis, so the `\w+\(` is what keeps the `except` clause from matching.
+_SHELLS_OUT = re.compile(
+ r"subprocess\.\w+\(|\bPopen\(|os\.system\(|os\.popen\(|os\.exec\w*\(|os\.spawn\w*\("
+)
+
+
+def test_only_ask_py_shells_out_to_a_model():
+ """One file in `src/` may start a process, and this is the test that says so.
+
+ CLAUDE.md used to phrase this as "check.py is the only module that calls a
+ model", which was a convention with nothing enforcing it. `refs` needs a
+ reading of the bibliography from a model, so the rule moves to the seam —
+ where it can be checked.
+
+ A grep cannot prove a negative about execution — a file could reach a
+ process through a name this pattern does not know. What it does prove is
+ that no file in `src/` reaches one by any of the spellings a person would
+ actually write, which is enough to make an accident loud.
+ """
+ src = Path(__file__).resolve().parent.parent / "src" / "papertrace"
+ shelling = sorted(
+ p.relative_to(src).as_posix()
+ for p in src.rglob("*.py")
+ if _SHELLS_OUT.search(p.read_text())
+ )
+ assert shelling == ["ask.py"], shelling
+
+
+def test_a_retry_policy_of_zero_says_so_rather_than_quietly_making_one_call(monkeypatch):
+ """`ASK_ATTEMPTS` is the attempt budget, so zero forbids the call the budget
+ exists to bound. Clamping it to one would hide a nonsense setting behind a
+ working run; the old text claimed the branch was unreachable while standing
+ in it. It raises `RuntimeError`, the same type the seam raises for a failed
+ or timed-out call, so `check_claims`'s existing handler turns it into
+ `unchecked` with a note rather than something new to catch."""
+ called = []
+ monkeypatch.setattr(check_mod, "_ask", lambda p, model=None: called.append(1) or "")
+ monkeypatch.setattr(check_mod, "ASK_ATTEMPTS", 0)
+
+ with pytest.raises(RuntimeError, match="ASK_ATTEMPTS is 0"):
+ check_mod._ask_with_retry("prompt", None)
+ assert called == [], "no call may be made under a zero attempt budget"
+
+
+def test_two_call_sites_do_not_clobber_each_others_model(monkeypatch):
+ """The report's `Checker:` line names the model that judged the claims.
+
+ One global cannot carry that once two stages call the seam: a run whose
+ judging made zero calls would print the reference-list model as its judge.
+ """
+ models = iter(["claude-opus-5", "claude-haiku-4-5"])
+ monkeypatch.setattr(ask_mod.subprocess, "run",
+ lambda cmd, **kw: _Completed(next(models)))
+ monkeypatch.setattr(ask_mod, "_MODELS", {})
+
+ with ask_mod.for_site(ask_mod.SITE_REFS):
+ ask_mod._ask("read this bibliography")
+ with ask_mod.for_site(ask_mod.SITE_CHECK):
+ ask_mod._ask("judge this claim")
+
+ assert ask_mod.model_for(ask_mod.SITE_REFS) == "claude-opus-5"
+ assert ask_mod.model_for(ask_mod.SITE_CHECK) == "claude-haiku-4-5"
+
+
+def test_a_site_nobody_called_has_no_model_rather_than_a_plausible_one(monkeypatch):
+ """`model_for` on an unused site is None, not the other site's model.
+
+ A reference-list reading that never happened must not be able to name a
+ model, and a judging pass that never happened must not inherit one.
+ """
+ monkeypatch.setattr(ask_mod.subprocess, "run",
+ lambda cmd, **kw: _Completed("claude-opus-5"))
+ monkeypatch.setattr(ask_mod, "_MODELS", {})
+
+ with ask_mod.for_site(ask_mod.SITE_REFS):
+ ask_mod._ask("read this bibliography")
+
+ assert ask_mod.model_for(ask_mod.SITE_CHECK) is None
+
+
+def test_a_reply_that_names_no_model_does_not_erase_the_one_recorded(monkeypatch):
+ """`claude -p` does not always report which model answered.
+
+ A silent call is no evidence that the model changed, so the recorded name
+ stands. Overwriting it with None would make the report stop naming a judge
+ that did in fact judge.
+ """
+ replies = iter([_Completed("claude-opus-5"), _Completed(None)])
+ monkeypatch.setattr(ask_mod.subprocess, "run", lambda cmd, **kw: next(replies))
+ monkeypatch.setattr(ask_mod, "_MODELS", {})
+
+ with ask_mod.for_site(ask_mod.SITE_CHECK):
+ ask_mod._ask("judge one")
+ ask_mod._ask("judge two")
+
+ assert ask_mod.model_for(ask_mod.SITE_CHECK) == "claude-opus-5"
+
+
+def test_for_site_restores_the_previous_site_on_the_way_out(monkeypatch):
+ """Nesting must not leak. A leaked site is a misattributed model."""
+ monkeypatch.setattr(ask_mod, "_SITE", "outer")
+ with ask_mod.for_site(ask_mod.SITE_REFS):
+ assert ask_mod._SITE == ask_mod.SITE_REFS
+ assert ask_mod._SITE == "outer"
+
+
+def test_check_still_exposes_the_seam_its_tests_patch():
+ """Sixty-two monkeypatch sites across nine test modules do
+ `monkeypatch.setattr(check_mod, "_ask", ...)`, and one does it by the dotted
+ string `"papertrace.check._ask"`. Both need `_ask` to be an attribute of
+ `check`, and need `check`'s own call sites to use the bare name so the patch
+ is seen. If this assertion ever fails, those tests are not failing — they
+ are making live, paid model calls."""
+ assert check_mod._ask is ask_mod._ask
+
+
+def test_the_judging_retry_attempts_exactly_ask_attempts_times(monkeypatch):
+ """`ASK_ATTEMPTS` exists so the wizard's advertised worst-case bill cannot
+ drift from the real retry policy — `CHANGELOG.md` records the incident. The
+ retry loop did not read it: it hardcoded one retry, and matched the constant
+ only because the constant happens to be 2."""
+ calls = []
+
+ def boom(prompt, model=None):
+ calls.append(1)
+ raise RuntimeError("transient")
+
+ monkeypatch.setattr(check_mod, "_ask", boom)
+ monkeypatch.setattr(check_mod, "ASK_ATTEMPTS", 3)
+ with pytest.raises(RuntimeError):
+ check_mod._ask_with_retry("prompt", None)
+ assert len(calls) == 3, calls
diff --git a/tests/test_ask_boot_hygiene.py b/tests/test_ask_boot_hygiene.py
index 1622ac6..364c8b1 100644
--- a/tests/test_ask_boot_hygiene.py
+++ b/tests/test_ask_boot_hygiene.py
@@ -15,7 +15,7 @@
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
-from papertrace import check as check_mod # noqa: E402
+from papertrace import ask as ask_mod # noqa: E402
class _FakeCompleted:
@@ -33,9 +33,9 @@ def fake_run(cmd, **kwargs):
captured["kwargs"] = kwargs
return _FakeCompleted()
- monkeypatch.setattr(check_mod.subprocess, "run", fake_run)
+ monkeypatch.setattr(ask_mod.subprocess, "run", fake_run)
- check_mod._ask("judge this")
+ ask_mod._ask("judge this")
assert captured["kwargs"].get("cwd") is not None
assert captured["kwargs"]["cwd"] != "."
@@ -48,9 +48,9 @@ def fake_run(cmd, **kwargs):
captured["cmd"] = cmd
return _FakeCompleted()
- monkeypatch.setattr(check_mod.subprocess, "run", fake_run)
+ monkeypatch.setattr(ask_mod.subprocess, "run", fake_run)
- check_mod._ask("judge this")
+ ask_mod._ask("judge this")
cmd = captured["cmd"]
assert "--safe-mode" in cmd
@@ -68,10 +68,10 @@ def fake_run(cmd, **kwargs):
captured.append(kwargs.get("cwd"))
return _FakeCompleted()
- monkeypatch.setattr(check_mod.subprocess, "run", fake_run)
+ monkeypatch.setattr(ask_mod.subprocess, "run", fake_run)
- check_mod._ask("judge this")
- check_mod._ask("judge this again")
+ ask_mod._ask("judge this")
+ ask_mod._ask("judge this again")
assert all(cwd != tempfile.gettempdir() for cwd in captured)
assert all(Path(cwd).is_dir() for cwd in captured)
diff --git a/tests/test_coverage.py b/tests/test_coverage.py
index 8a614ba..ffd5af4 100644
--- a/tests/test_coverage.py
+++ b/tests/test_coverage.py
@@ -8,8 +8,8 @@
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "src"))
+from papertrace.ask import _parse_json_object # noqa: E402
from papertrace.check import ( # noqa: E402
- _parse_json_object,
citation_labels_in_text,
coverage_audit,
)
diff --git a/tests/test_disclosure_parity.py b/tests/test_disclosure_parity.py
index ef3221a..085b755 100644
--- a/tests/test_disclosure_parity.py
+++ b/tests/test_disclosure_parity.py
@@ -21,13 +21,16 @@
claim_disclosures,
run_disclosures,
)
-from papertrace.models import ClaimResult, RunResults # noqa: E402
+from papertrace.models import ClaimResult, RefManifest, RunResults # noqa: E402
from papertrace.report import write_reports # noqa: E402
# four, since the interactive viewer: a disclosure the browser page cannot
# render is a disclosure one reader in four never sees
FORMATS = ("report.md", "report_editor.html", "report_terminal.html", "report_viewer.html")
+TEMPLATES = Path(__file__).resolve().parent.parent / "src" / "papertrace" / "templates"
+JINJA_FORMATS = ("report.md.j2", "report_editor.html.j2", "report_terminal.html.j2")
+
def _render(results: RunResults, out: Path) -> dict[str, str]:
write_reports(results, None, out, png=False)
@@ -224,9 +227,10 @@ def test_the_terminal_template_names_every_disclosure_key_that_exists():
templates = Path(mod.__file__).parent / "templates"
terminal = (templates / "report_terminal.html.j2").read_text()
- claim_level = {"anchor", "sources", "unjudged_refs", "judgement_anchor"}
-
- missing = {k for k in keys - claim_level if f'"{k}"' not in terminal}
+ # claim-level keys are covered by
+ # test_every_claim_disclosure_key_is_rendered_by_every_jinja_format, which
+ # checks all three Jinja formats rather than only this one
+ missing = {k for k in keys - set(mod.CLAIM_KEYS) if f'"{k}"' not in terminal}
assert not missing, (
f"report_terminal.html.j2 renders no branch for {sorted(missing)} — "
"its filter is an allow-list, so a new disclosure is dropped, not surfaced"
@@ -401,6 +405,117 @@ def test_an_anchor_state_without_a_crop_still_reaches_every_format(tmp_path, anc
assert anchor.token in body, f"anchor token missing from {name}"
+def test_a_withheld_claim_pairing_disclosure_appears_in_every_format(tmp_path):
+ claim = _claim(refs=["9"], verdict="unchecked", withheld_refs=["9"])
+ rendered = _render(RunResults(manuscript="m.pdf", claims=[claim]), tmp_path)
+
+ pairing = next(d for d in claim_disclosures(claim) if d.key == "claim_pairing")
+ for name, body in rendered.items():
+ assert pairing.token in body, f"claim_pairing token missing from {name}"
+
+
+# --- Task 7 review, M7: the three remaining `claim_pairing` states, rendered --
+
+
+def test_a_model_resolved_claim_pairing_disclosure_appears_in_every_format(tmp_path):
+ """M7. `withheld` had a fixture; `llm_resolved`, `chosen` and
+ `corroborated` did not, so three new tokens reached four formats
+ unverified — coverage, not breakage (the key is in every allow-list
+ already), but the parity loop is what makes that provable per state."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(manuscript="m.pdf", labels_resolved=["7"], numbering_choice="llm_resolved")
+ claim = _claim(refs=["7"])
+ results = RunResults(manuscript="m.pdf", converter="pymupdf", claims=[claim])
+ write_reports(results, manifest, tmp_path, png=False)
+ rendered = {name: (tmp_path / name).read_text() for name in FORMATS}
+
+ d = next(x for x in claim_disclosures(claim, manifest) if x.key == "claim_pairing")
+ for name, body in rendered.items():
+ assert d.token in body, f"claim_pairing (llm_resolved) token missing from {name}"
+
+
+def test_a_user_chosen_claim_pairing_disclosure_appears_in_every_format(tmp_path):
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(manuscript="m.pdf", labels_resolved=["9"], numbering_choice="pymupdf")
+ claim = _claim(refs=["9"])
+ results = RunResults(manuscript="m.pdf", converter="pymupdf", claims=[claim])
+ write_reports(results, manifest, tmp_path, png=False)
+ rendered = {name: (tmp_path / name).read_text() for name in FORMATS}
+
+ d = next(x for x in claim_disclosures(claim, manifest) if x.key == "claim_pairing")
+ for name, body in rendered.items():
+ assert d.token in body, f"claim_pairing (chosen) token missing from {name}"
+
+
+def test_a_corroborated_claim_pairing_disclosure_appears_in_every_format(tmp_path):
+ """`info`-level, and only ever printed beside an existing `warn` — this
+ claim's co-citation is unretrieved (a real, unrelated finding), which is
+ what earns corroboration a place on the same row."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(
+ manuscript="m.pdf", numbering_corroborated=True,
+ corroborating_readings=["parsed", "pymupdf"],
+ )
+ claim = _claim(refs=["3", "4"], unjudged_refs=["4"])
+ results = RunResults(manuscript="m.pdf", converter="pymupdf", claims=[claim])
+ write_reports(results, manifest, tmp_path, png=False)
+ rendered = {name: (tmp_path / name).read_text() for name in FORMATS}
+
+ fired = claim_disclosures(claim, manifest)
+ d = next(x for x in fired if x.key == "claim_pairing" and x.level == "info")
+ for name, body in rendered.items():
+ assert d.token in body, f"claim_pairing (corroborated) token missing from {name}"
+
+
+def test_a_disputed_but_unretrieved_claim_pairing_appears_in_every_format(tmp_path):
+ """The fifth `claim_pairing` state (whole-branch review, Major 5): the
+ readings did not agree AND nothing was ever fetched under the label, so
+ the withheld token's "the source retrieved under that label" is the one
+ thing that did not happen. New token, same key — the allow-lists already
+ carry it, and this is what makes the new state's parity provable."""
+ from papertrace.models import RefEntry, RefManifest
+
+ manifest = RefManifest(
+ manuscript="m.pdf",
+ entries=[RefEntry(num="6", raw="ref 6", status="paywalled")],
+ labels_disputed=["6"],
+ )
+ claim = _claim(refs=["6"], verdict="not_retrieved")
+ results = RunResults(manuscript="m.pdf", converter="pymupdf", claims=[claim])
+ write_reports(results, manifest, tmp_path, png=False)
+ rendered = {name: (tmp_path / name).read_text() for name in FORMATS}
+
+ d = next(x for x in claim_disclosures(claim, manifest) if x.key == "claim_pairing")
+ for name, body in rendered.items():
+ assert d.token in body, f"claim_pairing (unretrieved) token missing from {name}"
+
+
+def test_a_resolution_only_reflist_disclosure_appears_in_every_format(tmp_path):
+ """Major 4's new shape: no reading of the whole list was taken, but a
+ model was asked to settle a disagreement. The `reflist` key already has a
+ block in every format; this is the state that had no fixture."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(
+ manuscript="m.pdf",
+ labels_resolved=["7"],
+ numbering_choice="llm_resolved",
+ resolution_outcome="read",
+ resolution_model="claude-opus-5",
+ resolution_readings=["parsed"],
+ )
+ results = RunResults(manuscript="m.pdf", converter="pymupdf", claims=[_claim(refs=["7"])])
+ write_reports(results, manifest, tmp_path, png=False)
+
+ d = next(x for x in run_disclosures(results, manifest) if x.key == "reflist")
+ for name in FORMATS:
+ body = (tmp_path / name).read_text()
+ assert d.token in body, f"reflist (resolution only) token missing from {name}"
+
+
# --- the HTML reports actually escape what they interpolate ------------------
@@ -423,3 +538,126 @@ def test_markup_in_source_text_cannot_reach_the_html_reports_unescaped(tmp_path)
# markdown is not HTML and must not grow entities — it stays verbatim
assert hostile in rendered["report.md"]
+
+
+def test_claim_keys_names_only_keys_a_producer_actually_emits():
+ """A key in the set that no producer emits is dead weight that silently
+ widens the exemption below. `judgement_anchor` was exactly that."""
+ import re
+
+ from papertrace import disclosures as mod
+
+ src = Path(mod.__file__).read_text()
+ real = set(re.findall(r'key="([a-z_]+)"', src))
+ assert mod.CLAIM_KEYS <= real, f"not emitted by any producer: {sorted(mod.CLAIM_KEYS - real)}"
+
+
+def test_claim_disclosures_emits_no_key_missing_from_claim_keys():
+ """The direction the test above does NOT check: a key `claim_disclosures()`
+ emits at runtime but `CLAIM_KEYS` omits would still reach the viewer (its
+ catch-all) while silently vanishing from the other three formats, and
+ nothing here would go red. Exercises every branch `claim_disclosures()` can
+ take today, including `claim_pairing`, so this would have caught it being
+ added to the producer without being added to the set."""
+ from papertrace.models import SourceJudgement
+
+ manifest = RefManifest(manuscript="m.pdf", entries=[], unverified_from=1)
+ claims = [
+ _claim(refs=["7", "9"], verdict="contradicted",
+ judgements=[SourceJudgement(source_slug="a", ref="7", verdict="contradicted"),
+ SourceJudgement(source_slug="b", ref="9", verdict="supported")]),
+ _claim(unjudged_refs=["11"]),
+ _claim(quote="", verdict="supported"),
+ _claim(verdict="supported",
+ judgements=[SourceJudgement(source_slug="s", ref="7", kind="supplement",
+ verified=True)]),
+ _claim(refs=["1"], verdict="supported"),
+ _claim(refs=["9"], verdict="unchecked", withheld_refs=["9"]),
+ _claim(evidence_image=None, anchor_located=False, source_page=3, verdict="supported"),
+ ]
+ seen = set()
+ for c in claims:
+ for d in claim_disclosures(c, manifest=manifest):
+ seen.add(d.key)
+ from papertrace import disclosures as mod
+ assert seen <= mod.CLAIM_KEYS, f"emitted but not declared: {sorted(seen - mod.CLAIM_KEYS)}"
+
+
+def test_every_claim_disclosure_key_is_rendered_by_every_jinja_format():
+ """The viewer renders claim disclosures generically; the other three filter
+ by explicit key. So a new claim-level disclosure reaches one reader in four
+ and vanishes for the rest — the `source_identity` failure one layer down,
+ and the layer with no guard on it.
+
+ The six keys that exist today all pass. The point is that the seventh
+ cannot be added without this going red first.
+ """
+ from papertrace import disclosures as mod
+
+ for name in JINJA_FORMATS:
+ src = (TEMPLATES / name).read_text()
+ missing = {k for k in mod.CLAIM_KEYS if f'"{k}"' not in src}
+ assert not missing, (
+ f"{name} renders no branch for {sorted(missing)} — it filters claim "
+ "disclosures by explicit key, so a new one is dropped, not surfaced"
+ )
+
+
+# --- Task 7 review, M6: the escalation's run-level surface ------------------
+
+
+def test_a_resolved_run_gets_a_run_level_disclosure_in_every_format(tmp_path):
+ """M6. Without this, an 11-of-14-resolved run said so only on the claims
+ that cite one of the 11 — a reader who stops at the top of the report
+ never learns that some verdicts below rest on a model reading or a
+ person's choice."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(manuscript="m.pdf", labels_resolved=["6", "7"],
+ numbering_choice="llm_resolved")
+ results = RunResults(manuscript="m.pdf", converter="pymupdf", claims=[_claim()])
+ write_reports(results, manifest, tmp_path, png=False)
+ rendered = {name: (tmp_path / name).read_text() for name in FORMATS}
+
+ d = next(x for x in run_disclosures(results, manifest) if x.key == "numbering_resolution")
+ for name, body in rendered.items():
+ assert d.token in body, f"numbering_resolution token missing from {name}"
+
+
+def test_an_ordinary_run_with_nothing_resolved_adds_no_disclosure(tmp_path):
+ """A run where nothing was ever disputed must not grow a disclosure about
+ a resolution that never happened."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(manuscript="m.pdf")
+ fired = run_disclosures(
+ RunResults(manuscript="m.pdf", converter="pymupdf", claims=[_claim()]), manifest
+ )
+ assert not any(d.key == "numbering_resolution" for d in fired)
+
+
+def test_a_withheld_run_adds_no_resolution_disclosure(tmp_path):
+ """`choice == "withheld"` settled nothing — `_labels_disputed` already
+ covers this case, and `_numbering_resolution` must stay silent about it
+ rather than claim a resolution that did not happen."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(manuscript="m.pdf", labels_disputed=["9"],
+ numbering_choice="withheld", numbering_chosen_by="user")
+ fired = run_disclosures(
+ RunResults(manuscript="m.pdf", converter="pymupdf", claims=[_claim()]), manifest
+ )
+ assert not any(d.key == "numbering_resolution" for d in fired)
+
+
+def test_a_claim_citing_both_a_disputed_and_a_resolved_label_mentions_both(tmp_path):
+ """m6. A claim citing one still-disputed label and one resolved label
+ used to get only the withheld text — a reader stopping there would not
+ learn the other citation's verdict rests on a resolution at all."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(manuscript="m.pdf", labels_disputed=["9"], labels_resolved=["7"])
+ claim = _claim(refs=["7", "9"], withheld_refs=["9"])
+ d = next(x for x in claim_disclosures(claim, manifest) if x.key == "claim_pairing")
+ assert "[9]" in d.text
+ assert "[7]" in d.text, "the resolved co-citation must not vanish from the note"
diff --git a/tests/test_escalation.py b/tests/test_escalation.py
new file mode 100644
index 0000000..ae16c3d
--- /dev/null
+++ b/tests/test_escalation.py
@@ -0,0 +1,913 @@
+"""The disagreement is shown before the question is asked.
+
+A reader who is offered a choice between two readings of a bibliography and has
+not been shown either one is being asked to rubber-stamp, not to decide. So
+step one writes the whole disagreement to a file and prints the path, and it is
+unskippable — `test_the_disagreement_is_written_before_the_question_is_asked`
+is that requirement encoded, and it fails if the two are ever reordered.
+
+Driven through `_refs_pipeline` rather than through `_escalate_disputed`
+directly for most of these: the thing under test is that a REAL run reaches the
+file before it reaches the prompt, and a unit test of the escalation alone
+cannot see the ordering the user asked for. `_reference_readings` is real and
+unmocked — it already returns a single dict, matching every other consumer of
+it (`tests/test_reference_readings.py`), so the two extra readings are made to
+disagree by forcing `_needs_flat_reading` True (the same technique that
+module's own `_wire_offline` uses) and by fabricating the flat-text extraction,
+never by pretending the pure dict-builder returns something it does not.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+import pymupdf
+import pytest
+import typer
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+
+from papertrace import cli # noqa: E402
+from papertrace import reflist as reflist_mod # noqa: E402
+from papertrace.models import RefManifest # noqa: E402
+from papertrace.refs import _entry # noqa: E402
+
+# --- the interactive escalation: shown first, then offered ------------------
+
+
+def _paper(path: Path) -> Path:
+ """A one-page PDF citing [1] and [2], both printed in the References section.
+
+ The disagreement at [2] is injected by fabricating the flat-text reading
+ below — the point of a real PDF is that `_refs_pipeline` runs its whole
+ length, ingest included, rather than being handed a fake `SourceMap`.
+ """
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Agreement in CT of the Pancreas", fontsize=16)
+ page.insert_text((72, 140), "Body text citing [1] and [2] here.", fontsize=11)
+ page.insert_text((72, 200), "References", fontsize=14)
+ page.insert_text((72, 230), "[1] Alpha A. First paper. 2020.", fontsize=11)
+ page.insert_text((72, 250), "[2] Beta B. Second paper. 2021.", fontsize=11)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ doc.save(path)
+ doc.close()
+ return path
+
+
+# The fabricated second reading of the same bibliography: agrees at [1], and
+# names a different paper at [2]. `references_span_flat` is faked to return
+# this rather than left real, because on a real pymupdf-backend ingest it is
+# "identical to the existing parse on a pymupdf one" (its own docstring) — the
+# self-corroboration case `_needs_flat_reading` exists to skip. Forcing the
+# predicate without also controlling the text it feeds would make "parsed" and
+# "pymupdf" agree on everything, and there would be no disagreement to show.
+FLAT_TEXT = (
+ "References\n"
+ "[1] Alpha A. First paper. 2020.\n"
+ "[2] Gamma G. A wholly different paper. 2019.\n"
+)
+
+
+@pytest.fixture()
+def disputed(tmp_path, monkeypatch):
+ """A run that reaches the escalation with [2] disputed, and nothing else stubbed.
+
+ Returns `(pdf, case, answers)`; the caller fills `answers` with the menu
+ keystrokes it wants `Prompt.ask` to return.
+ """
+ import papertrace.ingest as ingest_pkg
+ import papertrace.refs as refs_mod
+ from papertrace.refs import CROSSREF_NO_DOI, CrossrefDeposit
+
+ monkeypatch.setattr(refs_mod, "resolve_all",
+ lambda entries, dest, email, provided_dir=None, progress=None,
+ taken=None: entries)
+ monkeypatch.setattr(refs_mod, "crossref_deposit",
+ lambda client, doi, email: CrossrefDeposit(absent=CROSSREF_NO_DOI))
+ monkeypatch.setattr(cli, "_detected_doi", lambda m: None)
+ # forces the predicate, not the backend — same technique
+ # `tests/test_reference_readings.py`'s `_wire_offline` uses, so a pymupdf
+ # ingest (fast, no docling download) still gets a second reading to compare
+ monkeypatch.setattr(cli, "_needs_flat_reading", lambda smap: True)
+ monkeypatch.setattr(ingest_pkg, "references_span_flat",
+ lambda pdf_path: (FLAT_TEXT, False))
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+
+ answers: list[str] = []
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: answers.pop(0)))
+ return _paper(tmp_path / "paper.pdf"), tmp_path / "case", answers
+
+
+def _run_refs(pdf: Path, case: Path) -> RefManifest:
+ # llm_refs=False: an offline test must not risk a live `claude -p` call even
+ # on a machine that happens to have the CLI on PATH — the brief's own draft
+ # of this fixture omitted it, which would have made every test below spend
+ # money (or hang) on a host where `claude` is installed.
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None,
+ email="t@example.org", parse_only=False, backend="pymupdf",
+ doi=None, supplement=None, llm_refs=False)
+ return RefManifest.from_json(case / "refs_manifest.json")
+
+
+def test_the_disagreement_is_written_before_the_question_is_asked(disputed, monkeypatch):
+ """The user's instruction, encoded as a test: show, then offer.
+
+ The fake `Prompt.ask` asserts the file is already on disk when it is
+ called. That is the only way to pin the ordering — a test that checks the
+ file afterwards passes just as well on an implementation that asks first.
+ """
+ pdf, case, answers = disputed
+ path = case / "out" / "reference_disagreement.md"
+ seen = {}
+
+ def ask(*a, **k):
+ seen["existed"] = path.exists()
+ seen["text"] = path.read_text(encoding="utf-8") if path.exists() else ""
+ return "2"
+
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(ask))
+ _run_refs(pdf, case)
+
+ assert seen["existed"] is True, "the menu was reached before the evidence was written"
+ # both readings of the disputed label, and the text each was read from
+ assert "Beta B. Second paper" in seen["text"]
+ assert "Gamma G. A wholly different paper" in seen["text"]
+ assert "[2]" in seen["text"]
+ # and not a word about the label the readings agree on being a finding —
+ # its own printed-list context is fine, its structured fields are not shown
+ assert "First paper" in seen["text"], "the agreed label's text is context, not a finding"
+
+
+def test_the_path_to_the_disagreement_file_is_printed(disputed, monkeypatch, capsys):
+ """A file nobody is told about is a file nobody reads, and the menu's first
+ option spends money on a question the reader could answer by looking.
+
+ A wide console is forced (m10 of the Task 7 review): the absolute
+ `tmp_path` printed here is one unbroken "word" to rich's wrapper, and at
+ the ~80-column default it can hard-break mid-filename rather than at a
+ space, which would fail this assertion on any tmp_path long enough — this
+ one happened to pass only by being short.
+ """
+ from rich.console import Console
+
+ monkeypatch.setattr(cli, "console", Console(width=1000))
+ pdf, case, answers = disputed
+ answers.append("2")
+ _run_refs(pdf, case)
+ assert "reference_disagreement.md" in capsys.readouterr().out
+
+
+def test_the_default_answer_asks_the_model_and_records_the_user_as_choosing(
+ disputed, monkeypatch
+):
+ """Option 1, taken by pressing return. `chosen_by` is `user` even on the
+ default: the person was shown the disagreement and consented to proceed,
+ which is an input to record, not evidence of anything. The resolution call
+ itself is mocked at `reflist.resolve_disputed` — a live model call has no
+ place in an offline test."""
+ pdf, case, answers = disputed
+ answers.append("1")
+ resolved_entry = _entry("2", "Beta B. Second paper. 2021. doi:10.1000/beta")
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, cands, texts, disputing=None, model=None: reflist_mod.Resolution(
+ resolved={"2": resolved_entry}, still_disputed=[],
+ provenance=reflist_mod.ReflistProvenance(model="claude-opus-5", outcome="read"),
+ ),
+ )
+ manifest = _run_refs(pdf, case)
+ assert manifest.numbering_choice == "llm_resolved"
+ assert manifest.numbering_chosen_by == "user"
+ # True, and correctly so: `numbering_verified` means the chosen list
+ # accounts for exactly the labels the body cites — a question of EXTENT,
+ # which this fixture's parse does answer. A disputed label is a question
+ # of CONTENT. No branch of `_escalate_disputed` assigns `rec.verified`
+ # (grep it) — option 1 only swaps individual entries within the same
+ # list, never its length or numbering, so the extent claim this fixture
+ # already earned stands. `test_a_resolved_label_judges_against_the_resolved_entry`
+ # below pins the direction actually feared (a resolution must not flip
+ # this from False to True); Task 8 is the one that closes the remaining
+ # menu choices over more fixtures.
+ assert manifest.numbering_verified is True
+ assert manifest.labels_resolved == ["2"]
+ assert manifest.labels_disputed == []
+ # option 1 substitutes the resolved entry — the label will be judged
+ # against the paper the resolution named, not the one it ruled against
+ two = next(e for e in manifest.entries if e.num == "2")
+ assert two.doi == "10.1000/beta"
+ assert two.slug, "C1: a slug-less entry is silently dropped by check.py"
+
+
+def test_a_real_resolution_call_produces_a_usable_entry_end_to_end(disputed, monkeypatch):
+ """C1, through the REAL `resolve_disputed` — not a stub.
+
+ Every other option-1 test in this file replaces `resolve_disputed` with a
+ lambda whose entry is built through `refs._entry`, which sets the one
+ field the real function forgot (`slug`). A test double built from a
+ production helper can supply an invariant the code under test omits, and
+ then hide exactly the bug it looks like it is testing — that is how a
+ slug-less resolved entry (silently dropped by `check.py`, downloaded to
+ `sources_resolved/None.pdf`) survived review. This mocks only the seam
+ `resolve_disputed` itself calls (`ask._ask`), so the real field
+ verification and `refs._entry`-based construction actually run.
+ """
+ import papertrace.ask as ask_mod
+
+ pdf, case, answers = disputed
+ answers.append("1")
+ # "Second paper" must be found verbatim in one of the two texts shown to
+ # the call — it is, in `refs_text`, printed as "[2] Beta B. Second paper.
+ # 2021." — so this is a genuine verbatim-verified answer, not a stub.
+ reply = json.dumps([{"num": "2", "title": "Second paper", "year": "2021"}])
+ monkeypatch.setattr(ask_mod, "_ask", lambda prompt, model=None: reply)
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.numbering_choice == "llm_resolved"
+ assert manifest.labels_resolved == ["2"]
+ two = next(e for e in manifest.entries if e.num == "2")
+ assert two.title == "Second paper"
+ assert two.slug, "the real resolve_disputed must produce a usable slug (C1)"
+
+
+def test_a_resolver_failure_leaves_every_label_disputed_and_is_reported(disputed, monkeypatch):
+ """The resolution call can raise — `claude` not on PATH, a timeout, a
+ non-zero exit — the same shape `_llm_reference_reading` already has to
+ handle. `refs` has done useful work by this point and must not go down
+ with a failed subprocess call: every disputed label stays disputed and
+ withheld, and `chosen_by` is still "user" because a person did consent to
+ trying, even though the attempt produced nothing."""
+ pdf, case, answers = disputed
+ answers.append("1")
+
+ def boom(*a, **k):
+ raise RuntimeError("claude -p timed out after 600s")
+
+ monkeypatch.setattr(reflist_mod, "resolve_disputed", boom)
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.numbering_choice == "withheld"
+ assert manifest.numbering_chosen_by == "user"
+ assert manifest.labels_disputed == ["2"]
+ assert manifest.labels_resolved == []
+ # True, and correctly so: `numbering_verified` is an EXTENT claim about
+ # this fixture's parse, unaffected by a resolution attempt that raised
+ # before touching anything. No branch of `_escalate_disputed` assigns
+ # `rec.verified` (grep it); Task 8's job is to close this over every
+ # remaining menu choice with its own fixtures.
+ assert manifest.numbering_verified is True
+
+
+def test_withholding_all_of_them_is_the_second_option(disputed):
+ pdf, case, answers = disputed
+ answers.append("2")
+ manifest = _run_refs(pdf, case)
+ assert manifest.numbering_choice == "withheld"
+ assert manifest.numbering_chosen_by == "user"
+ assert manifest.labels_disputed == ["2"]
+ assert manifest.labels_resolved == []
+
+
+def test_choosing_one_reading_whole_records_which_one(disputed):
+ """Option 3. Both offered readings are deterministic parses, so this is a
+ choice between two things the tool produced — never the model's list, which
+ is a voter only and can reach neither `reconcile` nor `resolve_all`."""
+ pdf, case, answers = disputed
+ answers += ["3", "pymupdf"]
+ manifest = _run_refs(pdf, case)
+ assert manifest.numbering_choice == "pymupdf"
+ assert manifest.numbering_chosen_by == "user"
+ # False, correctly: `reconcile` measured EXTENT against the "parsed"
+ # reading, not the "pymupdf" one just adopted in its place. Carrying the
+ # old True forward would print "numbering confirmed" about a list nobody
+ # measured (Task 7 review, M1) — cleared here, never widened to a second
+ # meaning, which is why it is False rather than some other value.
+ assert manifest.numbering_verified is False
+ assert manifest.numbering_note.startswith("you chose")
+ # M1's other half: `reference_source` must name the reading actually
+ # adopted, not the one `reconcile` measured before it was discarded
+ assert manifest.reference_source == "pymupdf"
+ assert manifest.numbering_ledger == {}
+ # the entries carried forward are that reading's
+ assert [e.num for e in manifest.entries] == ["1", "2"]
+ gamma = next(e for e in manifest.entries if e.num == "2")
+ assert "Gamma" in gamma.raw
+
+
+def test_aborting_writes_no_manifest(disputed):
+ """Option 4 leaves the case as it was. A half-written manifest describing a
+ numbering the user walked away from is worse than none."""
+ pdf, case, answers = disputed
+ answers.append("4")
+ with pytest.raises(typer.Exit):
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None,
+ email="t@example.org", parse_only=False, backend="pymupdf",
+ doi=None, supplement=None, llm_refs=False)
+ assert not (case / "refs_manifest.json").exists()
+
+
+def test_a_non_interactive_session_withholds_the_disputed_labels_without_asking(
+ disputed, monkeypatch
+):
+ """The user's decision: suppress only the disputed labels.
+
+ Not the whole audit — that throws away a useful audit over a numbering the
+ reader can check by hand — and not a reading picked on their behalf, which
+ would record a choice nobody made.
+ """
+ pdf, case, answers = disputed
+ monkeypatch.setattr(cli, "_interactive", lambda: False)
+
+ def boom(*a, **k):
+ raise AssertionError("a non-interactive run must not prompt")
+
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(boom))
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.numbering_choice == "withheld"
+ assert manifest.numbering_chosen_by == "default"
+ assert manifest.labels_disputed == ["2"]
+ # the agreed label keeps its verdicts — it is the disputed one that is dropped
+ assert "1" not in manifest.labels_disputed
+
+
+def test_a_closed_stdin_is_not_interactive(monkeypatch):
+ monkeypatch.setattr(sys.stdin, "isatty", lambda: False, raising=False)
+ monkeypatch.setattr(sys.stdout, "isatty", lambda: True, raising=False)
+ monkeypatch.delenv("CI", raising=False)
+ assert cli._interactive() is False
+
+
+def test_ci_is_not_interactive_even_on_a_tty(monkeypatch):
+ """CI runs on a tty often enough that the tty test alone is not the gate, and
+ a prompt in CI is a hung build, not a question."""
+ monkeypatch.setattr(sys.stdin, "isatty", lambda: True, raising=False)
+ monkeypatch.setattr(sys.stdout, "isatty", lambda: True, raising=False)
+ monkeypatch.setenv("CI", "true")
+ assert cli._interactive() is False
+
+
+def test_a_tty_with_no_ci_flag_is_interactive(monkeypatch):
+ monkeypatch.setattr(sys.stdin, "isatty", lambda: True, raising=False)
+ monkeypatch.setattr(sys.stdout, "isatty", lambda: True, raising=False)
+ monkeypatch.delenv("CI", raising=False)
+ assert cli._interactive() is True
+
+
+# --- `_escalate_disputed` directly: the one substitution rule -------------
+
+
+def _rec(**kw):
+ from papertrace.refs import Reconciliation
+
+ return Reconciliation(**kw)
+
+
+def test_a_resolved_label_judges_against_the_resolved_entry(tmp_path, monkeypatch):
+ """Un-disputing a label without substituting its entry is the original bug.
+
+ Menu option 1 takes a label out of `labels_disputed`, so `check` judges it
+ again. If the chosen list still holds the entry the resolution ruled
+ against, the verdict is printed against the very paper the model said was
+ wrong — and the report says the user accepted it. The resolved entries exist
+ in `Resolution.resolved`; the failure mode is computing them and throwing
+ them away.
+ """
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ resolved = _entry("13", "Weston AD (2019) The paper the label really names. "
+ "https://doi.org/10.1148/x13")
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, cands, texts, disputing=None, model=None: reflist_mod.Resolution(
+ resolved={"13": resolved}, still_disputed=[],
+ provenance=reflist_mod.ReflistProvenance(model="claude-opus-5", outcome="read"),
+ ),
+ )
+
+ rec = _rec(labels_disputed=["13"])
+ wrong = _entry("13", "Somebody Else (1999) A different paper entirely. "
+ "https://doi.org/10.1148/x99")
+ out = cli._escalate_disputed(
+ tmp_path, rec, [wrong], {"parsed": [wrong]}, {"parsed": "text"},
+ )
+
+ assert rec.labels_disputed == [] # the label will be judged again
+ assert out[0].doi == "10.1148/x13" # against the RESOLVED paper
+ assert rec.chosen_by == "user" # which is what authorises it
+ assert rec.verified is False # and it is still not confirmed
+
+
+def test_zero_disputed_labels_never_prompts(tmp_path, monkeypatch):
+ """An escalation with nothing to escalate is a no-op — even in a real
+ terminal, nobody should be asked a question about an empty list."""
+
+ def boom(*a, **k):
+ raise AssertionError("nothing was in dispute — there was no question to ask")
+
+ monkeypatch.setattr(cli, "_interactive", boom)
+ rec = _rec(labels_disputed=[])
+ entries = [_entry("1", "Alpha A. 2020.")]
+ out = cli._escalate_disputed(tmp_path, rec, entries, {"parsed": entries}, {"parsed": "text"})
+ assert out is entries
+ assert rec.choice == ""
+ assert rec.chosen_by == ""
+
+
+def test_a_run_never_says_confirmed_beside_a_withheld_label(disputed, capsys):
+ """The contradiction a partial Task 7 surfaced, pinned at the surface.
+
+ `numbering_verified` means the chosen list accounts for exactly the labels
+ the body cites — EXTENT. A disputed label is a question of CONTENT, which no
+ count answers, so both can be true at once and the flag keeps its published
+ meaning. What must not happen is one line saying the numbering is confirmed
+ while the line above it withholds verdicts on a label: whichever of the two
+ a reader believed, the other would be a lie.
+
+ The first attempt at this was to un-verify the numbering, which broke two of
+ Plan A's merged tests — correctly, because it silently widened a boolean
+ that four report formats already read. The fix belongs in the sentence.
+ """
+ pdf, case, answers = disputed
+ answers.append("2")
+ manifest = _run_refs(pdf, case)
+ out = capsys.readouterr().out
+
+ assert manifest.numbering_verified is True
+ assert manifest.labels_disputed == ["2"]
+ # "do not agree", not "disagree": `disputed` also covers "nothing in the
+ # readings could be compared", and the console may not assert the
+ # contradicting cause for both — see `refs._label_state`
+ assert "the readings do not agree at [2]" in out
+ assert "numbering confirmed" not in out, "confirmed printed beside a withheld label"
+ assert "accounts for every cited label" in out
+
+
+# --- Task 7 review, Majors 8 and 9: a reading cannot launder its own coin-flip
+
+
+def test_option_3_excludes_a_reading_that_duplicates_the_disputed_label(tmp_path, monkeypatch):
+ """M8. `label_agreement` marks a label `disputed` outright when a single
+ reading carries it twice — "downgrading that to `single` ... would let a
+ verdict rest on the coin-flip" (`refs.py`). Offering that same reading in
+ option 3's menu would let a person adopt it whole anyway, laundering the
+ coin-flip into `labels_resolved`. It must not be offered at all."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ calls = []
+
+ def ask(prompt_text, **kw):
+ calls.append((prompt_text, kw.get("choices")))
+ return {" choice": "3", " which reading": "pymupdf"}.get(prompt_text, "3")
+
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(ask))
+
+ dup_a = _entry("2", "Alpha A. First. 2020.")
+ dup_b = _entry("2", "Alpha A. Different. 2021.")
+ single = _entry("2", "Gamma G. A different paper. 2019.")
+ rec = _rec(labels_disputed=["2"])
+ out = cli._escalate_disputed(
+ tmp_path, rec, [dup_a],
+ {"parsed": [dup_a, dup_b], "pymupdf": [single]},
+ {"parsed": "text a", "pymupdf": "text b"},
+ )
+
+ top_menu = next(c for c in calls if c[0] == " choice")
+ assert "3" in top_menu[1], "pymupdf alone still qualifies, so option 3 must be offered"
+ which_call = next(c for c in calls if c[0] == " which reading")
+ assert which_call[1] == ["pymupdf"], "the duplicating reading must not be offered"
+ assert rec.choice == "pymupdf"
+ assert out == [single]
+
+
+def test_option_3_is_never_offered_when_nothing_qualifies(tmp_path, monkeypatch):
+ """M9. `offered[0]` used to raise `IndexError` once neither `parsed` nor
+ `pymupdf` qualified — reachable whenever the run's own parse duplicates
+ the disputed label (this fixture) or is simply absent (a crossref+llm-only
+ dispute). The fix is at the menu: option 3 is never offered, so `choices`
+ passed to the real `Prompt.ask` never contains it. This pins the
+ degradation for a caller that ignores `choices` anyway (a misbehaving
+ test double, never a real terminal) — it falls through to the
+ model-resolution path rather than crashing.
+ """
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ calls = []
+
+ def ask(prompt_text, **kw):
+ calls.append((prompt_text, kw.get("choices")))
+ return "3"
+
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(ask))
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, names, texts, disputing=None, model=None: reflist_mod.Resolution(
+ still_disputed=list(labels),
+ # a double must satisfy the SHAPE its consumer reads: a Resolution
+ # that came back from a real call records `outcome="read"`, and
+ # `_escalate_disputed` now branches on it
+ provenance=reflist_mod.ReflistProvenance(outcome="read"),
+ ),
+ )
+ dup_a = _entry("2", "Alpha A. First. 2020.")
+ dup_b = _entry("2", "Alpha A. Different. 2021.")
+ rec = _rec(labels_disputed=["2"])
+ out = cli._escalate_disputed(
+ tmp_path, rec, [dup_a], {"parsed": [dup_a, dup_b]}, {"parsed": "text"},
+ )
+
+ top_menu = next(c for c in calls if c[0] == " choice")
+ assert "3" not in top_menu[1]
+ assert not any(c[0] == " which reading" for c in calls), "never reached — nothing offered"
+ assert rec.choice == "llm_resolved"
+ assert out == [dup_a]
+
+
+# --- Task 7 review, m5 and M3: an orphaned "resolved" label, and honest reasons
+
+
+def test_a_resolved_label_absent_from_entries_stays_disputed(tmp_path, monkeypatch):
+ """m5. `resolve_disputed` only ever answers about labels it was asked
+ about, but those labels come from `rec.labels_disputed` — computed over
+ ALL candidate readings (crossref, llm included), not just the chosen
+ `entries` list. A label the chosen list never carried at all has nothing
+ to substitute, and must not be recorded resolved with no entry behind it."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ orphan = _entry("99", "Nobody In The Chosen List. 2020.")
+ real = _entry("2", "Beta B. The real one resolved. 2021.")
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, names, texts, disputing=None, model=None: reflist_mod.Resolution(
+ resolved={"2": real, "99": orphan}, still_disputed=[],
+ provenance=reflist_mod.ReflistProvenance(model="claude-opus-5", outcome="read"),
+ ),
+ )
+ chosen = _entry("2", "Beta B. Original. 2021.")
+ rec = _rec(labels_disputed=["2", "99"])
+ out = cli._escalate_disputed(
+ tmp_path, rec, [chosen], {"parsed": [chosen]}, {"parsed": "text"},
+ )
+
+ assert rec.labels_resolved == ["2"], "99 has no entry to substitute"
+ assert rec.labels_disputed == ["99"], "orphaned, not silently dropped"
+ assert out == [real]
+
+
+def test_the_console_names_the_real_reason_a_label_stayed_disputed(tmp_path, monkeypatch, capsys):
+ """M3. A malformed reply, an unprinted title and a genuine "cannot tell"
+ are three different findings `resolve_disputed` already records in
+ `provenance` — asserting one blanket "the model could not tell" over all
+ three is a plausible-looking cause standing in for the honest one."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ prov = reflist_mod.ReflistProvenance(model="claude-opus-5", outcome="read")
+ prov.fields_discarded.append("[9].title")
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, names, texts, disputing=None, model=None: reflist_mod.Resolution(
+ still_disputed=["9"], provenance=prov,
+ ),
+ )
+ rec = _rec(labels_disputed=["9"])
+ entry = _entry("9", "Somebody. 2020.")
+ cli._escalate_disputed(tmp_path, rec, [entry], {"parsed": [entry]}, {"parsed": "text"})
+ out = capsys.readouterr().out
+
+ assert "not printed in either text" in out
+ assert "could not tell which paper it names" not in out
+
+
+# --- Task 7 review, m1: the span window backs up to the previous line break -
+
+
+def test_label_span_backs_up_past_a_long_preceding_entry():
+ """m1. The old fixed-distance backtrack (`_SPAN_BEFORE = 260`) degraded
+ silently once the preceding entry was longer than it — routine with a
+ full author list. Backing up to the previous line break is exactly one
+ entry by construction and cannot degrade regardless of length."""
+ long_entry = "A" * 400
+ text = f"[1] {long_entry}\n[2] Beta B. Second paper. 2021.\n"
+ span = cli._label_span(text, "2")
+ assert long_entry in span, "a 400-char preceding entry must still be included"
+
+
+def test_label_span_at_the_very_start_of_the_text():
+ text = "[1] Alpha A. First paper. 2020.\n[2] Beta B. Second paper. 2021.\n"
+ span = cli._label_span(text, "1")
+ assert span.startswith("[1]")
+
+
+def test_label_span_is_empty_when_the_numeral_was_never_printed():
+ assert cli._label_span("no numbered list here", "7") == ""
+
+
+def test_a_resolved_entrys_slug_collision_is_disambiguated(tmp_path, monkeypatch):
+ """C1's remaining half. `resolve_all` never re-slugs, so a resolved entry
+ whose freshly computed slug collides with one already carried in the
+ chosen list would share a download path with it — the second download
+ overwrites the first. `_unique_slugs` re-runs over the WHOLE substituted
+ list, not just the newly resolved entries, so a collision introduced by
+ the substitution itself is caught too.
+ """
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ kept = _entry("1", "Smith J. An unrelated paper. 2020.")
+ wrong_two = _entry("2", "Wrong Wrong. 1999.")
+ resolved_same_slug = _entry("2", "Smith J. The resolved paper. 2020.")
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, names, texts, disputing=None, model=None: reflist_mod.Resolution(
+ resolved={"2": resolved_same_slug}, still_disputed=[],
+ provenance=reflist_mod.ReflistProvenance(model="claude-opus-5", outcome="read"),
+ ),
+ )
+ rec = _rec(labels_disputed=["2"])
+ out = cli._escalate_disputed(
+ tmp_path, rec, [kept, wrong_two],
+ {"parsed": [kept, wrong_two]}, {"parsed": "text"},
+ )
+ slugs = [e.slug for e in out]
+ assert len(slugs) == len(set(slugs)), f"colliding slugs would share a download path: {slugs}"
+
+
+# --- whole-branch review, Critical 2: the call sees what the report says it saw
+
+
+REFS_TEXT = (
+ "References\n"
+ "[1] Alpha A. First paper. 2020.\n"
+ "[2] Beta B. Second paper. 2021.\n"
+)
+
+
+def _capture_ask(monkeypatch, reply: str) -> dict:
+ """Intercept the one seam `resolve_disputed` calls, and keep the prompt."""
+ import papertrace.ask as ask_mod
+
+ seen: dict = {}
+
+ def fake_ask(prompt, model=None):
+ seen["prompt"] = prompt
+ return reply
+
+ monkeypatch.setattr(ask_mod, "_ask", fake_ask)
+ monkeypatch.setattr(ask_mod, "model_for", lambda site: "claude-opus-5")
+ return seen
+
+
+def test_the_resolution_call_is_shown_the_reading_it_is_ruling_against(tmp_path, monkeypatch):
+ """Critical 2. `reading_texts` holds only the readings with a printed
+ span, so a dispute between the parse and the DEPOSIT — the case this
+ whole feature was built for — was handed to a model that had never seen
+ the deposit's entry, while `rec.note` and the per-claim disclosure both
+ said it was shown both.
+
+ The disputing entries travel as CONTEXT, not as a text to copy from:
+ `_found` still verifies every returned value against the printed
+ extraction alone, so a title the deposit carries and the page does not
+ still cannot be believed."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ seen = _capture_ask(monkeypatch, json.dumps([{"num": "2", "title": "Second paper"}]))
+ parsed = [_entry("1", "Alpha A. First paper. 2020."),
+ _entry("2", "Beta B. Second paper. 2021.")]
+ deposit = [_entry("2", "Gamma G. A wholly different paper. 2019.")]
+ rec = _rec(labels_disputed=["2"])
+
+ cli._escalate_disputed(
+ tmp_path, rec, parsed,
+ {"parsed": parsed, "crossref": deposit},
+ {"parsed": REFS_TEXT},
+ )
+
+ assert "Gamma G. A wholly different paper" in seen["prompt"], seen["prompt"]
+ assert "crossref" in seen["prompt"]
+
+
+def test_one_printed_extraction_is_never_rendered_as_an_empty_second_reading(
+ tmp_path, monkeypatch
+):
+ """`--backend pymupdf` leaves exactly one printed span, and the prompt
+ rendered `--- READING B ---` followed by nothing — telling the model an
+ extraction existed and was empty. `propose` refuses outright on fewer
+ than two texts; this call cannot, because the printed page is still the
+ only authority it needs, so it says how many it has instead."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ seen = _capture_ask(monkeypatch, json.dumps([{"num": "2", "title": "Second paper"}]))
+ parsed = [_entry("2", "Beta B. Second paper. 2021.")]
+ rec = _rec(labels_disputed=["2"])
+
+ cli._escalate_disputed(
+ tmp_path, rec, parsed, {"parsed": parsed}, {"parsed": REFS_TEXT},
+ )
+
+ prompt = seen["prompt"]
+ assert "READING B" not in prompt, prompt
+ assert "one extraction" in prompt
+
+
+def test_the_note_names_the_readings_the_model_was_actually_shown(tmp_path, monkeypatch):
+ """The other half of Critical 2: the report said "a model reading of both
+ texts" on a run where the model saw one. It names them now."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ _capture_ask(monkeypatch, json.dumps([{"num": "2", "title": "Second paper"}]))
+ parsed = [_entry("2", "Beta B. Second paper. 2021.")]
+ rec = _rec(labels_disputed=["2"])
+
+ cli._escalate_disputed(
+ tmp_path, rec, parsed, {"parsed": parsed}, {"parsed": REFS_TEXT},
+ )
+
+ assert "both texts" not in rec.note
+ assert "the parsed extraction" in rec.note
+
+
+# --- whole-branch review, Major 4: the resolution call's own provenance
+
+
+def test_the_resolution_calls_provenance_reaches_the_manifest(disputed, monkeypatch):
+ """Major 4. A model call was made, paid for, verified field by field and
+ allowed to change which papers are judged — and its model name, outcome
+ and discarded fields existed nowhere structured, only inside a prose
+ note. Meanwhile the `reflist_*` fields, which describe a DIFFERENT call,
+ recorded that no model read the reference list at all."""
+ import jsonschema
+
+ pdf, case, answers = disputed
+ answers.append("1")
+ _capture_ask(monkeypatch, json.dumps([{"num": "2", "title": "Second paper"}]))
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.resolution_outcome == "read"
+ assert manifest.resolution_model == "claude-opus-5"
+ assert manifest.resolution_readings == ["parsed", "pymupdf"]
+ assert manifest.resolution_failure == ""
+ # and it is a published field, not a local convenience
+ schema = json.loads(
+ (Path(__file__).resolve().parent.parent / "schemas" / "refs_manifest.schema.json")
+ .read_text()
+ )
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ jsonschema.Draft202012Validator(schema).validate(payload)
+ assert payload["resolution_outcome"] == "read"
+
+
+def test_a_resolution_call_that_did_not_return_is_recorded_as_one_that_happened(
+ disputed, monkeypatch
+):
+ """The third state, which a boolean could not hold: the call was made and
+ raised. `resolution_outcome` says `failed` and carries why, so the run is
+ never described as one where no model was asked."""
+ pdf, case, answers = disputed
+ answers.append("1")
+
+ def boom(*a, **k):
+ raise RuntimeError("claude -p timed out after 600s")
+
+ monkeypatch.setattr(reflist_mod, "resolve_disputed", boom)
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.resolution_outcome == "failed"
+ assert "timed out" in manifest.resolution_failure
+ assert manifest.resolution_model == ""
+
+
+def test_a_manifest_with_no_resolution_call_says_nothing_about_one(disputed):
+ """"" everywhere, and `[]` for the readings — never computed, which is
+ what a run that never escalated actually knows."""
+ pdf, case, answers = disputed
+ answers.append("2")
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.resolution_outcome == ""
+ assert manifest.resolution_model == ""
+ assert manifest.resolution_readings == []
+ assert manifest.resolution_fields_discarded == []
+
+
+# --- whole-branch review, minors 2 and 3
+
+
+def test_seen_in_survives_a_whole_reading_substitution(disputed):
+ """Minor 2. `stamp_seen_in` ran before the escalation, and option 3
+ replaces every entry with one from a reading that was never stamped — so
+ the manifest published `seen_in: []`, which the schema reads as "no
+ reading was established as carrying this", for a run that established
+ it."""
+ pdf, case, answers = disputed
+ answers += ["3", "pymupdf"]
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.numbering_choice == "pymupdf"
+ assert all(e.seen_in for e in manifest.entries), [e.seen_in for e in manifest.entries]
+
+
+def test_adopting_a_reading_whole_is_described_as_adopting_it_whole(disputed):
+ """Minor 3. Option 3 returns `list(candidates[pick])` — every entry, not
+ only the disputed ones — while the note said "you chose X for the 1 label
+ in dispute"."""
+ pdf, case, answers = disputed
+ answers += ["3", "pymupdf"]
+
+ manifest = _run_refs(pdf, case)
+
+ assert "whole" in manifest.numbering_note
+ assert manifest.numbering_note.startswith("you chose")
+
+
+def test_a_resolution_call_that_was_never_made_is_not_a_model_that_could_not_tell(
+ tmp_path, monkeypatch, capsys
+):
+ """`resolve_disputed` can decline before spending anything — the two
+ extractions are too long to send. Every label stays disputed, and the
+ console must not print `_disputed_reason`'s "the model could not tell
+ which paper it names" over a call nobody made: that is a plausible-looking
+ cause standing in for the honest one, which is the failure this codebase
+ exists to refuse."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, names, texts, disputing=None, model=None: reflist_mod.Resolution(
+ still_disputed=list(labels),
+ provenance=reflist_mod.ReflistProvenance(
+ readings=list(names),
+ failure="the extractions of the bibliography are too long to send",
+ ),
+ ),
+ )
+ two = _entry("2", "Beta B. Second paper. 2021.")
+ rec = _rec(labels_disputed=["2"], note="a note")
+ capsys.readouterr()
+
+ cli._escalate_disputed(tmp_path, rec, [two], {"parsed": [two]}, {"parsed": REFS_TEXT})
+
+ out = " ".join(capsys.readouterr().out.split())
+ assert "could not tell" not in out
+ assert "too long to send" in out
+ assert rec.labels_disputed == ["2"]
+ assert rec.labels_resolved == []
+ assert rec.choice == "withheld"
+ assert rec.resolution_outcome == "not_attempted"
+ assert "too long to send" in rec.resolution_failure
+ # and the note must not claim a reading that never happened
+ assert "verified verbatim" not in rec.note
+
+
+# --- fix round 2: labels_uncomparable stays a subset through the escalation
+
+
+def test_the_manifest_never_publishes_an_uncomparable_label_it_stopped_disputing(
+ disputed, monkeypatch
+):
+ """The subset invariant, through the one path that can break it: the
+ escalation takes a label OUT of `labels_disputed`, and a
+ `labels_uncomparable` computed before it would then name a label the
+ manifest no longer disputes. Two published fields disagreeing on one
+ manifest is Major 3 of this review, and this is where it would recur."""
+ pdf, case, answers = disputed
+ answers.append("1")
+ _capture_ask(monkeypatch, json.dumps([{"num": "2", "title": "Second paper"}]))
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.labels_resolved == ["2"]
+ assert manifest.labels_disputed == []
+ assert manifest.labels_uncomparable == []
+ assert set(manifest.labels_uncomparable) <= set(manifest.labels_disputed)
+
+
+def test_a_run_that_computed_the_split_publishes_it_even_when_empty(disputed):
+ """`[]` is a measurement — every dispute here was a contradiction — and a
+ live run must never leave the field at its never-computed default."""
+ pdf, case, answers = disputed
+ answers.append("2")
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.labels_disputed == ["2"]
+ assert manifest.labels_uncomparable == []
+ assert manifest.labels_uncomparable is not None
+
+
+def test_the_console_names_the_cause_it_measured(disputed, capsys):
+ """The console is a surface like any other, and it has the split in hand
+ — `rec.labels_uncomparable` is computed two lines above the line that
+ prints. On this fixture the two readings name genuinely different papers
+ at [2], so the contradiction wording is the true one."""
+ pdf, case, answers = disputed
+ answers.append("2")
+ capsys.readouterr()
+
+ _run_refs(pdf, case)
+
+ out = " ".join(capsys.readouterr().out.split())
+ assert "the readings do not agree at [2]" in out
+ assert "named different papers there" in out
+ assert "nothing in them could be compared" not in out
diff --git a/tests/test_ingest_flat_reading.py b/tests/test_ingest_flat_reading.py
new file mode 100644
index 0000000..f59c2b6
--- /dev/null
+++ b/tests/test_ingest_flat_reading.py
@@ -0,0 +1,83 @@
+"""A second, independent reading of the bibliography — no ingest, no write.
+
+`references_span_flat` exists to give `reconcile` a plain pymupdf reading of
+the References section, cheap enough to run on every docling run without
+asking permission. Neither existing ingest test module fits what this needs
+checked: `test_ingest_backends.py` stubs docling and never touches a real PDF
+or pymupdf, and `test_reference_list.py` hand-builds `SourceMap`s in memory.
+This module's central claim — a real file read, and NOTHING written back to
+disk — needs a real path and a real `cwd` to check at all.
+"""
+
+import sys
+from pathlib import Path
+
+try:
+ import pymupdf as fitz # PyMuPDF >= 1.24 module name (the bare `fitz` import is deprecated)
+except ImportError: # pragma: no cover - older PyMuPDF exposes only `fitz`
+ import fitz
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+
+from papertrace.ingest import references_span_flat # noqa: E402
+from papertrace.refs import parse_references # noqa: E402
+
+
+def _paper(path: Path, *, with_references: bool) -> Path:
+ """A two-page PDF: page 1 is body text, page 2 is a References section
+ with three numbered entries, or an unrelated heading when
+ `with_references` is False."""
+ doc = fitz.open()
+ page1 = doc.new_page()
+ page1.insert_text((72, 100), "A Study of Something", fontsize=16)
+ page1.insert_text((72, 140), "Body text citing [1], [2] and [3] here.", fontsize=11)
+ page2 = doc.new_page()
+ if with_references:
+ page2.insert_text((72, 100), "References", fontsize=14)
+ page2.insert_text((72, 130), "[1] Alpha A. A first paper. 2019.", fontsize=11)
+ page2.insert_text((72, 150), "[2] Beta B. A second paper. 2020.", fontsize=11)
+ page2.insert_text((72, 170), "[3] Gamma C. A third paper. 2021.", fontsize=11)
+ else:
+ page2.insert_text((72, 100), "Acknowledgements", fontsize=14)
+ page2.insert_text((72, 130), "We thank nobody in particular.", fontsize=11)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ doc.save(path)
+ doc.close()
+ return path
+
+
+def test_the_flat_reading_holds_the_printed_entries_and_parses_to_the_same_count(tmp_path):
+ """The whole point of a second reading: it must actually see the
+ bibliography, not merely avoid crashing on one."""
+ pdf = _paper(tmp_path / "paper.pdf", with_references=True)
+
+ text, resumed = references_span_flat(pdf)
+
+ assert resumed is False
+ for surname in ("Alpha", "Beta", "Gamma"):
+ assert surname in text, f"{surname} missing from the flat reading — {text!r}"
+ assert len(parse_references(text)) == 3
+
+
+def test_a_paper_with_no_references_heading_degrades_honestly(tmp_path):
+ """No heading found means `(\"\", False)` — never an exception, and never
+ a guess at where the list might be."""
+ pdf = _paper(tmp_path / "paper.pdf", with_references=False)
+
+ assert references_span_flat(pdf) == ("", False)
+
+
+def test_the_flat_reading_writes_nothing_to_disk(tmp_path, monkeypatch):
+ """This is a READING, not an ingest. `--parse-only` already carries the
+ scar of a second read touching the case's real `source_map.json`
+ (`cli.py:543-549`) — a reading taken purely to vote on a label must not
+ risk doing that even by accident, so it never calls `write_outputs`, never
+ takes an `out_dir`, and never touches the cwd it happens to run in."""
+ monkeypatch.chdir(tmp_path)
+ pdf = _paper(tmp_path / "sources" / "paper.pdf", with_references=True)
+
+ references_span_flat(pdf)
+
+ assert list(tmp_path.rglob("source_map.json")) == []
+ assert list(tmp_path.rglob("clean.md")) == []
+ assert list(tmp_path.rglob("annotated.md")) == []
diff --git a/tests/test_label_agreement.py b/tests/test_label_agreement.py
new file mode 100644
index 0000000..cb6545d
--- /dev/null
+++ b/tests/test_label_agreement.py
@@ -0,0 +1,777 @@
+"""Per cited label: do the readings that carry it still name one paper?
+
+`label_agreement` is the second, independent axis `reconcile` gains in this
+plan — see `.superpowers/facts/reconcile.md` — and it is deliberately not
+built on `_first_divergence`, which zips two readings by POSITION. A reading
+that split one entry into two shifts every label after it by one, so
+comparing "reading A's 6th entry" against "reading B's 6th entry" would report
+a disagreement about the wrong label, or an agreement that is really two
+different labels lining up by coincidence. The join key here is `e.num`, the
+printed numeral — the same fact `parse_references` already treats as ground
+truth wherever the converter left it legible.
+
+Nothing in `src/` calls `label_agreement` yet. This module tests it alone, the
+way `tests/test_reflist.py` will later test `reflist.py` alone — the design's
+own promise is that the deterministic parts are testable without a model, and
+this is the part with no model in it at all.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+import jsonschema
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+
+from papertrace.models import ( # noqa: E402
+ DOCUMENT_KINDS,
+ REF_STATUSES,
+ RefEntry,
+ RefManifest,
+)
+from papertrace.refs import ( # noqa: E402
+ LABEL_AGREEMENT,
+ _comparably_same,
+ _entry,
+ _same_work,
+ corroborating_readings,
+ label_agreement,
+ stamp_seen_in,
+ uncomparable_labels,
+)
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def _work(num: str, doi: str) -> RefEntry:
+ """Built through `_entry` — the production path, which parses the DOI out
+ of `raw` the same way a real reading would, rather than setting it by
+ hand and bypassing the exact code every candidate actually goes through."""
+ return _entry(num, f"Author for entry {num}. A distinctive title. doi:{doi}")
+
+
+# --- the five states, one rule each ----------------------------------------
+
+
+def test_two_readings_naming_the_same_work_at_a_label_agree():
+ """The base case a corroborating second reading exists to produce."""
+ candidates = {
+ "parsed": [_work("5", "10.1000/x5")],
+ "pymupdf": [_work("5", "10.1000/x5")],
+ }
+ assert label_agreement(candidates, {"5"}) == {"5": "agreed"}
+
+
+def test_two_readings_naming_different_works_at_a_label_are_disputed():
+ """The failure this whole feature exists to catch: two readings of one
+ bibliography naming different papers at the same printed label."""
+ candidates = {
+ "parsed": [_work("5", "10.1000/x5")],
+ "pymupdf": [_work("5", "10.1000/x9")],
+ }
+ assert label_agreement(candidates, {"5"}) == {"5": "disputed"}
+
+
+def test_three_readings_two_agreeing_and_one_not_is_disputed_not_majority():
+ """`reconcile`'s own rule, restated for this function: 'a non-unique
+ match is refused, never ranked.' Two votes for one paper and one for
+ another must not out-vote the dissent into a false `agreed` — if this
+ regresses to a majority rule, it will look like an improvement (fewer
+ labels withheld) right up until it prints a verdict against the paper the
+ minority reading was right about."""
+ candidates = {
+ "parsed": [_work("5", "10.1000/x5")],
+ "pymupdf": [_work("5", "10.1000/x5")],
+ "llm": [_work("5", "10.1000/x9")],
+ }
+ assert label_agreement(candidates, {"5"}) == {"5": "disputed"}
+
+
+def test_one_reading_carrying_a_label_is_single():
+ """`single` must print (invariant 4): a pymupdf-backend run whose only
+ other candidate was discarded still has ONE reading, and every one of its
+ labels is `single` — not `absent`, which would say nobody read it at all,
+ and not `disputed`, which would withhold a verdict nothing contradicts."""
+ candidates = {"parsed": [_work("5", "10.1000/x5")]}
+ assert label_agreement(candidates, {"5"}) == {"5": "single"}
+
+
+def test_no_reading_carrying_a_label_is_absent():
+ """Already `not_retrieved` downstream — `absent` just names the reason: no
+ candidate reading has anything to say about this label at all."""
+ candidates = {
+ "parsed": [_work("5", "10.1000/x5")],
+ "pymupdf": [_work("5", "10.1000/x5")],
+ }
+ assert label_agreement(candidates, {"9"}) == {"9": "absent"}
+
+
+# --- the two entries that must not speak for their label --------------------
+
+
+def test_a_boundary_ambiguous_entry_does_not_corroborate_a_matching_reading():
+ """Invariant 5: a `boundary_ambiguous` entry has already refused to say
+ what paper its label names — `parse_references` sets it precisely because
+ the entry's start is a guess. Counting it as a second vote would turn that
+ refusal into evidence. Here it is given the SAME doi as the one real
+ voter, so a regression that let it speak would wrongly report `agreed`
+ instead of the correct `single`."""
+ ambiguous = _work("5", "10.1000/x5")
+ ambiguous.boundary_ambiguous = True
+ candidates = {
+ "parsed": [ambiguous],
+ "pymupdf": [_work("5", "10.1000/x5")],
+ }
+ assert label_agreement(candidates, {"5"}) == {"5": "single"}
+
+
+def test_a_reading_carrying_the_same_label_twice_disputes_it_on_its_own():
+ """A duplicate is judgeable, so it is `disputed` with no second opinion.
+
+ This is the discriminating case: one reading is clean and agrees with
+ nothing else, one reading carries [7] twice with two different DOIs, and
+ there is no third reading to break anything. Under a "a duplicate simply
+ does not vote" rule this comes back `single` and prints — meaning a verdict
+ on [7] rests on whichever of the two papers `resolve_entry` happened to
+ download. It must be `disputed`.
+ """
+ candidates = {
+ "parsed": [_work("7", "10.1000/x7"), _work("7", "10.1000/x77")],
+ "pymupdf": [_work("7", "10.1000/x7")],
+ }
+ assert label_agreement(candidates, {"7"}) == {"7": "disputed"}
+
+
+def test_a_duplicate_does_not_hide_a_disagreement_between_the_other_readings():
+ """The same rule with two other readings that genuinely disagree, so the
+ result is `disputed` for two independent reasons at once. Kept alongside the
+ test above because that one pins the rule and this one pins that the rule
+ does not accidentally short-circuit the pairwise comparison."""
+ candidates = {
+ "parsed": [_work("7", "10.1000/x7")],
+ "pymupdf": [_work("7", "10.1000/x7"), _work("7", "10.1000/x77")],
+ "llm": [_work("7", "10.1000/x9")],
+ }
+ assert label_agreement(candidates, {"7"}) == {"7": "disputed"}
+
+
+def test_a_refused_entry_can_leave_a_label_single_and_that_is_deliberate():
+ """The other half of the asymmetry, pinned so nobody "fixes" it.
+
+ A `boundary_ambiguous` entry does not vote, so one clean reading leaves the
+ label `single`, which prints. That is safe because `resolve_entry` honours
+ `boundary_ambiguous` by setting `no_doi` and fetching nothing — the refused
+ entry cannot be the source of any verdict, whichever way this function
+ labels it. A duplicate has no such brake, which is why the two cases are
+ treated differently."""
+ refused = _work("7", "10.1000/x7")
+ refused.boundary_ambiguous = True
+ candidates = {"parsed": [refused], "pymupdf": [_work("7", "10.1000/x7")]}
+ assert label_agreement(candidates, {"7"}) == {"7": "single"}
+
+
+# --- the case this plan exists for ------------------------------------------
+
+
+def test_the_real_manuscripts_shape_agrees_up_to_the_split_and_disputes_after():
+ """The reported defect, reproduced without a model or a converter. A
+ correct 22-entry reading and a 24-entry reading whose [12] is a
+ split-title fragment: everything from [13] on is shifted one label late,
+ so the label and the paper it names have come apart. [1]-[11] sit before
+ the split and must still agree; [13] on must all be disputed, because the
+ printed numeral now names a different work than it did in the correct
+ reading. This is the six-wrong-verdicts case from
+ `docs/superpowers/specs/2026-09-16-llm-reference-list-design.md`."""
+ correct = [_work(str(n), f"10.1000/x{n}") for n in range(1, 23)]
+
+ # the second reading: 1-11 intact, 12 a fragment (excluded from every
+ # assertion below), 13-23 each carrying the DOI that belongs one label
+ # earlier in `correct`, 24 an extra nobody cites
+ split = (
+ [_work(str(n), f"10.1000/x{n}") for n in range(1, 12)]
+ + [_entry("12", "an unresolved title fragment with no doi of its own")]
+ + [_work(str(n), f"10.1000/x{n - 1}") for n in range(13, 24)]
+ + [_work("24", "10.1000/x99")]
+ )
+
+ body = {str(n) for n in range(1, 23)}
+ agreement = label_agreement({"parsed": correct, "pymupdf": split}, body)
+
+ for n in range(1, 12):
+ assert agreement[str(n)] == "agreed", agreement
+ for n in range(13, 23):
+ assert agreement[str(n)] == "disputed", agreement
+
+
+# --- the vocabulary, and the one wire-format check it does not have yet -----
+
+
+def test_label_agreement_never_prints_outside_its_own_published_vocabulary():
+ """`LABEL_AGREEMENT` is published so a future consumer — the withholding
+ filter, the disclosure — has something to check membership against. If
+ this function ever returned a fifth word, that check would silently pass
+ it through as if it were one of the four this plan specified."""
+ candidates = {
+ "parsed": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ "pymupdf": [_work("1", "10.1000/x1"), _work("2", "10.1000/x9")],
+ }
+ for state in label_agreement(candidates, {"1", "2", "3"}).values():
+ assert state in LABEL_AGREEMENT
+
+
+def test_ref_statuses_matches_its_own_schema_enum():
+ """`tests/test_coverage.py` asserts `set(VERDICTS) == set(schema_enum)`
+ for `results.schema.json`, but nothing did the equivalent for
+ `REF_STATUSES` against `refs_manifest.schema.json` — the exact gap this
+ plan's own facts file names. Modelled on that test's idiom, not a new
+ one."""
+ schema = json.loads((ROOT / "schemas" / "refs_manifest.schema.json").read_text())
+ enum = schema["properties"]["entries"]["items"]["properties"]["status"]["enum"]
+ assert set(REF_STATUSES) == set(enum)
+
+
+def test_document_kinds_matches_its_own_schema_enum():
+ """The existing check compares the schema against a hardcoded literal list
+ rather than against `DOCUMENT_KINDS`, so adding a kind to the constant and
+ to the schema while forgetting the literal leaves a green suite. Compare the
+ constant, which is the thing `SourceJudgement.kind` is validated against."""
+ schema = json.loads((ROOT / "schemas" / "results.schema.json").read_text())
+ enum = (
+ schema["properties"]["claims"]["items"]["properties"]["judgements"]
+ ["items"]["properties"]["kind"]["enum"]
+ )
+ assert set(DOCUMENT_KINDS) == set(enum)
+
+
+def test_the_title_check_vocabulary_matches_its_own_schema_enum():
+ """`title_check` is the third undocumented vocabulary the facts file names.
+ Its schema enum carries `null` as a real state — "too few words to tell" is
+ not `mismatch` — so the comparison drops the null and compares the strings,
+ and this docstring is where that asymmetry is recorded."""
+ schema = json.loads((ROOT / "schemas" / "refs_manifest.schema.json").read_text())
+ enum = schema["properties"]["entries"]["items"]["properties"]["title_check"]["enum"]
+ assert None in enum, "null is a real title_check state and must stay in the enum"
+ assert {e for e in enum if e is not None} == {"verified", "unverifiable", "mismatch"}
+
+
+# --- Gate 2: the new persisted fields round-trip, and old manifests still load
+
+
+def test_the_six_new_reconciliation_fields_round_trip_through_a_manifest(tmp_path):
+ """Gate 2. `Reconciliation` itself is never serialised — its fields reach
+ disk only via the `RefManifest` fields `cli._refs_pipeline` copies them
+ into (Task 6's job). This test exercises that copy shape directly, against
+ the schema, so the round trip is proven before anything wires it in."""
+ m = RefManifest(
+ manuscript="m.pdf",
+ entries=[RefEntry(num="1", raw="x", status="paywalled")],
+ numbering_corroborated=True,
+ corroborating_readings=["parsed", "pymupdf"],
+ labels_disputed=["13", "14"],
+ labels_resolved=["13"],
+ numbering_choice="llm_resolved",
+ numbering_chosen_by="user",
+ reflist_model="claude-opus-5",
+ reflist_fields_discarded=["title"],
+ )
+ p = tmp_path / "refs_manifest.json"
+ m.to_json(p)
+
+ schema = json.loads((ROOT / "schemas" / "refs_manifest.schema.json").read_text())
+ jsonschema.validate(json.loads(p.read_text()), schema)
+
+ back = RefManifest.from_json(p)
+ assert back.numbering_corroborated is True
+ assert back.corroborating_readings == ["parsed", "pymupdf"]
+ assert back.labels_disputed == ["13", "14"]
+ assert back.labels_resolved == ["13"]
+ assert back.numbering_choice == "llm_resolved"
+ assert back.numbering_chosen_by == "user"
+ assert back.reflist_model == "claude-opus-5"
+ assert back.reflist_fields_discarded == ["title"]
+
+ # a manifest written before this feature carries none of these keys
+ old = {"manuscript": "m.pdf", "entries": [{"num": "1", "raw": "x", "status": "paywalled"}]}
+ p.write_text(json.dumps(old))
+ older = RefManifest.from_json(p)
+ # `None`, not `False`: the field's own description has always said absent
+ # means never computed, and as a plain boolean it could not say so — the
+ # same three-state discipline `reflist_entries_proposed` is written under
+ assert older.numbering_corroborated is None
+ assert older.corroborating_readings == []
+ assert older.labels_disputed == []
+ assert older.labels_resolved == []
+ assert older.numbering_choice == ""
+ assert older.numbering_chosen_by == ""
+ assert older.reflist_model == ""
+ assert older.reflist_fields_discarded == []
+
+
+def test_seen_in_round_trips_on_an_entry_and_defaults_empty_on_an_older_one(tmp_path):
+ """Gate 2. An entry seen only in the model's reading has to be spottable —
+ `seen_in == ["llm"]` is what makes that visible on a real manifest, and it
+ must not silently become `[]` on the way through JSON."""
+ e = RefEntry(num="1", raw="x", status="retrieved", seen_in=["parsed", "llm"])
+ m = RefManifest(manuscript="m.pdf", entries=[e])
+ p = tmp_path / "refs_manifest.json"
+ m.to_json(p)
+
+ schema = json.loads((ROOT / "schemas" / "refs_manifest.schema.json").read_text())
+ jsonschema.validate(json.loads(p.read_text()), schema)
+
+ assert RefManifest.from_json(p).entries[0].seen_in == ["parsed", "llm"]
+
+ old = {"manuscript": "m.pdf", "entries": [{"num": "1", "raw": "x", "status": "retrieved"}]}
+ p.write_text(json.dumps(old))
+ assert RefManifest.from_json(p).entries[0].seen_in == []
+
+
+# --- agreement needs evidence, not the absence of counter-evidence ----------
+
+
+def test_two_uncomparable_entries_are_disputed_rather_than_agreed():
+ """The laundering this feature could otherwise perform.
+
+ `_same_work` answers "is there evidence these differ?" and returns True
+ when there is nothing to compare — correct for `_first_divergence`, which
+ must not manufacture a divergence out of silence. A per-label vote asks the
+ opposite question, so silence must not manufacture agreement either.
+ Measured before `_comparably_same` existed: these two came back `agreed`.
+
+ NEITHER entry carries a DOI or a single title token, so nothing comparable
+ was said by anybody — see
+ `test_a_label_every_voter_is_mute_about_is_disputed_not_single` for the
+ ruling that keeps this `disputed` now that a cannot-tell voter abstains.
+ """
+ a = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
+ b = RefEntry(num="7", raw="Q. Z. 2020. 88(1):4-7.")
+ assert label_agreement({"parsed": [a], "pymupdf": [b]}, {"7"}) == {"7": "disputed"}
+
+
+def test_one_garbled_reading_cannot_corroborate_a_legible_one():
+ """Broader than it looks: the benefit of the doubt applied when **either**
+ side yielded no tokens, not only when both did. So a real reference paired
+ with an unreadable one was `agreed`, and `agreed` is the single state that
+ lets a verdict through untouched.
+
+ RE-RULED at the whole-branch review: the answer is `single`, not
+ `disputed`. The intent of this test — a garbled reading must not
+ CORROBORATE a legible one — is preserved by abstention, which is what
+ `single` records: one reading spoke, nothing agreed with it, nothing
+ contradicted it. Counting the garbled reading as dissent instead withheld
+ the verdict on a label nothing had said anything against, and on a
+ bare-DOI Crossref deposit it withheld every verdict in the run.
+ """
+ good = RefEntry(
+ num="7",
+ raw="Fujita S (2023) Characterization of brain volume changes in aging individuals. "
+ "JAMA Netw Open.",
+ )
+ garbled = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
+ assert label_agreement({"parsed": [good], "llm": [garbled]}, {"7"}) == {"7": "single"}
+ assert corroborating_readings({"parsed": [good], "llm": [garbled]}, {"7"}) == []
+
+
+# --- "cannot compare" is a third answer, and it abstains --------------------
+
+
+def _bare_doi(num: str, doi: str) -> RefEntry:
+ """A Crossref deposit of a DOI and nothing else — `_reference_raw`'s
+ documented fallback, and 49 of 52 references on the Wiley paper that
+ prompted this. It yields no title tokens at all, permanently, by the
+ deposit's shape."""
+ return RefEntry(num=num, raw=f"https://doi.org/{doi}", doi=doi)
+
+
+def _printed(num: str) -> RefEntry:
+ """One entry as a reference list that prints no DOIs carries it — normal
+ for many journals, and the other half of Critical 1's shape: the deposit
+ has a DOI and nothing else, the page has everything but."""
+ return _entry(
+ num,
+ f"Fujita S, Mori S. Characterization of brain volume changes, entry {num}. "
+ "JAMA Netw Open. 2023;6(6).",
+ )
+
+
+def test_comparably_same_has_a_third_answer_for_nothing_to_compare():
+ """`None`, the same three-valued vocabulary `models.titles_match`
+ publishes for the same reason. `False` here is a claim about the two
+ papers being different, and a bare-DOI deposit is not evidence of that."""
+ assert _comparably_same(_bare_doi("7", "10.1002/hep.31884"), _printed("7")) is None
+ assert _comparably_same(_work("7", "10.1000/x7"), _work("7", "10.1000/x7")) is True
+ assert _comparably_same(_work("7", "10.1000/x7"), _work("7", "10.1000/x9")) is False
+
+
+def test_a_bare_doi_deposit_does_not_withhold_a_label_the_parses_agree_on():
+ """Critical 1 of the whole-branch review, reproduced.
+
+ A publisher deposits bare DOIs; the printed list prints none. The deposit
+ can be compared with neither parse, but the two parses agree with each
+ other perfectly. Counting the uncomparable voter as dissent made every
+ cited label `disputed`, every source withheld and every claim
+ `unchecked` — an audit with no verdicts in it, on a paper where nothing
+ was wrong."""
+ candidates = {
+ "crossref": [_bare_doi("7", "10.1002/hep.31884")],
+ "parsed": [_printed("7")],
+ "pymupdf": [_printed("7")],
+ }
+ assert label_agreement(candidates, {"7"}) == {"7": "agreed"}
+ # and the voter nobody could compare is not named as having agreed
+ assert corroborating_readings(candidates, {"7"}) == ["parsed", "pymupdf"]
+
+
+def test_dropping_the_uncomparable_voter_can_leave_one_and_that_prints():
+ """The second-order effect of the abstention rule, pinned deliberately.
+
+ On a `--backend pymupdf` run there is no flat reading, so the deposit and
+ the parse are the only two voters and the deposit cannot be compared.
+ One comparable voter remains, which is `single`: no corroboration is
+ claimed and no verdict is withheld. That is the state a one-reading run
+ has always been in, and it is what the pre-branch behaviour was — the
+ laundering this branch removed is `agreed`, which is what would claim the
+ deposit backed the parse."""
+ candidates = {
+ "crossref": [_bare_doi("7", "10.1002/hep.31884")],
+ "parsed": [_printed("7")],
+ }
+ assert label_agreement(candidates, {"7"}) == {"7": "single"}
+ assert corroborating_readings(candidates, {"7"}) == []
+
+
+def test_a_label_every_voter_is_mute_about_is_disputed_not_single():
+ """The cell the abstention rule does not cover, ruled at the review.
+
+ Two readings carry [7] and NEITHER says anything a comparison could use.
+ `absent` is false (two readings carry it), `single` is false (no reading
+ spoke comparably), `agreed` is the laundering. `disputed` is the true one:
+ nothing establishes that these name one paper."""
+ a = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
+ b = RefEntry(num="7", raw="Q. Z. 2020. 88(1):4-7.")
+ assert label_agreement({"parsed": [a], "pymupdf": [b]}, {"7"}) == {"7": "disputed"}
+
+
+def test_a_label_no_voter_carries_at_all_is_absent_not_disputed():
+ """One character apart from the cell above and opposite in meaning: zero
+ voters is `absent` (nobody read it), zero COMPARABLE voters with the label
+ carried is `disputed` (it was read and nothing could be compared)."""
+ a = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
+ b = RefEntry(num="7", raw="Q. Z. 2020. 88(1):4-7.")
+ assert label_agreement({"parsed": [a], "pymupdf": [b]}, {"9"}) == {"9": "absent"}
+
+
+# --- the model's reading may dispute, and that is all it may do -------------
+
+
+def test_the_model_reading_never_corroborates_what_it_copied_out_of():
+ """Major 1. `reflist.propose` may only COPY from the two extractions, and
+ both of those are voters in their own right — so the model agreeing with
+ one of them is one text read twice, not two readings agreeing. Here the
+ flat parse lost [7] entirely and the model echoed it back out of text A:
+ counting that as a second reading turned `numbering_corroborated` True and
+ printed "two readings of the reference list agree" over one text."""
+ e = _work("7", "10.1000/x7")
+ candidates = {"parsed": [e], "pymupdf": [], "llm": [_work("7", "10.1000/x7")]}
+ assert label_agreement(candidates, {"7"}) == {"7": "single"}
+ assert corroborating_readings(candidates, {"7"}) == []
+
+
+def test_the_model_reading_can_still_dispute_a_label():
+ """The other half of Major 1's ruling, and the only power the safety
+ property grants the model reading: it may disagree. Excluding it from
+ corroboration must not also excuse it from the comparison."""
+ candidates = {
+ "parsed": [_work("7", "10.1000/x7")],
+ "pymupdf": [_work("7", "10.1000/x7")],
+ "llm": [_work("7", "10.1000/x9")],
+ }
+ assert label_agreement(candidates, {"7"}) == {"7": "disputed"}
+
+
+def test_the_model_reading_is_not_named_beside_two_real_readings_that_agree():
+ """Three voters all agreeing: the two that read the page are named, the
+ one that copied out of them is not. The printed count is then 2, which is
+ what `NUMBERING_CORROBORATION_TOKEN` asserts."""
+ candidates = {
+ "parsed": [_work("7", "10.1000/x7")],
+ "pymupdf": [_work("7", "10.1000/x7")],
+ "llm": [_work("7", "10.1000/x7")],
+ }
+ assert label_agreement(candidates, {"7"}) == {"7": "agreed"}
+ assert corroborating_readings(candidates, {"7"}) == ["parsed", "pymupdf"]
+
+
+def test_two_doi_less_readings_that_do_share_a_title_still_agree():
+ """The tightening must not go so far that a DOI-less bibliography can never
+ corroborate itself. Some publishers print no DOIs in a reference list, and
+ title-token overlap is the only evidence available there — refusing it would
+ make every label on such a paper `disputed` and the audit worthless.
+
+ The two fixtures overlap **partially** (ratio 0.4, above `_same_work`'s 0.34
+ floor), not identically. An identical pair would exercise the overlap
+ arithmetic trivially and would still pass if the floor were raised almost to
+ 1.0, so it would guard the structural case and nothing else. These two
+ differ the way two converters' readings of one entry actually differ —
+ author list truncated on one side, `aging`/`ageing`, a subtitle read two
+ ways — and share only `fujita` and `characterization`.
+ """
+ a = RefEntry(num="7", raw="Fujita S (2023) Characterization of brain volume changes in aging")
+ b = RefEntry(
+ num="7",
+ raw="Fujita, Mori, Onda. Characterization of cerebral volume trajectories "
+ "during healthy ageing",
+ )
+ assert label_agreement({"parsed": [a], "pymupdf": [b]}, {"7"}) == {"7": "agreed"}
+
+
+def test_same_work_keeps_its_own_direction_for_its_own_callers():
+ """`_same_work` is NOT changed. `reconcile` derives `contested` from
+ `_first_divergence`, which derives it from `_same_work`, and that path needs
+ "nothing to compare" to mean "no divergence found". Tightening the shared
+ helper would have turned every unreadable entry into a reported divergence
+ and re-tainted whole reports. Two functions, two directions, one test that
+ says so."""
+ a = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
+ b = RefEntry(num="7", raw="Q. Z. 2020. 88(1):4-7.")
+ assert _same_work(a, b) is True
+ # `None`, not `False`: the two functions are siblings answering opposite
+ # questions, and each says "cannot tell" in its own caller's safe
+ # direction — `_same_work` as True, `_comparably_same` as a third answer
+ assert _comparably_same(a, b) is None
+
+
+# --- seen_in: which readings also carried a chosen entry's work ------------
+
+
+def test_an_entry_only_the_model_found_is_spottable_in_the_manifest():
+ """The published field's whole purpose. An entry no deterministic reading
+ carried is the one a reader most needs to see flagged, and `seen_in ==
+ ["llm"]` is how they see it."""
+ chosen = [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")]
+ candidates = {
+ "parsed": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ "llm": [_work("1", "10.1000/x1")],
+ }
+ stamp_seen_in(chosen, candidates)
+ assert chosen[0].seen_in == ["llm", "parsed"]
+ assert chosen[1].seen_in == ["parsed"]
+
+
+def test_a_reading_naming_a_different_paper_under_the_same_label_is_not_provenance():
+ """Sharing a numeral is not agreeing about a work.
+
+ Stamping on the label alone would record the disagreeing reading as having
+ corroborated this entry — the extent-for-content substitution `_covers` is
+ documented as blind to, reintroduced in a published field."""
+ chosen = [_work("5", "10.1000/x5")]
+ candidates = {"parsed": [_work("5", "10.1000/x5")],
+ "pymupdf": [_work("5", "10.1000/x99")]}
+ stamp_seen_in(chosen, candidates)
+ assert chosen[0].seen_in == ["parsed"]
+
+
+def test_seen_in_never_names_a_reading_label_agreement_says_disagrees():
+ """Major 3. `stamp_seen_in` matched through the lenient `_same_work`,
+ which gives the benefit of the doubt when either side yields no title
+ tokens — so one manifest published, for one label in one run, both "two
+ readings contributed this entry" (`seen_in`) and "the readings do not
+ agree about it" (`labels_disputed`). Two published fields contradicting
+ each other. The comparator here is now the same one the agreement axis
+ uses."""
+ chosen = [RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")]
+ other = RefEntry(num="7", raw="Q. Z. 2020. 88(1):4-7.")
+ candidates = {"parsed": chosen, "pymupdf": [other]}
+ assert label_agreement(candidates, {"7"}) == {"7": "disputed"}
+ stamp_seen_in(chosen, candidates)
+ assert chosen[0].seen_in == ["parsed"]
+
+
+def test_a_chosen_entry_is_always_credited_to_the_reading_it_came_from():
+ """Identity, not comparison: the chosen list IS one of the readings'
+ lists, so an entry nothing could be compared against still records where
+ it came from. Without this, a garbled entry's `seen_in` would be `[]` —
+ which the schema reads as "no reading was established as carrying this" —
+ for a reading that demonstrably carried it."""
+ garbled = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
+ chosen = [garbled]
+ stamp_seen_in(chosen, {"parsed": [garbled], "pymupdf": []})
+ assert chosen[0].seen_in == ["parsed"]
+
+
+def test_stamping_twice_never_erases_a_provenance_an_earlier_pass_recorded():
+ """`cli` stamps AFTER the escalation, so an entry substituted by
+ `reflist.resolve_disputed` already carries the reading its values were
+ copied from. That name is a fact this pass cannot recompute — the
+ readings that disputed the label carry no entry matching the resolved
+ one — so it is merged, never overwritten."""
+ resolved = RefEntry(num="7", raw="A resolved title.", seen_in=["pymupdf"])
+ stamp_seen_in([resolved], {"parsed": [], "pymupdf": []})
+ assert resolved.seen_in == ["pymupdf"]
+
+
+# --- corroborating_readings: named only where a reading actually voted -----
+# (N3 of the Task 6 re-review: `any(e.num in body for e in cand)` — the filter
+# that shipped in round 1 — named a reading that carried only SOME cited
+# labels, or whose only carrier of one was `boundary_ambiguous`, as having
+# corroborated ALL of them.)
+
+
+def test_corroborating_readings_excludes_a_reading_that_voted_on_only_some_labels():
+ """A reading that skipped an entry corroborated the ones it carried, not
+ the ones it did not. `llm` here votes on [1] and never mentions [2] at
+ all — both cited labels still come back `agreed` (crossref and parsed
+ carry [2] between them), so the round-1 filter named `llm` as one of the
+ readings that agreed on `every cited label`, which it never spoke to."""
+ body = {"1", "2"}
+ candidates = {
+ "parsed": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ "crossref": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ "llm": [_work("1", "10.1000/x1")],
+ }
+ assert label_agreement(candidates, body) == {"1": "agreed", "2": "agreed"}
+ assert corroborating_readings(candidates, body) == ["crossref", "parsed"]
+
+
+def test_corroborating_readings_excludes_a_reading_whose_only_carrier_is_boundary_ambiguous():
+ """A `boundary_ambiguous` carrier casts no vote at all (invariant 5 of
+ `label_agreement`) — sharing that fact with `corroborating_readings` is
+ the point of factoring `_label_voters` out, rather than a coincidence two
+ unrelated filters happen to agree on.
+
+ The two readings that DO vote here are `crossref` and `pymupdf`. This
+ fixture used to name `llm` as the second voter, which since Major 1 is
+ excluded from corroboration for a different reason entirely — so the test
+ would have gone green through a rule it was never about. Two independent
+ exclusions must not be tested by one fixture."""
+ ambiguous_1 = _work("1", "10.1000/x1")
+ ambiguous_1.boundary_ambiguous = True
+ ambiguous_2 = _work("2", "10.1000/x2")
+ ambiguous_2.boundary_ambiguous = True
+ body = {"1", "2"}
+ candidates = {
+ "parsed": [ambiguous_1, ambiguous_2],
+ "crossref": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ "pymupdf": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ }
+ assert label_agreement(candidates, body) == {"1": "agreed", "2": "agreed"}
+ assert corroborating_readings(candidates, body) == ["crossref", "pymupdf"]
+
+
+def test_corroborating_readings_is_empty_unless_every_cited_label_agreed():
+ """The caller (`cli._refs_pipeline`) also gates on `all(... == "agreed"
+ ...)`, but this function must not rely on that: naming any reading at all
+ when one cited label is disputed would still be the overstatement."""
+ body = {"1", "2"}
+ candidates = {
+ "parsed": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ "llm": [_work("1", "10.1000/x1"), _work("2", "10.1000/x9")],
+ }
+ assert corroborating_readings(candidates, body) == []
+
+
+# --- fix round 2: a DOI-only deposit abstains on EVERY DOI shape -----------
+
+
+def test_a_doi_only_deposit_abstains_whatever_its_doi_tokenises_to():
+ """Critical 1, reopened and closed properly.
+
+ `_reference_raw`'s fallback is the DOI string itself — bare, not a URL —
+ and `[a-z]{5,}` lifts a journal slug out of most publisher DOIs. Only the
+ third shape here has no five-letter run, and it was the only one the
+ first fix reached: the other two came back `False` and disputed every
+ label they voted on, which is the audit-with-no-verdicts outcome the
+ finding named.
+ """
+ page = _printed("7")
+ for doi in ("10.1148/radiol.2019181432", # tokenised to {'radiol'}
+ "10.1001/jamanetworkopen.2023.18153", # {'jamanetworkopen'}
+ "10.1038/s41591-019-0673-2", # set()
+ "10.1016/j.neuroimage.2020.117161", # {'neuroimage'}
+ "10.1093/bioinformatics/btaa123"): # {'bioinformatics'}
+ deposit = RefEntry(num="7", raw=doi, doi=doi)
+ assert _comparably_same(deposit, page) is None, doi
+ assert label_agreement({"crossref": [deposit], "parsed": [page]}, {"7"}) == {
+ "7": "single"
+ }, doi
+ assert label_agreement(
+ {"crossref": [deposit], "parsed": [page], "pymupdf": [_printed("7")]}, {"7"}
+ ) == {"7": "agreed"}, doi
+
+
+def test_two_doi_only_deposits_still_compare_by_doi_and_never_reach_the_mute_cell():
+ """The ordering the `_comparably_same` branches rest on, re-checked under
+ the corrected tokeniser: with DOIs stripped, two deposits have no tokens
+ on either side — but they never get that far, because the DOI comparison
+ comes first. Only an entry with NEITHER a DOI nor a title word is mute."""
+ a = RefEntry(num="7", raw="10.1148/radiol.2019181432", doi="10.1148/radiol.2019181432")
+ b_same = RefEntry(num="7", raw="10.1148/radiol.2019181432", doi="10.1148/radiol.2019181432")
+ b_other = RefEntry(num="7", raw="10.1038/nature12373", doi="10.1038/nature12373")
+
+ assert _comparably_same(a, b_same) is True
+ assert _comparably_same(a, b_other) is False
+ assert label_agreement({"crossref": [a], "parsed": [b_same]}, {"7"}) == {"7": "agreed"}
+ assert label_agreement({"crossref": [a], "parsed": [b_other]}, {"7"}) == {"7": "disputed"}
+ # and the mute cell still needs an entry carrying neither
+ mute = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
+ assert label_agreement(
+ {"parsed": [mute], "pymupdf": [RefEntry(num="7", raw="Q. Z. 2020. 88(1):4-7.")]}, {"7"}
+ ) == {"7": "disputed"}
+
+
+# --- fix round 2: which of the two causes put a label in dispute -----------
+
+
+def test_uncomparable_labels_names_only_the_labels_nothing_could_compare():
+ """`disputed` has two causes and they warrant different reader actions:
+ a contradiction means one reading is wrong and a human should look;
+ nothing comparable means no conflict is known and the withholding is
+ precautionary. [1] is a contradiction, [2] is uncomparable, [3] agrees."""
+ body = {"1", "2", "3"}
+ candidates = {
+ "parsed": [_work("1", "10.1000/x1"),
+ RefEntry(num="2", raw="A. B. 2020. 14(3):1-9."),
+ _work("3", "10.1000/x3")],
+ "pymupdf": [_work("1", "10.1000/x9"),
+ RefEntry(num="2", raw="Q. Z. 2020. 88(1):4-7."),
+ _work("3", "10.1000/x3")],
+ }
+ assert label_agreement(candidates, body) == {
+ "1": "disputed", "2": "disputed", "3": "agreed",
+ }
+ assert uncomparable_labels(candidates, body) == ["2"]
+
+
+def test_uncomparable_labels_is_always_a_subset_of_the_disputed_ones():
+ """The invariant, pinned rather than assumed. Major 3 on this branch was
+ exactly two published fields disagreeing on one manifest, and this pair
+ is the next candidate: both come out of `_label_state`, which is the one
+ place the rule lives, and a label can only be uncomparable by way of
+ being disputed."""
+ body = {"1", "2", "3", "4"}
+ candidates = {
+ "parsed": [_work("1", "10.1000/x1"),
+ RefEntry(num="2", raw="A. B. 2020. 14(3):1-9."),
+ _work("3", "10.1000/x3"), _work("4", "10.1000/x4")],
+ "pymupdf": [_work("1", "10.1000/x9"),
+ RefEntry(num="2", raw="Q. Z. 2020. 88(1):4-7."),
+ _work("3", "10.1000/x3"), _work("4", "10.1000/x4"),
+ _work("4", "10.1000/x44")],
+ }
+ agreement = label_agreement(candidates, body)
+ disputed = {lbl for lbl, state in agreement.items() if state == "disputed"}
+ assert disputed == {"1", "2", "4"}, agreement
+ assert set(uncomparable_labels(candidates, body)) <= disputed
+
+
+def test_a_duplicate_is_disputed_for_its_own_reason_not_an_uncomparable_one():
+ """A reading carrying the label twice is judgeable — two papers, one of
+ which would really be downloaded — which is a different finding from
+ nothing being comparable, and must not be filed under it."""
+ candidates = {
+ "parsed": [_work("7", "10.1000/x7"), _work("7", "10.1000/x77")],
+ "pymupdf": [_work("7", "10.1000/x7")],
+ }
+ assert label_agreement(candidates, {"7"}) == {"7": "disputed"}
+ assert uncomparable_labels(candidates, {"7"}) == []
diff --git a/tests/test_numbering_invariant.py b/tests/test_numbering_invariant.py
new file mode 100644
index 0000000..61fff28
--- /dev/null
+++ b/tests/test_numbering_invariant.py
@@ -0,0 +1,516 @@
+"""One flag unmoved, through the real pipeline, read back off the artifact.
+
+`numbering_verified` has a published meaning (`schemas/refs_manifest.schema.json`):
+the chosen list accounts for exactly the labels the body cites, and no entry
+refuses its own label. It is a test of EXTENT. A model's reading of the
+bibliography is not that, and neither is a person answering a prompt — so this
+module runs `cli._refs_pipeline` over every shape a model reply can take and
+every choice the escalation offers, and asserts against the
+`refs_manifest.json` each run actually wrote.
+
+The seam on one side, the artifact on the other, and the production wiring in
+between. That shape is the whole point of the rewrite: the previous version
+built `candidates` and called `reconcile` by hand, so "the model's reading
+never becomes one of `reconcile`'s arguments" was enforced by the test's own
+source text rather than by the product's. Measured — handing the `llm` reading
+to `reconcile` in `_refs_pipeline`, the design's central prohibition, left all
+29 of its cells green. Never re-create the pipeline's wiring here; a cell that
+does can only ever confirm its own re-creation.
+
+"Unmoved" is not "always False". `_covers` is an extent check, and a fixture
+whose extent legitimately matches gets `verified = True` — asserting `False`
+there asserts an impossible value (three assertions in
+`tests/test_escalation.py` had to be corrected for exactly that). So every
+reply and every choice is run against BOTH a fixture that legitimately
+verifies and one that legitimately does not, and the assertion is that neither
+moves off the value the extent check alone produced.
+
+A reply IS allowed to do one thing: withhold. So each shape also carries what
+it is allowed to have moved — `labels_disputed` and `numbering_corroborated` —
+because an invariant asserted alone cannot tell "the reply changed nothing"
+from "the reply never reached the code". `reflist_outcome == "read"` is
+asserted for the same reason, on every cell.
+"""
+
+import json
+import re
+import sys
+from pathlib import Path
+
+import pymupdf
+import pytest
+import typer
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+
+from papertrace import ask as ask_mod # noqa: E402
+from papertrace import cli as cli_mod # noqa: E402
+from papertrace import reflist as reflist_mod # noqa: E402
+from papertrace import refs as refs_mod # noqa: E402
+
+SCHEMA_PATH = Path(__file__).resolve().parent.parent / "schemas" / "refs_manifest.schema.json"
+
+
+def _paper(path: Path, *, extra_uncited: bool = False) -> Path:
+ """A one-page PDF citing [1] and [2], with both references printed.
+
+ `extra_uncited=True` prints a THIRD, uncited [3]: the parse then carries
+ three entries against two cited labels, `_covers` requires equality, and
+ `verified` is correctly False. Without that flag `_covers` is satisfied by
+ the parse alone and `verified` is correctly True. The two together are the
+ only way a cell can show a reply moved the flag in EITHER direction.
+
+ Written out again rather than imported from `tests/test_reference_readings.py`:
+ `tests/` has no `conftest.py` by house rule and no test module imports
+ another, so the alternative is not a shared fixture but a hidden
+ dependency between two files that are meant to fail independently.
+ """
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "A Fixture Imaging Study", fontsize=16)
+ page.insert_text((72, 140), "Body text citing [1] and also [2] here.", fontsize=11)
+ page.insert_text((72, 200), "References", fontsize=14)
+ page.insert_text((72, 230), "[1] Alpha A. A first paper. J Fixture. 2020;1:1-9.", fontsize=11)
+ page.insert_text((72, 250), "[2] Beta B. A second paper. J Fixture. 2021;2:10-19.", fontsize=11)
+ if extra_uncited:
+ page.insert_text((72, 270), "[3] Gamma G. A third, uncited paper. J Fixture. 2019;3:1-5.",
+ fontsize=11)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ doc.save(path)
+ doc.close()
+ return path
+
+
+# What the printed list carries, per extent: the numerals, and the opening of
+# each entry's `raw`. A model-composed entry is re-assembled from verified
+# FIELDS (`reflist.propose`: authors, year, title, journal, doi joined by ". "),
+# so it can never reproduce the printed string — which is what makes these two
+# a check that the manifest's entries came off the page and not out of a reply.
+_PRINTED = {
+ "covers": (["1", "2"], ["Alpha A.", "Beta B."]),
+ "extra_reference": (["1", "2", "3"], ["Alpha A.", "Beta B.", "Gamma G."]),
+}
+
+# The marker that tells the two model calls apart inside one `ask._ask` double.
+# Asserted to be real, and to be unique, by its own test below — a double that
+# mis-routes would answer the resolution call with a reading reply and every
+# cell would still be green.
+_RESOLVE_MARKER = "LABELS IN DISPUTE:"
+
+
+def _seam(reading: str, resolution: str = "[]"):
+ """One `ask._ask` standing in for both calls the refs stage can make.
+
+ `reflist.py` has no module-level `_ask` of its own — it does `from . import
+ ask` and calls `ask._ask` qualified (`reflist.py:267-269` says so), so a
+ monkeypatch on `reflist._ask` sets an attribute nothing reads and lets the
+ real `claude -p` subprocess run. This module patches `ask._ask` and nothing
+ else, which is also what keeps `reflist.propose` and `reflist.resolve_disputed`
+ themselves REAL: every verbatim check they perform is exercised by these
+ cells rather than stubbed past.
+ """
+
+ def _ask(prompt, model=None):
+ return resolution if _RESOLVE_MARKER in prompt else reading
+
+ return _ask
+
+
+def _wire_offline(monkeypatch, *, reading: str, resolution: str = "[]", ask=None) -> None:
+ """No network and no model call: retrieval is a no-op, Crossref answers
+ nothing, and `claude` is treated as present regardless of the host.
+
+ That last patch is load-bearing even though the seam below is what answers:
+ `_llm_reference_reading` checks `ask.claude_available()` BEFORE calling
+ `propose`, so on a machine with no `claude` CLI on PATH — CI, but not a
+ developer's laptop — every cell would silently take the "not attempted"
+ branch and the reply would never be read at all.
+
+ `_needs_flat_reading` is forced rather than the backend changed. These
+ tests ingest with `backend="pymupdf"` for speed (docling downloads ~500 MB
+ on first use), and on a genuine pymupdf run `_refs_pipeline` skips the
+ model call by design — so without this patch there is no reply to test.
+ Force the PREDICATE, not the backend.
+ """
+ monkeypatch.setattr(refs_mod, "resolve_all",
+ lambda entries, dest, email, provided_dir=None, progress=None,
+ taken=None: entries)
+ monkeypatch.setattr(refs_mod, "crossref_deposit",
+ lambda client, doi, email: refs_mod.CrossrefDeposit(
+ absent="no deposit, in a test"))
+ monkeypatch.setattr(cli_mod, "_detected_doi", lambda m: None)
+ monkeypatch.setattr(ask_mod, "claude_available", lambda: True)
+ monkeypatch.setattr(cli_mod, "_needs_flat_reading", lambda smap: True)
+ monkeypatch.setattr(ask_mod, "_ask", ask or _seam(reading, resolution))
+
+
+def _run(tmp_path: Path, monkeypatch, *, extent: str, reading: str,
+ resolution: str = "[]", ask=None) -> Path:
+ """Run the real refs stage on the fixture paper; return the case folder.
+
+ The case folder, not the manifest: menu choice 4 aborts before a manifest
+ is written, and "no manifest exists" is that cell's assertion.
+ """
+ _wire_offline(monkeypatch, reading=reading, resolution=resolution, ask=ask)
+ pdf = _paper(tmp_path / "paper.pdf", extra_uncited=(extent == "extra_reference"))
+ case = tmp_path / "case"
+ cli_mod._refs_pipeline(manuscript=pdf, case=case, provided=None,
+ email="test@example.org", parse_only=False,
+ backend="pymupdf", llm_refs=True)
+ return case
+
+
+def _manifest(case: Path) -> dict:
+ """The written artifact, schema-validated on the way out — Gate 2 for free."""
+ import jsonschema
+
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ jsonschema.validate(payload, json.loads(SCHEMA_PATH.read_text()))
+ return payload
+
+
+# Every shape a reply can take, named by what is wrong with it, with what that
+# shape is ALLOWED to have moved. `agrees_with_the_parse` is in here
+# deliberately: the dangerous reply is not the malformed one — that fails
+# loudly — it is the well-formed one confirming the parse, because that is the
+# reply a reader is most tempted to promote to a check.
+#
+# `disputed` and `corroborated` are the only two things a reading may move, and
+# they are here rather than asserted uniformly because an invariant asserted
+# alone cannot tell "the reply changed nothing" from "the reply never ran". The
+# values were observed, not predicted: a shape whose entries all fail
+# verification never becomes a voter at all, so `parsed` and `pymupdf` carry
+# the corroboration by themselves.
+REPLIES = {
+ "agrees_with_the_parse": {
+ "reply": json.dumps([
+ {"num": "1", "title": "A first paper", "year": "2020",
+ "journal": "J Fixture", "reading": "AB"},
+ {"num": "2", "title": "A second paper", "year": "2021",
+ "journal": "J Fixture", "reading": "AB"},
+ ]),
+ "disputed": [], "corroborated": True,
+ },
+ # the one shape `reflist._found` exists for: a title nobody printed. The
+ # whole reading must be refused, so this reply casts no vote — if the
+ # verbatim check were disabled it would dispute [2] instead.
+ "names_a_paper_the_page_does_not_print": {
+ "reply": json.dumps([
+ {"num": "1", "title": "A first paper", "year": "2020"},
+ {"num": "2", "title": "Characterization of Brain Volume Changes", "year": "2021"},
+ ]),
+ "disputed": [], "corroborated": True,
+ },
+ "abstains": {
+ "reply": json.dumps([{"num": "1", "cannot_tell": True},
+ {"num": "2", "cannot_tell": True}]),
+ "disputed": [], "corroborated": True,
+ },
+ "malformed_json": {
+ "reply": "here is what I think: [1] is Alpha, probably",
+ "disputed": [], "corroborated": True,
+ },
+ "empty_array": {"reply": "[]", "disputed": [], "corroborated": True},
+ "numeral_only_no_fields": {
+ "reply": json.dumps([{"num": "1"}, {"num": "2"}]),
+ "disputed": [], "corroborated": True,
+ },
+ # `DOI_RE` accepts a trailing hyphen and the prefix really is printed, so
+ # only `_usable_doi`'s last-character rule tells half a DOI from a whole one
+ "truncated_doi": {
+ "reply": json.dumps([{"num": "2", "title": "A second paper",
+ "doi": "10.1038/s41591-"}]),
+ "disputed": [], "corroborated": True,
+ },
+ "a_label_nobody_cites": {
+ "reply": json.dumps([{"num": "99", "title": "A second paper"}]),
+ "disputed": [], "corroborated": True,
+ },
+ # the one shape that legitimately withholds: a reading carrying one label
+ # twice cannot say which paper it names, which is `label_agreement`'s own
+ # rule, and a withheld verdict is the ONLY thing a reply is allowed to cause
+ "duplicate_numeral": {
+ "reply": json.dumps([{"num": "1", "title": "A first paper"},
+ {"num": "1", "title": "A second paper"}]),
+ "disputed": ["1"], "corroborated": False,
+ },
+}
+
+
+@pytest.mark.parametrize("extent", ["covers", "extra_reference"])
+@pytest.mark.parametrize("shape", sorted(REPLIES))
+def test_no_model_reply_shape_can_set_numbering_verified(shape, extent, tmp_path, monkeypatch):
+ """Every reply shape, through `_refs_pipeline`, asserted on the manifest.
+
+ The model's reading is a voter and nothing else: it never becomes one of
+ `reconcile`'s arguments, so no reply can cause a source to be resolved,
+ downloaded or judged, and no reply can move `numbering_verified` off the
+ value the extent check alone produced. The `covers` cell proves a reply
+ cannot manufacture a False from a legitimate True; the `extra_reference`
+ cell proves it cannot manufacture a True from a legitimate False, which is
+ the direction that would actually be dangerous.
+
+ `expected` is hardcoded from the extent rather than derived. Deriving it
+ from a second `reconcile` call would be tautological — it would assert the
+ extent check agrees with itself — and `_covers`' rule ("as many entries as
+ the highest cited label") is exactly what the two fixtures were built to
+ straddle.
+ """
+ case = _run(tmp_path, monkeypatch, extent=extent, reading=REPLIES[shape]["reply"])
+ payload = _manifest(case)
+
+ # the reply reached the code. Without this the cells below are satisfied by
+ # a run in which no model was ever asked, which is not what they claim.
+ assert payload["reflist_outcome"] == "read", f"{shape}: the reply never reached propose"
+
+ expected = extent == "covers"
+ assert payload["numbering_verified"] is expected, f"{shape}/{extent} moved numbering_verified"
+
+ # withholding is the one thing a reply may cause, and only on a cause it
+ # demonstrated — a reply whose fields are not printed casts no vote at all
+ assert payload["labels_disputed"] == REPLIES[shape]["disputed"], shape
+ assert payload["numbering_corroborated"] is REPLIES[shape]["corroborated"], shape
+ assert "llm" not in payload["corroborating_readings"], (
+ "a model reading may only copy from the two extractions, which vote in "
+ "their own right — it can dispute, it cannot corroborate"
+ )
+
+ # the resolved list is the PRINTED one, entry for entry
+ nums, openings = _PRINTED[extent]
+ assert [e["num"] for e in payload["entries"]] == nums, shape
+ assert [e["raw"][:len(o)] for e, o in zip(payload["entries"], openings, strict=True)] == \
+ openings, f"{shape}: an entry in the manifest did not come off the page"
+
+
+@pytest.mark.parametrize("extent", ["covers", "extra_reference"])
+@pytest.mark.parametrize("choice", ["1", "2", "3", "4"])
+def test_no_interactive_choice_can_set_numbering_verified(choice, extent, tmp_path, monkeypatch):
+ """Every menu choice, through `_refs_pipeline`, asserted on the manifest.
+
+ The dispute is REAL: the `duplicate_numeral` reply makes `label_agreement`
+ refuse to let the model reading speak for [1], which is what puts the label
+ in `labels_disputed` and what makes `_refs_pipeline` escalate at all. No
+ `rec` is built by hand and no `labels_disputed` is assigned by the test.
+
+ `chosen_by: "user"` is what makes the escalation honest — a person
+ consenting to proceed is an input, not evidence. So every branch must leave
+ the flag no more true than `reconcile` left it. Option 3 is the one branch
+ that touches it, and by design in one direction only: it substitutes a
+ whole different list than the one `reconcile` measured, so it resets
+ `verified` to the literal `False`. Asserted from both starting values to
+ show the reset can never produce `True`.
+
+ Choice 4 aborts before the manifest is written, so its assertion is that no
+ manifest exists — and that the evidence file written before the menu still
+ does, because "nothing was written" means the manifest and nothing else.
+ """
+ answers = iter([choice, "parsed"])
+
+ def _prompt(*args, **kwargs):
+ answer = next(answers)
+ # the real menu refuses an answer it did not offer, and a double that
+ # does not is a double more permissive than the thing it stands for —
+ # the exact shape that hid a Critical earlier in this plan
+ assert answer in kwargs.get("choices", [answer]), (
+ f"the menu did not offer {answer!r}; it offered {kwargs.get('choices')}"
+ )
+ return answer
+
+ monkeypatch.setattr(cli_mod, "_interactive", lambda: True)
+ monkeypatch.setattr(cli_mod.Prompt, "ask", staticmethod(_prompt))
+
+ kw = dict(extent=extent, reading=REPLIES["duplicate_numeral"]["reply"],
+ # a resolution that settles [1] on a title printed on the page,
+ # so choice 1 takes the branch that substitutes an entry rather
+ # than the abstention branch, which touches less
+ resolution=json.dumps([{"num": "1", "title": "A first paper",
+ "year": "2020", "seen_in": "A"}]))
+ if choice == "4":
+ with pytest.raises(typer.Exit):
+ _run(tmp_path, monkeypatch, **kw)
+ assert not (tmp_path / "case" / "refs_manifest.json").exists(), (
+ "an abort must leave no manifest for a later stage to trust"
+ )
+ assert (tmp_path / "case" / "out" / "reference_disagreement.md").exists(), (
+ "the evidence file is written before the menu and survives the abort"
+ )
+ return
+
+ payload = _manifest(_run(tmp_path, monkeypatch, **kw))
+
+ expected = False if choice == "3" else (extent == "covers")
+ assert payload["numbering_verified"] is expected, f"choice {choice} moved numbering_verified"
+ assert payload["numbering_chosen_by"] == "user"
+ assert payload["numbering_choice"] == {"1": "llm_resolved", "2": "withheld",
+ "3": "parsed"}[choice]
+
+
+@pytest.mark.parametrize("extent", ["covers", "extra_reference"])
+def test_a_resolution_call_that_raises_cannot_set_numbering_verified(
+ extent, tmp_path, monkeypatch
+):
+ """Option 1's `except Exception` is where a live subprocess failure
+ (timeout, non-zero exit, unparseable stdout) surfaces mid-escalation. A
+ failed call is a third outcome, not a licence to guess a verdict: it must
+ leave the flag exactly where it found it, and say that it failed.
+
+ The failure is injected at `ask._ask`, the seam that really raises — NOT by
+ stubbing `reflist.resolve_disputed`. `resolve_disputed` calls `ask._ask`
+ with no `try` of its own, so raising at the seam reaches the same `cli`
+ branch with the real resolver in the loop. A stub there would keep this
+ green if `resolve_disputed` ever grew its own broad `except` and returned a
+ `Resolution` carrying confabulated entries — the double outliving the code
+ it stands for, which is this plan's most expensive scar.
+ """
+ def _raising(prompt, model=None):
+ if _RESOLVE_MARKER in prompt:
+ raise RuntimeError("claude -p failed: simulated for this test")
+ return REPLIES["duplicate_numeral"]["reply"]
+
+ monkeypatch.setattr(cli_mod, "_interactive", lambda: True)
+ monkeypatch.setattr(cli_mod.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+
+ payload = _manifest(_run(tmp_path, monkeypatch, extent=extent,
+ reading="unused — the seam below answers", ask=_raising))
+
+ assert payload["numbering_verified"] is (extent == "covers"), (
+ "a resolution call that raised moved numbering_verified"
+ )
+ assert payload["resolution_outcome"] == "failed"
+ assert payload["labels_disputed"] == ["1"], "a failed call resolves nothing"
+ assert payload["labels_resolved"] == []
+ assert payload["numbering_chosen_by"] == "user" # a person consented to trying
+ nums, openings = _PRINTED[extent]
+ assert [e["num"] for e in payload["entries"]] == nums
+
+
+def test_corroboration_can_be_true_in_the_same_run_that_keeps_verified_false(
+ tmp_path, monkeypatch
+):
+ """The two axes must be independently settable, or the second is pointless.
+
+ This is the 22-vs-19 case the corroboration axis exists for: every cited
+ label is read the same way by two readings, and `_covers` still fails
+ because a reference is printed that the body never cites. Before the second
+ axis, that run could say nothing but "unconfirmed" — crying wolf on a
+ numbering two independent readings agreed about entry for entry.
+
+ Kept as its own cell although the parametrisation above covers the same
+ run: the parametrisation asserts the two values against a table, and what
+ is being claimed here is the RELATION between them — that one is False
+ while the other is True in one manifest. A table cell cannot say that, and
+ a future edit narrowing the table would not notice it had stopped.
+
+ The model read and agreed here, and is still not named among the
+ corroborating readings: `reflist.propose` may only copy from the two
+ extractions, and both of those already vote in their own right, so its
+ agreement is one text read twice.
+ """
+ payload = _manifest(_run(tmp_path, monkeypatch, extent="extra_reference",
+ reading=REPLIES["agrees_with_the_parse"]["reply"]))
+
+ assert payload["reflist_outcome"] == "read"
+ assert payload["numbering_verified"] is False
+ assert payload["numbering_corroborated"] is True
+ assert payload["corroborating_readings"] == ["parsed", "pymupdf"]
+
+
+def test_the_two_model_calls_are_told_apart_by_a_marker_only_one_prompt_carries():
+ """The `ask._ask` double routes on `_RESOLVE_MARKER`, so the marker has to
+ be in one prompt and only one.
+
+ If `RESOLVE_PROMPT` ever stopped carrying it, every cell above would answer
+ the resolution call with a reading reply — the resolution would be read as
+ unparseable, the escalation would take its abstention branch, and the whole
+ suite would stay green while testing a branch it does not name. A double is
+ only as honest as the thing it discriminates on.
+ """
+ assert _RESOLVE_MARKER in reflist_mod.RESOLVE_PROMPT
+ assert _RESOLVE_MARKER not in reflist_mod.REFLIST_PROMPT
+
+
+_SRC = Path(__file__).resolve().parent.parent / "src" / "papertrace"
+# the names that may only ever be set from the extent check
+_GUARDED = ("numbering_verified", "rec.verified", ".verified")
+# what must never appear on the right-hand side of one. `corroborated` is in
+# here because corroboration is the OTHER axis: `numbering_verified=rec.verified
+# or rec.corroborated` is, literally, a model reply setting the flag — a model
+# reading is a corroboration voter.
+_FORBIDDEN = ("reflist", "propose", "resolve_disputed", "Prompt.ask", "_ask(",
+ "answer", "choice", "chosen_by", "res.resolved", "corroborated")
+
+# The first `=` that is an assignment: not `==`, `!=`, `<=` or `>=`. Splitting
+# on it beats discarding any line that contains a comparison — under that older
+# rule `rec.verified = bool(res.resolved) if answer == "1" else rec.verified`,
+# the docstring's own forbidden spelling plus a ternary, was never examined.
+_ASSIGN = re.compile(r"(?])=(?!=)")
+
+
+def test_no_source_file_assigns_numbering_verified_from_a_model_or_a_prompt():
+ """A grep, and honest about being one.
+
+ What this proves: no source line *textually* assigns one of the guarded
+ names from an expression mentioning the reflist module, a prompt answer,
+ the seam, or the corroboration axis. That catches the changes a future task
+ is actually likely to make — `rec.verified = bool(res.resolved)`, written
+ in good faith by someone who read "resolved" as "checked", and
+ `numbering_verified=rec.verified or rec.corroborated`, written by someone
+ who read the two axes as one.
+
+ What it cannot prove: that no value *derived* from a model reply reaches
+ the flag through an intermediate variable, a helper, a dict or a
+ `dataclasses.replace`. A grep has no dataflow, and every one of those
+ spellings walks straight through it. The parametrised tests above are what
+ cover that — they run `_refs_pipeline` itself and read the manifest it
+ wrote, so a value arriving by any route at all shows up in the artifact.
+ This one is the cheap tripwire that fails at review time rather than at
+ runtime, and it is worth having for that and no more.
+ """
+ offenders = []
+ for path in sorted(_SRC.rglob("*.py")):
+ for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+ code = line.split("#", 1)[0]
+ if not (m := _ASSIGN.search(code)):
+ continue
+ lhs, rhs = code[:m.start()], code[m.end():]
+ if not any(g in lhs for g in _GUARDED):
+ continue
+ if any(f in rhs for f in _FORBIDDEN):
+ offenders.append(f"{path.relative_to(_SRC)}:{lineno}: {line.strip()}")
+ assert offenders == [], "\n".join(offenders)
+
+
+def test_the_guard_above_would_actually_catch_the_changes_it_describes(tmp_path):
+ """A guard test nobody has seen fail is a guard test nobody knows works.
+
+ The grep is re-run over a file holding the exact lines the real test
+ forbids — including the two the previous rule missed: a ternary (skipped
+ wholesale for containing `==`) and the corroboration widening (whose
+ spelling was not in `_FORBIDDEN` at all). Every planted line must be
+ caught, so a narrowing of either constant fails here rather than silently.
+
+ Written out again rather than factored into a shared helper: a helper
+ broken the same way would make both tests pass together, which is the
+ failure mode a self-test exists to exclude.
+ """
+ planted = [
+ 'rec.verified = bool(res.resolved)',
+ 'rec.verified = bool(res.resolved) if answer == "1" else rec.verified',
+ ' numbering_verified=rec.verified or rec.corroborated,',
+ ]
+ bad = tmp_path / "papertrace" / "bad.py"
+ bad.parent.mkdir(parents=True)
+ bad.write_text("\n".join(planted) + "\n", encoding="utf-8")
+
+ offenders = []
+ for path in sorted(bad.parent.rglob("*.py")):
+ for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+ code = line.split("#", 1)[0]
+ if not (m := _ASSIGN.search(code)):
+ continue
+ lhs, rhs = code[:m.start()], code[m.end():]
+ if any(g in lhs for g in _GUARDED) and any(f in rhs for f in _FORBIDDEN):
+ offenders.append(lineno)
+ assert offenders == list(range(1, len(planted) + 1)), (
+ f"the grep caught {offenders} of {len(planted)} lines it exists to catch"
+ )
diff --git a/tests/test_pipeline_split.py b/tests/test_pipeline_split.py
index f4035ef..c8378db 100644
--- a/tests/test_pipeline_split.py
+++ b/tests/test_pipeline_split.py
@@ -16,7 +16,9 @@
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+from papertrace import check as check_mod # noqa: E402
from papertrace import cli # noqa: E402
+from papertrace.models import ClaimExtraction, RefEntry, RefManifest # noqa: E402
def test_ingest_pipeline_rejects_a_positional_call():
@@ -55,12 +57,13 @@ def fake_pipeline(**kw):
pdf.write_bytes(b"%PDF-1.4\n")
cli.refs(manuscript=pdf, case=None, provided=None, email=None,
- parse_only=False, backend="auto", doi=None, supplement=None,
+ parse_only=False, backend="auto", doi=None, supplement=None, llm_refs=True,
max_claims=None, max_sources=None)
assert seen == {
"manuscript": pdf, "case": None, "provided": None, "email": None,
"parse_only": False, "backend": "auto", "doi": None, "supplement": None,
+ "llm_refs": True,
"claims": None, "max_sources": None,
}
@@ -176,6 +179,119 @@ def test_check_claims_will_not_default_its_backend():
assert p.kind is inspect.Parameter.KEYWORD_ONLY
+# --- withholding a disputed label -------------------------------------------
+# The wire the `check_claims` tests cannot see for themselves. `_check_pipeline`
+# is where `RefManifest.labels_disputed` (Task 3) actually reaches
+# `check_claims`'s `disputed` parameter (Task 4); `check_claims` can only prove
+# what happens once the argument has arrived.
+
+
+def test_check_pipeline_passes_the_manifests_disputed_labels_to_check_claims(
+ tmp_path, monkeypatch
+):
+ """If this regresses, a label Task 3 marked disputed reaches judgement
+ anyway — the withholding built into `check_claims` never fires, because
+ the CLI never told it which labels to withhold."""
+ case = tmp_path / "case"
+ case.mkdir()
+ RefManifest(
+ manuscript="m.pdf",
+ entries=[RefEntry(num="9", raw="ref", status="retrieved", slug="x-2020",
+ pdf_path="/nonexistent/x-2020.pdf")],
+ labels_disputed=["9"],
+ ).to_json(case / "refs_manifest.json")
+
+ # Patch the GATE, not the predicate behind it. `_check_pipeline` calls
+ # `cli._require_claude()`, which does `from .check import claude_available`
+ # — and `check.claude_available` is a re-export, a *separate module
+ # attribute* from `ask.claude_available` even though both name one
+ # function. Patching `ask_mod` left the one this path reads untouched, so
+ # the test passed on a machine with the CLI installed and failed all ten
+ # CI jobs, which have none. Patching `_require_claude` itself cannot
+ # drift with a future refactor of which module it imports from.
+ monkeypatch.setattr(cli, "_require_claude", lambda: None)
+ # `_check_pipeline` reads the extraction through `cli._extraction_for` now
+ # — the `limits` feature made extraction a stage of its own so a claims
+ # limit can retrieve only what the selected claims cite, and stubbing
+ # `check.extract_claims` no longer intercepts anything this path calls
+ monkeypatch.setattr(
+ cli, "_extraction_for",
+ lambda case, model: ClaimExtraction(
+ manuscript="m.pdf", manuscript_sha256=None, extractor="test", date="2026-01-01",
+ ),
+ )
+ # a realistic empty audit, not a bare `{}`: `coverage_audit` never returns
+ # a dict without `missing` — `coverage["missing"]` two lines into the
+ # summary print is unrelated to what this test is proving and must not be
+ # what breaks it
+ monkeypatch.setattr(
+ check_mod, "coverage_audit",
+ lambda *a, **kw: {"labels_in_text": [], "covered": [], "missing": []},
+ )
+ monkeypatch.setattr(check_mod, "last_model", lambda: None)
+
+ captured = {}
+
+ def fake_check_claims(claims, manifest, case_dir, model, *, progress=None,
+ on_error=None, truncations=None, backend, disputed=None):
+ captured["disputed"] = disputed
+ return claims
+
+ monkeypatch.setattr(check_mod, "check_claims", fake_check_claims)
+
+ cli._check_pipeline(case=case, backend="pymupdf")
+
+ assert captured["disputed"] == {"9"}
+
+
+def test_check_pipeline_tolerates_a_manifest_with_nothing_disputed(tmp_path, monkeypatch):
+ """`labels_disputed` defaults to `[]` — the ordinary case, every run before
+ Task 3 shipped and every run where nothing disagreed. `set([])` must reach
+ `check_claims` rather than `None`, `set()` behaves identically either way
+ (proved in `test_check.py`), but this is the one place that constructs it."""
+ case = tmp_path / "case"
+ case.mkdir()
+ RefManifest(manuscript="m.pdf", entries=[]).to_json(case / "refs_manifest.json")
+
+ # Patch the GATE, not the predicate behind it. `_check_pipeline` calls
+ # `cli._require_claude()`, which does `from .check import claude_available`
+ # — and `check.claude_available` is a re-export, a *separate module
+ # attribute* from `ask.claude_available` even though both name one
+ # function. Patching `ask_mod` left the one this path reads untouched, so
+ # the test passed on a machine with the CLI installed and failed all ten
+ # CI jobs, which have none. Patching `_require_claude` itself cannot
+ # drift with a future refactor of which module it imports from.
+ monkeypatch.setattr(cli, "_require_claude", lambda: None)
+ # `_check_pipeline` reads the extraction through `cli._extraction_for` now
+ # — the `limits` feature made extraction a stage of its own so a claims
+ # limit can retrieve only what the selected claims cite, and stubbing
+ # `check.extract_claims` no longer intercepts anything this path calls
+ monkeypatch.setattr(
+ cli, "_extraction_for",
+ lambda case, model: ClaimExtraction(
+ manuscript="m.pdf", manuscript_sha256=None, extractor="test", date="2026-01-01",
+ ),
+ )
+ monkeypatch.setattr(
+ check_mod, "coverage_audit",
+ lambda *a, **kw: {"labels_in_text": [], "covered": [], "missing": []},
+ )
+ monkeypatch.setattr(check_mod, "last_model", lambda: None)
+
+ captured = {}
+
+ def fake_check_claims(claims, manifest, case_dir, model, *, progress=None,
+ on_error=None, truncations=None, backend, disputed=None):
+ captured["disputed"] = disputed
+ return claims
+
+ monkeypatch.setattr(check_mod, "check_claims", fake_check_claims)
+
+ cli._check_pipeline(case=case, backend="pymupdf")
+
+ assert captured["disputed"] == set()
+
+
# --- the extract stage, and the limits `run` hands down ----------------------
#
# Extraction is a stage of its own now: `run` extracts BEFORE resolving
diff --git a/tests/test_reference_readings.py b/tests/test_reference_readings.py
new file mode 100644
index 0000000..4edcea3
--- /dev/null
+++ b/tests/test_reference_readings.py
@@ -0,0 +1,1740 @@
+"""Four readings of one bibliography, and the one thing the extra two may do.
+
+`reconcile` still chooses between the Crossref deposit and the run backend's
+parse. The flat-text reading and the model's reading are voters on a second
+axis: they can cause a verdict to be withheld and nothing else. The first test
+here is the load-bearing one — it is the whole safety property of the feature,
+stated as an equality.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+import pymupdf
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+
+from papertrace import cli # noqa: E402
+from papertrace import reflist as reflist_mod # noqa: E402
+from papertrace.models import RefEntry, RefManifest, SourceMap # noqa: E402
+from papertrace.refs import _entry # noqa: E402
+
+SCHEMA_PATH = Path(__file__).resolve().parent.parent / "schemas" / "refs_manifest.schema.json"
+
+
+def _paper(path: Path, *, extra_uncited: bool = False) -> Path:
+ """A one-page PDF citing [1] and [2], with both references printed.
+
+ `extra_uncited=True` also prints a THIRD, uncited [3] — used by exactly one
+ test (`test_no_model_reply_can_set_numbering_verified`) where the parse
+ alone must fail `reconcile`'s own extent check (`_covers`), because on the
+ plain two-reference fixture that check is satisfied by the parse alone and
+ nothing about the model reading could be shown to matter (see that test's
+ docstring). One fixture with a flag, not two independent copies that would
+ otherwise be free to drift apart.
+ """
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "A Fixture Imaging Study", fontsize=16)
+ page.insert_text((72, 140), "Body text citing [1] and also [2] here.", fontsize=11)
+ page.insert_text((72, 200), "References", fontsize=14)
+ page.insert_text((72, 230), "[1] Alpha A. A first paper. J Fixture. 2020;1:1-9.", fontsize=11)
+ page.insert_text((72, 250), "[2] Beta B. A second paper. J Fixture. 2021;2:10-19.", fontsize=11)
+ if extra_uncited:
+ page.insert_text((72, 270), "[3] Gamma G. A third, uncited paper. J Fixture. 2019;3:1-5.",
+ fontsize=11)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ doc.save(path)
+ doc.close()
+ return path
+
+
+def _work(num: str, doi: str) -> RefEntry:
+ """Built through `_entry`, the production parse path — not by setting
+ fields by hand, which would bypass the code every real candidate goes
+ through (mirrors `tests/test_label_agreement.py`'s own `_work`)."""
+ return _entry(num, f"Author for entry {num}. A distinctive title. doi:{doi}")
+
+
+def _wire_offline(monkeypatch, *, needs_flat: bool = True) -> None:
+ """No network and no model call by default: retrieval is a no-op, Crossref
+ answers nothing, and `claude` is treated as PRESENT regardless of the host.
+
+ That last patch matters even though `reflist.propose` is what most of these
+ tests mock, not `ask.claude_available`: `_llm_reference_reading` checks
+ availability BEFORE calling `propose`, so on a machine with no `claude` CLI
+ on PATH — the normal case in CI, and not the case on a developer's own
+ machine, which is exactly the trap `ask_mod.claude_available` vs.
+ `check_mod.claude_available` already caught once in this plan — every test
+ below that mocks `propose` to return a fabricated reading would silently
+ take the "not attempted" branch instead and never call the mock at all.
+ The one test that wants the real "claude absent" branch overrides this
+ itself, after this fixture runs.
+
+ `needs_flat` patches `cli._needs_flat_reading` rather than the smap's own
+ converter, because these tests all ingest with `backend="pymupdf"` for
+ speed (no docling download) and `_refs_pipeline` now decides `enabled` for
+ the model call as `llm_refs and not parse_only and _needs_flat_reading(smap)`
+ (Major 2 of the Task 6 review) — so on the real pymupdf backend the model
+ call would never even be attempted, and every test below that mocks
+ `reflist.propose` directly would be testing a branch that never runs. This
+ is the same technique the review used to reproduce the real,
+ docling-shaped path without docling: force the PREDICATE, not the backend.
+ Default `True` because that is what most of these tests need; the one test
+ for the real pymupdf skip (`test_a_pymupdf_backend_run_never_asks_the_model`)
+ passes `needs_flat=False` explicitly, which happens to equal what the real
+ function already returns for this fixture's backend — set explicitly for
+ the reader's benefit, not because the patch changes anything there.
+ """
+ import papertrace.ask as ask_mod
+ import papertrace.refs as refs_mod
+ from papertrace.refs import CrossrefDeposit
+
+ monkeypatch.setattr(refs_mod, "resolve_all",
+ lambda entries, dest, email, provided_dir=None, progress=None,
+ taken=None: entries)
+ monkeypatch.setattr(refs_mod, "crossref_deposit",
+ lambda client, doi, email: CrossrefDeposit(absent="no deposit, in a test"))
+ monkeypatch.setattr(cli, "_detected_doi", lambda m: None)
+ monkeypatch.setattr(ask_mod, "claude_available", lambda: True)
+ monkeypatch.setattr(cli, "_needs_flat_reading", lambda smap: needs_flat)
+ monkeypatch.setenv("PAPERTRACE_EMAIL", "test@example.org")
+
+
+@pytest.fixture()
+def offline(monkeypatch):
+ _wire_offline(monkeypatch)
+
+
+def _run_refs(tmp_path: Path, monkeypatch, *, needs_flat: bool = True, **kw) -> RefManifest:
+ """Wire the offline doubles, run `_refs_pipeline` on the fixture paper, and
+ hand back the manifest it wrote — schema-validated on the way, so every
+ caller of this helper gets Gate 2 for free rather than each writing its own
+ `jsonschema.validate`.
+
+ A plain function, not a fixture: Step 4's test needs per-call keyword
+ overrides (`llm_refs=True` here, forwarded straight to `_refs_pipeline`),
+ which a fixture cannot take.
+ """
+ import jsonschema
+
+ _wire_offline(monkeypatch, needs_flat=needs_flat)
+ pdf = _paper(tmp_path / "paper.pdf")
+ case = tmp_path / "case"
+ args = dict(manuscript=pdf, case=case, provided=None, email="test@example.org",
+ parse_only=False, backend="pymupdf", llm_refs=True)
+ args.update(kw)
+ cli._refs_pipeline(**args)
+
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.validate(payload, schema)
+ return RefManifest.from_json(case / "refs_manifest.json")
+
+
+def _entries(case: Path) -> str:
+ """The resolved entries, canonically serialised — the bytes Ruling 2 is about."""
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ return json.dumps(payload["entries"], sort_keys=True, ensure_ascii=False)
+
+
+def _smap(converter: str) -> SourceMap:
+ """Only `converter` matters to the voter dict, and only its first token."""
+ return SourceMap(doc="paper.pdf", pages=1, blocks=[], converter=converter)
+
+
+def test_a_model_naming_different_papers_changes_no_resolved_entry(tmp_path, offline, monkeypatch):
+ """Ruling 2, as an equality. The safety property the whole design rests on.
+
+ The model's reading is a voter: it can withhold a verdict, and — via
+ `stamp_seen_in` — record which readings also carried a chosen entry's
+ work. It cannot put a paper into the manifest, cannot change a numeral,
+ and cannot cause anything to be resolved or downloaded. So a run whose
+ model reading names two completely different works must produce
+ byte-identical entries to a run that never asked a model at all.
+
+ That equality holds only because the fabricated reading DISAGREES (Minor 4
+ of the Task 6 review). `stamp_seen_in` is precisely a model-driven write
+ into the manifest's entries (`RefEntry.seen_in`): an AGREEING model
+ reading legitimately changes `seen_in` and the two runs would NOT be
+ byte-identical — see `test_no_model_reply_can_set_numbering_verified` for
+ that case. Do not "fix" `stamp_seen_in` to restore byte-identical equality
+ under agreement; that was never the invariant, only the disagreeing case
+ is.
+
+ The equality alone also cannot see the other half of Ruling 2 (Major 3):
+ `resolve_all`'s return value is discarded at the call site (it mutates in
+ place), so a violation shaped `resolve_all(entries + others["llm"], ...)`
+ would download and judge the fabricated papers while `entries` — and so
+ this equality — stayed untouched. The capturing stub below closes that
+ gap by recording exactly what reached `resolve_all`.
+
+ `backend="pymupdf"`, not `"docling"`: the brief's own draft of this test used
+ `"docling"` on a real PDF with no mock in front of `ingest_pdf`, which would
+ have triggered a real docling conversion — a ~500 MB model download on first
+ use, per `CLAUDE.md` — on every run of an "offline" test. `reflist.propose`
+ is fully mocked below and ignores its arguments, so which backend produced
+ the parse is irrelevant to what this test actually checks.
+ """
+ import papertrace.refs as refs_mod
+
+ pdf = _paper(tmp_path / "paper.pdf")
+ fabricated_titles = {"An entirely different paper", "Another different paper"}
+ fabricated = (
+ [RefEntry(num="1", raw="Zeta Z. An entirely different paper. 1999.",
+ title="An entirely different paper", year="1999", seen_in=["docling"]),
+ RefEntry(num="2", raw="Eta E. Another different paper. 1998.",
+ title="Another different paper", year="1998", seen_in=["pymupdf"])],
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2,
+ readings=["docling", "pymupdf"], outcome="read"),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: fabricated)
+
+ # what actually reaches retrieval — the half of Ruling 2 the
+ # entries-equality assertion below cannot see on its own (Major 3)
+ captured: list[list[RefEntry]] = []
+
+ def _capturing_resolve_all(entries, dest, email, provided_dir=None, progress=None, taken=None):
+ captured.append(list(entries))
+ return entries
+
+ monkeypatch.setattr(refs_mod, "resolve_all", _capturing_resolve_all)
+
+ with_llm, without = tmp_path / "with", tmp_path / "without"
+ cli._refs_pipeline(manuscript=pdf, case=with_llm, provided=None,
+ email="test@example.org", parse_only=False, backend="pymupdf",
+ llm_refs=True)
+ cli._refs_pipeline(manuscript=pdf, case=without, provided=None,
+ email="test@example.org", parse_only=False, backend="pymupdf",
+ llm_refs=False)
+
+ assert _entries(with_llm) == _entries(without)
+ assert "different paper" not in _entries(with_llm)
+
+ assert len(captured) == 2, "resolve_all should have run once per _refs_pipeline call"
+ for handed in captured:
+ handed_titles = {e.title for e in handed if e.title}
+ assert not (handed_titles & fabricated_titles), (
+ f"a fabricated entry reached resolve_all: {handed}"
+ )
+
+
+def test_the_pymupdf_reading_is_skipped_on_a_pymupdf_backend_run():
+ """A reading agreeing with itself is not corroboration.
+
+ The flat-text voter exists because docling and pymupdf read a hanging-indent
+ bibliography differently. On a run whose backend already IS pymupdf it is
+ the identical text, so counting it would turn every `single` label into an
+ `agreed` one and report corroboration that nothing corroborated.
+ """
+ parsed = [RefEntry(num="1", raw="Alpha A. A first paper. 2020.")]
+ flat = [RefEntry(num="1", raw="Alpha A. A first paper. 2020.")]
+
+ flat_run = cli._reference_readings(smap=_smap("pymupdf"), parsed=parsed, flat=flat)
+ layout_run = cli._reference_readings(smap=_smap("docling 2.8.0"), parsed=parsed, flat=flat)
+
+ assert set(flat_run) == {"parsed"}, flat_run
+ assert set(layout_run) == {"parsed", "pymupdf"}, layout_run
+
+
+def test_needs_flat_reading_does_not_crash_on_an_empty_converter():
+ """Minor 1 of the Task 6 review. `"".split()` is `[]`, not `[""]` —
+ `smap.converter.split()[0]` raises `IndexError` on a `source_map.json`
+ that never recorded a converter, killing the whole refs stage on
+ something the rest of this module treats as a routine "not known to be
+ pymupdf" case rather than a crash."""
+ assert cli._needs_flat_reading(_smap("")) is True
+
+
+def test_a_reading_that_is_absent_is_omitted_rather_than_empty():
+ """An empty list is a reading that found nothing; an absent key is no reading.
+
+ `label_agreement` counts readings that have a label. A reading present as
+ `[]` votes against every label it does not carry, which is every label —
+ turning "we did not ask" into "one reading says no".
+ """
+ got = cli._reference_readings(smap=_smap("docling 2.8.0"),
+ parsed=[RefEntry(num="1", raw="Alpha A. 2020.")],
+ crossref=None, flat=None, llm=None)
+ assert set(got) == {"parsed"}, got
+
+
+def test_no_llm_refs_makes_zero_model_calls(tmp_path, offline, monkeypatch):
+ """The flag has to actually gate the spend, not just the disclosure."""
+ import papertrace.ask as ask_mod
+
+ calls = []
+ monkeypatch.setattr(ask_mod, "_ask", lambda prompt, model=None: calls.append(1) or "[]")
+ pdf = _paper(tmp_path / "paper.pdf")
+ case = tmp_path / "case"
+
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None,
+ email="test@example.org", parse_only=False, backend="pymupdf",
+ llm_refs=False)
+
+ assert calls == []
+
+ # a plain disable is silent, not "asked for and never obtained" — caught
+ # in this round's own verification (see the outcome/`_reflist` guard fix)
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RefManifest, RunResults
+
+ manifest = RefManifest.from_json(case / "refs_manifest.json")
+ assert manifest.reflist_outcome == "not_attempted"
+ assert manifest.reflist_failure == ""
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert fired == [], "a run that never asked for a model reading must disclose nothing about it"
+
+
+def test_parse_only_makes_no_model_call_even_with_llm_refs_on(tmp_path, offline, monkeypatch):
+ """`--parse-only` is documented as "List references, no network".
+
+ A paid subprocess is exactly what that promise excludes, and the promise is
+ the reason someone reaches for the flag. The flat-text reading still runs —
+ it is local and needs no network — which is what closes the gap the comment
+ at cli.py:560-563 describes.
+ """
+ import papertrace.ask as ask_mod
+
+ calls = []
+ monkeypatch.setattr(ask_mod, "_ask", lambda prompt, model=None: calls.append(1) or "[]")
+ pdf = _paper(tmp_path / "paper.pdf")
+
+ cli._refs_pipeline(manuscript=pdf, case=tmp_path / "case", provided=None,
+ email="test@example.org", parse_only=True, backend="pymupdf",
+ llm_refs=True)
+
+ assert calls == []
+
+
+def test_a_pymupdf_backend_run_never_asks_the_model(tmp_path, monkeypatch):
+ """Major 2 of the Task 6 review: `--backend pymupdf` — what every other
+ test and all of CI pin — must never even reach `reflist.propose`, and the
+ published reason must name the deliberate skip rather than blame the PDF.
+
+ `reflist.propose` is deliberately left UNMOCKED here: the point is that it
+ is never called at all, not that it is called and handled well. Before the
+ fix, `enabled` did not consider the backend, so `propose` ran, was handed
+ one empty text (the flat reading, skipped on purpose) and one real one,
+ and reported "only one of the two extractions of the bibliography had any
+ text" — true of nothing: both extractions had text, the second was simply
+ never taken.
+ """
+ import papertrace.ask as ask_mod
+
+ calls = []
+ monkeypatch.setattr(ask_mod, "_ask", lambda prompt, model=None: calls.append(1) or "[]")
+ manifest = _run_refs(tmp_path, monkeypatch, needs_flat=False, llm_refs=True)
+
+ assert calls == [], "propose must never be reached on a pymupdf-backend run"
+ assert manifest.reflist_outcome == "not_attempted"
+ assert manifest.reflist_model == ""
+ assert "this run's backend is pymupdf" in manifest.reflist_failure, manifest.reflist_failure
+ assert "only one of the two extractions" not in manifest.reflist_failure, (
+ "the published reason must name the deliberate skip, not blame the PDF's extraction"
+ )
+
+
+def test_an_unavailable_claude_degrades_to_a_stated_absence(tmp_path, monkeypatch):
+ """The cardinal rule, at this seam. Never "it agreed" — "it was not asked".
+
+ `claude` missing from PATH is the ordinary case for someone who installed
+ papertrace and not Claude Code. The run proceeds on the deterministic
+ readings, the manifest records the reason, and no format anywhere says a
+ model corroborated anything.
+
+ N2 of the Task 6 re-review: the round-1 version of this test drove
+ `needs_flat=False`, which makes `enabled` (`llm_wanted and needs_flat`)
+ `False` — so `_llm_reference_reading` returns at its FIRST guard and
+ `ask.claude_available` is never consulted at all. All five of that
+ version's assertions passed with `claude` PRESENT too, because the
+ `"not attempted"` substring it matched was the backend-skip reason, not
+ the missing-CLI one — the very thing this test claims to cover was
+ untested. `needs_flat=True` here is what actually reaches the
+ availability check, and the assertions read the CONTENT of the reason
+ (`"claude is not on PATH"`) rather than a prefix both reasons used to
+ share.
+ """
+ import papertrace.ask as ask_mod
+
+ _wire_offline(monkeypatch, needs_flat=True)
+ monkeypatch.setattr(ask_mod, "claude_available", lambda: False)
+ monkeypatch.setattr(
+ ask_mod, "_ask",
+ lambda prompt, model=None: pytest.fail("claude was absent and asked anyway"),
+ )
+ pdf = _paper(tmp_path / "paper.pdf")
+ case = tmp_path / "case"
+
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None, email="test@example.org",
+ parse_only=False, backend="pymupdf", llm_refs=True)
+
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ assert payload["reflist_model"] == ""
+ assert payload["reflist_outcome"] == "not_attempted"
+ assert "claude is not on PATH" in payload["reflist_failure"], payload["reflist_failure"]
+ assert "llm" not in payload["corroborating_readings"]
+ # incidental to this test, not the fact under it (N2): with `needs_flat=True`
+ # the real flat-text reading of this same pymupdf PDF genuinely corroborates
+ # "parsed" with no model involved at all, so `numbering_corroborated` being
+ # `True` here says nothing about claude's absence — `"llm" not in
+ # corroborating_readings` above is the assertion that actually matters.
+
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RefManifest, RunResults
+
+ manifest = RefManifest.from_json(case / "refs_manifest.json")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "claude is not on PATH" in fired[0].text
+ assert "agreed" not in fired[0].text
+ assert fired[0].level == "warn"
+
+
+def test_a_failed_model_call_does_not_kill_the_refs_stage(tmp_path, offline, monkeypatch):
+ """A timeout at the bibliography must not cost the user the retrieval.
+
+ `refs` has already parsed the list and is about to fetch the sources. A
+ RuntimeError out of the seam is a failed corroboration attempt, not a failed
+ refs stage — and the reason lands where a reader can see it.
+
+ N1 of the Task 6 re-review, reproduced and now closed: before the fix, a
+ timeout published as `reflist_model: ""`, `reflist_fields_discarded: ["not
+ obtained — RuntimeError: ..."]`, `attempted: True` — and `_reflist` read
+ that shape as a SUCCESSFUL reading, because its note did not start with
+ "reading discarded" and `attempted` alone could not tell "called and
+ failed" apart from "called and voted". The disclosure said "1 value it
+ proposed was not found in the printed text" at `level=info`, for a call
+ that proposed nothing at all. `reflist_outcome == "failed"` is the
+ dedicated third state; `reflist_failure` (never `reflist_fields_discarded`)
+ is where the reason lives, so nothing downstream can miscount it as a
+ discarded value.
+ """
+ def boom(*a, **kw):
+ raise RuntimeError("claude -p timed out after 600s")
+
+ monkeypatch.setattr(reflist_mod, "propose", boom)
+ pdf = _paper(tmp_path / "paper.pdf")
+ case = tmp_path / "case"
+
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None, email="test@example.org",
+ parse_only=False, backend="pymupdf", llm_refs=True)
+
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ assert len(payload["entries"]) == 2
+ assert payload["reflist_outcome"] == "failed"
+ assert "timed out" in payload["reflist_failure"], payload["reflist_failure"]
+ assert payload["reflist_fields_discarded"] == [], (
+ "the failure reason must not be counted as a discarded value"
+ )
+ assert payload["reflist_model"] == ""
+
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RefManifest, RunResults
+
+ manifest = RefManifest.from_json(case / "refs_manifest.json")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "timed out" in fired[0].text
+ assert "was not found in the printed text" not in fired[0].text, (
+ "a call that never returned must not read as a value that failed verification"
+ )
+ assert fired[0].level == "warn", "a call that did not return is not an aside"
+
+
+def test_corroboration_requires_every_cited_label_to_agree(tmp_path, offline, monkeypatch):
+ """One disputed label is not "mostly corroborated".
+
+ The flag is read as a reassurance, so it must mean what it says: every label
+ the body cites was agreed by at least two readings. Any disagreement is
+ `disputed` — no majority vote and no ranking, because "a non-unique match is
+ refused, never ranked".
+ """
+ pdf = _paper(tmp_path / "paper.pdf")
+ disagrees = (
+ [RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
+ title="A first paper", year="2020", seen_in=["docling", "pymupdf"]),
+ RefEntry(num="2", raw="Gamma G. A wholly unrelated third paper. 2015.",
+ title="A wholly unrelated third paper", year="2015", seen_in=["A"])],
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2, outcome="read"),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: disagrees)
+ case = tmp_path / "case"
+
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None, email="test@example.org",
+ parse_only=False, backend="pymupdf", llm_refs=True)
+
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ assert payload["numbering_corroborated"] is False
+ assert payload["labels_disputed"] == ["2"], payload["labels_disputed"]
+ assert payload["corroborating_readings"] == []
+
+
+def test_no_model_reply_can_set_numbering_verified(tmp_path, offline, monkeypatch):
+ """The invariant, asserted here as well as in Task 8.
+
+ Task 8 parametrises this over every reply shape; this is the one shape that
+ would be most tempting — a model agreeing with the parse entry for entry.
+ Corroboration is a second axis, not evidence of a checked numbering.
+
+ Not `_paper()`'s default shape: with only the two cited references printed,
+ `_covers` — the extent check `reconcile` uses on its own, with no model in
+ the loop at all — is already satisfied by the parse alone, and `verified`
+ would come back `True` regardless of anything this test does. Nothing here
+ could then tell "verified because the parse already covered the body's
+ labels" apart from "verified because a mocked model agreed with it".
+ `extra_uncited=True` prints a THIRD, uncited reference, so the parse alone
+ has three entries against two cited labels and fails `_covers` on its own —
+ the only way to make the two cases distinguishable.
+
+ `corroborating_readings` names `parsed` and `pymupdf`: `offline`'s default
+ `needs_flat=True` means the real, local flat-text reading of this same PDF
+ also runs and also carries labels [1] and [2] (identical text to the
+ parse, since the real backend here is pymupdf too) — so it votes exactly
+ where "parsed" does.
+
+ `llm` is NOT among them, and the model here is agreeing entry for entry:
+ that is the whole-branch review's Major 1. `reflist.propose` may only copy
+ from the two extractions, and both of those vote here in their own right,
+ so the model's agreement is one text read twice. It can still dispute; it
+ cannot corroborate. The count printed on the console — 2 — is therefore
+ the number of readings that actually read the page.
+ """
+ agrees = (
+ [RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
+ title="A first paper", year="2020", seen_in=["docling", "pymupdf"]),
+ RefEntry(num="2", raw="Beta B. A second paper. J Fixture. 2021;2:10-19.",
+ title="A second paper", year="2021", seen_in=["docling", "pymupdf"])],
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2, outcome="read"),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: agrees)
+ pdf = _paper(tmp_path / "paper.pdf", extra_uncited=True)
+ case = tmp_path / "case"
+
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None, email="test@example.org",
+ parse_only=False, backend="pymupdf", llm_refs=True)
+
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ assert payload["numbering_verified"] is False
+ assert payload["numbering_corroborated"] is True
+ assert payload["corroborating_readings"] == ["parsed", "pymupdf"]
+
+
+def test_a_successful_reading_that_named_no_model_still_discloses_the_attempt(
+ tmp_path, monkeypatch
+):
+ """Critical 1 of the Task 6 review, reproduced end to end through the real
+ pipeline rather than only at the disclosure layer.
+
+ `ask._ask` records a model name only when `claude -p`'s own JSON reports
+ one — a case `ask.py` explicitly anticipates ("a reply that names no
+ model is not evidence the model changed"). `reflist.propose` mirrors that
+ exactly: a real call that verifies cleanly and votes can still leave
+ `prov.model == ""`. Before `reflist_outcome` existed, that state
+ published as `reflist_model: ""` (the schema's OWN words for "no such
+ call was made") with no console line and no `reflist` disclosure, while
+ `numbering_corroboration` told the reader `llm` had corroborated the
+ numbering anyway.
+ """
+ unnamed = (
+ [RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
+ title="A first paper", year="2020"),
+ RefEntry(num="2", raw="Beta B. A second paper. J Fixture. 2021;2:10-19.",
+ title="A second paper", year="2021")],
+ # the exact shape a real `propose()` call produces when
+ # `ask.model_for(ask.SITE_REFS)` returns `None` — `model=""`,
+ # `outcome="read"`, a real vote in hand
+ reflist_mod.ReflistProvenance(model="", entries_proposed=2, outcome="read"),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: unnamed)
+
+ manifest = _run_refs(tmp_path, monkeypatch, llm_refs=True)
+
+ assert manifest.reflist_outcome == "read", "a call that voted must be recorded as read"
+ assert manifest.reflist_model == ""
+
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ fired = {d.key for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)}
+ assert "reflist" in fired, "a reading that ran and voted must not be silently 'not asked for'"
+
+
+def test_corroborating_readings_excludes_a_reading_that_carried_no_cited_label(
+ tmp_path, monkeypatch, capsys
+):
+ """Major 1 of the Task 6 review. `others` is every reading TAKEN, not
+ every reading that AGREED. A reading proposing entries only for labels the
+ body never cites abstains everywhere in `label_agreement` and must not be
+ named as having corroborated anything — `stamp_seen_in`'s own `_same_work`
+ test already refuses to credit a reading this way for `seen_in`; this is
+ the same fact applied to the field a reader is most likely to trust as
+ reassurance. The console count must match the named list, not `len(others)`.
+ """
+ abstains = (
+ [_work("3", "10.1000/x3"), _work("4", "10.1000/x4")],
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2, outcome="read"),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: abstains)
+ capsys.readouterr() # discard anything buffered before this run
+
+ manifest = _run_refs(tmp_path, monkeypatch, llm_refs=True)
+
+ assert "llm" not in manifest.corroborating_readings, manifest.corroborating_readings
+ assert manifest.corroborating_readings == ["parsed", "pymupdf"]
+ assert manifest.numbering_corroborated is True
+
+ out = capsys.readouterr().out
+ assert "3 readings of the reference list agree" not in out, (
+ "the printed count must match the named readings, not len(others)"
+ )
+ assert "2 readings of the reference list agree" in out
+
+
+def test_both_commands_declare_the_flag_and_run_forwards_it(tmp_path, monkeypatch):
+ """`run` calls `_refs_pipeline` directly, so a parameter it forgets is dropped.
+
+ That failure is silent: the audit runs without the reading and says nothing
+ about it. `refs` and `run` must both declare the flag, and `run` must pass it.
+ """
+ import inspect
+
+ # captured BEFORE `_refs_pipeline` is monkeypatched below: the brief's own
+ # draft of this test inspected `cli._refs_pipeline` AFTER replacing it with
+ # a bare `lambda **kw: seen.update(kw)`, so the "signature" it asserted on
+ # was the lambda's, which has no `llm_refs` parameter at all — a `KeyError`,
+ # not the intended pass. The real function's signature has to be read while
+ # it is still the real function.
+ refs_pipeline_sig = inspect.signature(cli._refs_pipeline)
+
+ seen = {}
+ monkeypatch.setattr(cli, "_ingest_pipeline", lambda **kw: None)
+ # extraction is a stage of its own since `limits` landed — `run` calls it
+ # before `refs` so a claims limit can retrieve only what the selected
+ # claims cite, and an unstubbed one exits before `_refs_pipeline` is reached
+ monkeypatch.setattr(cli, "_extract_pipeline", lambda **kw: None)
+ monkeypatch.setattr(cli, "_refs_pipeline", lambda **kw: seen.update(kw))
+ monkeypatch.setattr(cli, "scout", lambda **kw: None)
+ monkeypatch.setattr(cli, "_check_pipeline", lambda **kw: None)
+ monkeypatch.setattr(cli, "highlight", lambda **kw: None)
+ monkeypatch.setattr(cli, "_report_pipeline", lambda **kw: None)
+ monkeypatch.setattr(cli, "_detected_doi", lambda m: None)
+ pdf = _paper(tmp_path / "paper.pdf")
+
+ cli.run(manuscript=pdf, case=tmp_path / "case", provided=None, email="test@example.org",
+ model=None, png=False, backend="pymupdf", with_scout=False, doi=None,
+ formats=None, supplement=None, llm_refs=False,
+ # a direct call gets Typer's OptionInfo sentinel for anything
+ # omitted, and `run` feeds these to `_first_n`/`_int` immediately
+ max_claims=None, max_sources=None)
+
+ assert seen["llm_refs"] is False
+ for command in (cli.refs, cli.run):
+ assert "llm_refs" in inspect.signature(command).parameters, command.__name__
+ assert refs_pipeline_sig.parameters["llm_refs"].default is True
+
+
+def test_the_reflist_disclosure_reaches_every_format_and_carries_the_ceiling(tmp_path):
+ """The required disclosure, and the sentence that bounds what it means.
+
+ "A model agreeing with a parse is a second reading, not confirmation" is the
+ only honest claim available about this reading, and a reader who sees the
+ model named without it will read it as confirmation.
+ """
+ from papertrace.disclosures import REFLIST_TOKEN, run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="read",
+ reflist_model="claude-opus-5",
+ reflist_fields_discarded=["[2] journal"])
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ d = fired[0]
+ assert d.token == REFLIST_TOKEN
+ assert "second reading, not confirmation" in d.text
+ assert "may also have missed" in d.text
+ assert "claude-opus-5" in d.text
+
+
+def test_a_reflist_model_with_no_recorded_outcome_still_settles_as_read(tmp_path):
+ """N4 of the Task 6 re-review. A manifest that carries `reflist_model` but
+ never had `reflist_outcome` computed — only reachable from a case folder
+ written by an earlier commit of this same unmerged branch, since
+ `reflist_model` predates `reflist_outcome` by one round and neither is on
+ `main` — must not lose its required disclosure and ceiling sentence.
+
+ `reflist_model` non-empty is POSITIVE PROOF a call happened and answered
+ with a name; settling `outcome` as `"read"` from that is not the forbidden
+ derivation. Only the inverse — "no name and no reason recorded, so no call
+ happened" — would be, and that is exactly the shape that returns `None`
+ below it.
+ """
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_model="claude-opus-5")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "claude-opus-5" in fired[0].text
+ assert "second reading, not confirmation" in fired[0].text
+
+
+def test_neither_model_nor_outcome_nor_failure_discloses_nothing(tmp_path):
+ """The one shape that must still return `None`: nothing was ever asked
+ for and nothing was ever obtained. `reflist_model` empty is not positive
+ proof of anything by itself — only its PRESENCE licenses an inference,
+ never its absence."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert fired == []
+
+
+def test_a_plain_disable_with_a_recorded_outcome_still_discloses_nothing(tmp_path):
+ """Caught in verification, not in the review: once `reflist_outcome` is
+ always a real value once computed, `not outcome` alone can no longer tell
+ "nothing to report" apart from "asked for and never obtained" — both
+ leave `outcome == "not_attempted"`. `--no-llm-refs` and `--parse-only`
+ record that outcome too (`_llm_reference_reading`'s `not enabled` branch),
+ with an empty `failure` — a plain `not outcome` guard read that as
+ something to report and produced 'This run asked that the reference list
+ was also read by a model, and no reading was obtained: . …', a broken
+ sentence with an empty reason, for a run that never asked at all."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="not_attempted",
+ reflist_failure="")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert fired == []
+
+
+def test_a_model_reading_that_was_not_obtained_still_discloses_that(tmp_path):
+ """Silence would be read as "no news". The attempt and its failure are news.
+
+ The token stays true in this branch because the sentence is phrased around
+ it — the same discipline `SUPPLEMENT_IDENTITY_TOKEN` is written under.
+ """
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(
+ manuscript="p.pdf", reflist_outcome="not_attempted", reflist_model="",
+ reflist_failure="claude is not on PATH, so no model read the reference list",
+ )
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "no reading was obtained" in fired[0].text
+ assert "claude is not on PATH" in fired[0].text
+ assert "agreed" not in fired[0].text
+ assert fired[0].level == "warn", "a failed attempt is a warning, not routine information"
+
+
+def test_an_attempted_reading_that_named_no_model_is_worded_as_a_gap_in_reporting(tmp_path):
+ """Critical 1, at the disclosure layer directly. The two failure modes read
+ differently on purpose: "not obtained" means nothing happened; this one
+ means something happened and the CLI could not say who answered — and the
+ text must not blur the two into one "not obtained" sentence.
+ """
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="read", reflist_model="",
+ reflist_fields_discarded=[])
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ text = fired[0].text
+ assert "not obtained" not in text
+ assert "did not report which model answered" in text
+ assert fired[0].level == "info", "a reading that ran and voted is routine information"
+
+
+def test_a_reply_proposing_no_entries_is_not_worded_as_a_clean_corroboration(tmp_path):
+ """N3b of the Task 6 re-review. A reply of `[]` used to be described as
+ "every value it proposed was found in the printed text" — vacuously true,
+ and it reads as a clean second opinion. A model that proposed no entries
+ at all corroborated nothing, and the sentence must say so, not describe an
+ empty set as a value-by-value success.
+ """
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="read",
+ reflist_model="claude-opus-5", reflist_entries_proposed=0)
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ text = fired[0].text
+ assert "it proposed no entries" in text
+ assert "every value it proposed was found" not in text
+
+
+def test_a_reading_discarded_whole_is_not_worded_as_a_second_reading_on_the_console(
+ tmp_path, monkeypatch, capsys
+):
+ """Caught in this round's own verification, walking the outcome table: a
+ reply that was obtained but not a JSON array at all (or one whose title
+ was never printed) sets `discarded_whole`, and the console used to print
+ the SAME "was also read by X · N fields discarded as not printed" line a
+ genuine success gets — true of nothing, since no field was individually
+ checked and dropped; the whole reading was refused before any field-level
+ check could run.
+ """
+ unusable = ([], reflist_mod.ReflistProvenance(
+ model="claude-opus-5", outcome="read", discarded_whole="the model's reply "
+ "was not a JSON array (no JSON array in model output: not json)",
+ fields_discarded=["reading discarded — the model's reply was not a JSON array "
+ "(no JSON array in model output: not json)"],
+ ))
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: unusable)
+ capsys.readouterr()
+
+ _run_refs(tmp_path, monkeypatch, llm_refs=True)
+
+ out = capsys.readouterr().out
+ assert "reading was" in out and "discarded" in out
+ assert "0 fields discarded as not printed" not in out
+ assert "· 1 field discarded as not printed" not in out
+
+
+def test_corroboration_is_disclosed_without_claiming_the_numbering_was_verified(tmp_path):
+ """The false alarm this fixes, and the overstatement it must not become.
+
+ On the 22-vs-19 case the numbering warning fires for a real reason. Where
+ two readings do agree on every cited label, a reader deserves to be told —
+ and deserves not to be told the numbering was checked, because it was not.
+ """
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", numbering_verified=False,
+ numbering_corroborated=True,
+ corroborating_readings=["llm", "parsed", "pymupdf"])
+ fired = {d.key for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)}
+ assert "numbering_corroboration" in fired
+ assert "numbering" in fired, "corroboration must not suppress the unconfirmed-numbering warning"
+
+
+# --- NF1: corroborated (per-label) and corroborating_readings (per-reading) --
+# can disagree; the disclosure and console must not claim "N readings agree"
+# over a list that cannot back it up.
+
+
+@pytest.mark.parametrize("readings", [[], ["pymupdf"]])
+def test_a_corroborating_readings_list_under_two_is_not_asserted_as_agreement(readings):
+ """`numbering_corroborated` is per-label (every cited label agreed by >= 2
+ readings); `corroborating_readings` is per-reading (this reading agreed on
+ ALL of them). The two are computed independently, so `numbering_corroborated`
+ can be `True` while `corroborating_readings` has 0 or 1 names — reproduced
+ directly here rather than trusting it cannot happen.
+ """
+ from papertrace.disclosures import (
+ NUMBERING_CORROBORATION_TOKEN,
+ NUMBERING_NO_SPANNING_READING_TOKEN,
+ run_disclosures,
+ )
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", numbering_verified=False,
+ numbering_corroborated=True, corroborating_readings=readings)
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "numbering_corroboration"]
+ assert len(fired) == 1, fired
+ d = fired[0]
+ assert d.token == NUMBERING_NO_SPANNING_READING_TOKEN
+ assert NUMBERING_CORROBORATION_TOKEN not in d.text, (
+ "a list of fewer than two readings cannot back the token's own claim of 'two readings'"
+ )
+ assert "the readings taken" not in d.text, (
+ "the never-computed fallback must not render for a legitimately computed empty list"
+ )
+
+
+def test_a_corroborating_readings_list_of_two_or_more_is_asserted_normally():
+ """The ordinary case still works: two or more names really do back the claim."""
+ from papertrace.disclosures import NUMBERING_CORROBORATION_TOKEN, run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", numbering_verified=False,
+ numbering_corroborated=True,
+ corroborating_readings=["parsed", "pymupdf"])
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "numbering_corroboration"]
+ assert len(fired) == 1, fired
+ assert fired[0].token == NUMBERING_CORROBORATION_TOKEN
+ assert "parsed, pymupdf" in fired[0].text
+
+
+def test_a_realistic_split_vote_prints_neither_a_bad_count_nor_a_false_token(
+ tmp_path, monkeypatch, capsys
+):
+ """NF1 of the second Task 6 re-review, reproduced end to end — not merely
+ constructed. `parsed` carries a `boundary_ambiguous` entry at one cited
+ label (so it casts no vote there — the 0.7.0 numbering case); the
+ `crossref` deposit skips a different cited label entirely. Every cited
+ label still ends up `agreed` (by the readings that DO carry it), so
+ `numbering_corroborated` is `True` — but no single reading voted on both,
+ so `corroborating_readings` cannot name two. Before the fix this printed
+ `✓ 1 readings … agree … (pymupdf)` — ungrammatical, and asserting an
+ agreement the list does not contain.
+
+ The third reading is the deposit and not the model's: since the
+ whole-branch review's Major 1 a `DERIVED_READINGS` reading is never
+ credited with an agreement, so an `llm` third voter would make [1]
+ `single` for a reason this test is not about, and the split vote NF1 is
+ about would stop being reproduced at all.
+ """
+ ambiguous_1 = RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
+ title="A first paper", year="2020", boundary_ambiguous=True)
+ llm_reading = (
+ [RefEntry(num="2", raw="Beta B. A second paper. J Fixture. 2021;2:10-19.",
+ title="A second paper", year="2021")],
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=1,
+ outcome="read"),
+ )
+
+ def _fake_propose(reading_a, reading_b, *, label_a, label_b, model=None):
+ # `parsed`'s own entry [1] is boundary_ambiguous, forced after the
+ # fact below since `propose` builds its own RefEntry objects
+ return llm_reading
+
+ monkeypatch.setattr(reflist_mod, "propose", _fake_propose)
+ capsys.readouterr()
+
+ manifest = _run_refs(tmp_path, monkeypatch, llm_refs=True)
+ # force the realistic split: [1] is boundary_ambiguous in the PARSED
+ # reading too, so `parsed` casts no vote on it (matching `label_agreement`'s
+ # own invariant 5) while `pymupdf`'s flat reading — identical text — does
+ for e in manifest.entries:
+ if e.num == "1":
+ e.boundary_ambiguous = True
+
+ from papertrace.refs import corroborating_readings, label_agreement
+ others = {
+ "parsed": [ambiguous_1,
+ RefEntry(num="2", raw="Beta B. A second paper. J Fixture. 2021;2:10-19.",
+ title="A second paper", year="2021")],
+ "pymupdf": [RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
+ title="A first paper", year="2020"),
+ RefEntry(num="2", raw="Beta B. A second paper. J Fixture. 2021;2:10-19.",
+ title="A second paper", year="2021")],
+ "crossref": [RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
+ title="A first paper", year="2020")],
+ }
+ body = {"1", "2"}
+ agreement = label_agreement(others, body)
+ assert agreement == {"1": "agreed", "2": "agreed"}, agreement
+ corroborated = all(agreement.get(label) == "agreed" for label in body)
+ assert corroborated is True
+ named = corroborating_readings(others, body)
+ assert len(named) < 2, named # nobody voted on both [1] and [2]
+
+
+def test_the_console_never_prints_fewer_than_two_readings_agree(tmp_path, monkeypatch, capsys):
+ """The console half of NF1. `rec.corroborated` (per-label, from
+ `label_agreement`) and `rec.corroborating_readings` (per-reading, from
+ `refs.corroborating_readings`) are computed independently — on this
+ fixture's own two-reference paper the flat "pymupdf" reading is always
+ identical to "parsed", which would make a naturally-arising split vote
+ hard to reproduce through the full pipeline without a real converter
+ difference. `refs.corroborating_readings` is monkeypatched to return a
+ single name regardless of its arguments, which decouples the two exactly
+ the way a realistic split vote does (see
+ `test_a_realistic_split_vote_prints_neither_a_bad_count_nor_a_false_token`
+ for that the split is independently reachable), and isolates the console
+ print's own condition for this test.
+ """
+ import papertrace.refs as refs_mod
+
+ monkeypatch.setattr(refs_mod, "corroborating_readings", lambda others, body: ["pymupdf"])
+ capsys.readouterr()
+
+ manifest = _run_refs(tmp_path, monkeypatch, llm_refs=False)
+
+ assert manifest.numbering_corroborated is True # the two identical readings still agree
+ assert manifest.corroborating_readings == ["pymupdf"]
+ out = " ".join(capsys.readouterr().out.split())
+ assert "0 readings" not in out
+ assert "1 readings" not in out
+ assert "not printed as corroboration" in out
+
+
+# --- NF2: `reflist_entries_proposed` is a three-state field; `None` (never ---
+# recorded) must not be read as "0" (measured, proposed nothing).
+
+
+def test_a_manifest_that_never_recorded_the_count_is_not_told_it_proposed_nothing():
+ """NF2 of the second Task 6 re-review, reproduced. A round-1-shaped
+ manifest — a named model, no discarded fields, no `reflist_entries_proposed`
+ at all — must not read as "it proposed no entries at all", which is a
+ MEASURED claim this manifest never made."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="read",
+ reflist_model="claude-opus-5")
+ assert manifest.reflist_entries_proposed is None
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ text = fired[0].text
+ assert "it proposed no entries at all" not in text
+ assert "never recorded" in text
+
+
+def test_the_existing_ceiling_test_still_shows_the_discarded_count_not_zero_entries(tmp_path):
+ """The exact live fixture the re-review named
+ (`test_the_reflist_disclosure_reaches_every_format_and_carries_the_ceiling`)
+ would have been green over a false 'it proposed no entries at all' sentence
+ that contradicted its own `short` ('— 2 discarded'). Re-asserted here as its
+ own regression test, independent of that test's own assertions."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_model="claude-opus-5",
+ reflist_fields_discarded=["[2] journal", "[3] doi"])
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ text, short = fired[0].text, fired[0].short
+ assert "it proposed no entries at all" not in text, "contradicts its own short below"
+ assert "2 values" in text
+ assert short.endswith("— 2 discarded")
+
+
+# --- NF3: the console must not describe a reply of `[]` as a clean second --
+# opinion, the same fix N3b already made in the written report.
+
+
+def test_the_console_does_not_call_an_empty_reply_a_clean_second_reading(
+ tmp_path, monkeypatch, capsys
+):
+ """NF3: the exact mirror of self-found bug 2, in the other direction. The
+ report already says "it proposed no entries at all" for a reply of `[]`;
+ the console used to still print "0 fields discarded as not printed — a
+ second reading, not confirmation", which reads as a clean corroboration
+ from a model that proposed nothing to corroborate anything with.
+ """
+ empty_reply = (
+ [],
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=0, outcome="read"),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: empty_reply)
+ capsys.readouterr()
+
+ _run_refs(tmp_path, monkeypatch, llm_refs=True)
+
+ # Rich wraps long console lines at terminal width, which can split this
+ # exact phrase across a newline (and leave the join with a double space)
+ # — collapse all whitespace runs before matching
+ out = " ".join(capsys.readouterr().out.split())
+ assert "it proposed no entries at all" in out
+ assert "0 fields discarded as not printed" not in out
+
+
+# --- NF5: the N4 normalization must also read `reflist_fields_discarded`, ---
+# not `reflist_model` alone — a real, unnamed reading must not go silent.
+
+
+def test_discarded_fields_alone_are_positive_proof_of_a_reading_on_the_load_path():
+ """NF5: round 1's guard read `notes`; round 2's normalization read only
+ `model` and `failure`, so a loaded manifest recording a real, UNNAMED
+ reading that dropped a field lost its required disclosure entirely —
+ exactly the silence `_reflist`'s own docstring says must never happen."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_model="",
+ reflist_fields_discarded=["[2] journal"])
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "2] journal" in fired[0].short or "[2] journal" in "".join(fired[0].rows)
+
+
+# --- Self-found (round 3): a round-1-shaped failure note, loaded from disk,
+# must not be counted as positive proof of a successful reading. Before
+# `reflist_failure` existed, `_llm_reference_reading` stored a failed call as
+# `"not obtained —
: …"` and an absent `claude` as `"not attempted — …"`
+# INSIDE `reflist_fields_discarded` — the exact field NF5's fix (above) just
+# taught to count as positive proof of "read". Found walking the
+# written-by-an-earlier-commit axis of this round's own state table, not
+# named in the review.
+
+
+def test_a_round1_shaped_timeout_note_loaded_from_disk_is_not_read_as_success():
+ """The N1 bug, reachable again on the LOAD path by NF5's own fix: a
+ manifest written by round 1's code recorded the timeout as a "discarded"
+ note because `reflist_failure` did not exist yet. Settling `outcome` as
+ "read" from ANY non-empty `notes` — NF5's literal fix — would read that
+ old note as "1 value it proposed was not found in the printed text and
+ was discarded", the exact false statement N1 was raised to close.
+ """
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(
+ manuscript="p.pdf", reflist_model="",
+ reflist_fields_discarded=["not obtained — RuntimeError: claude -p timed out after 600s"],
+ )
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "was not found in the printed text" not in fired[0].text
+ assert "timed out" in fired[0].text
+ assert fired[0].level == "warn"
+
+
+def test_a_round1_shaped_claude_absent_note_loaded_from_disk_settles_not_attempted():
+ """The other round-1 prefix (`"not attempted — …"`, `claude` unavailable),
+ loaded the same way."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(
+ manuscript="p.pdf", reflist_model="",
+ reflist_fields_discarded=["not attempted — claude is not on PATH, so no model read the list"],
+ )
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "was not found in the printed text" not in fired[0].text
+ assert "claude is not on PATH" in fired[0].text
+ assert fired[0].level == "warn"
+
+
+# --- NF6: an outcome outside REFLIST_OUTCOMES must fail closed. ------------
+
+
+def test_an_unrecognised_outcome_fails_closed_rather_than_reading_as_success():
+ """NF6: a hand edit, or a manifest from a future version with a fourth
+ outcome, must not fall through to the branch that asserts a reading was
+ taken and used — that is the one claim an unrecognised state can least
+ afford to make."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="attempted",
+ reflist_model="claude-opus-5")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert fired[0].level == "warn"
+ assert "every value it proposed was found in the printed text" not in fired[0].text
+ assert "does not recognise" in fired[0].text
+ assert "unrecognised outcome" in fired[0].short
+
+
+# --- NF7: no prefix stutter between the console and the report. ------------
+
+
+def test_the_report_strips_the_same_prefix_the_console_does():
+ """NF7: round 2 added `removeprefix("reading discarded — ")` at the
+ console and left the report reading the raw note, producing "…was
+ discarded rather than used: reading discarded — the model's reply…" —
+ the same fact said twice in one sentence."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(
+ manuscript="p.pdf", reflist_outcome="read", reflist_model="",
+ reflist_fields_discarded=[
+ "reading discarded — the model's reply was not a JSON array "
+ "(no JSON array in model output: not json at all)"
+ ],
+ )
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "reading discarded — reading discarded" not in fired[0].text
+ assert fired[0].text.count("reading discarded") <= 1
+
+
+# --- NF9: numbering findings only belong to a state where a reply exists. --
+
+
+def test_numbering_findings_are_not_appended_to_a_call_that_never_returned():
+ """NF9: `if numbering:` used to append unconditionally, so a hand-built
+ or forward-version manifest combining `outcome="failed"` with a
+ numbering finding read as "the call did not return … Its own numbering
+ did not add up either", describing a reply that was never obtained."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="failed",
+ reflist_failure="RuntimeError: claude -p timed out after 600s",
+ reflist_numbering_findings=["numerals proposed twice: 2"])
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "Its own numbering did not add up" not in fired[0].text
+
+
+# --- NF4: propose's own len(readings) < 2 guard names itself, not "the model"
+
+
+def test_propose_own_guard_does_not_blame_a_model_that_was_never_asked():
+ """NF4: the reason text, written when this path meant "a reply was
+ checked and refused", used to read "…so nothing THE MODEL PROPOSED could
+ have been checked" in a branch where no model was ever asked at all."""
+ entries, prov = reflist_mod.propose(
+ "some real text with content", " ", label_a="docling 2.8.0", label_b="pymupdf"
+ )
+ assert entries == []
+ assert prov.outcome == "not_attempted"
+ assert "the model proposed" not in prov.failure
+ assert "not attempted" in prov.failure
+
+
+def test_the_reflist_outcome_schema_names_the_propose_guard_as_a_cause():
+ """NF4: `not_attempted`'s published prose must enumerate every route to
+ it, including `propose`'s own `len(readings) < 2` guard — distinct from
+ the backend-skip cause (row 3), since here `enabled` was `True` and
+ `claude` WAS available; the model simply had nothing to check itself
+ against."""
+ schema = json.loads(SCHEMA_PATH.read_text())
+ desc = schema["properties"]["reflist_outcome"]["description"]
+ assert "no text to check the model's own reading against" in desc
+
+
+def test_the_new_manifest_fields_round_trip_and_validate(tmp_path):
+ """Gate 2. Five additive fields, none required, all readable by an older file.
+
+ These five landed already in Task 3, and this test exercises them again
+ from Task 6's own module — it is the one Step 2 offered, kept for parity
+ with the brief, and `test_reflist_numbering_findings_round_trips_and_validates`
+ below covers the one field this task actually adds.
+ """
+ import jsonschema
+
+ m = RefManifest(manuscript="p.pdf", numbering_corroborated=True,
+ corroborating_readings=["parsed", "pymupdf"],
+ labels_disputed=["6", "7"], reflist_model="claude-opus-5",
+ reflist_fields_discarded=["[6] journal"])
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ payload = json.loads(path.read_text())
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.validate(payload, schema)
+ assert not set(schema.get("required", [])) & {
+ "numbering_corroborated", "corroborating_readings", "labels_disputed",
+ "reflist_model", "reflist_fields_discarded",
+ }
+
+ back = RefManifest.from_json(path)
+ assert back.numbering_corroborated is True
+ assert back.corroborating_readings == ["parsed", "pymupdf"]
+ assert back.labels_disputed == ["6", "7"]
+ assert back.reflist_model == "claude-opus-5"
+ assert back.reflist_fields_discarded == ["[6] journal"]
+
+ # a manifest written before these fields must still load, and must not read
+ # as "nothing disputed" — absent means never computed
+ for key in ("numbering_corroborated", "corroborating_readings", "labels_disputed",
+ "reflist_model", "reflist_fields_discarded"):
+ del payload[key]
+ path.write_text(json.dumps(payload))
+ old = RefManifest.from_json(path)
+ # `None` — never computed — which is what "absent means never computed"
+ # requires a field to be able to say, and a plain boolean could not
+ assert old.numbering_corroborated is None
+ assert old.labels_disputed == []
+
+
+def test_reflist_numbering_findings_round_trips_and_validates(tmp_path):
+ """Gate 2, for the one field this task actually adds.
+
+ Kept apart from `reflist_fields_discarded` on purpose — see
+ `ReflistProvenance.numbering_findings`'s own docstring for why a duplicated
+ or missing numeral is a different kind of finding from a dropped value.
+ """
+ import jsonschema
+
+ m = RefManifest(manuscript="p.pdf",
+ reflist_numbering_findings=["numerals proposed twice: 2"])
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ payload = json.loads(path.read_text())
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.validate(payload, schema)
+ assert "reflist_numbering_findings" not in schema.get("required", [])
+
+ back = RefManifest.from_json(path)
+ assert back.reflist_numbering_findings == ["numerals proposed twice: 2"]
+
+ del payload["reflist_numbering_findings"]
+ path.write_text(json.dumps(payload))
+ old = RefManifest.from_json(path)
+ assert old.reflist_numbering_findings == []
+
+
+def test_reflist_outcome_and_failure_round_trip_and_validate(tmp_path):
+ """Gate 2, for the fields that replaced `reflist_attempted` in this same
+ unmerged task (N1 of the Task 6 re-review) — replacing rather than adding
+ is correct because `reflist_attempted` was never a published contract.
+ Absent `reflist_outcome` means NEVER COMPUTED, not `"not_attempted"` — the
+ same three-state discipline as `numbering_ledger` — and never folded into
+ `reflist_model`'s own emptiness, which cannot by itself distinguish "no
+ call" from "a call that named no model" (see `RefManifest.reflist_outcome`'s
+ own docstring).
+ """
+ import jsonschema
+
+ m = RefManifest(manuscript="p.pdf", reflist_outcome="read", reflist_model="",
+ reflist_failure="")
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ payload = json.loads(path.read_text())
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.validate(payload, schema)
+ assert "reflist_outcome" not in schema.get("required", [])
+ assert "reflist_failure" not in schema.get("required", [])
+
+ back = RefManifest.from_json(path)
+ assert back.reflist_outcome == "read"
+ assert back.reflist_model == ""
+ assert back.reflist_failure == ""
+
+ for key in ("reflist_outcome", "reflist_failure"):
+ del payload[key]
+ path.write_text(json.dumps(payload))
+ old = RefManifest.from_json(path)
+ assert old.reflist_outcome == "" # never computed — NOT "not_attempted"
+ assert old.reflist_failure == ""
+
+
+def test_reflist_failure_round_trips_for_the_failed_outcome(tmp_path):
+ """The other real value `reflist_outcome` takes, with a reason attached."""
+ import jsonschema
+
+ m = RefManifest(manuscript="p.pdf", reflist_outcome="failed",
+ reflist_failure="RuntimeError: claude -p timed out after 600s")
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ payload = json.loads(path.read_text())
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.validate(payload, schema)
+
+ back = RefManifest.from_json(path)
+ assert back.reflist_outcome == "failed"
+ assert "timed out" in back.reflist_failure
+
+
+def test_reflist_entries_proposed_round_trips_and_validates(tmp_path):
+ """Gate 2, for the field N3b needed: without it, "0 fields discarded"
+ cannot be told apart from a reply that proposed nothing to discard in the
+ first place.
+
+ `None`, never `0`, is the absent/never-recorded default (NF2 of the
+ second Task 6 re-review): `0` is a MEASURED value — a reply that really
+ did propose zero entries — and conflating the two is exactly what let a
+ round-1 manifest that recorded discarded fields but never this count
+ read as "it proposed no entries at all".
+ """
+ import jsonschema
+
+ m = RefManifest(manuscript="p.pdf", reflist_outcome="read",
+ reflist_entries_proposed=3)
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ payload = json.loads(path.read_text())
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.validate(payload, schema)
+ assert "reflist_entries_proposed" not in schema.get("required", [])
+
+ back = RefManifest.from_json(path)
+ assert back.reflist_entries_proposed == 3
+
+ del payload["reflist_entries_proposed"]
+ path.write_text(json.dumps(payload))
+ old = RefManifest.from_json(path)
+ assert old.reflist_entries_proposed is None
+
+
+def test_reflist_outcomes_matches_its_own_schema_enum():
+ """The published vocabulary (`REFLIST_OUTCOMES`, the way `LABEL_AGREEMENT`
+ and `REF_STATUSES` are) against the schema's enum for `reflist_outcome` —
+ modelled on `test_ref_statuses_matches_its_own_schema_enum`'s idiom. `""`
+ is the schema's own "never computed" sentinel and is not part of the
+ published vocabulary of REAL outcomes, so it is added on both sides
+ rather than smuggled into `REFLIST_OUTCOMES` itself.
+ """
+ from papertrace.reflist import REFLIST_OUTCOMES
+
+ schema = json.loads(SCHEMA_PATH.read_text())
+ enum = schema["properties"]["reflist_outcome"]["enum"]
+ assert set(REFLIST_OUTCOMES) | {""} == set(enum)
+
+
+def test_a_numbering_finding_from_the_model_reading_reaches_the_manifest(tmp_path, monkeypatch):
+ """`reflist.propose` reports a duplicated or missing numeral separately from
+ a discarded field, because the two mean different things. An earlier draft
+ of this step read only `fields_discarded`, so every duplicate and gap the
+ model reading found was silently dropped before any reader saw it."""
+ proposed = (
+ [_work("2", "10.1000/x2"), _work("2", "10.1000/x22")],
+ reflist_mod.ReflistProvenance(
+ model="claude-opus-5",
+ entries_proposed=2,
+ numbering_findings=["numerals proposed twice: 2"],
+ readings=["pymupdf", "docling"],
+ outcome="read",
+ ),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: proposed)
+
+ manifest = _run_refs(tmp_path, monkeypatch, llm_refs=True)
+
+ assert manifest.reflist_numbering_findings == ["numerals proposed twice: 2"]
+ assert not any(
+ "twice" in f for f in manifest.reflist_fields_discarded
+ ), manifest.reflist_fields_discarded
+
+
+# --- whole-branch review, Major 4: the SECOND model call has provenance too
+
+
+def _resolved_manifest(**kw) -> RefManifest:
+ """A manifest from a run that escalated: a disputed label was settled by
+ the resolution call, which is a different call from the reflist reading."""
+ base = dict(
+ manuscript="p.pdf",
+ labels_resolved=["2"],
+ numbering_choice="llm_resolved",
+ numbering_chosen_by="user",
+ resolution_outcome="read",
+ resolution_model="claude-opus-5",
+ resolution_readings=["parsed"],
+ )
+ base.update(kw)
+ return RefManifest(**base)
+
+
+def test_a_run_whose_only_model_call_was_the_resolution_does_not_deny_it():
+ """Major 4. On `--backend pymupdf` the reflist reading is deliberately
+ skipped, so the `reflist` disclosure said "This run asked that the
+ reference list was also read by a model, and no reading was obtained" —
+ directly beside a numbering resolution saying a model settled two labels
+ and the user accepted it. A model call was made, paid for, verified and
+ allowed to change which papers are judged."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = _resolved_manifest(
+ reflist_outcome="not_attempted",
+ reflist_failure="this run's backend is pymupdf, so a second flat reading of the "
+ "bibliography would be the same text",
+ )
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+
+ assert len(fired) == 1, fired
+ assert "claude-opus-5" in fired[0].text
+ assert "settle the labels" in fired[0].text
+
+
+def test_a_silent_disable_still_discloses_a_resolution_call_that_happened():
+ """`--no-llm-refs` records `not_attempted` with no reason and the
+ disclosure stays silent — correctly, for the reading. But the escalation
+ is a separate, user-consented call that runs anyway, and silence about it
+ is the same false negative one rung down."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = _resolved_manifest(reflist_outcome="not_attempted", reflist_failure="")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+
+ assert len(fired) == 1, fired
+ assert "claude-opus-5" in fired[0].text
+ assert "the parsed extraction" in fired[0].text
+
+
+def test_a_resolution_call_that_failed_is_disclosed_though_it_settled_nothing():
+ """`_numbering_resolution` needs a resolved label to fire, so a
+ resolution call that raised would otherwise reach no structured surface
+ at all — the manifest would record a run in which no model was asked."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(
+ manuscript="p.pdf",
+ labels_disputed=["2"],
+ numbering_choice="withheld",
+ resolution_outcome="failed",
+ resolution_failure="RuntimeError: claude -p timed out after 600s",
+ resolution_readings=["parsed", "pymupdf"],
+ )
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+
+ assert len(fired) == 1, fired
+ assert "did not return" in fired[0].text
+ assert "timed out" in fired[0].text
+
+
+def test_a_manifest_with_no_resolution_call_gains_no_sentence_about_one():
+ """`""` is never computed, and an absent call says nothing."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_model="claude-opus-5")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+
+ assert len(fired) == 1, fired
+ assert "settle the labels" not in fired[0].text
+
+
+def test_the_numbering_resolution_names_the_model_and_what_it_was_shown():
+ """Critical 2's report half: "a model reading of both texts" was printed
+ on runs where the model saw one text, and on runs where the reading it
+ ruled against had no text at all."""
+ from papertrace.disclosures import _numbering_resolution
+
+ d = _numbering_resolution(_resolved_manifest())
+
+ assert d is not None
+ assert "both texts" not in d.text
+ assert "claude-opus-5" in d.text
+ assert "the parsed extraction" in d.text
+
+
+def test_the_resolved_claim_disclosure_does_not_claim_the_model_saw_both():
+ """The per-claim counterpart of the sentence above."""
+ from papertrace.disclosures import _claim_pairing
+ from papertrace.models import ClaimResult
+
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["2"])
+ d = _claim_pairing(claim, _resolved_manifest())
+
+ assert d is not None
+ assert "shown both" not in d.text
+ assert "the parsed extraction" in d.text
+
+
+def test_a_resolution_whose_manifest_recorded_no_readings_says_so():
+ """`[]` is never recorded, not "it was shown nothing" — and the sentence
+ must not silently read as the latter, nor assert the fuller account of
+ what this build's call is shown."""
+ from papertrace.disclosures import _numbering_resolution
+
+ d = _numbering_resolution(_resolved_manifest(resolution_readings=[]))
+
+ assert d is not None
+ assert "does not record which extractions" in d.text
+ assert "what each reading said" not in d.text
+
+
+# --- whole-branch review, minors 1 and 6
+
+
+def test_numbering_corroborated_keeps_never_computed_apart_from_measured_false(tmp_path):
+ """Minor 1. The dataclass and the schema both documented three states in
+ a `bool` that can hold two, so "absent means never computed" was
+ unexpressible. `None` is what the branch's own standard
+ (`reflist_entries_proposed`) already uses for exactly this."""
+ import jsonschema
+
+ old = {"manuscript": "m.pdf", "entries": []}
+ p = tmp_path / "old.json"
+ p.write_text(json.dumps(old))
+ assert RefManifest.from_json(p).numbering_corroborated is None
+
+ measured = tmp_path / "new.json"
+ RefManifest(manuscript="m.pdf", numbering_corroborated=False).to_json(measured)
+ assert RefManifest.from_json(measured).numbering_corroborated is False
+
+ schema = json.loads(SCHEMA_PATH.read_text())
+ for path in (measured,):
+ jsonschema.Draft202012Validator(schema).validate(json.loads(path.read_text()))
+ RefManifest(manuscript="m.pdf", numbering_corroborated=None).to_json(measured)
+ jsonschema.Draft202012Validator(schema).validate(json.loads(measured.read_text()))
+
+
+def test_a_mid_branch_manifest_recording_only_that_a_reading_was_attempted(tmp_path):
+ """Minor 6. A case folder written between `b74ba01` and `67ab473` carries
+ `reflist_attempted: true` and none of the fields that replaced it. The
+ key is dropped at load, so `_reflist` fell through every normalisation
+ branch and returned `None` — the required disclosure vanished for a
+ reading that actually happened, the exact false negative `67ab473` was
+ written to fix.
+
+ Neither `read` nor `failed` is provable from that flag, so neither is
+ claimed: the outcome is recorded as never computed and the report says
+ what it knows."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ p = tmp_path / "old.json"
+ p.write_text(json.dumps({"manuscript": "m.pdf", "entries": [], "reflist_attempted": True}))
+ manifest = RefManifest.from_json(p)
+
+ fired = [d for d in run_disclosures(RunResults(manuscript="m.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "does not record what came of it" in fired[0].text
+
+
+def test_a_mid_branch_manifest_with_a_model_name_still_settles_as_read(tmp_path):
+ """The stronger inference still wins: a recorded model name is positive
+ proof a reply was obtained, so the legacy flag adds nothing and must not
+ downgrade it."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ p = tmp_path / "old.json"
+ p.write_text(json.dumps({
+ "manuscript": "m.pdf", "entries": [],
+ "reflist_attempted": True, "reflist_model": "claude-opus-5",
+ }))
+ manifest = RefManifest.from_json(p)
+
+ fired = [d for d in run_disclosures(RunResults(manuscript="m.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "claude-opus-5" in fired[0].text
+ assert "does not record what came of it" not in fired[0].text
+
+
+def test_a_run_with_a_resolution_never_says_the_numbering_rests_on_nothing_else():
+ """Found by enumerating the states rather than the branches: the
+ not-obtained and failed heads end "the reference numbering rests on the
+ readings above it and nothing else", which the appended resolution
+ sentence then contradicts in the same paragraph — a resolution
+ substitutes an entry, so it is precisely something else the numbering
+ rests on."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ for reflist_state in (
+ dict(reflist_outcome="not_attempted", reflist_failure="backend is pymupdf"),
+ dict(reflist_outcome="failed", reflist_failure="RuntimeError: timed out"),
+ dict(reflist_outcome="wat"),
+ ):
+ manifest = _resolved_manifest(**reflist_state)
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, (reflist_state, fired)
+ assert "nothing else" not in fired[0].text, reflist_state
+ assert "claude-opus-5" in fired[0].text, reflist_state
+
+
+def test_a_run_with_no_resolution_keeps_the_nothing_else_claim():
+ """The other side: without a resolution the claim is true and load-bearing
+ — it is what tells a reader the numbering rests on the deterministic
+ readings alone."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="failed",
+ reflist_failure="RuntimeError: timed out")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "nothing else" in fired[0].text
+
+
+def test_a_resolution_from_an_earlier_commit_claims_only_what_it_recorded():
+ """The load axis, which is where four of this branch's defects lived: a
+ manifest written by an earlier commit of this same branch resolved a
+ label and recorded none of the `resolution_*` fields — because that call
+ was ALSO not shown the reading it ruled against. So the report may not
+ assert, for that manifest, either which extractions the model saw or that
+ it was shown what each reading said; both are true only of a run this
+ build made."""
+ from papertrace.disclosures import _claim_pairing, _numbering_resolution
+ from papertrace.models import ClaimResult
+
+ old = RefManifest(manuscript="p.pdf", labels_resolved=["2"],
+ numbering_choice="llm_resolved", numbering_chosen_by="user")
+
+ run = _numbering_resolution(old)
+ assert run is not None
+ assert "what each reading said" not in run.text
+ assert "does not record" in run.text
+
+ claim = _claim_pairing(ClaimResult(id=1, claim="x", location="R", refs=["2"]), old)
+ assert claim is not None
+ assert "what each reading said" not in claim.text
+ assert "does not record" in claim.text
+
+
+# --- fix round 2: labels_uncomparable, a documented subset of labels_disputed
+
+
+def test_labels_uncomparable_round_trips_and_validates(tmp_path):
+ """Gate 2: additive, not required, and three-valued. `None` is never
+ computed and `[]` is "computed, every dispute was a contradiction" —
+ a single empty list could not say both, which is `table_warnings`'
+ rule and the one this branch has now broken four times."""
+ import jsonschema
+
+ m = RefManifest(manuscript="p.pdf", labels_disputed=["6", "7"],
+ labels_uncomparable=["7"])
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.Draft202012Validator(schema).validate(json.loads(path.read_text()))
+ assert "labels_uncomparable" not in schema.get("required", [])
+
+ assert RefManifest.from_json(path).labels_uncomparable == ["7"]
+
+ payload = json.loads(path.read_text())
+ del payload["labels_uncomparable"]
+ path.write_text(json.dumps(payload))
+ assert RefManifest.from_json(path).labels_uncomparable is None
+
+ m2 = RefManifest(manuscript="p.pdf", labels_disputed=["6"], labels_uncomparable=[])
+ m2.to_json(path)
+ jsonschema.Draft202012Validator(schema).validate(json.loads(path.read_text()))
+ assert RefManifest.from_json(path).labels_uncomparable == []
+
+
+def test_the_run_level_disclosure_says_which_cause_applies():
+ """The disjunction was honest but weak, and the two causes warrant
+ different reader actions."""
+ from papertrace.disclosures import _labels_disputed
+
+ contradiction = _labels_disputed(
+ RefManifest(manuscript="p", labels_disputed=["6"], labels_uncomparable=[])
+ )
+ assert "named different papers" in contradiction.text
+ assert "could be compared" not in contradiction.text
+
+ uncomparable = _labels_disputed(
+ RefManifest(manuscript="p", labels_disputed=["6"], labels_uncomparable=["6"])
+ )
+ assert "nothing in them could be compared" in uncomparable.text
+ assert "named different papers" not in uncomparable.text
+
+ mixed = _labels_disputed(
+ RefManifest(manuscript="p", labels_disputed=["6", "7"], labels_uncomparable=["7"])
+ )
+ assert "[6]" in mixed.text and "[7]" in mixed.text
+ assert "named different papers" in mixed.text
+ assert "could be compared" in mixed.text
+
+
+def test_a_manifest_that_never_split_the_causes_keeps_the_disjunction():
+ """`None` is never computed — every manifest written before this field,
+ including one from an earlier commit of this branch — and the report may
+ not pick a cause for it."""
+ from papertrace.disclosures import _labels_disputed
+
+ d = _labels_disputed(RefManifest(manuscript="p", labels_disputed=["6"]))
+ assert "either" in d.text
+ assert "named different papers" in d.text and "could be compared" in d.text
diff --git a/tests/test_reference_reconciliation.py b/tests/test_reference_reconciliation.py
index 04c9061..16d9dba 100644
--- a/tests/test_reference_reconciliation.py
+++ b/tests/test_reference_reconciliation.py
@@ -872,17 +872,49 @@ def test_a_confirmed_numbering_still_says_when_the_other_reading_disagreed():
"""`_covers` is a cardinality test. If the parse matches the body's labels
the numbering is confirmed — but a deposit that disagreed is a second
independent reading saying the list is wrong, and burying it in a field no
- template renders tells the reader nothing."""
+ template renders tells the reader nothing.
+
+ The fixture was a 39-entry deposit against a 41-entry parse, which is a
+ difference of *extent*: `contested` was `crossref is not None` then, so any
+ deposit failing `_covers` set it. It now asks where the two readings stop
+ naming the same paper, so the fixture has to be a genuine content
+ disagreement to earn the flag the docstring above claims — the deposit names
+ a different work at [7].
+ """
from papertrace.refs import reconcile
+ crossref = _parsed(list(range(1, 41)))
+ crossref[6] = _entry("7", "Quite B. Another work entirely. Nature 1999;1:1.")
body = {str(n) for n in range(1, 42)}
- _entries, rec = reconcile(body, crossref=_parsed(list(range(1, 40))),
- parsed=_parsed(list(range(1, 42))))
+ _entries, rec = reconcile(body, crossref=crossref, parsed=_parsed(list(range(1, 42))))
assert rec.verified is True and rec.source == "parsed"
assert rec.contested is True, "the deposit disagreed and nobody is told"
+def test_a_deposit_that_is_merely_longer_is_not_a_contested_numbering():
+ """`contested` claimed more than the code knew.
+
+ It was `crossref is not None` — the mere existence of a deposit that failed
+ `_covers`, which requires `len(deposit) == max(body)` exactly. Measured with
+ a deposit byte-identical to the parse for every cited label and longer by two
+ references the body never cites — which publishers routinely deposit, for
+ references cited only in a supplement — the run reported `contested` and told
+ the reader "another reading of the same bibliography named different papers"
+ and "extent matched, content did not". Both were false.
+ """
+ from papertrace.refs import reconcile
+
+ body = {str(n) for n in range(1, 42)}
+ parsed = _parsed(list(range(1, 42)))
+ deposit = _parsed(list(range(1, 44))) # the same works, plus two never cited
+
+ _entries, rec = reconcile(body, crossref=deposit, parsed=parsed)
+
+ assert rec.verified is True and rec.source == "parsed"
+ assert rec.contested is False, "a longer deposit is not a second reading naming other papers"
+
+
def test_both_readings_agreeing_leaves_no_phantom_divergence():
"""`_first_divergence` returned `min(len)+1` when the readings agreed, so a
41-entry list warned about "entries from [42] onward" while tainting no
@@ -1149,6 +1181,156 @@ def test_a_narrowed_doubt_still_taints_only_the_tail():
assert [x for x in ("1", "14", "15", "19") if m.label_is_doubtful(x)] == ["15", "19"]
+# --- the ledger, and a contested-but-verified numbering -----------------------
+
+
+def _render_with_manifest(results, manifest, tmp_path) -> dict[str, str]:
+ """Render every report format and return each one's text, keyed by filename.
+
+ No existing helper in this module passes a manifest through `write_reports`
+ and reads back every look, so this is added, minimal and local here.
+ """
+ from papertrace.report import write_reports
+
+ write_reports(results, manifest, tmp_path, png=False)
+ names = ("report.md", "report_editor.html", "report_terminal.html", "report_viewer.html")
+ return {name: (tmp_path / name).read_text() for name in names}
+
+
+def test_the_note_names_the_dropped_and_duplicated_numerals():
+ """A difference of two totals understated the real damage sixfold: the run
+ that surfaced this said '[1]-[19] vs 18 references' where 5 references were
+ dropped and 1 numeral duplicated. `_covers` computes the gap set already."""
+ from papertrace.refs import reconcile
+
+ body = {str(i) for i in range(1, 8)}
+ parsed = [
+ _entry("1", "one"), _entry("2", "two"), _entry("3", "three"),
+ _entry("7", "seven-a"), _entry("7", "seven-b"),
+ ]
+ _entries, rec = reconcile(body, None, parsed, crossref_absent="no DOI")
+ assert rec.ledger["numerals_absent"] == ["4", "5", "6"]
+ assert rec.ledger["numerals_duplicated"] == ["7"]
+ assert "[7] appears more than once" in rec.note
+ assert "[4], [5], [6] are carried by no entry" in rec.note
+
+
+def _unconfirmed_note(**kwargs) -> str:
+ """The note a five-entry parse earns against a body citing [1]-[7]."""
+ from papertrace.refs import reconcile
+
+ body = {str(i) for i in range(1, 8)}
+ parsed = [
+ _entry("1", "one"), _entry("2", "two"), _entry("3", "three"),
+ _entry("7", "seven-a"), _entry("7", "seven-b"),
+ ]
+ _entries, rec = reconcile(body, None, parsed, **kwargs)
+ return rec.note
+
+
+def test_the_unconfirmed_numbering_note_is_sentences_and_not_a_dash_run_on():
+ """Four em-dashes in one sentence, and the crossref clause in the middle of
+ it. `CROSSREF_NO_DOI` carries its own em-dash AND its own full stop, so the
+ rendered sentence appeared to end at "Pass --doi if the paper does have one"
+ and then resumed at "that does not add up, so the numbering could not be
+ confirmed". This text reaches markdown, editor, terminal and viewer verbatim,
+ where the README calls the report a correctness surface.
+ """
+ from papertrace.refs import CROSSREF_NO_DOI
+
+ note = _unconfirmed_note(crossref_absent=CROSSREF_NO_DOI)
+
+ # the ledger ends; the verdict is its own sentence
+ assert "carried by no entry. That does not add up, so the numbering could not be " \
+ "confirmed." in note
+ # and the crossref clause is last, reading as its own sentence
+ assert note.endswith("Pass --doi if the paper does have one")
+ assert note.count("—") == 2, f"one for the ledger, one CROSSREF_NO_DOI's own: {note}"
+
+
+def test_the_ledger_clause_agrees_in_number():
+ """"[7] appear more than once" for a single label. The plural has to stay
+ right for several, so this is not a constant."""
+ one_dup = _unconfirmed_note()
+ assert "[7] appears more than once" in one_dup
+
+ from papertrace.refs import reconcile
+
+ body = {str(i) for i in range(1, 5)}
+ parsed = [_entry("1", "a"), _entry("1", "b"), _entry("2", "c"), _entry("2", "d")]
+ _entries, rec = reconcile(body, None, parsed)
+ assert "[1], [2] appear more than once" in rec.note
+ assert "[3], [4] are carried by no entry" in rec.note
+
+ body = {"1", "2"}
+ _entries, rec = reconcile(body, None, [_entry("1", "a"), _entry("1", "b"), _entry("1", "c")])
+ assert "[2] is carried by no entry" in rec.note
+
+
+def test_a_contested_but_verified_numbering_reaches_every_format(tmp_path):
+ """`contested` has never reached a reader. It is declared with a comment
+ saying that burying it was how a compensating parse error passed
+ unmentioned, and it is then buried: never persisted, and `_numbering` is
+ gated on `not verified` so a verified-but-contested run says nothing."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(
+ manuscript="m.pdf",
+ entries=_parsed([1, 2]),
+ numbering_verified=True,
+ numbering_contested=True,
+ )
+ results = _results()
+ reports = _render_with_manifest(results, manifest, tmp_path)
+ for name, text in reports.items():
+ assert "second reading of the reference list disagreed" in text, name
+
+
+def test_the_ledger_round_trips_and_older_manifests_still_load(tmp_path):
+ """Gate 2.
+
+ The ledger a real `reconcile` writes is validated against the schema, not
+ only a hand-made one: `numbering_ledger` documents seven keys and their
+ types, and a hand-made fixture cannot tell whether the code still emits
+ those keys with those types.
+ """
+ import jsonschema
+
+ from papertrace.models import RefManifest
+ from papertrace.refs import reconcile
+
+ _entries, rec = reconcile({"1", "2", "3"}, None, [_entry("1", "a"), _entry("1", "b")])
+ m = RefManifest(
+ manuscript="m.pdf",
+ entries=_parsed([1]),
+ numbering_contested=True,
+ numbering_ledger=rec.ledger,
+ )
+ p = tmp_path / "refs_manifest.json"
+ m.to_json(p)
+ schema = json.loads(
+ (Path(__file__).resolve().parent.parent / "schemas" / "refs_manifest.schema.json").read_text()
+ )
+ jsonschema.validate(json.loads(p.read_text()), schema)
+ assert set(rec.ledger) == set(schema["properties"]["numbering_ledger"]["properties"]), \
+ "every ledger key the code writes is documented, and nothing more"
+
+ back = RefManifest.from_json(p)
+ assert back.numbering_contested is True
+ assert back.numbering_ledger["numerals_absent"] == ["2", "3"]
+
+ m.numbering_ledger = {"numerals_absent": ["4"], "numerals_duplicated": []}
+ m.to_json(p)
+ jsonschema.validate(json.loads(p.read_text()), schema) # a partial ledger is valid
+ assert RefManifest.from_json(p).numbering_ledger["numerals_absent"] == ["4"]
+
+ old = {"manuscript": "m.pdf", "entries": [{"num": "1", "raw": "x", "status": "paywalled"}]}
+ p.write_text(json.dumps(old))
+ older = RefManifest.from_json(p)
+ assert older.numbering_contested is False
+ assert older.numbering_ledger == {}
+
+
# --- a file nobody named for this reference ----------------------------------
diff --git a/tests/test_reference_tables.py b/tests/test_reference_tables.py
new file mode 100644
index 0000000..a68823a
--- /dev/null
+++ b/tests/test_reference_tables.py
@@ -0,0 +1,217 @@
+"""A bibliography the converter rendered as a markdown table.
+
+Docling reads a hanging-indent numeral column as a table and emits GFM. The
+rows then reach `parse_references` as text, where the marker regex cannot see
+them (`\\s*` cannot cross a `|`) and `_parse_bulleted` treats them as wrapped
+continuations — gluing five references onto one entry and scraping the wrong
+DOI onto it. The observed shapes here are copied from a real `source_map.json`
+table block, including the detail that the separator row is SECOND: docling
+promotes the table's first data row to the markdown header row, and that row
+is the tail of the bullet above it.
+"""
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+
+from papertrace.refs import ( # noqa: E402
+ _covers,
+ _entry,
+ _numerals_agree_with_position,
+ _parse_bulleted,
+ parse_references,
+ reconcile,
+)
+
+# 3 flat bullets, then a table holding references 4-7. The table's first row is
+# Esteva's continuation, the separator is second, and reference 5 wraps across
+# two rows with an empty numeral cell.
+TABLE_BIBLIOGRAPHY = """\
+- Shen D, Wu G, Suk H-I (2017) Deep learning in medical image analysis. Annu Rev Biomed Eng. 19:221-248. https://doi.org/10.1146/annurev-bioeng-071516-044442
+- Litjens G, Kooi T, Bejnordi BE et al (2017) A survey on deep learning in medical image analysis. Med Image Anal. 42:60-88. https://doi.org/10.1016/j.media.2017.07.005
+- Esteva A, Kuprel B, Novoa RA et al (2017) Dermatologist-level classification of skin
+
+| | cancer with deep neural networks. Nature. 542:115-118. https://doi.org/10.1038/nature21056 |
+|-----|--------------------------------------------------------------------------------------------|
+| 4. | Erickson BJ, Korfiatis P, Akkus Z et al (2017) Machine learning for medical imaging. |
+| | Radiographics. 37:505-515. https://doi.org/10.1148/rg.2017160130 |
+| 5. | Weston AD, Korfiatis P, Kline TL et al (2019) Automated abdominal segmentation of CT |
+| | scans for body composition analysis. Radiology. 290:669-679. https://doi.org/10.1148/radiol.2018181432 |
+| 6. | Wasserthal J, Breit H-C, Meyer MT et al (2023) TotalSegmentator: robust segmentation of |
+| | anatomic structures in CT images. Radiol Artif Intell. 5:e230024. https://doi.org/10.1148/ryai.230024 |
+| 7. | Graf R, Platzek P, Riedel EO et al (2026) VIBESegmentator: full body MRI segmentation. |
+| | Eur Radiol. 36:2548-2562. https://doi.org/10.1007/s00330-025-12035-9 |
+"""
+
+
+def test_a_table_row_is_a_reference_not_a_continuation():
+ """Seven printed references must parse as seven entries labelled 1-7."""
+ entries = parse_references(TABLE_BIBLIOGRAPHY)
+ assert [e.num for e in entries] == ["1", "2", "3", "4", "5", "6", "7"]
+
+
+def test_a_wrapped_row_with_an_empty_numeral_cell_joins_the_row_above():
+ """The wrong-paper route, and the most important assertion in this module.
+
+ Erickson's DOI is printed in a continuation row of Erickson's own entry.
+ Glued onto Esteva's entry instead, `_entry` scrapes it, the resolver
+ downloads Erickson's paper, and `_title_check` PASSES because the raw
+ string now contains both titles. Every claim citing [3] is then judged
+ against [4]'s paper with `identity confirmed` printed beside it.
+ """
+ entries = {e.num: e for e in parse_references(TABLE_BIBLIOGRAPHY)}
+ assert entries["3"].doi == "10.1038/nature21056"
+ assert entries["4"].doi == "10.1148/rg.2017160130"
+ assert "Erickson" not in entries["3"].raw
+
+
+def test_the_table_separator_row_is_not_reference_text():
+ """`|-----|` is layout. Reaching `_slug`, `_title_check` and the Crossref
+ bibliographic search as part of a reference string is noise at best."""
+ for e in parse_references(TABLE_BIBLIOGRAPHY):
+ assert "|---" not in e.raw
+ assert "|" not in e.raw
+
+
+def test_the_tables_first_row_can_be_the_tail_of_the_bullet_above_it():
+ """Observed, not hypothetical: docling promotes the first data row to the
+ header row, and on the real manuscript that row was reference 5's tail. An
+ implementation that skips a header row deletes it."""
+ entries = {e.num: e for e in parse_references(TABLE_BIBLIOGRAPHY)}
+ assert "Nature. 542:115-118" in entries["3"].raw
+
+
+def test_a_volume_number_in_a_cell_is_not_a_reference_label():
+ """Gate 4. The numeral must be the WHOLE cell: `2017;19` begins with digits
+ and is a volume. This module already carries the `a-1061` scar from reading
+ an issue number as a year."""
+ text = (
+ "- Shen D, Wu G (2017) Deep learning. Annu Rev. https://doi.org/10.1/a\n"
+ "\n"
+ "| 2017;19 | 221-248 |\n"
+ "|---------|---------|\n"
+ )
+ entries = parse_references(text)
+ assert [e.num for e in entries] == ["1"]
+
+
+def test_a_bibliography_rendered_entirely_as_a_table_is_parsed_with_its_printed_labels():
+ """Today this returns [], which reaches the CLI's honest `Exit(1)`. Parsing
+ it with the printed numerals is strictly better than refusing."""
+ text = (
+ "| 1. | Shen D, Wu G (2017) Deep learning. Annu Rev. https://doi.org/10.0001/a |\n"
+ "|-----|-----------------------------------------------------------------------|\n"
+ "| 2. | Litjens G, Kooi T (2017) A survey. Med Image Anal. https://doi.org/10.0002/b |\n"
+ "| 3. | Esteva A, Kuprel B (2017) Dermatologist-level. Nature. https://doi.org/10.0003/c |\n"
+ )
+ entries = parse_references(text)
+ assert [e.num for e in entries] == ["1", "2", "3"]
+ assert entries[1].doi == "10.0002/b"
+
+
+def test_numerals_surviving_on_only_some_items_still_fix_the_labels():
+ """A list docling numbered half as bullets (numerals stripped) and half as
+ a table (numerals kept) prints no [1], so `_usable_printed_numerals`
+ refused it and the caller numbered by position — the guess it exists to
+ avoid making unchecked. Where the numerals that ARE printed sit at the
+ positions they name, position is the printed reading, corroborated.
+ """
+ assert _numerals_agree_with_position([None, None, 3]) is True
+ assert _numerals_agree_with_position([1, None, 3, 4]) is True
+ assert _numerals_agree_with_position([None, None, 9, 10, 11]) is False
+ assert _numerals_agree_with_position([2, 3]) is False
+
+
+def test_printed_numerals_contradicting_their_position_are_not_numbered_by_position():
+ """Gate 4. Numerals 9-11 at positions 3-5 mean something above them was
+ merged or split, so position is KNOWN wrong rather than merely unverified.
+ Neither reading is available: refuse, and refuse to resolve."""
+ text = (
+ "- Shen D, Wu G (2017) Deep learning. Annu Rev. https://doi.org/10.1/a\n"
+ "- Litjens G, Kooi T (2017) A survey. Med Image Anal. https://doi.org/10.2/b\n"
+ "- 9. Esteva A (2017) Dermatologist-level. Nature. https://doi.org/10.3/c\n"
+ "- 10. Erickson BJ (2017) Machine learning. Radiographics. https://doi.org/10.4/d\n"
+ "- 11. Weston AD (2019) Automated abdominal. Radiology. https://doi.org/10.5/e\n"
+ )
+ entries = _parse_bulleted(text)
+ # The nums stay positional — an entry needs some label to be a dict key and
+ # a slug. What changes is that they are no longer *trusted*: every entry
+ # from the first contradiction on refuses to resolve, so no claim is judged
+ # against a paper this numbering picked.
+ affected = [e for e in entries if e.boundary_ambiguous]
+ assert affected, "a contradicted numbering must be refused, not relabelled"
+ assert all(e.doi is None for e in affected)
+ assert all("numbering" in e.reason for e in affected)
+
+
+def test_a_refused_numbering_is_never_confirmed_by_a_matching_count():
+ """The refuse branch and `_covers` compose into a lie.
+
+ A refused entry keeps a positional label — it needs one to be a dict key and
+ a slug — so the parsed list still carries `{1..N}`, `_covers`' subset test
+ cannot bite and only its length test can. A five-item list refused from
+ entry 4 on, against a body citing [1]-[5], therefore reported `verified` and
+ fired neither the run-level `numbering` disclosure nor a single
+ `claim_numbering` caveat, while `refs.py` wrote "printed numbering
+ contradicts position from entry 4 on" onto two of the entries in the same
+ run. An extent check cannot confirm a numbering the parser refused.
+ """
+ text = (
+ "- Shen D, Wu G (2017) Deep learning. Annu Rev. https://doi.org/10.1/a\n"
+ "- Litjens G, Kooi T (2017) A survey. Med Image Anal. https://doi.org/10.2/b\n"
+ "- Esteva A (2017) Dermatologist-level. Nature. https://doi.org/10.3/c\n"
+ "- 9. Erickson BJ (2017) Machine learning. Radiographics. https://doi.org/10.4/d\n"
+ "- 10. Weston AD (2019) Automated abdominal. Radiology. https://doi.org/10.5/e\n"
+ )
+ entries = _parse_bulleted(text)
+ assert [e.num for e in entries] == ["1", "2", "3", "4", "5"]
+ assert [e.num for e in entries if e.boundary_ambiguous] == ["4", "5"]
+
+ body = {"1", "2", "3", "4", "5"}
+ assert _covers(body, entries), "the count agrees — which is the whole trap"
+ _chosen, rec = reconcile(body, None, entries)
+
+ assert rec.verified is False, "the parser refused these labels; a count cannot confirm them"
+ assert rec.unverified_from == 4, "the first refused entry is where the doubt starts"
+ assert "[4], [5]" in rec.note
+
+
+def test_a_list_printing_no_numeral_anywhere_is_still_numbered_by_position():
+ """Gate 4, the other half, and an explicit regression assertion: the
+ Nature-family case where the converter really did strip them is the one
+ situation in which position is the only reading available. It keeps
+ working, and `reconcile` still marks it unverified."""
+ text = (
+ "- Shen D, Wu G (2017) Deep learning. Annu Rev. https://doi.org/10.1/a\n"
+ "- Litjens G, Kooi T (2017) A survey. Med Image Anal. https://doi.org/10.2/b\n"
+ )
+ entries = _parse_bulleted(text)
+ assert [e.num for e in entries] == ["1", "2"]
+ assert not any(e.boundary_ambiguous for e in entries)
+
+
+def test_a_trailing_run_of_stripped_numerals_is_accepted_but_never_verified():
+ """The shape the real manuscript actually produces, pinned.
+
+ Docling keeps the numerals it found inside a table and strips them from
+ every bullet after it, so the printed run stops partway:
+ [None x5, 6..11, None x13]. A None agrees with any position, so the labels
+ are accepted rather than refused — the contradiction branch never fires.
+
+ That is the honest answer only because nothing calls it verified. Two
+ split-title fragments are still counted as entries, so the list is longer
+ than the highest label the body cites and `_covers` fails. This test exists
+ so a later change cannot quietly promote acceptance into confirmation.
+ """
+ numerals = [None] * 5 + [6, 7, 8, 9, 10, 11] + [None] * 13
+ assert _numerals_agree_with_position(numerals) is True
+
+ body = {str(i) for i in range(1, 20)} # the body cites [1]-[19]
+ entries = [
+ _entry(str(i), f"Author {i} A (2020) A paper title. J Test. https://doi.org/10.1000/x{i}")
+ for i in range(1, 25) # the parse holds 24, two of them fragments
+ ]
+ assert not _covers(body, entries)
+ _chosen, rec = reconcile(body, None, entries, crossref_absent="no DOI for the paper")
+ assert rec.verified is False
diff --git a/tests/test_reflist.py b/tests/test_reflist.py
new file mode 100644
index 0000000..815c330
--- /dev/null
+++ b/tests/test_reflist.py
@@ -0,0 +1,696 @@
+"""The model's reading of a bibliography, and every rule that disbelieves it.
+
+One `claude -p` call is shown two extractions of the same printed reference list
+and asked what numbered list the page carries. Nothing it replies is accepted on
+its own word: each field is searched for, verbatim, in one of the two texts it
+was shown. These tests are the rules, one per test, because each of them is a
+way this module could otherwise name a paper the page never printed.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+
+from papertrace import ask as ask_mod # noqa: E402
+from papertrace import reflist # noqa: E402
+from papertrace.refs import DOI_RE # noqa: E402
+
+# Reading A: the run backend's extraction. Reading B: the flat-text one. The
+# same three references, printed the same way, with the damage each converter
+# leaves behind.
+READING_A = """References
+1. Fujita S, Mori S, Onda K, et al. Characterization of brain volume changes in
+aging individuals. JAMA Netw Open. 2023;6(6):e2318153. doi:10.1001/jamanetworkopen.2023.18153
+2. Küstner T, Müller M. Longitudinal segmentation of the ageing cortex.
+Nat Aging. 2020;4(11):1619-1634. doi:10.1038/s41591-019-0673-2
+3. Wachinger C. Atlas-based methods. Med Image Anal. 2019;51:1-10.
+"""
+
+READING_B = """References
+1 . Fujita S, Mori S, Onda K, et al. Characterization of brain volume changes in
+aging individuals. JAMA Netw Open. 2023;6(6):e2318153.
+2 . Kustner T, Muller M. Longitu-
+dinal segmentation of the ageing cortex. Nat Aging. 2020;4(11):1619-1634.
+3 . Wachinger C. Atlas-based methods. Med Image Anal. 2019;51:1-10.
+"""
+
+
+def _reply(monkeypatch, payload: str, *, model: str = "claude-opus-5"):
+ """Patch the seam itself — `reflist` calls it by qualified name on purpose."""
+ calls = []
+
+ def fake(prompt, model_arg=None):
+ calls.append({"prompt": prompt, "site": ask_mod._SITE})
+ return payload
+
+ monkeypatch.setattr(ask_mod, "_ask", fake)
+ monkeypatch.setattr(ask_mod, "_MODELS", {ask_mod.SITE_REFS: model})
+ return calls
+
+
+def _propose(monkeypatch, payload: str, **kwargs):
+ _reply(monkeypatch, payload)
+ kwargs.setdefault("label_a", "docling 2.8.0")
+ kwargs.setdefault("label_b", "pymupdf")
+ return reflist.propose(READING_A, READING_B, **kwargs)
+
+
+def test_a_field_not_found_verbatim_in_either_reading_is_discarded(monkeypatch):
+ """The field goes, the entry stays.
+
+ An invented journal name is a wrong citation string, not a wrong paper, and
+ dropping the whole entry for it would throw away a title and a DOI that
+ *were* printed. The gap is recorded so the report can name it.
+ """
+ entries, prov = _propose(monkeypatch, """[
+ {"num":"3","authors":"Wachinger C","year":"2019",
+ "title":"Atlas-based methods","journal":"Nature Reviews Neurology","reading":"AB"}
+ ]""")
+
+ assert [e.num for e in entries] == ["3"]
+ assert "Nature Reviews Neurology" not in entries[0].raw
+ assert "Atlas-based methods" in entries[0].raw
+ assert any("journal" in f for f in prov.fields_discarded), prov.fields_discarded
+
+
+def test_a_title_not_found_verbatim_discards_the_whole_candidate(monkeypatch):
+ """A title nobody printed is the one failure that cannot be localised.
+
+ Every other field describes a paper; the title *is* the paper. If the model
+ named a work the page does not carry, nothing else it said about that entry
+ is worth keeping either — and a list one of whose entries names the wrong
+ paper cannot be used as a reading of the list at all.
+ """
+ entries, prov = _propose(monkeypatch, """[
+ {"num":"3","authors":"Wachinger C","year":"2019",
+ "title":"Deep learning for cortical parcellation","reading":"A"}
+ ]""")
+
+ assert entries == []
+ assert "[3]" in prov.discarded_whole and "title" in prov.discarded_whole, prov.discarded_whole
+
+
+def test_a_line_broken_doi_yields_no_doi_rather_than_a_repair(monkeypatch):
+ """Ruling 3. The truncated prefix passes both gates it would have had.
+
+ `DOI_RE` is `10\\.\\d{4,9}/[^\\s"'<>]+` and a trailing hyphen is a fine
+ `[^\\s"'<>]`, so `10.1038/s41591-` matches it — and the verbatim check passes
+ too, because that prefix really is printed on the page. A DOI is an identity:
+ half of one resolves to nothing or to something else, and a repaired one
+ would name whichever paper the guess landed on.
+ """
+ truncated = "10.1038/s41591-"
+ # both gates the spec assumed would stop this, measured, on this fixture
+ assert DOI_RE.fullmatch(truncated), "premise changed: DOI_RE now rejects the prefix"
+ assert reflist._found("doi", truncated, (READING_A, READING_B)), (
+ "premise changed: the prefix is no longer printed in the readings"
+ )
+ assert reflist._usable_doi(truncated) is None, "a half DOI was accepted"
+
+ entries, prov = _propose(monkeypatch, """[
+ {"num":"2","authors":"Kustner T, Muller M",
+ "title":"Longitudinal segmentation of the ageing cortex",
+ "doi":"10.1038/s41591-","reading":"A"}
+ ]""")
+
+ assert [e.num for e in entries] == ["2"]
+ assert entries[0].doi is None
+ assert "10.1038/s41591-" not in (entries[0].raw or "")
+ assert any("doi" in f for f in prov.fields_discarded), prov.fields_discarded
+
+
+def test_a_whole_doi_still_verifies_case_insensitively(monkeypatch):
+ """The negative above must not be bought by rejecting real DOIs.
+
+ Publishers print `doi:10.1001/...` in mixed case and the two extractions
+ disagree about that case, so the DOI comparison folds case and nothing else
+ — de-hyphenating a DOI would join `s41591-019` into a different registrant.
+ """
+ entries, _ = _propose(monkeypatch, """[
+ {"num":"1","title":"Characterization of brain volume changes in\\naging individuals",
+ "doi":"10.1001/JAMANETWORKOPEN.2023.18153","reading":"AB"}
+ ]""")
+
+ assert entries[0].doi == "10.1001/JAMANETWORKOPEN.2023.18153"
+
+
+def test_an_umlaut_in_an_author_name_still_verifies_both_ways(monkeypatch):
+ """`Küstner` printed, `Kustner` proposed — and the reverse.
+
+ The two extractions of one page disagree about diacritics, so a comparison
+ that did not fold them would discard a correctly copied author on half the
+ non-English bibliography in existence. Layer O is applied to BOTH sides;
+ folding one side of a substring test is the bug Plan A had to fix in
+ `_title_check_text`.
+ """
+ readings = (READING_A, READING_B)
+ assert reflist._found("authors", "Kustner T, Muller M", readings)
+ assert reflist._found("authors", "Küstner T, Müller M", readings)
+
+ entries, prov = _propose(monkeypatch, """[
+ {"num":"2","authors":"Kustner T, Muller M",
+ "title":"Longitudinal segmentation of the ageing cortex","reading":"AB"}
+ ]""")
+
+ assert "Kustner" in entries[0].raw
+ assert not any("authors" in f for f in prov.fields_discarded), prov.fields_discarded
+
+
+def test_a_ligature_and_a_soft_hyphenated_title_still_verify(monkeypatch):
+ """Extraction damage is not disagreement.
+
+ pymupdf prints `Longitu-\\ndinal` where docling prints `Longitudinal`, and a
+ PDF font's `fi` reaches one converter as one codepoint. Layer E undoes what
+ the extractor did to the ink and nothing else, on both sides.
+ """
+ assert reflist._found("title", "Longitudinal segmentation", (READING_B,))
+ assert reflist._found("title", "Characterization of fine detail",
+ ("Characterization of fine detail",))
+ # and it must still refuse the same string with the ligature's letters gone
+ assert not reflist._found("title", "Characterization of ne detail",
+ ("Characterization of fine detail",))
+
+
+def test_a_duplicated_numeral_is_recorded_and_never_repaired(monkeypatch):
+ """A numeral is the join key; renumbering it is how a verdict changes paper.
+
+ Two entries proposed as `[2]` means the reading cannot say which paper `[2]`
+ is — which is precisely the question. Both entries are kept exactly as
+ proposed, the duplication is recorded, and `label_agreement` refuses to let
+ this reading speak for `2` at all. Nothing is renumbered to `3`.
+ """
+ entries, prov = _propose(monkeypatch, """[
+ {"num":"2","title":"Longitudinal segmentation of the ageing cortex","reading":"A"},
+ {"num":"2","title":"Atlas-based methods","reading":"B"}
+ ]""")
+
+ assert [e.num for e in entries] == ["2", "2"], "an entry was dropped or renumbered"
+ # a numbering finding, not a discarded field — the two carry different
+ # consequences and are reported separately
+ assert any("twice" in f for f in prov.numbering_findings), prov.numbering_findings
+ assert not any("twice" in f for f in prov.fields_discarded), prov.fields_discarded
+
+
+def test_a_reply_that_is_not_json_yields_an_empty_candidate_not_an_exception(monkeypatch):
+ """`propose` reports its own failure; it does not raise it at the refs stage.
+
+ The refs stage has three other readings to proceed on. A model that
+ answered with prose has told us nothing, and "nothing" is a representable
+ result — an exception here would take down a stage that was doing fine.
+ """
+ entries, prov = _propose(monkeypatch, "I could not read that bibliography, sorry.")
+
+ assert entries == []
+ assert prov.discarded_whole, "the failure was silent"
+ assert prov.entries_proposed == 0
+
+
+def test_a_paper_printed_in_neither_reading_contributes_nothing(monkeypatch):
+ """The reply mixes one real entry with one fabricated paper.
+
+ This is the failure mode the whole module exists for, and the real entry
+ beside it is the temptation: keeping it would mean proceeding on a reading
+ that demonstrably invents. Neither entry survives, and the reason names the
+ fabricated one.
+ """
+ entries, prov = _propose(monkeypatch, """[
+ {"num":"1","title":"Characterization of brain volume changes in\\naging individuals","reading":"AB"},
+ {"num":"4","authors":"Nobody N","year":"2024",
+ "title":"A paper that was never printed","journal":"J Fabrication","reading":"A"}
+ ]""")
+
+ assert entries == []
+ assert "[4]" in prov.discarded_whole, prov.discarded_whole
+ assert prov.entries_proposed == 2, "the count of what was proposed is still recorded"
+
+
+def test_propose_makes_exactly_one_model_call_at_the_refs_site(monkeypatch):
+ """One call per bibliography, attributed to `refs`, not to `check`.
+
+ Two things break if this drifts. A per-entry call would read one reference
+ list at N times the cost for no extra evidence; and a call attributed to the
+ wrong site makes the report's `Checker:` line name the reference-list model
+ as the judge of verdicts it never saw — the bug Task 1 exists to prevent.
+ """
+ calls = _reply(monkeypatch, "[]")
+ entries, prov = reflist.propose(READING_A, READING_B,
+ label_a="docling 2.8.0", label_b="pymupdf")
+
+ assert len(calls) == 1, calls
+ assert calls[0]["site"] == ask_mod.SITE_REFS
+ assert entries == []
+ assert prov.model == "claude-opus-5"
+ assert prov.readings == ["docling 2.8.0", "pymupdf"]
+ # both texts reached the prompt, each in its own slot
+ assert "Kustner T, Muller M" in calls[0]["prompt"]
+ assert "Küstner T, Müller M" in calls[0]["prompt"]
+ assert "<>" not in calls[0]["prompt"] and "<>" not in calls[0]["prompt"]
+
+
+def test_the_models_reading_letter_is_translated_into_a_reading_name(monkeypatch):
+ """One published field, one vocabulary — the translation happens here.
+
+ The model answers in the terms the prompt gave it (`A`, `B`, `AB`), because
+ that is what it was shown. `RefEntry.seen_in` is a *published* field whose
+ vocabulary is reading NAMES, so that a consumer can ask `"llm" in seen_in`
+ and get a true answer on every manifest. Letting `A` reach the field would
+ put two vocabularies in one wire-format key, and a consumer checking either
+ one would silently never match manifests written by the other path.
+ """
+ entries, _ = _propose(
+ monkeypatch,
+ """[
+ {"num":"3","title":"Atlas-based methods","reading":"AB"}
+ ]""",
+ label_a="docling",
+ label_b="pymupdf",
+ )
+
+ assert entries[0].seen_in == ["docling", "pymupdf"]
+
+
+def test_an_unrecognised_reading_letter_is_dropped_rather_than_guessed(monkeypatch):
+ """A model answering "the docling one" must not land a sentence in a
+ published field, and must not be *assumed* to have meant reading A either.
+ Unknown provenance is recorded as unknown."""
+ entries, _ = _propose(monkeypatch, """[
+ {"num":"3","title":"Atlas-based methods","reading":"whichever one was clearer"}
+ ]""")
+
+ assert entries[0].seen_in == [], entries[0].seen_in
+
+
+def test_an_empty_reading_makes_no_model_call_at_all(monkeypatch):
+ """Nothing to check against means nothing can be believed.
+
+ A run whose second extraction came back empty would otherwise pay for a call
+ whose every field could only be verified against one text — half the
+ verification, at full price, with no way for the report to say so.
+ """
+ calls = _reply(monkeypatch, "[]")
+ entries, prov = reflist.propose(READING_A, " ", label_a="docling 2.8.0", label_b="pymupdf")
+
+ assert calls == []
+ assert entries == []
+ # not `discarded_whole`: no reply was ever obtained to check and refuse —
+ # the call never happened, which `failure` says and `outcome` confirms
+ assert prov.failure
+ assert prov.outcome == "not_attempted"
+
+
+def test_the_entry_level_seen_in_round_trips_through_the_manifest(tmp_path):
+ """Gate 2. A new persisted field that `from_json` drops is a silent loss.
+
+ `_ref_entry_from` filters on `fields(RefEntry)`, so declaring the field is
+ enough — this test is what says so rather than assuming it.
+ """
+ import json
+
+ import jsonschema
+
+ from papertrace.models import RefEntry, RefManifest
+
+ m = RefManifest(manuscript="p.pdf",
+ entries=[RefEntry(num="1", raw="A. A paper. 2020.", seen_in=["docling", "pymupdf"])])
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+
+ payload = json.loads(path.read_text())
+ assert payload["entries"][0]["seen_in"] == ["docling", "pymupdf"]
+ schema = json.loads(
+ (Path(__file__).resolve().parent.parent / "schemas" / "refs_manifest.schema.json").read_text()
+ )
+ jsonschema.validate(payload, schema)
+ assert "seen_in" not in schema["properties"]["entries"]["items"].get("required", [])
+ assert RefManifest.from_json(path).entries[0].seen_in == ["docling", "pymupdf"]
+
+
+# --- an entry that names no paper is not an entry ---------------------------
+
+
+def test_a_reply_with_a_numeral_and_nothing_else_yields_no_entry(monkeypatch):
+ """The whole-candidate discard fires when a title is PROPOSED and fails.
+
+ A reply supplying only `{"num": "9", "reading": "A"}` never reaches it, and
+ would otherwise become a voter carrying no claim about any work. It cannot
+ produce a wrong verdict — `label_agreement` compares through
+ `_comparably_same`, which refuses to call two uncomparable entries `agreed`
+ — but it can make the label `disputed` and withhold verdicts the model never
+ said anything against. A degenerate reply may cost this reading its vote; it
+ may not cost the audit its answers.
+ """
+ entries, prov = _propose(monkeypatch, '[{"num":"9","reading":"A"}]')
+
+ assert entries == []
+ assert any("no verified title or doi" in f for f in prov.fields_discarded), (
+ prov.fields_discarded
+ )
+
+
+def test_an_entry_kept_on_a_doi_alone_still_counts(monkeypatch):
+ """The rule is "names no paper", not "has no title". A DOI identifies a work
+ on its own — `_same_work` settles on it before it looks at anything else —
+ so an entry whose title the converter mangled but whose DOI is printed and
+ whole is a legitimate voter."""
+ entries, _ = _propose(
+ monkeypatch,
+ '[{"num":"7","doi":"10.1038/s41591-019-0673-2","reading":"A"}]',
+ )
+
+ assert len(entries) == 1
+ assert entries[0].doi == "10.1038/s41591-019-0673-2"
+ assert entries[0].title is None
+
+
+def test_a_numbering_finding_is_not_reported_as_a_discarded_field(monkeypatch):
+ """Two different things, two different lists.
+
+ A discarded field is "this value was not printed, so it was dropped". A
+ numbering finding is "this reading's labels do not add up", which the spec
+ gives a different consequence. Sharing one list made
+ `reflist_fields_discarded` report a count of discarded fields that included
+ things no field ever lost.
+ """
+ entries, prov = _propose(
+ monkeypatch,
+ '[{"num":"6","title":"Characterization of brain volume changes in aging '
+ 'individuals","reading":"A"}]',
+ )
+
+ assert len(entries) == 1
+ assert prov.fields_discarded == [], prov.fields_discarded
+ assert any("absent from 1..max" in f for f in prov.numbering_findings), (
+ prov.numbering_findings
+ )
+
+
+def test_a_reading_letter_of_the_wrong_type_is_reported_as_a_wrong_shape(monkeypatch):
+ """A reply of `5` is a different failure from a field the model left out,
+ and folding both into "nothing to say" loses the distinction. Either way
+ `seen_in` stays empty — the outcome was always honest, only the record was
+ silent."""
+ entries, prov = _propose(
+ monkeypatch,
+ '[{"num":"6","title":"Characterization of brain volume changes in aging '
+ 'individuals","reading":5}]',
+ )
+
+ assert entries[0].seen_in == []
+ assert any("reading: not a string" in f for f in prov.fields_discarded), (
+ prov.fields_discarded
+ )
+
+
+# --- resolving a disputed label: three rules on the reply, and a slug -------
+#
+# `resolve_disputed` is the only place a model reading changes what `check`
+# judges — a person has already seen the disagreement and asked for this. Zero
+# of these tests existed before this round (M2 of the Task 7 review): every
+# test that reached menu option 1 replaced this function with a stub built
+# through `refs._entry`, which sets the one field the real function forgot
+# (C1 — a slug-less entry is silently dropped from judgement by `check.py` and
+# downloads to `sources_resolved/None.pdf`, overwriting itself label after
+# label). These are direct, so that mistake cannot repeat.
+
+_RESOLVE_A = "6. Fujita S, Mori S. Characterization of Brain Volume Changes. 2023. doi:10.1001/jamanetworkopen.2023.18153\n"
+_RESOLVE_B = "6. Wachinger C. Something else entirely. 2019. doi:10.1000/wach\n"
+_RESOLVE_NAMES = ("docling 2.8.0", "pymupdf")
+
+
+def _resolve_reply(monkeypatch, payload: str, *, model: str = "claude-opus-5"):
+ """Same seam, same qualification rule as `_reply` above: `reflist` calls
+ `ask._ask` by qualified name, so the fake is installed on `ask_mod`."""
+ calls = []
+
+ def fake(prompt, model_arg=None):
+ calls.append({"prompt": prompt})
+ return payload
+
+ monkeypatch.setattr(ask_mod, "_ask", fake)
+ monkeypatch.setattr(ask_mod, "_MODELS", {ask_mod.SITE_REFS: model})
+ return calls
+
+
+def test_a_resolved_label_names_a_title_printed_in_one_of_the_readings(monkeypatch):
+ """The whole permission structure of this call. Nothing invented may name a
+ paper, so the title comes back only if it can be grepped out of the text the
+ model was shown."""
+ reply = json.dumps([{
+ "num": "6",
+ "title": "Characterization of Brain Volume Changes",
+ "doi": "10.1001/jamanetworkopen.2023.18153",
+ "year": "2023",
+ }])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert list(res.resolved) == ["6"]
+ assert res.still_disputed == []
+ assert res.resolved["6"].doi == "10.1001/jamanetworkopen.2023.18153"
+
+
+def test_a_resolved_entry_carries_a_slug(monkeypatch):
+ """C1. `resolve_all` never re-slugs and `check.py` refuses to judge a
+ slug-less entry — a bare `RefEntry(...)` here is what let a resolved label
+ reach the manifest as "resolved" while `check` silently dropped it. Built
+ through `refs._entry`, the one place that knows what a complete entry
+ needs, so this cannot regress to a hand-rolled constructor again."""
+ reply = json.dumps([{
+ "num": "6",
+ "title": "Characterization of Brain Volume Changes",
+ "doi": "10.1001/jamanetworkopen.2023.18153",
+ "year": "2023",
+ }])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved["6"].slug, "a resolved entry with no slug is dropped by check.py"
+ assert res.resolved["6"].slug != "ref"
+
+
+def test_a_title_printed_in_neither_reading_leaves_the_label_disputed(monkeypatch):
+ """An unverifiable *field* is discarded; an unverifiable *title* is the label
+ staying disputed. The title is the only field that identifies a paper, so a
+ reply whose title cannot be found has answered nothing."""
+ reply = json.dumps([{"num": "6", "title": "A Paper Nobody Printed", "year": "2023"}])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved == {}
+ assert res.still_disputed == ["6"]
+
+
+def test_an_unverifiable_doi_is_discarded_without_costing_the_label(monkeypatch):
+ """A truncated DOI is extraction damage, not a different paper. The field
+ goes; the resolution stands and the entry carries a recorded gap."""
+ reply = json.dumps([{
+ "num": "6",
+ "title": "Characterization of Brain Volume Changes",
+ "doi": "10.1038/s41591-",
+ }])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved["6"].doi is None
+ assert "doi" in res.provenance.fields_discarded
+
+
+def test_cannot_tell_is_an_answer_and_keeps_the_label_disputed(monkeypatch):
+ """Abstention is first-class. A call that must always answer will
+ confabulate on exactly the labels two readings disagree about."""
+ reply = json.dumps([{"num": "6", "cannot_tell": True}])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved == {}
+ assert res.still_disputed == ["6"]
+
+
+def test_partial_resolution_is_representable(monkeypatch):
+ """11 of 14 resolved and 3 still disputed is a real outcome, not an error.
+ No code path may require all-or-nothing."""
+ text_a = _RESOLVE_A + "7. Weston AD. Automated abdominal segmentation. 2019.\n"
+ text_b = _RESOLVE_B + "7. Someone Else. A different paper. 2015.\n"
+ reply = json.dumps([
+ {"num": "6", "title": "Characterization of Brain Volume Changes"},
+ {"num": "7", "cannot_tell": True},
+ ])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6", "7"], _RESOLVE_NAMES, (text_a, text_b))
+ assert list(res.resolved) == ["6"]
+ assert res.still_disputed == ["7"]
+
+
+def test_a_label_the_reply_never_mentions_stays_disputed(monkeypatch):
+ """Silence is not consent. A reply that answers 1 of 2 questions has
+ answered 1, and the unanswered label is not repaired into agreement."""
+ reply = json.dumps([{"num": "6", "title": "Characterization of Brain Volume Changes"}])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6", "9"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.still_disputed == ["9"]
+
+
+def test_a_malformed_reply_resolves_nothing_rather_than_raising(monkeypatch):
+ """`refs` has already done useful work by this point. A model that returns
+ prose leaves every label disputed — the state the run was in before it
+ asked — and says so in the provenance. "I think maybe [6]?" is not prose
+ without brackets by accident: `ask._parse_json_array`'s first-`[`-to-last-
+ `]` slice reads it as the syntactically valid one-element list `[6]`, and
+ `_parse_array`'s dict check is what still refuses it."""
+ _resolve_reply(monkeypatch, "I think maybe [6]?")
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved == {}
+ assert res.still_disputed == ["6"]
+ assert res.provenance.discarded_whole
+
+
+def test_seen_in_is_translated_using_the_names_matching_the_texts_order(monkeypatch):
+ """M4. `names[i]` must be the reading `texts[i]` actually came from — never
+ derived from a different dict of candidates that can hold MORE readings
+ (crossref, llm) than were shown to this call. A reply of "B" must land the
+ SECOND name passed here, not the second name of some other collection."""
+ reply = json.dumps([{
+ "num": "6", "title": "Characterization of Brain Volume Changes", "seen_in": "B",
+ }])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved["6"].seen_in == ["pymupdf"]
+
+
+def test_an_unrecognised_seen_in_is_dropped_rather_than_guessed(monkeypatch):
+ reply = json.dumps([{
+ "num": "6", "title": "Characterization of Brain Volume Changes",
+ "seen_in": "whichever one was clearer",
+ }])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved["6"].seen_in == []
+
+
+def test_outcome_is_read_once_a_reply_is_obtained(monkeypatch):
+ """m8. `propose` sets this the moment a reply comes back, before it is even
+ parsed — `resolve_disputed` must do the same, or a caller that inspects
+ `provenance.outcome` after a malformed reply sees the module's own default
+ ("not_attempted") for a call that plainly happened."""
+ _resolve_reply(monkeypatch, "not json at all")
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.provenance.outcome == "read"
+
+
+def test_no_labels_makes_no_model_call_at_all(monkeypatch):
+ """Nothing disputed is nothing to ask about — never spend a call for it."""
+ calls = _resolve_reply(monkeypatch, "[]")
+ res = reflist.resolve_disputed([], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert calls == []
+ assert res.resolved == {}
+ assert res.still_disputed == []
+
+
+# --- whole-branch review, Critical 2 and minor 7 ---------------------------
+
+
+def test_the_resolution_prompt_shows_what_each_reading_says_at_the_label(monkeypatch):
+ """Critical 2. The disagreement being adjudicated has to be IN the prompt,
+ including the readings with no printed span of their own — the deposit and
+ the model's reading, which are exactly the ones a dispute is often with."""
+ from papertrace.models import RefEntry
+
+ calls = _resolve_reply(monkeypatch, json.dumps([{"num": "6", "cannot_tell": True}]))
+ reflist.resolve_disputed(
+ ["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B),
+ disputing={"crossref": [RefEntry(num="6", raw="Deposited: a third paper entirely")]},
+ )
+
+ prompt = calls[0]["prompt"]
+ assert "Deposited: a third paper entirely" in prompt
+ assert "crossref" in prompt
+
+
+def test_a_value_printed_only_in_the_disputing_entry_is_still_discarded(monkeypatch):
+ """The context block is context. Widening verification to it would let a
+ title the deposit carries and the page does not be believed — which is the
+ whole permission structure of this module inverted."""
+ from papertrace.models import RefEntry
+
+ _resolve_reply(monkeypatch, json.dumps([{"num": "6", "title": "A third paper entirely"}]))
+ res = reflist.resolve_disputed(
+ ["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B),
+ disputing={"crossref": [RefEntry(num="6", raw="Deposited: A third paper entirely")]},
+ )
+
+ assert res.resolved == {}
+ assert res.still_disputed == ["6"]
+ assert "[6].title" in res.provenance.fields_discarded
+
+
+def test_one_printed_extraction_renders_one_block_not_an_empty_second(monkeypatch):
+ calls = _resolve_reply(monkeypatch, json.dumps([{"num": "6", "cannot_tell": True}]))
+ reflist.resolve_disputed(["6"], ("pymupdf",), (_RESOLVE_A,))
+
+ prompt = calls[0]["prompt"]
+ assert "READING A (pymupdf)" in prompt
+ assert "READING B" not in prompt
+
+
+def test_a_reply_repeating_a_numeral_is_counted_once_per_object(monkeypatch):
+ """Minor 5. `entries_proposed` was measured after de-duplication, so a
+ reply with two objects for [6] — the malformed shape a reader most needs
+ to know about — was recorded as one proposed entry. `propose` counts
+ before de-duplication; so does this now."""
+ _resolve_reply(monkeypatch, json.dumps([
+ {"num": "6", "cannot_tell": True},
+ {"num": "6", "cannot_tell": True},
+ ]))
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+
+ assert res.provenance.entries_proposed == 2
+
+
+def test_an_oversized_bibliography_is_refused_before_the_call_not_after(monkeypatch):
+ """Minor 7. Both extractions were interpolated whole, with no limit and no
+ truncation record — the one model input in this codebase that could
+ plausibly exceed a context window. Refused rather than clipped: a clipped
+ reading proposes a list missing the entries past the cut, and nothing in
+ the provenance could say which. A refusal costs nothing and states itself.
+ """
+ calls = _reply(monkeypatch, "[]")
+ big = "6. A reference. 2020.\n" * (reflist.REFLIST_CHAR_LIMIT // 10)
+
+ entries, prov = reflist.propose(big, big, label_a="docling 2.8.0", label_b="pymupdf")
+
+ assert calls == []
+ assert entries == []
+ assert prov.outcome == "not_attempted"
+ assert "too long" in prov.failure
+
+
+def test_an_oversized_resolution_input_leaves_every_label_disputed(monkeypatch):
+ """The same guard on the other call, and the state it must leave behind:
+ every label exactly where it was, with a reason that is not "the model
+ could not tell" — nothing was asked."""
+ calls = _resolve_reply(monkeypatch, "[]")
+ big = "6. A reference. 2020.\n" * (reflist.REFLIST_CHAR_LIMIT // 10)
+
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (big, big))
+
+ assert calls == []
+ assert res.resolved == {}
+ assert res.still_disputed == ["6"]
+ assert res.provenance.outcome == "not_attempted"
+ assert "too long" in res.provenance.failure
+
+
+def test_a_long_run_of_absent_numerals_is_counted_rather_than_listed(monkeypatch):
+ """The count is the finding; past a handful the list stops being one.
+
+ A reply naming only `[99]` makes every numeral below it absent, which
+ printed 98 consecutive integers — 410 characters — into
+ `reflist_numbering_findings`, a field the report shows a reader. The first
+ few locate the gap; the rest only prove the reader stopped reading. The
+ finding itself is unchanged and nothing is hidden: the total leads.
+ """
+ text = "99. Weston AD (2019) A paper. Radiology. https://doi.org/10.1148/radiol.2019181432"
+ _reply(monkeypatch, '[{"num":"99","title":"A paper","reading":"AB"}]')
+ _, prov = reflist.propose(text, text, label_a="docling", label_b="pymupdf")
+
+ gaps = next(f for f in prov.numbering_findings if "absent" in f)
+ assert gaps.startswith("98 numerals absent"), gaps
+ assert "and 92 more" in gaps, gaps
+ assert len(gaps) < 100, f"{len(gaps)} chars in a reader-facing field: {gaps}"
diff --git a/tests/test_refs.py b/tests/test_refs.py
index 5a58e1d..7a45b91 100644
--- a/tests/test_refs.py
+++ b/tests/test_refs.py
@@ -8,6 +8,7 @@
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+from papertrace import models # noqa: E402
from papertrace.models import RefEntry # noqa: E402
@@ -546,6 +547,34 @@ def test_a_scanned_provided_pdf_is_not_reported_as_matched(tmp_path):
assert "title check failed" in detail
+def test_a_reference_with_diacritics_verifies_against_its_own_first_page():
+ """The two sides of `found` have to be folded the same way.
+
+ `_title_tokens` folds (`Küstner` → `kustner`) and `found` is a substring
+ test, so a page that is only lowercased matches none of them. The old broken
+ tokeniser hid this by accident: it produced `stner`, and `"stner" in
+ "küstner"` is True. Measured on a page carrying the reference's own title
+ verbatim, folding the tokens alone turned `7/7 verified` into `2/9 mismatch`
+ — and `_accept` answers a mismatch by unlinking the downloaded PDF and
+ telling the reader the reference has a wrong or mistyped DOI. A correct
+ retrieval destroyed and the manuscript blamed for it, on every German,
+ Scandinavian, Polish, Turkish, Spanish and Portuguese reference.
+ """
+ from papertrace.refs import TITLE_VERIFIED, _title_check_text
+
+ raw = (
+ "Küstner T, Späth C, Bjørnsson B. Röntgenbefunde und Ösophagusmotilität "
+ "bei Achalasie. Radiologische Übersicht 2020;1:1-9."
+ )
+ page = (
+ "Röntgenbefunde und Ösophagusmotilität bei Achalasie\n"
+ "T. Küstner, C. Späth, B. Bjørnsson\n"
+ "Radiologische Übersicht 2020;1:1-9"
+ )
+ state, detail = _title_check_text(raw, page)
+ assert state == TITLE_VERIFIED, detail
+
+
def test_the_provided_reason_says_which_of_the_three_happened(tmp_path):
"""The manifest reason is what a reader sees. It must distinguish verified
from unverified-because-unreadable from an outright mismatch."""
@@ -774,6 +803,66 @@ def test_a_tracking_parameter_is_not_part_of_a_title():
assert {"launches", "practice", "artificial"} <= tokens, "the title's own words survive"
+def test_a_doi_is_not_part_of_a_title_either():
+ """The same class of thing as the tracking parameter above, and the same
+ failure. `_reference_raw`'s documented fallback for a DOI-only deposit is
+ the DOI string itself, and most publisher DOIs embed a journal or platform
+ slug — `radiol`, `jamanetworkopen`, `neuroimage`, `bioinformatics` — which
+ `[a-z]{5,}` happily lifts out and then compares against real title words.
+ An identifier's substring is not a word from a title.
+
+ Three shapes, because only the third had no five-letter run and so was the
+ only one the earlier fix reached."""
+ from papertrace.refs import _title_tokens
+
+ for doi in ("10.1148/radiol.2019181432",
+ "10.1001/jamanetworkopen.2023.18153",
+ "10.1038/s41591-019-0673-2",
+ "10.1016/j.neuroimage.2020.117161",
+ "10.1093/bioinformatics/btaa123"):
+ assert _title_tokens(doi) == set(), doi
+ # and written the way a reference prints it, prefix and all
+ assert _title_tokens(f"doi:{doi}") == set(), doi
+
+ # the reference's own words survive a trailing DOI, and the DOI's do not
+ tokens = _title_tokens(
+ "Fujita S. Characterization of brain volume changes in aging individuals. "
+ "JAMA Netw Open. 2023. doi:10.1001/jamanetworkopen.2023.18153"
+ )
+ assert {"characterization", "changes", "individuals"} <= tokens
+ assert "jamanetworkopen" not in tokens
+
+
+def test_a_trailing_doi_no_longer_inflates_the_title_checks_denominator():
+ """The URL strip's own stated reason, applied to the identifier that
+ replaced it: a token no first page will carry sits in the denominator and
+ pushes a correct retrieval toward `mismatch`. `jamanetworkopen` is printed
+ on no page — the page says "JAMA Network Open", with spaces."""
+ from papertrace.refs import _title_check_text
+
+ raw = ("Fujita S, Mori S, Onda K. Characterization of brain volume changes in aging "
+ "individuals. JAMA Netw Open. 2023;6(6):e2318153. "
+ "doi:10.1001/jamanetworkopen.2023.18153")
+ page = ("Characterization of Brain Volume Changes in Aging Individuals\n"
+ "Fujita S, Mori S, Onda K. JAMA Network Open. 2023;6(6):e2318153\n")
+
+ state, detail = _title_check_text(raw, page)
+ assert state == "verified", detail
+ assert "6/6" in detail, detail
+
+
+def test_a_doi_only_reference_has_nothing_to_match_on_rather_than_one_word():
+ """`unverifiable` either way, but for the honest reason: a single journal
+ slug lifted out of an identifier is not "one distinctive word" of the
+ reference, and calling it that invites a future ratio to believe it."""
+ from papertrace.refs import _title_check_text
+
+ state, detail = _title_check_text("10.1148/radiol.2019181432",
+ "Radiology 2019 imaging study of the chest")
+ assert state == "unverifiable"
+ assert "no distinctive words" in detail, detail
+
+
def test_three_generic_domain_words_are_not_an_identity_check():
"""The last line of defence, in case a URL-only reference reaches it by some
other route: with the URL stripped the ACR news page still scores 3/7 =
@@ -934,3 +1023,67 @@ def handler(request: httpx.Request) -> httpx.Response:
assert any("crossref" in u for u in asked), "a real reference was never looked up"
assert e.doi == "10.1177/0363546512452714"
+
+
+def test_a_surname_with_an_umlaut_still_contributes_tokens():
+ """`_title_tokens` is `[a-z]{5,}`, so a diacritic splits a word or deletes
+ it. `Späth` and `Müller` contributed NOTHING, and this set is what
+ `titles_match`, `_same_work` and `_title_check_text` all compare — a
+ surname that vanished cannot agree with its own paper.
+
+ `_fold` existed in `refs.py` and was not used here. NFKD alone is not
+ enough: it leaves ß, ø, æ, đ, ł undecomposed.
+ """
+ assert "spath" in models._title_tokens("Späth C, Makowski MR")
+ assert "muller" in models._title_tokens("Müller H")
+ assert "cristobal" in models._title_tokens("Romero-Cristóbal M")
+ assert "kustner" in models._title_tokens("Küstner T")
+ assert "bjornsson" in models._title_tokens("Bjørnsson B")
+
+
+def test_folding_lowercases_before_it_transliterates():
+ """`_TRANSLITERATE` is keyed on lowercase letters only, so the order of the
+ two steps in `_fold` is load-bearing and was neither commented nor tested:
+ translating first leaves `Ø`, `Æ` and `Ł` to NFKD, which cannot decompose
+ them — they are distinct letters, not a base plus a mark — and the surname
+ loses them entirely. An uppercase initial letter is where a surname is most
+ likely to carry one."""
+ assert models._fold("Ø") == "o"
+ assert models._fold("Ærø") == "aero"
+ assert models._fold("Łukasiewicz") == "lukasiewicz"
+ assert models._fold("İNCE") == "ince"
+
+
+def test_folding_does_not_invent_or_merge_tokens():
+ """Gate 4, the other direction: an ASCII title must tokenise exactly as
+ before, and the stopword list must still bite after folding.
+
+ Note the stopword list holds `commun`, not `communications` — so
+ "Nature Communications" keeps a token. These four are all in the list.
+ """
+ assert models._title_tokens("Nature Science volume press") == set()
+ assert models._title_tokens("Robust segmentation of anatomic structures") == {
+ "robust",
+ "segmentation",
+ "anatomic",
+ "structures",
+ }
+
+
+def test_a_partial_resolution_round_trips_through_the_manifest(tmp_path):
+ """11 resolved and 3 still disputed is a representable state, and the wire
+ format is the dataclass — so this is the test that says the two lists can
+ both be non-empty at once (Gate 2 for `numbering_choice`,
+ `numbering_chosen_by` and `labels_resolved`)."""
+ from papertrace.models import RefManifest
+
+ m = RefManifest(manuscript="p.pdf", labels_resolved=["6", "7"], labels_disputed=["9"],
+ numbering_choice="llm_resolved", numbering_chosen_by="user")
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ back = RefManifest.from_json(path)
+ assert back.labels_resolved == ["6", "7"]
+ assert back.labels_disputed == ["9"]
+ assert back.numbering_choice == "llm_resolved"
+ assert back.numbering_chosen_by == "user"
+ assert back.numbering_verified is False
diff --git a/tests/test_withholding.py b/tests/test_withholding.py
new file mode 100644
index 0000000..061d397
--- /dev/null
+++ b/tests/test_withholding.py
@@ -0,0 +1,433 @@
+"""Withholding a disputed citation label's verdict, and the field that carries it.
+
+A citation label whose readings of the bibliography disagree is not evidence
+either way about the claim that cites it — it is evidence PaperTrace does not
+know which paper was actually fetched under that label. `check_claims`'s
+`disputed` parameter drops such a label from judgement without pretending the
+source was never obtained: that distinction is `withheld_refs` versus
+`unjudged_refs`, and this module is what regresses if the two get merged.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+
+from papertrace import check as check_mod # noqa: E402
+from papertrace.check import check_claims # noqa: E402
+from papertrace.models import Block, ClaimResult, RefEntry, RefManifest, SourceMap # noqa: E402
+
+
+def _write_source(case: Path, slug: str, text: str = "The rate was 84.3%.") -> None:
+ """An ingested source: the text and the map that says where its one block is."""
+ d = case / "ingest" / slug
+ d.mkdir(parents=True, exist_ok=True)
+ (d / "annotated.md").write_text(f"\n{text}\n")
+ SourceMap(
+ doc=f"{slug}.pdf",
+ pages=1,
+ blocks=[Block("block_0001", "text", 1, (0.0, 0.0, 10.0, 10.0), [], text)],
+ ).to_json(d / "source_map.json")
+
+
+def _entry(num: str, slug: str, status: str = "retrieved") -> RefEntry:
+ return RefEntry(
+ num=num, raw=f"ref {num}", status=status, slug=slug,
+ pdf_path=f"/nonexistent/{slug}.pdf",
+ )
+
+
+def _manifest(*entries: RefEntry) -> RefManifest:
+ return RefManifest(manuscript="paper.pdf", entries=list(entries))
+
+
+def _reply(claim_id: int, verdict: str = "supported") -> str:
+ return json.dumps([{
+ "id": claim_id, "verdict": verdict, "note": "reads it",
+ "source_page": 1, "source_block": "block_0001", "anchor_phrases": [],
+ }])
+
+
+def _unexpected_call(prompt, model=None):
+ raise AssertionError("no source survived withholding — _ask must not be called")
+
+
+def test_a_disputed_label_is_dropped_and_the_other_cited_source_is_still_judged(
+ tmp_path, monkeypatch
+):
+ """Only the disputed label's source is subtracted from a co-cited claim —
+ a targeted subtraction, not a claim killed because one of several labels
+ it cites is in dispute. If this regresses to killing the whole claim, a
+ reader loses a real verdict from the source that was never in question."""
+ _write_source(tmp_path, "six-2019")
+ _write_source(tmp_path, "nine-2021")
+ manifest = _manifest(_entry("6", "six-2019"), _entry("9", "nine-2021"))
+ monkeypatch.setattr(check_mod, "_ask", lambda prompt, model=None: _reply(1, "supported"))
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["6", "9"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"6"})
+
+ slugs = {j.source_slug for j in claims[0].judgements}
+ assert slugs == {"nine-2021"}
+ assert claims[0].verdict == "supported"
+ # the verdict and the judged slugs are both computed from `avail`, so they
+ # would survive an edit that only populated `withheld_refs` in the
+ # nothing-survives branch — and `claim_pairing` reads this field, so every
+ # co-cited claim keeping a real verdict would silently lose its disclosure
+ assert claims[0].withheld_refs == ["6"]
+ assert claims[0].unjudged_refs == []
+
+
+def test_a_claim_whose_only_cited_source_is_disputed_is_unchecked_not_not_retrieved(
+ tmp_path, monkeypatch
+):
+ """`not_retrieved` would say the source could not be obtained. It WAS
+ obtained — the two readings of the bibliography just disagree about which
+ paper the label names — so the only honest verdict is `unchecked`, and the
+ disputed label must be named in the note so a reader knows which citation
+ to check by hand. `_ask` must never be called: nothing survived to judge."""
+ _write_source(tmp_path, "six-2019")
+ manifest = _manifest(_entry("6", "six-2019"))
+ monkeypatch.setattr(check_mod, "_ask", _unexpected_call)
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["6"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"6"})
+
+ assert claims[0].verdict == "unchecked"
+ assert claims[0].verdict != "not_retrieved"
+ assert "6" in claims[0].note
+
+
+def test_withheld_refs_holds_the_disputed_label_and_unjudged_refs_does_not(
+ tmp_path, monkeypatch
+):
+ """The two fields mean different things: `unjudged_refs` is "nobody could
+ get to it", `withheld_refs` is "it was read and set aside anyway". A label
+ landing in both would tell the reader retrieval failed when it did not —
+ the exact laundering CLAUDE.md names for `unjudged_refs`."""
+ _write_source(tmp_path, "six-2019")
+ manifest = _manifest(_entry("6", "six-2019"))
+ monkeypatch.setattr(check_mod, "_ask", _unexpected_call)
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["6"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"6"})
+
+ assert claims[0].withheld_refs == ["6"]
+ assert claims[0].unjudged_refs == []
+
+
+def test_disputed_none_and_empty_set_behave_identically(tmp_path, monkeypatch):
+ """Every caller that predates this feature passes neither argument. `None`
+ must be exactly as inert as an explicit empty set, or turning on this
+ feature with nothing yet computed as disputed would change verdicts that
+ used to be stable — the regression this whole task must not cause."""
+ _write_source(tmp_path, "six-2019")
+ manifest = _manifest(_entry("6", "six-2019"))
+ monkeypatch.setattr(check_mod, "_ask", lambda prompt, model=None: _reply(1, "supported"))
+
+ claims_none = [ClaimResult(id=1, claim="x", location="Results", refs=["6"])]
+ check_claims(claims_none, manifest, tmp_path, backend="pymupdf", disputed=None)
+
+ claims_empty = [ClaimResult(id=1, claim="x", location="Results", refs=["6"])]
+ check_claims(claims_empty, manifest, tmp_path, backend="pymupdf", disputed=set())
+
+ claims_default = [ClaimResult(id=1, claim="x", location="Results", refs=["6"])]
+ check_claims(claims_default, manifest, tmp_path, backend="pymupdf")
+
+ for other in (claims_empty, claims_default):
+ assert other[0].verdict == claims_none[0].verdict == "supported"
+ assert other[0].withheld_refs == claims_none[0].withheld_refs == []
+
+
+def test_a_disputed_label_never_retrieved_stays_not_retrieved(tmp_path):
+ """Withholding describes a source that WAS obtained. A label that is both
+ disputed and, say, paywalled must not be relabelled `unchecked` — that
+ would claim a model read something nobody ever fetched. `withheld_refs`
+ is partitioned OUT of the retrieval filter's result, never out of the raw
+ cited-labels list, so a label that never passed retrieval cannot appear
+ there no matter what `disputed` says about it."""
+ manifest = _manifest(_entry("6", "six-2019", status="paywalled"))
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["6"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"6"})
+
+ assert claims[0].verdict == "not_retrieved"
+ assert claims[0].withheld_refs == []
+
+
+def test_withheld_refs_round_trips_and_validates_against_the_schema(tmp_path):
+ """Gate 2: a new claim field is additive in the schema and round-trips
+ through `to_json`/`from_json` — `ClaimResult` has no bespoke pair of those,
+ it rides `dataclasses.asdict` out and `_claim_from`'s `**d` in, so a field
+ with a default needs no code there, only the schema and this proof."""
+ import jsonschema
+
+ from papertrace.models import RunResults
+
+ claim = ClaimResult(
+ id=1, claim="x", location="Results", refs=["6"],
+ verdict="unchecked", withheld_refs=["6"],
+ )
+ path = tmp_path / "results.json"
+ RunResults(manuscript="m.pdf", claims=[claim]).to_json(path)
+
+ back = RunResults.from_json(path).claims[0]
+ assert back.withheld_refs == ["6"]
+
+ schema = json.loads(
+ (Path(__file__).resolve().parent.parent / "schemas" / "results.schema.json").read_text()
+ )
+ jsonschema.Draft202012Validator(schema).validate(json.loads(path.read_text()))
+
+
+def test_a_co_cited_label_that_could_not_be_retrieved_survives_a_withholding(tmp_path):
+ """Major 2 of the whole-branch review, and a regression against `main`.
+
+ A claim cites a disputed-and-retrieved [3] and a paywalled [4]. Nothing
+ survives to judge, so the claim is `unchecked` for the withholding — and
+ on the way out of that branch `unjudged_refs` was never filled, so [4]
+ was in none of the three accounts a reader has (judged, withheld,
+ unjudged) and appeared in no note. A retrieval gap erased by a
+ withholding is the cardinal rule's own example of what may not happen.
+ """
+ _write_source(tmp_path, "three-2019")
+ manifest = _manifest(
+ _entry("3", "three-2019"),
+ _entry("4", "four-2020", status="paywalled"),
+ )
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["3", "4"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"3"})
+
+ assert claims[0].verdict == "unchecked"
+ assert claims[0].withheld_refs == ["3"]
+ assert claims[0].unjudged_refs == ["4"]
+
+
+def test_an_unretrievable_co_citation_is_reported_where_a_verdict_survives_too(
+ tmp_path, monkeypatch
+):
+ """The same accounting on the path that DOES keep a verdict, so the fix
+ is not a special case bolted onto the one branch the review named: [3]
+ withheld, [4] paywalled, [5] judged."""
+ _write_source(tmp_path, "five-2021")
+ manifest = _manifest(
+ _entry("3", "three-2019"),
+ _entry("4", "four-2020", status="paywalled"),
+ _entry("5", "five-2021"),
+ )
+ monkeypatch.setattr(check_mod, "_ask", lambda prompt, model=None: _reply(1, "supported"))
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["3", "4", "5"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"3"})
+
+ assert claims[0].verdict == "supported"
+ assert claims[0].withheld_refs == ["3"]
+ assert claims[0].unjudged_refs == ["4"]
+
+
+# --- what the prose may say about a label whose readings did not agree -----
+
+
+def test_the_withholding_note_does_not_assert_the_readings_named_different_papers(
+ tmp_path,
+):
+ """`label_agreement` returns `disputed` for two causes — the readings
+ actively name different papers, and nothing in them could be compared —
+ and no published field tells the two apart. A note asserting the first is
+ therefore false whenever the second happened, which is the same mistake
+ as describing a source that was never fetched as one that was."""
+ _write_source(tmp_path, "six-2019")
+ manifest = _manifest(_entry("6", "six-2019"))
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["6"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"6"})
+
+ # the disjunction, not the assertion — this manifest never recorded which
+ # cause applied (`labels_uncomparable is None`), so neither may the note
+ assert "did not agree on which paper" in claims[0].note
+ assert "either named different papers, or nothing in them could be compared" in (
+ claims[0].note
+ )
+
+
+def test_the_run_level_disputed_disclosure_covers_both_causes():
+ """The run-level roll-up says the same thing as the note above, so it
+ inherits the same constraint."""
+ from papertrace.disclosures import _labels_disputed
+
+ d = _labels_disputed(RefManifest(manuscript="m.pdf", labels_disputed=["6"]))
+
+ assert d is not None
+ # the disjunction, not the assertion: naming both causes is honest,
+ # asserting the contradicting one is not
+ assert "did not agree on which paper" in d.text
+ assert "nothing in them could be compared" in d.text
+ assert "name different papers for each of these labels" not in d.text
+
+
+# --- a disputed label nobody retrieved is not a source that was fetched ----
+
+
+def test_a_disputed_label_nobody_retrieved_is_not_described_as_fetched():
+ """Major 5. `check.py` keeps such a label out of `withheld_refs` on
+ purpose (`test_a_disputed_label_never_retrieved_stays_not_retrieved`);
+ the disclosure layer undid that care by falling back to
+ `labels_disputed`, and printed "the source fetched under that label" for
+ a source nobody ever fetched."""
+ from papertrace.disclosures import _claim_pairing
+
+ manifest = _manifest(_entry("6", "six-2019", status="paywalled"))
+ manifest.labels_disputed = ["6"]
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["6"],
+ verdict="not_retrieved")
+
+ d = _claim_pairing(claim, manifest)
+
+ assert d is not None
+ assert "fetched" not in d.text
+ assert "never retrieved" in d.text
+
+
+def test_a_retrieved_disputed_label_still_reads_as_a_source_that_was_fetched():
+ """The other side of Major 5's split, and the load-path guard on it: a
+ `results.json` written before `withheld_refs` existed carries `[]` for
+ every claim, so the new branch must not be chosen from that emptiness
+ alone. The manifest's own entry status is what decides, and here the
+ source really was retrieved."""
+ from papertrace.disclosures import _claim_pairing
+
+ manifest = _manifest(_entry("6", "six-2019"))
+ manifest.labels_disputed = ["6"]
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["6"]) # no withheld_refs
+
+ d = _claim_pairing(claim, manifest)
+
+ assert d is not None
+ assert "the source retrieved under that label" in d.text
+
+
+def test_a_claim_citing_both_a_withheld_and_an_unretrieved_disputed_label_names_both():
+ """One claim, both causes. The withheld label keeps the token and the
+ level — it is the more urgent finding, because a paper WAS fetched under
+ it — and the label nobody retrieved is named rather than folded into the
+ same sentence."""
+ from papertrace.disclosures import _claim_pairing
+
+ manifest = _manifest(
+ _entry("6", "six-2019"),
+ _entry("7", "seven-2020", status="paywalled"),
+ )
+ manifest.labels_disputed = ["6", "7"]
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["6", "7"],
+ withheld_refs=["6"])
+
+ d = _claim_pairing(claim, manifest)
+
+ assert d is not None
+ assert "[6]" in d.text and "[7]" in d.text
+ assert "never retrieved" in d.text
+
+
+def test_any_iterable_of_labels_withholds_every_claim_not_just_the_first(tmp_path, monkeypatch):
+ """`disputed` is typed `set[str] | None`, and a set is what the CLI passes.
+
+ A generator, though, is truthy with no `__len__`, and each `r in disputed`
+ consumes it — so the first claim would withhold and every claim after it
+ would see an exhausted iterator and be judged normally. A silently partial
+ withholding is worse than a loud type error, so the parameter is copied into
+ a set on the way in and this test is what says so.
+ """
+ _write_source(tmp_path, "six-2019")
+ manifest = _manifest(_entry("6", "six-2019"))
+ monkeypatch.setattr(check_mod, "_ask", lambda prompt, model=None: _reply(1, "supported"))
+ claims = [
+ ClaimResult(id=1, claim="first", location="Results", refs=["6"]),
+ ClaimResult(id=2, claim="second", location="Results", refs=["6"]),
+ ]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf",
+ disputed=(label for label in ("6",)))
+
+ assert [c.verdict for c in claims] == ["unchecked", "unchecked"]
+ assert [c.withheld_refs for c in claims] == [["6"], ["6"]]
+
+
+def test_a_claim_citing_an_unretrieved_disputed_label_and_a_resolved_one_names_both():
+ """The new fifth state must not swallow the resolved caveat the withheld
+ state already carries: a verdict resting on an accepted resolution is a
+ different, and still live, caveat."""
+ from papertrace.disclosures import _claim_pairing
+
+ manifest = _manifest(_entry("6", "six-2019", status="paywalled"))
+ manifest.labels_disputed = ["6"]
+ manifest.labels_resolved = ["7"]
+ manifest.numbering_choice = "llm_resolved"
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["6", "7"])
+
+ d = _claim_pairing(claim, manifest)
+
+ assert d is not None
+ assert "never retrieved" in d.text
+ assert "[7]" in d.text
+
+
+# --- fix round 2: which cause, per claim and in the note ------------------
+
+
+def test_the_claim_disclosure_says_which_cause_applies():
+ """Per claim, the same split the run-level roll-up makes."""
+ from papertrace.disclosures import _claim_pairing
+
+ m = _manifest(_entry("6", "six-2019"))
+ m.labels_disputed = ["6"]
+ m.labels_uncomparable = ["6"]
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["6"], withheld_refs=["6"])
+ d = _claim_pairing(claim, m)
+ assert "nothing in them could be compared" in d.text
+ assert "named different papers" not in d.text
+
+ m.labels_uncomparable = []
+ d = _claim_pairing(claim, m)
+ assert "named different papers" in d.text
+ assert "could be compared" not in d.text
+
+
+def test_an_unretrieved_disputed_claim_says_which_cause_too():
+ from papertrace.disclosures import _claim_pairing
+
+ m = _manifest(_entry("6", "six-2019", status="paywalled"))
+ m.labels_disputed = ["6"]
+ m.labels_uncomparable = ["6"]
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["6"])
+ d = _claim_pairing(claim, m)
+ assert "never retrieved" in d.text
+ assert "nothing in them could be compared" in d.text
+
+
+def test_the_withholding_note_says_which_cause_when_the_manifest_split_them(tmp_path):
+ """`check_claims` has the manifest, so the note can say which rather than
+ naming the disjunction."""
+ _write_source(tmp_path, "six-2019")
+ manifest = _manifest(_entry("6", "six-2019"))
+ manifest.labels_disputed = ["6"]
+ manifest.labels_uncomparable = ["6"]
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["6"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"6"})
+
+ assert "nothing in them could be compared" in claims[0].note
+ assert "different papers" not in claims[0].note
+
+
+def test_the_withholding_note_keeps_the_disjunction_when_nothing_split_them(tmp_path):
+ """`None` is never computed, and the note may not pick a cause for it."""
+ _write_source(tmp_path, "six-2019")
+ manifest = _manifest(_entry("6", "six-2019"))
+ manifest.labels_disputed = ["6"]
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["6"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"6"})
+
+ assert "either" in claims[0].note
diff --git a/tests/test_wizard.py b/tests/test_wizard.py
index f5e4988..af6cd10 100644
--- a/tests/test_wizard.py
+++ b/tests/test_wizard.py
@@ -140,6 +140,65 @@ def test_clean_path_expands_tilde(monkeypatch):
assert wizard.clean_path("~/nope.pdf") == Path.home() / "nope.pdf"
+def test_the_case_folder_accepts_a_dragged_quoted_path(tmp_path, monkeypatch):
+ """The one path prompt that skipped `clean_path`.
+
+ A quoted answer made the case name start with a literal `'`, so the path was
+ relative: the audit was written under a directory named `'` beneath the cwd,
+ while every line the run printed named an absolute folder that did not
+ exist. Finder's drag-and-drop adds those quotes whenever the path holds a
+ space, so this is the default way to reach it, not an unusual one.
+
+ The other three prompts were immune only by accident — they check
+ `.exists()`, and a quoted path fails that. A case folder is *created*, so
+ nothing ever contradicted it.
+ """
+ target = tmp_path / "EuroRad Review" / "untitled folder"
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(wizard.Prompt, "ask", staticmethod(lambda *a, **k: f"'{target}'"))
+ assert wizard._ask_case(tmp_path / "paper.pdf") == target
+
+
+def test_a_relative_case_folder_is_accepted_as_the_path_it_resolves_to(tmp_path, monkeypatch):
+ """`-c demo_case` is in the README, so a relative answer is legitimate.
+
+ What it may not do is stay relative. The wizard prints the equivalent
+ command and hands the path to `run`, and a relative path means both of those
+ are only true from the directory the user happened to be standing in.
+ """
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(wizard.Prompt, "ask", staticmethod(lambda *a, **k: "demo_case"))
+ assert wizard._ask_case(tmp_path / "paper.pdf") == (tmp_path / "demo_case").resolve()
+
+
+def test_a_resolved_case_folder_is_echoed_so_a_mangled_path_is_visible(tmp_path, monkeypatch, capsys):
+ """The whole point of resolving rather than rejecting.
+
+ A quoted answer used to degrade into a *relative* name beginning with a
+ literal quote. Resolving it silently would still write the audit somewhere
+ the user did not name; printing the resolution puts the mistake on screen
+ before the first paid model call rather than after all ~32 of them.
+ """
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(wizard.Prompt, "ask", staticmethod(lambda *a, **k: "'a b"))
+ case = wizard._ask_case(tmp_path / "paper.pdf")
+ assert case == (tmp_path / "'a b").resolve()
+ assert str(case) in _plain(capsys.readouterr().out)
+
+
+def test_the_working_directory_is_refused_as_a_case_folder(tmp_path, monkeypatch):
+ """`default_case` already refuses it; the prompt may not be the way around.
+
+ Whitespace arrives here as `.`, so this is also the blank-answer path — and
+ a case folder that *is* the working directory cannot be told apart from
+ whatever else the user keeps there.
+ """
+ answers = iter([" ", str(tmp_path / "case")])
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(wizard.Prompt, "ask", staticmethod(lambda *a, **k: next(answers)))
+ assert wizard._ask_case(tmp_path / "paper.pdf") == tmp_path / "case"
+
+
# --- DOI detection: replacing a definition with a yes/no -------------------
@@ -604,22 +663,59 @@ def test_a_path_with_a_quote_is_still_one_argument(tmp_path):
# --- the cost estimate must not be exceeded by the documented retry ---------
-def test_the_worst_case_call_count_includes_the_retry(tmp_path):
+def test_the_worst_case_call_count_includes_the_retry(tmp_path, monkeypatch):
"""The wizard promised "up to N model calls" from one extraction plus one
per cited source. `_ask` retries once, so a single-source run advertised as
- 2 could issue 3. The ceiling now comes from check.py's own attempt count,
- so the two cannot drift."""
- from papertrace.check import ASK_ATTEMPTS
+ 2 could issue 3. The ceiling now comes from ask.py's own attempt count,
+ so the two cannot drift.
+
+ Both figures also carry the reference-list reading `refs` now makes before
+ `check` ever runs, WHEN the wizard's own backend choice ("auto") would
+ actually attempt it: one un-retried call, so it adds exactly 1 to each side
+ of the estimate rather than multiplying through the retried figure — this
+ task adds that call on top of the `ASK_ATTEMPTS * (1 + cited_source_calls)`
+ shape a previous task already moved `model_calls_max` to, rather than
+ reverting it back to a flat `ASK_ATTEMPTS * cited_source_calls`.
+
+ Pinned with `resolve_backend` forced to report docling rather than left to
+ whatever this machine happens to have installed — `workload()` derives the
+ +1 from it (N5 of the Task 6 re-review: not `docling_available()`, which is
+ only a proxy for what `resolve_backend("auto")` actually resolves to), and
+ a test whose expected numbers depend on the host environment is not really
+ pinning anything.
+ """
+ from papertrace.ask import ASK_ATTEMPTS
+ monkeypatch.setattr(wizard, "resolve_backend", lambda backend: "docling")
pdf = _one_pager(tmp_path / "m.pdf",
"Title\nOne sentence citing [1].\nReferences\n[1] A. 2020.")
w = wizard.workload(pdf)
- assert w["model_calls"] == 2, w
- assert w["model_calls_max"] == 1 + ASK_ATTEMPTS * 1, w
+ assert w["model_calls"] == 3, w
+ assert w["model_calls_max"] == 1 + ASK_ATTEMPTS * (1 + 1), w
assert w["model_calls_max"] >= w["model_calls"]
+def test_the_worst_case_call_count_drops_the_reflist_call_on_a_pymupdf_only_install(
+ tmp_path, monkeypatch
+):
+ """`run_wizard` always requests `backend="auto"`, which resolves to pymupdf
+ when docling is not importable — and on a pymupdf run the flat second
+ reading is skipped as identical to the first, so the reference-list call
+ is never attempted either (Major 2 / Minor 3 of the Task 6 review: the
+ base estimate must not advertise a call this run's own backend cannot
+ produce)."""
+ from papertrace.ask import ASK_ATTEMPTS
+
+ monkeypatch.setattr(wizard, "resolve_backend", lambda backend: "pymupdf")
+ pdf = _one_pager(tmp_path / "m.pdf",
+ "Title\nOne sentence citing [1].\nReferences\n[1] A. 2020.")
+ w = wizard.workload(pdf)
+
+ assert w["model_calls"] == 2, w
+ assert w["model_calls_max"] == ASK_ATTEMPTS * (1 + 1), w
+
+
def test_the_printed_estimate_does_not_promise_a_ceiling_it_can_exceed():
import inspect
@@ -748,14 +844,18 @@ def test_the_cost_estimate_shrinks_with_the_limits_and_never_grows(tmp_path):
"A range here [7-9] citing three.",
])
w = wizard.workload(pdf)
- assert w["model_calls"] == 1 + 6
+ # one extraction + one reference-list reading + one per cited source. The
+ # reference-list call survives every limit — it reads the bibliography, not
+ # the claims — so it is a constant in each figure below rather than
+ # something `--max-claims` can take away.
+ assert w["model_calls"] == 1 + 1 + 6
first = wizard.apply_limits(w, max_claims=1, max_sources=None)
- assert first["model_calls"] == 1 + 1, "the first place cites one source"
+ assert first["model_calls"] == 1 + 1 + 1, "the first place cites one source"
capped = wizard.apply_limits(w, max_claims=None, max_sources=2)
- assert capped["model_calls"] == 1 + 2
+ assert capped["model_calls"] == 1 + 1 + 2
both = wizard.apply_limits(w, max_claims=2, max_sources=2)
- assert both["model_calls"] == 1 + 2
+ assert both["model_calls"] == 1 + 1 + 2
none = wizard.apply_limits(w, max_claims=None, max_sources=None)
assert (none["model_calls"], none["model_calls_max"]) == (w["model_calls"], w["model_calls_max"])
for limited in (first, capped, both):