>" not in calls[0]["prompt"]
+
+
+def test_the_models_reading_letter_is_translated_into_a_reading_name(monkeypatch):
+ """One published field, one vocabulary — the translation happens here.
+
+ The model answers in the terms the prompt gave it (`A`, `B`, `AB`), because
+ that is what it was shown. `RefEntry.seen_in` is a *published* field whose
+ vocabulary is reading NAMES, so that a consumer can ask `"llm" in seen_in`
+ and get a true answer on every manifest. Letting `A` reach the field would
+ put two vocabularies in one wire-format key, and a consumer checking either
+ one would silently never match manifests written by the other path.
+ """
+ entries, _ = _propose(
+ monkeypatch,
+ """[
+ {"num":"3","title":"Atlas-based methods","reading":"AB"}
+ ]""",
+ label_a="docling",
+ label_b="pymupdf",
+ )
+
+ assert entries[0].seen_in == ["docling", "pymupdf"]
+
+
+def test_an_unrecognised_reading_letter_is_dropped_rather_than_guessed(monkeypatch):
+ """A model answering "the docling one" must not land a sentence in a
+ published field, and must not be *assumed* to have meant reading A either.
+ Unknown provenance is recorded as unknown."""
+ entries, _ = _propose(monkeypatch, """[
+ {"num":"3","title":"Atlas-based methods","reading":"whichever one was clearer"}
+ ]""")
+
+ assert entries[0].seen_in == [], entries[0].seen_in
+
+
+def test_an_empty_reading_makes_no_model_call_at_all(monkeypatch):
+ """Nothing to check against means nothing can be believed.
+
+ A run whose second extraction came back empty would otherwise pay for a call
+ whose every field could only be verified against one text — half the
+ verification, at full price, with no way for the report to say so.
+ """
+ calls = _reply(monkeypatch, "[]")
+ entries, prov = reflist.propose(READING_A, " ", label_a="docling 2.8.0", label_b="pymupdf")
+
+ assert calls == []
+ assert entries == []
+ assert prov.discarded_whole
+
+
+def test_the_entry_level_seen_in_round_trips_through_the_manifest(tmp_path):
+ """Gate 2. A new persisted field that `from_json` drops is a silent loss.
+
+ `_ref_entry_from` filters on `fields(RefEntry)`, so declaring the field is
+ enough — this test is what says so rather than assuming it.
+ """
+ import json
+
+ import jsonschema
+
+ from papertrace.models import RefEntry, RefManifest
+
+ m = RefManifest(manuscript="p.pdf",
+ entries=[RefEntry(num="1", raw="A. A paper. 2020.", seen_in=["docling", "pymupdf"])])
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+
+ payload = json.loads(path.read_text())
+ assert payload["entries"][0]["seen_in"] == ["docling", "pymupdf"]
+ schema = json.loads(
+ (Path(__file__).resolve().parent.parent / "schemas" / "refs_manifest.schema.json").read_text()
+ )
+ jsonschema.validate(payload, schema)
+ assert "seen_in" not in schema["properties"]["entries"]["items"].get("required", [])
+ assert RefManifest.from_json(path).entries[0].seen_in == ["docling", "pymupdf"]
From 3140270fd2d4702a407efda25d0aa3142cfc7de4 Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Thu, 17 Sep 2026 21:32:07 +0200
Subject: [PATCH 23/44] reflist: an entry that names no paper is not an entry
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Three review findings.
A reply of `{"num": "9", "reading": "A"}` became a voter carrying no claim about
any work. The whole-candidate discard only fires when a title was PROPOSED and
failed verification, so a reply that omits the identity field entirely never
reached it. An entry with neither a verified title nor a verified DOI is now
dropped with a recorded reason.
The review diagnosed this as a wrong-verdict risk via `_same_work`'s trivial
agreement. That is not the live path: `label_agreement` compares through
`_comparably_same`, which refuses to call two uncomparable entries agreed, so
the phantom made the label `disputed`, not `agreed`. The hazard is the other
direction — a degenerate reply withholding verdicts it never said anything
against. Measured both ways before fixing.
Numbering findings move to their own list. A discarded field is "this value was
not printed"; a numeral duplicate or gap is "this reading's labels do not add
up", which the spec gives a different consequence — so one list reported a
count of discarded fields including things no field ever lost. No `covers` flag
comes with it, and the comment says why: `label_agreement` already enforces
"not covering", and a flag would be a published field with no consumer.
A `reading` value of the wrong type is now reported as a wrong shape rather than
folded into "the model said nothing".
---
src/papertrace/reflist.py | 38 ++++++++++++++++--
tests/test_reflist.py | 81 ++++++++++++++++++++++++++++++++++++++-
2 files changed, 115 insertions(+), 4 deletions(-)
diff --git a/src/papertrace/reflist.py b/src/papertrace/reflist.py
index bf02b02..a178abf 100644
--- a/src/papertrace/reflist.py
+++ b/src/papertrace/reflist.py
@@ -131,6 +131,19 @@ class ReflistProvenance:
model: str = ""
entries_proposed: int = 0
fields_discarded: list[str] = field(default_factory=list)
+ # Kept apart from `fields_discarded` on purpose. A discarded field is "this
+ # value was not printed, so it was dropped"; a numbering finding is "this
+ # reading's labels do not add up", which the spec gives a different
+ # consequence — it marks the candidate as not covering, not as missing a
+ # value. One list holding both made `reflist_fields_discarded` report a
+ # count of discarded fields that included things no field ever lost.
+ #
+ # There is deliberately no `covers` flag: "not covering" needs no separate
+ # signal because `label_agreement` already enforces it, refusing to let a
+ # reading speak for a label it carries twice, and simply not seeing a label
+ # the reading never carried. A flag would be a published field with no
+ # consumer.
+ numbering_findings: list[str] = field(default_factory=list)
# non-empty means the whole reading was refused, and says why. It is never
# "the model agreed": a reading that could not be checked is not a reading.
discarded_whole: str = ""
@@ -261,7 +274,13 @@ def propose(
# "A" — landing a sentence the model wrote in a published field whose
# only legal values are "A", "B" and "AB".
said = item.get("reading")
- said_norm = said.strip().upper() if isinstance(said, str) else ""
+ if said is not None and not isinstance(said, str):
+ # a reply of `5` or `{"text": "A"}` is a wrong SHAPE, which is a
+ # different report from a field the model simply left out
+ prov.fields_discarded.append(f"[{num}] reading: not a string")
+ said_norm = ""
+ else:
+ said_norm = said.strip().upper() if isinstance(said, str) else ""
letters = list(said_norm) if said_norm in _READING_LETTERS else []
seen_in = [{"A": label_a, "B": label_b}[c] for c in letters]
if said_norm and not letters:
@@ -274,6 +293,19 @@ def propose(
# `_title_tokens` read `raw` to decide whether two readings name one
# paper. Composing it from unverified values would be the invention this
# module refuses; composing it from verified ones is re-assembly.
+ # An entry that names no paper is not an entry. The whole-candidate
+ # discard above fires only when a title was PROPOSED and failed; a reply
+ # supplying a numeral and nothing else never reaches it, and would
+ # otherwise become a voter carrying no claim about any work. That voter
+ # cannot produce a wrong verdict — `label_agreement` compares through
+ # `_comparably_same`, which refuses to call two uncomparable entries
+ # `agreed` — but it can make a label `disputed` and withhold verdicts
+ # the model never said anything against. A degenerate reply may cost
+ # this reading its vote; it may not cost the audit its answers.
+ if "title" not in kept and "doi" not in kept:
+ prov.fields_discarded.append(f"[{num}]: no verified title or doi, so no entry")
+ continue
+
parts = [kept[k] for k in ("authors", "year", "title", "journal") if k in kept]
raw_text = ". ".join(parts)
if "doi" in kept:
@@ -304,9 +336,9 @@ def propose(
nums = [e.num for e in out]
twice = sorted({n for n in nums if nums.count(n) > 1}, key=_label_key)
if twice:
- prov.fields_discarded.append("numerals proposed twice: " + ", ".join(twice))
+ prov.numbering_findings.append("numerals proposed twice: " + ", ".join(twice))
if gaps := _numeral_gaps(nums):
- prov.fields_discarded.append("numerals absent from 1..max: " + ", ".join(gaps))
+ prov.numbering_findings.append("numerals absent from 1..max: " + ", ".join(gaps))
return out, prov
diff --git a/tests/test_reflist.py b/tests/test_reflist.py
index 6ee1dc8..7c3b8dc 100644
--- a/tests/test_reflist.py
+++ b/tests/test_reflist.py
@@ -186,7 +186,10 @@ def test_a_duplicated_numeral_is_recorded_and_never_repaired(monkeypatch):
]""")
assert [e.num for e in entries] == ["2", "2"], "an entry was dropped or renumbered"
- assert any("twice" in f for f in prov.fields_discarded), prov.fields_discarded
+ # a numbering finding, not a discarded field — the two carry different
+ # consequences and are reported separately
+ assert any("twice" in f for f in prov.numbering_findings), prov.numbering_findings
+ assert not any("twice" in f for f in prov.fields_discarded), prov.fields_discarded
def test_a_reply_that_is_not_json_yields_an_empty_candidate_not_an_exception(monkeypatch):
@@ -318,3 +321,79 @@ def test_the_entry_level_seen_in_round_trips_through_the_manifest(tmp_path):
jsonschema.validate(payload, schema)
assert "seen_in" not in schema["properties"]["entries"]["items"].get("required", [])
assert RefManifest.from_json(path).entries[0].seen_in == ["docling", "pymupdf"]
+
+
+# --- an entry that names no paper is not an entry ---------------------------
+
+
+def test_a_reply_with_a_numeral_and_nothing_else_yields_no_entry(monkeypatch):
+ """The whole-candidate discard fires when a title is PROPOSED and fails.
+
+ A reply supplying only `{"num": "9", "reading": "A"}` never reaches it, and
+ would otherwise become a voter carrying no claim about any work. It cannot
+ produce a wrong verdict — `label_agreement` compares through
+ `_comparably_same`, which refuses to call two uncomparable entries `agreed`
+ — but it can make the label `disputed` and withhold verdicts the model never
+ said anything against. A degenerate reply may cost this reading its vote; it
+ may not cost the audit its answers.
+ """
+ entries, prov = _propose(monkeypatch, '[{"num":"9","reading":"A"}]')
+
+ assert entries == []
+ assert any("no verified title or doi" in f for f in prov.fields_discarded), (
+ prov.fields_discarded
+ )
+
+
+def test_an_entry_kept_on_a_doi_alone_still_counts(monkeypatch):
+ """The rule is "names no paper", not "has no title". A DOI identifies a work
+ on its own — `_same_work` settles on it before it looks at anything else —
+ so an entry whose title the converter mangled but whose DOI is printed and
+ whole is a legitimate voter."""
+ entries, _ = _propose(
+ monkeypatch,
+ '[{"num":"7","doi":"10.1038/s41591-019-0673-2","reading":"A"}]',
+ )
+
+ assert len(entries) == 1
+ assert entries[0].doi == "10.1038/s41591-019-0673-2"
+ assert entries[0].title is None
+
+
+def test_a_numbering_finding_is_not_reported_as_a_discarded_field(monkeypatch):
+ """Two different things, two different lists.
+
+ A discarded field is "this value was not printed, so it was dropped". A
+ numbering finding is "this reading's labels do not add up", which the spec
+ gives a different consequence. Sharing one list made
+ `reflist_fields_discarded` report a count of discarded fields that included
+ things no field ever lost.
+ """
+ entries, prov = _propose(
+ monkeypatch,
+ '[{"num":"6","title":"Characterization of brain volume changes in aging '
+ 'individuals","reading":"A"}]',
+ )
+
+ assert len(entries) == 1
+ assert prov.fields_discarded == [], prov.fields_discarded
+ assert any("absent from 1..max" in f for f in prov.numbering_findings), (
+ prov.numbering_findings
+ )
+
+
+def test_a_reading_letter_of_the_wrong_type_is_reported_as_a_wrong_shape(monkeypatch):
+ """A reply of `5` is a different failure from a field the model left out,
+ and folding both into "nothing to say" loses the distinction. Either way
+ `seen_in` stays empty — the outcome was always honest, only the record was
+ silent."""
+ entries, prov = _propose(
+ monkeypatch,
+ '[{"num":"6","title":"Characterization of brain volume changes in aging '
+ 'individuals","reading":5}]',
+ )
+
+ assert entries[0].seen_in == []
+ assert any("reading: not a string" in f for f in prov.fields_discarded), (
+ prov.fields_discarded
+ )
From 87504b2758eb1ffec361d86dff6b3461ad3452d1 Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Thu, 17 Sep 2026 21:38:51 +0200
Subject: [PATCH 24/44] docs: Task 6 must not drop the model reading's
numbering findings
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Task 5's re-review found that Task 6 was written against the old provenance
shape: it read only `fields_discarded`, so every duplicate numeral and gap
`reflist.propose` reports would have been dropped before any reader saw it.
`_llm_reference_reading` now returns the provenance object, which is what the
task's own Interfaces block already claimed and its code contradicted — a
3-tuple of (entries, notes, model) can carry only one of the two kinds of
finding. The findings get their own manifest field, their own schema
description, their own console line and their own clause in the reflist
disclosure, because "3 values discarded" and "it numbered one entry twice" are
different facts.
Step 4 also claimed five new manifest fields that Task 3 already landed. It now
says so, tells the implementer to verify rather than rewrite, and adds only the
one field that is genuinely new.
---
.../plans/2026-09-17-llm-reference-list.md | 146 +++++++++++++++---
1 file changed, 126 insertions(+), 20 deletions(-)
diff --git a/docs/superpowers/plans/2026-09-17-llm-reference-list.md b/docs/superpowers/plans/2026-09-17-llm-reference-list.md
index af8c591..e4426a9 100644
--- a/docs/superpowers/plans/2026-09-17-llm-reference-list.md
+++ b/docs/superpowers/plans/2026-09-17-llm-reference-list.md
@@ -3363,7 +3363,8 @@ subject follows the house pattern rather than inventing a catch-all.
and `_llm_reference_reading(...) -> tuple[list[RefEntry], ReflistProvenance | None]`.
- Produces in `models.RefManifest`: `numbering_corroborated: bool = False`,
`corroborating_readings: list[str]`, `labels_disputed: list[str]`,
- `reflist_model: str = ""`, `reflist_fields_discarded: list[str]`.
+ `reflist_model: str = ""`, `reflist_fields_discarded: list[str]`,
+ `reflist_numbering_findings: list[str]`.
- **Left to Task 7, deliberately:** `labels_resolved`, `numbering_choice`,
`numbering_chosen_by`. Nothing in this task can set them — they record an
interactive choice — and a field written only as `""` by the task that added
@@ -3815,7 +3816,24 @@ Expected: `TypeError: _refs_pipeline() got an unexpected keyword argument
'llm_refs'` and `AttributeError: module 'papertrace.cli' has no attribute
'_reference_readings'`. Paste it.
-- [ ] **Step 4: `RefManifest` gains five fields, and the schema says what absent means**
+- [ ] **Step 4: `RefManifest` gains ONE field, and the schema says what absent means**
+
+**Read this before editing.** An earlier draft of this step added six fields
+here. **Five of them already landed in Task 3** — verify with:
+
+```bash
+~/anaconda3/bin/python -c "import sys,dataclasses; sys.path.insert(0,'src'); from papertrace.models import RefManifest; print([f.name for f in dataclasses.fields(RefManifest)])"
+```
+
+Expected to be present already: `numbering_corroborated`,
+`corroborating_readings`, `labels_disputed`, `labels_resolved`,
+`reflist_model`, `reflist_fields_discarded`. **The only field this task adds is
+`reflist_numbering_findings`.** The five declarations below are reproduced so
+you can confirm the ones on disk match — if any differs, report it rather than
+rewriting it; Task 3 is reviewed and closed. Add only the last one, and only its
+`to_json` / `from_json` / schema entries.
+
+(`numbering_choice` and `numbering_chosen_by` belong to Task 7. Do not add them.)
In `src/papertrace/models.py`, in `RefManifest`, after `numbering_ledger`:
@@ -3842,6 +3860,13 @@ In `src/papertrace/models.py`, in `RefManifest`, after `numbering_ledger`:
# What that reading cost in dropped values, and — when there was no reading —
# why there was none. Empty means no model reading was asked for.
reflist_fields_discarded: list[str] = field(default_factory=list)
+ # A different kind of finding, kept in its own field for that reason: the
+ # model reading's own labels did not add up — a numeral proposed twice, or a
+ # gap in 1..max. Not a value that went missing, so counting it among the
+ # discarded fields would report a number of dropped values that never
+ # dropped. Empty means none found, which on a reading that produced no
+ # entries is not the same as none present.
+ reflist_numbering_findings: list[str] = field(default_factory=list)
```
In `to_json`'s payload, beside `numbering_ledger`:
@@ -3852,6 +3877,7 @@ In `to_json`'s payload, beside `numbering_ledger`:
"labels_disputed": self.labels_disputed,
"reflist_model": self.reflist_model,
"reflist_fields_discarded": self.reflist_fields_discarded,
+ "reflist_numbering_findings": self.reflist_numbering_findings,
```
In `from_json`, `.get(...)` for each so an older file still loads:
@@ -3862,6 +3888,7 @@ In `from_json`, `.get(...)` for each so an older file still loads:
labels_disputed=data.get("labels_disputed", []),
reflist_model=data.get("reflist_model", ""),
reflist_fields_discarded=data.get("reflist_fields_discarded", []),
+ reflist_numbering_findings=data.get("reflist_numbering_findings", []),
```
In `schemas/refs_manifest.schema.json`, additive in `properties`, nothing added
@@ -3890,9 +3917,51 @@ to `required` (`["manuscript", "entries"]`):
"type": "array",
"items": { "type": "string" },
"description": "Values a model proposed for the reference list that were not found verbatim in either extraction of the printed page, and so were discarded — plus, when no reading was obtained at all, the reason. Empty means no model reading was asked for."
+ },
+ "reflist_numbering_findings": {
+ "type": "array",
+ "items": { "type": "string" },
+ "description": "Findings about the model reading's own labels rather than its values: a numeral it proposed twice, or a gap in 1..max. Kept apart from `reflist_fields_discarded` because nothing went missing — folding the two would report a count of discarded values that includes things no value ever lost. Empty means none were found, which on a reading that produced no entries is not the same as none being present."
}
```
+Add the round-trip assertions for it beside the existing `reflist_*` ones, and
+one behavioural test that a duplicate numeral actually reaches the manifest —
+the whole point of this field is that `reflist.propose` reports it and an
+earlier draft of this plan dropped it on the floor:
+
+```python
+def test_a_numbering_finding_from_the_model_reading_reaches_the_manifest(
+ tmp_path, monkeypatch
+):
+ """`reflist.propose` reports a duplicated or missing numeral separately from
+ a discarded field, because the two mean different things. An earlier draft
+ of this step read only `fields_discarded`, so every duplicate and gap the
+ model reading found was silently dropped before any reader saw it."""
+ proposed = (
+ [_work("2", "10.1000/x2"), _work("2", "10.1000/x22")],
+ reflist_mod.ReflistProvenance(
+ model="claude-opus-5",
+ entries_proposed=2,
+ numbering_findings=["numerals proposed twice: 2"],
+ readings=["pymupdf", "docling"],
+ ),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: proposed)
+
+ manifest = _run_refs(tmp_path, monkeypatch, llm_refs=True)
+
+ assert manifest.reflist_numbering_findings == ["numerals proposed twice: 2"]
+ assert not any(
+ "twice" in f for f in manifest.reflist_fields_discarded
+ ), manifest.reflist_fields_discarded
+```
+
+Use whatever fixture helper the tests in Step 2 established for driving
+`_refs_pipeline` and reading the manifest back; `_run_refs` above is a
+placeholder for that helper's real name, which Step 2 fixes — **do not add a
+second way to run the stage.**
+
- [ ] **Step 5: `cli._reference_readings` — the voter dict, and nothing impure**
In `src/papertrace/cli.py`, above `_refs_pipeline` (`:496`):
@@ -3953,24 +4022,34 @@ def _llm_reference_reading(
label_a: str,
label_b: str,
enabled: bool,
-) -> tuple[list["RefEntry"], list[str], str]:
+) -> tuple[list["RefEntry"], "ReflistProvenance"]:
"""A model's reading of the bibliography, or a stated reason there is none.
- Returns `(entries, notes, model)`. `notes` is never empty when `enabled`:
+ Returns `(entries, provenance)` — the provenance object itself, **always**,
+ never `None` and never a tuple of loose pieces. `reflist.propose` reports
+ two distinct kinds of finding (`fields_discarded`, a value that was not
+ printed; `numbering_findings`, a reading whose labels do not add up) and a
+ 3-tuple of `(entries, notes, model)` could only carry one of them. An
+ earlier draft of this plan did exactly that and would have dropped every
+ duplicate and gap on the floor, unreported.
+
+ `provenance.fields_discarded` is never empty when `enabled`:
an attempt that produced nothing has to leave the reason where the manifest
and the report can read it, because silence here would be read as "no news"
about a reading that was asked for and did not happen.
"""
from . import ask
- from .reflist import propose
+ from .reflist import ReflistProvenance, propose
if not enabled:
- return [], [], ""
+ return [], ReflistProvenance()
if not ask.claude_available():
# the ordinary case for someone who installed papertrace and not Claude
# Code. The run proceeds on the deterministic readings and says so —
# never that a model agreed with them.
- return [], ["not attempted — claude is not on PATH, so no model read the list"], ""
+ return [], ReflistProvenance(
+ fields_discarded=["not attempted — claude is not on PATH, so no model read the list"]
+ )
try:
entries, prov = propose(reading_a, reading_b, label_a=label_a, label_b=label_b)
except Exception as e: # noqa: BLE001 — a failed corroboration is not a failed refs stage
@@ -3982,12 +4061,14 @@ def _llm_reference_reading(
# closed on the next one. The failure is recorded, not swallowed:
# `reflist_fields_discarded` carries it into the manifest and a console
# line says it at the time. Same shape as `check.py:996`.
- return [], [f"not obtained — {type(e).__name__}: {str(e)[:200]}"], ""
- notes = list(prov.fields_discarded)
+ return [], ReflistProvenance(
+ fields_discarded=[f"not obtained — {type(e).__name__}: {str(e)[:200]}"]
+ )
if prov.discarded_whole:
- # the entries are already `[]` in this branch; the reason is what travels
- notes.insert(0, f"reading discarded — {prov.discarded_whole}")
- return entries, notes, prov.model
+ # the entries are already `[]` in this branch; the reason is what travels,
+ # and it goes FIRST so the console line quotes it rather than a field name
+ prov.fields_discarded.insert(0, f"reading discarded — {prov.discarded_whole}")
+ return entries, prov
```
Add `RefEntry` to `cli.py`'s type-checking imports if it is not already
@@ -4033,7 +4114,7 @@ Immediately **before** the `reconcile` call at `:638`:
if _needs_flat_reading(smap):
flat_text, _ = references_span_flat(manuscript)
flat_entries = parse_references(flat_text)
- llm_entries, reflist_notes, reflist_model = _llm_reference_reading(
+ llm_entries, reflist_prov = _llm_reference_reading(
refs_text,
flat_text,
label_a=smap.converter,
@@ -4047,14 +4128,22 @@ Immediately **before** the `reconcile` call at `:638`:
smap=smap, crossref=crossref_entries, parsed=parsed,
flat=flat_entries, llm=llm_entries,
)
- if reflist_model:
+ if reflist_prov.model:
+ dropped = len(reflist_prov.fields_discarded)
console.print(
- f"the reference list was also read by [bold]{reflist_model}[/bold] · "
- f"{len(reflist_notes)} field{'' if len(reflist_notes) == 1 else 's'} discarded as "
+ f"the reference list was also read by [bold]{reflist_prov.model}[/bold] · "
+ f"{dropped} field{'' if dropped == 1 else 's'} discarded as "
f"not printed [dim]— a second reading, not confirmation[/dim]"
)
- elif reflist_notes:
- console.print(f"[yellow]⚠ no model reading of the reference list[/yellow] — {reflist_notes[0]}")
+ # reported separately, because it is a different kind of finding: not a
+ # value that was missing, but a reading whose own labels do not add up
+ for finding in reflist_prov.numbering_findings:
+ console.print(f" [yellow]⚠ the model reading's {finding}[/yellow]")
+ elif reflist_prov.fields_discarded:
+ console.print(
+ "[yellow]⚠ no model reading of the reference list[/yellow] — "
+ f"{reflist_prov.fields_discarded[0]}"
+ )
```
Add `references_span_flat` to the lazy `from .ingest import ...` at `:534` and
@@ -4100,7 +4189,8 @@ And in `RefManifest(...)` at `:702-714`:
corroborating_readings=rec.corroborating_readings,
labels_disputed=rec.labels_disputed,
reflist_model=reflist_model,
- reflist_fields_discarded=reflist_notes,
+ reflist_fields_discarded=reflist_prov.fields_discarded,
+ reflist_numbering_findings=reflist_prov.numbering_findings,
```
**Note the ordering constraint:** the agreement block must sit above the
@@ -4231,6 +4321,10 @@ def _reflist(manifest) -> Disclosure | None:
"""
model = getattr(manifest, "reflist_model", "") or ""
notes = list(getattr(manifest, "reflist_fields_discarded", []) or [])
+ # a different kind of finding, and it must reach the reader too: a reading
+ # whose own labels do not add up is worth knowing about even when every
+ # value it proposed was printed
+ numbering = list(getattr(manifest, "reflist_numbering_findings", []) or [])
if not model and not notes:
return None # no model reading was asked for
ceiling = (
@@ -4264,13 +4358,25 @@ def _reflist(manifest) -> Disclosure | None:
f"bibliography and asked what numbered list it carries; {dropped}."
)
short = f"{REFLIST_TOKEN} ({model}) — {len(notes)} discarded"
+ if numbering:
+ # appended rather than folded into `dropped`: "3 values discarded" and
+ # "it numbered one entry twice" are different facts, and a reader who
+ # sees them as one number cannot tell which happened
+ head += (
+ " Its own numbering did not add up either — "
+ + "; ".join(numbering)
+ + "."
+ )
return Disclosure(
key="reflist",
level="info",
token=REFLIST_TOKEN,
text=f"{head} {ceiling}",
short=short,
- rows=tuple(notes[:6]),
+ # both kinds, numbering first: it describes the reading as a whole, and
+ # a truncated list should not lose the structural finding to six field
+ # names
+ rows=tuple((numbering + notes)[:6]),
)
From b74ba01139248c80c0788c6aca924064515f1825 Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Thu, 17 Sep 2026 21:57:17 +0200
Subject: [PATCH 25/44] refs: four readings of the bibliography, and a second
axis to say so
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The flat-text reading and the model's reading are voters. Neither reaches
reconcile's arguments or resolve_all, so no model output can cause a source to be
resolved, downloaded or judged — only a verdict to be withheld. A run whose model
reading names two different papers produces byte-identical entries to --no-llm-refs,
and that equality is a test.
The pymupdf reading is skipped when the run backend already is pymupdf: a reading
agreeing with itself is not corroboration. --parse-only keeps it and loses the
model call, because "no network" is what the flag promises.
numbering_corroborated is a second axis and numbering_verified keeps its exact
meaning. Corroboration is what lets the numbering warning stop being the only
thing a clean reference list gets told about.
claude absent, or the call failing, leaves a stated reason in the manifest and on
the console — never that the model agreed.
---
schemas/refs_manifest.schema.json | 7 +
src/papertrace/cli.py | 220 ++++++-
src/papertrace/disclosures.py | 109 ++++
src/papertrace/models.py | 9 +
src/papertrace/refs.py | 21 +
.../templates/report_terminal.html.j2 | 6 +
src/papertrace/wizard.py | 11 +-
tests/test_label_agreement.py | 31 +
tests/test_pipeline_split.py | 3 +-
tests/test_reference_readings.py | 537 ++++++++++++++++++
tests/test_wizard.py | 14 +-
11 files changed, 954 insertions(+), 14 deletions(-)
create mode 100644 tests/test_reference_readings.py
diff --git a/schemas/refs_manifest.schema.json b/schemas/refs_manifest.schema.json
index d76d535..7e72803 100644
--- a/schemas/refs_manifest.schema.json
+++ b/schemas/refs_manifest.schema.json
@@ -288,6 +288,13 @@
"items": {
"type": "string"
}
+ },
+ "reflist_numbering_findings": {
+ "type": "array",
+ "description": "Findings about the model reading's own labels rather than its values: a numeral it proposed twice, or a gap in 1..max. Kept apart from `reflist_fields_discarded` because nothing went missing — folding the two would report a count of discarded values that includes things no value ever lost. Empty means none were found, which on a reading that produced no entries is not the same as none being present.",
+ "items": {
+ "type": "string"
+ }
}
}
}
diff --git a/src/papertrace/cli.py b/src/papertrace/cli.py
index c848a69..93c79b1 100644
--- a/src/papertrace/cli.py
+++ b/src/papertrace/cli.py
@@ -14,6 +14,7 @@
import sys
import tempfile
from pathlib import Path
+from typing import TYPE_CHECKING
import typer
from rich.console import Console
@@ -21,7 +22,13 @@
from . import __version__
from .brand import BANNER
-from .models import ClaimResult, RefManifest, RunResults, manuscript_fingerprint
+from .models import ClaimResult, RefEntry, RefManifest, RunResults, manuscript_fingerprint
+
+if TYPE_CHECKING:
+ # not at module level: `.reflist` pulls in `.refs`, which this module only
+ # ever imports lazily inside `_refs_pipeline` to keep `httpx` and the
+ # network-facing stack out of every other command's import cost
+ from .reflist import ReflistProvenance
app = typer.Typer(add_completion=False, rich_markup_mode="rich", invoke_without_command=True)
console = Console()
@@ -493,6 +500,111 @@ def _detected_doi(manuscript: Path) -> str | None:
return detect_doi(manuscript)
+def _needs_flat_reading(smap) -> bool:
+ """Is a flat-text reading of this bibliography a second reading at all?
+
+ `smap.converter` is `"pymupdf"` or `"docling "`. On a run whose
+ backend already IS pymupdf, `references_span_flat` returns the same text
+ `references_span` did, so the cost buys nothing — and counting it as a
+ reading would turn every `single` label into `agreed` and report
+ corroboration that nothing corroborated.
+ """
+ return smap.converter.split()[0] != "pymupdf"
+
+
+def _reference_readings(
+ *,
+ smap,
+ crossref: list[RefEntry] | None = None,
+ parsed: list[RefEntry] | None = None,
+ flat: list[RefEntry] | None = None,
+ llm: list[RefEntry] | None = None,
+) -> dict[str, list[RefEntry]]:
+ """The readings of the bibliography that are entitled to a vote, by name.
+
+ Pure, so the agreement axis is testable without a PDF, a network or a model.
+
+ A reading that is ABSENT is omitted, never present as `[]`: `label_agreement`
+ counts the readings that carry a label, so an empty list votes against every
+ label it does not have — which is all of them — and turns "we did not ask"
+ into "one reading says no".
+
+ NOTHING here reaches `reconcile`'s arguments or `resolve_all`. `reconcile`
+ still chooses between the deposit and the run backend's parse; this dict is
+ a second, orthogonal axis whose only power is to withhold a verdict.
+ """
+ out: dict[str, list[RefEntry]] = {}
+ if crossref:
+ out["crossref"] = crossref
+ if parsed:
+ out["parsed"] = parsed
+ # double-guarded on purpose: the caller skips the *work* on a pymupdf run,
+ # and this refuses the *vote* even if handed one, so a future caller cannot
+ # reintroduce self-corroboration by passing the text in anyway
+ if flat and _needs_flat_reading(smap):
+ out["pymupdf"] = flat
+ if llm:
+ out["llm"] = llm
+ return out
+
+
+def _llm_reference_reading(
+ reading_a: str,
+ reading_b: str,
+ *,
+ label_a: str,
+ label_b: str,
+ enabled: bool,
+) -> tuple[list[RefEntry], ReflistProvenance]:
+ """A model's reading of the bibliography, or a stated reason there is none.
+
+ Returns `(entries, provenance)` — the provenance object itself, **always**,
+ never `None` and never a tuple of loose pieces. `reflist.propose` reports
+ two distinct kinds of finding (`fields_discarded`, a value that was not
+ printed; `numbering_findings`, a reading whose labels do not add up) and a
+ 3-tuple of `(entries, notes, model)` could only carry one of them. An
+ earlier draft of this plan did exactly that and would have dropped every
+ duplicate and gap on the floor, unreported.
+
+ `provenance.fields_discarded` is never empty when `enabled` and the model
+ was not asked or the call failed: an attempt that produced nothing has to
+ leave the reason where the manifest and the report can read it, because
+ silence here would be read as "no news" about a reading that was asked for
+ and did not happen.
+ """
+ from . import ask
+ from .reflist import ReflistProvenance, propose
+
+ if not enabled:
+ return [], ReflistProvenance()
+ if not ask.claude_available():
+ # the ordinary case for someone who installed papertrace and not Claude
+ # Code. The run proceeds on the deterministic readings and says so —
+ # never that a model agreed with them.
+ return [], ReflistProvenance(
+ fields_discarded=["not attempted — claude is not on PATH, so no model read the list"]
+ )
+ try:
+ entries, prov = propose(reading_a, reading_b, label_a=label_a, label_b=label_b)
+ except Exception as e: # noqa: BLE001 — a failed corroboration is not a failed refs stage
+ # Blanket, and here specifically: `refs` has already parsed the list and
+ # is about to fetch the sources, and this call is the one thing in the
+ # stage that leaves the machine. The seam raises RuntimeError for a
+ # timeout and a non-zero exit and ValueError for unparseable stdout, and
+ # a subprocess can surface OSError besides — enumerating them would fail
+ # closed on the next one. The failure is recorded, not swallowed:
+ # `reflist_fields_discarded` carries it into the manifest and a console
+ # line says it at the time. Same shape as `check.py:996`.
+ return [], ReflistProvenance(
+ fields_discarded=[f"not obtained — {type(e).__name__}: {str(e)[:200]}"]
+ )
+ if prov.discarded_whole:
+ # the entries are already `[]` in this branch; the reason is what travels,
+ # and it goes FIRST so the console line quotes it rather than a field name
+ prov.fields_discarded.insert(0, f"reading discarded — {prov.discarded_whole}")
+ return entries, prov
+
+
def _refs_pipeline(
*,
manuscript: Path,
@@ -503,6 +615,7 @@ def _refs_pipeline(
backend: str = "auto",
doi: str | None = None,
supplement: list[Path] | None = None,
+ llm_refs: bool = True,
) -> None:
"""Parse the References section, then retrieve open-access copies with an honest manifest.
@@ -513,17 +626,19 @@ def _refs_pipeline(
the value. This function is what they actually call; `refs()` is a thin CLI
adapter over it.
"""
- from .ingest import ingest_pdf, references_span
+ from .ingest import ingest_pdf, references_span, references_span_flat
from .models import SourceMap, citation_labels, is_references_heading, paper_title
from .refs import (
_client,
crossref_deposit,
deposit_corroborates,
deposit_is_this_paper,
+ label_agreement,
manuscript_supplements,
parse_references,
reconcile,
resolve_all,
+ stamp_seen_in,
unused_provided,
)
@@ -556,11 +671,15 @@ def _refs_pipeline(
console.print("[red]No numbered references found — is there a References section?[/red]")
raise typer.Exit(1)
console.print(f"parsed [bold]{len(entries)}[/bold] numbered references")
+ # `entries` is rebound to the reconciled list below, which destroys the parse
+ # as a separate reading — and the parse is one of the four voters
+ parsed = entries
# The manuscript's own [N] markers arbitrate. Free, offline, and the only
- # one of the three readings that is definitionally right about what the
- # paper cites — the parse and the deposit are both candidates measured
- # against it. --parse-only stays offline, so it gets no second candidate.
+ # one of the readings that is definitionally right about what the paper
+ # cites — every reading of the bibliography is a candidate measured against
+ # it. --parse-only stays offline: it gets the flat-text reading, which is
+ # local, and never the model's, which is spend.
body_labels = _body_citation_labels(smap, citation_labels, is_references_heading)
crossref_entries, absent, identity_note = None, "", ""
if not parse_only:
@@ -635,11 +754,80 @@ def _refs_pipeline(
f"{'confirmed by title' if identity else 'confirmed by bibliography' if corroboration and corroboration.confirms else 'unverified'})[/dim]"
)
+ # The bibliography, read again. Both extra readings are voters only: neither
+ # reaches `reconcile`'s arguments or `resolve_all`, so no model output and no
+ # flat-text reading can cause a source to be resolved, downloaded or judged.
+ # They can only cause a verdict to be withheld.
+ flat_text, flat_entries = "", None
+ if _needs_flat_reading(smap):
+ flat_text, _ = references_span_flat(manuscript)
+ flat_entries = parse_references(flat_text)
+ llm_entries, reflist_prov = _llm_reference_reading(
+ refs_text,
+ flat_text,
+ label_a=smap.converter,
+ label_b="pymupdf",
+ # --parse-only promises "no network"; a `claude -p` subprocess is both
+ # network and spend, so the flag is forced off here rather than trusted
+ # to the caller
+ enabled=llm_refs and not parse_only,
+ )
+ others = _reference_readings(
+ smap=smap, crossref=crossref_entries, parsed=parsed,
+ flat=flat_entries, llm=llm_entries,
+ )
+ if reflist_prov.model:
+ dropped = len(reflist_prov.fields_discarded)
+ console.print(
+ f"the reference list was also read by [bold]{reflist_prov.model}[/bold] · "
+ f"{dropped} field{'' if dropped == 1 else 's'} discarded as "
+ f"not printed [dim]— a second reading, not confirmation[/dim]"
+ )
+ # reported separately, because it is a different kind of finding: not a
+ # value that was missing, but a reading whose own labels do not add up
+ for finding in reflist_prov.numbering_findings:
+ console.print(f" [yellow]⚠ the model reading's {finding}[/yellow]")
+ elif reflist_prov.fields_discarded:
+ console.print(
+ "[yellow]⚠ no model reading of the reference list[/yellow] — "
+ f"{reflist_prov.fields_discarded[0]}"
+ )
+
entries, rec = reconcile(body_labels, crossref_entries, entries, crossref_absent=absent)
if rec.source == "crossref" and identity_note:
# the note is what a reader of `refs_manifest.json` gets, so a list taken
# from a publisher's record says on whose authority it was adopted
rec.note += identity_note
+
+ # A second, independent axis. `rec.verified` keeps its exact meaning and
+ # nothing here may touch it: two readings of one document agreeing says
+ # nothing about a reference the layout destroyed in both.
+ agreement = label_agreement(others, body_labels)
+ rec.labels_disputed = sorted(
+ (label for label, state in agreement.items() if state == "disputed"),
+ key=lambda s: (0, int(s), "") if s.isdigit() else (1, 0, s),
+ )
+ # every cited label, or it is not corroboration. One disputed label is not
+ # "mostly corroborated", and `single` is not agreement — it is one reading
+ rec.corroborated = bool(body_labels) and all(
+ agreement.get(label) == "agreed" for label in body_labels
+ )
+ # named only when it holds: an empty list means no corroboration was
+ # established, never that these readings disagreed
+ rec.corroborating_readings = sorted(others) if rec.corroborated else []
+ if rec.corroborated:
+ console.print(
+ f"[green]✓ {len(others)} readings of the reference list agree[/green] on every "
+ f"cited label [dim]({', '.join(sorted(others))}) — corroboration, not a "
+ f"confirmed numbering[/dim]"
+ )
+ if rec.labels_disputed:
+ console.print(
+ f"[yellow]⚠ readings disagree at [{'], ['.join(rec.labels_disputed)}][/yellow] — "
+ "claims citing those labels will be reported unchecked, not guessed"
+ )
+ stamp_seen_in(entries, others)
+
if rec.verified:
console.print(f"[green]✓ numbering confirmed[/green] — {rec.note}")
else:
@@ -711,6 +899,12 @@ def tick(e):
unverified_from=rec.unverified_from,
numbering_contested=rec.contested,
numbering_ledger=rec.ledger,
+ numbering_corroborated=rec.corroborated,
+ corroborating_readings=rec.corroborating_readings,
+ labels_disputed=rec.labels_disputed,
+ reflist_model=reflist_prov.model,
+ reflist_fields_discarded=reflist_prov.fields_discarded,
+ reflist_numbering_findings=reflist_prov.numbering_findings,
)
manifest.to_json(case / "refs_manifest.json")
ok = len(manifest.retrieved)
@@ -749,10 +943,16 @@ def refs(
"supplement needs no flag — drop it in the sources folder named after "
"the reference, e.g. pyrros-2023-supplement.pdf",
),
+ llm_refs: bool = typer.Option(
+ True, "--llm-refs/--no-llm-refs",
+ help="Also have a model read the printed reference list as a second opinion "
+ "on the numbering (one extra model call; off under --parse-only)",
+ ),
) -> None:
"""Parse the References section, then retrieve open-access copies with an honest manifest."""
_refs_pipeline(manuscript=manuscript, case=case, provided=provided, email=email,
- parse_only=parse_only, backend=backend, doi=doi, supplement=supplement)
+ parse_only=parse_only, backend=backend, doi=doi, supplement=supplement,
+ llm_refs=llm_refs)
@app.command(rich_help_panel="Pipeline stages — `run` calls these in order")
@@ -1179,6 +1379,11 @@ def run(
"supplement needs no flag — drop it in the sources folder named after "
"the reference, e.g. pyrros-2023-supplement.pdf",
),
+ llm_refs: bool = typer.Option(
+ True, "--llm-refs/--no-llm-refs",
+ help="Also have a model read the printed reference list as a second opinion "
+ "on the numbering (one extra model call; off under --parse-only)",
+ ),
) -> None:
"""Full pipeline: ingest → refs → scout → check → highlight → report."""
console.print(BANNER)
@@ -1209,7 +1414,8 @@ def run(
# without erroring.
doi = doi or _detected_doi(manuscript)
_refs_pipeline(manuscript=manuscript, case=case, provided=provided, email=email,
- parse_only=False, backend=backend, doi=doi, supplement=supplement)
+ parse_only=False, backend=backend, doi=doi, supplement=supplement,
+ llm_refs=llm_refs)
if with_scout:
scout(case=case, doi=doi, email=email)
_check_pipeline(case=case, model=model, backend=backend)
diff --git a/src/papertrace/disclosures.py b/src/papertrace/disclosures.py
index ed4ba97..3c28863 100644
--- a/src/papertrace/disclosures.py
+++ b/src/papertrace/disclosures.py
@@ -59,6 +59,11 @@
# ANCHOR_NOT_LOCATED_TOKEN / ANCHOR_UNKNOWN_TOKEN are one Disclosure key with
# one token per state actually reached. Only this state's token exists yet.
CLAIM_PAIRING_WITHHELD_TOKEN = "verdict withheld — the reference list readings disagree"
+# no apostrophe, `&`, `<` or `>` in either — same HTML-autoescape rule as
+# LABELS_DISPUTED_TOKEN above, since both are asserted verbatim in two formats
+# that escape and two that do not
+REFLIST_TOKEN = "the reference list was also read by a model"
+NUMBERING_CORROBORATION_TOKEN = "two readings of the reference list agree on every cited label"
# The keys `claim_disclosures()` can emit. Declared here, where the producers
# live, because three of the four report formats filter claim disclosures by
# explicit key and so drop an unlisted one without erroring. The viewer is
@@ -542,6 +547,106 @@ def _labels_disputed(manifest) -> Disclosure | None:
)
+def _reflist(manifest) -> Disclosure | None:
+ """The required disclosure for the model reading, in all three of its outcomes.
+
+ Fires on a reading used, a reading discarded, and a reading asked for and not
+ obtained — never only on success, because silence about an attempt reads as
+ no news. The token is phrased AROUND in every branch, the way
+ `SUPPLEMENT_IDENTITY_TOKEN` is: it is asserted verbatim in all four formats,
+ so a branch where it is not literally true would make one format lie.
+ """
+ model = getattr(manifest, "reflist_model", "") or ""
+ notes = list(getattr(manifest, "reflist_fields_discarded", []) or [])
+ # a different kind of finding, and it must reach the reader too: a reading
+ # whose own labels do not add up is worth knowing about even when every
+ # value it proposed was printed
+ numbering = list(getattr(manifest, "reflist_numbering_findings", []) or [])
+ if not model and not notes:
+ return None # no model reading was asked for
+ ceiling = (
+ "A model agreeing with a parse is a second reading, not confirmation: it read "
+ "the same document, so a reference the layout destroyed is one it may also "
+ "have missed."
+ )
+ if not model:
+ head = (
+ f"This run asked that {REFLIST_TOKEN}, and no reading was obtained: "
+ f"{notes[0]}. The reference numbering therefore rests on the readings above "
+ f"it and nothing else."
+ )
+ short = f"{REFLIST_TOKEN}: asked for, not obtained — {notes[0]}"
+ elif any(n.startswith("reading discarded") for n in notes):
+ why = next(n for n in notes if n.startswith("reading discarded"))
+ head = (
+ f"Here {REFLIST_TOKEN} ({model}), and its reading was discarded rather than "
+ f"used: {why}. Nothing it proposed contributed to the list below."
+ )
+ short = f"{REFLIST_TOKEN} ({model}) — discarded as unusable"
+ else:
+ dropped = (
+ f"{len(notes)} value{'' if len(notes) == 1 else 's'} it proposed "
+ f"{'was' if len(notes) == 1 else 'were'} not found in the printed text and "
+ f"{'was' if len(notes) == 1 else 'were'} discarded"
+ if notes else "every value it proposed was found in the printed text"
+ )
+ head = (
+ f"Here {REFLIST_TOKEN} ({model}), shown two extractions of the same printed "
+ f"bibliography and asked what numbered list it carries; {dropped}."
+ )
+ short = f"{REFLIST_TOKEN} ({model}) — {len(notes)} discarded"
+ if numbering:
+ # appended rather than folded into `dropped`: "3 values discarded" and
+ # "it numbered one entry twice" are different facts, and a reader who
+ # sees them as one number cannot tell which happened
+ head += (
+ " Its own numbering did not add up either — "
+ + "; ".join(numbering)
+ + "."
+ )
+ return Disclosure(
+ key="reflist",
+ level="info",
+ token=REFLIST_TOKEN,
+ text=f"{head} {ceiling}",
+ short=short,
+ # both kinds, numbering first: it describes the reading as a whole, and
+ # a truncated list should not lose the structural finding to six field
+ # names
+ rows=tuple((numbering + notes)[:6]),
+ )
+
+
+def _numbering_corroboration(manifest) -> Disclosure | None:
+ """Fires on agreement, and deliberately NOT gated on `numbering_verified`.
+
+ The unconfirmed-numbering warning is right to fire whenever nothing checked
+ the numbering, and on a paper whose readings all agree it is also the whole
+ of what the report says about the reference list — which is how it came to
+ read as an alarm about the 22-vs-19 case on papers where nothing was wrong.
+ This is the other half of that sentence, and it is careful not to become the
+ claim the warning is about: agreement between readings is not a checked
+ numbering.
+ """
+ if not getattr(manifest, "numbering_corroborated", False):
+ return None
+ readings = list(getattr(manifest, "corroborating_readings", []) or [])
+ named = ", ".join(readings) if readings else "the readings taken"
+ return Disclosure(
+ key="numbering_corroboration",
+ level="info",
+ token=NUMBERING_CORROBORATION_TOKEN,
+ text=(
+ f"On this run {NUMBERING_CORROBORATION_TOKEN} ({named}). That is corroboration, "
+ "not confirmation, and it does not make the numbering verified: these are "
+ "readings of one printed page, so a reference its layout destroyed is one they "
+ "can all have missed in the same way. Any label they disagreed about is listed "
+ "separately, and no verdict was printed for it."
+ ),
+ short=f"{NUMBERING_CORROBORATION_TOKEN} ({named})",
+ )
+
+
def _claim_numbering(claim, manifest) -> Disclosure:
"""The run-level warning, said again where the verdict is read.
@@ -781,6 +886,10 @@ def run_disclosures(results, manifest=None) -> list[Disclosure]:
out.append(d)
if d := _labels_disputed(manifest):
out.append(d)
+ if d := _numbering_corroboration(manifest):
+ out.append(d)
+ if d := _reflist(manifest):
+ out.append(d)
return out
diff --git a/src/papertrace/models.py b/src/papertrace/models.py
index e6f215f..8542daf 100644
--- a/src/papertrace/models.py
+++ b/src/papertrace/models.py
@@ -603,6 +603,13 @@ class RefManifest:
# Which fields of the LLM's proposed reading could not be found verbatim
# in either text it was shown, and were discarded rather than trusted.
reflist_fields_discarded: list[str] = field(default_factory=list)
+ # A different kind of finding, kept in its own field for that reason: the
+ # model reading's own labels did not add up — a numeral proposed twice, or a
+ # gap in 1..max. Not a value that went missing, so counting it among the
+ # discarded fields would report a number of dropped values that never
+ # dropped. Empty means none found, which on a reading that produced no
+ # entries is not the same as none present.
+ reflist_numbering_findings: list[str] = field(default_factory=list)
def document(self, slug: str) -> Document | None:
"""The judgeable file this slug names, article or supplement, or None."""
@@ -688,6 +695,7 @@ def to_json(self, path: Path) -> None:
"numbering_chosen_by": self.numbering_chosen_by,
"reflist_model": self.reflist_model,
"reflist_fields_discarded": self.reflist_fields_discarded,
+ "reflist_numbering_findings": self.reflist_numbering_findings,
"summary": {
"total": len(self.entries),
"available": len(self.retrieved),
@@ -729,6 +737,7 @@ def from_json(cls, path: Path) -> RefManifest:
numbering_chosen_by=data.get("numbering_chosen_by", ""),
reflist_model=data.get("reflist_model", ""),
reflist_fields_discarded=data.get("reflist_fields_discarded", []),
+ reflist_numbering_findings=data.get("reflist_numbering_findings", []),
)
diff --git a/src/papertrace/refs.py b/src/papertrace/refs.py
index 36a94e2..0f2f22a 100644
--- a/src/papertrace/refs.py
+++ b/src/papertrace/refs.py
@@ -828,6 +828,27 @@ def label_agreement(
return result
+def stamp_seen_in(chosen: list[RefEntry], candidates: dict[str, list[RefEntry]]) -> None:
+ """Record, per chosen entry, which readings also carried that work.
+
+ In place, because the entries are already the manifest's. An entry carried
+ by one reading only is the interesting case — especially `["llm"]`, which
+ means no deterministic reading found it at all.
+
+ Matched on the label AND `_same_work`: a reading that carries [12] naming a
+ different paper has not corroborated this entry, and stamping its name here
+ on the strength of the shared numeral would turn a disagreement into
+ provenance. That is the same mistake `_covers` makes about extent, and it is
+ the reason this is not simply `label in {e.num for e in cand}`.
+ """
+ for e in chosen:
+ e.seen_in = sorted(
+ name
+ for name, cand in candidates.items()
+ if any(o.num == e.num and _same_work(e, o) for o in cand)
+ )
+
+
def reconcile(
body_labels: set[str],
crossref: list[RefEntry] | None,
diff --git a/src/papertrace/templates/report_terminal.html.j2 b/src/papertrace/templates/report_terminal.html.j2
index de3893a..7394643 100644
--- a/src/papertrace/templates/report_terminal.html.j2
+++ b/src/papertrace/templates/report_terminal.html.j2
@@ -105,6 +105,12 @@
{% endfor %}
{% for d in disclosures if d.key == "labels_disputed" %}
▸ check⚠ {{ d.short }}
+{% endfor %}
+{% for d in disclosures if d.key == "numbering_corroboration" %}
+ ▸ resolve{{ d.short }}
+{% endfor %}
+{% for d in disclosures if d.key == "reflist" %}
+ ▸ resolve{{ d.short }}
{% endfor %}
▸ check{{ r.claims|length }} citation-backed claims · reading each against its cited page{% if r.uncited %} · {{ r.uncited|length }} uncited assertions flagged{% endif %}
{% if scout and not scout.error %}
diff --git a/src/papertrace/wizard.py b/src/papertrace/wizard.py
index 13caa68..8af83d2 100644
--- a/src/papertrace/wizard.py
+++ b/src/papertrace/wizard.py
@@ -217,11 +217,16 @@ def workload(pdf: Path) -> dict:
"places": len(groups),
"multi": multi,
"labels": len(labels),
- "model_calls": 1 + cited_source_calls,
+ # one reference-list reading, one extraction call, then one per cited
+ # source. The reference-list call gets no retry — `reflist.propose` asks
+ # once and reports what it got — so it adds exactly 1 to the base figure
+ # and 1 to the ceiling, on top of the retried figure below rather than
+ # inside it.
+ "model_calls": 2 + cited_source_calls,
# the retry is real spend: ASK_ATTEMPTS attempts for extraction, and
# ASK_ATTEMPTS attempts per judging call. Derived from ask.py rather
# than a local multiplier, so the estimate cannot drift from the policy.
- "model_calls_max": ASK_ATTEMPTS * (1 + cited_source_calls),
+ "model_calls_max": 1 + ASK_ATTEMPTS * (1 + cited_source_calls),
"style_unrecognised": len(groups) == 0,
}
@@ -578,7 +583,7 @@ def _n(count: int, noun: str) -> str:
run_cmd(
manuscript=paper, case=case, provided=provided, email=email, model=None,
png=png, backend="auto", with_scout=with_scout, doi=doi, formats=formats,
- supplement=supplement,
+ supplement=supplement, llm_refs=True,
)
diff --git a/tests/test_label_agreement.py b/tests/test_label_agreement.py
index 59099ac..c087540 100644
--- a/tests/test_label_agreement.py
+++ b/tests/test_label_agreement.py
@@ -36,6 +36,7 @@
_entry,
_same_work,
label_agreement,
+ stamp_seen_in,
)
ROOT = Path(__file__).resolve().parent.parent
@@ -386,3 +387,33 @@ def test_same_work_keeps_its_own_direction_for_its_own_callers():
b = RefEntry(num="7", raw="Q. Z. 2020. 88(1):4-7.")
assert _same_work(a, b) is True
assert _comparably_same(a, b) is False
+
+
+# --- seen_in: which readings also carried a chosen entry's work ------------
+
+
+def test_an_entry_only_the_model_found_is_spottable_in_the_manifest():
+ """The published field's whole purpose. An entry no deterministic reading
+ carried is the one a reader most needs to see flagged, and `seen_in ==
+ ["llm"]` is how they see it."""
+ chosen = [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")]
+ candidates = {
+ "parsed": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ "llm": [_work("1", "10.1000/x1")],
+ }
+ stamp_seen_in(chosen, candidates)
+ assert chosen[0].seen_in == ["llm", "parsed"]
+ assert chosen[1].seen_in == ["parsed"]
+
+
+def test_a_reading_naming_a_different_paper_under_the_same_label_is_not_provenance():
+ """Sharing a numeral is not agreeing about a work.
+
+ Stamping on the label alone would record the disagreeing reading as having
+ corroborated this entry — the extent-for-content substitution `_covers` is
+ documented as blind to, reintroduced in a published field."""
+ chosen = [_work("5", "10.1000/x5")]
+ candidates = {"parsed": [_work("5", "10.1000/x5")],
+ "pymupdf": [_work("5", "10.1000/x99")]}
+ stamp_seen_in(chosen, candidates)
+ assert chosen[0].seen_in == ["parsed"]
diff --git a/tests/test_pipeline_split.py b/tests/test_pipeline_split.py
index 5a112fe..cef2dff 100644
--- a/tests/test_pipeline_split.py
+++ b/tests/test_pipeline_split.py
@@ -58,11 +58,12 @@ def fake_pipeline(**kw):
pdf.write_bytes(b"%PDF-1.4\n")
cli.refs(manuscript=pdf, case=None, provided=None, email=None,
- parse_only=False, backend="auto", doi=None, supplement=None)
+ parse_only=False, backend="auto", doi=None, supplement=None, llm_refs=True)
assert seen == {
"manuscript": pdf, "case": None, "provided": None, "email": None,
"parse_only": False, "backend": "auto", "doi": None, "supplement": None,
+ "llm_refs": True,
}
diff --git a/tests/test_reference_readings.py b/tests/test_reference_readings.py
new file mode 100644
index 0000000..0e7800f
--- /dev/null
+++ b/tests/test_reference_readings.py
@@ -0,0 +1,537 @@
+"""Four readings of one bibliography, and the one thing the extra two may do.
+
+`reconcile` still chooses between the Crossref deposit and the run backend's
+parse. The flat-text reading and the model's reading are voters on a second
+axis: they can cause a verdict to be withheld and nothing else. The first test
+here is the load-bearing one — it is the whole safety property of the feature,
+stated as an equality.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+import pymupdf
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+
+from papertrace import cli # noqa: E402
+from papertrace import reflist as reflist_mod # noqa: E402
+from papertrace.models import RefEntry, RefManifest, SourceMap # noqa: E402
+from papertrace.refs import _entry # noqa: E402
+
+SCHEMA_PATH = Path(__file__).resolve().parent.parent / "schemas" / "refs_manifest.schema.json"
+
+
+def _paper(path: Path) -> Path:
+ """A one-page PDF citing [1] and [2], with both references printed."""
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "A Fixture Imaging Study", fontsize=16)
+ page.insert_text((72, 140), "Body text citing [1] and also [2] here.", fontsize=11)
+ page.insert_text((72, 200), "References", fontsize=14)
+ page.insert_text((72, 230), "[1] Alpha A. A first paper. J Fixture. 2020;1:1-9.", fontsize=11)
+ page.insert_text((72, 250), "[2] Beta B. A second paper. J Fixture. 2021;2:10-19.", fontsize=11)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ doc.save(path)
+ doc.close()
+ return path
+
+
+def _work(num: str, doi: str) -> RefEntry:
+ """Built through `_entry`, the production parse path — not by setting
+ fields by hand, which would bypass the code every real candidate goes
+ through (mirrors `tests/test_label_agreement.py`'s own `_work`)."""
+ return _entry(num, f"Author for entry {num}. A distinctive title. doi:{doi}")
+
+
+def _wire_offline(monkeypatch) -> None:
+ """No network and no model call by default: retrieval is a no-op, Crossref
+ answers nothing, and `claude` is treated as PRESENT regardless of the host.
+
+ That last patch matters even though `reflist.propose` is what most of these
+ tests mock, not `ask.claude_available`: `_llm_reference_reading` checks
+ availability BEFORE calling `propose`, so on a machine with no `claude` CLI
+ on PATH — the normal case in CI, and not the case on a developer's own
+ machine, which is exactly the trap `ask_mod.claude_available` vs.
+ `check_mod.claude_available` already caught once in this plan — every test
+ below that mocks `propose` to return a fabricated reading would silently
+ take the "not attempted" branch instead and never call the mock at all.
+ The one test that wants the real "claude absent" branch overrides this
+ itself, after this fixture runs.
+ """
+ import papertrace.ask as ask_mod
+ import papertrace.refs as refs_mod
+ from papertrace.refs import CrossrefDeposit
+
+ monkeypatch.setattr(refs_mod, "resolve_all",
+ lambda entries, dest, email, provided_dir=None, progress=None,
+ taken=None: entries)
+ monkeypatch.setattr(refs_mod, "crossref_deposit",
+ lambda client, doi, email: CrossrefDeposit(absent="no deposit, in a test"))
+ monkeypatch.setattr(cli, "_detected_doi", lambda m: None)
+ monkeypatch.setattr(ask_mod, "claude_available", lambda: True)
+ monkeypatch.setenv("PAPERTRACE_EMAIL", "test@example.org")
+
+
+@pytest.fixture()
+def offline(monkeypatch):
+ _wire_offline(monkeypatch)
+
+
+def _run_refs(tmp_path: Path, monkeypatch, **kw) -> RefManifest:
+ """Wire the offline doubles, run `_refs_pipeline` on the fixture paper, and
+ hand back the manifest it wrote — schema-validated on the way, so every
+ caller of this helper gets Gate 2 for free rather than each writing its own
+ `jsonschema.validate`.
+
+ A plain function, not a fixture: Step 4's test needs per-call keyword
+ overrides (`llm_refs=True` here, forwarded straight to `_refs_pipeline`),
+ which a fixture cannot take.
+ """
+ import jsonschema
+
+ _wire_offline(monkeypatch)
+ pdf = _paper(tmp_path / "paper.pdf")
+ case = tmp_path / "case"
+ args = dict(manuscript=pdf, case=case, provided=None, email="test@example.org",
+ parse_only=False, backend="pymupdf", llm_refs=True)
+ args.update(kw)
+ cli._refs_pipeline(**args)
+
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.validate(payload, schema)
+ return RefManifest.from_json(case / "refs_manifest.json")
+
+
+def _entries(case: Path) -> str:
+ """The resolved entries, canonically serialised — the bytes Ruling 2 is about."""
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ return json.dumps(payload["entries"], sort_keys=True, ensure_ascii=False)
+
+
+def _smap(converter: str) -> SourceMap:
+ """Only `converter` matters to the voter dict, and only its first token."""
+ return SourceMap(doc="paper.pdf", pages=1, blocks=[], converter=converter)
+
+
+def test_a_model_naming_different_papers_changes_no_resolved_entry(tmp_path, offline, monkeypatch):
+ """Ruling 2, as an equality. The safety property the whole design rests on.
+
+ The model's reading is a voter. It can withhold a verdict; it cannot put a
+ paper into the manifest, cannot change a numeral, and cannot cause anything
+ to be resolved or downloaded. So a run whose model reading names two
+ completely different works must produce byte-identical entries to a run that
+ never asked a model at all. If this test ever fails, `others["llm"]` has
+ reached `reconcile`'s arguments or `resolve_all`, and no other test in this
+ suite would notice.
+
+ `backend="pymupdf"`, not `"docling"`: the brief's own draft of this test used
+ `"docling"` on a real PDF with no mock in front of `ingest_pdf`, which would
+ have triggered a real docling conversion — a ~500 MB model download on first
+ use, per `CLAUDE.md` — on every run of an "offline" test. `reflist.propose`
+ is fully mocked below and ignores its arguments, so which backend produced
+ the parse is irrelevant to what this test actually checks.
+ """
+ pdf = _paper(tmp_path / "paper.pdf")
+ fabricated = (
+ [RefEntry(num="1", raw="Zeta Z. An entirely different paper. 1999.",
+ title="An entirely different paper", year="1999", seen_in=["docling"]),
+ RefEntry(num="2", raw="Eta E. Another different paper. 1998.",
+ title="Another different paper", year="1998", seen_in=["pymupdf"])],
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2,
+ readings=["docling", "pymupdf"]),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: fabricated)
+
+ with_llm, without = tmp_path / "with", tmp_path / "without"
+ cli._refs_pipeline(manuscript=pdf, case=with_llm, provided=None,
+ email="test@example.org", parse_only=False, backend="pymupdf",
+ llm_refs=True)
+ cli._refs_pipeline(manuscript=pdf, case=without, provided=None,
+ email="test@example.org", parse_only=False, backend="pymupdf",
+ llm_refs=False)
+
+ assert _entries(with_llm) == _entries(without)
+ assert "different paper" not in _entries(with_llm)
+
+
+def test_the_pymupdf_reading_is_skipped_on_a_pymupdf_backend_run():
+ """A reading agreeing with itself is not corroboration.
+
+ The flat-text voter exists because docling and pymupdf read a hanging-indent
+ bibliography differently. On a run whose backend already IS pymupdf it is
+ the identical text, so counting it would turn every `single` label into an
+ `agreed` one and report corroboration that nothing corroborated.
+ """
+ parsed = [RefEntry(num="1", raw="Alpha A. A first paper. 2020.")]
+ flat = [RefEntry(num="1", raw="Alpha A. A first paper. 2020.")]
+
+ flat_run = cli._reference_readings(smap=_smap("pymupdf"), parsed=parsed, flat=flat)
+ layout_run = cli._reference_readings(smap=_smap("docling 2.8.0"), parsed=parsed, flat=flat)
+
+ assert set(flat_run) == {"parsed"}, flat_run
+ assert set(layout_run) == {"parsed", "pymupdf"}, layout_run
+
+
+def test_a_reading_that_is_absent_is_omitted_rather_than_empty():
+ """An empty list is a reading that found nothing; an absent key is no reading.
+
+ `label_agreement` counts readings that have a label. A reading present as
+ `[]` votes against every label it does not carry, which is every label —
+ turning "we did not ask" into "one reading says no".
+ """
+ got = cli._reference_readings(smap=_smap("docling 2.8.0"),
+ parsed=[RefEntry(num="1", raw="Alpha A. 2020.")],
+ crossref=None, flat=None, llm=None)
+ assert set(got) == {"parsed"}, got
+
+
+def test_no_llm_refs_makes_zero_model_calls(tmp_path, offline, monkeypatch):
+ """The flag has to actually gate the spend, not just the disclosure."""
+ import papertrace.ask as ask_mod
+
+ calls = []
+ monkeypatch.setattr(ask_mod, "_ask", lambda prompt, model=None: calls.append(1) or "[]")
+ pdf = _paper(tmp_path / "paper.pdf")
+
+ cli._refs_pipeline(manuscript=pdf, case=tmp_path / "case", provided=None,
+ email="test@example.org", parse_only=False, backend="pymupdf",
+ llm_refs=False)
+
+ assert calls == []
+
+
+def test_parse_only_makes_no_model_call_even_with_llm_refs_on(tmp_path, offline, monkeypatch):
+ """`--parse-only` is documented as "List references, no network".
+
+ A paid subprocess is exactly what that promise excludes, and the promise is
+ the reason someone reaches for the flag. The flat-text reading still runs —
+ it is local and needs no network — which is what closes the gap the comment
+ at cli.py:560-563 describes.
+ """
+ import papertrace.ask as ask_mod
+
+ calls = []
+ monkeypatch.setattr(ask_mod, "_ask", lambda prompt, model=None: calls.append(1) or "[]")
+ pdf = _paper(tmp_path / "paper.pdf")
+
+ cli._refs_pipeline(manuscript=pdf, case=tmp_path / "case", provided=None,
+ email="test@example.org", parse_only=True, backend="pymupdf",
+ llm_refs=True)
+
+ assert calls == []
+
+
+def test_an_unavailable_claude_degrades_to_a_stated_absence(tmp_path, offline, monkeypatch):
+ """The cardinal rule, at this seam. Never "it agreed" — "it was not asked".
+
+ `claude` missing from PATH is the ordinary case for someone who installed
+ papertrace and not Claude Code. The run proceeds on the deterministic
+ readings, the manifest records the reason, and no format anywhere says a
+ model corroborated anything.
+ """
+ import papertrace.ask as ask_mod
+
+ monkeypatch.setattr(ask_mod, "claude_available", lambda: False)
+ monkeypatch.setattr(ask_mod, "_ask",
+ lambda prompt, model=None: pytest.fail("claude was absent and asked anyway"))
+ pdf = _paper(tmp_path / "paper.pdf")
+ case = tmp_path / "case"
+
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None, email="test@example.org",
+ parse_only=False, backend="pymupdf", llm_refs=True)
+
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ assert payload["reflist_model"] == ""
+ assert any("not attempted" in f for f in payload["reflist_fields_discarded"]), payload
+ assert "llm" not in payload["corroborating_readings"]
+ assert payload["numbering_corroborated"] is False
+
+
+def test_a_failed_model_call_does_not_kill_the_refs_stage(tmp_path, offline, monkeypatch):
+ """A timeout at the bibliography must not cost the user the retrieval.
+
+ `refs` has already parsed the list and is about to fetch the sources. A
+ RuntimeError out of the seam is a failed corroboration attempt, not a failed
+ refs stage — and the reason lands where a reader can see it.
+ """
+ def boom(*a, **kw):
+ raise RuntimeError("claude -p timed out after 600s")
+
+ monkeypatch.setattr(reflist_mod, "propose", boom)
+ pdf = _paper(tmp_path / "paper.pdf")
+ case = tmp_path / "case"
+
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None, email="test@example.org",
+ parse_only=False, backend="pymupdf", llm_refs=True)
+
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ assert len(payload["entries"]) == 2
+ assert any("timed out" in f for f in payload["reflist_fields_discarded"]), payload
+ assert payload["reflist_model"] == ""
+
+
+def test_corroboration_requires_every_cited_label_to_agree(tmp_path, offline, monkeypatch):
+ """One disputed label is not "mostly corroborated".
+
+ The flag is read as a reassurance, so it must mean what it says: every label
+ the body cites was agreed by at least two readings. Any disagreement is
+ `disputed` — no majority vote and no ranking, because "a non-unique match is
+ refused, never ranked".
+ """
+ pdf = _paper(tmp_path / "paper.pdf")
+ disagrees = (
+ [RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
+ title="A first paper", year="2020", seen_in=["docling", "pymupdf"]),
+ RefEntry(num="2", raw="Gamma G. A wholly unrelated third paper. 2015.",
+ title="A wholly unrelated third paper", year="2015", seen_in=["A"])],
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: disagrees)
+ case = tmp_path / "case"
+
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None, email="test@example.org",
+ parse_only=False, backend="pymupdf", llm_refs=True)
+
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ assert payload["numbering_corroborated"] is False
+ assert payload["labels_disputed"] == ["2"], payload["labels_disputed"]
+ assert payload["corroborating_readings"] == []
+
+
+def test_no_model_reply_can_set_numbering_verified(tmp_path, offline, monkeypatch):
+ """The invariant, asserted here as well as in Task 8.
+
+ Task 8 parametrises this over every reply shape; this is the one shape that
+ would be most tempting — a model agreeing with the parse entry for entry.
+ Corroboration is a second axis, not evidence of a checked numbering.
+
+ Not `_paper()`: that fixture cites exactly the two references it prints, so
+ `_covers` — the extent check `reconcile` uses on its own, with no model in
+ the loop at all — is already satisfied by the parse alone, and `verified`
+ would come back `True` regardless of anything this test does. Nothing here
+ could then tell "verified because the parse already covered the body's
+ labels" apart from "verified because a mocked model agreed with it". This
+ fixture prints a THIRD, uncited reference, so the parse alone has three
+ entries against two cited labels and fails `_covers` on its own — the only
+ way to make the two cases distinguishable.
+ """
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "A Fixture Imaging Study", fontsize=16)
+ page.insert_text((72, 140), "Body text citing [1] and also [2] here.", fontsize=11)
+ page.insert_text((72, 200), "References", fontsize=14)
+ page.insert_text((72, 230), "[1] Alpha A. A first paper. J Fixture. 2020;1:1-9.", fontsize=11)
+ page.insert_text((72, 250), "[2] Beta B. A second paper. J Fixture. 2021;2:10-19.", fontsize=11)
+ page.insert_text((72, 270), "[3] Gamma G. A third, uncited paper. J Fixture. 2019;3:1-5.",
+ fontsize=11)
+ pdf = tmp_path / "paper.pdf"
+ doc.save(pdf)
+ doc.close()
+ agrees = (
+ [RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
+ title="A first paper", year="2020", seen_in=["docling", "pymupdf"]),
+ RefEntry(num="2", raw="Beta B. A second paper. J Fixture. 2021;2:10-19.",
+ title="A second paper", year="2021", seen_in=["docling", "pymupdf"])],
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: agrees)
+ case = tmp_path / "case"
+
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None, email="test@example.org",
+ parse_only=False, backend="pymupdf", llm_refs=True)
+
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ assert payload["numbering_verified"] is False
+ assert payload["numbering_corroborated"] is True
+ assert payload["corroborating_readings"] == ["llm", "parsed"]
+
+
+def test_both_commands_declare_the_flag_and_run_forwards_it(tmp_path, monkeypatch):
+ """`run` calls `_refs_pipeline` directly, so a parameter it forgets is dropped.
+
+ That failure is silent: the audit runs without the reading and says nothing
+ about it. `refs` and `run` must both declare the flag, and `run` must pass it.
+ """
+ import inspect
+
+ # captured BEFORE `_refs_pipeline` is monkeypatched below: the brief's own
+ # draft of this test inspected `cli._refs_pipeline` AFTER replacing it with
+ # a bare `lambda **kw: seen.update(kw)`, so the "signature" it asserted on
+ # was the lambda's, which has no `llm_refs` parameter at all — a `KeyError`,
+ # not the intended pass. The real function's signature has to be read while
+ # it is still the real function.
+ refs_pipeline_sig = inspect.signature(cli._refs_pipeline)
+
+ seen = {}
+ monkeypatch.setattr(cli, "_ingest_pipeline", lambda **kw: None)
+ monkeypatch.setattr(cli, "_refs_pipeline", lambda **kw: seen.update(kw))
+ monkeypatch.setattr(cli, "scout", lambda **kw: None)
+ monkeypatch.setattr(cli, "_check_pipeline", lambda **kw: None)
+ monkeypatch.setattr(cli, "highlight", lambda **kw: None)
+ monkeypatch.setattr(cli, "_report_pipeline", lambda **kw: None)
+ monkeypatch.setattr(cli, "_detected_doi", lambda m: None)
+ pdf = _paper(tmp_path / "paper.pdf")
+
+ cli.run(manuscript=pdf, case=tmp_path / "case", provided=None, email="test@example.org",
+ model=None, png=False, backend="pymupdf", with_scout=False, doi=None,
+ formats=None, supplement=None, llm_refs=False)
+
+ assert seen["llm_refs"] is False
+ for command in (cli.refs, cli.run):
+ assert "llm_refs" in inspect.signature(command).parameters, command.__name__
+ assert refs_pipeline_sig.parameters["llm_refs"].default is True
+
+
+def test_the_reflist_disclosure_reaches_every_format_and_carries_the_ceiling(tmp_path):
+ """The required disclosure, and the sentence that bounds what it means.
+
+ "A model agreeing with a parse is a second reading, not confirmation" is the
+ only honest claim available about this reading, and a reader who sees the
+ model named without it will read it as confirmation.
+ """
+ from papertrace.disclosures import REFLIST_TOKEN, run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_model="claude-opus-5",
+ reflist_fields_discarded=["[2] journal"])
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ d = fired[0]
+ assert d.token == REFLIST_TOKEN
+ assert "second reading, not confirmation" in d.text
+ assert "may also have missed" in d.text
+ assert "claude-opus-5" in d.text
+
+
+def test_a_model_reading_that_was_not_obtained_still_discloses_that(tmp_path):
+ """Silence would be read as "no news". The attempt and its failure are news.
+
+ The token stays true in this branch because the sentence is phrased around
+ it — the same discipline `SUPPLEMENT_IDENTITY_TOKEN` is written under.
+ """
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_model="",
+ reflist_fields_discarded=["not attempted — claude is not on PATH"])
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "not attempted" in fired[0].text
+ assert "agreed" not in fired[0].text
+
+
+def test_corroboration_is_disclosed_without_claiming_the_numbering_was_verified(tmp_path):
+ """The false alarm this fixes, and the overstatement it must not become.
+
+ On the 22-vs-19 case the numbering warning fires for a real reason. Where
+ two readings do agree on every cited label, a reader deserves to be told —
+ and deserves not to be told the numbering was checked, because it was not.
+ """
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", numbering_verified=False,
+ numbering_corroborated=True,
+ corroborating_readings=["llm", "parsed", "pymupdf"])
+ fired = {d.key for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)}
+ assert "numbering_corroboration" in fired
+ assert "numbering" in fired, "corroboration must not suppress the unconfirmed-numbering warning"
+
+
+def test_the_new_manifest_fields_round_trip_and_validate(tmp_path):
+ """Gate 2. Five additive fields, none required, all readable by an older file.
+
+ These five landed already in Task 3, and this test exercises them again
+ from Task 6's own module — it is the one Step 2 offered, kept for parity
+ with the brief, and `test_reflist_numbering_findings_round_trips_and_validates`
+ below covers the one field this task actually adds.
+ """
+ import jsonschema
+
+ m = RefManifest(manuscript="p.pdf", numbering_corroborated=True,
+ corroborating_readings=["parsed", "pymupdf"],
+ labels_disputed=["6", "7"], reflist_model="claude-opus-5",
+ reflist_fields_discarded=["[6] journal"])
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ payload = json.loads(path.read_text())
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.validate(payload, schema)
+ assert not set(schema.get("required", [])) & {
+ "numbering_corroborated", "corroborating_readings", "labels_disputed",
+ "reflist_model", "reflist_fields_discarded",
+ }
+
+ back = RefManifest.from_json(path)
+ assert back.numbering_corroborated is True
+ assert back.corroborating_readings == ["parsed", "pymupdf"]
+ assert back.labels_disputed == ["6", "7"]
+ assert back.reflist_model == "claude-opus-5"
+ assert back.reflist_fields_discarded == ["[6] journal"]
+
+ # a manifest written before these fields must still load, and must not read
+ # as "nothing disputed" — absent means never computed
+ for key in ("numbering_corroborated", "corroborating_readings", "labels_disputed",
+ "reflist_model", "reflist_fields_discarded"):
+ del payload[key]
+ path.write_text(json.dumps(payload))
+ old = RefManifest.from_json(path)
+ assert old.numbering_corroborated is False
+ assert old.labels_disputed == []
+
+
+def test_reflist_numbering_findings_round_trips_and_validates(tmp_path):
+ """Gate 2, for the one field this task actually adds.
+
+ Kept apart from `reflist_fields_discarded` on purpose — see
+ `ReflistProvenance.numbering_findings`'s own docstring for why a duplicated
+ or missing numeral is a different kind of finding from a dropped value.
+ """
+ import jsonschema
+
+ m = RefManifest(manuscript="p.pdf",
+ reflist_numbering_findings=["numerals proposed twice: 2"])
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ payload = json.loads(path.read_text())
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.validate(payload, schema)
+ assert "reflist_numbering_findings" not in schema.get("required", [])
+
+ back = RefManifest.from_json(path)
+ assert back.reflist_numbering_findings == ["numerals proposed twice: 2"]
+
+ del payload["reflist_numbering_findings"]
+ path.write_text(json.dumps(payload))
+ old = RefManifest.from_json(path)
+ assert old.reflist_numbering_findings == []
+
+
+def test_a_numbering_finding_from_the_model_reading_reaches_the_manifest(tmp_path, monkeypatch):
+ """`reflist.propose` reports a duplicated or missing numeral separately from
+ a discarded field, because the two mean different things. An earlier draft
+ of this step read only `fields_discarded`, so every duplicate and gap the
+ model reading found was silently dropped before any reader saw it."""
+ proposed = (
+ [_work("2", "10.1000/x2"), _work("2", "10.1000/x22")],
+ reflist_mod.ReflistProvenance(
+ model="claude-opus-5",
+ entries_proposed=2,
+ numbering_findings=["numerals proposed twice: 2"],
+ readings=["pymupdf", "docling"],
+ ),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: proposed)
+
+ manifest = _run_refs(tmp_path, monkeypatch, llm_refs=True)
+
+ assert manifest.reflist_numbering_findings == ["numerals proposed twice: 2"]
+ assert not any(
+ "twice" in f for f in manifest.reflist_fields_discarded
+ ), manifest.reflist_fields_discarded
diff --git a/tests/test_wizard.py b/tests/test_wizard.py
index 76293c7..5e39a10 100644
--- a/tests/test_wizard.py
+++ b/tests/test_wizard.py
@@ -666,15 +666,23 @@ def test_the_worst_case_call_count_includes_the_retry(tmp_path):
"""The wizard promised "up to N model calls" from one extraction plus one
per cited source. `_ask` retries once, so a single-source run advertised as
2 could issue 3. The ceiling now comes from ask.py's own attempt count,
- so the two cannot drift."""
+ so the two cannot drift.
+
+ Both figures also carry the reference-list reading `refs` now makes before
+ `check` ever runs: one un-retried call, so it adds exactly 1 to each side
+ of the estimate rather than multiplying through the retried figure — this
+ task adds that call on top of the `ASK_ATTEMPTS * (1 + cited_source_calls)`
+ shape a previous task already moved `model_calls_max` to, rather than
+ reverting it back to a flat `ASK_ATTEMPTS * cited_source_calls`.
+ """
from papertrace.ask import ASK_ATTEMPTS
pdf = _one_pager(tmp_path / "m.pdf",
"Title\nOne sentence citing [1].\nReferences\n[1] A. 2020.")
w = wizard.workload(pdf)
- assert w["model_calls"] == 2, w
- assert w["model_calls_max"] == ASK_ATTEMPTS * (1 + 1), w
+ assert w["model_calls"] == 3, w
+ assert w["model_calls_max"] == 1 + ASK_ATTEMPTS * (1 + 1), w
assert w["model_calls_max"] >= w["model_calls"]
From 67ab473015204f4338004f196fdd78c5b64ebdee Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Thu, 17 Sep 2026 23:15:21 +0200
Subject: [PATCH 26/44] refs: a model reading that voted is never published as
one that never happened
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Review round 1 on the four-readings task. Branch on whether the model was
actually asked (ReflistProvenance.attempted / RefManifest.reflist_attempted),
never on whether the reply named itself — a call that verifies cleanly and
votes can still leave reflist_model empty, and reading that emptiness as "no
call was made" is the cardinal rule's own failure mode.
corroborating_readings now names only readings that carried a cited label,
not every reading `others` happened to hold — a reading proposing entries for
labels the body never cites abstains everywhere in label_agreement and must
not be credited as having corroborated anything.
The pymupdf-backend skip is decided where the backend is known, and reports
itself honestly instead of blaming the PDF for "only one of two extractions
had text" when both did and the second was never taken on purpose. The
wizard's own cost estimate and the --llm-refs help text now say the same
conditional thing.
The load-bearing safety test now captures what actually reaches resolve_all,
closing the half of Ruling 2 the entries-equality assertion could not see on
its own.
Plus: an IndexError on an empty converter, a disclosure level that disagreed
with its own console line, a triplicated sort key, and a duplicated PDF
fixture.
---
schemas/refs_manifest.schema.json | 6 +-
src/papertrace/cli.py | 85 +++--
src/papertrace/disclosures.py | 33 +-
src/papertrace/models.py | 21 +-
src/papertrace/reflist.py | 12 +
.../templates/report_terminal.html.j2 | 2 +-
src/papertrace/wizard.py | 23 +-
tests/test_reference_readings.py | 296 +++++++++++++++---
tests/test_wizard.py | 31 +-
9 files changed, 434 insertions(+), 75 deletions(-)
diff --git a/schemas/refs_manifest.schema.json b/schemas/refs_manifest.schema.json
index 7e72803..891bbd8 100644
--- a/schemas/refs_manifest.schema.json
+++ b/schemas/refs_manifest.schema.json
@@ -278,9 +278,13 @@
],
"description": "Who is responsible for `numbering_choice`: '' when no escalation ran, 'default' when nobody was actually asked, 'user' when a person consented to it. A person consenting to proceed is an input, never evidence that the numbering was verified — `numbering_verified` cannot be set from this."
},
+ "reflist_attempted": {
+ "type": "boolean",
+ "description": "Whether `refs` actually invoked a model to read the bibliography, regardless of whether the reply named itself. False means the call was never made at all — --llm-refs was off, --parse-only forced it off, the backend left no second reading to compare against, claude was not on PATH, or the subprocess failed before any reply was verified. Distinguishes \"no call happened\" from \"a call happened and voted but reported no name\", which `reflist_model` alone cannot: it is empty in both cases."
+ },
"reflist_model": {
"type": "string",
- "description": "The model that produced the LLM's structured reading of the bibliography, when --llm-refs ran and produced anything usable. '' means no such call was made, or nothing it proposed survived verbatim verification against the two texts it was shown."
+ "description": "The model that produced the LLM's structured reading of the bibliography, when --llm-refs ran, produced anything usable, AND the reply itself reported a model name. '' means EITHER no such call was made OR one was made, verified and voted but did not report which model answered — see `reflist_attempted` for which; this field alone cannot tell them apart, and never means the reading was rejected."
},
"reflist_fields_discarded": {
"type": "array",
diff --git a/src/papertrace/cli.py b/src/papertrace/cli.py
index 93c79b1..47a7eac 100644
--- a/src/papertrace/cli.py
+++ b/src/papertrace/cli.py
@@ -509,7 +509,10 @@ def _needs_flat_reading(smap) -> bool:
reading would turn every `single` label into `agreed` and report
corroboration that nothing corroborated.
"""
- return smap.converter.split()[0] != "pymupdf"
+ # `"".split()` is `[]`, not `[""]` — guard rather than let a `source_map.json`
+ # that never recorded a converter (empty string) crash the whole refs stage
+ # on an IndexError instead of just treating it as "not known to be pymupdf"
+ return (smap.converter.split() or [""])[0] != "pymupdf"
def _reference_readings(
@@ -555,6 +558,7 @@ def _llm_reference_reading(
label_a: str,
label_b: str,
enabled: bool,
+ disabled_reason: str = "",
) -> tuple[list[RefEntry], ReflistProvenance]:
"""A model's reading of the bibliography, or a stated reason there is none.
@@ -571,12 +575,20 @@ def _llm_reference_reading(
leave the reason where the manifest and the report can read it, because
silence here would be read as "no news" about a reading that was asked for
and did not happen.
+
+ `disabled_reason` names a STRUCTURAL cause when `not enabled` — the run's
+ own backend leaves no second reading to compare against — as opposed to a
+ bare, silent `ReflistProvenance()` for the two causes that are simply the
+ caller's choice (`--no-llm-refs`, `--parse-only`) and need no explaining.
"""
from . import ask
from .reflist import ReflistProvenance, propose
if not enabled:
- return [], ReflistProvenance()
+ # a caller's own choice is silent (empty `fields_discarded`); a
+ # structural reason the caller had no choice about is disclosed, same
+ # as "claude not on PATH" below
+ return [], ReflistProvenance(fields_discarded=[disabled_reason] if disabled_reason else [])
if not ask.claude_available():
# the ordinary case for someone who installed papertrace and not Claude
# Code. The run proceeds on the deterministic readings and says so —
@@ -595,8 +607,13 @@ def _llm_reference_reading(
# closed on the next one. The failure is recorded, not swallowed:
# `reflist_fields_discarded` carries it into the manifest and a console
# line says it at the time. Same shape as `check.py:996`.
+ # `attempted=True`: every exception reaching here came out of `propose`
+ # actually invoking the subprocess (its own two early returns for "not
+ # enough text" and "not a JSON array" return normally, not raise) — a
+ # failed call is still a call that happened, not one that never was.
return [], ReflistProvenance(
- fields_discarded=[f"not obtained — {type(e).__name__}: {str(e)[:200]}"]
+ attempted=True,
+ fields_discarded=[f"not obtained — {type(e).__name__}: {str(e)[:200]}"],
)
if prov.discarded_whole:
# the entries are already `[]` in this branch; the reason is what travels,
@@ -628,6 +645,11 @@ def _refs_pipeline(
"""
from .ingest import ingest_pdf, references_span, references_span_flat
from .models import SourceMap, citation_labels, is_references_heading, paper_title
+
+ # `_label_key` is shared with `reflist.propose`'s own numbering-findings
+ # sort, rather than a third inline copy of the same tuple key — a
+ # divergence between them would sort the same labels two ways in one report
+ from .reflist import _label_key
from .refs import (
_client,
crossref_deposit,
@@ -759,27 +781,41 @@ def _refs_pipeline(
# flat-text reading can cause a source to be resolved, downloaded or judged.
# They can only cause a verdict to be withheld.
flat_text, flat_entries = "", None
- if _needs_flat_reading(smap):
+ needs_flat = _needs_flat_reading(smap)
+ if needs_flat:
flat_text, _ = references_span_flat(manuscript)
flat_entries = parse_references(flat_text)
+ # --parse-only promises "no network"; a `claude -p` subprocess is both
+ # network and spend, so the flag is forced off here rather than trusted to
+ # the caller. `needs_flat` is decided HERE, where the backend is known —
+ # on a pymupdf run the model would be handed the same text twice, which
+ # `reflist.propose` can only read as "one of the two extractions had no
+ # text" and blame on the PDF rather than on this deliberate skip.
+ llm_wanted = llm_refs and not parse_only
llm_entries, reflist_prov = _llm_reference_reading(
refs_text,
flat_text,
label_a=smap.converter,
label_b="pymupdf",
- # --parse-only promises "no network"; a `claude -p` subprocess is both
- # network and spend, so the flag is forced off here rather than trusted
- # to the caller
- enabled=llm_refs and not parse_only,
+ enabled=llm_wanted and needs_flat,
+ disabled_reason=(
+ "not attempted — this run's backend is pymupdf, so a second flat "
+ "reading of the bibliography would be the same text"
+ if llm_wanted and not needs_flat
+ else ""
+ ),
)
others = _reference_readings(
smap=smap, crossref=crossref_entries, parsed=parsed,
flat=flat_entries, llm=llm_entries,
)
- if reflist_prov.model:
+ if reflist_prov.attempted:
+ # named, even when the reply itself did not — see `ReflistProvenance.attempted`'s
+ # docstring for why `model` alone cannot stand in for "this happened"
+ named_model = reflist_prov.model or "an unnamed model"
dropped = len(reflist_prov.fields_discarded)
console.print(
- f"the reference list was also read by [bold]{reflist_prov.model}[/bold] · "
+ f"the reference list was also read by [bold]{named_model}[/bold] · "
f"{dropped} field{'' if dropped == 1 else 's'} discarded as "
f"not printed [dim]— a second reading, not confirmation[/dim]"
)
@@ -805,20 +841,30 @@ def _refs_pipeline(
agreement = label_agreement(others, body_labels)
rec.labels_disputed = sorted(
(label for label, state in agreement.items() if state == "disputed"),
- key=lambda s: (0, int(s), "") if s.isdigit() else (1, 0, s),
+ key=_label_key,
)
# every cited label, or it is not corroboration. One disputed label is not
# "mostly corroborated", and `single` is not agreement — it is one reading
rec.corroborated = bool(body_labels) and all(
agreement.get(label) == "agreed" for label in body_labels
)
- # named only when it holds: an empty list means no corroboration was
- # established, never that these readings disagreed
- rec.corroborating_readings = sorted(others) if rec.corroborated else []
+ # Named only when corroboration holds, and only the readings that actually
+ # voted on a cited label — never every reading `others` happened to carry.
+ # `sorted(others)` names a reading that carried none of the cited labels
+ # (it proposed only entries for labels the body never cites) as having
+ # corroborated them, which `stamp_seen_in`'s own `_same_work` test would
+ # never do for that same reading. Since `rec.corroborated` already requires
+ # every cited label to be `agreed`, a reading that carries any cited label
+ # at all necessarily agreed on it — filtering to "carries one" is enough.
+ rec.corroborating_readings = (
+ sorted(name for name, cand in others.items() if any(e.num in body_labels for e in cand))
+ if rec.corroborated else []
+ )
if rec.corroborated:
console.print(
- f"[green]✓ {len(others)} readings of the reference list agree[/green] on every "
- f"cited label [dim]({', '.join(sorted(others))}) — corroboration, not a "
+ f"[green]✓ {len(rec.corroborating_readings)} readings of the reference list "
+ f"agree[/green] on every cited label "
+ f"[dim]({', '.join(rec.corroborating_readings)}) — corroboration, not a "
f"confirmed numbering[/dim]"
)
if rec.labels_disputed:
@@ -902,6 +948,7 @@ def tick(e):
numbering_corroborated=rec.corroborated,
corroborating_readings=rec.corroborating_readings,
labels_disputed=rec.labels_disputed,
+ reflist_attempted=reflist_prov.attempted,
reflist_model=reflist_prov.model,
reflist_fields_discarded=reflist_prov.fields_discarded,
reflist_numbering_findings=reflist_prov.numbering_findings,
@@ -946,7 +993,8 @@ def refs(
llm_refs: bool = typer.Option(
True, "--llm-refs/--no-llm-refs",
help="Also have a model read the printed reference list as a second opinion "
- "on the numbering (one extra model call; off under --parse-only)",
+ "on the numbering (one extra model call — none under --parse-only, or on a "
+ "pymupdf backend, which leaves no second reading to compare)",
),
) -> None:
"""Parse the References section, then retrieve open-access copies with an honest manifest."""
@@ -1382,7 +1430,8 @@ def run(
llm_refs: bool = typer.Option(
True, "--llm-refs/--no-llm-refs",
help="Also have a model read the printed reference list as a second opinion "
- "on the numbering (one extra model call; off under --parse-only)",
+ "on the numbering (one extra model call — none under --parse-only, or on a "
+ "pymupdf backend, which leaves no second reading to compare)",
),
) -> None:
"""Full pipeline: ingest → refs → scout → check → highlight → report."""
diff --git a/src/papertrace/disclosures.py b/src/papertrace/disclosures.py
index 3c28863..3d86ff7 100644
--- a/src/papertrace/disclosures.py
+++ b/src/papertrace/disclosures.py
@@ -555,34 +555,48 @@ def _reflist(manifest) -> Disclosure | None:
no news. The token is phrased AROUND in every branch, the way
`SUPPLEMENT_IDENTITY_TOKEN` is: it is asserted verbatim in all four formats,
so a branch where it is not literally true would make one format lie.
+
+ Branches on `reflist_attempted`, never on whether `reflist_model` is set.
+ `claude -p` only reports a model name when its own JSON does — a reply
+ naming none is not evidence nothing was asked — so a call that happened,
+ verified cleanly and voted can still leave `reflist_model == ""`. Reading
+ that emptiness as "no reading was obtained" is precisely the state a real
+ run reached before `reflist_attempted` existed: one real call, nothing on
+ the console, the manifest asserting no such call was made.
"""
model = getattr(manifest, "reflist_model", "") or ""
+ attempted = bool(getattr(manifest, "reflist_attempted", False))
notes = list(getattr(manifest, "reflist_fields_discarded", []) or [])
# a different kind of finding, and it must reach the reader too: a reading
# whose own labels do not add up is worth knowing about even when every
# value it proposed was printed
numbering = list(getattr(manifest, "reflist_numbering_findings", []) or [])
- if not model and not notes:
+ if not attempted and not notes:
return None # no model reading was asked for
ceiling = (
"A model agreeing with a parse is a second reading, not confirmation: it read "
"the same document, so a reference the layout destroyed is one it may also "
"have missed."
)
- if not model:
+ # named even when the reply itself did not — see this function's own
+ # docstring for why `model` alone cannot stand in for "this happened"
+ named = model or "a model that did not report its own name"
+ if not attempted:
head = (
f"This run asked that {REFLIST_TOKEN}, and no reading was obtained: "
f"{notes[0]}. The reference numbering therefore rests on the readings above "
f"it and nothing else."
)
short = f"{REFLIST_TOKEN}: asked for, not obtained — {notes[0]}"
+ level = "warn" # a failed attempt, not routine information
elif any(n.startswith("reading discarded") for n in notes):
why = next(n for n in notes if n.startswith("reading discarded"))
head = (
- f"Here {REFLIST_TOKEN} ({model}), and its reading was discarded rather than "
+ f"Here {REFLIST_TOKEN} ({named}), and its reading was discarded rather than "
f"used: {why}. Nothing it proposed contributed to the list below."
)
- short = f"{REFLIST_TOKEN} ({model}) — discarded as unusable"
+ short = f"{REFLIST_TOKEN} ({named}) — discarded as unusable"
+ level = "info"
else:
dropped = (
f"{len(notes)} value{'' if len(notes) == 1 else 's'} it proposed "
@@ -591,10 +605,15 @@ def _reflist(manifest) -> Disclosure | None:
if notes else "every value it proposed was found in the printed text"
)
head = (
- f"Here {REFLIST_TOKEN} ({model}), shown two extractions of the same printed "
+ f"Here {REFLIST_TOKEN} ({named}), shown two extractions of the same printed "
f"bibliography and asked what numbered list it carries; {dropped}."
)
- short = f"{REFLIST_TOKEN} ({model}) — {len(notes)} discarded"
+ if not model:
+ # a reading was taken and voted — the gap is what the CLI could
+ # report about it, not whether anything happened at all
+ head += " The CLI did not report which model answered."
+ short = f"{REFLIST_TOKEN} ({named}) — {len(notes)} discarded"
+ level = "info"
if numbering:
# appended rather than folded into `dropped`: "3 values discarded" and
# "it numbered one entry twice" are different facts, and a reader who
@@ -606,7 +625,7 @@ def _reflist(manifest) -> Disclosure | None:
)
return Disclosure(
key="reflist",
- level="info",
+ level=level,
token=REFLIST_TOKEN,
text=f"{head} {ceiling}",
short=short,
diff --git a/src/papertrace/models.py b/src/papertrace/models.py
index 8542daf..f572a16 100644
--- a/src/papertrace/models.py
+++ b/src/papertrace/models.py
@@ -594,11 +594,22 @@ class RefManifest:
# never ran. Never set from a model's own say-so alone.
numbering_choice: str = "" # "" | withheld | llm_resolved | parsed | pymupdf
numbering_chosen_by: str = "" # "" | default | user
+ # Whether `refs` actually invoked a model to read the bibliography — set
+ # regardless of whether the reply named itself. False means the call was
+ # never made at all: --llm-refs was off, --parse-only forced it off, the
+ # backend left no second reading to compare against, claude was not on
+ # PATH, or the subprocess itself failed before any reply was verified.
+ # This is the field that distinguishes "no call happened" from "a call
+ # happened and voted, but the CLI did not learn its name" — `reflist_model`
+ # alone cannot, because `claude -p` only reports a model name when its own
+ # JSON does, and a reply naming none is not evidence nothing was asked.
+ reflist_attempted: bool = False
# The model that produced the LLM's structured reading of the
- # bibliography, when --llm-refs ran and produced anything usable. "" means
- # no such call was made, or nothing it proposed survived verbatim
- # verification against the two texts it was shown — same sentence as the
- # schema's, on purpose: two accounts of what "" means is one too many.
+ # bibliography, when --llm-refs ran, produced anything usable, AND the
+ # subprocess's own reply reported a name. "" means EITHER no such call was
+ # made OR one was made and answered but did not report which model —
+ # `reflist_attempted` is what tells those two apart; this field alone
+ # cannot.
reflist_model: str = ""
# Which fields of the LLM's proposed reading could not be found verbatim
# in either text it was shown, and were discarded rather than trusted.
@@ -693,6 +704,7 @@ def to_json(self, path: Path) -> None:
"labels_resolved": self.labels_resolved,
"numbering_choice": self.numbering_choice,
"numbering_chosen_by": self.numbering_chosen_by,
+ "reflist_attempted": self.reflist_attempted,
"reflist_model": self.reflist_model,
"reflist_fields_discarded": self.reflist_fields_discarded,
"reflist_numbering_findings": self.reflist_numbering_findings,
@@ -735,6 +747,7 @@ def from_json(cls, path: Path) -> RefManifest:
labels_resolved=data.get("labels_resolved", []),
numbering_choice=data.get("numbering_choice", ""),
numbering_chosen_by=data.get("numbering_chosen_by", ""),
+ reflist_attempted=bool(data.get("reflist_attempted", False)),
reflist_model=data.get("reflist_model", ""),
reflist_fields_discarded=data.get("reflist_fields_discarded", []),
reflist_numbering_findings=data.get("reflist_numbering_findings", []),
diff --git a/src/papertrace/reflist.py b/src/papertrace/reflist.py
index a178abf..251fc6b 100644
--- a/src/papertrace/reflist.py
+++ b/src/papertrace/reflist.py
@@ -148,6 +148,15 @@ class ReflistProvenance:
# "the model agreed": a reading that could not be checked is not a reading.
discarded_whole: str = ""
readings: list[str] = field(default_factory=list)
+ # Whether `claude -p` was actually invoked — set the instant `propose`
+ # commits to the call, never derived from `model`. `ask._ask` records a
+ # model name only when the subprocess's own JSON reports one ("a reply
+ # that names no model is not evidence the model changed", `ask.py`), so a
+ # call that succeeded, verified cleanly and voted can still leave `model`
+ # `""`. A caller that read `model` as the proxy for "did this happen at
+ # all" would publish that vote as "no such call was made" — the exact
+ # state a real run reached before this field existed.
+ attempted: bool = False
# order matters only for `raw` below, which is re-assembled in printed order
@@ -187,6 +196,9 @@ def propose(
"nothing the model proposed could have been checked against a second reading"
)
return [], prov
+ # committed to the call from here on — everything past this point is a real
+ # attempt, whatever it returns
+ prov.attempted = True
# labels before texts: a label is a short fixed string ("pymupdf",
# "docling 2.8.0") and cannot contain a placeholder, while page text
diff --git a/src/papertrace/templates/report_terminal.html.j2 b/src/papertrace/templates/report_terminal.html.j2
index 7394643..5e02df3 100644
--- a/src/papertrace/templates/report_terminal.html.j2
+++ b/src/papertrace/templates/report_terminal.html.j2
@@ -110,7 +110,7 @@
▸ resolve{{ d.short }}
{% endfor %}
{% for d in disclosures if d.key == "reflist" %}
- ▸ resolve{{ d.short }}
+ ▸ resolve{{ '⚠ ' if d.level == 'warn' else '' }}{{ d.short }}
{% endfor %}
▸ check{{ r.claims|length }} citation-backed claims · reading each against its cited page{% if r.uncited %} · {{ r.uncited|length }} uncited assertions flagged{% endif %}
{% if scout and not scout.error %}
diff --git a/src/papertrace/wizard.py b/src/papertrace/wizard.py
index 8af83d2..b7e2910 100644
--- a/src/papertrace/wizard.py
+++ b/src/papertrace/wizard.py
@@ -212,21 +212,30 @@ def workload(pdf: Path) -> dict:
# one extraction call, then one per cited source. Sources are judged in
# groups, so this is an upper bound on the judging calls, not a promise.
cited_source_calls = sum(len(g) for g in groups)
+ # `run_wizard` always requests `backend="auto"`, which resolves to docling
+ # when it is importable and to pymupdf otherwise (`ingest.resolve_backend`).
+ # On a pymupdf run the flat second reading is skipped as identical to the
+ # first — `cli._needs_flat_reading`'s own predicate, mirrored here rather
+ # than imported, since importing `.ingest`'s `resolve_backend` would do —
+ # so the reference-list call is never attempted either, and the estimate
+ # must not promise a call this run's own backend will not produce.
+ llm_call = 1 if docling_available() else 0
return {
"pages": pages,
"places": len(groups),
"multi": multi,
"labels": len(labels),
- # one reference-list reading, one extraction call, then one per cited
- # source. The reference-list call gets no retry — `reflist.propose` asks
- # once and reports what it got — so it adds exactly 1 to the base figure
- # and 1 to the ceiling, on top of the retried figure below rather than
- # inside it.
- "model_calls": 2 + cited_source_calls,
+ # one reference-list reading (when the backend leaves a second text to
+ # check it against), one extraction call, then one per cited source.
+ # The reference-list call gets no retry — `reflist.propose` asks once
+ # and reports what it got — so it adds exactly `llm_call` to the base
+ # figure and to the ceiling, on top of the retried figure below rather
+ # than inside it.
+ "model_calls": 1 + llm_call + cited_source_calls,
# the retry is real spend: ASK_ATTEMPTS attempts for extraction, and
# ASK_ATTEMPTS attempts per judging call. Derived from ask.py rather
# than a local multiplier, so the estimate cannot drift from the policy.
- "model_calls_max": 1 + ASK_ATTEMPTS * (1 + cited_source_calls),
+ "model_calls_max": llm_call + ASK_ATTEMPTS * (1 + cited_source_calls),
"style_unrecognised": len(groups) == 0,
}
diff --git a/tests/test_reference_readings.py b/tests/test_reference_readings.py
index 0e7800f..22216f2 100644
--- a/tests/test_reference_readings.py
+++ b/tests/test_reference_readings.py
@@ -24,8 +24,17 @@
SCHEMA_PATH = Path(__file__).resolve().parent.parent / "schemas" / "refs_manifest.schema.json"
-def _paper(path: Path) -> Path:
- """A one-page PDF citing [1] and [2], with both references printed."""
+def _paper(path: Path, *, extra_uncited: bool = False) -> Path:
+ """A one-page PDF citing [1] and [2], with both references printed.
+
+ `extra_uncited=True` also prints a THIRD, uncited [3] — used by exactly one
+ test (`test_no_model_reply_can_set_numbering_verified`) where the parse
+ alone must fail `reconcile`'s own extent check (`_covers`), because on the
+ plain two-reference fixture that check is satisfied by the parse alone and
+ nothing about the model reading could be shown to matter (see that test's
+ docstring). One fixture with a flag, not two independent copies that would
+ otherwise be free to drift apart.
+ """
doc = pymupdf.open()
page = doc.new_page()
page.insert_text((72, 100), "A Fixture Imaging Study", fontsize=16)
@@ -33,6 +42,9 @@ def _paper(path: Path) -> Path:
page.insert_text((72, 200), "References", fontsize=14)
page.insert_text((72, 230), "[1] Alpha A. A first paper. J Fixture. 2020;1:1-9.", fontsize=11)
page.insert_text((72, 250), "[2] Beta B. A second paper. J Fixture. 2021;2:10-19.", fontsize=11)
+ if extra_uncited:
+ page.insert_text((72, 270), "[3] Gamma G. A third, uncited paper. J Fixture. 2019;3:1-5.",
+ fontsize=11)
path.parent.mkdir(parents=True, exist_ok=True)
doc.save(path)
doc.close()
@@ -46,7 +58,7 @@ def _work(num: str, doi: str) -> RefEntry:
return _entry(num, f"Author for entry {num}. A distinctive title. doi:{doi}")
-def _wire_offline(monkeypatch) -> None:
+def _wire_offline(monkeypatch, *, needs_flat: bool = True) -> None:
"""No network and no model call by default: retrieval is a no-op, Crossref
answers nothing, and `claude` is treated as PRESENT regardless of the host.
@@ -60,6 +72,21 @@ def _wire_offline(monkeypatch) -> None:
take the "not attempted" branch instead and never call the mock at all.
The one test that wants the real "claude absent" branch overrides this
itself, after this fixture runs.
+
+ `needs_flat` patches `cli._needs_flat_reading` rather than the smap's own
+ converter, because these tests all ingest with `backend="pymupdf"` for
+ speed (no docling download) and `_refs_pipeline` now decides `enabled` for
+ the model call as `llm_refs and not parse_only and _needs_flat_reading(smap)`
+ (Major 2 of the Task 6 review) — so on the real pymupdf backend the model
+ call would never even be attempted, and every test below that mocks
+ `reflist.propose` directly would be testing a branch that never runs. This
+ is the same technique the review used to reproduce the real,
+ docling-shaped path without docling: force the PREDICATE, not the backend.
+ Default `True` because that is what most of these tests need; the one test
+ for the real pymupdf skip (`test_a_pymupdf_backend_run_never_asks_the_model`)
+ passes `needs_flat=False` explicitly, which happens to equal what the real
+ function already returns for this fixture's backend — set explicitly for
+ the reader's benefit, not because the patch changes anything there.
"""
import papertrace.ask as ask_mod
import papertrace.refs as refs_mod
@@ -72,6 +99,7 @@ def _wire_offline(monkeypatch) -> None:
lambda client, doi, email: CrossrefDeposit(absent="no deposit, in a test"))
monkeypatch.setattr(cli, "_detected_doi", lambda m: None)
monkeypatch.setattr(ask_mod, "claude_available", lambda: True)
+ monkeypatch.setattr(cli, "_needs_flat_reading", lambda smap: needs_flat)
monkeypatch.setenv("PAPERTRACE_EMAIL", "test@example.org")
@@ -80,7 +108,7 @@ def offline(monkeypatch):
_wire_offline(monkeypatch)
-def _run_refs(tmp_path: Path, monkeypatch, **kw) -> RefManifest:
+def _run_refs(tmp_path: Path, monkeypatch, *, needs_flat: bool = True, **kw) -> RefManifest:
"""Wire the offline doubles, run `_refs_pipeline` on the fixture paper, and
hand back the manifest it wrote — schema-validated on the way, so every
caller of this helper gets Gate 2 for free rather than each writing its own
@@ -92,7 +120,7 @@ def _run_refs(tmp_path: Path, monkeypatch, **kw) -> RefManifest:
"""
import jsonschema
- _wire_offline(monkeypatch)
+ _wire_offline(monkeypatch, needs_flat=needs_flat)
pdf = _paper(tmp_path / "paper.pdf")
case = tmp_path / "case"
args = dict(manuscript=pdf, case=case, provided=None, email="test@example.org",
@@ -120,13 +148,28 @@ def _smap(converter: str) -> SourceMap:
def test_a_model_naming_different_papers_changes_no_resolved_entry(tmp_path, offline, monkeypatch):
"""Ruling 2, as an equality. The safety property the whole design rests on.
- The model's reading is a voter. It can withhold a verdict; it cannot put a
- paper into the manifest, cannot change a numeral, and cannot cause anything
- to be resolved or downloaded. So a run whose model reading names two
- completely different works must produce byte-identical entries to a run that
- never asked a model at all. If this test ever fails, `others["llm"]` has
- reached `reconcile`'s arguments or `resolve_all`, and no other test in this
- suite would notice.
+ The model's reading is a voter: it can withhold a verdict, and — via
+ `stamp_seen_in` — record which readings also carried a chosen entry's
+ work. It cannot put a paper into the manifest, cannot change a numeral,
+ and cannot cause anything to be resolved or downloaded. So a run whose
+ model reading names two completely different works must produce
+ byte-identical entries to a run that never asked a model at all.
+
+ That equality holds only because the fabricated reading DISAGREES (Minor 4
+ of the Task 6 review). `stamp_seen_in` is precisely a model-driven write
+ into the manifest's entries (`RefEntry.seen_in`): an AGREEING model
+ reading legitimately changes `seen_in` and the two runs would NOT be
+ byte-identical — see `test_no_model_reply_can_set_numbering_verified` for
+ that case. Do not "fix" `stamp_seen_in` to restore byte-identical equality
+ under agreement; that was never the invariant, only the disagreeing case
+ is.
+
+ The equality alone also cannot see the other half of Ruling 2 (Major 3):
+ `resolve_all`'s return value is discarded at the call site (it mutates in
+ place), so a violation shaped `resolve_all(entries + others["llm"], ...)`
+ would download and judge the fabricated papers while `entries` — and so
+ this equality — stayed untouched. The capturing stub below closes that
+ gap by recording exactly what reached `resolve_all`.
`backend="pymupdf"`, not `"docling"`: the brief's own draft of this test used
`"docling"` on a real PDF with no mock in front of `ingest_pdf`, which would
@@ -135,17 +178,30 @@ def test_a_model_naming_different_papers_changes_no_resolved_entry(tmp_path, off
is fully mocked below and ignores its arguments, so which backend produced
the parse is irrelevant to what this test actually checks.
"""
+ import papertrace.refs as refs_mod
+
pdf = _paper(tmp_path / "paper.pdf")
+ fabricated_titles = {"An entirely different paper", "Another different paper"}
fabricated = (
[RefEntry(num="1", raw="Zeta Z. An entirely different paper. 1999.",
title="An entirely different paper", year="1999", seen_in=["docling"]),
RefEntry(num="2", raw="Eta E. Another different paper. 1998.",
title="Another different paper", year="1998", seen_in=["pymupdf"])],
reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2,
- readings=["docling", "pymupdf"]),
+ readings=["docling", "pymupdf"], attempted=True),
)
monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: fabricated)
+ # what actually reaches retrieval — the half of Ruling 2 the
+ # entries-equality assertion below cannot see on its own (Major 3)
+ captured: list[list[RefEntry]] = []
+
+ def _capturing_resolve_all(entries, dest, email, provided_dir=None, progress=None, taken=None):
+ captured.append(list(entries))
+ return entries
+
+ monkeypatch.setattr(refs_mod, "resolve_all", _capturing_resolve_all)
+
with_llm, without = tmp_path / "with", tmp_path / "without"
cli._refs_pipeline(manuscript=pdf, case=with_llm, provided=None,
email="test@example.org", parse_only=False, backend="pymupdf",
@@ -157,6 +213,13 @@ def test_a_model_naming_different_papers_changes_no_resolved_entry(tmp_path, off
assert _entries(with_llm) == _entries(without)
assert "different paper" not in _entries(with_llm)
+ assert len(captured) == 2, "resolve_all should have run once per _refs_pipeline call"
+ for handed in captured:
+ handed_titles = {e.title for e in handed if e.title}
+ assert not (handed_titles & fabricated_titles), (
+ f"a fabricated entry reached resolve_all: {handed}"
+ )
+
def test_the_pymupdf_reading_is_skipped_on_a_pymupdf_backend_run():
"""A reading agreeing with itself is not corroboration.
@@ -176,6 +239,15 @@ def test_the_pymupdf_reading_is_skipped_on_a_pymupdf_backend_run():
assert set(layout_run) == {"parsed", "pymupdf"}, layout_run
+def test_needs_flat_reading_does_not_crash_on_an_empty_converter():
+ """Minor 1 of the Task 6 review. `"".split()` is `[]`, not `[""]` —
+ `smap.converter.split()[0]` raises `IndexError` on a `source_map.json`
+ that never recorded a converter, killing the whole refs stage on
+ something the rest of this module treats as a routine "not known to be
+ pymupdf" case rather than a crash."""
+ assert cli._needs_flat_reading(_smap("")) is True
+
+
def test_a_reading_that_is_absent_is_omitted_rather_than_empty():
"""An empty list is a reading that found nothing; an absent key is no reading.
@@ -225,16 +297,55 @@ def test_parse_only_makes_no_model_call_even_with_llm_refs_on(tmp_path, offline,
assert calls == []
-def test_an_unavailable_claude_degrades_to_a_stated_absence(tmp_path, offline, monkeypatch):
+def test_a_pymupdf_backend_run_never_asks_the_model(tmp_path, monkeypatch):
+ """Major 2 of the Task 6 review: `--backend pymupdf` — what every other
+ test and all of CI pin — must never even reach `reflist.propose`, and the
+ published reason must name the deliberate skip rather than blame the PDF.
+
+ `reflist.propose` is deliberately left UNMOCKED here: the point is that it
+ is never called at all, not that it is called and handled well. Before the
+ fix, `enabled` did not consider the backend, so `propose` ran, was handed
+ one empty text (the flat reading, skipped on purpose) and one real one,
+ and reported "only one of the two extractions of the bibliography had any
+ text" — true of nothing: both extractions had text, the second was simply
+ never taken.
+ """
+ import papertrace.ask as ask_mod
+
+ calls = []
+ monkeypatch.setattr(ask_mod, "_ask", lambda prompt, model=None: calls.append(1) or "[]")
+ manifest = _run_refs(tmp_path, monkeypatch, needs_flat=False, llm_refs=True)
+
+ assert calls == [], "propose must never be reached on a pymupdf-backend run"
+ assert manifest.reflist_attempted is False
+ assert manifest.reflist_model == ""
+ assert any(
+ "this run's backend is pymupdf" in f for f in manifest.reflist_fields_discarded
+ ), manifest.reflist_fields_discarded
+ assert not any(
+ "only one of the two extractions" in f for f in manifest.reflist_fields_discarded
+ ), "the published reason must name the deliberate skip, not blame the PDF's extraction"
+
+
+def test_an_unavailable_claude_degrades_to_a_stated_absence(tmp_path, monkeypatch):
"""The cardinal rule, at this seam. Never "it agreed" — "it was not asked".
`claude` missing from PATH is the ordinary case for someone who installed
papertrace and not Claude Code. The run proceeds on the deterministic
readings, the manifest records the reason, and no format anywhere says a
model corroborated anything.
+
+ `needs_flat=False` (not the shared fixture's default): with the flat
+ reading also forced on, "parsed" and the real flat-text reading of this
+ same pymupdf-backend PDF are identical and genuinely corroborate each
+ other with no model involved at all, which would make
+ `numbering_corroborated` legitimately `True` for a reason this test is not
+ about. Keeping the fixture single-voter here is what isolates "claude was
+ unavailable" as the one fact under test.
"""
import papertrace.ask as ask_mod
+ _wire_offline(monkeypatch, needs_flat=False)
monkeypatch.setattr(ask_mod, "claude_available", lambda: False)
monkeypatch.setattr(ask_mod, "_ask",
lambda prompt, model=None: pytest.fail("claude was absent and asked anyway"))
@@ -246,6 +357,7 @@ def test_an_unavailable_claude_degrades_to_a_stated_absence(tmp_path, offline, m
payload = json.loads((case / "refs_manifest.json").read_text())
assert payload["reflist_model"] == ""
+ assert payload["reflist_attempted"] is False
assert any("not attempted" in f for f in payload["reflist_fields_discarded"]), payload
assert "llm" not in payload["corroborating_readings"]
assert payload["numbering_corroborated"] is False
@@ -288,7 +400,7 @@ def test_corroboration_requires_every_cited_label_to_agree(tmp_path, offline, mo
title="A first paper", year="2020", seen_in=["docling", "pymupdf"]),
RefEntry(num="2", raw="Gamma G. A wholly unrelated third paper. 2015.",
title="A wholly unrelated third paper", year="2015", seen_in=["A"])],
- reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2),
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2, attempted=True),
)
monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: disagrees)
case = tmp_path / "case"
@@ -309,36 +421,32 @@ def test_no_model_reply_can_set_numbering_verified(tmp_path, offline, monkeypatc
would be most tempting — a model agreeing with the parse entry for entry.
Corroboration is a second axis, not evidence of a checked numbering.
- Not `_paper()`: that fixture cites exactly the two references it prints, so
+ Not `_paper()`'s default shape: with only the two cited references printed,
`_covers` — the extent check `reconcile` uses on its own, with no model in
the loop at all — is already satisfied by the parse alone, and `verified`
would come back `True` regardless of anything this test does. Nothing here
could then tell "verified because the parse already covered the body's
- labels" apart from "verified because a mocked model agreed with it". This
- fixture prints a THIRD, uncited reference, so the parse alone has three
- entries against two cited labels and fails `_covers` on its own — the only
- way to make the two cases distinguishable.
+ labels" apart from "verified because a mocked model agreed with it".
+ `extra_uncited=True` prints a THIRD, uncited reference, so the parse alone
+ has three entries against two cited labels and fails `_covers` on its own —
+ the only way to make the two cases distinguishable.
+
+ `corroborating_readings` names three, not two: `offline`'s default
+ `needs_flat=True` means the real, local flat-text reading of this same PDF
+ also runs and also carries labels [1] and [2] (identical text to the
+ parse, since the real backend here is pymupdf too) — so it votes exactly
+ where "parsed" does, and Major 1's fix (name only readings that actually
+ carried a cited label) correctly counts it alongside "llm" and "parsed".
"""
- doc = pymupdf.open()
- page = doc.new_page()
- page.insert_text((72, 100), "A Fixture Imaging Study", fontsize=16)
- page.insert_text((72, 140), "Body text citing [1] and also [2] here.", fontsize=11)
- page.insert_text((72, 200), "References", fontsize=14)
- page.insert_text((72, 230), "[1] Alpha A. A first paper. J Fixture. 2020;1:1-9.", fontsize=11)
- page.insert_text((72, 250), "[2] Beta B. A second paper. J Fixture. 2021;2:10-19.", fontsize=11)
- page.insert_text((72, 270), "[3] Gamma G. A third, uncited paper. J Fixture. 2019;3:1-5.",
- fontsize=11)
- pdf = tmp_path / "paper.pdf"
- doc.save(pdf)
- doc.close()
agrees = (
[RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
title="A first paper", year="2020", seen_in=["docling", "pymupdf"]),
RefEntry(num="2", raw="Beta B. A second paper. J Fixture. 2021;2:10-19.",
title="A second paper", year="2021", seen_in=["docling", "pymupdf"])],
- reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2),
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2, attempted=True),
)
monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: agrees)
+ pdf = _paper(tmp_path / "paper.pdf", extra_uncited=True)
case = tmp_path / "case"
cli._refs_pipeline(manuscript=pdf, case=case, provided=None, email="test@example.org",
@@ -347,7 +455,76 @@ def test_no_model_reply_can_set_numbering_verified(tmp_path, offline, monkeypatc
payload = json.loads((case / "refs_manifest.json").read_text())
assert payload["numbering_verified"] is False
assert payload["numbering_corroborated"] is True
- assert payload["corroborating_readings"] == ["llm", "parsed"]
+ assert payload["corroborating_readings"] == ["llm", "parsed", "pymupdf"]
+
+
+def test_a_successful_reading_that_named_no_model_still_discloses_the_attempt(tmp_path, monkeypatch):
+ """Critical 1 of the Task 6 review, reproduced end to end through the real
+ pipeline rather than only at the disclosure layer.
+
+ `ask._ask` records a model name only when `claude -p`'s own JSON reports
+ one — a case `ask.py` explicitly anticipates ("a reply that names no
+ model is not evidence the model changed"). `reflist.propose` mirrors that
+ exactly: a real call that verifies cleanly and votes can still leave
+ `prov.model == ""`. Before `reflist_attempted` existed, that state
+ published as `reflist_model: ""` (the schema's OWN words for "no such
+ call was made") with no console line and no `reflist` disclosure, while
+ `numbering_corroboration` told the reader `llm` had corroborated the
+ numbering anyway.
+ """
+ unnamed = (
+ [RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
+ title="A first paper", year="2020"),
+ RefEntry(num="2", raw="Beta B. A second paper. J Fixture. 2021;2:10-19.",
+ title="A second paper", year="2021")],
+ # the exact shape a real `propose()` call produces when
+ # `ask.model_for(ask.SITE_REFS)` returns `None` — `model=""`,
+ # `attempted=True`, a real vote in hand
+ reflist_mod.ReflistProvenance(model="", entries_proposed=2, attempted=True),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: unnamed)
+
+ manifest = _run_refs(tmp_path, monkeypatch, llm_refs=True)
+
+ assert manifest.reflist_attempted is True, "a call that voted must be recorded as attempted"
+ assert manifest.reflist_model == ""
+
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ fired = {d.key for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)}
+ assert "reflist" in fired, "a reading that ran and voted must not be silently 'not asked for'"
+
+
+def test_corroborating_readings_excludes_a_reading_that_carried_no_cited_label(
+ tmp_path, monkeypatch, capsys
+):
+ """Major 1 of the Task 6 review. `others` is every reading TAKEN, not
+ every reading that AGREED. A reading proposing entries only for labels the
+ body never cites abstains everywhere in `label_agreement` and must not be
+ named as having corroborated anything — `stamp_seen_in`'s own `_same_work`
+ test already refuses to credit a reading this way for `seen_in`; this is
+ the same fact applied to the field a reader is most likely to trust as
+ reassurance. The console count must match the named list, not `len(others)`.
+ """
+ abstains = (
+ [_work("3", "10.1000/x3"), _work("4", "10.1000/x4")],
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2, attempted=True),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: abstains)
+ capsys.readouterr() # discard anything buffered before this run
+
+ manifest = _run_refs(tmp_path, monkeypatch, llm_refs=True)
+
+ assert "llm" not in manifest.corroborating_readings, manifest.corroborating_readings
+ assert manifest.corroborating_readings == ["parsed", "pymupdf"]
+ assert manifest.numbering_corroborated is True
+
+ out = capsys.readouterr().out
+ assert "3 readings of the reference list agree" not in out, (
+ "the printed count must match the named readings, not len(others)"
+ )
+ assert "2 readings of the reference list agree" in out
def test_both_commands_declare_the_flag_and_run_forwards_it(tmp_path, monkeypatch):
@@ -396,7 +573,8 @@ def test_the_reflist_disclosure_reaches_every_format_and_carries_the_ceiling(tmp
from papertrace.disclosures import REFLIST_TOKEN, run_disclosures
from papertrace.models import RunResults
- manifest = RefManifest(manuscript="p.pdf", reflist_model="claude-opus-5",
+ manifest = RefManifest(manuscript="p.pdf", reflist_attempted=True,
+ reflist_model="claude-opus-5",
reflist_fields_discarded=["[2] journal"])
fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
if d.key == "reflist"]
@@ -424,6 +602,26 @@ def test_a_model_reading_that_was_not_obtained_still_discloses_that(tmp_path):
assert len(fired) == 1, fired
assert "not attempted" in fired[0].text
assert "agreed" not in fired[0].text
+ assert fired[0].level == "warn", "a failed attempt is a warning, not routine information"
+
+
+def test_an_attempted_reading_that_named_no_model_is_worded_as_a_gap_in_reporting(tmp_path):
+ """Critical 1, at the disclosure layer directly. The two failure modes read
+ differently on purpose: "not obtained" means nothing happened; this one
+ means something happened and the CLI could not say who answered — and the
+ text must not blur the two into one "not obtained" sentence.
+ """
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_attempted=True, reflist_model="",
+ reflist_fields_discarded=[])
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ text = fired[0].text
+ assert "not obtained" not in text
+ assert "did not report which model answered" in text
def test_corroboration_is_disclosed_without_claiming_the_numbering_was_verified(tmp_path):
@@ -513,6 +711,33 @@ def test_reflist_numbering_findings_round_trips_and_validates(tmp_path):
assert old.reflist_numbering_findings == []
+def test_reflist_attempted_round_trips_and_validates(tmp_path):
+ """Gate 2, for Critical 1's field. Absent means never computed / no call
+ ever made — the same three-state discipline as `numbering_ledger` — never
+ "the model agreed" and never folded into `reflist_model`'s own emptiness,
+ which cannot by itself distinguish "no call" from "a call that named no
+ model" (see `RefManifest.reflist_attempted`'s own docstring).
+ """
+ import jsonschema
+
+ m = RefManifest(manuscript="p.pdf", reflist_attempted=True, reflist_model="")
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ payload = json.loads(path.read_text())
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.validate(payload, schema)
+ assert "reflist_attempted" not in schema.get("required", [])
+
+ back = RefManifest.from_json(path)
+ assert back.reflist_attempted is True
+ assert back.reflist_model == ""
+
+ del payload["reflist_attempted"]
+ path.write_text(json.dumps(payload))
+ old = RefManifest.from_json(path)
+ assert old.reflist_attempted is False
+
+
def test_a_numbering_finding_from_the_model_reading_reaches_the_manifest(tmp_path, monkeypatch):
"""`reflist.propose` reports a duplicated or missing numeral separately from
a discarded field, because the two mean different things. An earlier draft
@@ -525,6 +750,7 @@ def test_a_numbering_finding_from_the_model_reading_reaches_the_manifest(tmp_pat
entries_proposed=2,
numbering_findings=["numerals proposed twice: 2"],
readings=["pymupdf", "docling"],
+ attempted=True,
),
)
monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: proposed)
diff --git a/tests/test_wizard.py b/tests/test_wizard.py
index 5e39a10..6c97f20 100644
--- a/tests/test_wizard.py
+++ b/tests/test_wizard.py
@@ -662,21 +662,28 @@ def test_a_path_with_a_quote_is_still_one_argument(tmp_path):
# --- the cost estimate must not be exceeded by the documented retry ---------
-def test_the_worst_case_call_count_includes_the_retry(tmp_path):
+def test_the_worst_case_call_count_includes_the_retry(tmp_path, monkeypatch):
"""The wizard promised "up to N model calls" from one extraction plus one
per cited source. `_ask` retries once, so a single-source run advertised as
2 could issue 3. The ceiling now comes from ask.py's own attempt count,
so the two cannot drift.
Both figures also carry the reference-list reading `refs` now makes before
- `check` ever runs: one un-retried call, so it adds exactly 1 to each side
+ `check` ever runs, WHEN the wizard's own backend choice ("auto") would
+ actually attempt it: one un-retried call, so it adds exactly 1 to each side
of the estimate rather than multiplying through the retried figure — this
task adds that call on top of the `ASK_ATTEMPTS * (1 + cited_source_calls)`
shape a previous task already moved `model_calls_max` to, rather than
reverting it back to a flat `ASK_ATTEMPTS * cited_source_calls`.
+
+ Pinned with `docling_available` forced `True` rather than left to whatever
+ this machine happens to have installed — `workload()` derives the +1 from
+ it, and a test whose expected numbers depend on the host environment is
+ not really pinning anything.
"""
from papertrace.ask import ASK_ATTEMPTS
+ monkeypatch.setattr(wizard, "docling_available", lambda: True)
pdf = _one_pager(tmp_path / "m.pdf",
"Title\nOne sentence citing [1].\nReferences\n[1] A. 2020.")
w = wizard.workload(pdf)
@@ -686,6 +693,26 @@ def test_the_worst_case_call_count_includes_the_retry(tmp_path):
assert w["model_calls_max"] >= w["model_calls"]
+def test_the_worst_case_call_count_drops_the_reflist_call_on_a_pymupdf_only_install(
+ tmp_path, monkeypatch
+):
+ """`run_wizard` always requests `backend="auto"`, which resolves to pymupdf
+ when docling is not importable — and on a pymupdf run the flat second
+ reading is skipped as identical to the first, so the reference-list call
+ is never attempted either (Major 2 / Minor 3 of the Task 6 review: the
+ base estimate must not advertise a call this run's own backend cannot
+ produce)."""
+ from papertrace.ask import ASK_ATTEMPTS
+
+ monkeypatch.setattr(wizard, "docling_available", lambda: False)
+ pdf = _one_pager(tmp_path / "m.pdf",
+ "Title\nOne sentence citing [1].\nReferences\n[1] A. 2020.")
+ w = wizard.workload(pdf)
+
+ assert w["model_calls"] == 2, w
+ assert w["model_calls_max"] == ASK_ATTEMPTS * (1 + 1), w
+
+
def test_the_printed_estimate_does_not_promise_a_ceiling_it_can_exceed():
import inspect
From 9acce4729ed902263f688fb208ebec27c8fb4aef Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Thu, 17 Sep 2026 23:39:58 +0200
Subject: [PATCH 27/44] docs: a resolved label must be judged against the
resolved entry
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Task 7's menu option 1 took a label out of labels_disputed — so check judges it
again — and returned the entry list unchanged. The resolved entries sit in
Resolution.resolved and were computed and thrown away, leaving the verdict to be
printed against the very paper the resolution ruled against, under a report line
saying the user accepted it. The original wrong-paper bug, reached through the
one path a human consented to.
The Interfaces block said the list is changed only by option 3, which is what
made the omission read as intentional. Both now substitute, option 1 per
resolved label, and a test pins it because nothing else in the plan would
notice.
---
.../plans/2026-09-17-llm-reference-list.md | 58 ++++++++++++++++++-
1 file changed, 56 insertions(+), 2 deletions(-)
diff --git a/docs/superpowers/plans/2026-09-17-llm-reference-list.md b/docs/superpowers/plans/2026-09-17-llm-reference-list.md
index e4426a9..26643dc 100644
--- a/docs/superpowers/plans/2026-09-17-llm-reference-list.md
+++ b/docs/superpowers/plans/2026-09-17-llm-reference-list.md
@@ -4607,7 +4607,11 @@ renders the disclosure twice.
- `cli._interactive() -> bool`
- `cli._write_disagreement(case, labels, candidates, texts) -> Path`
- `cli._escalate_disputed(...) -> list[RefEntry]` — mutates `rec` in place and
- returns the entry list to carry forward (changed only by menu option 3)
+ returns the entry list to carry forward. **Changed by menu option 3 (a whole
+ reading substituted) and by menu option 1 (each resolved label's entry
+ substituted).** An earlier draft of this block said "option 3 only", which
+ would leave option 1 un-disputing a label while judging it against the entry
+ the resolution ruled against — see Step 5's comment and its test.
- `reflist.Resolution` — `resolved: dict[str, RefEntry]`,
`still_disputed: list[str]`, `provenance: ReflistProvenance`
- `reflist.resolve_disputed(labels, entries, texts, *, model=None) -> Resolution`
@@ -5363,12 +5367,62 @@ def _escalate_disputed(
"accepted that. It is a reading, not a confirmation — the numbering is still "
"unconfirmed"
)
- return entries
+ # The resolved entry REPLACES the one in the chosen list, in place, same
+ # order, same length. Returning `entries` unchanged here — which an earlier
+ # draft of this plan did — takes the label out of `labels_disputed` so
+ # `check` judges it again, while leaving the entry the resolution just ruled
+ # AGAINST as the paper it judges against. That is the original wrong-paper
+ # bug reached through the one path a user consented to, and the report would
+ # say "you accepted that" over it.
+ #
+ # This is the ONLY place in the codebase where a model reading changes what
+ # gets resolved and judged, and it is legal only because
+ # `rec.chosen_by == "user"` is recorded beside it. `rec.verified` stays
+ # False regardless; Task 8 proves that over every reply shape and choice.
+ return [res.resolved.get(e.num, e) for e in entries]
```
`_label_list` lives in `refs.py`; add it to the `from .refs import (…)` block
inside `_refs_pipeline` rather than reimplementing the bracket join.
+**A test this step must carry**, because nothing else in the plan would notice:
+
+```python
+def test_a_resolved_label_judges_against_the_resolved_entry(tmp_path, monkeypatch):
+ """Un-disputing a label without substituting its entry is the original bug.
+
+ Menu option 1 takes a label out of `labels_disputed`, so `check` judges it
+ again. If the chosen list still holds the entry the resolution ruled
+ against, the verdict is printed against the very paper the model said was
+ wrong — and the report says the user accepted it. The resolved entries exist
+ in `Resolution.resolved`; the failure mode is computing them and throwing
+ them away.
+ """
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ resolved = _entry("13", "Weston AD (2019) The paper the label really names. "
+ "https://doi.org/10.1148/x13")
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, cands, texts, model=None: reflist_mod.Resolution(
+ resolved={"13": resolved}, still_disputed=[],
+ provenance=reflist_mod.ReflistProvenance(model="claude-opus-5"),
+ ),
+ )
+
+ rec = _rec(labels_disputed=["13"])
+ wrong = _entry("13", "Somebody Else (1999) A different paper entirely. "
+ "https://doi.org/10.1148/x99")
+ out = cli._escalate_disputed(
+ tmp_path, rec, [wrong], {"parsed": [wrong]}, {"parsed": "text"},
+ )
+
+ assert rec.labels_disputed == [] # the label will be judged again
+ assert out[0].doi == "10.1148/x13" # against the RESOLVED paper
+ assert rec.chosen_by == "user" # which is what authorises it
+ assert rec.verified is False # and it is still not confirmed
+```
+
Wire it in at `cli.py:659`, immediately after the `if parse_only: … return`
block:
From b00af17e7b309651c72db9eedecd490dbc215bf0 Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Thu, 17 Sep 2026 23:56:22 +0200
Subject: [PATCH 28/44] refs: a failed model call is a third outcome, not a
boolean's blind spot
Review round 2. attempted: bool could not tell "never called" apart from
"called and failed" -- a timeout's own note didn't start with "reading
discarded", so it fell into the success branch and printed "1 value it
proposed was not found in the printed text" for a call that proposed
nothing. reflist_outcome is now one of not_attempted/failed/read, published
as REFLIST_OUTCOMES; reflist_failure carries why, kept apart from
fields_discarded so a reason can never be miscounted as a discarded value.
The claude-absent test drove needs_flat=False, which returns before
ask.claude_available is ever consulted -- the exact thing it claimed to
cover went untested, and the "not attempted" substring it matched was the
backend-skip reason. Fixed to reach the guard and assert on the reason's
content; the two reasons no longer share a prefix.
corroborating_readings now intersects the per-label voters label_agreement
itself computes, rather than approximating "carried a cited label" -- which
named a reading that skipped an entry, or whose only carrier was
boundary_ambiguous, as having agreed on all of them.
Walking every outcome by hand caught two more: a plain --no-llm-refs run
started printing a broken disclosure once outcome was always a real string,
and the console line for a reading discarded whole read like an ordinary
success. Both fixed with regression tests.
---
schemas/refs_manifest.schema.json | 17 +-
src/papertrace/cli.py | 117 +++++++----
src/papertrace/disclosures.py | 104 +++++++---
src/papertrace/models.py | 41 ++--
src/papertrace/reflist.py | 54 +++--
src/papertrace/refs.py | 65 +++++-
src/papertrace/wizard.py | 16 +-
tests/test_label_agreement.py | 55 +++++
tests/test_reference_readings.py | 321 +++++++++++++++++++++++++-----
tests/test_reflist.py | 5 +-
tests/test_wizard.py | 14 +-
11 files changed, 638 insertions(+), 171 deletions(-)
diff --git a/schemas/refs_manifest.schema.json b/schemas/refs_manifest.schema.json
index 891bbd8..d4daa8a 100644
--- a/schemas/refs_manifest.schema.json
+++ b/schemas/refs_manifest.schema.json
@@ -278,13 +278,18 @@
],
"description": "Who is responsible for `numbering_choice`: '' when no escalation ran, 'default' when nobody was actually asked, 'user' when a person consented to it. A person consenting to proceed is an input, never evidence that the numbering was verified — `numbering_verified` cannot be set from this."
},
- "reflist_attempted": {
- "type": "boolean",
- "description": "Whether `refs` actually invoked a model to read the bibliography, regardless of whether the reply named itself. False means the call was never made at all — --llm-refs was off, --parse-only forced it off, the backend left no second reading to compare against, claude was not on PATH, or the subprocess failed before any reply was verified. Distinguishes \"no call happened\" from \"a call happened and voted but reported no name\", which `reflist_model` alone cannot: it is empty in both cases."
+ "reflist_outcome": {
+ "type": "string",
+ "enum": ["", "not_attempted", "failed", "read"],
+ "description": "Which of the three real outcomes this run's model reading of the bibliography reached: 'not_attempted' (no subprocess call was ever made — --llm-refs off, --parse-only, the backend left no second reading to compare against, or claude not on PATH), 'failed' (a call was made and did not return an answer), or 'read' (a reply was obtained, whether or not anything in it survived verification). Absent or '' means NEVER COMPUTED, not 'not_attempted' — a manifest written before this field existed asserts nothing about whether a call was made. A two-valued flag could not tell 'never called' apart from 'called and failed', which is exactly the bug this field replaced: a failed call published as a successful reading."
+ },
+ "reflist_failure": {
+ "type": "string",
+ "description": "Why there is no reading, when `reflist_outcome` is not 'read' — never a value that failed verification (`reflist_fields_discarded`'s job). '' when `reflist_outcome` is 'read', or when neither has ever been computed."
},
"reflist_model": {
"type": "string",
- "description": "The model that produced the LLM's structured reading of the bibliography, when --llm-refs ran, produced anything usable, AND the reply itself reported a model name. '' means EITHER no such call was made OR one was made, verified and voted but did not report which model answered — see `reflist_attempted` for which; this field alone cannot tell them apart, and never means the reading was rejected."
+ "description": "The model that produced the LLM's structured reading of the bibliography, when --llm-refs ran, produced anything usable, AND the reply itself reported a model name. '' means EITHER no such call was made OR one was made, verified and voted but did not report which model answered — see `reflist_outcome` for which; this field alone cannot tell them apart, and never means the reading was rejected."
},
"reflist_fields_discarded": {
"type": "array",
@@ -299,6 +304,10 @@
"items": {
"type": "string"
}
+ },
+ "reflist_entries_proposed": {
+ "type": "integer",
+ "description": "How many entries the model's reply proposed, before verification, whenever `reflist_outcome` is 'read' — including 0. Absent or 0 on a manifest where `reflist_outcome` is not 'read' means never computed, not 'proposed nothing'. Without this a reader cannot tell a model that proposed no entries at all from one that proposed several and had none discarded — both otherwise print '0 fields discarded'."
}
}
}
diff --git a/src/papertrace/cli.py b/src/papertrace/cli.py
index 47a7eac..7fc9bcd 100644
--- a/src/papertrace/cli.py
+++ b/src/papertrace/cli.py
@@ -570,11 +570,12 @@ def _llm_reference_reading(
earlier draft of this plan did exactly that and would have dropped every
duplicate and gap on the floor, unreported.
- `provenance.fields_discarded` is never empty when `enabled` and the model
- was not asked or the call failed: an attempt that produced nothing has to
- leave the reason where the manifest and the report can read it, because
- silence here would be read as "no news" about a reading that was asked for
- and did not happen.
+ `provenance.outcome` is set to exactly one of `reflist.REFLIST_OUTCOMES` on
+ every return, and `provenance.failure` carries why whenever `outcome` is
+ not `"read"`. A boolean here could not tell "never called" apart from
+ "called and failed" — a failed call used to fall through to the same
+ branch a successful one takes, published as a value that failed
+ verification when nothing was ever proposed at all.
`disabled_reason` names a STRUCTURAL cause when `not enabled` — the run's
own backend leaves no second reading to compare against — as opposed to a
@@ -585,16 +586,18 @@ def _llm_reference_reading(
from .reflist import ReflistProvenance, propose
if not enabled:
- # a caller's own choice is silent (empty `fields_discarded`); a
- # structural reason the caller had no choice about is disclosed, same
- # as "claude not on PATH" below
- return [], ReflistProvenance(fields_discarded=[disabled_reason] if disabled_reason else [])
+ # a caller's own choice is silent (empty `failure`); a structural
+ # reason the caller had no choice about is disclosed, same as "claude
+ # not on PATH" below. `outcome` stays at its "not_attempted" default.
+ return [], ReflistProvenance(failure=disabled_reason)
if not ask.claude_available():
# the ordinary case for someone who installed papertrace and not Claude
# Code. The run proceeds on the deterministic readings and says so —
- # never that a model agreed with them.
+ # never that a model agreed with them. Worded with no substring in
+ # common with the backend-skip reason above, so a test (or a reader)
+ # asserting on the CONTENT of one can never be satisfied by the other.
return [], ReflistProvenance(
- fields_discarded=["not attempted — claude is not on PATH, so no model read the list"]
+ failure="claude is not on PATH, so no model read the reference list"
)
try:
entries, prov = propose(reading_a, reading_b, label_a=label_a, label_b=label_b)
@@ -605,15 +608,17 @@ def _llm_reference_reading(
# timeout and a non-zero exit and ValueError for unparseable stdout, and
# a subprocess can surface OSError besides — enumerating them would fail
# closed on the next one. The failure is recorded, not swallowed:
- # `reflist_fields_discarded` carries it into the manifest and a console
- # line says it at the time. Same shape as `check.py:996`.
- # `attempted=True`: every exception reaching here came out of `propose`
- # actually invoking the subprocess (its own two early returns for "not
- # enough text" and "not a JSON array" return normally, not raise) — a
- # failed call is still a call that happened, not one that never was.
+ # `reflist_failure` carries it into the manifest and a console line
+ # says it at the time. Same shape as `check.py:996`.
+ # `outcome="failed"`, never "not_attempted": every exception reaching
+ # here came out of `propose` actually invoking the subprocess — its own
+ # two early returns for "not enough text" and "not a JSON array" return
+ # normally, they do not raise. A fresh `ReflistProvenance` is built
+ # rather than reusing `propose`'s local one, which never gets returned:
+ # the exception propagates past `propose`'s own `return` statement.
return [], ReflistProvenance(
- attempted=True,
- fields_discarded=[f"not obtained — {type(e).__name__}: {str(e)[:200]}"],
+ outcome="failed",
+ failure=f"{type(e).__name__}: {str(e)[:200]}",
)
if prov.discarded_whole:
# the entries are already `[]` in this branch; the reason is what travels,
@@ -652,6 +657,7 @@ def _refs_pipeline(
from .reflist import _label_key
from .refs import (
_client,
+ corroborating_readings,
crossref_deposit,
deposit_corroborates,
deposit_is_this_paper,
@@ -798,9 +804,12 @@ def _refs_pipeline(
label_a=smap.converter,
label_b="pymupdf",
enabled=llm_wanted and needs_flat,
+ # worded with no substring in common with "claude is not on PATH" —
+ # the two structural reasons must never be mistaken for each other by
+ # anything matching on their text
disabled_reason=(
- "not attempted — this run's backend is pymupdf, so a second flat "
- "reading of the bibliography would be the same text"
+ "this run's backend is pymupdf, so a second flat reading of the "
+ "bibliography would be the same text"
if llm_wanted and not needs_flat
else ""
),
@@ -809,24 +818,48 @@ def _refs_pipeline(
smap=smap, crossref=crossref_entries, parsed=parsed,
flat=flat_entries, llm=llm_entries,
)
- if reflist_prov.attempted:
- # named, even when the reply itself did not — see `ReflistProvenance.attempted`'s
- # docstring for why `model` alone cannot stand in for "this happened"
- named_model = reflist_prov.model or "an unnamed model"
- dropped = len(reflist_prov.fields_discarded)
- console.print(
- f"the reference list was also read by [bold]{named_model}[/bold] · "
- f"{dropped} field{'' if dropped == 1 else 's'} discarded as "
- f"not printed [dim]— a second reading, not confirmation[/dim]"
+ # Three states, printed three different ways — never fewer, or a call
+ # that failed prints as one that succeeded (the regression this replaced):
+ # see `reflist.ReflistProvenance.outcome`'s docstring.
+ if reflist_prov.outcome == "read":
+ # named, even when the reply itself did not — same wording
+ # `disclosures._reflist` uses for this state, so the console and the
+ # written report never describe one call two different ways
+ named_model = reflist_prov.model or "a model that did not report its own name"
+ whole = next(
+ (n.removeprefix("reading discarded — ")
+ for n in reflist_prov.fields_discarded if n.startswith("reading discarded")),
+ "",
)
+ if whole:
+ # a reply was obtained and the WHOLE reading was refused — never
+ # the same line as a success with fields dropped, which read as
+ # "N fields discarded as not printed" for a reply that was not a
+ # JSON array at all, or named a paper the page does not print
+ console.print(
+ f"the reference list was also read by [bold]{named_model}[/bold], "
+ f"and its reading was [yellow]discarded[/yellow] — {whole}"
+ )
+ else:
+ dropped = len(reflist_prov.fields_discarded)
+ console.print(
+ f"the reference list was also read by [bold]{named_model}[/bold] · "
+ f"{dropped} field{'' if dropped == 1 else 's'} discarded as "
+ f"not printed [dim]— a second reading, not confirmation[/dim]"
+ )
# reported separately, because it is a different kind of finding: not a
# value that was missing, but a reading whose own labels do not add up
for finding in reflist_prov.numbering_findings:
console.print(f" [yellow]⚠ the model reading's {finding}[/yellow]")
- elif reflist_prov.fields_discarded:
+ elif reflist_prov.outcome == "failed":
+ console.print(
+ "[yellow]⚠ the reference list could not be read by a model — "
+ f"the call did not return: {reflist_prov.failure}[/yellow]"
+ )
+ elif reflist_prov.failure:
console.print(
"[yellow]⚠ no model reading of the reference list[/yellow] — "
- f"{reflist_prov.fields_discarded[0]}"
+ f"{reflist_prov.failure}"
)
entries, rec = reconcile(body_labels, crossref_entries, entries, crossref_absent=absent)
@@ -849,16 +882,14 @@ def _refs_pipeline(
agreement.get(label) == "agreed" for label in body_labels
)
# Named only when corroboration holds, and only the readings that actually
- # voted on a cited label — never every reading `others` happened to carry.
- # `sorted(others)` names a reading that carried none of the cited labels
- # (it proposed only entries for labels the body never cites) as having
- # corroborated them, which `stamp_seen_in`'s own `_same_work` test would
- # never do for that same reading. Since `rec.corroborated` already requires
- # every cited label to be `agreed`, a reading that carries any cited label
- # at all necessarily agreed on it — filtering to "carries one" is enough.
+ # voted `agreed` on EVERY cited label — `refs.corroborating_readings`
+ # recomputes each label's voters with `label_agreement`'s own rule and
+ # intersects their names, rather than approximating "carried a cited
+ # label" (which named a reading that skipped an entry, or whose only
+ # carrier of one was `boundary_ambiguous` and so cast no vote at all —
+ # both measured; see the Task 6 re-review's N3).
rec.corroborating_readings = (
- sorted(name for name, cand in others.items() if any(e.num in body_labels for e in cand))
- if rec.corroborated else []
+ corroborating_readings(others, body_labels) if rec.corroborated else []
)
if rec.corroborated:
console.print(
@@ -948,10 +979,12 @@ def tick(e):
numbering_corroborated=rec.corroborated,
corroborating_readings=rec.corroborating_readings,
labels_disputed=rec.labels_disputed,
- reflist_attempted=reflist_prov.attempted,
+ reflist_outcome=reflist_prov.outcome,
+ reflist_failure=reflist_prov.failure,
reflist_model=reflist_prov.model,
reflist_fields_discarded=reflist_prov.fields_discarded,
reflist_numbering_findings=reflist_prov.numbering_findings,
+ reflist_entries_proposed=reflist_prov.entries_proposed,
)
manifest.to_json(case / "refs_manifest.json")
ok = len(manifest.retrieved)
diff --git a/src/papertrace/disclosures.py b/src/papertrace/disclosures.py
index 3d86ff7..a2907a3 100644
--- a/src/papertrace/disclosures.py
+++ b/src/papertrace/disclosures.py
@@ -548,47 +548,85 @@ def _labels_disputed(manifest) -> Disclosure | None:
def _reflist(manifest) -> Disclosure | None:
- """The required disclosure for the model reading, in all three of its outcomes.
-
- Fires on a reading used, a reading discarded, and a reading asked for and not
- obtained — never only on success, because silence about an attempt reads as
- no news. The token is phrased AROUND in every branch, the way
- `SUPPLEMENT_IDENTITY_TOKEN` is: it is asserted verbatim in all four formats,
- so a branch where it is not literally true would make one format lie.
-
- Branches on `reflist_attempted`, never on whether `reflist_model` is set.
- `claude -p` only reports a model name when its own JSON does — a reply
- naming none is not evidence nothing was asked — so a call that happened,
- verified cleanly and voted can still leave `reflist_model == ""`. Reading
- that emptiness as "no reading was obtained" is precisely the state a real
- run reached before `reflist_attempted` existed: one real call, nothing on
- the console, the manifest asserting no such call was made.
+ """The required disclosure for the model reading, across every real outcome.
+
+ Fires on a reading used, a reading discarded, a reading asked for and not
+ obtained, and a reading asked for whose call did not return — never only
+ on success, because silence about an attempt reads as no news. The token
+ is phrased AROUND in every branch, the way `SUPPLEMENT_IDENTITY_TOKEN` is:
+ it is asserted verbatim in all four formats, so a branch where it is not
+ literally true would make one format lie.
+
+ Branches on `reflist_outcome` — one of `reflist.REFLIST_OUTCOMES`, never on
+ whether `reflist_model` is set and never on a single boolean. A boolean
+ cannot tell "never called" apart from "called and failed": that shape of
+ field once sent a failed call (`outcome` would have been `True` either
+ way) down the SAME prose as a successful one, because the failure's own
+ note did not happen to start with the one prefix that branch checked for —
+ a plausible-looking "N values … discarded" in place of an admission that
+ the call never returned at all. `claude -p` also only reports a model name
+ when its own JSON does — a reply naming none is not evidence nothing was
+ asked — so a call that happened, verified cleanly and voted can still
+ leave `reflist_model == ""`; that is a SEPARATE fact from `outcome` and is
+ handled by `named` below, never by conflating the two.
"""
+ outcome = getattr(manifest, "reflist_outcome", "") or ""
model = getattr(manifest, "reflist_model", "") or ""
- attempted = bool(getattr(manifest, "reflist_attempted", False))
+ failure = getattr(manifest, "reflist_failure", "") or ""
notes = list(getattr(manifest, "reflist_fields_discarded", []) or [])
# a different kind of finding, and it must reach the reader too: a reading
# whose own labels do not add up is worth knowing about even when every
# value it proposed was printed
numbering = list(getattr(manifest, "reflist_numbering_findings", []) or [])
- if not attempted and not notes:
- return None # no model reading was asked for
+ entries_proposed = int(getattr(manifest, "reflist_entries_proposed", 0) or 0)
+ if not outcome:
+ # An older manifest from before `reflist_outcome` existed. A reported
+ # model name is positive proof a call happened and answered — settle
+ # as "read". Absent that, a `failure` reason alone is enough to say
+ # SOMETHING was asked for even though which state it reached was never
+ # recorded; treat it as "not_attempted" rather than manufacture a
+ # third guess. Only the inverse ("no name and no reason -> no call")
+ # would be the forbidden derivation, and that is exactly the case
+ # below that returns `None`.
+ outcome = "read" if model else ("not_attempted" if failure else "")
+ if not outcome or (outcome == "not_attempted" and not failure):
+ # the second clause is the caller's own silent choice (`--no-llm-refs`,
+ # `--parse-only`): `outcome` is recorded as `"not_attempted"` either
+ # way, so `not outcome` alone cannot tell "nothing to report" apart
+ # from "asked for, and never happened" — only the absence of a reason
+ # can, the same distinction `_llm_reference_reading`'s `disabled_reason`
+ # makes when it decides whether to fill `failure` at all
+ return None # no model reading was asked for, and nothing to report
ceiling = (
"A model agreeing with a parse is a second reading, not confirmation: it read "
"the same document, so a reference the layout destroyed is one it may also "
"have missed."
)
# named even when the reply itself did not — see this function's own
- # docstring for why `model` alone cannot stand in for "this happened"
+ # docstring for why `model` alone cannot stand in for "this happened".
+ # Identical wording to `cli._refs_pipeline`'s console line for the same
+ # state, so the two surfaces never describe one call two different ways.
named = model or "a model that did not report its own name"
- if not attempted:
+ if outcome == "not_attempted":
head = (
f"This run asked that {REFLIST_TOKEN}, and no reading was obtained: "
- f"{notes[0]}. The reference numbering therefore rests on the readings above "
+ f"{failure}. The reference numbering therefore rests on the readings above "
f"it and nothing else."
)
- short = f"{REFLIST_TOKEN}: asked for, not obtained — {notes[0]}"
- level = "warn" # a failed attempt, not routine information
+ short = f"{REFLIST_TOKEN}: asked for, not obtained — {failure}"
+ level = "warn" # a structural gap, not routine information
+ elif outcome == "failed":
+ # textually distinct from the branch above on purpose: this is a call
+ # that WAS made and did not return, never "no reading was obtained" —
+ # the two must not read as the same fact
+ head = (
+ f"This run asked that {REFLIST_TOKEN}, and the call did not return an "
+ f"answer: {failure}. Nothing it might have proposed reached the list "
+ f"below, and the reference numbering rests on the readings above this "
+ f"and nothing else."
+ )
+ short = f"{REFLIST_TOKEN}: the call failed — {failure}"
+ level = "warn" # a call that did not return is not an aside
elif any(n.startswith("reading discarded") for n in notes):
why = next(n for n in notes if n.startswith("reading discarded"))
head = (
@@ -598,12 +636,20 @@ def _reflist(manifest) -> Disclosure | None:
short = f"{REFLIST_TOKEN} ({named}) — discarded as unusable"
level = "info"
else:
- dropped = (
- f"{len(notes)} value{'' if len(notes) == 1 else 's'} it proposed "
- f"{'was' if len(notes) == 1 else 'were'} not found in the printed text and "
- f"{'was' if len(notes) == 1 else 'were'} discarded"
- if notes else "every value it proposed was found in the printed text"
- )
+ if entries_proposed == 0:
+ # distinct from "every value it proposed was found" below: THAT
+ # sentence describes a reading that proposed something and none of
+ # it was dropped, which reads as a clean corroboration — a reply
+ # that proposed nothing at all corroborated nothing
+ dropped = "it proposed no entries at all"
+ elif notes:
+ dropped = (
+ f"{len(notes)} value{'' if len(notes) == 1 else 's'} it proposed "
+ f"{'was' if len(notes) == 1 else 'were'} not found in the printed text and "
+ f"{'was' if len(notes) == 1 else 'were'} discarded"
+ )
+ else:
+ dropped = "every value it proposed was found in the printed text"
head = (
f"Here {REFLIST_TOKEN} ({named}), shown two extractions of the same printed "
f"bibliography and asked what numbered list it carries; {dropped}."
diff --git a/src/papertrace/models.py b/src/papertrace/models.py
index f572a16..7e3d500 100644
--- a/src/papertrace/models.py
+++ b/src/papertrace/models.py
@@ -594,21 +594,26 @@ class RefManifest:
# never ran. Never set from a model's own say-so alone.
numbering_choice: str = "" # "" | withheld | llm_resolved | parsed | pymupdf
numbering_chosen_by: str = "" # "" | default | user
- # Whether `refs` actually invoked a model to read the bibliography — set
- # regardless of whether the reply named itself. False means the call was
- # never made at all: --llm-refs was off, --parse-only forced it off, the
- # backend left no second reading to compare against, claude was not on
- # PATH, or the subprocess itself failed before any reply was verified.
- # This is the field that distinguishes "no call happened" from "a call
- # happened and voted, but the CLI did not learn its name" — `reflist_model`
- # alone cannot, because `claude -p` only reports a model name when its own
- # JSON does, and a reply naming none is not evidence nothing was asked.
- reflist_attempted: bool = False
+ # Which of `reflist.REFLIST_OUTCOMES` this run's model reading reached.
+ # "" means NEVER COMPUTED — not "not_attempted" — the same three-state
+ # discipline as `numbering_ledger`; a manifest written before this field
+ # existed asserts nothing about whether a call was made. Once computed it
+ # is always one of the three real values: a boolean could not tell "never
+ # called" apart from "called and failed", which is exactly the shape of
+ # bug that produced a failed call published as a successful reading.
+ reflist_outcome: str = ""
+ # Why there is no reading, when `reflist_outcome` is not `"read"` — never
+ # a value that failed verification (`reflist_fields_discarded`'s job) and
+ # never a reading that was checked and refused whole
+ # (`reflist_fields_discarded`'s "reading discarded —" entries, which
+ # require a reply to have been obtained at all). "" when `reflist_outcome`
+ # is `"read"`, or when neither has ever been computed.
+ reflist_failure: str = ""
# The model that produced the LLM's structured reading of the
# bibliography, when --llm-refs ran, produced anything usable, AND the
# subprocess's own reply reported a name. "" means EITHER no such call was
# made OR one was made and answered but did not report which model —
- # `reflist_attempted` is what tells those two apart; this field alone
+ # `reflist_outcome` is what tells those two apart; this field alone
# cannot.
reflist_model: str = ""
# Which fields of the LLM's proposed reading could not be found verbatim
@@ -621,6 +626,12 @@ class RefManifest:
# dropped. Empty means none found, which on a reading that produced no
# entries is not the same as none present.
reflist_numbering_findings: list[str] = field(default_factory=list)
+ # How many entries the model's reply proposed, before verification —
+ # ALWAYS recorded when `reflist_outcome == "read"`, including 0. Without
+ # this, "0 fields discarded" reads identically whether the model proposed
+ # nothing at all or proposed several entries that all verified cleanly,
+ # and the report cannot tell a reader which happened.
+ reflist_entries_proposed: int = 0
def document(self, slug: str) -> Document | None:
"""The judgeable file this slug names, article or supplement, or None."""
@@ -704,10 +715,12 @@ def to_json(self, path: Path) -> None:
"labels_resolved": self.labels_resolved,
"numbering_choice": self.numbering_choice,
"numbering_chosen_by": self.numbering_chosen_by,
- "reflist_attempted": self.reflist_attempted,
+ "reflist_outcome": self.reflist_outcome,
+ "reflist_failure": self.reflist_failure,
"reflist_model": self.reflist_model,
"reflist_fields_discarded": self.reflist_fields_discarded,
"reflist_numbering_findings": self.reflist_numbering_findings,
+ "reflist_entries_proposed": self.reflist_entries_proposed,
"summary": {
"total": len(self.entries),
"available": len(self.retrieved),
@@ -747,10 +760,12 @@ def from_json(cls, path: Path) -> RefManifest:
labels_resolved=data.get("labels_resolved", []),
numbering_choice=data.get("numbering_choice", ""),
numbering_chosen_by=data.get("numbering_chosen_by", ""),
- reflist_attempted=bool(data.get("reflist_attempted", False)),
+ reflist_outcome=data.get("reflist_outcome", ""),
+ reflist_failure=data.get("reflist_failure", ""),
reflist_model=data.get("reflist_model", ""),
reflist_fields_discarded=data.get("reflist_fields_discarded", []),
reflist_numbering_findings=data.get("reflist_numbering_findings", []),
+ reflist_entries_proposed=data.get("reflist_entries_proposed", 0),
)
diff --git a/src/papertrace/reflist.py b/src/papertrace/reflist.py
index 251fc6b..4fd8a4f 100644
--- a/src/papertrace/reflist.py
+++ b/src/papertrace/reflist.py
@@ -146,17 +146,36 @@ class ReflistProvenance:
numbering_findings: list[str] = field(default_factory=list)
# non-empty means the whole reading was refused, and says why. It is never
# "the model agreed": a reading that could not be checked is not a reading.
+ # ALWAYS set after a reply was actually obtained (`outcome == "read"`) —
+ # never before the call, and never for a call that did not return one.
discarded_whole: str = ""
readings: list[str] = field(default_factory=list)
- # Whether `claude -p` was actually invoked — set the instant `propose`
- # commits to the call, never derived from `model`. `ask._ask` records a
- # model name only when the subprocess's own JSON reports one ("a reply
- # that names no model is not evidence the model changed", `ask.py`), so a
- # call that succeeded, verified cleanly and voted can still leave `model`
- # `""`. A caller that read `model` as the proxy for "did this happen at
- # all" would publish that vote as "no such call was made" — the exact
- # state a real run reached before this field existed.
- attempted: bool = False
+ # Three states, not a boolean paired with a string — a boolean asked to
+ # discriminate three outcomes (never called; called and failed; called and
+ # answered) sends a failed call down the same branch as a successful one
+ # whenever the failure happens to leave no name attached, which is exactly
+ # the regression a two-valued `attempted` produced. Set by `propose` to
+ # `"read"` only once a reply was actually obtained (never derived from
+ # `model`: `ask._ask` records a name only when the subprocess's own JSON
+ # reports one, so a call that answered and verified cleanly can still
+ # leave `model == ""`). Set by `_llm_reference_reading` to `"failed"` in
+ # its `except` branch. Left at the default for every case where no
+ # subprocess call was ever made — disabled, `--parse-only`, the backend
+ # leaving no second reading, or `claude` not on PATH.
+ outcome: str = "not_attempted"
+ # Why there is no reading, when `outcome` is not `"read"` — never a value
+ # that failed verification (that is `fields_discarded`'s job) and never a
+ # reading that was checked and refused (that is `discarded_whole`'s job,
+ # and requires a reply to have been checked at all). This is the one slot
+ # for "why is there nothing to show", whether nothing was ever attempted
+ # or an attempt was made and did not return.
+ failure: str = ""
+
+
+# published so a consumer has something to check membership against, and so a
+# future fifth branch of "why there is no reading" cannot be added as a bare
+# string nobody enumerated — the same discipline as `LABEL_AGREEMENT`
+REFLIST_OUTCOMES = ("not_attempted", "failed", "read")
# order matters only for `raw` below, which is re-assembled in printed order
@@ -190,15 +209,16 @@ def propose(
if len(readings) < 2:
# one text is half the verification at full price, and no way to say so
# in the report — a value found in the only reading available has been
- # checked against nothing but itself
- prov.discarded_whole = (
+ # checked against nothing but itself. `outcome` stays "not_attempted":
+ # no subprocess call happens on this path at all, so nothing was
+ # "checked and refused" (`discarded_whole`'s meaning) — the reason
+ # belongs in `failure`, the same slot a disabled or claude-absent call
+ # uses for "why there is no reading"
+ prov.failure = (
"only one of the two extractions of the bibliography had any text, so "
"nothing the model proposed could have been checked against a second reading"
)
return [], prov
- # committed to the call from here on — everything past this point is a real
- # attempt, whatever it returns
- prov.attempted = True
# labels before texts: a label is a short fixed string ("pymupdf",
# "docling 2.8.0") and cannot contain a placeholder, while page text
@@ -215,6 +235,12 @@ def propose(
# `ask._ask`, which only a qualified call sees.
with ask.for_site(ask.SITE_REFS):
raw = ask._ask(prompt, model)
+ # a reply was obtained — everything from here on describes what came back,
+ # never whether anything came back at all. If `ask._ask` raised instead,
+ # this line is never reached and `prov` (with `outcome` still at its
+ # "not_attempted" default) is never returned — `_llm_reference_reading`'s
+ # `except` builds a fresh one with `outcome="failed"` instead.
+ prov.outcome = "read"
prov.model = ask.model_for(ask.SITE_REFS) or ""
try:
diff --git a/src/papertrace/refs.py b/src/papertrace/refs.py
index 0f2f22a..d8291a2 100644
--- a/src/papertrace/refs.py
+++ b/src/papertrace/refs.py
@@ -808,14 +808,7 @@ def label_agreement(
"""
result: dict[str, str] = {}
for label in body:
- voters: list[RefEntry] = []
- duplicated = False
- for entries in candidates.values():
- carriers = [e for e in entries if e.num == label and not e.boundary_ambiguous]
- if len(carriers) == 1:
- voters.append(carriers[0])
- elif carriers:
- duplicated = True # see the docstring: a duplicate is judgeable
+ voters, duplicated = _label_voters(candidates, label)
if duplicated:
result[label] = "disputed"
elif not voters:
@@ -823,11 +816,65 @@ def label_agreement(
elif len(voters) == 1:
result[label] = "single"
else:
- agree = all(_comparably_same(x, y) for x, y in itertools.combinations(voters, 2))
+ agree = all(
+ _comparably_same(x, y) for (_, x), (_, y) in itertools.combinations(voters, 2)
+ )
result[label] = "agreed" if agree else "disputed"
return result
+def _label_voters(
+ candidates: dict[str, list[RefEntry]], label: str
+) -> tuple[list[tuple[str, RefEntry]], bool]:
+ """The `(reading name, entry)` pairs entitled to a vote on this one label,
+ and whether any reading carried it more than once.
+
+ Factored out of `label_agreement` so `corroborating_readings` below can
+ name the readings that actually cast a vote, rather than approximating it
+ a third way. Private: the label-by-label state machine belongs to
+ `label_agreement`, and this is only its inner loop.
+ """
+ voters: list[tuple[str, RefEntry]] = []
+ duplicated = False
+ for name, entries in candidates.items():
+ carriers = [e for e in entries if e.num == label and not e.boundary_ambiguous]
+ if len(carriers) == 1:
+ voters.append((name, carriers[0]))
+ elif carriers:
+ duplicated = True # see label_agreement's docstring: judgeable, not silent
+ return voters, duplicated
+
+
+def corroborating_readings(candidates: dict[str, list[RefEntry]], body: set[str]) -> list[str]:
+ """Which readings actually voted `agreed` on EVERY cited label — named,
+ sorted, or `[]` if any label failed to reach `agreed` on its own.
+
+ Not `sorted(candidates)`: that names every reading TAKEN, whether or not it
+ carried the label at all. Not `any(e.num in body for e in cand)` either: a
+ reading that carries only SOME cited labels — a model reading that skipped
+ an entry — is not one that agreed on all of them, and a reading whose only
+ carrier of a label is `boundary_ambiguous` cast no vote on it at all. Both
+ were measured to overstate `corroborating_readings` in exactly this
+ function's absence. This recomputes each label's voters with
+ `_label_voters` — the same test `label_agreement` uses — and intersects
+ their names across every cited label, so a reading is named only when it
+ voted, and agreed, everywhere it was asked to.
+ """
+ names: set[str] | None = None
+ for label in body:
+ voters, duplicated = _label_voters(candidates, label)
+ if duplicated or len(voters) < 2:
+ return []
+ agree = all(
+ _comparably_same(x, y) for (_, x), (_, y) in itertools.combinations(voters, 2)
+ )
+ if not agree:
+ return []
+ label_names = {name for name, _ in voters}
+ names = label_names if names is None else (names & label_names)
+ return sorted(names) if names else []
+
+
def stamp_seen_in(chosen: list[RefEntry], candidates: dict[str, list[RefEntry]]) -> None:
"""Record, per chosen entry, which readings also carried that work.
diff --git a/src/papertrace/wizard.py b/src/papertrace/wizard.py
index b7e2910..3f5eafb 100644
--- a/src/papertrace/wizard.py
+++ b/src/papertrace/wizard.py
@@ -28,6 +28,7 @@
from .ask import ASK_ATTEMPTS, claude_available
from .brand import BANNER
from .ingest import _docling_available as docling_available
+from .ingest import resolve_backend
from .models import _LABEL_GROUP, _expand_label_group, is_references_heading
from .refs import DOI_RE
@@ -212,14 +213,13 @@ def workload(pdf: Path) -> dict:
# one extraction call, then one per cited source. Sources are judged in
# groups, so this is an upper bound on the judging calls, not a promise.
cited_source_calls = sum(len(g) for g in groups)
- # `run_wizard` always requests `backend="auto"`, which resolves to docling
- # when it is importable and to pymupdf otherwise (`ingest.resolve_backend`).
- # On a pymupdf run the flat second reading is skipped as identical to the
- # first — `cli._needs_flat_reading`'s own predicate, mirrored here rather
- # than imported, since importing `.ingest`'s `resolve_backend` would do —
- # so the reference-list call is never attempted either, and the estimate
- # must not promise a call this run's own backend will not produce.
- llm_call = 1 if docling_available() else 0
+ # `run_wizard` always requests `backend="auto"`, which `resolve_backend`
+ # turns into docling when it is importable and pymupdf otherwise. On a
+ # pymupdf run the flat second reading is skipped as identical to the first
+ # (`cli._needs_flat_reading`'s own predicate) — so the reference-list call
+ # is never attempted either, and the estimate must not promise a call this
+ # run's own backend will not produce.
+ llm_call = 1 if resolve_backend("auto") != "pymupdf" else 0
return {
"pages": pages,
"places": len(groups),
diff --git a/tests/test_label_agreement.py b/tests/test_label_agreement.py
index c087540..35bb5ec 100644
--- a/tests/test_label_agreement.py
+++ b/tests/test_label_agreement.py
@@ -35,6 +35,7 @@
_comparably_same,
_entry,
_same_work,
+ corroborating_readings,
label_agreement,
stamp_seen_in,
)
@@ -417,3 +418,57 @@ def test_a_reading_naming_a_different_paper_under_the_same_label_is_not_provenan
"pymupdf": [_work("5", "10.1000/x99")]}
stamp_seen_in(chosen, candidates)
assert chosen[0].seen_in == ["parsed"]
+
+
+# --- corroborating_readings: named only where a reading actually voted -----
+# (N3 of the Task 6 re-review: `any(e.num in body for e in cand)` — the filter
+# that shipped in round 1 — named a reading that carried only SOME cited
+# labels, or whose only carrier of one was `boundary_ambiguous`, as having
+# corroborated ALL of them.)
+
+
+def test_corroborating_readings_excludes_a_reading_that_voted_on_only_some_labels():
+ """A reading that skipped an entry corroborated the ones it carried, not
+ the ones it did not. `llm` here votes on [1] and never mentions [2] at
+ all — both cited labels still come back `agreed` (crossref and parsed
+ carry [2] between them), so the round-1 filter named `llm` as one of the
+ readings that agreed on `every cited label`, which it never spoke to."""
+ body = {"1", "2"}
+ candidates = {
+ "parsed": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ "crossref": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ "llm": [_work("1", "10.1000/x1")],
+ }
+ assert label_agreement(candidates, body) == {"1": "agreed", "2": "agreed"}
+ assert corroborating_readings(candidates, body) == ["crossref", "parsed"]
+
+
+def test_corroborating_readings_excludes_a_reading_whose_only_carrier_is_boundary_ambiguous():
+ """A `boundary_ambiguous` carrier casts no vote at all (invariant 5 of
+ `label_agreement`) — sharing that fact with `corroborating_readings` is
+ the point of factoring `_label_voters` out, rather than a coincidence two
+ unrelated filters happen to agree on."""
+ ambiguous_1 = _work("1", "10.1000/x1")
+ ambiguous_1.boundary_ambiguous = True
+ ambiguous_2 = _work("2", "10.1000/x2")
+ ambiguous_2.boundary_ambiguous = True
+ body = {"1", "2"}
+ candidates = {
+ "parsed": [ambiguous_1, ambiguous_2],
+ "crossref": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ "llm": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ }
+ assert label_agreement(candidates, body) == {"1": "agreed", "2": "agreed"}
+ assert corroborating_readings(candidates, body) == ["crossref", "llm"]
+
+
+def test_corroborating_readings_is_empty_unless_every_cited_label_agreed():
+ """The caller (`cli._refs_pipeline`) also gates on `all(... == "agreed"
+ ...)`, but this function must not rely on that: naming any reading at all
+ when one cited label is disputed would still be the overstatement."""
+ body = {"1", "2"}
+ candidates = {
+ "parsed": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ "llm": [_work("1", "10.1000/x1"), _work("2", "10.1000/x9")],
+ }
+ assert corroborating_readings(candidates, body) == []
diff --git a/tests/test_reference_readings.py b/tests/test_reference_readings.py
index 22216f2..e7a9d3a 100644
--- a/tests/test_reference_readings.py
+++ b/tests/test_reference_readings.py
@@ -188,7 +188,7 @@ def test_a_model_naming_different_papers_changes_no_resolved_entry(tmp_path, off
RefEntry(num="2", raw="Eta E. Another different paper. 1998.",
title="Another different paper", year="1998", seen_in=["pymupdf"])],
reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2,
- readings=["docling", "pymupdf"], attempted=True),
+ readings=["docling", "pymupdf"], outcome="read"),
)
monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: fabricated)
@@ -268,13 +268,26 @@ def test_no_llm_refs_makes_zero_model_calls(tmp_path, offline, monkeypatch):
calls = []
monkeypatch.setattr(ask_mod, "_ask", lambda prompt, model=None: calls.append(1) or "[]")
pdf = _paper(tmp_path / "paper.pdf")
+ case = tmp_path / "case"
- cli._refs_pipeline(manuscript=pdf, case=tmp_path / "case", provided=None,
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None,
email="test@example.org", parse_only=False, backend="pymupdf",
llm_refs=False)
assert calls == []
+ # a plain disable is silent, not "asked for and never obtained" — caught
+ # in this round's own verification (see the outcome/`_reflist` guard fix)
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RefManifest, RunResults
+
+ manifest = RefManifest.from_json(case / "refs_manifest.json")
+ assert manifest.reflist_outcome == "not_attempted"
+ assert manifest.reflist_failure == ""
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert fired == [], "a run that never asked for a model reading must disclose nothing about it"
+
def test_parse_only_makes_no_model_call_even_with_llm_refs_on(tmp_path, offline, monkeypatch):
"""`--parse-only` is documented as "List references, no network".
@@ -317,14 +330,12 @@ def test_a_pymupdf_backend_run_never_asks_the_model(tmp_path, monkeypatch):
manifest = _run_refs(tmp_path, monkeypatch, needs_flat=False, llm_refs=True)
assert calls == [], "propose must never be reached on a pymupdf-backend run"
- assert manifest.reflist_attempted is False
+ assert manifest.reflist_outcome == "not_attempted"
assert manifest.reflist_model == ""
- assert any(
- "this run's backend is pymupdf" in f for f in manifest.reflist_fields_discarded
- ), manifest.reflist_fields_discarded
- assert not any(
- "only one of the two extractions" in f for f in manifest.reflist_fields_discarded
- ), "the published reason must name the deliberate skip, not blame the PDF's extraction"
+ assert "this run's backend is pymupdf" in manifest.reflist_failure, manifest.reflist_failure
+ assert "only one of the two extractions" not in manifest.reflist_failure, (
+ "the published reason must name the deliberate skip, not blame the PDF's extraction"
+ )
def test_an_unavailable_claude_degrades_to_a_stated_absence(tmp_path, monkeypatch):
@@ -335,17 +346,21 @@ def test_an_unavailable_claude_degrades_to_a_stated_absence(tmp_path, monkeypatc
readings, the manifest records the reason, and no format anywhere says a
model corroborated anything.
- `needs_flat=False` (not the shared fixture's default): with the flat
- reading also forced on, "parsed" and the real flat-text reading of this
- same pymupdf-backend PDF are identical and genuinely corroborate each
- other with no model involved at all, which would make
- `numbering_corroborated` legitimately `True` for a reason this test is not
- about. Keeping the fixture single-voter here is what isolates "claude was
- unavailable" as the one fact under test.
+ N2 of the Task 6 re-review: the round-1 version of this test drove
+ `needs_flat=False`, which makes `enabled` (`llm_wanted and needs_flat`)
+ `False` — so `_llm_reference_reading` returns at its FIRST guard and
+ `ask.claude_available` is never consulted at all. All five of that
+ version's assertions passed with `claude` PRESENT too, because the
+ `"not attempted"` substring it matched was the backend-skip reason, not
+ the missing-CLI one — the very thing this test claims to cover was
+ untested. `needs_flat=True` here is what actually reaches the
+ availability check, and the assertions read the CONTENT of the reason
+ (`"claude is not on PATH"`) rather than a prefix both reasons used to
+ share.
"""
import papertrace.ask as ask_mod
- _wire_offline(monkeypatch, needs_flat=False)
+ _wire_offline(monkeypatch, needs_flat=True)
monkeypatch.setattr(ask_mod, "claude_available", lambda: False)
monkeypatch.setattr(ask_mod, "_ask",
lambda prompt, model=None: pytest.fail("claude was absent and asked anyway"))
@@ -357,10 +372,25 @@ def test_an_unavailable_claude_degrades_to_a_stated_absence(tmp_path, monkeypatc
payload = json.loads((case / "refs_manifest.json").read_text())
assert payload["reflist_model"] == ""
- assert payload["reflist_attempted"] is False
- assert any("not attempted" in f for f in payload["reflist_fields_discarded"]), payload
+ assert payload["reflist_outcome"] == "not_attempted"
+ assert "claude is not on PATH" in payload["reflist_failure"], payload["reflist_failure"]
assert "llm" not in payload["corroborating_readings"]
- assert payload["numbering_corroborated"] is False
+ # incidental to this test, not the fact under it (N2): with `needs_flat=True`
+ # the real flat-text reading of this same pymupdf PDF genuinely corroborates
+ # "parsed" with no model involved at all, so `numbering_corroborated` being
+ # `True` here says nothing about claude's absence — `"llm" not in
+ # corroborating_readings` above is the assertion that actually matters.
+
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RefManifest, RunResults
+
+ manifest = RefManifest.from_json(case / "refs_manifest.json")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "claude is not on PATH" in fired[0].text
+ assert "agreed" not in fired[0].text
+ assert fired[0].level == "warn"
def test_a_failed_model_call_does_not_kill_the_refs_stage(tmp_path, offline, monkeypatch):
@@ -369,6 +399,18 @@ def test_a_failed_model_call_does_not_kill_the_refs_stage(tmp_path, offline, mon
`refs` has already parsed the list and is about to fetch the sources. A
RuntimeError out of the seam is a failed corroboration attempt, not a failed
refs stage — and the reason lands where a reader can see it.
+
+ N1 of the Task 6 re-review, reproduced and now closed: before the fix, a
+ timeout published as `reflist_model: ""`, `reflist_fields_discarded: ["not
+ obtained — RuntimeError: ..."]`, `attempted: True` — and `_reflist` read
+ that shape as a SUCCESSFUL reading, because its note did not start with
+ "reading discarded" and `attempted` alone could not tell "called and
+ failed" apart from "called and voted". The disclosure said "1 value it
+ proposed was not found in the printed text" at `level=info`, for a call
+ that proposed nothing at all. `reflist_outcome == "failed"` is the
+ dedicated third state; `reflist_failure` (never `reflist_fields_discarded`)
+ is where the reason lives, so nothing downstream can miscount it as a
+ discarded value.
"""
def boom(*a, **kw):
raise RuntimeError("claude -p timed out after 600s")
@@ -382,9 +424,26 @@ def boom(*a, **kw):
payload = json.loads((case / "refs_manifest.json").read_text())
assert len(payload["entries"]) == 2
- assert any("timed out" in f for f in payload["reflist_fields_discarded"]), payload
+ assert payload["reflist_outcome"] == "failed"
+ assert "timed out" in payload["reflist_failure"], payload["reflist_failure"]
+ assert payload["reflist_fields_discarded"] == [], (
+ "the failure reason must not be counted as a discarded value"
+ )
assert payload["reflist_model"] == ""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RefManifest, RunResults
+
+ manifest = RefManifest.from_json(case / "refs_manifest.json")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "timed out" in fired[0].text
+ assert "was not found in the printed text" not in fired[0].text, (
+ "a call that never returned must not read as a value that failed verification"
+ )
+ assert fired[0].level == "warn", "a call that did not return is not an aside"
+
def test_corroboration_requires_every_cited_label_to_agree(tmp_path, offline, monkeypatch):
"""One disputed label is not "mostly corroborated".
@@ -400,7 +459,7 @@ def test_corroboration_requires_every_cited_label_to_agree(tmp_path, offline, mo
title="A first paper", year="2020", seen_in=["docling", "pymupdf"]),
RefEntry(num="2", raw="Gamma G. A wholly unrelated third paper. 2015.",
title="A wholly unrelated third paper", year="2015", seen_in=["A"])],
- reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2, attempted=True),
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2, outcome="read"),
)
monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: disagrees)
case = tmp_path / "case"
@@ -443,7 +502,7 @@ def test_no_model_reply_can_set_numbering_verified(tmp_path, offline, monkeypatc
title="A first paper", year="2020", seen_in=["docling", "pymupdf"]),
RefEntry(num="2", raw="Beta B. A second paper. J Fixture. 2021;2:10-19.",
title="A second paper", year="2021", seen_in=["docling", "pymupdf"])],
- reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2, attempted=True),
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2, outcome="read"),
)
monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: agrees)
pdf = _paper(tmp_path / "paper.pdf", extra_uncited=True)
@@ -466,7 +525,7 @@ def test_a_successful_reading_that_named_no_model_still_discloses_the_attempt(tm
one — a case `ask.py` explicitly anticipates ("a reply that names no
model is not evidence the model changed"). `reflist.propose` mirrors that
exactly: a real call that verifies cleanly and votes can still leave
- `prov.model == ""`. Before `reflist_attempted` existed, that state
+ `prov.model == ""`. Before `reflist_outcome` existed, that state
published as `reflist_model: ""` (the schema's OWN words for "no such
call was made") with no console line and no `reflist` disclosure, while
`numbering_corroboration` told the reader `llm` had corroborated the
@@ -479,14 +538,14 @@ def test_a_successful_reading_that_named_no_model_still_discloses_the_attempt(tm
title="A second paper", year="2021")],
# the exact shape a real `propose()` call produces when
# `ask.model_for(ask.SITE_REFS)` returns `None` — `model=""`,
- # `attempted=True`, a real vote in hand
- reflist_mod.ReflistProvenance(model="", entries_proposed=2, attempted=True),
+ # `outcome="read"`, a real vote in hand
+ reflist_mod.ReflistProvenance(model="", entries_proposed=2, outcome="read"),
)
monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: unnamed)
manifest = _run_refs(tmp_path, monkeypatch, llm_refs=True)
- assert manifest.reflist_attempted is True, "a call that voted must be recorded as attempted"
+ assert manifest.reflist_outcome == "read", "a call that voted must be recorded as read"
assert manifest.reflist_model == ""
from papertrace.disclosures import run_disclosures
@@ -509,7 +568,7 @@ def test_corroborating_readings_excludes_a_reading_that_carried_no_cited_label(
"""
abstains = (
[_work("3", "10.1000/x3"), _work("4", "10.1000/x4")],
- reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2, attempted=True),
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=2, outcome="read"),
)
monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: abstains)
capsys.readouterr() # discard anything buffered before this run
@@ -573,7 +632,7 @@ def test_the_reflist_disclosure_reaches_every_format_and_carries_the_ceiling(tmp
from papertrace.disclosures import REFLIST_TOKEN, run_disclosures
from papertrace.models import RunResults
- manifest = RefManifest(manuscript="p.pdf", reflist_attempted=True,
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="read",
reflist_model="claude-opus-5",
reflist_fields_discarded=["[2] journal"])
fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
@@ -586,6 +645,64 @@ def test_the_reflist_disclosure_reaches_every_format_and_carries_the_ceiling(tmp
assert "claude-opus-5" in d.text
+def test_a_reflist_model_with_no_recorded_outcome_still_settles_as_read(tmp_path):
+ """N4 of the Task 6 re-review. A manifest that carries `reflist_model` but
+ never had `reflist_outcome` computed — only reachable from a case folder
+ written by an earlier commit of this same unmerged branch, since
+ `reflist_model` predates `reflist_outcome` by one round and neither is on
+ `main` — must not lose its required disclosure and ceiling sentence.
+
+ `reflist_model` non-empty is POSITIVE PROOF a call happened and answered
+ with a name; settling `outcome` as `"read"` from that is not the forbidden
+ derivation. Only the inverse — "no name and no reason recorded, so no call
+ happened" — would be, and that is exactly the shape that returns `None`
+ below it.
+ """
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_model="claude-opus-5")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "claude-opus-5" in fired[0].text
+ assert "second reading, not confirmation" in fired[0].text
+
+
+def test_neither_model_nor_outcome_nor_failure_discloses_nothing(tmp_path):
+ """The one shape that must still return `None`: nothing was ever asked
+ for and nothing was ever obtained. `reflist_model` empty is not positive
+ proof of anything by itself — only its PRESENCE licenses an inference,
+ never its absence."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert fired == []
+
+
+def test_a_plain_disable_with_a_recorded_outcome_still_discloses_nothing(tmp_path):
+ """Caught in verification, not in the review: once `reflist_outcome` is
+ always a real value once computed, `not outcome` alone can no longer tell
+ "nothing to report" apart from "asked for and never obtained" — both
+ leave `outcome == "not_attempted"`. `--no-llm-refs` and `--parse-only`
+ record that outcome too (`_llm_reference_reading`'s `not enabled` branch),
+ with an empty `failure` — a plain `not outcome` guard read that as
+ something to report and produced 'This run asked that the reference list
+ was also read by a model, and no reading was obtained: . …', a broken
+ sentence with an empty reason, for a run that never asked at all."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="not_attempted",
+ reflist_failure="")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert fired == []
+
+
def test_a_model_reading_that_was_not_obtained_still_discloses_that(tmp_path):
"""Silence would be read as "no news". The attempt and its failure are news.
@@ -595,12 +712,13 @@ def test_a_model_reading_that_was_not_obtained_still_discloses_that(tmp_path):
from papertrace.disclosures import run_disclosures
from papertrace.models import RunResults
- manifest = RefManifest(manuscript="p.pdf", reflist_model="",
- reflist_fields_discarded=["not attempted — claude is not on PATH"])
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="not_attempted", reflist_model="",
+ reflist_failure="claude is not on PATH, so no model read the reference list")
fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
if d.key == "reflist"]
assert len(fired) == 1, fired
- assert "not attempted" in fired[0].text
+ assert "no reading was obtained" in fired[0].text
+ assert "claude is not on PATH" in fired[0].text
assert "agreed" not in fired[0].text
assert fired[0].level == "warn", "a failed attempt is a warning, not routine information"
@@ -614,7 +732,7 @@ def test_an_attempted_reading_that_named_no_model_is_worded_as_a_gap_in_reportin
from papertrace.disclosures import run_disclosures
from papertrace.models import RunResults
- manifest = RefManifest(manuscript="p.pdf", reflist_attempted=True, reflist_model="",
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="read", reflist_model="",
reflist_fields_discarded=[])
fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
if d.key == "reflist"]
@@ -622,6 +740,55 @@ def test_an_attempted_reading_that_named_no_model_is_worded_as_a_gap_in_reportin
text = fired[0].text
assert "not obtained" not in text
assert "did not report which model answered" in text
+ assert fired[0].level == "info", "a reading that ran and voted is routine information"
+
+
+def test_a_reply_proposing_no_entries_is_not_worded_as_a_clean_corroboration(tmp_path):
+ """N3b of the Task 6 re-review. A reply of `[]` used to be described as
+ "every value it proposed was found in the printed text" — vacuously true,
+ and it reads as a clean second opinion. A model that proposed no entries
+ at all corroborated nothing, and the sentence must say so, not describe an
+ empty set as a value-by-value success.
+ """
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="read",
+ reflist_model="claude-opus-5", reflist_entries_proposed=0)
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ text = fired[0].text
+ assert "it proposed no entries" in text
+ assert "every value it proposed was found" not in text
+
+
+def test_a_reading_discarded_whole_is_not_worded_as_a_second_reading_on_the_console(
+ tmp_path, monkeypatch, capsys
+):
+ """Caught in this round's own verification, walking the outcome table: a
+ reply that was obtained but not a JSON array at all (or one whose title
+ was never printed) sets `discarded_whole`, and the console used to print
+ the SAME "was also read by X · N fields discarded as not printed" line a
+ genuine success gets — true of nothing, since no field was individually
+ checked and dropped; the whole reading was refused before any field-level
+ check could run.
+ """
+ unusable = ([], reflist_mod.ReflistProvenance(
+ model="claude-opus-5", outcome="read", discarded_whole="the model's reply "
+ "was not a JSON array (no JSON array in model output: not json)",
+ fields_discarded=["reading discarded — the model's reply was not a JSON array "
+ "(no JSON array in model output: not json)"],
+ ))
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: unusable)
+ capsys.readouterr()
+
+ _run_refs(tmp_path, monkeypatch, llm_refs=True)
+
+ out = capsys.readouterr().out
+ assert "reading was" in out and "discarded" in out
+ assert "0 fields discarded as not printed" not in out
+ assert "· 1 field discarded as not printed" not in out
def test_corroboration_is_disclosed_without_claiming_the_numbering_was_verified(tmp_path):
@@ -711,31 +878,95 @@ def test_reflist_numbering_findings_round_trips_and_validates(tmp_path):
assert old.reflist_numbering_findings == []
-def test_reflist_attempted_round_trips_and_validates(tmp_path):
- """Gate 2, for Critical 1's field. Absent means never computed / no call
- ever made — the same three-state discipline as `numbering_ledger` — never
- "the model agreed" and never folded into `reflist_model`'s own emptiness,
- which cannot by itself distinguish "no call" from "a call that named no
- model" (see `RefManifest.reflist_attempted`'s own docstring).
+def test_reflist_outcome_and_failure_round_trip_and_validate(tmp_path):
+ """Gate 2, for the fields that replaced `reflist_attempted` in this same
+ unmerged task (N1 of the Task 6 re-review) — replacing rather than adding
+ is correct because `reflist_attempted` was never a published contract.
+ Absent `reflist_outcome` means NEVER COMPUTED, not `"not_attempted"` — the
+ same three-state discipline as `numbering_ledger` — and never folded into
+ `reflist_model`'s own emptiness, which cannot by itself distinguish "no
+ call" from "a call that named no model" (see `RefManifest.reflist_outcome`'s
+ own docstring).
"""
import jsonschema
- m = RefManifest(manuscript="p.pdf", reflist_attempted=True, reflist_model="")
+ m = RefManifest(manuscript="p.pdf", reflist_outcome="read", reflist_model="",
+ reflist_failure="")
path = tmp_path / "refs_manifest.json"
m.to_json(path)
payload = json.loads(path.read_text())
schema = json.loads(SCHEMA_PATH.read_text())
jsonschema.validate(payload, schema)
- assert "reflist_attempted" not in schema.get("required", [])
+ assert "reflist_outcome" not in schema.get("required", [])
+ assert "reflist_failure" not in schema.get("required", [])
back = RefManifest.from_json(path)
- assert back.reflist_attempted is True
+ assert back.reflist_outcome == "read"
assert back.reflist_model == ""
+ assert back.reflist_failure == ""
- del payload["reflist_attempted"]
+ for key in ("reflist_outcome", "reflist_failure"):
+ del payload[key]
path.write_text(json.dumps(payload))
old = RefManifest.from_json(path)
- assert old.reflist_attempted is False
+ assert old.reflist_outcome == "" # never computed — NOT "not_attempted"
+ assert old.reflist_failure == ""
+
+
+def test_reflist_failure_round_trips_for_the_failed_outcome(tmp_path):
+ """The other real value `reflist_outcome` takes, with a reason attached."""
+ import jsonschema
+
+ m = RefManifest(manuscript="p.pdf", reflist_outcome="failed",
+ reflist_failure="RuntimeError: claude -p timed out after 600s")
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ payload = json.loads(path.read_text())
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.validate(payload, schema)
+
+ back = RefManifest.from_json(path)
+ assert back.reflist_outcome == "failed"
+ assert "timed out" in back.reflist_failure
+
+
+def test_reflist_entries_proposed_round_trips_and_validates(tmp_path):
+ """Gate 2, for the field N3b needed: without it, "0 fields discarded"
+ cannot be told apart from a reply that proposed nothing to discard in the
+ first place."""
+ import jsonschema
+
+ m = RefManifest(manuscript="p.pdf", reflist_outcome="read",
+ reflist_entries_proposed=3)
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ payload = json.loads(path.read_text())
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.validate(payload, schema)
+ assert "reflist_entries_proposed" not in schema.get("required", [])
+
+ back = RefManifest.from_json(path)
+ assert back.reflist_entries_proposed == 3
+
+ del payload["reflist_entries_proposed"]
+ path.write_text(json.dumps(payload))
+ old = RefManifest.from_json(path)
+ assert old.reflist_entries_proposed == 0
+
+
+def test_reflist_outcomes_matches_its_own_schema_enum():
+ """The published vocabulary (`REFLIST_OUTCOMES`, the way `LABEL_AGREEMENT`
+ and `REF_STATUSES` are) against the schema's enum for `reflist_outcome` —
+ modelled on `test_ref_statuses_matches_its_own_schema_enum`'s idiom. `""`
+ is the schema's own "never computed" sentinel and is not part of the
+ published vocabulary of REAL outcomes, so it is added on both sides
+ rather than smuggled into `REFLIST_OUTCOMES` itself.
+ """
+ from papertrace.reflist import REFLIST_OUTCOMES
+
+ schema = json.loads(SCHEMA_PATH.read_text())
+ enum = schema["properties"]["reflist_outcome"]["enum"]
+ assert set(REFLIST_OUTCOMES) | {""} == set(enum)
def test_a_numbering_finding_from_the_model_reading_reaches_the_manifest(tmp_path, monkeypatch):
@@ -750,7 +981,7 @@ def test_a_numbering_finding_from_the_model_reading_reaches_the_manifest(tmp_pat
entries_proposed=2,
numbering_findings=["numerals proposed twice: 2"],
readings=["pymupdf", "docling"],
- attempted=True,
+ outcome="read",
),
)
monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: proposed)
diff --git a/tests/test_reflist.py b/tests/test_reflist.py
index 7c3b8dc..aa2627f 100644
--- a/tests/test_reflist.py
+++ b/tests/test_reflist.py
@@ -293,7 +293,10 @@ def test_an_empty_reading_makes_no_model_call_at_all(monkeypatch):
assert calls == []
assert entries == []
- assert prov.discarded_whole
+ # not `discarded_whole`: no reply was ever obtained to check and refuse —
+ # the call never happened, which `failure` says and `outcome` confirms
+ assert prov.failure
+ assert prov.outcome == "not_attempted"
def test_the_entry_level_seen_in_round_trips_through_the_manifest(tmp_path):
diff --git a/tests/test_wizard.py b/tests/test_wizard.py
index 6c97f20..2aae30c 100644
--- a/tests/test_wizard.py
+++ b/tests/test_wizard.py
@@ -676,14 +676,16 @@ def test_the_worst_case_call_count_includes_the_retry(tmp_path, monkeypatch):
shape a previous task already moved `model_calls_max` to, rather than
reverting it back to a flat `ASK_ATTEMPTS * cited_source_calls`.
- Pinned with `docling_available` forced `True` rather than left to whatever
- this machine happens to have installed — `workload()` derives the +1 from
- it, and a test whose expected numbers depend on the host environment is
- not really pinning anything.
+ Pinned with `resolve_backend` forced to report docling rather than left to
+ whatever this machine happens to have installed — `workload()` derives the
+ +1 from it (N5 of the Task 6 re-review: not `docling_available()`, which is
+ only a proxy for what `resolve_backend("auto")` actually resolves to), and
+ a test whose expected numbers depend on the host environment is not really
+ pinning anything.
"""
from papertrace.ask import ASK_ATTEMPTS
- monkeypatch.setattr(wizard, "docling_available", lambda: True)
+ monkeypatch.setattr(wizard, "resolve_backend", lambda backend: "docling")
pdf = _one_pager(tmp_path / "m.pdf",
"Title\nOne sentence citing [1].\nReferences\n[1] A. 2020.")
w = wizard.workload(pdf)
@@ -704,7 +706,7 @@ def test_the_worst_case_call_count_drops_the_reflist_call_on_a_pymupdf_only_inst
produce)."""
from papertrace.ask import ASK_ATTEMPTS
- monkeypatch.setattr(wizard, "docling_available", lambda: False)
+ monkeypatch.setattr(wizard, "resolve_backend", lambda backend: "pymupdf")
pdf = _one_pager(tmp_path / "m.pdf",
"Title\nOne sentence citing [1].\nReferences\n[1] A. 2020.")
w = wizard.workload(pdf)
From c9fa04271f8cb912f38a38f9d3efe9487a3f35fa Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Fri, 18 Sep 2026 14:12:36 +0200
Subject: [PATCH 29/44] refs: show the disagreement, then offer to resolve it
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The escalation. A run that found two readings naming different papers at a
cited label writes every reading of every disputed label to
case/out/reference_disagreement.md, with the verbatim text each was read from,
and prints the path — before any question is asked. Nobody is asked to choose
blind, and the test that pins it asserts the file is on disk at the moment
Prompt.ask is called, because a test that checks afterwards passes just as well
on an implementation that asks first.
Four choices: resolve the disputed labels with a model, withhold them, adopt one
reading whole, or abort. Option 1 substitutes each resolved entry into the list
it returns — un-disputing a label while leaving the entry the resolution ruled
against would judge the label against that paper, under a report line saying a
person accepted it. A non-interactive run does not prompt: it withholds the
disputed labels and keeps every other verdict, recording choice=withheld and
chosen_by=default rather than a choice nobody made.
chosen_by is what makes this honest. It is the only place a model reading can
change what gets resolved and judged, and it is legal because a person is
recorded as having asked for it. numbering_verified is untouched by every branch.
A window onto the printed list now opens ABOVE the disputed label, not at it:
when a reading splits one entry, the damage started in the entry before the
first disputed one, so a window beginning at the label showed the symptom and
hid the cause. The constant's own comment already claimed the entries either
side were visible.
And a run no longer prints "numbering confirmed" beside a line withholding a
label's verdicts. numbering_verified means extent; a disputed label is content.
Widening the flag to cover both — the first attempt — broke two of Plan A's
merged tests, correctly: four report formats read that boolean. The fix belonged
in the sentence, not the field.
---
src/papertrace/cli.py | 285 +++++++++++++++++++++++-
src/papertrace/disclosures.py | 111 ++++++++--
src/papertrace/reflist.py | 206 ++++++++++++++++++
tests/test_escalation.py | 393 ++++++++++++++++++++++++++++++++++
tests/test_refs.py | 19 ++
5 files changed, 991 insertions(+), 23 deletions(-)
create mode 100644 tests/test_escalation.py
diff --git a/src/papertrace/cli.py b/src/papertrace/cli.py
index 7fc9bcd..ec33810 100644
--- a/src/papertrace/cli.py
+++ b/src/papertrace/cli.py
@@ -489,6 +489,114 @@ def _body_citation_labels(smap, citation_labels, is_references_heading) -> set[s
return citation_labels("\n".join(body))
+def _interactive() -> bool:
+ """May this run stop and ask a person something?
+
+ Both streams, and not under CI. `stdin.isatty()` alone is not the gate: CI
+ runners frequently allocate a tty, and a prompt in CI is a hung build rather
+ than a question. `stdout` is checked too because a run whose output is being
+ piped into a file has a reader who is not watching.
+ """
+ return sys.stdin.isatty() and sys.stdout.isatty() and not os.environ.get("CI")
+
+
+_DISAGREEMENT_NAME = "reference_disagreement.md"
+# How much of the printed list to quote around a disputed label. Enough to show
+# the entries either side of it — a merge or a split is only visible against its
+# neighbours, and the neighbours are what say which reading dropped a reference.
+# Two constants because the window is not centred: the entry ABOVE the first
+# disputed label is where a split began, so it is worth about one reference of
+# backtrack, while the entries below are what shifted.
+_SPAN_BEFORE = 260
+_SPAN_CHARS = 600
+
+
+def _label_span(text: str, label: str) -> str:
+ """The printed text around this label, or "" when the numeral is not printed.
+
+ A window, not a parse. This file exists so a reader can check the parse, so
+ a second parse deciding what to show them would fail in the same place and
+ hide the same reference. `""` where the numeral was never printed is the
+ honest answer, and that absence is often the disagreement itself.
+ """
+ m = re.search(rf"(?:(?<=\n)|\A)\s*\[?{re.escape(label)}[\].:]?\s", text)
+ if m is None:
+ return ""
+ # Open the window ABOVE the label, not at it. When a reading splits one
+ # entry into two, every label after the split is shifted and the damage
+ # started in the entry *before* the first disputed one — a window that
+ # begins at the disputed label shows the symptom and hides the cause. Snap
+ # forward to the next line break so it never opens mid-reference.
+ start = max(0, m.start() - _SPAN_BEFORE)
+ if start:
+ nl = text.find("\n", start)
+ start = nl + 1 if 0 <= nl < m.start() else start
+ return text[start : m.start() + _SPAN_CHARS].strip()
+
+
+def _write_disagreement(
+ case: Path,
+ labels: list[str],
+ candidates: dict[str, list[RefEntry]],
+ texts: dict[str, str],
+) -> Path:
+ """Every reading of every disputed label, with the text each was read from.
+
+ Written before any question is asked. The structured fields alone are not
+ enough to settle a disagreement — they are what disagreed — so the verbatim
+ source span travels with them, and a reader who wants to answer the question
+ themselves can, without opening the PDF.
+ """
+ out = case / "out"
+ out.mkdir(parents=True, exist_ok=True) # not created by `_refs_pipeline` before this point
+ lines = [
+ "# Reference numbering — where the readings disagree",
+ "",
+ "One section per citation label whose readings of the bibliography name "
+ "different papers. Each reading's structured fields sit above the verbatim "
+ "text it was read from, so the disagreement can be settled by eye against "
+ "the printed list.",
+ "",
+ f"Readings compared: {', '.join(sorted(candidates))}.",
+ "",
+ "Verdicts on claims citing these labels are withheld — reported "
+ "`unchecked`, never guessed — unless a reading is adopted for them.",
+ "",
+ ]
+ for label in labels:
+ lines += [f"## [{label}]", ""]
+ for name in sorted(candidates):
+ hits = [e for e in candidates[name] if e.num == label]
+ lines += [f"### {name}", ""]
+ if not hits:
+ lines += ["Carries no entry for this label.", ""]
+ continue
+ # more than one hit IS the finding on a reading that carried the
+ # label twice — which of the two it means is precisely the question,
+ # so both are printed rather than the first
+ for e in hits:
+ lines += [
+ f"- doi: `{e.doi or '—'}`",
+ f"- year: `{e.year or '—'}`",
+ f"- slug: `{e.slug or '—'}`",
+ f"- refused to number itself: `{e.boundary_ambiguous}`",
+ "",
+ "Read from:",
+ "",
+ "```",
+ e.raw,
+ "```",
+ "",
+ ]
+ for name in sorted(texts):
+ if span := _label_span(texts[name], label):
+ lines += [f"### {name} — the printed list around it", "",
+ "```", span, "```", ""]
+ path = out / _DISAGREEMENT_NAME
+ path.write_text("\n".join(lines), encoding="utf-8")
+ return path
+
+
def _detected_doi(manuscript: Path) -> str | None:
"""The DOI printed on the paper's own front matter, or None.
@@ -627,6 +735,157 @@ def _llm_reference_reading(
return entries, prov
+_READING_LABEL = {"parsed": "the run backend's parse", "pymupdf": "the flat-text parse"}
+
+
+def _escalate_disputed(
+ case: Path,
+ rec,
+ entries: list[RefEntry],
+ candidates: dict[str, list[RefEntry]],
+ texts: dict[str, str],
+ model: str | None = None,
+) -> list[RefEntry]:
+ """Show the disagreement, then offer to resolve it. Mutates `rec`, returns the list to use.
+
+ Step 1 is unskippable and happens before the menu: nobody is asked to choose
+ blind. Nothing here may set `rec.verified` — the honest record is *which*
+ choice was made and *by whom*. `chosen_by: "user"` is the whole point: a
+ person consenting to proceed is an input, not evidence. This codebase
+ already ranks that signal — "a filename carries the user's assertion,
+ content carries none."
+
+ Changed by menu option 3 (a whole reading substituted) AND by option 1
+ (each resolved label's entry substituted) — never option 1 leaving the
+ chosen list untouched while un-disputing the label. That would let `check`
+ judge the label again against the very entry the resolution ruled against,
+ under a report line saying a person accepted it: the original wrong-paper
+ bug, reached through the one path a human authorised.
+ """
+ from .refs import _label_list # top-level in `refs.py`; not visible from `_refs_pipeline`'s
+ # own local import, since this is a different function's frame
+
+ labels = list(rec.labels_disputed)
+ n = len(labels)
+ if not n:
+ return entries
+
+ if not _interactive():
+ # Suppress the disputed labels and nothing else. Not the whole audit —
+ # that throws away a useful audit over a numbering the reader can check
+ # by hand — and not a reading picked on the user's behalf, which would
+ # record a choice nobody made. The evidence file is still written (no
+ # network, no model, no prompt): nobody was asked, but the next person
+ # to open this case folder should not have to re-run interactively just
+ # to see what disagreed.
+ path = _write_disagreement(case, labels, candidates, texts)
+ rec.choice, rec.chosen_by = "withheld", "default"
+ console.print(
+ f"[yellow]⚠ {n} citation label{'' if n == 1 else 's'} in dispute[/yellow] — "
+ f"{_label_list(labels)}. Not a tty, so nothing was asked: verdicts on claims "
+ "citing them are withheld and reported unchecked. Re-run in a terminal to "
+ f"settle them. [dim]full disagreement: {path}[/dim]"
+ )
+ return entries
+
+ # Step 1, unskippable: the file exists before the first question is asked.
+ path = _write_disagreement(case, labels, candidates, texts)
+ console.print(
+ f"\n[yellow]⚠ {n} citation label{'' if n == 1 else 's'} in dispute[/yellow] — "
+ f"{_label_list(labels)}"
+ )
+ for label in labels:
+ for name in sorted(candidates):
+ hits = [e for e in candidates[name] if e.num == label]
+ shown = hits[0].raw[:70] if hits else "— no entry —"
+ console.print(f" [{label:>3}] [cyan]{name:<8}[/cyan] {shown}")
+ console.print(f" [dim]full disagreement, with the text each reading came from:[/dim] {path}")
+
+ offered = [r for r in ("parsed", "pymupdf") if r in candidates]
+ console.print(
+ f"\n1. Ask the model to resolve the {n} disputed label{'' if n == 1 else 's'} "
+ "[dim][default][/dim]\n"
+ f"2. Withhold verdicts on all {n}\n"
+ f"3. Use one reading whole: {' / '.join(offered)}\n"
+ "4. Abort"
+ )
+ answer = Prompt.ask(" choice", choices=["1", "2", "3", "4"], default="1")
+
+ if answer == "4":
+ # nothing written: a manifest describing a numbering the user walked
+ # away from is worse than no manifest, because every later stage trusts it
+ console.print("[dim]aborted — nothing was written.[/dim]")
+ raise typer.Exit(1)
+
+ if answer == "2":
+ rec.choice, rec.chosen_by = "withheld", "user"
+ return entries
+
+ if answer == "3":
+ pick = Prompt.ask(" which reading", choices=offered, default=offered[0])
+ rec.choice, rec.chosen_by = pick, "user"
+ rec.labels_resolved = sorted(labels, key=int)
+ rec.labels_disputed = []
+ rec.note += (
+ f"; you chose {_READING_LABEL.get(pick, pick)} for the {n} label"
+ f"{'' if n == 1 else 's'} in dispute. That is your reading, not a check of it "
+ "— the numbering is still unconfirmed"
+ )
+ return list(candidates[pick])
+
+ from .reflist import resolve_disputed
+
+ try:
+ res = resolve_disputed(labels, candidates, tuple(texts[k] for k in sorted(texts)),
+ model=model)
+ except Exception as e: # noqa: BLE001 — same shape as `_llm_reference_reading`: the refs
+ # stage has already done useful work and must not go down with a failed
+ # subprocess call. Every disputed label is left exactly where it was —
+ # disputed and withheld — and the failure is named rather than swallowed.
+ # `chosen_by` stays "user": a person did consent to trying, even though
+ # the attempt itself produced nothing.
+ console.print(
+ f"[yellow]⚠ the model could not resolve the disputed label"
+ f"{'' if n == 1 else 's'} — the call did not return: "
+ f"{type(e).__name__}: {str(e)[:200]}[/yellow]"
+ )
+ rec.choice, rec.chosen_by = "withheld", "user"
+ rec.note += (
+ f"; asked a model to resolve the {n} disputed label{'' if n == 1 else 's'} and "
+ "the call failed, so they remain withheld"
+ )
+ return entries
+
+ rec.choice, rec.chosen_by = "llm_resolved", "user"
+ rec.labels_resolved = sorted(res.resolved, key=int)
+ rec.labels_disputed = sorted(res.still_disputed, key=int)
+ done, left = len(rec.labels_resolved), len(rec.labels_disputed)
+ console.print(
+ f" [green]{done} resolved[/green]"
+ + (f" · [yellow]{left} still disputed[/yellow]" if left else "")
+ )
+ if left:
+ # partial resolution is the normal outcome, and the abstentions are named
+ # because they are the labels a reader should check by hand
+ console.print(
+ f" [dim]{_label_list(rec.labels_disputed)} — the model could not tell which "
+ "paper these name, so their verdicts stay withheld.[/dim]"
+ )
+ rec.note += (
+ f"; {done} of the {n} disputed label{'' if n == 1 else 's'} were resolved by a "
+ "model reading of both texts, with every field verified verbatim, and you "
+ "accepted that. It is a reading, not a confirmation — the numbering is still "
+ "unconfirmed"
+ )
+ # The resolved entry REPLACES the one in the chosen list, in place, same
+ # order, same length. Returning `entries` unchanged here takes the label out
+ # of `labels_disputed` — so `check` judges it again — while leaving the
+ # entry the resolution just ruled AGAINST as the paper it judges against.
+ # That is the original wrong-paper bug reached through the one path a user
+ # consented to, under a report line saying they accepted it.
+ return [res.resolved.get(e.num, e) for e in entries]
+
+
def _refs_pipeline(
*,
manuscript: Path,
@@ -905,7 +1164,19 @@ def _refs_pipeline(
)
stamp_seen_in(entries, others)
- if rec.verified:
+ if rec.verified and rec.labels_disputed:
+ # `numbering_verified` means EXTENT — the chosen list accounts for
+ # exactly the labels the body cites — and a disputed label is a question
+ # of CONTENT, which no count can answer. Both can be true at once, and
+ # the flag's published meaning is not widened to cover the second (that
+ # would redefine a boolean four report formats already read). What must
+ # not happen is this line saying "confirmed" beside the line above
+ # withholding verdicts: whichever of the two a reader believed, the
+ # other would be a lie.
+ console.print(
+ f"[green]✓ numbering accounts for every cited label[/green] — {rec.note}"
+ )
+ elif rec.verified:
console.print(f"[green]✓ numbering confirmed[/green] — {rec.note}")
else:
console.print(f"[yellow]⚠ numbering unconfirmed[/yellow] — {rec.note}")
@@ -923,6 +1194,15 @@ def _refs_pipeline(
console.print(f" [{e.num:>3}] {e.raw[:90]}")
return
+ # After --parse-only returns: the escalation can spend a model call, and
+ # --parse-only promises no network. Only "parsed" and "pymupdf" carry a
+ # printed text to show a reader — "crossref" and "llm" are voters with no
+ # separate span of the page, so they are omitted here rather than passed as "".
+ reading_texts: dict[str, str] = {"parsed": refs_text}
+ if needs_flat:
+ reading_texts["pymupdf"] = flat_text
+ entries = _escalate_disputed(case, rec, entries, others, reading_texts, model=None)
+
if provided is not None and provided.is_file():
console.print(
f"[red]--provided expects a folder of PDFs, got a file:[/red] {provided}\n"
@@ -979,6 +1259,9 @@ def tick(e):
numbering_corroborated=rec.corroborated,
corroborating_readings=rec.corroborating_readings,
labels_disputed=rec.labels_disputed,
+ labels_resolved=rec.labels_resolved,
+ numbering_choice=rec.choice,
+ numbering_chosen_by=rec.chosen_by,
reflist_outcome=reflist_prov.outcome,
reflist_failure=reflist_prov.failure,
reflist_model=reflist_prov.model,
diff --git a/src/papertrace/disclosures.py b/src/papertrace/disclosures.py
index a2907a3..29704ac 100644
--- a/src/papertrace/disclosures.py
+++ b/src/papertrace/disclosures.py
@@ -53,12 +53,15 @@
# carries one passes here and fails the parity test that asserts it verbatim
# in all four — "reference list's" was tried first and had to be reworded.
LABELS_DISPUTED_TOKEN = "labels where the reference list readings disagree"
-# `claim_pairing` gains three more states in Tasks 6 and 7 (resolved by a
-# model call, chosen by the user, corroborated by a second agreeing reading) —
-# each needs its own token, the same way ANCHOR_LOCATED_TOKEN /
-# ANCHOR_NOT_LOCATED_TOKEN / ANCHOR_UNKNOWN_TOKEN are one Disclosure key with
-# one token per state actually reached. Only this state's token exists yet.
+# `claim_pairing` gains three more states in Task 7 (resolved by a model call,
+# chosen by the user, corroborated by a second agreeing reading) — each needs
+# its own token, the same way ANCHOR_LOCATED_TOKEN / ANCHOR_NOT_LOCATED_TOKEN /
+# ANCHOR_UNKNOWN_TOKEN are one Disclosure key with one token per state actually
+# reached.
CLAIM_PAIRING_WITHHELD_TOKEN = "verdict withheld — the reference list readings disagree"
+CLAIM_PAIRING_RESOLVED_TOKEN = "resolved by a model reading of both texts, accepted by you"
+CLAIM_PAIRING_CHOSEN_TOKEN = "you chose which reading of the reference list to use"
+CLAIM_PAIRING_CORROBORATED_TOKEN = "agreed by every reading of the reference list"
# no apostrophe, `&`, `<` or `>` in either — same HTML-autoescape rule as
# LABELS_DISPUTED_TOKEN above, since both are asserted verbatim in two formats
# that escape and two that do not
@@ -712,6 +715,11 @@ def _numbering_corroboration(manifest) -> Disclosure | None:
)
+def _label_group(labels: list[str]) -> str:
+ """`["4", "5"]` -> `[4], [5]` — a citation-label group as the reader sees it."""
+ return f"[{'], ['.join(labels)}]"
+
+
def _claim_numbering(claim, manifest) -> Disclosure:
"""The run-level warning, said again where the verdict is read.
@@ -723,7 +731,7 @@ def _claim_numbering(claim, manifest) -> Disclosure:
(r for r in claim.refs if manifest.label_is_doubtful(r)),
key=lambda r: int(r),
)
- labels = f"[{'], ['.join(doubtful)}]"
+ labels = _label_group(doubtful)
return Disclosure(
key="claim_numbering",
level="warn",
@@ -1060,18 +1068,25 @@ def _no_quote(claim) -> Disclosure:
)
-def _claim_pairing(claim) -> Disclosure | None:
+def _claim_pairing(claim, manifest=None) -> Disclosure | None:
"""Which pairing state this claim's cited labels are in, if not a clean one.
- Only `withheld` exists yet — `withheld_refs` is the only field a pairing
- state has written to `ClaimResult` so far. Tasks 6 and 7 add the
- model-resolved, user-chosen and corroborated states as further branches
- reading their own new fields, never a second function — so `CLAIM_KEYS`
- keeps exactly one claim-level entry covering all four.
+ Four states, four tokens — the same one-key-many-tokens shape `anchor`
+ uses. `withheld` reads only `claim.withheld_refs`, so it fires with no
+ `manifest` at all (as it always has); the other three read the manifest's
+ escalation fields and are silent without one. `corroborated` is the only
+ `info`-level state, and `claim_disclosures` gates it on the claim already
+ carrying a `warn`: reassurance repeated on every clean claim is noise a
+ reader stops reading.
"""
- if claim.withheld_refs:
- labels = f"[{'], ['.join(claim.withheld_refs)}]"
- one = len(claim.withheld_refs) == 1
+ cited = set(claim.refs)
+ withheld = sorted(cited & set(claim.withheld_refs), key=int)
+ disputed = sorted(cited & set(getattr(manifest, "labels_disputed", None) or []), key=int)
+ resolved = sorted(cited & set(getattr(manifest, "labels_resolved", None) or []), key=int)
+
+ if withheld or disputed:
+ labels = _label_group(withheld or disputed)
+ one = len(withheld or disputed) == 1
return Disclosure(
key="claim_pairing",
level="warn",
@@ -1086,11 +1101,57 @@ def _claim_pairing(claim) -> Disclosure | None:
),
short=f"{labels} {CLAIM_PAIRING_WITHHELD_TOKEN}",
)
- # Task 6: `resolved_by_model` — the resolution call named a reading and the
- # default (non-interactive) run accepted it.
- # Task 7: `chosen_by_user` — an interactive session picked one reading
- # whole; `corroborated`, level="info", fires only when the claim already
- # carries a warn-level disclosure, so a clean claim stays silent.
+
+ if resolved and manifest is not None and manifest.numbering_choice == "llm_resolved":
+ labels = _label_group(resolved)
+ return Disclosure(
+ key="claim_pairing",
+ level="warn",
+ token=CLAIM_PAIRING_RESOLVED_TOKEN,
+ text=(
+ f"This claim cites {labels}, {CLAIM_PAIRING_RESOLVED_TOKEN}. Two readings "
+ "of the bibliography disagreed there. A model was shown both, every field "
+ "of its answer was found verbatim in one of them, and a person accepted "
+ "the result — so this verdict rests on a reading nobody checked against "
+ "the printed page, chosen by a person. The numbering is still recorded as "
+ "unconfirmed."
+ ),
+ short=f"{labels} {CLAIM_PAIRING_RESOLVED_TOKEN}",
+ )
+
+ if resolved and manifest is not None and manifest.numbering_choice in ("parsed", "pymupdf"):
+ labels = _label_group(resolved)
+ return Disclosure(
+ key="claim_pairing",
+ level="warn",
+ token=CLAIM_PAIRING_CHOSEN_TOKEN,
+ text=(
+ f"This claim cites {labels}. Two readings of the bibliography disagreed "
+ f"there and {CLAIM_PAIRING_CHOSEN_TOKEN} — the `{manifest.numbering_choice}` "
+ "one, for every disputed label. That is an assertion about which reading "
+ "is right, not a check of it, and this verdict is about whichever paper "
+ "that reading names."
+ ),
+ short=f"{labels} {CLAIM_PAIRING_CHOSEN_TOKEN} ({manifest.numbering_choice})",
+ )
+
+ # `info`, and gated by the caller on this claim already carrying a `warn`:
+ # corroboration is reassurance, and reassurance on a clean claim is noise
+ if manifest is not None and getattr(manifest, "numbering_corroborated", False) and cited:
+ labels = _label_group(sorted(cited, key=int))
+ readings = ", ".join(manifest.corroborating_readings)
+ return Disclosure(
+ key="claim_pairing",
+ level="info",
+ token=CLAIM_PAIRING_CORROBORATED_TOKEN,
+ text=(
+ f"{labels} was {CLAIM_PAIRING_CORROBORATED_TOKEN} ({readings}). That is "
+ "evidence for this pairing, not confirmation of it — every reading read "
+ "the same document, so a reference the layout destroyed is one they may "
+ "all have missed."
+ ),
+ short=f"{labels} {CLAIM_PAIRING_CORROBORATED_TOKEN}",
+ )
return None
@@ -1119,8 +1180,14 @@ def claim_disclosures(claim, manifest=None) -> list[Disclosure]:
out.append(_supplement_headline(claim))
if manifest is not None and any(manifest.label_is_doubtful(r) for r in claim.refs):
out.append(_claim_numbering(claim, manifest))
- if (d := _claim_pairing(claim)) is not None:
- out.append(d)
+ # `claim_pairing` at info level (corroboration) is reassurance, and
+ # reassurance on a claim with nothing else to say about it is noise on
+ # every row of the report — so it appends only beside an existing `warn`.
+ # A `warn`-level pairing disclosure is unconditional, `manifest` or not:
+ # `withheld` reads only `claim.withheld_refs`.
+ if (d := _claim_pairing(claim, manifest)) is not None:
+ if d.level == "warn" or any(x.level == "warn" for x in out):
+ out.append(d)
if (d := anchor_disclosure(claim)) is not None:
out.append(d)
return out
diff --git a/src/papertrace/reflist.py b/src/papertrace/reflist.py
index 4fd8a4f..c040c7c 100644
--- a/src/papertrace/reflist.py
+++ b/src/papertrace/reflist.py
@@ -397,3 +397,209 @@ def _numeral_gaps(nums: list[str]) -> list[str]:
return []
present = set(numeric)
return [str(n) for n in range(1, max(numeric) + 1) if n not in present]
+
+
+# ---------------------------------------------------------------------------
+# resolving a disputed label — a person has already seen the disagreement
+# (`cli._write_disagreement`) and asked for a model's opinion on it
+# ---------------------------------------------------------------------------
+
+RESOLVE_PROMPT = """You are shown two readings of one printed bibliography, and a list of
+citation labels the two readings disagree about — they name different papers at
+that label.
+
+For each label, decide which paper the label actually names, using ONLY the two
+texts below. Reply with a JSON array, one object per label you can answer:
+
+[{"num": "6",
+ "title": "the paper's title, copied character for character from one of the texts",
+ "authors": "copied character for character, or omit",
+ "year": "copied character for character, or omit",
+ "journal": "copied character for character, or omit",
+ "doi": "copied character for character, or omit",
+ "seen_in": "A" or "B"}]
+
+If you cannot tell which paper a label names, reply for it with
+{"num": "6", "cannot_tell": true}. THAT IS A CORRECT ANSWER and is preferred
+over a guess: these are exactly the labels where two readings contradict each
+other, so a guess here is the error this tool exists to prevent. Answering some
+labels and abstaining on others is expected.
+
+Every value you give is checked against the two texts verbatim. A value that is
+not printed in one of them is discarded, and a title that is not printed leaves
+the label unresolved — so do not correct, expand, complete or tidy anything.
+
+LABELS IN DISPUTE: <>
+
+--- READING A ---
+<>
+
+--- READING B ---
+<>
+"""
+
+
+@dataclass
+class Resolution:
+ """What the resolution call settled, what it did not, and on whose words.
+
+ `still_disputed` is not the complement of `resolved` by arithmetic — it is
+ built by listing what came back unanswered, abstained on, or unverifiable.
+ Deriving it as `set(asked) - set(resolved)` gives the same answer today and
+ would silently start counting a dropped label as resolved the first time a
+ field-verification branch forgets to record one.
+ """
+
+ resolved: dict[str, RefEntry] = field(default_factory=dict)
+ still_disputed: list[str] = field(default_factory=list)
+ provenance: ReflistProvenance = field(default_factory=ReflistProvenance)
+
+
+def _parse_array(raw: str) -> list[dict] | None:
+ """A lenient JSON-array read for a resolution reply: `None` on anything
+ that is not a list of objects, never a raise.
+
+ `resolve_disputed` has already done useful refs-stage work by the time this
+ runs, and a reply shaped wrong must leave every label in the state the run
+ was already in, not take the stage down with it. Requiring every element to
+ be a dict — not just the outer shape to be a list — is what tells a genuine
+ reply apart from a stray bracketed numeral inside a sentence of prose:
+ `ask._parse_json_array`'s first-`[`-to-last-`]` slice reads
+ "I think maybe [6]?" as the syntactically valid one-element list `[6]`, and
+ without this check that would be believed as an answer instead of refused.
+ """
+ try:
+ items = ask._parse_json_array(raw)
+ except ValueError:
+ return None
+ if not isinstance(items, list) or not all(isinstance(x, dict) for x in items):
+ return None
+ return items
+
+
+# only fields the model is asked to copy, in the order `raw` is re-assembled —
+# "doi" is verified separately below, through `_usable_doi`, not this loop
+_RESOLVE_FIELDS = ("authors", "year", "journal")
+
+
+def resolve_disputed(
+ labels: list[str],
+ entries: dict[str, list[RefEntry]],
+ texts: tuple[str, ...],
+ *,
+ model: str | None = None,
+) -> Resolution:
+ """Ask the model which paper each disputed label names, and believe none of it on its word.
+
+ Three rules, in this order, and the second is the one that keeps this call
+ honest: **abstention is a first-class answer.** `cannot_tell` keeps the
+ label disputed and withheld. A call that must always answer will confabulate
+ on exactly the labels two readings disagree about — that is the population,
+ by construction, and it is why this returns a `Resolution` with two lists
+ rather than a dict of answers.
+
+ 1. Every field is verified verbatim against `texts` through `_found` (Layer
+ E for everything, Layer O for authors and titles) and, for a DOI,
+ `_usable_doi`. An unverifiable field is discarded; an unverifiable title
+ leaves the label disputed, because the title is the only field that
+ identifies a paper.
+ 2. `cannot_tell`, a label the reply never mentions, and a reply that is not
+ a JSON array of objects at all all leave their labels disputed.
+ 3. Partial resolution is the normal outcome, not a degraded one. Nothing
+ downstream may require all-or-nothing.
+
+ `entries` names each candidate reading, and its sorted keys are read in the
+ same order the one caller (`cli._escalate_disputed`) builds `texts` in — so
+ a reply's `seen_in: "A"/"B"` is translated into an actual reading NAME
+ before it reaches `RefEntry.seen_in`, never stored as the bare letter.
+ `RefEntry.seen_in`'s published vocabulary is reading names (`propose`
+ enforces the same rule for its own "A"/"B"/"AB"), and a letter leaking
+ through would put two vocabularies in one wire-format key.
+ """
+ prov = ReflistProvenance(readings=list(texts))
+ if not labels:
+ return Resolution(provenance=prov)
+
+ reading_names = sorted(entries)
+ prompt = (
+ RESOLVE_PROMPT.replace("<>", ", ".join(f"[{n}]" for n in labels))
+ .replace("<>", texts[0] if texts else "")
+ .replace("<>", texts[1] if len(texts) > 1 else "")
+ )
+ with ask.for_site(ask.SITE_REFS):
+ raw = ask._ask(prompt, model)
+ prov.model = ask.model_for(ask.SITE_REFS) or ""
+
+ items = _parse_array(raw)
+ if items is None:
+ # A reply that is not a JSON array of objects has answered nothing.
+ # Every label stays in the state the run was already in — the safe
+ # one — and the reason is recorded rather than the failure being
+ # indistinguishable from an all-abstention reply.
+ prov.discarded_whole = (
+ "the reply was not a JSON array of objects, so no label was resolved"
+ )
+ return Resolution(still_disputed=list(labels), provenance=prov)
+
+ by_num = {str(o.get("num", "")).strip(): o for o in items}
+ prov.entries_proposed = len(by_num)
+ resolved: dict[str, RefEntry] = {}
+ unresolved: list[str] = []
+ for label in labels:
+ obj = by_num.get(label)
+ if obj is None or obj.get("cannot_tell"):
+ unresolved.append(label)
+ continue
+
+ title = str(obj.get("title") or "")
+ if not _found("title", title, texts):
+ # not a discarded FIELD — a discarded ANSWER: nothing else in the
+ # object can name a paper on its own, so the whole label stays
+ # disputed rather than resolving on the strength of its neighbours
+ prov.fields_discarded.append(f"[{label}].title")
+ unresolved.append(label)
+ continue
+
+ kept: dict[str, str] = {"title": title}
+ for name in _RESOLVE_FIELDS:
+ value = str(obj.get(name) or "")
+ if not value:
+ continue # an omitted field is a gap the model was told to leave
+ if _found(name, value, texts):
+ kept[name] = value
+ else:
+ prov.fields_discarded.append(f"[{label}].{name}")
+
+ doi = _usable_doi(str(obj.get("doi") or ""))
+ if doi is not None and not _found("doi", doi, texts):
+ doi = None
+ if obj.get("doi") and doi is None:
+ # bare "doi", not "[label].doi" — a recorded gap the entry's
+ # `no_doi` resolution will independently rediscover
+ prov.fields_discarded.append("doi")
+ elif doi is not None:
+ kept["doi"] = doi
+
+ parts = [kept[k] for k in ("authors", "year", "title", "journal") if k in kept]
+ raw_text = ". ".join(parts)
+ if "doi" in kept:
+ raw_text = f"{raw_text} doi:{kept['doi']}".strip()
+
+ seen_in: list[str] = []
+ letter = obj.get("seen_in")
+ if isinstance(letter, str):
+ idx = {"A": 0, "B": 1}.get(letter.strip().upper())
+ if idx is not None and idx < len(reading_names):
+ seen_in = [reading_names[idx]]
+
+ resolved[label] = RefEntry(
+ num=label,
+ raw=raw_text,
+ doi=kept.get("doi"),
+ title=kept.get("title"),
+ year=kept.get("year"),
+ reason="a disputed label, resolved by a model reading of both texts and "
+ "accepted by a person — a reading, not a confirmed numbering",
+ seen_in=seen_in,
+ )
+ return Resolution(resolved=resolved, still_disputed=unresolved, provenance=prov)
diff --git a/tests/test_escalation.py b/tests/test_escalation.py
new file mode 100644
index 0000000..85f9af2
--- /dev/null
+++ b/tests/test_escalation.py
@@ -0,0 +1,393 @@
+"""The disagreement is shown before the question is asked.
+
+A reader who is offered a choice between two readings of a bibliography and has
+not been shown either one is being asked to rubber-stamp, not to decide. So
+step one writes the whole disagreement to a file and prints the path, and it is
+unskippable — `test_the_disagreement_is_written_before_the_question_is_asked`
+is that requirement encoded, and it fails if the two are ever reordered.
+
+Driven through `_refs_pipeline` rather than through `_escalate_disputed`
+directly for most of these: the thing under test is that a REAL run reaches the
+file before it reaches the prompt, and a unit test of the escalation alone
+cannot see the ordering the user asked for. `_reference_readings` is real and
+unmocked — it already returns a single dict, matching every other consumer of
+it (`tests/test_reference_readings.py`), so the two extra readings are made to
+disagree by forcing `_needs_flat_reading` True (the same technique that
+module's own `_wire_offline` uses) and by fabricating the flat-text extraction,
+never by pretending the pure dict-builder returns something it does not.
+"""
+
+import sys
+from pathlib import Path
+
+import pymupdf
+import pytest
+import typer
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+
+from papertrace import cli # noqa: E402
+from papertrace import reflist as reflist_mod # noqa: E402
+from papertrace.models import RefManifest # noqa: E402
+from papertrace.refs import _entry # noqa: E402
+
+# --- the interactive escalation: shown first, then offered ------------------
+
+
+def _paper(path: Path) -> Path:
+ """A one-page PDF citing [1] and [2], both printed in the References section.
+
+ The disagreement at [2] is injected by fabricating the flat-text reading
+ below — the point of a real PDF is that `_refs_pipeline` runs its whole
+ length, ingest included, rather than being handed a fake `SourceMap`.
+ """
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Agreement in CT of the Pancreas", fontsize=16)
+ page.insert_text((72, 140), "Body text citing [1] and [2] here.", fontsize=11)
+ page.insert_text((72, 200), "References", fontsize=14)
+ page.insert_text((72, 230), "[1] Alpha A. First paper. 2020.", fontsize=11)
+ page.insert_text((72, 250), "[2] Beta B. Second paper. 2021.", fontsize=11)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ doc.save(path)
+ doc.close()
+ return path
+
+
+# The fabricated second reading of the same bibliography: agrees at [1], and
+# names a different paper at [2]. `references_span_flat` is faked to return
+# this rather than left real, because on a real pymupdf-backend ingest it is
+# "identical to the existing parse on a pymupdf one" (its own docstring) — the
+# self-corroboration case `_needs_flat_reading` exists to skip. Forcing the
+# predicate without also controlling the text it feeds would make "parsed" and
+# "pymupdf" agree on everything, and there would be no disagreement to show.
+FLAT_TEXT = (
+ "References\n"
+ "[1] Alpha A. First paper. 2020.\n"
+ "[2] Gamma G. A wholly different paper. 2019.\n"
+)
+
+
+@pytest.fixture()
+def disputed(tmp_path, monkeypatch):
+ """A run that reaches the escalation with [2] disputed, and nothing else stubbed.
+
+ Returns `(pdf, case, answers)`; the caller fills `answers` with the menu
+ keystrokes it wants `Prompt.ask` to return.
+ """
+ import papertrace.ingest as ingest_pkg
+ import papertrace.refs as refs_mod
+ from papertrace.refs import CROSSREF_NO_DOI, CrossrefDeposit
+
+ monkeypatch.setattr(refs_mod, "resolve_all",
+ lambda entries, dest, email, provided_dir=None, progress=None,
+ taken=None: entries)
+ monkeypatch.setattr(refs_mod, "crossref_deposit",
+ lambda client, doi, email: CrossrefDeposit(absent=CROSSREF_NO_DOI))
+ monkeypatch.setattr(cli, "_detected_doi", lambda m: None)
+ # forces the predicate, not the backend — same technique
+ # `tests/test_reference_readings.py`'s `_wire_offline` uses, so a pymupdf
+ # ingest (fast, no docling download) still gets a second reading to compare
+ monkeypatch.setattr(cli, "_needs_flat_reading", lambda smap: True)
+ monkeypatch.setattr(ingest_pkg, "references_span_flat",
+ lambda pdf_path: (FLAT_TEXT, False))
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+
+ answers: list[str] = []
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: answers.pop(0)))
+ return _paper(tmp_path / "paper.pdf"), tmp_path / "case", answers
+
+
+def _run_refs(pdf: Path, case: Path) -> RefManifest:
+ # llm_refs=False: an offline test must not risk a live `claude -p` call even
+ # on a machine that happens to have the CLI on PATH — the brief's own draft
+ # of this fixture omitted it, which would have made every test below spend
+ # money (or hang) on a host where `claude` is installed.
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None,
+ email="t@example.org", parse_only=False, backend="pymupdf",
+ doi=None, supplement=None, llm_refs=False)
+ return RefManifest.from_json(case / "refs_manifest.json")
+
+
+def test_the_disagreement_is_written_before_the_question_is_asked(disputed, monkeypatch):
+ """The user's instruction, encoded as a test: show, then offer.
+
+ The fake `Prompt.ask` asserts the file is already on disk when it is
+ called. That is the only way to pin the ordering — a test that checks the
+ file afterwards passes just as well on an implementation that asks first.
+ """
+ pdf, case, answers = disputed
+ path = case / "out" / "reference_disagreement.md"
+ seen = {}
+
+ def ask(*a, **k):
+ seen["existed"] = path.exists()
+ seen["text"] = path.read_text(encoding="utf-8") if path.exists() else ""
+ return "2"
+
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(ask))
+ _run_refs(pdf, case)
+
+ assert seen["existed"] is True, "the menu was reached before the evidence was written"
+ # both readings of the disputed label, and the text each was read from
+ assert "Beta B. Second paper" in seen["text"]
+ assert "Gamma G. A wholly different paper" in seen["text"]
+ assert "[2]" in seen["text"]
+ # and not a word about the label the readings agree on being a finding —
+ # its own printed-list context is fine, its structured fields are not shown
+ assert "First paper" in seen["text"], "the agreed label's text is context, not a finding"
+
+
+def test_the_path_to_the_disagreement_file_is_printed(disputed, capsys):
+ """A file nobody is told about is a file nobody reads, and the menu's first
+ option spends money on a question the reader could answer by looking."""
+ pdf, case, answers = disputed
+ answers.append("2")
+ _run_refs(pdf, case)
+ assert "reference_disagreement.md" in capsys.readouterr().out
+
+
+def test_the_default_answer_asks_the_model_and_records_the_user_as_choosing(
+ disputed, monkeypatch
+):
+ """Option 1, taken by pressing return. `chosen_by` is `user` even on the
+ default: the person was shown the disagreement and consented to proceed,
+ which is an input to record, not evidence of anything. The resolution call
+ itself is mocked at `reflist.resolve_disputed` — a live model call has no
+ place in an offline test."""
+ pdf, case, answers = disputed
+ answers.append("1")
+ resolved_entry = _entry("2", "Beta B. Second paper. 2021. doi:10.1000/beta")
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, cands, texts, model=None: reflist_mod.Resolution(
+ resolved={"2": resolved_entry}, still_disputed=[],
+ provenance=reflist_mod.ReflistProvenance(model="claude-opus-5", outcome="read"),
+ ),
+ )
+ manifest = _run_refs(pdf, case)
+ assert manifest.numbering_choice == "llm_resolved"
+ assert manifest.numbering_chosen_by == "user"
+ # True, and correctly so: `numbering_verified` means the chosen list
+ # accounts for exactly the labels the body cites — a question of EXTENT,
+ # which this fixture's parse does answer. A disputed label is a question
+ # of CONTENT. The escalation may not touch this flag in either
+ # direction, and asserting it is unchanged is what pins that;
+ # `tests/test_numbering_invariant.py` proves no model path can set it
+ # where it is legitimately False.
+ assert manifest.numbering_verified is True
+ assert manifest.labels_resolved == ["2"]
+ assert manifest.labels_disputed == []
+ # option 1 substitutes the resolved entry — the label will be judged
+ # against the paper the resolution named, not the one it ruled against
+ two = next(e for e in manifest.entries if e.num == "2")
+ assert two.doi == "10.1000/beta"
+
+
+def test_a_resolver_failure_leaves_every_label_disputed_and_is_reported(disputed, monkeypatch):
+ """The resolution call can raise — `claude` not on PATH, a timeout, a
+ non-zero exit — the same shape `_llm_reference_reading` already has to
+ handle. `refs` has done useful work by this point and must not go down
+ with a failed subprocess call: every disputed label stays disputed and
+ withheld, and `chosen_by` is still "user" because a person did consent to
+ trying, even though the attempt produced nothing."""
+ pdf, case, answers = disputed
+ answers.append("1")
+
+ def boom(*a, **k):
+ raise RuntimeError("claude -p timed out after 600s")
+
+ monkeypatch.setattr(reflist_mod, "resolve_disputed", boom)
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.numbering_choice == "withheld"
+ assert manifest.numbering_chosen_by == "user"
+ assert manifest.labels_disputed == ["2"]
+ assert manifest.labels_resolved == []
+ # True, and correctly so: `numbering_verified` means the chosen list
+ # accounts for exactly the labels the body cites — a question of EXTENT,
+ # which this fixture's parse does answer. A disputed label is a question
+ # of CONTENT. The escalation may not touch this flag in either
+ # direction, and asserting it is unchanged is what pins that;
+ # `tests/test_numbering_invariant.py` proves no model path can set it
+ # where it is legitimately False.
+ assert manifest.numbering_verified is True
+
+
+def test_withholding_all_of_them_is_the_second_option(disputed):
+ pdf, case, answers = disputed
+ answers.append("2")
+ manifest = _run_refs(pdf, case)
+ assert manifest.numbering_choice == "withheld"
+ assert manifest.numbering_chosen_by == "user"
+ assert manifest.labels_disputed == ["2"]
+ assert manifest.labels_resolved == []
+
+
+def test_choosing_one_reading_whole_records_which_one(disputed):
+ """Option 3. Both offered readings are deterministic parses, so this is a
+ choice between two things the tool produced — never the model's list, which
+ is a voter only and can reach neither `reconcile` nor `resolve_all`."""
+ pdf, case, answers = disputed
+ answers += ["3", "pymupdf"]
+ manifest = _run_refs(pdf, case)
+ assert manifest.numbering_choice == "pymupdf"
+ assert manifest.numbering_chosen_by == "user"
+ # True, and correctly so: `numbering_verified` means the chosen list
+ # accounts for exactly the labels the body cites — a question of EXTENT,
+ # which this fixture's parse does answer. A disputed label is a question
+ # of CONTENT. The escalation may not touch this flag in either
+ # direction, and asserting it is unchanged is what pins that;
+ # `tests/test_numbering_invariant.py` proves no model path can set it
+ # where it is legitimately False.
+ assert manifest.numbering_verified is True
+ # the entries carried forward are that reading's
+ assert [e.num for e in manifest.entries] == ["1", "2"]
+ gamma = next(e for e in manifest.entries if e.num == "2")
+ assert "Gamma" in gamma.raw
+
+
+def test_aborting_writes_no_manifest(disputed):
+ """Option 4 leaves the case as it was. A half-written manifest describing a
+ numbering the user walked away from is worse than none."""
+ pdf, case, answers = disputed
+ answers.append("4")
+ with pytest.raises(typer.Exit):
+ cli._refs_pipeline(manuscript=pdf, case=case, provided=None,
+ email="t@example.org", parse_only=False, backend="pymupdf",
+ doi=None, supplement=None, llm_refs=False)
+ assert not (case / "refs_manifest.json").exists()
+
+
+def test_a_non_interactive_session_withholds_the_disputed_labels_without_asking(
+ disputed, monkeypatch
+):
+ """The user's decision: suppress only the disputed labels.
+
+ Not the whole audit — that throws away a useful audit over a numbering the
+ reader can check by hand — and not a reading picked on their behalf, which
+ would record a choice nobody made.
+ """
+ pdf, case, answers = disputed
+ monkeypatch.setattr(cli, "_interactive", lambda: False)
+
+ def boom(*a, **k):
+ raise AssertionError("a non-interactive run must not prompt")
+
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(boom))
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.numbering_choice == "withheld"
+ assert manifest.numbering_chosen_by == "default"
+ assert manifest.labels_disputed == ["2"]
+ # the agreed label keeps its verdicts — it is the disputed one that is dropped
+ assert "1" not in manifest.labels_disputed
+
+
+def test_a_closed_stdin_is_not_interactive(monkeypatch):
+ monkeypatch.setattr(sys.stdin, "isatty", lambda: False, raising=False)
+ monkeypatch.setattr(sys.stdout, "isatty", lambda: True, raising=False)
+ monkeypatch.delenv("CI", raising=False)
+ assert cli._interactive() is False
+
+
+def test_ci_is_not_interactive_even_on_a_tty(monkeypatch):
+ """CI runs on a tty often enough that the tty test alone is not the gate, and
+ a prompt in CI is a hung build, not a question."""
+ monkeypatch.setattr(sys.stdin, "isatty", lambda: True, raising=False)
+ monkeypatch.setattr(sys.stdout, "isatty", lambda: True, raising=False)
+ monkeypatch.setenv("CI", "true")
+ assert cli._interactive() is False
+
+
+def test_a_tty_with_no_ci_flag_is_interactive(monkeypatch):
+ monkeypatch.setattr(sys.stdin, "isatty", lambda: True, raising=False)
+ monkeypatch.setattr(sys.stdout, "isatty", lambda: True, raising=False)
+ monkeypatch.delenv("CI", raising=False)
+ assert cli._interactive() is True
+
+
+# --- `_escalate_disputed` directly: the one substitution rule -------------
+
+
+def _rec(**kw):
+ from papertrace.refs import Reconciliation
+
+ return Reconciliation(**kw)
+
+
+def test_a_resolved_label_judges_against_the_resolved_entry(tmp_path, monkeypatch):
+ """Un-disputing a label without substituting its entry is the original bug.
+
+ Menu option 1 takes a label out of `labels_disputed`, so `check` judges it
+ again. If the chosen list still holds the entry the resolution ruled
+ against, the verdict is printed against the very paper the model said was
+ wrong — and the report says the user accepted it. The resolved entries exist
+ in `Resolution.resolved`; the failure mode is computing them and throwing
+ them away.
+ """
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ resolved = _entry("13", "Weston AD (2019) The paper the label really names. "
+ "https://doi.org/10.1148/x13")
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, cands, texts, model=None: reflist_mod.Resolution(
+ resolved={"13": resolved}, still_disputed=[],
+ provenance=reflist_mod.ReflistProvenance(model="claude-opus-5"),
+ ),
+ )
+
+ rec = _rec(labels_disputed=["13"])
+ wrong = _entry("13", "Somebody Else (1999) A different paper entirely. "
+ "https://doi.org/10.1148/x99")
+ out = cli._escalate_disputed(
+ tmp_path, rec, [wrong], {"parsed": [wrong]}, {"parsed": "text"},
+ )
+
+ assert rec.labels_disputed == [] # the label will be judged again
+ assert out[0].doi == "10.1148/x13" # against the RESOLVED paper
+ assert rec.chosen_by == "user" # which is what authorises it
+ assert rec.verified is False # and it is still not confirmed
+
+
+def test_zero_disputed_labels_never_prompts(tmp_path, monkeypatch):
+ """An escalation with nothing to escalate is a no-op — even in a real
+ terminal, nobody should be asked a question about an empty list."""
+
+ def boom(*a, **k):
+ raise AssertionError("nothing was in dispute — there was no question to ask")
+
+ monkeypatch.setattr(cli, "_interactive", boom)
+ rec = _rec(labels_disputed=[])
+ entries = [_entry("1", "Alpha A. 2020.")]
+ out = cli._escalate_disputed(tmp_path, rec, entries, {"parsed": entries}, {"parsed": "text"})
+ assert out is entries
+ assert rec.choice == ""
+ assert rec.chosen_by == ""
+
+
+def test_a_run_never_says_confirmed_beside_a_withheld_label(disputed, capsys):
+ """The contradiction a partial Task 7 surfaced, pinned at the surface.
+
+ `numbering_verified` means the chosen list accounts for exactly the labels
+ the body cites — EXTENT. A disputed label is a question of CONTENT, which no
+ count answers, so both can be true at once and the flag keeps its published
+ meaning. What must not happen is one line saying the numbering is confirmed
+ while the line above it withholds verdicts on a label: whichever of the two
+ a reader believed, the other would be a lie.
+
+ The first attempt at this was to un-verify the numbering, which broke two of
+ Plan A's merged tests — correctly, because it silently widened a boolean
+ that four report formats already read. The fix belongs in the sentence.
+ """
+ pdf, case, answers = disputed
+ answers.append("2")
+ manifest = _run_refs(pdf, case)
+ out = capsys.readouterr().out
+
+ assert manifest.numbering_verified is True
+ assert manifest.labels_disputed == ["2"]
+ assert "readings disagree at [2]" in out
+ assert "numbering confirmed" not in out, "confirmed printed beside a withheld label"
+ assert "accounts for every cited label" in out
diff --git a/tests/test_refs.py b/tests/test_refs.py
index 60ff2c7..3944bcd 100644
--- a/tests/test_refs.py
+++ b/tests/test_refs.py
@@ -1008,3 +1008,22 @@ def test_folding_does_not_invent_or_merge_tokens():
"anatomic",
"structures",
}
+
+
+def test_a_partial_resolution_round_trips_through_the_manifest(tmp_path):
+ """11 resolved and 3 still disputed is a representable state, and the wire
+ format is the dataclass — so this is the test that says the two lists can
+ both be non-empty at once (Gate 2 for `numbering_choice`,
+ `numbering_chosen_by` and `labels_resolved`)."""
+ from papertrace.models import RefManifest
+
+ m = RefManifest(manuscript="p.pdf", labels_resolved=["6", "7"], labels_disputed=["9"],
+ numbering_choice="llm_resolved", numbering_chosen_by="user")
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ back = RefManifest.from_json(path)
+ assert back.labels_resolved == ["6", "7"]
+ assert back.labels_disputed == ["9"]
+ assert back.numbering_choice == "llm_resolved"
+ assert back.numbering_chosen_by == "user"
+ assert back.numbering_verified is False
From 8f45c8a368e7e41fcc55bf98df88bf648650e937 Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Fri, 18 Sep 2026 14:35:49 +0200
Subject: [PATCH 30/44] refs: a zero-or-empty value cannot mean both measured
and never measured
Review round 3. reflist_entries_proposed, corroborating_readings and the
console's reply-of-[] line all asked the same empty value to carry two
meanings -- exactly the class table_warnings was already documented against
("a list, [], and null -- nobody watched").
reflist_entries_proposed is now int | None: None means never recorded, 0
means measured and zero. A round-1 manifest that recorded discarded fields
but never this count no longer reads as "proposed no entries at all", which
contradicted its own short text in the repo's own fixture.
corroborating_readings can legitimately name 0 or 1 readings even when
numbering_corroborated is True (it's a per-reading fact, corroborated is
per-label) -- round 2's intersection made that state real. Fixed the
sentence, not the field: the console and the disclosure now gate the "N
readings agree" claim on the list actually having two names, under a
different token when it doesn't, rather than redefine what
numbering_corroborated means (the thing the hard constraint this round
added says not to do).
An unrecognised outcome now fails closed instead of reading as success; the
propose guard names itself instead of blaming a model that was never asked;
the report strips the same prefix the console already does; numbering
findings stop appending to a call that never returned.
Walking the four-axis table (live/loaded, written-by-an-earlier-commit)
past the rows the review named surfaced one more: a round-1-shaped "not
obtained" note, loaded from a case folder, was being read as a discarded
value by this round's own fix for the silence bug -- reopening N1 on the
load path. Split the normalization to tell a real discarded value apart
from round 1's own failure prefixes.
---
schemas/refs_manifest.schema.json | 6 +-
src/papertrace/cli.py | 34 ++-
src/papertrace/disclosures.py | 149 +++++++++--
src/papertrace/models.py | 19 +-
src/papertrace/reflist.py | 8 +-
tests/test_reference_readings.py | 403 +++++++++++++++++++++++++++++-
6 files changed, 584 insertions(+), 35 deletions(-)
diff --git a/schemas/refs_manifest.schema.json b/schemas/refs_manifest.schema.json
index d4daa8a..02044ad 100644
--- a/schemas/refs_manifest.schema.json
+++ b/schemas/refs_manifest.schema.json
@@ -281,7 +281,7 @@
"reflist_outcome": {
"type": "string",
"enum": ["", "not_attempted", "failed", "read"],
- "description": "Which of the three real outcomes this run's model reading of the bibliography reached: 'not_attempted' (no subprocess call was ever made — --llm-refs off, --parse-only, the backend left no second reading to compare against, or claude not on PATH), 'failed' (a call was made and did not return an answer), or 'read' (a reply was obtained, whether or not anything in it survived verification). Absent or '' means NEVER COMPUTED, not 'not_attempted' — a manifest written before this field existed asserts nothing about whether a call was made. A two-valued flag could not tell 'never called' apart from 'called and failed', which is exactly the bug this field replaced: a failed call published as a successful reading."
+ "description": "Which of the three real outcomes this run's model reading of the bibliography reached: 'not_attempted' (no subprocess call was ever made — --llm-refs off, --parse-only, the backend left no second reading to compare against, claude not on PATH, or the flat-text reading itself came back with no text to check the model's own reading against), 'failed' (a call was made and did not return an answer), or 'read' (a reply was obtained, whether or not anything in it survived verification). Absent or '' means NEVER COMPUTED, not 'not_attempted' — a manifest written before this field existed asserts nothing about whether a call was made. A two-valued flag could not tell 'never called' apart from 'called and failed', which is exactly the bug this field replaced: a failed call published as a successful reading."
},
"reflist_failure": {
"type": "string",
@@ -306,8 +306,8 @@
}
},
"reflist_entries_proposed": {
- "type": "integer",
- "description": "How many entries the model's reply proposed, before verification, whenever `reflist_outcome` is 'read' — including 0. Absent or 0 on a manifest where `reflist_outcome` is not 'read' means never computed, not 'proposed nothing'. Without this a reader cannot tell a model that proposed no entries at all from one that proposed several and had none discarded — both otherwise print '0 fields discarded'."
+ "type": ["integer", "null"],
+ "description": "How many entries the model's reply proposed, before verification, whenever `reflist_outcome` is 'read' — including 0. `null` (or absent) means NEVER RECORDED — either no reading happened, or this manifest was written before this field existed — and is NOT the same as `0`, which means a reading happened and measured zero proposed entries. A manifest that recorded `reflist_model` or `reflist_fields_discarded` without this field is exactly the case `null` exists for: without the distinction, a model that proposed nothing was told apart from one whose count was simply never recorded, which this field alone can settle."
}
}
}
diff --git a/src/papertrace/cli.py b/src/papertrace/cli.py
index ec33810..e083979 100644
--- a/src/papertrace/cli.py
+++ b/src/papertrace/cli.py
@@ -1099,6 +1099,17 @@ def _refs_pipeline(
f"the reference list was also read by [bold]{named_model}[/bold], "
f"and its reading was [yellow]discarded[/yellow] — {whole}"
)
+ elif reflist_prov.entries_proposed == 0:
+ # NF3: the exact mirror of the `whole` branch above, in the other
+ # direction — a reply of `[]` is a real reply, not a refused one,
+ # but "0 fields discarded as not printed — a second reading, not
+ # confirmation" reads as a clean corroboration from a model that
+ # proposed nothing to corroborate anything with
+ console.print(
+ f"the reference list was also read by [bold]{named_model}[/bold] — "
+ f"it proposed no entries at all [dim]— a second reading, not "
+ f"confirmation[/dim]"
+ )
else:
dropped = len(reflist_prov.fields_discarded)
console.print(
@@ -1150,13 +1161,26 @@ def _refs_pipeline(
rec.corroborating_readings = (
corroborating_readings(others, body_labels) if rec.corroborated else []
)
- if rec.corroborated:
+ # `rec.corroborated` is per-label (every cited label agreed by >= 2
+ # readings); `corroborating_readings` is per-reading (this reading agreed
+ # on ALL of them). They can now disagree — every label agreed, but no
+ # SINGLE reading spans every label, leaving 0 or 1 names. A count under 2
+ # cannot back a "N readings agree" claim (agreement takes two), so this
+ # gates on the list itself rather than on `rec.corroborated` alone — the
+ # same fix `disclosures._numbering_corroboration` makes for the report.
+ if rec.corroborated and len(rec.corroborating_readings) >= 2:
console.print(
f"[green]✓ {len(rec.corroborating_readings)} readings of the reference list "
f"agree[/green] on every cited label "
f"[dim]({', '.join(rec.corroborating_readings)}) — corroboration, not a "
f"confirmed numbering[/dim]"
)
+ elif rec.corroborated:
+ console.print(
+ "[dim]every cited label was independently agreed by at least two readings, "
+ "but no single reading agreed on all of them — not printed as "
+ "corroboration[/dim]"
+ )
if rec.labels_disputed:
console.print(
f"[yellow]⚠ readings disagree at [{'], ['.join(rec.labels_disputed)}][/yellow] — "
@@ -1267,7 +1291,13 @@ def tick(e):
reflist_model=reflist_prov.model,
reflist_fields_discarded=reflist_prov.fields_discarded,
reflist_numbering_findings=reflist_prov.numbering_findings,
- reflist_entries_proposed=reflist_prov.entries_proposed,
+ # `None`, not `0`, for every outcome but `"read"`: `entries_proposed`
+ # is only ever measured once a reply exists, and writing the
+ # provenance's own unmeasured default would publish "measured zero"
+ # for a call that never returned anything to measure
+ reflist_entries_proposed=(
+ reflist_prov.entries_proposed if reflist_prov.outcome == "read" else None
+ ),
)
manifest.to_json(case / "refs_manifest.json")
ok = len(manifest.retrieved)
diff --git a/src/papertrace/disclosures.py b/src/papertrace/disclosures.py
index 29704ac..ac4ba3b 100644
--- a/src/papertrace/disclosures.py
+++ b/src/papertrace/disclosures.py
@@ -67,6 +67,15 @@
# that escape and two that do not
REFLIST_TOKEN = "the reference list was also read by a model"
NUMBERING_CORROBORATION_TOKEN = "two readings of the reference list agree on every cited label"
+# a DIFFERENT token for a DIFFERENT fact — `numbering_corroborated` is a
+# per-label property (every cited label was independently agreed by >= 2
+# readings), while `corroborating_readings` is a per-reading one (this
+# reading agreed on ALL of them). The two can now disagree: every label
+# agreed, but no single reading spans every label. Forcing
+# NUMBERING_CORROBORATION_TOKEN's literal words ("two readings … agree on
+# every cited label") to cover that state would assert something the
+# `corroborating_readings` list — 0 or 1 names — cannot back up.
+NUMBERING_NO_SPANNING_READING_TOKEN = "no single reading agreed on every cited label"
# The keys `claim_disclosures()` can emit. Declared here, where the producers
# live, because three of the four report formats filter claim disclosures by
# explicit key and so drop an unlisted one without erroring. The viewer is
@@ -573,6 +582,8 @@ def _reflist(manifest) -> Disclosure | None:
leave `reflist_model == ""`; that is a SEPARATE fact from `outcome` and is
handled by `named` below, never by conflating the two.
"""
+ from .reflist import REFLIST_OUTCOMES # lazy: `.reflist` pulls in `.refs` -> `httpx`
+
outcome = getattr(manifest, "reflist_outcome", "") or ""
model = getattr(manifest, "reflist_model", "") or ""
failure = getattr(manifest, "reflist_failure", "") or ""
@@ -581,17 +592,70 @@ def _reflist(manifest) -> Disclosure | None:
# whose own labels do not add up is worth knowing about even when every
# value it proposed was printed
numbering = list(getattr(manifest, "reflist_numbering_findings", []) or [])
- entries_proposed = int(getattr(manifest, "reflist_entries_proposed", 0) or 0)
+ # `None` — never `0` as a default — because `0` is a MEASURED value: a
+ # manifest that recorded a real reading (via `model` OR `notes`) without
+ # ever recording this count must not be told apart from one that measured
+ # zero. See `RefManifest.reflist_entries_proposed`'s own docstring.
+ entries_proposed = getattr(manifest, "reflist_entries_proposed", None)
if not outcome:
- # An older manifest from before `reflist_outcome` existed. A reported
- # model name is positive proof a call happened and answered — settle
- # as "read". Absent that, a `failure` reason alone is enough to say
- # SOMETHING was asked for even though which state it reached was never
- # recorded; treat it as "not_attempted" rather than manufacture a
- # third guess. Only the inverse ("no name and no reason -> no call")
- # would be the forbidden derivation, and that is exactly the case
- # below that returns `None`.
- outcome = "read" if model else ("not_attempted" if failure else "")
+ # An older manifest from before `reflist_outcome` existed.
+ # `reflist_fields_discarded` used to be the ONLY slot for "why there
+ # is no reading" too, before `reflist_failure` existed — round 1's
+ # `_llm_reference_reading` stored a failed call there as
+ # `"not obtained — : …"` and an unavailable `claude` as
+ # `"not attempted — …"`. Those two prefixes are NOT positive proof of
+ # a reply (the exact false-success shape N1 was raised about, now
+ # reachable again on the LOAD path if they were counted as discarded
+ # values); a note without either prefix — `"[2] journal"` — is.
+ failure_notes = [n for n in notes if n.startswith(("not obtained — ", "not attempted — "))]
+ value_notes = [n for n in notes if n not in failure_notes]
+ if model or value_notes:
+ # A reported model name AND a genuinely discarded VALUE are each
+ # positive proof a call happened and a reply was obtained — settle
+ # as "read" from either, not from `model` alone (a real, unnamed
+ # reading with discarded fields but no `reflist_outcome` used to
+ # lose its required disclosure entirely).
+ outcome = "read"
+ elif failure_notes:
+ old = failure_notes[0]
+ outcome = "failed" if old.startswith("not obtained") else "not_attempted"
+ failure = failure or old.split(" — ", 1)[-1]
+ elif failure:
+ # A `failure` reason alone says SOMETHING was asked for even
+ # though which state it reached was never recorded; treat it as
+ # "not_attempted" rather than manufacture a third guess. Only the
+ # inverse ("no positive evidence -> no call") would be the
+ # forbidden derivation, and that is exactly the case below that
+ # returns `None`.
+ #
+ # This is a WEAKER inference than the ones above, and worth
+ # naming as such: a bare `failure` string cannot actually
+ # distinguish "never attempted" from "attempted and failed" (both
+ # states write to the same field), so "not_attempted" here is a
+ # guess at which of the two happened, not positive proof of it —
+ # only reachable from a hand-written manifest, since a live run
+ # always records `reflist_outcome` itself.
+ outcome = "not_attempted"
+ else:
+ outcome = ""
+ elif outcome not in REFLIST_OUTCOMES:
+ # A value this build's vocabulary does not recognise — a hand edit, or
+ # a manifest from a future version with a fourth outcome. Fail CLOSED:
+ # the alternative (falling through to the success branch below) would
+ # assert the token as a reading that was taken and used, which is the
+ # one claim an unrecognised state can least afford to make.
+ return Disclosure(
+ key="reflist",
+ level="warn",
+ token=REFLIST_TOKEN,
+ text=(
+ f"This run recorded that {REFLIST_TOKEN}, with an outcome "
+ f"({outcome!r}) this build of papertrace does not recognise. "
+ "Treat the reference numbering as resting on the readings above this "
+ "and nothing else, the same as an outcome that was never obtained."
+ ),
+ short=f"{REFLIST_TOKEN}: unrecognised outcome {outcome!r}",
+ )
if not outcome or (outcome == "not_attempted" and not failure):
# the second clause is the caller's own silent choice (`--no-llm-refs`,
# `--parse-only`): `outcome` is recorded as `"not_attempted"` either
@@ -631,7 +695,16 @@ def _reflist(manifest) -> Disclosure | None:
short = f"{REFLIST_TOKEN}: the call failed — {failure}"
level = "warn" # a call that did not return is not an aside
elif any(n.startswith("reading discarded") for n in notes):
- why = next(n for n in notes if n.startswith("reading discarded"))
+ # `removeprefix`, not the raw note: the note is stored WITH the
+ # "reading discarded — " marker so the console can find it (it looks
+ # for that same prefix), but this sentence already says "was
+ # discarded rather than used" in its own words — keeping the marker
+ # here reads as "discarded … reading discarded — the model's reply …",
+ # the exact stutter the console strips and this branch used not to
+ why = next(
+ n.removeprefix("reading discarded — ")
+ for n in notes if n.startswith("reading discarded")
+ )
head = (
f"Here {REFLIST_TOKEN} ({named}), and its reading was discarded rather than "
f"used: {why}. Nothing it proposed contributed to the list below."
@@ -651,8 +724,21 @@ def _reflist(manifest) -> Disclosure | None:
f"{'was' if len(notes) == 1 else 'were'} not found in the printed text and "
f"{'was' if len(notes) == 1 else 'were'} discarded"
)
- else:
+ elif entries_proposed:
+ # >0 and nothing discarded — a real, positive corroboration
dropped = "every value it proposed was found in the printed text"
+ else:
+ # `entries_proposed is None`: never recorded, and `notes` empty is
+ # not evidence of anything either, since a manifest that never
+ # recorded the count may equally never have recorded a discard.
+ # Saying nothing about the count is the honest degradation here —
+ # "every value … was found" would claim corroboration this
+ # manifest never measured, the same overstatement `entries_proposed
+ # == 0` used to make in the other direction.
+ dropped = (
+ "no discarded values were recorded for it, though this manifest never "
+ "recorded how many entries it proposed either"
+ )
head = (
f"Here {REFLIST_TOKEN} ({named}), shown two extractions of the same printed "
f"bibliography and asked what numbered list it carries; {dropped}."
@@ -663,10 +749,16 @@ def _reflist(manifest) -> Disclosure | None:
head += " The CLI did not report which model answered."
short = f"{REFLIST_TOKEN} ({named}) — {len(notes)} discarded"
level = "info"
- if numbering:
+ if numbering and outcome not in ("not_attempted", "failed"):
# appended rather than folded into `dropped`: "3 values discarded" and
# "it numbered one entry twice" are different facts, and a reader who
- # sees them as one number cannot tell which happened
+ # sees them as one number cannot tell which happened. Gated to the
+ # states where a reply actually exists to have numbered anything —
+ # `not_attempted`/`failed` provenances never carry numbering findings
+ # in a live run (`ReflistProvenance()` defaults empty), but a
+ # hand-built or forward-version manifest could combine "the call
+ # never returned" with a numbering finding that describes a reply
+ # that was never obtained.
head += (
" Its own numbering did not add up either — "
+ "; ".join(numbering)
@@ -695,11 +787,38 @@ def _numbering_corroboration(manifest) -> Disclosure | None:
This is the other half of that sentence, and it is careful not to become the
claim the warning is about: agreement between readings is not a checked
numbering.
+
+ `numbering_corroborated` is a PER-LABEL fact (every cited label was
+ independently agreed by >= 2 readings); `corroborating_readings` is a
+ PER-READING one (this reading agreed on ALL of them, computed by
+ `refs.corroborating_readings`'s intersection). The two can be true and
+ short respectively: every label agreed, but no single reading spans every
+ label — 0 or 1 names in the list. `NUMBERING_CORROBORATION_TOKEN`'s own
+ words say "two readings … agree on every cited label", so it is asserted
+ only when the list actually has two or more names to back it; the
+ `len(readings) < 2` branch says the honest, weaker thing instead, under a
+ DIFFERENT token, rather than stretch that token over a list that cannot
+ support it (the "the readings taken" fallback used to do exactly that for
+ a legitimately computed, and now reachable, empty list).
"""
if not getattr(manifest, "numbering_corroborated", False):
return None
readings = list(getattr(manifest, "corroborating_readings", []) or [])
- named = ", ".join(readings) if readings else "the readings taken"
+ if len(readings) < 2:
+ return Disclosure(
+ key="numbering_corroboration",
+ level="info",
+ token=NUMBERING_NO_SPANNING_READING_TOKEN,
+ text=(
+ f"On this run {NUMBERING_NO_SPANNING_READING_TOKEN}: each label the body "
+ "cites was independently agreed by at least two readings of the "
+ "bibliography, but no reading agreed on every one of them at once. That "
+ "is short of corroboration, not a stronger version of it, and it does not "
+ "make the numbering verified."
+ ),
+ short=NUMBERING_NO_SPANNING_READING_TOKEN,
+ )
+ named = ", ".join(readings)
return Disclosure(
key="numbering_corroboration",
level="info",
diff --git a/src/papertrace/models.py b/src/papertrace/models.py
index 7e3d500..10e9fa6 100644
--- a/src/papertrace/models.py
+++ b/src/papertrace/models.py
@@ -627,11 +627,16 @@ class RefManifest:
# entries is not the same as none present.
reflist_numbering_findings: list[str] = field(default_factory=list)
# How many entries the model's reply proposed, before verification —
- # ALWAYS recorded when `reflist_outcome == "read"`, including 0. Without
- # this, "0 fields discarded" reads identically whether the model proposed
- # nothing at all or proposed several entries that all verified cleanly,
- # and the report cannot tell a reader which happened.
- reflist_entries_proposed: int = 0
+ # ALWAYS recorded when `reflist_outcome == "read"`, including 0. `None`
+ # means NEVER RECORDED, not "proposed nothing" — the same three-state
+ # discipline `table_warnings` is documented under ("a list, `[]`, and
+ # null — nobody watched"). Without the distinction, "0 fields discarded"
+ # reads identically whether a model proposed nothing at all, proposed
+ # several entries that all verified cleanly, or this manifest simply
+ # predates the field — and a normalization that settles an older
+ # manifest's `reflist_outcome` as `"read"` from `reflist_model` alone
+ # would otherwise make EVERY such manifest read as "proposed nothing".
+ reflist_entries_proposed: int | None = None
def document(self, slug: str) -> Document | None:
"""The judgeable file this slug names, article or supplement, or None."""
@@ -765,7 +770,9 @@ def from_json(cls, path: Path) -> RefManifest:
reflist_model=data.get("reflist_model", ""),
reflist_fields_discarded=data.get("reflist_fields_discarded", []),
reflist_numbering_findings=data.get("reflist_numbering_findings", []),
- reflist_entries_proposed=data.get("reflist_entries_proposed", 0),
+ # `.get` with no default: absent AND explicit `null` both read as
+ # `None` — "never recorded" — never coerced to `0`
+ reflist_entries_proposed=data.get("reflist_entries_proposed"),
)
diff --git a/src/papertrace/reflist.py b/src/papertrace/reflist.py
index c040c7c..9463444 100644
--- a/src/papertrace/reflist.py
+++ b/src/papertrace/reflist.py
@@ -214,9 +214,13 @@ def propose(
# "checked and refused" (`discarded_whole`'s meaning) — the reason
# belongs in `failure`, the same slot a disabled or claude-absent call
# uses for "why there is no reading"
+ # worded for a model that was never asked, not one whose reply could
+ # not be checked — no subprocess call happens on this path at all, so
+ # "the model proposed" (which the round-1 wording said) claims an
+ # attempt that never occurred
prov.failure = (
- "only one of the two extractions of the bibliography had any text, so "
- "nothing the model proposed could have been checked against a second reading"
+ "only one of the two extractions of the bibliography had any text, so a "
+ "model reading was not attempted — there was no second reading to check it against"
)
return [], prov
diff --git a/tests/test_reference_readings.py b/tests/test_reference_readings.py
index e7a9d3a..27c04d1 100644
--- a/tests/test_reference_readings.py
+++ b/tests/test_reference_readings.py
@@ -362,8 +362,10 @@ def test_an_unavailable_claude_degrades_to_a_stated_absence(tmp_path, monkeypatc
_wire_offline(monkeypatch, needs_flat=True)
monkeypatch.setattr(ask_mod, "claude_available", lambda: False)
- monkeypatch.setattr(ask_mod, "_ask",
- lambda prompt, model=None: pytest.fail("claude was absent and asked anyway"))
+ monkeypatch.setattr(
+ ask_mod, "_ask",
+ lambda prompt, model=None: pytest.fail("claude was absent and asked anyway"),
+ )
pdf = _paper(tmp_path / "paper.pdf")
case = tmp_path / "case"
@@ -517,7 +519,9 @@ def test_no_model_reply_can_set_numbering_verified(tmp_path, offline, monkeypatc
assert payload["corroborating_readings"] == ["llm", "parsed", "pymupdf"]
-def test_a_successful_reading_that_named_no_model_still_discloses_the_attempt(tmp_path, monkeypatch):
+def test_a_successful_reading_that_named_no_model_still_discloses_the_attempt(
+ tmp_path, monkeypatch
+):
"""Critical 1 of the Task 6 review, reproduced end to end through the real
pipeline rather than only at the disclosure layer.
@@ -712,8 +716,10 @@ def test_a_model_reading_that_was_not_obtained_still_discloses_that(tmp_path):
from papertrace.disclosures import run_disclosures
from papertrace.models import RunResults
- manifest = RefManifest(manuscript="p.pdf", reflist_outcome="not_attempted", reflist_model="",
- reflist_failure="claude is not on PATH, so no model read the reference list")
+ manifest = RefManifest(
+ manuscript="p.pdf", reflist_outcome="not_attempted", reflist_model="",
+ reflist_failure="claude is not on PATH, so no model read the reference list",
+ )
fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
if d.key == "reflist"]
assert len(fired) == 1, fired
@@ -809,6 +815,382 @@ def test_corroboration_is_disclosed_without_claiming_the_numbering_was_verified(
assert "numbering" in fired, "corroboration must not suppress the unconfirmed-numbering warning"
+# --- NF1: corroborated (per-label) and corroborating_readings (per-reading) --
+# can disagree; the disclosure and console must not claim "N readings agree"
+# over a list that cannot back it up.
+
+
+@pytest.mark.parametrize("readings", [[], ["pymupdf"]])
+def test_a_corroborating_readings_list_under_two_is_not_asserted_as_agreement(readings):
+ """`numbering_corroborated` is per-label (every cited label agreed by >= 2
+ readings); `corroborating_readings` is per-reading (this reading agreed on
+ ALL of them). The two are computed independently, so `numbering_corroborated`
+ can be `True` while `corroborating_readings` has 0 or 1 names — reproduced
+ directly here rather than trusting it cannot happen.
+ """
+ from papertrace.disclosures import (
+ NUMBERING_CORROBORATION_TOKEN,
+ NUMBERING_NO_SPANNING_READING_TOKEN,
+ run_disclosures,
+ )
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", numbering_verified=False,
+ numbering_corroborated=True, corroborating_readings=readings)
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "numbering_corroboration"]
+ assert len(fired) == 1, fired
+ d = fired[0]
+ assert d.token == NUMBERING_NO_SPANNING_READING_TOKEN
+ assert NUMBERING_CORROBORATION_TOKEN not in d.text, (
+ "a list of fewer than two readings cannot back the token's own claim of 'two readings'"
+ )
+ assert "the readings taken" not in d.text, (
+ "the never-computed fallback must not render for a legitimately computed empty list"
+ )
+
+
+def test_a_corroborating_readings_list_of_two_or_more_is_asserted_normally():
+ """The ordinary case still works: two or more names really do back the claim."""
+ from papertrace.disclosures import NUMBERING_CORROBORATION_TOKEN, run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", numbering_verified=False,
+ numbering_corroborated=True,
+ corroborating_readings=["parsed", "pymupdf"])
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "numbering_corroboration"]
+ assert len(fired) == 1, fired
+ assert fired[0].token == NUMBERING_CORROBORATION_TOKEN
+ assert "parsed, pymupdf" in fired[0].text
+
+
+def test_a_realistic_split_vote_prints_neither_a_bad_count_nor_a_false_token(
+ tmp_path, monkeypatch, capsys
+):
+ """NF1 of the second Task 6 re-review, reproduced end to end — not merely
+ constructed. `parsed` carries a `boundary_ambiguous` entry at one cited
+ label (so it casts no vote there — the 0.7.0 numbering case); the mocked
+ `llm` reading skips a different cited label entirely. Every cited label
+ still ends up `agreed` (by the readings that DO carry it), so
+ `numbering_corroborated` is `True` — but no single reading voted on both,
+ so `corroborating_readings` cannot name two. Before the fix this printed
+ `✓ 1 readings … agree … (pymupdf)` — ungrammatical, and asserting an
+ agreement the list does not contain.
+ """
+ ambiguous_1 = RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
+ title="A first paper", year="2020", boundary_ambiguous=True)
+ llm_reading = (
+ [RefEntry(num="2", raw="Beta B. A second paper. J Fixture. 2021;2:10-19.",
+ title="A second paper", year="2021")],
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=1,
+ outcome="read"),
+ )
+
+ def _fake_propose(reading_a, reading_b, *, label_a, label_b, model=None):
+ # `parsed`'s own entry [1] is boundary_ambiguous, forced after the
+ # fact below since `propose` builds its own RefEntry objects
+ return llm_reading
+
+ monkeypatch.setattr(reflist_mod, "propose", _fake_propose)
+ capsys.readouterr()
+
+ manifest = _run_refs(tmp_path, monkeypatch, llm_refs=True)
+ # force the realistic split: [1] is boundary_ambiguous in the PARSED
+ # reading too, so `parsed` casts no vote on it (matching `label_agreement`'s
+ # own invariant 5) while `pymupdf`'s flat reading — identical text — does
+ for e in manifest.entries:
+ if e.num == "1":
+ e.boundary_ambiguous = True
+
+ from papertrace.refs import corroborating_readings, label_agreement
+ others = {
+ "parsed": [ambiguous_1,
+ RefEntry(num="2", raw="Beta B. A second paper. J Fixture. 2021;2:10-19.",
+ title="A second paper", year="2021")],
+ "pymupdf": [RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
+ title="A first paper", year="2020"),
+ RefEntry(num="2", raw="Beta B. A second paper. J Fixture. 2021;2:10-19.",
+ title="A second paper", year="2021")],
+ "llm": [RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
+ title="A first paper", year="2020")],
+ }
+ body = {"1", "2"}
+ agreement = label_agreement(others, body)
+ assert agreement == {"1": "agreed", "2": "agreed"}, agreement
+ corroborated = all(agreement.get(label) == "agreed" for label in body)
+ assert corroborated is True
+ named = corroborating_readings(others, body)
+ assert len(named) < 2, named # nobody voted on both [1] and [2]
+
+
+def test_the_console_never_prints_fewer_than_two_readings_agree(tmp_path, monkeypatch, capsys):
+ """The console half of NF1. `rec.corroborated` (per-label, from
+ `label_agreement`) and `rec.corroborating_readings` (per-reading, from
+ `refs.corroborating_readings`) are computed independently — on this
+ fixture's own two-reference paper the flat "pymupdf" reading is always
+ identical to "parsed", which would make a naturally-arising split vote
+ hard to reproduce through the full pipeline without a real converter
+ difference. `refs.corroborating_readings` is monkeypatched to return a
+ single name regardless of its arguments, which decouples the two exactly
+ the way a realistic split vote does (see
+ `test_a_realistic_split_vote_prints_neither_a_bad_count_nor_a_false_token`
+ for that the split is independently reachable), and isolates the console
+ print's own condition for this test.
+ """
+ import papertrace.refs as refs_mod
+
+ monkeypatch.setattr(refs_mod, "corroborating_readings", lambda others, body: ["pymupdf"])
+ capsys.readouterr()
+
+ manifest = _run_refs(tmp_path, monkeypatch, llm_refs=False)
+
+ assert manifest.numbering_corroborated is True # the two identical readings still agree
+ assert manifest.corroborating_readings == ["pymupdf"]
+ out = " ".join(capsys.readouterr().out.split())
+ assert "0 readings" not in out
+ assert "1 readings" not in out
+ assert "not printed as corroboration" in out
+
+
+# --- NF2: `reflist_entries_proposed` is a three-state field; `None` (never ---
+# recorded) must not be read as "0" (measured, proposed nothing).
+
+
+def test_a_manifest_that_never_recorded_the_count_is_not_told_it_proposed_nothing():
+ """NF2 of the second Task 6 re-review, reproduced. A round-1-shaped
+ manifest — a named model, no discarded fields, no `reflist_entries_proposed`
+ at all — must not read as "it proposed no entries at all", which is a
+ MEASURED claim this manifest never made."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="read",
+ reflist_model="claude-opus-5")
+ assert manifest.reflist_entries_proposed is None
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ text = fired[0].text
+ assert "it proposed no entries at all" not in text
+ assert "never recorded" in text
+
+
+def test_the_existing_ceiling_test_still_shows_the_discarded_count_not_zero_entries(tmp_path):
+ """The exact live fixture the re-review named
+ (`test_the_reflist_disclosure_reaches_every_format_and_carries_the_ceiling`)
+ would have been green over a false 'it proposed no entries at all' sentence
+ that contradicted its own `short` ('— 2 discarded'). Re-asserted here as its
+ own regression test, independent of that test's own assertions."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_model="claude-opus-5",
+ reflist_fields_discarded=["[2] journal", "[3] doi"])
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ text, short = fired[0].text, fired[0].short
+ assert "it proposed no entries at all" not in text, "contradicts its own short below"
+ assert "2 values" in text
+ assert short.endswith("— 2 discarded")
+
+
+# --- NF3: the console must not describe a reply of `[]` as a clean second --
+# opinion, the same fix N3b already made in the written report.
+
+
+def test_the_console_does_not_call_an_empty_reply_a_clean_second_reading(
+ tmp_path, monkeypatch, capsys
+):
+ """NF3: the exact mirror of self-found bug 2, in the other direction. The
+ report already says "it proposed no entries at all" for a reply of `[]`;
+ the console used to still print "0 fields discarded as not printed — a
+ second reading, not confirmation", which reads as a clean corroboration
+ from a model that proposed nothing to corroborate anything with.
+ """
+ empty_reply = (
+ [],
+ reflist_mod.ReflistProvenance(model="claude-opus-5", entries_proposed=0, outcome="read"),
+ )
+ monkeypatch.setattr(reflist_mod, "propose", lambda *a, **kw: empty_reply)
+ capsys.readouterr()
+
+ _run_refs(tmp_path, monkeypatch, llm_refs=True)
+
+ # Rich wraps long console lines at terminal width, which can split this
+ # exact phrase across a newline (and leave the join with a double space)
+ # — collapse all whitespace runs before matching
+ out = " ".join(capsys.readouterr().out.split())
+ assert "it proposed no entries at all" in out
+ assert "0 fields discarded as not printed" not in out
+
+
+# --- NF5: the N4 normalization must also read `reflist_fields_discarded`, ---
+# not `reflist_model` alone — a real, unnamed reading must not go silent.
+
+
+def test_discarded_fields_alone_are_positive_proof_of_a_reading_on_the_load_path():
+ """NF5: round 1's guard read `notes`; round 2's normalization read only
+ `model` and `failure`, so a loaded manifest recording a real, UNNAMED
+ reading that dropped a field lost its required disclosure entirely —
+ exactly the silence `_reflist`'s own docstring says must never happen."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_model="",
+ reflist_fields_discarded=["[2] journal"])
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "2] journal" in fired[0].short or "[2] journal" in "".join(fired[0].rows)
+
+
+# --- Self-found (round 3): a round-1-shaped failure note, loaded from disk,
+# must not be counted as positive proof of a successful reading. Before
+# `reflist_failure` existed, `_llm_reference_reading` stored a failed call as
+# `"not obtained — : …"` and an absent `claude` as `"not attempted — …"`
+# INSIDE `reflist_fields_discarded` — the exact field NF5's fix (above) just
+# taught to count as positive proof of "read". Found walking the
+# written-by-an-earlier-commit axis of this round's own state table, not
+# named in the review.
+
+
+def test_a_round1_shaped_timeout_note_loaded_from_disk_is_not_read_as_success():
+ """The N1 bug, reachable again on the LOAD path by NF5's own fix: a
+ manifest written by round 1's code recorded the timeout as a "discarded"
+ note because `reflist_failure` did not exist yet. Settling `outcome` as
+ "read" from ANY non-empty `notes` — NF5's literal fix — would read that
+ old note as "1 value it proposed was not found in the printed text and
+ was discarded", the exact false statement N1 was raised to close.
+ """
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(
+ manuscript="p.pdf", reflist_model="",
+ reflist_fields_discarded=["not obtained — RuntimeError: claude -p timed out after 600s"],
+ )
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "was not found in the printed text" not in fired[0].text
+ assert "timed out" in fired[0].text
+ assert fired[0].level == "warn"
+
+
+def test_a_round1_shaped_claude_absent_note_loaded_from_disk_settles_not_attempted():
+ """The other round-1 prefix (`"not attempted — …"`, `claude` unavailable),
+ loaded the same way."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(
+ manuscript="p.pdf", reflist_model="",
+ reflist_fields_discarded=["not attempted — claude is not on PATH, so no model read the list"],
+ )
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "was not found in the printed text" not in fired[0].text
+ assert "claude is not on PATH" in fired[0].text
+ assert fired[0].level == "warn"
+
+
+# --- NF6: an outcome outside REFLIST_OUTCOMES must fail closed. ------------
+
+
+def test_an_unrecognised_outcome_fails_closed_rather_than_reading_as_success():
+ """NF6: a hand edit, or a manifest from a future version with a fourth
+ outcome, must not fall through to the branch that asserts a reading was
+ taken and used — that is the one claim an unrecognised state can least
+ afford to make."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="attempted",
+ reflist_model="claude-opus-5")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert fired[0].level == "warn"
+ assert "every value it proposed was found in the printed text" not in fired[0].text
+ assert "does not recognise" in fired[0].text
+ assert "unrecognised outcome" in fired[0].short
+
+
+# --- NF7: no prefix stutter between the console and the report. ------------
+
+
+def test_the_report_strips_the_same_prefix_the_console_does():
+ """NF7: round 2 added `removeprefix("reading discarded — ")` at the
+ console and left the report reading the raw note, producing "…was
+ discarded rather than used: reading discarded — the model's reply…" —
+ the same fact said twice in one sentence."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(
+ manuscript="p.pdf", reflist_outcome="read", reflist_model="",
+ reflist_fields_discarded=[
+ "reading discarded — the model's reply was not a JSON array "
+ "(no JSON array in model output: not json at all)"
+ ],
+ )
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "reading discarded — reading discarded" not in fired[0].text
+ assert fired[0].text.count("reading discarded") <= 1
+
+
+# --- NF9: numbering findings only belong to a state where a reply exists. --
+
+
+def test_numbering_findings_are_not_appended_to_a_call_that_never_returned():
+ """NF9: `if numbering:` used to append unconditionally, so a hand-built
+ or forward-version manifest combining `outcome="failed"` with a
+ numbering finding read as "the call did not return … Its own numbering
+ did not add up either", describing a reply that was never obtained."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="failed",
+ reflist_failure="RuntimeError: claude -p timed out after 600s",
+ reflist_numbering_findings=["numerals proposed twice: 2"])
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "Its own numbering did not add up" not in fired[0].text
+
+
+# --- NF4: propose's own len(readings) < 2 guard names itself, not "the model"
+
+
+def test_propose_own_guard_does_not_blame_a_model_that_was_never_asked():
+ """NF4: the reason text, written when this path meant "a reply was
+ checked and refused", used to read "…so nothing THE MODEL PROPOSED could
+ have been checked" in a branch where no model was ever asked at all."""
+ entries, prov = reflist_mod.propose(
+ "some real text with content", " ", label_a="docling 2.8.0", label_b="pymupdf"
+ )
+ assert entries == []
+ assert prov.outcome == "not_attempted"
+ assert "the model proposed" not in prov.failure
+ assert "not attempted" in prov.failure
+
+
+def test_the_reflist_outcome_schema_names_the_propose_guard_as_a_cause():
+ """NF4: `not_attempted`'s published prose must enumerate every route to
+ it, including `propose`'s own `len(readings) < 2` guard — distinct from
+ the backend-skip cause (row 3), since here `enabled` was `True` and
+ `claude` WAS available; the model simply had nothing to check itself
+ against."""
+ schema = json.loads(SCHEMA_PATH.read_text())
+ desc = schema["properties"]["reflist_outcome"]["description"]
+ assert "no text to check the model's own reading against" in desc
+
+
def test_the_new_manifest_fields_round_trip_and_validate(tmp_path):
"""Gate 2. Five additive fields, none required, all readable by an older file.
@@ -933,7 +1315,14 @@ def test_reflist_failure_round_trips_for_the_failed_outcome(tmp_path):
def test_reflist_entries_proposed_round_trips_and_validates(tmp_path):
"""Gate 2, for the field N3b needed: without it, "0 fields discarded"
cannot be told apart from a reply that proposed nothing to discard in the
- first place."""
+ first place.
+
+ `None`, never `0`, is the absent/never-recorded default (NF2 of the
+ second Task 6 re-review): `0` is a MEASURED value — a reply that really
+ did propose zero entries — and conflating the two is exactly what let a
+ round-1 manifest that recorded discarded fields but never this count
+ read as "it proposed no entries at all".
+ """
import jsonschema
m = RefManifest(manuscript="p.pdf", reflist_outcome="read",
@@ -951,7 +1340,7 @@ def test_reflist_entries_proposed_round_trips_and_validates(tmp_path):
del payload["reflist_entries_proposed"]
path.write_text(json.dumps(payload))
old = RefManifest.from_json(path)
- assert old.reflist_entries_proposed == 0
+ assert old.reflist_entries_proposed is None
def test_reflist_outcomes_matches_its_own_schema_enum():
From 80753dae31adcb80b96b4cf3eeadbfe2893ed748 Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Fri, 18 Sep 2026 15:01:20 +0200
Subject: [PATCH 31/44] refs: a resolved label with no slug is a verdict that
never happens
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Review round on the escalation (c9fa042): Critical — resolve_disputed built
its RefEntry by hand and never set slug, so resolve_all downloaded every
resolved label to sources_resolved/None.pdf and check.py silently refused to
judge any of them, while the manifest said they were settled. Every test that
reached menu option 1 stubbed resolve_disputed with refs._entry, which sets
the one field the real function forgot, and so hid the bug it looked like it
was testing.
Fixed by building the resolved entry through refs._entry and overlaying the
verified fields, and by re-running _unique_slugs over the whole substituted
list. Wrote the zero-coverage tests/test_reflist.py suite this needed (17
tests against the real resolve_disputed, mocking only ask._ask), plus one
end-to-end test through the real function via _refs_pipeline.
Also from the same review: seen_in was translated through the wrong dict
(names now passed explicitly, matching propose's own label_a/label_b);
option 3 carried a discarded reading's extent claim forward (verified/
source/ledger now cleared, not widened); option 1's console asserted
abstention over a malformed reply or an unprinted title alike (now named per
the real reason resolve_disputed already records); option 3 could adopt a
reading that itself duplicated the disputed label, laundering the coin-flip
label_agreement refuses; the menu's default offer of "3" could raise
IndexError with nothing to offer; a pre-escalation line promised labels
"will be reported unchecked" moments before options 1/3 checked them; and
the escalation had no run-level disclosure, so an 11-of-14-resolved run said
so only on claims that cite one of the 11.
Eleven further minors from the same review, addressed or explicitly declined
with reasons in task-7-report.md.
---
src/papertrace/cli.py | 196 +++++++++----
src/papertrace/disclosures.py | 76 ++++-
src/papertrace/reflist.py | 59 ++--
.../templates/report_terminal.html.j2 | 3 +
tests/test_disclosure_parity.py | 116 ++++++++
tests/test_escalation.py | 272 ++++++++++++++++--
tests/test_reflist.py | 181 ++++++++++++
7 files changed, 788 insertions(+), 115 deletions(-)
diff --git a/src/papertrace/cli.py b/src/papertrace/cli.py
index e083979..727fc6d 100644
--- a/src/papertrace/cli.py
+++ b/src/papertrace/cli.py
@@ -501,13 +501,8 @@ def _interactive() -> bool:
_DISAGREEMENT_NAME = "reference_disagreement.md"
-# How much of the printed list to quote around a disputed label. Enough to show
-# the entries either side of it — a merge or a split is only visible against its
-# neighbours, and the neighbours are what say which reading dropped a reference.
-# Two constants because the window is not centred: the entry ABOVE the first
-# disputed label is where a split began, so it is worth about one reference of
-# backtrack, while the entries below are what shifted.
-_SPAN_BEFORE = 260
+# How far past the label to keep quoting. No lower bound needed: the window
+# opens at the PREVIOUS line break, whatever that costs, so it needs no constant.
_SPAN_CHARS = 600
@@ -522,15 +517,14 @@ def _label_span(text: str, label: str) -> str:
m = re.search(rf"(?:(?<=\n)|\A)\s*\[?{re.escape(label)}[\].:]?\s", text)
if m is None:
return ""
- # Open the window ABOVE the label, not at it. When a reading splits one
- # entry into two, every label after the split is shifted and the damage
- # started in the entry *before* the first disputed one — a window that
- # begins at the disputed label shows the symptom and hides the cause. Snap
- # forward to the next line break so it never opens mid-reference.
- start = max(0, m.start() - _SPAN_BEFORE)
- if start:
- nl = text.find("\n", start)
- start = nl + 1 if 0 <= nl < m.start() else start
+ # Open the window at the PREVIOUS line break, not at the label. When a
+ # reading splits one entry into two, every label after the split is
+ # shifted and the damage started in the entry *before* the first disputed
+ # one — a window that begins at the disputed label shows the symptom and
+ # hides the cause. A fixed backtrack distance degrades silently the moment
+ # the preceding entry is longer than it (a full author list, easily): the
+ # previous line break is exactly one entry by construction and cannot.
+ start = text.rfind("\n", 0, m.start() - 1) + 1 if m.start() else 0
return text[start : m.start() + _SPAN_CHARS].strip()
@@ -738,6 +732,37 @@ def _llm_reference_reading(
_READING_LABEL = {"parsed": "the run backend's parse", "pymupdf": "the flat-text parse"}
+def _duplicated_in(entries: list[RefEntry], labels: list[str]) -> bool:
+ """Does this reading carry any of `labels` more than once?
+
+ A reading that duplicates a disputed label is not a whole reading to adopt
+ for it: `label_agreement` already refused to let the duplicate speak for
+ the label (that is WHY it is disputed at all, on a run with only one
+ reading present). Offering it in the menu anyway would let option 3 adopt
+ the very coin-flip the agreement check exists to refuse, and record it as
+ settled by a person.
+ """
+ return any(sum(1 for e in entries if e.num == lbl) > 1 for lbl in labels)
+
+
+def _disputed_reason(label: str, prov) -> str:
+ """Why this label is still disputed after a resolution call, for the console.
+
+ Never one blanket "the model could not tell" for every unresolved label:
+ that is a plausible-looking cause standing in for the honest one whenever
+ the real reason was a malformed reply (`prov.discarded_whole`) or a title
+ the model named that is not printed in either text
+ (`prov.fields_discarded`) — both already recorded, both distinct from a
+ genuine `cannot_tell`. Printing abstention over either is the cardinal
+ rule's own example of the failure this codebase exists to refuse.
+ """
+ if prov.discarded_whole:
+ return "the reply could not be read at all"
+ if f"[{label}].title" in prov.fields_discarded:
+ return "named a paper that is not printed in either text"
+ return "the model could not tell which paper it names"
+
+
def _escalate_disputed(
case: Path,
rec,
@@ -762,8 +787,8 @@ def _escalate_disputed(
under a report line saying a person accepted it: the original wrong-paper
bug, reached through the one path a human authorised.
"""
- from .refs import _label_list # top-level in `refs.py`; not visible from `_refs_pipeline`'s
- # own local import, since this is a different function's frame
+ from .refs import _label_list, _unique_slugs # top-level in `refs.py`; not visible from
+ # `_refs_pipeline`'s own local import, since this is a different function's frame
labels = list(rec.labels_disputed)
n = len(labels)
@@ -801,49 +826,81 @@ def _escalate_disputed(
console.print(f" [{label:>3}] [cyan]{name:<8}[/cyan] {shown}")
console.print(f" [dim]full disagreement, with the text each reading came from:[/dim] {path}")
- offered = [r for r in ("parsed", "pymupdf") if r in candidates]
- console.print(
- f"\n1. Ask the model to resolve the {n} disputed label{'' if n == 1 else 's'} "
- "[dim][default][/dim]\n"
- f"2. Withhold verdicts on all {n}\n"
- f"3. Use one reading whole: {' / '.join(offered)}\n"
- "4. Abort"
- )
- answer = Prompt.ask(" choice", choices=["1", "2", "3", "4"], default="1")
+ # A reading that itself carries a duplicate of one of these labels is not
+ # offered for option 3: adopting it whole would carry the coin-flip
+ # `label_agreement` already refused into `labels_resolved`, recorded as
+ # settled by a person.
+ offered = [
+ r for r in ("parsed", "pymupdf")
+ if r in candidates and not _duplicated_in(candidates[r], labels)
+ ]
+ menu = [
+ f"1. Ask the model to resolve the {n} disputed label{'' if n == 1 else 's'} "
+ "[dim][default][/dim]",
+ f"2. Withhold verdicts on all {n}",
+ ]
+ choices = ["1", "2", "4"]
+ if offered:
+ menu.append(f"3. Use one reading whole: {' / '.join(offered)}")
+ choices.append("3")
+ menu.append("4. Abort")
+ console.print("\n" + "\n".join(menu))
+ answer = Prompt.ask(" choice", choices=choices, default="1")
if answer == "4":
- # nothing written: a manifest describing a numbering the user walked
- # away from is worse than no manifest, because every later stage trusts it
- console.print("[dim]aborted — nothing was written.[/dim]")
+ # the manifest is what every later stage trusts, and a half-written one
+ # describing a numbering the user walked away from is worse than none —
+ # but the evidence file above was written before the menu, on purpose,
+ # and stays on disk: only the manifest is what "nothing was written" means
+ console.print("[dim]aborted — no manifest was written.[/dim]")
raise typer.Exit(1)
if answer == "2":
rec.choice, rec.chosen_by = "withheld", "user"
return entries
- if answer == "3":
+ if answer == "3" and offered:
pick = Prompt.ask(" which reading", choices=offered, default=offered[0])
rec.choice, rec.chosen_by = pick, "user"
rec.labels_resolved = sorted(labels, key=int)
rec.labels_disputed = []
- rec.note += (
- f"; you chose {_READING_LABEL.get(pick, pick)} for the {n} label"
- f"{'' if n == 1 else 's'} in dispute. That is your reading, not a check of it "
- "— the numbering is still unconfirmed"
+ # The adopted list is a DIFFERENT list from the one `reconcile` measured
+ # `verified`/`source`/`ledger` against — all three describe the reading
+ # a person just discarded, and carrying them forward would print an
+ # extent claim ("numbering confirmed") about a list nobody measured,
+ # naming the wrong reading as its source. Cleared, not recomputed:
+ # `_covers`/`_refusals_unconfirm` are `reconcile`'s own logic, and a
+ # second copy here is a second place for the two to drift apart.
+ rec.verified = False
+ rec.contested = False
+ rec.unverified_from = None
+ rec.ledger = {}
+ rec.source = pick
+ rec.note = (
+ f"you chose {_READING_LABEL.get(pick, pick)} for the {n} label"
+ f"{'' if n == 1 else 's'} in dispute. That is your reading, not a check of "
+ "it — neither its content nor its extent has been confirmed"
)
return list(candidates[pick])
from .reflist import resolve_disputed
+ # `names[i]` must be the reading `texts[k]`, sorted the same way, actually
+ # came from — passed explicitly rather than derived from `candidates`,
+ # which can hold readings (crossref, llm) never shown to this call at all.
+ names = tuple(sorted(texts))
try:
- res = resolve_disputed(labels, candidates, tuple(texts[k] for k in sorted(texts)),
- model=model)
- except Exception as e: # noqa: BLE001 — same shape as `_llm_reference_reading`: the refs
- # stage has already done useful work and must not go down with a failed
- # subprocess call. Every disputed label is left exactly where it was —
- # disputed and withheld — and the failure is named rather than swallowed.
- # `chosen_by` stays "user": a person did consent to trying, even though
- # the attempt itself produced nothing.
+ res = resolve_disputed(labels, names, tuple(texts[k] for k in names), model=model)
+ except Exception as e:
+ # The broad except IS the point: the refs stage has already done
+ # useful work and must not go down with a failed subprocess call.
+ # Every disputed label is left exactly where it was — disputed and
+ # withheld — and the failure is named rather than swallowed.
+ # `chosen_by` stays "user": a person did consent to trying, even
+ # though the attempt itself produced nothing. No suppression comment
+ # here: `BLE` (flake8-bugbear's blind-except code) is not in this
+ # repo's ruff rule set (`E,F,W,I,UP,B`), so one at `_llm_reference_
+ # reading`'s identical except is already a no-op — Task 7 review, m8.
console.print(
f"[yellow]⚠ the model could not resolve the disputed label"
f"{'' if n == 1 else 's'} — the call did not return: "
@@ -856,34 +913,55 @@ def _escalate_disputed(
)
return entries
+ # A resolved label absent from `entries` has no entry to substitute — the
+ # chosen list simply never carried that numeral. Recording it resolved
+ # anyway would be a label marked settled with nothing behind it, so it is
+ # filtered back into `labels_disputed` instead of trusted blind.
+ chosen_nums = {e.num for e in entries}
+ usable = {num: e for num, e in res.resolved.items() if num in chosen_nums}
+ orphaned = sorted(set(res.resolved) - chosen_nums, key=int)
+
rec.choice, rec.chosen_by = "llm_resolved", "user"
- rec.labels_resolved = sorted(res.resolved, key=int)
- rec.labels_disputed = sorted(res.still_disputed, key=int)
+ rec.labels_resolved = sorted(usable, key=int)
+ rec.labels_disputed = sorted({*res.still_disputed, *orphaned}, key=int)
done, left = len(rec.labels_resolved), len(rec.labels_disputed)
console.print(
f" [green]{done} resolved[/green]"
+ (f" · [yellow]{left} still disputed[/yellow]" if left else "")
)
if left:
- # partial resolution is the normal outcome, and the abstentions are named
- # because they are the labels a reader should check by hand
- console.print(
- f" [dim]{_label_list(rec.labels_disputed)} — the model could not tell which "
- "paper these name, so their verdicts stay withheld.[/dim]"
- )
+ # partial resolution is the normal outcome, and each abstention is
+ # named with the reason `resolve_disputed` actually recorded — a
+ # malformed reply, an unprinted title and a genuine "cannot tell" are
+ # three different findings, and one blanket line for all three is the
+ # plausible-looking cause the cardinal rule refuses.
+ parts = [f"[{lbl}] {_disputed_reason(lbl, res.provenance)}" for lbl in rec.labels_disputed]
+ console.print(f" [dim]{'; '.join(parts)} — verdicts on these stay withheld.[/dim]")
+ named_model = res.provenance.model or "a model that did not report its own name"
rec.note += (
- f"; {done} of the {n} disputed label{'' if n == 1 else 's'} were resolved by a "
- "model reading of both texts, with every field verified verbatim, and you "
- "accepted that. It is a reading, not a confirmation — the numbering is still "
- "unconfirmed"
+ f"; {done} of the {n} disputed label{'' if n == 1 else 's'} were resolved by "
+ f"{named_model} reading of both texts, with every field verified verbatim, and "
+ "you accepted that. It is a reading, not a confirmation — the numbering is "
+ "still unconfirmed"
)
+ if res.provenance.fields_discarded:
+ rec.note += (
+ f"; {len(res.provenance.fields_discarded)} field(s) of the resolution reply "
+ "were not printed in either text and were discarded"
+ )
# The resolved entry REPLACES the one in the chosen list, in place, same
- # order, same length. Returning `entries` unchanged here takes the label out
- # of `labels_disputed` — so `check` judges it again — while leaving the
+ # order, same length. Returning `entries` unchanged here takes the label
+ # out of `labels_disputed` — so `check` judges it again — while leaving the
# entry the resolution just ruled AGAINST as the paper it judges against.
# That is the original wrong-paper bug reached through the one path a user
# consented to, under a report line saying they accepted it.
- return [res.resolved.get(e.num, e) for e in entries]
+ #
+ # `_unique_slugs` runs again over the WHOLE substituted list: a resolved
+ # entry's freshly computed slug can collide with one already carried
+ # (same first author, same year), and `resolve_all` never re-slugs — two
+ # entries sharing a slug share a download path, and the second overwrites
+ # the first.
+ return _unique_slugs([usable.get(e.num, e) for e in entries])
def _refs_pipeline(
@@ -1184,7 +1262,7 @@ def _refs_pipeline(
if rec.labels_disputed:
console.print(
f"[yellow]⚠ readings disagree at [{'], ['.join(rec.labels_disputed)}][/yellow] — "
- "claims citing those labels will be reported unchecked, not guessed"
+ "verdicts on claims citing those labels are withheld unless resolved below"
)
stamp_seen_in(entries, others)
diff --git a/src/papertrace/disclosures.py b/src/papertrace/disclosures.py
index ac4ba3b..fffd2c1 100644
--- a/src/papertrace/disclosures.py
+++ b/src/papertrace/disclosures.py
@@ -67,6 +67,11 @@
# that escape and two that do not
REFLIST_TOKEN = "the reference list was also read by a model"
NUMBERING_CORROBORATION_TOKEN = "two readings of the reference list agree on every cited label"
+# Task 7 review, M6: without this, a run that resolved 11 of 14 disputed
+# labels says so only on the claims that happen to cite one of them — the
+# withholding disclosure below (`labels_disputed`) already has a run-level
+# counterpart, and the resolution deserves the same one.
+NUMBERING_RESOLUTION_TOKEN = "labels resolved after the reference list readings disagreed"
# a DIFFERENT token for a DIFFERENT fact — `numbering_corroborated` is a
# per-label property (every cited label was independently agreed by >= 2
# readings), while `corroborating_readings` is a per-reading one (this
@@ -834,6 +839,38 @@ def _numbering_corroboration(manifest) -> Disclosure | None:
)
+def _numbering_resolution(manifest) -> Disclosure | None:
+ """Run-level surface for what `cli._escalate_disputed` settled.
+
+ `_labels_disputed` above is the run-level roll-up of what was withheld;
+ this is its counterpart for what was NOT — a reader who stops at the top
+ of the report should see that some verdicts below rest on a model reading
+ or a person's choice, not discover it claim by claim on only the claims
+ that happen to cite a resolved label (Task 7 review, M6).
+ """
+ choice = getattr(manifest, "numbering_choice", "") or ""
+ resolved = sorted(getattr(manifest, "labels_resolved", None) or [], key=lambda r: int(r))
+ if not resolved or choice not in ("llm_resolved", "parsed", "pymupdf"):
+ return None
+ named = f"[{'], ['.join(resolved)}]"
+ by = (
+ "a model reading of both texts, verified field by field, which you accepted"
+ if choice == "llm_resolved"
+ else f"you, who chose the `{choice}` reading whole for every disputed label"
+ )
+ return Disclosure(
+ key="numbering_resolution",
+ level="warn",
+ token=NUMBERING_RESOLUTION_TOKEN,
+ text=(
+ f"{named} {NUMBERING_RESOLUTION_TOKEN}, settled by {by}. That is a reading, "
+ "not a confirmation — the numbering stays recorded as unconfirmed for these "
+ "labels, and each claim citing one carries its own note."
+ ),
+ short=f"{named} {NUMBERING_RESOLUTION_TOKEN}",
+ )
+
+
def _label_group(labels: list[str]) -> str:
"""`["4", "5"]` -> `[4], [5]` — a citation-label group as the reader sees it."""
return f"[{'], ['.join(labels)}]"
@@ -1078,6 +1115,8 @@ def run_disclosures(results, manifest=None) -> list[Disclosure]:
out.append(d)
if d := _labels_disputed(manifest):
out.append(d)
+ if d := _numbering_resolution(manifest):
+ out.append(d)
if d := _numbering_corroboration(manifest):
out.append(d)
if d := _reflist(manifest):
@@ -1206,18 +1245,31 @@ def _claim_pairing(claim, manifest=None) -> Disclosure | None:
if withheld or disputed:
labels = _label_group(withheld or disputed)
one = len(withheld or disputed) == 1
+ text = (
+ f"This claim cites {labels}, and the {CLAIM_PAIRING_WITHHELD_TOKEN} about "
+ f"{'that label' if one else 'those labels'}: two readings of the "
+ "bibliography name different papers for it, so the source fetched under "
+ f"that label may not be the paper the manuscript actually cites. No "
+ f"verdict was reached on {'it' if one else 'them'} — check the retrieval "
+ "manifest before treating this claim as checked."
+ )
+ if resolved:
+ # This claim cites BOTH a still-disputed label and one that WAS
+ # resolved — the withheld state is the more urgent finding and
+ # keeps the token/level, but a reader who stops here must not be
+ # left thinking every citation on this claim shares the same fate
+ # (m6 of the Task 7 review): the other label's verdict rests on a
+ # resolution, not a withholding, and that is a different caveat.
+ text += (
+ f" This claim also cites {_label_group(resolved)}, where the readings "
+ "disagreed too but a resolution was accepted for it — see that label's "
+ "own reason in the retrieval manifest."
+ )
return Disclosure(
key="claim_pairing",
level="warn",
token=CLAIM_PAIRING_WITHHELD_TOKEN,
- text=(
- f"This claim cites {labels}, and the {CLAIM_PAIRING_WITHHELD_TOKEN} about "
- f"{'that label' if one else 'those labels'}: two readings of the "
- "bibliography name different papers for it, so the source fetched under "
- f"that label may not be the paper the manuscript actually cites. No "
- f"verdict was reached on {'it' if one else 'them'} — check the retrieval "
- "manifest before treating this claim as checked."
- ),
+ text=text,
short=f"{labels} {CLAIM_PAIRING_WITHHELD_TOKEN}",
)
@@ -1230,10 +1282,10 @@ def _claim_pairing(claim, manifest=None) -> Disclosure | None:
text=(
f"This claim cites {labels}, {CLAIM_PAIRING_RESOLVED_TOKEN}. Two readings "
"of the bibliography disagreed there. A model was shown both, every field "
- "of its answer was found verbatim in one of them, and a person accepted "
- "the result — so this verdict rests on a reading nobody checked against "
- "the printed page, chosen by a person. The numbering is still recorded as "
- "unconfirmed."
+ "of its answer was found verbatim in one of them, and you accepted the "
+ "result — so this verdict rests on a reading nobody checked against the "
+ "printed page, and you are who accepted it. The numbering is still "
+ "recorded as unconfirmed."
),
short=f"{labels} {CLAIM_PAIRING_RESOLVED_TOKEN}",
)
diff --git a/src/papertrace/reflist.py b/src/papertrace/reflist.py
index 9463444..2cf70f7 100644
--- a/src/papertrace/reflist.py
+++ b/src/papertrace/reflist.py
@@ -15,7 +15,7 @@
from . import ask
from .models import RefEntry, _fold
-from .refs import DOI_RE
+from .refs import DOI_RE, _entry
REFLIST_PROMPT = """You are the reference-list reading step of a peer-review fact-checker.
@@ -488,7 +488,7 @@ def _parse_array(raw: str) -> list[dict] | None:
def resolve_disputed(
labels: list[str],
- entries: dict[str, list[RefEntry]],
+ names: tuple[str, ...],
texts: tuple[str, ...],
*,
model: str | None = None,
@@ -512,19 +512,21 @@ def resolve_disputed(
3. Partial resolution is the normal outcome, not a degraded one. Nothing
downstream may require all-or-nothing.
- `entries` names each candidate reading, and its sorted keys are read in the
- same order the one caller (`cli._escalate_disputed`) builds `texts` in — so
- a reply's `seen_in: "A"/"B"` is translated into an actual reading NAME
- before it reaches `RefEntry.seen_in`, never stored as the bare letter.
- `RefEntry.seen_in`'s published vocabulary is reading names (`propose`
- enforces the same rule for its own "A"/"B"/"AB"), and a letter leaking
- through would put two vocabularies in one wire-format key.
+ `names[i]` MUST be the reading `texts[i]` actually came from — passed in
+ explicitly, the way `propose`'s `label_a`/`label_b` are, rather than derived
+ from a separate dict of candidates. A caller's candidates dict can hold more
+ readings (crossref, llm) than were ever shown to this call, and translating
+ a reply's `seen_in: "A"/"B"` through `sorted(that dict)` instead of through
+ the two names actually paired with `texts` can name the wrong reading
+ outright. `RefEntry.seen_in`'s published vocabulary is reading names
+ (`propose` enforces the same rule for its own "A"/"B"/"AB"), and a
+ mistranslated name is exactly the "two vocabularies in one wire-format key"
+ mistake that rule exists to prevent.
"""
- prov = ReflistProvenance(readings=list(texts))
+ prov = ReflistProvenance(readings=list(names))
if not labels:
return Resolution(provenance=prov)
- reading_names = sorted(entries)
prompt = (
RESOLVE_PROMPT.replace("<>", ", ".join(f"[{n}]" for n in labels))
.replace("<>", texts[0] if texts else "")
@@ -532,6 +534,9 @@ def resolve_disputed(
)
with ask.for_site(ask.SITE_REFS):
raw = ask._ask(prompt, model)
+ # a reply was obtained — everything from here on describes what came back,
+ # never whether anything came back at all (same rule `propose` follows).
+ prov.outcome = "read"
prov.model = ask.model_for(ask.SITE_REFS) or ""
items = _parse_array(raw)
@@ -593,17 +598,25 @@ def resolve_disputed(
letter = obj.get("seen_in")
if isinstance(letter, str):
idx = {"A": 0, "B": 1}.get(letter.strip().upper())
- if idx is not None and idx < len(reading_names):
- seen_in = [reading_names[idx]]
-
- resolved[label] = RefEntry(
- num=label,
- raw=raw_text,
- doi=kept.get("doi"),
- title=kept.get("title"),
- year=kept.get("year"),
- reason="a disputed label, resolved by a model reading of both texts and "
- "accepted by a person — a reading, not a confirmed numbering",
- seen_in=seen_in,
+ if idx is not None and idx < len(names):
+ seen_in = [names[idx]]
+
+ # Built through `_entry` — the one place that already knows what a
+ # complete entry needs (it parses a doi/year out of `raw` and computes
+ # the slug) — then overlaid with the fields VERIFIED above, never left
+ # to `_entry`'s own regexes for something already checked verbatim. A
+ # bare `RefEntry(...)` here is what let `slug` go missing: `resolve_all`
+ # never re-slugs a substituted entry, and `check.py` refuses to judge
+ # one with none, so a resolved label reached the manifest as settled
+ # while silently receiving no verdict at all.
+ e = _entry(label, raw_text)
+ e.doi = kept.get("doi")
+ e.title = kept.get("title")
+ e.year = kept.get("year")
+ e.reason = (
+ "a disputed label, resolved by a model reading of both texts and "
+ "accepted by a person — a reading, not a confirmed numbering"
)
+ e.seen_in = seen_in
+ resolved[label] = e
return Resolution(resolved=resolved, still_disputed=unresolved, provenance=prov)
diff --git a/src/papertrace/templates/report_terminal.html.j2 b/src/papertrace/templates/report_terminal.html.j2
index 5e02df3..6b128a3 100644
--- a/src/papertrace/templates/report_terminal.html.j2
+++ b/src/papertrace/templates/report_terminal.html.j2
@@ -106,6 +106,9 @@
{% for d in disclosures if d.key == "labels_disputed" %}
▸ check⚠ {{ d.short }}
{% endfor %}
+{% for d in disclosures if d.key == "numbering_resolution" %}
+ ▸ resolve⚠ {{ d.short }}
+{% endfor %}
{% for d in disclosures if d.key == "numbering_corroboration" %}
▸ resolve{{ d.short }}
{% endfor %}
diff --git a/tests/test_disclosure_parity.py b/tests/test_disclosure_parity.py
index d734d9b..6b838b4 100644
--- a/tests/test_disclosure_parity.py
+++ b/tests/test_disclosure_parity.py
@@ -414,6 +414,62 @@ def test_a_withheld_claim_pairing_disclosure_appears_in_every_format(tmp_path):
assert pairing.token in body, f"claim_pairing token missing from {name}"
+# --- Task 7 review, M7: the three remaining `claim_pairing` states, rendered --
+
+
+def test_a_model_resolved_claim_pairing_disclosure_appears_in_every_format(tmp_path):
+ """M7. `withheld` had a fixture; `llm_resolved`, `chosen` and
+ `corroborated` did not, so three new tokens reached four formats
+ unverified — coverage, not breakage (the key is in every allow-list
+ already), but the parity loop is what makes that provable per state."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(manuscript="m.pdf", labels_resolved=["7"], numbering_choice="llm_resolved")
+ claim = _claim(refs=["7"])
+ results = RunResults(manuscript="m.pdf", converter="pymupdf", claims=[claim])
+ write_reports(results, manifest, tmp_path, png=False)
+ rendered = {name: (tmp_path / name).read_text() for name in FORMATS}
+
+ d = next(x for x in claim_disclosures(claim, manifest) if x.key == "claim_pairing")
+ for name, body in rendered.items():
+ assert d.token in body, f"claim_pairing (llm_resolved) token missing from {name}"
+
+
+def test_a_user_chosen_claim_pairing_disclosure_appears_in_every_format(tmp_path):
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(manuscript="m.pdf", labels_resolved=["9"], numbering_choice="pymupdf")
+ claim = _claim(refs=["9"])
+ results = RunResults(manuscript="m.pdf", converter="pymupdf", claims=[claim])
+ write_reports(results, manifest, tmp_path, png=False)
+ rendered = {name: (tmp_path / name).read_text() for name in FORMATS}
+
+ d = next(x for x in claim_disclosures(claim, manifest) if x.key == "claim_pairing")
+ for name, body in rendered.items():
+ assert d.token in body, f"claim_pairing (chosen) token missing from {name}"
+
+
+def test_a_corroborated_claim_pairing_disclosure_appears_in_every_format(tmp_path):
+ """`info`-level, and only ever printed beside an existing `warn` — this
+ claim's co-citation is unretrieved (a real, unrelated finding), which is
+ what earns corroboration a place on the same row."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(
+ manuscript="m.pdf", numbering_corroborated=True,
+ corroborating_readings=["parsed", "pymupdf"],
+ )
+ claim = _claim(refs=["3", "4"], unjudged_refs=["4"])
+ results = RunResults(manuscript="m.pdf", converter="pymupdf", claims=[claim])
+ write_reports(results, manifest, tmp_path, png=False)
+ rendered = {name: (tmp_path / name).read_text() for name in FORMATS}
+
+ fired = claim_disclosures(claim, manifest)
+ d = next(x for x in fired if x.key == "claim_pairing" and x.level == "info")
+ for name, body in rendered.items():
+ assert d.token in body, f"claim_pairing (corroborated) token missing from {name}"
+
+
# --- the HTML reports actually escape what they interpolate ------------------
@@ -499,3 +555,63 @@ def test_every_claim_disclosure_key_is_rendered_by_every_jinja_format():
f"{name} renders no branch for {sorted(missing)} — it filters claim "
"disclosures by explicit key, so a new one is dropped, not surfaced"
)
+
+
+# --- Task 7 review, M6: the escalation's run-level surface ------------------
+
+
+def test_a_resolved_run_gets_a_run_level_disclosure_in_every_format(tmp_path):
+ """M6. Without this, an 11-of-14-resolved run said so only on the claims
+ that cite one of the 11 — a reader who stops at the top of the report
+ never learns that some verdicts below rest on a model reading or a
+ person's choice."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(manuscript="m.pdf", labels_resolved=["6", "7"],
+ numbering_choice="llm_resolved")
+ results = RunResults(manuscript="m.pdf", converter="pymupdf", claims=[_claim()])
+ write_reports(results, manifest, tmp_path, png=False)
+ rendered = {name: (tmp_path / name).read_text() for name in FORMATS}
+
+ d = next(x for x in run_disclosures(results, manifest) if x.key == "numbering_resolution")
+ for name, body in rendered.items():
+ assert d.token in body, f"numbering_resolution token missing from {name}"
+
+
+def test_an_ordinary_run_with_nothing_resolved_adds_no_disclosure(tmp_path):
+ """A run where nothing was ever disputed must not grow a disclosure about
+ a resolution that never happened."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(manuscript="m.pdf")
+ fired = run_disclosures(
+ RunResults(manuscript="m.pdf", converter="pymupdf", claims=[_claim()]), manifest
+ )
+ assert not any(d.key == "numbering_resolution" for d in fired)
+
+
+def test_a_withheld_run_adds_no_resolution_disclosure(tmp_path):
+ """`choice == "withheld"` settled nothing — `_labels_disputed` already
+ covers this case, and `_numbering_resolution` must stay silent about it
+ rather than claim a resolution that did not happen."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(manuscript="m.pdf", labels_disputed=["9"],
+ numbering_choice="withheld", numbering_chosen_by="user")
+ fired = run_disclosures(
+ RunResults(manuscript="m.pdf", converter="pymupdf", claims=[_claim()]), manifest
+ )
+ assert not any(d.key == "numbering_resolution" for d in fired)
+
+
+def test_a_claim_citing_both_a_disputed_and_a_resolved_label_mentions_both(tmp_path):
+ """m6. A claim citing one still-disputed label and one resolved label
+ used to get only the withheld text — a reader stopping there would not
+ learn the other citation's verdict rests on a resolution at all."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(manuscript="m.pdf", labels_disputed=["9"], labels_resolved=["7"])
+ claim = _claim(refs=["7", "9"], withheld_refs=["9"])
+ d = next(x for x in claim_disclosures(claim, manifest) if x.key == "claim_pairing")
+ assert "[9]" in d.text
+ assert "[7]" in d.text, "the resolved co-citation must not vanish from the note"
diff --git a/tests/test_escalation.py b/tests/test_escalation.py
index 85f9af2..40a13ce 100644
--- a/tests/test_escalation.py
+++ b/tests/test_escalation.py
@@ -17,6 +17,7 @@
never by pretending the pure dict-builder returns something it does not.
"""
+import json
import sys
from pathlib import Path
@@ -138,9 +139,19 @@ def ask(*a, **k):
assert "First paper" in seen["text"], "the agreed label's text is context, not a finding"
-def test_the_path_to_the_disagreement_file_is_printed(disputed, capsys):
+def test_the_path_to_the_disagreement_file_is_printed(disputed, monkeypatch, capsys):
"""A file nobody is told about is a file nobody reads, and the menu's first
- option spends money on a question the reader could answer by looking."""
+ option spends money on a question the reader could answer by looking.
+
+ A wide console is forced (m10 of the Task 7 review): the absolute
+ `tmp_path` printed here is one unbroken "word" to rich's wrapper, and at
+ the ~80-column default it can hard-break mid-filename rather than at a
+ space, which would fail this assertion on any tmp_path long enough — this
+ one happened to pass only by being short.
+ """
+ from rich.console import Console
+
+ monkeypatch.setattr(cli, "console", Console(width=1000))
pdf, case, answers = disputed
answers.append("2")
_run_refs(pdf, case)
@@ -171,10 +182,13 @@ def test_the_default_answer_asks_the_model_and_records_the_user_as_choosing(
# True, and correctly so: `numbering_verified` means the chosen list
# accounts for exactly the labels the body cites — a question of EXTENT,
# which this fixture's parse does answer. A disputed label is a question
- # of CONTENT. The escalation may not touch this flag in either
- # direction, and asserting it is unchanged is what pins that;
- # `tests/test_numbering_invariant.py` proves no model path can set it
- # where it is legitimately False.
+ # of CONTENT. No branch of `_escalate_disputed` assigns `rec.verified`
+ # (grep it) — option 1 only swaps individual entries within the same
+ # list, never its length or numbering, so the extent claim this fixture
+ # already earned stands. `test_a_resolved_label_judges_against_the_resolved_entry`
+ # below pins the direction actually feared (a resolution must not flip
+ # this from False to True); Task 8 is the one that closes the remaining
+ # menu choices over more fixtures.
assert manifest.numbering_verified is True
assert manifest.labels_resolved == ["2"]
assert manifest.labels_disputed == []
@@ -182,6 +196,39 @@ def test_the_default_answer_asks_the_model_and_records_the_user_as_choosing(
# against the paper the resolution named, not the one it ruled against
two = next(e for e in manifest.entries if e.num == "2")
assert two.doi == "10.1000/beta"
+ assert two.slug, "C1: a slug-less entry is silently dropped by check.py"
+
+
+def test_a_real_resolution_call_produces_a_usable_entry_end_to_end(disputed, monkeypatch):
+ """C1, through the REAL `resolve_disputed` — not a stub.
+
+ Every other option-1 test in this file replaces `resolve_disputed` with a
+ lambda whose entry is built through `refs._entry`, which sets the one
+ field the real function forgot (`slug`). A test double built from a
+ production helper can supply an invariant the code under test omits, and
+ then hide exactly the bug it looks like it is testing — that is how a
+ slug-less resolved entry (silently dropped by `check.py`, downloaded to
+ `sources_resolved/None.pdf`) survived review. This mocks only the seam
+ `resolve_disputed` itself calls (`ask._ask`), so the real field
+ verification and `refs._entry`-based construction actually run.
+ """
+ import papertrace.ask as ask_mod
+
+ pdf, case, answers = disputed
+ answers.append("1")
+ # "Second paper" must be found verbatim in one of the two texts shown to
+ # the call — it is, in `refs_text`, printed as "[2] Beta B. Second paper.
+ # 2021." — so this is a genuine verbatim-verified answer, not a stub.
+ reply = json.dumps([{"num": "2", "title": "Second paper", "year": "2021"}])
+ monkeypatch.setattr(ask_mod, "_ask", lambda prompt, model=None: reply)
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.numbering_choice == "llm_resolved"
+ assert manifest.labels_resolved == ["2"]
+ two = next(e for e in manifest.entries if e.num == "2")
+ assert two.title == "Second paper"
+ assert two.slug, "the real resolve_disputed must produce a usable slug (C1)"
def test_a_resolver_failure_leaves_every_label_disputed_and_is_reported(disputed, monkeypatch):
@@ -204,13 +251,11 @@ def boom(*a, **k):
assert manifest.numbering_chosen_by == "user"
assert manifest.labels_disputed == ["2"]
assert manifest.labels_resolved == []
- # True, and correctly so: `numbering_verified` means the chosen list
- # accounts for exactly the labels the body cites — a question of EXTENT,
- # which this fixture's parse does answer. A disputed label is a question
- # of CONTENT. The escalation may not touch this flag in either
- # direction, and asserting it is unchanged is what pins that;
- # `tests/test_numbering_invariant.py` proves no model path can set it
- # where it is legitimately False.
+ # True, and correctly so: `numbering_verified` is an EXTENT claim about
+ # this fixture's parse, unaffected by a resolution attempt that raised
+ # before touching anything. No branch of `_escalate_disputed` assigns
+ # `rec.verified` (grep it); Task 8's job is to close this over every
+ # remaining menu choice with its own fixtures.
assert manifest.numbering_verified is True
@@ -233,14 +278,17 @@ def test_choosing_one_reading_whole_records_which_one(disputed):
manifest = _run_refs(pdf, case)
assert manifest.numbering_choice == "pymupdf"
assert manifest.numbering_chosen_by == "user"
- # True, and correctly so: `numbering_verified` means the chosen list
- # accounts for exactly the labels the body cites — a question of EXTENT,
- # which this fixture's parse does answer. A disputed label is a question
- # of CONTENT. The escalation may not touch this flag in either
- # direction, and asserting it is unchanged is what pins that;
- # `tests/test_numbering_invariant.py` proves no model path can set it
- # where it is legitimately False.
- assert manifest.numbering_verified is True
+ # False, correctly: `reconcile` measured EXTENT against the "parsed"
+ # reading, not the "pymupdf" one just adopted in its place. Carrying the
+ # old True forward would print "numbering confirmed" about a list nobody
+ # measured (Task 7 review, M1) — cleared here, never widened to a second
+ # meaning, which is why it is False rather than some other value.
+ assert manifest.numbering_verified is False
+ assert manifest.numbering_note.startswith("you chose")
+ # M1's other half: `reference_source` must name the reading actually
+ # adopted, not the one `reconcile` measured before it was discarded
+ assert manifest.reference_source == "pymupdf"
+ assert manifest.numbering_ledger == {}
# the entries carried forward are that reading's
assert [e.num for e in manifest.entries] == ["1", "2"]
gamma = next(e for e in manifest.entries if e.num == "2")
@@ -391,3 +439,185 @@ def test_a_run_never_says_confirmed_beside_a_withheld_label(disputed, capsys):
assert "readings disagree at [2]" in out
assert "numbering confirmed" not in out, "confirmed printed beside a withheld label"
assert "accounts for every cited label" in out
+
+
+# --- Task 7 review, Majors 8 and 9: a reading cannot launder its own coin-flip
+
+
+def test_option_3_excludes_a_reading_that_duplicates_the_disputed_label(tmp_path, monkeypatch):
+ """M8. `label_agreement` marks a label `disputed` outright when a single
+ reading carries it twice — "downgrading that to `single` ... would let a
+ verdict rest on the coin-flip" (`refs.py`). Offering that same reading in
+ option 3's menu would let a person adopt it whole anyway, laundering the
+ coin-flip into `labels_resolved`. It must not be offered at all."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ calls = []
+
+ def ask(prompt_text, **kw):
+ calls.append((prompt_text, kw.get("choices")))
+ return {" choice": "3", " which reading": "pymupdf"}.get(prompt_text, "3")
+
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(ask))
+
+ dup_a = _entry("2", "Alpha A. First. 2020.")
+ dup_b = _entry("2", "Alpha A. Different. 2021.")
+ single = _entry("2", "Gamma G. A different paper. 2019.")
+ rec = _rec(labels_disputed=["2"])
+ out = cli._escalate_disputed(
+ tmp_path, rec, [dup_a],
+ {"parsed": [dup_a, dup_b], "pymupdf": [single]},
+ {"parsed": "text a", "pymupdf": "text b"},
+ )
+
+ top_menu = next(c for c in calls if c[0] == " choice")
+ assert "3" in top_menu[1], "pymupdf alone still qualifies, so option 3 must be offered"
+ which_call = next(c for c in calls if c[0] == " which reading")
+ assert which_call[1] == ["pymupdf"], "the duplicating reading must not be offered"
+ assert rec.choice == "pymupdf"
+ assert out == [single]
+
+
+def test_option_3_is_never_offered_when_nothing_qualifies(tmp_path, monkeypatch):
+ """M9. `offered[0]` used to raise `IndexError` once neither `parsed` nor
+ `pymupdf` qualified — reachable whenever the run's own parse duplicates
+ the disputed label (this fixture) or is simply absent (a crossref+llm-only
+ dispute). The fix is at the menu: option 3 is never offered, so `choices`
+ passed to the real `Prompt.ask` never contains it. This pins the
+ degradation for a caller that ignores `choices` anyway (a misbehaving
+ test double, never a real terminal) — it falls through to the
+ model-resolution path rather than crashing.
+ """
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ calls = []
+
+ def ask(prompt_text, **kw):
+ calls.append((prompt_text, kw.get("choices")))
+ return "3"
+
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(ask))
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, names, texts, model=None: reflist_mod.Resolution(
+ still_disputed=list(labels), provenance=reflist_mod.ReflistProvenance(),
+ ),
+ )
+ dup_a = _entry("2", "Alpha A. First. 2020.")
+ dup_b = _entry("2", "Alpha A. Different. 2021.")
+ rec = _rec(labels_disputed=["2"])
+ out = cli._escalate_disputed(
+ tmp_path, rec, [dup_a], {"parsed": [dup_a, dup_b]}, {"parsed": "text"},
+ )
+
+ top_menu = next(c for c in calls if c[0] == " choice")
+ assert "3" not in top_menu[1]
+ assert not any(c[0] == " which reading" for c in calls), "never reached — nothing offered"
+ assert rec.choice == "llm_resolved"
+ assert out == [dup_a]
+
+
+# --- Task 7 review, m5 and M3: an orphaned "resolved" label, and honest reasons
+
+
+def test_a_resolved_label_absent_from_entries_stays_disputed(tmp_path, monkeypatch):
+ """m5. `resolve_disputed` only ever answers about labels it was asked
+ about, but those labels come from `rec.labels_disputed` — computed over
+ ALL candidate readings (crossref, llm included), not just the chosen
+ `entries` list. A label the chosen list never carried at all has nothing
+ to substitute, and must not be recorded resolved with no entry behind it."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ orphan = _entry("99", "Nobody In The Chosen List. 2020.")
+ real = _entry("2", "Beta B. The real one resolved. 2021.")
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, names, texts, model=None: reflist_mod.Resolution(
+ resolved={"2": real, "99": orphan}, still_disputed=[],
+ provenance=reflist_mod.ReflistProvenance(model="claude-opus-5"),
+ ),
+ )
+ chosen = _entry("2", "Beta B. Original. 2021.")
+ rec = _rec(labels_disputed=["2", "99"])
+ out = cli._escalate_disputed(
+ tmp_path, rec, [chosen], {"parsed": [chosen]}, {"parsed": "text"},
+ )
+
+ assert rec.labels_resolved == ["2"], "99 has no entry to substitute"
+ assert rec.labels_disputed == ["99"], "orphaned, not silently dropped"
+ assert out == [real]
+
+
+def test_the_console_names_the_real_reason_a_label_stayed_disputed(tmp_path, monkeypatch, capsys):
+ """M3. A malformed reply, an unprinted title and a genuine "cannot tell"
+ are three different findings `resolve_disputed` already records in
+ `provenance` — asserting one blanket "the model could not tell" over all
+ three is a plausible-looking cause standing in for the honest one."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ prov = reflist_mod.ReflistProvenance(model="claude-opus-5")
+ prov.fields_discarded.append("[9].title")
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, names, texts, model=None: reflist_mod.Resolution(
+ still_disputed=["9"], provenance=prov,
+ ),
+ )
+ rec = _rec(labels_disputed=["9"])
+ entry = _entry("9", "Somebody. 2020.")
+ cli._escalate_disputed(tmp_path, rec, [entry], {"parsed": [entry]}, {"parsed": "text"})
+ out = capsys.readouterr().out
+
+ assert "not printed in either text" in out
+ assert "could not tell which paper it names" not in out
+
+
+# --- Task 7 review, m1: the span window backs up to the previous line break -
+
+
+def test_label_span_backs_up_past_a_long_preceding_entry():
+ """m1. The old fixed-distance backtrack (`_SPAN_BEFORE = 260`) degraded
+ silently once the preceding entry was longer than it — routine with a
+ full author list. Backing up to the previous line break is exactly one
+ entry by construction and cannot degrade regardless of length."""
+ long_entry = "A" * 400
+ text = f"[1] {long_entry}\n[2] Beta B. Second paper. 2021.\n"
+ span = cli._label_span(text, "2")
+ assert long_entry in span, "a 400-char preceding entry must still be included"
+
+
+def test_label_span_at_the_very_start_of_the_text():
+ text = "[1] Alpha A. First paper. 2020.\n[2] Beta B. Second paper. 2021.\n"
+ span = cli._label_span(text, "1")
+ assert span.startswith("[1]")
+
+
+def test_label_span_is_empty_when_the_numeral_was_never_printed():
+ assert cli._label_span("no numbered list here", "7") == ""
+
+
+def test_a_resolved_entrys_slug_collision_is_disambiguated(tmp_path, monkeypatch):
+ """C1's remaining half. `resolve_all` never re-slugs, so a resolved entry
+ whose freshly computed slug collides with one already carried in the
+ chosen list would share a download path with it — the second download
+ overwrites the first. `_unique_slugs` re-runs over the WHOLE substituted
+ list, not just the newly resolved entries, so a collision introduced by
+ the substitution itself is caught too.
+ """
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ kept = _entry("1", "Smith J. An unrelated paper. 2020.")
+ wrong_two = _entry("2", "Wrong Wrong. 1999.")
+ resolved_same_slug = _entry("2", "Smith J. The resolved paper. 2020.")
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, names, texts, model=None: reflist_mod.Resolution(
+ resolved={"2": resolved_same_slug}, still_disputed=[],
+ provenance=reflist_mod.ReflistProvenance(model="claude-opus-5"),
+ ),
+ )
+ rec = _rec(labels_disputed=["2"])
+ out = cli._escalate_disputed(
+ tmp_path, rec, [kept, wrong_two],
+ {"parsed": [kept, wrong_two]}, {"parsed": "text"},
+ )
+ slugs = [e.slug for e in out]
+ assert len(slugs) == len(set(slugs)), f"colliding slugs would share a download path: {slugs}"
diff --git a/tests/test_reflist.py b/tests/test_reflist.py
index aa2627f..d9a9ef0 100644
--- a/tests/test_reflist.py
+++ b/tests/test_reflist.py
@@ -7,6 +7,7 @@
way this module could otherwise name a paper the page never printed.
"""
+import json
import sys
from pathlib import Path
@@ -400,3 +401,183 @@ def test_a_reading_letter_of_the_wrong_type_is_reported_as_a_wrong_shape(monkeyp
assert any("reading: not a string" in f for f in prov.fields_discarded), (
prov.fields_discarded
)
+
+
+# --- resolving a disputed label: three rules on the reply, and a slug -------
+#
+# `resolve_disputed` is the only place a model reading changes what `check`
+# judges — a person has already seen the disagreement and asked for this. Zero
+# of these tests existed before this round (M2 of the Task 7 review): every
+# test that reached menu option 1 replaced this function with a stub built
+# through `refs._entry`, which sets the one field the real function forgot
+# (C1 — a slug-less entry is silently dropped from judgement by `check.py` and
+# downloads to `sources_resolved/None.pdf`, overwriting itself label after
+# label). These are direct, so that mistake cannot repeat.
+
+_RESOLVE_A = "6. Fujita S, Mori S. Characterization of Brain Volume Changes. 2023. doi:10.1001/jamanetworkopen.2023.18153\n"
+_RESOLVE_B = "6. Wachinger C. Something else entirely. 2019. doi:10.1000/wach\n"
+_RESOLVE_NAMES = ("docling 2.8.0", "pymupdf")
+
+
+def _resolve_reply(monkeypatch, payload: str, *, model: str = "claude-opus-5"):
+ """Same seam, same qualification rule as `_reply` above: `reflist` calls
+ `ask._ask` by qualified name, so the fake is installed on `ask_mod`."""
+ calls = []
+
+ def fake(prompt, model_arg=None):
+ calls.append({"prompt": prompt})
+ return payload
+
+ monkeypatch.setattr(ask_mod, "_ask", fake)
+ monkeypatch.setattr(ask_mod, "_MODELS", {ask_mod.SITE_REFS: model})
+ return calls
+
+
+def test_a_resolved_label_names_a_title_printed_in_one_of_the_readings(monkeypatch):
+ """The whole permission structure of this call. Nothing invented may name a
+ paper, so the title comes back only if it can be grepped out of the text the
+ model was shown."""
+ reply = json.dumps([{
+ "num": "6",
+ "title": "Characterization of Brain Volume Changes",
+ "doi": "10.1001/jamanetworkopen.2023.18153",
+ "year": "2023",
+ }])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert list(res.resolved) == ["6"]
+ assert res.still_disputed == []
+ assert res.resolved["6"].doi == "10.1001/jamanetworkopen.2023.18153"
+
+
+def test_a_resolved_entry_carries_a_slug(monkeypatch):
+ """C1. `resolve_all` never re-slugs and `check.py` refuses to judge a
+ slug-less entry — a bare `RefEntry(...)` here is what let a resolved label
+ reach the manifest as "resolved" while `check` silently dropped it. Built
+ through `refs._entry`, the one place that knows what a complete entry
+ needs, so this cannot regress to a hand-rolled constructor again."""
+ reply = json.dumps([{
+ "num": "6",
+ "title": "Characterization of Brain Volume Changes",
+ "doi": "10.1001/jamanetworkopen.2023.18153",
+ "year": "2023",
+ }])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved["6"].slug, "a resolved entry with no slug is dropped by check.py"
+ assert res.resolved["6"].slug != "ref"
+
+
+def test_a_title_printed_in_neither_reading_leaves_the_label_disputed(monkeypatch):
+ """An unverifiable *field* is discarded; an unverifiable *title* is the label
+ staying disputed. The title is the only field that identifies a paper, so a
+ reply whose title cannot be found has answered nothing."""
+ reply = json.dumps([{"num": "6", "title": "A Paper Nobody Printed", "year": "2023"}])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved == {}
+ assert res.still_disputed == ["6"]
+
+
+def test_an_unverifiable_doi_is_discarded_without_costing_the_label(monkeypatch):
+ """A truncated DOI is extraction damage, not a different paper. The field
+ goes; the resolution stands and the entry carries a recorded gap."""
+ reply = json.dumps([{
+ "num": "6",
+ "title": "Characterization of Brain Volume Changes",
+ "doi": "10.1038/s41591-",
+ }])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved["6"].doi is None
+ assert "doi" in res.provenance.fields_discarded
+
+
+def test_cannot_tell_is_an_answer_and_keeps_the_label_disputed(monkeypatch):
+ """Abstention is first-class. A call that must always answer will
+ confabulate on exactly the labels two readings disagree about."""
+ reply = json.dumps([{"num": "6", "cannot_tell": True}])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved == {}
+ assert res.still_disputed == ["6"]
+
+
+def test_partial_resolution_is_representable(monkeypatch):
+ """11 of 14 resolved and 3 still disputed is a real outcome, not an error.
+ No code path may require all-or-nothing."""
+ text_a = _RESOLVE_A + "7. Weston AD. Automated abdominal segmentation. 2019.\n"
+ text_b = _RESOLVE_B + "7. Someone Else. A different paper. 2015.\n"
+ reply = json.dumps([
+ {"num": "6", "title": "Characterization of Brain Volume Changes"},
+ {"num": "7", "cannot_tell": True},
+ ])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6", "7"], _RESOLVE_NAMES, (text_a, text_b))
+ assert list(res.resolved) == ["6"]
+ assert res.still_disputed == ["7"]
+
+
+def test_a_label_the_reply_never_mentions_stays_disputed(monkeypatch):
+ """Silence is not consent. A reply that answers 1 of 2 questions has
+ answered 1, and the unanswered label is not repaired into agreement."""
+ reply = json.dumps([{"num": "6", "title": "Characterization of Brain Volume Changes"}])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6", "9"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.still_disputed == ["9"]
+
+
+def test_a_malformed_reply_resolves_nothing_rather_than_raising(monkeypatch):
+ """`refs` has already done useful work by this point. A model that returns
+ prose leaves every label disputed — the state the run was in before it
+ asked — and says so in the provenance. "I think maybe [6]?" is not prose
+ without brackets by accident: `ask._parse_json_array`'s first-`[`-to-last-
+ `]` slice reads it as the syntactically valid one-element list `[6]`, and
+ `_parse_array`'s dict check is what still refuses it."""
+ _resolve_reply(monkeypatch, "I think maybe [6]?")
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved == {}
+ assert res.still_disputed == ["6"]
+ assert res.provenance.discarded_whole
+
+
+def test_seen_in_is_translated_using_the_names_matching_the_texts_order(monkeypatch):
+ """M4. `names[i]` must be the reading `texts[i]` actually came from — never
+ derived from a different dict of candidates that can hold MORE readings
+ (crossref, llm) than were shown to this call. A reply of "B" must land the
+ SECOND name passed here, not the second name of some other collection."""
+ reply = json.dumps([{
+ "num": "6", "title": "Characterization of Brain Volume Changes", "seen_in": "B",
+ }])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved["6"].seen_in == ["pymupdf"]
+
+
+def test_an_unrecognised_seen_in_is_dropped_rather_than_guessed(monkeypatch):
+ reply = json.dumps([{
+ "num": "6", "title": "Characterization of Brain Volume Changes",
+ "seen_in": "whichever one was clearer",
+ }])
+ _resolve_reply(monkeypatch, reply)
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.resolved["6"].seen_in == []
+
+
+def test_outcome_is_read_once_a_reply_is_obtained(monkeypatch):
+ """m8. `propose` sets this the moment a reply comes back, before it is even
+ parsed — `resolve_disputed` must do the same, or a caller that inspects
+ `provenance.outcome` after a malformed reply sees the module's own default
+ ("not_attempted") for a call that plainly happened."""
+ _resolve_reply(monkeypatch, "not json at all")
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert res.provenance.outcome == "read"
+
+
+def test_no_labels_makes_no_model_call_at_all(monkeypatch):
+ """Nothing disputed is nothing to ask about — never spend a call for it."""
+ calls = _resolve_reply(monkeypatch, "[]")
+ res = reflist.resolve_disputed([], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+ assert calls == []
+ assert res.resolved == {}
+ assert res.still_disputed == []
From 17f75202b74ff46066e895849db22db2eaa4228e Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Fri, 18 Sep 2026 15:10:55 +0200
Subject: [PATCH 32/44] docs: the authorisation ADR 0002 withheld, and the seam
rule that replaces a convention
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
ADR 0002 pre-specified this remedy — a third candidate reading behind an opt-in
flag, not a replacement of the parser — and explicitly declined to authorise
it. ADR 0003 is the authorisation, with the trigger now evidenced: 22 printed
references read as 18, six substantive verdicts about papers the manuscript
never cited, every one of them carrying title_check verified. It records the
four rejected alternatives so they are not re-proposed.
CLAUDE.md's "check.py is the only module that calls a model" becomes "one file
in src/ shells out, and a test says which" — the rule was protecting the seam,
not the module. The refs.py section gains per-label agreement, why the join is
the printed numeral and not the position, and why single prints.
README gains a does bullet and a does-not bullet stating the ceiling:
corroborated is not verified, every reading read the same document, and nothing
a model returns or a user answers sets numbering_verified. Line 216 is
untouched and nothing is retracted. No accuracy figure anywhere — none has been
measured.
evals/DESIGN.md gets a pointer to the sibling eval plan, not the plan.
---
CHANGELOG.md | 94 +++++++++++++++++
CLAUDE.md | 62 +++++++++--
README.md | 44 ++++++--
docs/adr/0003-llm-reference-list.md | 156 ++++++++++++++++++++++++++++
evals/DESIGN.md | 37 +++++++
5 files changed, 380 insertions(+), 13 deletions(-)
create mode 100644 docs/adr/0003-llm-reference-list.md
diff --git a/CHANGELOG.md b/CHANGELOG.md
index db0178c..0cd3b98 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -107,6 +107,100 @@ before the first paid model call rather than after all of them. An answer of
whitespace arrives as `.` and is refused, for the reason `default_case` already
refuses the working directory: an audit needs a folder of its own.
+### Added — a third and fourth reading of the reference list, and a per-label verdict on the numbering
+
+`reconcile` arbitrated between two readings of the bibliography: the tool's own
+parse and the reference list the publisher deposited with Crossref. Both can be
+absent — an unpublished manuscript has no DOI to look up — and where the parse
+is the only reading, nothing can contradict it.
+
+Two more candidates now stand beside them. A flat-text (pymupdf) parse of the
+same PDF, free on a docling run and skipped on a pymupdf one because it would
+be identical. And a structured reference list proposed by a model, which is
+shown both texts and given no authority over either: **every field of its
+reply must be found verbatim in one of them or it is discarded**, one entry's
+unverifiable title discards the model's **whole reading** — a title is the
+one field that names the paper, so a reply that got one wrong cannot be
+trusted about the rest — and the model's list never becomes `reconcile`'s
+chosen reading. It is a voter. On by default, `--no-llm-refs` to turn it off,
+on both `refs` and `run`.
+
+Agreement is then computed **per printed citation label** — joined on the
+numeral the page carries, never on position — and reported as `agreed`,
+`single`, `disputed` or `absent`. Where two readings name different papers at a
+label, verdicts on claims citing it are **withheld**: the claim is reported
+`unchecked` with the label named, not `not_retrieved`, because the source was
+obtained and read. `single` deliberately prints, with the caveat it already
+carried. `numbering_corroborated` is a new, independent axis beside
+`numbering_verified`, which keeps its exact meaning.
+
+Where nothing accounted for the body's labels and labels are in dispute, an
+interactive run writes the full disagreement to
+`case/out/reference_disagreement.md` — every reading's fields and the verbatim
+text each was read from — **prints the path, and only then asks** what to do
+about it. Four options: resolve the disputed labels with a model, withhold
+verdicts on all of them, adopt one reading whole, or abort. Whatever is
+answered is recorded as `numbering_chosen_by: "user"`; a person consenting to
+proceed is an input, not evidence, and nothing a user answers can set
+`numbering_verified`. A non-interactive run withholds the disputed labels and
+asks nothing.
+
+Partial resolution is a normal, representable, reported outcome:
+`labels_resolved` and `labels_disputed` are both non-empty on a run where some
+labels were settled and some were not, and "cannot tell" from the resolution
+call is a correct answer that keeps a label withheld.
+
+### Added — `ask.py`, the one seam, with the model recorded per call site
+
+Every model call went through one `_ask` in `check.py`, which was a convention
+stated in `CLAUDE.md` with nothing enforcing it. It is now a file —
+`src/papertrace/ask.py`, the only file in `src/` that runs a subprocess — and
+`tests/test_ask.py` greps `src/` and asserts exactly that. Same flags, same
+timeout, same sandbox: `--safe-mode`, `--tools ""`, a private 0700 scratch cwd.
+
+### Fixed — a run that judged nothing could name a judge
+
+The model that judged the claims was a single module global overwritten by
+every `_ask` call, and the report's `Checker:` line rendered it. With `refs`
+also calling the seam, a run whose judging made zero calls — every cited
+source not retrieved, so nothing to judge — would have printed the
+**reference-list** model as the judge of verdicts it never saw. The model is
+now recorded per call site, and `last_model()` reads the judging site only:
+`None` where no judging happened, never the other site's answer.
+
+### Fixed — claim extraction had no retry, and the wizard's cost ceiling assumed it did
+
+`ASK_ATTEMPTS` exists so the wizard's advertised worst-case bill cannot drift
+from the real retry policy. The retry loop did not read it — it hardcoded one
+retry and matched the constant only because the constant happens to be 2 — and
+the single extraction call, whose failure fails the whole run, had no retry at
+all. Extraction now retries like everything else, and the wizard's ceiling moved
+with it.
+
+### Changed — the coverage of `unjudged_refs`, and a second axis on the numbering
+
+`ClaimResult.withheld_refs` is a new field and is **not** `unjudged_refs`. An
+entry in `unjudged_refs` means the source could not be obtained — nobody read
+it. A withheld reference **was** obtained; what is in doubt is whether it is the
+paper the label names. Putting it in `unjudged_refs` would report a retrieval
+gap that does not exist.
+
+`numbering_verified` is unchanged in meaning and unchanged in what can set it.
+`numbering_corroborated` is reported beside it, which lets the numbering
+disclosure stop crying wolf on the case it was firing on wrongly: a list two
+readings agree about entry for entry, longer than the highest cited label
+because three of its references are cited only in the supplement.
+
+### Known staleness — `examples/demo/output/` predates this feature
+
+`--llm-refs` defaults on, so a fresh demo run now makes one extra model call
+and, on the docling backend the demo uses, emits the `reflist` disclosure in
+all four formats. The committed showcase in `examples/demo/output/` was
+generated before this feature existed and shows none of it. That is staleness
+in the committed artefact, not a defect in the feature — the showcase needs a
+fresh end-to-end run (network, a logged-in `claude` CLI) to pick it up, and
+that run is a branch-level step, not part of this change.
+
## [0.6.0] — 2026-09-13 (beta)
Carries 0.4.1 and 0.5.0 with it. Neither was ever published, so neither has a
diff --git a/CLAUDE.md b/CLAUDE.md
index f4f1e86..0b6b99e 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -144,15 +144,63 @@ ingest → refs → scout → check → highlight → report
subset test is satisfied by exactly the labels in doubt, and
`_refusals_unconfirm` is what stops a matching count printing "numbering
confirmed" over a numbering the parser declined to stand behind.
-- **`check.py`** — the **only** module that calls a model, and only through the
- `_ask()` seam (`claude -p` subprocess; inherits the user's Claude Code login,
- no API key). Two prompts: `EXTRACT_PROMPT` then `CHECK_PROMPT`, one call per
+ **Agreement is per printed label, and disagreement is refused rather than
+ ranked** (0.7.0). `reconcile` arbitrates between the Crossref deposit and the
+ run backend's parse, unchanged; a flat-text pymupdf reading and a model
+ reading (`reflist.py`, `--llm-refs`) are **voters only** and reach neither
+ `reconcile`'s arguments nor `resolve_all`, so no model output can cause a
+ source to be resolved, downloaded or judged — only a verdict to be withheld.
+ `label_agreement` joins on `e.num`, the printed numeral, never on position:
+ `_first_divergence` zips, and would compare docling's 6th entry against
+ pymupdf's 6th and report divergence for the wrong reason. Any pair failing
+ `_same_work` is `disputed` — no majority vote, because *"a non-unique match is
+ refused, never ranked"* — and `single` deliberately **prints**, since a
+ pymupdf-backend run whose model candidate was discarded has one reading, every
+ label would be `single`, and the audit would report nothing at all. A
+ `boundary_ambiguous` entry does not speak for its label and neither does a
+ reading carrying that label twice: the first has said it cannot stand behind
+ the label, and for the second, which of the two it means is the question.
+ Every field of the model's reply must be found **verbatim** in one of the two
+ texts it was shown or it is discarded, and an unverifiable title discards the
+ answer — nothing invented may name a paper. `numbering_corroborated` is a
+ second, independent axis: `numbering_verified` keeps its exact meaning and
+ stays `False` through every model reply and every interactive choice, which
+ `tests/test_numbering_invariant.py` parametrises over and which is the one
+ thing in this feature that may not be relaxed for convenience.
+- **`ask.py`** — the **only** file in `src/` that runs a subprocess, and the
+ only place this codebase shells out to a model (`claude -p --safe-mode
+ --tools ""` in a private scratch cwd; inherits the user's Claude Code login,
+ no API key). "`check.py` is the only module that calls a model" was the rule
+ until `refs` needed a reading of the bibliography too — the rule was
+ protecting the seam, not the module, and `tests/test_ask.py` greps `src/` and
+ asserts exactly one file, which makes it enforceable rather than
+ conventional. Two callers, `check.py` and `refs.py` (through `reflist.py`),
+ and no third without that test going red. The model is recorded **per call
+ site** (`for_site`, `model_for`, `SITE_CHECK`, `SITE_REFS`), not in one
+ global: `_LAST_MODEL` was overwritten by every call, so a run whose judging
+ made zero calls — every cited source `not_retrieved`, nothing to judge —
+ printed the reference-list model as the `Checker:` of verdicts it never saw.
+ `_ask`'s signature is frozen at `(prompt, model=None)`; sixty-two test sites
+ patch it with a two-argument lambda, which is why the site travels out of
+ band in a context manager instead of as a third parameter.
+- **`check.py`** — every prompt and every verdict rule, and no subprocess of
+ its own. Two prompts: `EXTRACT_PROMPT` then `CHECK_PROMPT`, one call per
**document** so context stays small — an article, each of its supplements,
- and each of the audited paper's own are separate calls with separate verdicts. Also holds `coverage_audit()`, which is
- deliberately **mechanical and prompt-independent** — a regex
+ and each of the audited paper's own are separate calls with separate
+ verdicts. Call the bare name `_ask(...)`, imported `from .ask import _ask`, so
+ `monkeypatch.setattr(check_mod, "_ask", …)` still intercepts; rewriting a
+ call site as `ask._ask(...)` bypasses every patch and turns sixty-two offline
+ tests into live paid calls. `_ask_with_retry` wraps the seam and reads
+ `ASK_ATTEMPTS` rather than hardcoding one retry — the wizard prints a
+ worst-case bill derived from that constant. Also holds `coverage_audit()`,
+ which is deliberately **mechanical and prompt-independent** — a regex
(`_LABEL_GROUP`) over bracketed numeric labels, so a citation the extractor
- missed still surfaces. The module global `_LAST_MODEL` carries the judging
- model out to the report; truncation travels in a per-run `Truncations`.
+ missed still surfaces. **A `disputed` label is dropped from `avail` before any
+ call is made**, so a source whose identity two readings contradict is never
+ judged; if nothing survives the claim is `unchecked` with the labels named,
+ never `not_retrieved` — that source was obtained, and `withheld_refs` is a
+ different field from `unjudged_refs` for exactly that reason. `last_model()`
+ reads the `check` site; truncation travels in a per-run `Truncations`.
- **`highlight.py`** — the division of labour that keeps evidence trustworthy:
the model proposes page, block and verbatim anchor phrases; **Python** locates
them with PyMuPDF `page.search_for` and draws the boxes. Boxes are never
diff --git a/README.md b/README.md
index 9b918f4..ad75f87 100644
--- a/README.md
+++ b/README.md
@@ -95,12 +95,18 @@ went uncited?**
couldn't be retrieved is `⊘ not retrieved` — recorded, never guessed.
- **Check its own reference numbering before trusting it.** The citation label
is the join key between a claim and the source it is judged against, so a
- list off by one produces a confident audit of the *wrong papers*. Two
- independent readings are taken — the tool's parse of the printed list, and
- the reference list the publisher deposited with Crossref (`refs --doi`,
- defaulting to the DOI printed on page 1) — and the manuscript's own `[N]`
- markers arbitrate between them. A reading is used only if it accounts for
- exactly the labels the body cites. When neither does, the audit continues,
+ list off by one produces a confident audit of the *wrong papers*. Up to four
+ independent readings are taken — the tool's parse of the printed list, a
+ flat-text parse of the same PDF, the reference list the publisher deposited
+ with Crossref (`refs --doi`, defaulting to the DOI printed on page 1), and a
+ structured list proposed by a model that is shown the two texts and given
+ authority over neither (`--no-llm-refs` turns it off) — and the manuscript's
+ own `[N]` markers arbitrate between them. Only the parse and the deposit can
+ be *adopted* as the list; the other two are readings that can disagree, and a
+ disagreement is all they can do. Every field of the model's reply is checked
+ against the text it was shown and discarded if it is not printed there, so
+ nothing it invented can name a paper. A reading is used only if it accounts
+ for exactly the labels the body cites. When neither does, the audit continues,
the report says the numbering is unconfirmed, and every verdict on a claim
citing a doubtful label carries that caveat beside it. Crossref is a second
reading, **not** an oracle. A deposit this tool can only partly read is set
@@ -118,6 +124,22 @@ went uncited?**
publisher deposited are looked for in the reference list printed in the
paper — and where neither can, the list is used and the manifest says the
identity behind it was never confirmed.
+- **Withhold a verdict where two readings of the bibliography name different
+ papers.** The citation label is the join key, so a label whose readings
+ disagree is a label nothing can say the cited paper for — and a confident
+ verdict there is about a paper the manuscript may never have cited. Agreement
+ is computed for each printed label separately, on the numeral the page
+ carries. Any disagreement withholds: the claim is reported `⚠ not checked`
+ with the label named, **not** `⊘ not retrieved`, because that source was
+ obtained and read — what is in doubt is whether it is the paper the label
+ names. Where nothing accounted for the body's labels and labels are in
+ dispute, an interactive run writes the whole disagreement to
+ `case/out/reference_disagreement.md` — every reading's fields and the
+ verbatim printed text each came from — prints the path, and only then asks
+ whether to resolve them, withhold them, adopt one reading, or stop. Nothing
+ you answer marks the numbering as confirmed; the report records which choice
+ was made and that you made it. A run that is not attached to a terminal
+ withholds the disputed labels and asks nothing.
- Report every citation **occurrence** — each bracketed marker at its own place
in the text — that no extracted claim reached, so a second sentence citing an
already-checked reference is not silently counted as covered. It also
@@ -173,6 +195,16 @@ went uncited?**
**PaperTrace does not**
+- **Confirm** a reference numbering by agreement between its readings. Where
+ they agree, the report says the numbering was *corroborated*, and says which
+ readings corroborated it — never that it was verified. They all read the same
+ document: a reference the page layout destroyed is one every reading may have
+ missed, and a model agreeing with the parse read the same damaged text the
+ parse did. *"Verified"* still means one thing only — the manuscript's own
+ `[N]` markers accounted for exactly the labels a candidate list carried — and
+ nothing a model returns and no answer you give at a prompt can set it. Nor
+ does the model ever see the page: it is shown extracted text, the same as
+ every other model call in this tool.
- **Always** establish that a supplement belongs to the work it was attached
to. Where its own title or DOI names that work, it does; where only its
filename matched, nothing read it, and the audited paper's own are whatever
diff --git a/docs/adr/0003-llm-reference-list.md b/docs/adr/0003-llm-reference-list.md
new file mode 100644
index 0000000..e330740
--- /dev/null
+++ b/docs/adr/0003-llm-reference-list.md
@@ -0,0 +1,156 @@
+# ADR 0003 — An LLM reading of the reference list, as a fourth candidate
+
+- **Status:** accepted
+- **Date:** 2026-09-17
+- **Related:** ADR [0002](0002-no-grobid.md), ADR [0001](0001-no-gold-benchmark.md),
+ `src/papertrace/reflist.py`, `src/papertrace/ask.py`,
+ `docs/superpowers/specs/2026-09-16-llm-reference-list-design.md`
+
+## Context
+
+ADR 0002 declined to adopt GROBID and, in the same breath, specified the
+escalation it *would* accept and refused to authorise it:
+
+> If reference parsing ever becomes the dominant source of wrong audits, the
+> cheap escalation is **a third candidate reading** wired into the existing
+> `reconcile` arbitration behind an opt-in flag — not a replacement of the
+> parser. That path is strictly additive and leaves the two-reading behaviour
+> unchanged when the third is absent. This ADR does not authorise it; it
+> records it as the shape a future proposal should take.
+
+This ADR is that authorisation. The trigger is now met with evidence.
+
+A manuscript under review had 22 printed references parse as 18. Docling
+renders a hanging-indent numeral column as a GFM table, so five reference rows
+arrived as `| 6. | Author A (2019) … |` and were glued onto the entry above
+them as wrapped continuations. Every claim citing `[6]` or above was judged
+against a different paper than the one cited — the citation label is the join
+key, so the verdicts were confident and wrong.
+
+The verdicts were not merely wrong, they were **confident**. Gluing five
+references onto one entry left `_entry` scraping the next reference's DOI onto
+it, and the title check *passed* on that DOI, because the glued raw string held
+the words of both papers. `title_check: verified` was printed on every one of
+them. A stronger title check would not have helped: the reference string
+really did name the paper that was downloaded, alongside the one the label
+meant. That is the distinction this ADR turns on — the broken join is
+`label → entry`, and every check the tool had verified `entry → PDF`.
+
+Three root causes, all reproduced offline and all fixed deterministically
+first (0.7.0: `_unwrap_table_rows`, `_numerals_agree_with_position`, and the
+`_covers` conflation that let a refused entry's positional label still satisfy
+the extent check). Those fixes are not the subject of this ADR and needed no
+authorisation — they are parser bugs. What they cannot do is certify
+themselves, which is the sentence ADR 0002 already wrote about GROBID: *"a
+parser cannot certify itself."* Two readings of one bibliography, both
+produced by this tool from the same converter's output, share a failure mode.
+A third and fourth reading that do not are the only thing that can say so.
+
+## Decision
+
+**A model reading of the reference list is adopted as a candidate — a voter
+with no vote of its own — behind `--llm-refs/--no-llm-refs`, default on.**
+
+The reconciliation now weighs four readings of one bibliography: the Crossref
+deposit, the run backend's parse, a flat-text (pymupdf) parse of the same PDF,
+and a structured list proposed by a model. `reconcile()` itself is unchanged
+and still takes exactly two arguments — the deposit and the parse — because
+only those two can be *adopted* as the list a source is resolved and downloaded
+against. The flat-text and model readings never reach `reconcile`'s arguments
+or `resolve_all`; they are folded into a second, orthogonal computation,
+`label_agreement`, whose only power is to withhold a verdict. Five constraints
+define the shape, and each is the answer to a way this could have gone wrong:
+
+1. **The model's reading can only subtract verdicts.** `reconcile` still
+ chooses between the deposit and the parse. The flat-text and model readings
+ are voters only, so no model output can cause a source to be resolved,
+ downloaded or judged — it can only cause a verdict to be withheld. This is
+ the safety property the design rests on and
+ `tests/test_numbering_invariant.py` tests it directly.
+2. **Every field of the model's reply is found verbatim in one of the two
+ texts it was shown, or discarded.** A discarded field costs a recorded gap;
+ an accepted field is one a reader can grep out of the source text. Nothing
+ invented survives to name a paper. A title is the one field that names the
+ paper, so one entry's unverifiable title discards the model's **whole
+ reading** — never just that entry — and the run says the model's reading
+ was unusable, never that it agreed.
+3. **Agreement is computed per printed citation label, never by position.**
+ `_first_divergence` zips two readings by position and is right for its own
+ job — deciding whether Crossref's deposit and the parse still describe the
+ same paper — but reused here it would compare docling's 6th entry against
+ pymupdf's 6th and report a divergence for the wrong reason on any list a
+ reading split or merged. `label_agreement` joins on `e.num`, the printed
+ numeral, instead. Reading-order zipping has been rejected repeatedly in
+ this codebase and is rejected again here.
+4. **Any disagreement is `disputed`. No majority vote, no ranking.** The
+ codebase's own rule: *"a non-unique match is refused, never ranked."* A
+ disputed label is dropped from the sources `check.py` judges, and the claim
+ lands `unchecked` with a note — explicitly not `not_retrieved`, which would
+ claim a retrieval failure that did not happen.
+5. **`numbering_verified` keeps its exact current meaning and stays `False`
+ through every model reply and every interactive choice.** Corroboration is
+ a **second, independent axis** (`numbering_corroborated`), reported beside
+ it. No published boolean is redefined and no model path can flip one.
+
+Four alternatives were weighed and rejected:
+
+- **A pairwise visual vote** — "does this bibliography crop match that title
+ crop." It verifies `entry → PDF`, which is already deterministically
+ title-checked and which *passed* on every one of the wrong verdicts above.
+ The broken join is `label → entry`, and no crop carries a label, because the
+ numeral is what the converter destroyed. It also has no deterministic
+ verifier, so a false "match" would overwrite a correct
+ `numbering_verified = False` with a plausible confirmation — the one thing
+ CLAUDE.md forbids. It is moot besides: the real docling table block has
+ `regions=1` for all five glued references, so no per-entry rectangle exists
+ to crop.
+- **The LLM as arbiter.** The body's own `[N]` markers are the only reading
+ definitionally right about what the paper cites. Replacing a free, offline,
+ definitionally-correct arbiter with a probabilistic one is a strict loss.
+- **Field-level splicing across candidates by position.** pymupdf loses DOIs
+ entirely on some entries, so the moment one list is short by even one field,
+ a `num` from one reading would be stapled to another reading's `doi` at the
+ wrong position.
+- **Images through the seam.** `claude -p` has no image channel. Granting it
+ one would dismantle `--tools ""` and the private scratch cwd, and would
+ falsify the README's statement that the judge never receives a page image.
+
+`ask.py` is extracted as part of this decision. CLAUDE.md's rule *"`check.py`
+is the only module that calls a model"* was protecting the seam, not the
+module; with `refs` (through `reflist.py`) calling it too, the rule becomes
+*"one file in `src/` shells out, and a test says which"* — enforceable rather
+than conventional (`tests/test_ask.py::test_only_ask_py_shells_out_to_a_model`).
+It also fixes a live misattribution: the model that answered was one module
+global overwritten by every call, so a run whose judging made zero calls (every
+cited source `not_retrieved`, nothing to judge) would have printed the
+reference-list model as the `Checker:` of verdicts it never saw. The model is
+now recorded per call site.
+
+**No accuracy figure is claimed for any of this, and none may be.** ADR 0001
+governs: a figure needs a gold set this project's authors did not construct,
+labelled by at least two people who did not write the prompts, on **both**
+backends — the converter is the cause here, not a nuisance parameter. A
+pairing evaluation for the `(citation label, printed reference entry)` join is
+specified as a sibling task in `evals/DESIGN.md` and is not part of this ADR.
+
+## Consequences
+
+- Crossref remains a **candidate, not an oracle**, and so does the model.
+ Adding a fourth reading does not soften that; it is the same rule applied to
+ a fourth voter.
+- Two stages now call the model seam, so a per-call-site model record is
+ load-bearing rather than tidy. Anything that reads "which model did this"
+ must name a site.
+- A disputed label costs verdicts. That direction is deliberate and it is the
+ expensive one: the cost and the safety of this join are two different
+ metrics in `evals/DESIGN.md`'s sibling pairing task, reported separately and
+ never blended.
+- Where nothing accounted for the body's labels and labels are in dispute, an
+ interactive run writes the full disagreement to
+ `case/out/reference_disagreement.md` **before** it asks anything, and
+ records `numbering_chosen_by: "user"` for whatever the user answers. A
+ person consenting to proceed is an input, not evidence. A non-interactive
+ run withholds the disputed labels and asks nothing.
+- ADR 0002's decision is unchanged. GROBID is still not adopted and still not
+ benchmarked; this ADR takes the escalation ADR 0002 named, not the one it
+ refused.
diff --git a/evals/DESIGN.md b/evals/DESIGN.md
index d270bff..35bac6b 100644
--- a/evals/DESIGN.md
+++ b/evals/DESIGN.md
@@ -50,6 +50,43 @@ a correct answer. A case whose labellers could not agree gets
`gold_verdict: null`, is kept in the set, and is excluded from every
denominator — deleting it would quietly make the set easier.
+### A second unit, specified elsewhere: (citation label, printed reference entry)
+
+The unit above is `(manuscript claim, cited source)` and it grades a model's
+judgement. It cannot grade the join that *chooses* the source. That join broke
+on a real manuscript — 22 printed references read as 18, and six substantive
+verdicts naming papers the manuscript never cited — and every one of them
+carried `title_check: verified`, because the glued reference string really did
+contain the downloaded paper's words alongside the cited paper's.
+
+So there is a second task, never folded into `judgment_accuracy`, whose unit is
+`(citation label, printed reference entry)`. Deliberately **not**
+`(label, retrieved PDF)`: that conflates the two joins, and it is exactly the
+conflation that let `verified` be printed on six wrong papers. Its gold rows
+carry the entry text **verbatim as printed** and no entry index — a gold set
+keyed on position breaks whenever parsing changes.
+
+Its two error directions are reported **separately and never blended**: labels
+the parse got wrong and the tool confirmed anyway (the safety metric), and
+labels the parse got right and the tool refused (the cost). Superscript papers
+have no arbiter and cannot be scored at all; they are excluded with a reason and
+counted, never deleted.
+
+**It is specified and implemented in its own plan, not here.** It needs
+`metrics.confusion()` parameterised off `JUDGMENT_VERDICTS`, a new population in
+`scoring._populations`, its own pass in `eligibility.py`, and
+`provenance.prompt_fingerprint` extended to cover `REFLIST_PROMPT` and
+`RESOLVE_PROMPT` — hashing only `EXTRACT_PROMPT` and `CHECK_PROMPT` while two
+more prompts decide the answers is the classic eval failure. `align.py` is not
+reusable for it and must not be touched: its candidate scoring is `difflib` over
+`(normalize_claim(text), set(labels))` with a 0.60 floor, and a pairing task
+wants an exact join on the label. `missing` keeps its `list[str]` meaning byte
+for byte — a refused pairing must never be unioned into it, because a
+wrongly-paired label is a numbering fault, not an extraction gap.
+
+Nothing in this section has been measured. The Status note at the top of this
+document governs it too.
+
## Paired cases
A pair is the same fact from the same source in two versions: one stated
From 0a876ebd238fcaedac5ce225b2e4923428d6c6f7 Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Fri, 18 Sep 2026 15:12:14 +0200
Subject: [PATCH 33/44] tests: numbering_verified stays False, over every reply
and every choice
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Most of this design's safety lives in one flag. Parametrised over eight model
reply shapes crossed with two extents (a fixture that legitimately verifies
and one that legitimately does not) — including the well-formed reply that
agrees with the parse, which is the tempting case, not the malformed one —
plus all four menu choices crossed with both starting values of the flag, plus
a resolver call that raises.
The brief prescribing this test had two defects of its own: its reply-shape
fixture asserted verified=False against a fixture that legitimately verifies
True, and its monkeypatch target (reflist_mod._ask) is never read by the real
code, which imports the whole ask module and calls ask._ask qualified — as
written it would have let every reply-shape test fall through to a live
subprocess call. Both fixed; see task-8-report.md for detail, including a
mutation from the brief's own verification step that turned out to be a no-op
against the real code.
A companion asserts numbering_corroborated CAN be True in the same run: the two
axes are independent, and a second axis that moves only with the first would
leave the 22-vs-19 false alarm firing.
Plus a grep guard over src/, whose docstring says what a grep cannot prove and
which has a self-test so it has been observed failing.
---
tests/test_numbering_invariant.py | 357 ++++++++++++++++++++++++++++++
1 file changed, 357 insertions(+)
create mode 100644 tests/test_numbering_invariant.py
diff --git a/tests/test_numbering_invariant.py b/tests/test_numbering_invariant.py
new file mode 100644
index 0000000..a89874e
--- /dev/null
+++ b/tests/test_numbering_invariant.py
@@ -0,0 +1,357 @@
+"""One flag staying False, across every reply shape and every interactive choice.
+
+`numbering_verified` means exactly one thing: the manuscript's own `[N]` markers
+accounted for exactly the labels a candidate reading carried. A model reading of
+the bibliography is not that, and neither is a person answering a prompt — so
+this module parametrises over every shape a model reply can take and every
+choice the escalation offers, and asserts the flag is unmoved by every one of
+them.
+
+"Unmoved" is not "always False". `_covers` is an extent check, and a fixture
+whose extent legitimately matches gets `verified = True` — asserting `False`
+there would assert an impossible value (the parametrisation task's own brief
+made exactly this mistake, in the same shape three assertions in
+`tests/test_escalation.py` had to be corrected for already: see the report for
+this task). So the reply-shape test below runs every reply against BOTH a
+fixture that legitimately verifies and one that legitimately does not, and
+checks that no reply — however it is shaped — moves either one off the value
+`_covers` alone determined.
+
+The companion test asserts `numbering_corroborated` CAN be True in the same
+run. The two are independent axes on purpose; a second axis that moves only
+when the first does is not a second axis, and the false-alarm corroboration
+exists to silence would still be firing.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
+
+from papertrace import ask as ask_mod # noqa: E402
+from papertrace import cli as cli_mod # noqa: E402
+from papertrace import reflist as reflist_mod # noqa: E402
+from papertrace.models import RefEntry, RefManifest # noqa: E402
+from papertrace.refs import ( # noqa: E402,E501
+ LABEL_AGREEMENT,
+ corroborating_readings,
+ label_agreement,
+ reconcile,
+)
+
+# Reading A and reading B of one bibliography, as text. Every fixture DOI has a
+# 4-9 digit registrant: `10.1/a` does not match `DOI_RE` and a plan already lost
+# a round to that.
+_TEXT_A = (
+ "1. Alpha A. First paper on registration. 2020. doi:10.1000/alpha\n"
+ "2. Beta B. Second paper on segmentation. 2021. doi:10.1000/beta\n"
+)
+_TEXT_B = (
+ "1. Alpha A. First paper on registration. 2020. doi:10.1000/alpha\n"
+ "2. Gamma G. A wholly different paper. 2019. doi:10.0002/b\n"
+)
+
+_BODY = {"1", "2"}
+
+
+def _reading(pairs: list[tuple[str, str, str | None]]) -> list[RefEntry]:
+ return [RefEntry(num=n, raw=raw, doi=doi) for n, raw, doi in pairs]
+
+
+def _parsed() -> list[RefEntry]:
+ return _reading([
+ ("1", "Alpha A. First paper on registration. 2020.", "10.1000/alpha"),
+ ("2", "Beta B. Second paper on segmentation. 2021.", "10.1000/beta"),
+ ])
+
+
+def _flat() -> list[RefEntry]:
+ return _reading([
+ ("1", "Alpha A. First paper on registration. 2020.", "10.1000/alpha"),
+ ("2", "Gamma G. A wholly different paper. 2019.", "10.0002/b"),
+ ])
+
+
+# Every shape a reply can take, named by what is wrong with it. `agrees` is in
+# here deliberately: the dangerous case is not the malformed reply — that fails
+# loudly — it is the well-formed one that confirms the parse, because that is
+# the reply a reader is most tempted to promote to a check.
+REPLIES = {
+ "agrees_with_the_parse": json.dumps([
+ {"num": "1", "title": "First paper on registration", "doi": "10.1000/alpha"},
+ {"num": "2", "title": "Second paper on segmentation", "doi": "10.1000/beta"},
+ ]),
+ "agrees_with_nothing": json.dumps([
+ {"num": "1", "title": "A title printed in neither reading"},
+ {"num": "2", "title": "Nor is this one"},
+ ]),
+ "abstains": json.dumps([{"num": "1", "cannot_tell": True},
+ {"num": "2", "cannot_tell": True}]),
+ "malformed_json": "here is what I think: [1] is Alpha, probably",
+ "empty_array": "[]",
+ "title_printed_nowhere": json.dumps([
+ {"num": "2", "title": "Characterization of Brain Volume Changes",
+ "doi": "10.1000/beta"},
+ ]),
+ "truncated_doi": json.dumps([
+ {"num": "2", "title": "Second paper on segmentation", "doi": "10.1038/s41591-"},
+ ]),
+ "a_label_nobody_cites": json.dumps([
+ {"num": "99", "title": "Second paper on segmentation"},
+ ]),
+}
+
+# Two extents, both legitimate. "covers" is `_parsed()` unchanged: it carries
+# exactly the body's two labels, so `_covers` passes and `verified` is
+# correctly True. "extra_reference" adds one entry the body never cites (the
+# 22-vs-19 shape: a reference cited only in a supplement) — `_covers` requires
+# `len(entries) == max(body)` exactly, so this one fails it and `verified` is
+# correctly False. Both are computed once, by the real `reconcile`, rather than
+# asserted from outside — a hand-picked expectation here would be exactly the
+# mistake this module exists to catch elsewhere.
+_EXTRA = RefEntry(num="3", raw="Delta D. Cited only in the supplement. 2022.")
+
+
+def _extent_parsed(extent: str) -> list[RefEntry]:
+ return _parsed() + [_EXTRA] if extent == "extra_reference" else _parsed()
+
+
+@pytest.mark.parametrize("extent", ["covers", "extra_reference"])
+@pytest.mark.parametrize("shape", sorted(REPLIES))
+def test_no_model_reply_shape_can_set_numbering_verified(shape, extent, monkeypatch):
+ """Parametrised over every model reply shape, on both a verified-True and a
+ verified-False fixture.
+
+ Most of this design's safety lives in one flag staying at whatever
+ `reconcile`'s extent check put it at. The model's reading is a voter: it
+ never becomes one of `reconcile`'s arguments (`reconcile(_BODY, None,
+ parsed)` below never mentions `proposed`), so no reply can cause a source
+ to be resolved, downloaded or judged, and no reply can move `verified` off
+ the value the extent check alone produced — in either direction. The
+ `covers` cell proves a reply cannot manufacture a False from a legitimate
+ True; the `extra_reference` cell proves a reply cannot manufacture a True
+ from a legitimate False, which is the direction that would actually be
+ dangerous.
+
+ Patches `ask._ask`, not `reflist._ask` — `reflist.py` has no such name of
+ its own; it imports the whole `ask` module and calls `ask._ask` qualified.
+ A monkeypatch on the wrong attribute sets an attribute nothing reads and
+ lets the real subprocess seam run, which is exactly the live-model-call
+ this suite's offline gate exists to forbid.
+ """
+ monkeypatch.setattr(ask_mod, "_ask", lambda prompt, model=None: REPLIES[shape])
+ proposed, prov = reflist_mod.propose(_TEXT_A, _TEXT_B, label_a="parsed", label_b="pymupdf")
+ assert prov.outcome == "read" # the call happened; what came back is the point
+
+ parsed = _extent_parsed(extent)
+ candidates = {"parsed": parsed, "pymupdf": _flat(), "llm": proposed}
+ states = label_agreement(candidates, _BODY)
+
+ # the parse and the deposit are what reconcile arbitrates between; `llm` is
+ # not passed, and a task that passes it has broken the design
+ entries, rec = reconcile(_BODY, None, parsed)
+ manifest = RefManifest(
+ manuscript="p.pdf",
+ entries=entries,
+ numbering_verified=rec.verified,
+ labels_disputed=[k for k, v in states.items() if v == "disputed"],
+ )
+
+ expected = extent == "covers"
+ assert rec.verified is expected, f"{shape}/{extent} moved rec.verified"
+ assert manifest.numbering_verified is expected, f"{shape}/{extent} moved the manifest"
+ assert all(v in LABEL_AGREEMENT for v in states.values())
+
+
+@pytest.mark.parametrize("initial_verified", [True, False], ids=["was_verified", "was_unverified"])
+@pytest.mark.parametrize("choice", ["1", "2", "3", "4"])
+def test_no_interactive_choice_can_set_numbering_verified(choice, initial_verified, monkeypatch, tmp_path):
+ """Parametrised over all four menu choices, including abort, on both a
+ verified-True and a verified-False starting point.
+
+ `chosen_by: "user"` is what makes the escalation honest — a person
+ consenting to proceed is an input, not evidence. So every branch of the
+ menu must leave `rec.verified` no more true than `reconcile` left it: for
+ choices 1, 2 and 4 that means exactly unchanged (nothing in their code
+ paths touches `rec.verified` at all), and this is the test that says so for
+ the branch a reviewer would be most tempted to "improve" — option 3, where
+ the user positively asserts which reading is right. Option 3 is the one
+ exception, and by design: it substitutes a whole different list than the
+ one `reconcile` measured, so it explicitly resets `verified` to the literal
+ `False` — never derived from the answer, never derived from which reading
+ was picked — because the extent of the newly adopted list was never
+ checked. That reset can only ever produce `False`; it is asserted here on
+ both starting points to show it never produces `True`.
+ """
+ import typer
+
+ candidates = {"parsed": _parsed(), "pymupdf": _flat()}
+ texts = {"parsed": _TEXT_A, "pymupdf": _TEXT_B}
+ # `_parsed()` legitimately verifies (see the test above); overridden here
+ # because this test is about `_escalate_disputed`'s own behaviour, not
+ # `reconcile`'s — a disputed label and a verified extent are independent
+ # findings, so the escalation must be exercised from both starting values.
+ _entries, rec = reconcile(_BODY, None, _parsed())
+ rec.labels_disputed = ["2"]
+ rec.verified = initial_verified
+
+ answers = iter([choice, "pymupdf"])
+ monkeypatch.setattr(cli_mod, "_interactive", lambda: True)
+ monkeypatch.setattr(cli_mod.Prompt, "ask", staticmethod(lambda *a, **k: next(answers)))
+ monkeypatch.setattr(ask_mod, "_ask", lambda prompt, model=None: REPLIES["agrees_with_the_parse"])
+
+ if choice == "4":
+ with pytest.raises(typer.Exit):
+ cli_mod._escalate_disputed(tmp_path, rec, _parsed(), candidates, texts)
+ else:
+ cli_mod._escalate_disputed(tmp_path, rec, _parsed(), candidates, texts)
+
+ expected = False if choice == "3" else initial_verified
+ assert rec.verified is expected, f"menu choice {choice} moved rec.verified"
+ manifest = RefManifest(
+ manuscript="p.pdf",
+ numbering_verified=rec.verified,
+ numbering_choice=rec.choice,
+ numbering_chosen_by=rec.chosen_by,
+ labels_resolved=rec.labels_resolved,
+ labels_disputed=rec.labels_disputed,
+ )
+ assert manifest.numbering_verified is expected
+ if choice != "4":
+ assert manifest.numbering_chosen_by == "user"
+
+
+@pytest.mark.parametrize("initial_verified", [True, False], ids=["was_verified", "was_unverified"])
+def test_a_raising_resolver_cannot_set_numbering_verified(initial_verified, monkeypatch, tmp_path):
+ """Option 1's `except Exception` is where a live subprocess failure
+ (timeout, non-zero exit, unparseable stdout) surfaces mid-escalation — a
+ failed call is a third outcome, not a licence to guess a verdict. It must
+ leave `rec.verified` exactly where it found it, same as every other branch
+ that does not explicitly reset it.
+
+ `reflist.resolve_disputed` is patched as a module attribute, not handed to
+ `cli` directly: `_escalate_disputed` does `from .reflist import
+ resolve_disputed` inside its own body, so the name is looked up fresh from
+ `reflist`'s namespace on every call and a monkeypatch on the module
+ attribute is what a function-local import actually sees.
+ """
+ candidates = {"parsed": _parsed(), "pymupdf": _flat()}
+ texts = {"parsed": _TEXT_A, "pymupdf": _TEXT_B}
+ _entries, rec = reconcile(_BODY, None, _parsed())
+ rec.labels_disputed = ["2"]
+ rec.verified = initial_verified
+
+ monkeypatch.setattr(cli_mod, "_interactive", lambda: True)
+ monkeypatch.setattr(cli_mod.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+
+ def _boom(*args, **kwargs):
+ raise RuntimeError("claude -p failed: simulated for this test")
+
+ monkeypatch.setattr(reflist_mod, "resolve_disputed", _boom)
+
+ entries_out = cli_mod._escalate_disputed(tmp_path, rec, _parsed(), candidates, texts)
+
+ assert entries_out == _parsed() # nothing substituted — the call never answered
+ assert rec.verified is initial_verified, "a raising resolver moved rec.verified"
+ assert rec.chosen_by == "user" # a person consented to trying, even though it failed
+ manifest = RefManifest(
+ manuscript="p.pdf",
+ numbering_verified=rec.verified,
+ numbering_chosen_by=rec.chosen_by,
+ )
+ assert manifest.numbering_verified is initial_verified
+
+
+def test_corroboration_can_be_true_in_the_same_run_that_keeps_verified_false():
+ """The two axes must be independently settable, or the second is pointless.
+
+ This is the 22-vs-19 case the corroboration axis exists for: every cited
+ label is read the same way by two readings, and `_covers` still fails
+ because three of the references are cited only in the supplement. Before
+ the second axis, that run could say nothing but "unconfirmed" — crying
+ wolf on a numbering two independent readings agreed about entry for entry.
+ """
+ candidates = {"parsed": _parsed(), "pymupdf": _parsed()}
+ states = label_agreement(candidates, _BODY)
+ assert set(states.values()) == {"agreed"}
+
+ # a list longer than the highest cited label: `_covers` requires equality
+ longer = _parsed() + [RefEntry(num="3", raw="Delta D. Cited only in the supplement. 2022.")]
+ _entries, rec = reconcile(_BODY, None, longer)
+ assert rec.verified is False, "the extent check still fails — that is the premise"
+
+ manifest = RefManifest(
+ manuscript="p.pdf",
+ numbering_verified=rec.verified,
+ numbering_corroborated=all(v == "agreed" for v in states.values()),
+ corroborating_readings=corroborating_readings(candidates, _BODY),
+ )
+ assert manifest.numbering_verified is False
+ assert manifest.numbering_corroborated is True
+ assert manifest.corroborating_readings == ["parsed", "pymupdf"]
+
+
+_SRC = Path(__file__).resolve().parent.parent / "src" / "papertrace"
+# the two names that may only ever be set from the extent check
+_GUARDED = ("numbering_verified", "rec.verified", ".verified")
+# what must never appear on the right-hand side of one
+_FORBIDDEN = ("reflist", "propose", "resolve_disputed", "Prompt.ask", "_ask(",
+ "answer", "choice", "chosen_by", "res.resolved")
+
+
+def test_no_source_file_assigns_numbering_verified_from_a_model_or_a_prompt():
+ """A grep, and honest about being one.
+
+ What this proves: no source line *textually* assigns one of the guarded
+ names from an expression mentioning the reflist module, a prompt answer or
+ the seam. That catches the change a future task is actually likely to make
+ — `rec.verified = bool(res.resolved)`, written in good faith by someone who
+ read "resolved" as "checked".
+
+ What it cannot prove: that no value *derived* from a model reply reaches
+ the flag through an intermediate variable, a helper or a dict. A grep has
+ no dataflow. The parametrised tests above are what cover that, by running
+ the real code over every reply shape and every choice; this one is the
+ cheap tripwire that fails at review time rather than at runtime, and it is
+ worth having for exactly that reason and no more.
+ """
+ offenders = []
+ for path in sorted(_SRC.rglob("*.py")):
+ for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+ code = line.split("#", 1)[0]
+ if "=" not in code or "==" in code:
+ continue
+ lhs, _, rhs = code.partition("=")
+ if not any(g in lhs for g in _GUARDED):
+ continue
+ if any(f in rhs for f in _FORBIDDEN):
+ offenders.append(f"{path.relative_to(_SRC)}:{lineno}: {line.strip()}")
+ assert offenders == [], "\n".join(offenders)
+
+
+def test_the_guard_above_would_actually_catch_the_change_it_describes(tmp_path):
+ """A guard test nobody has seen fail is a guard test nobody knows works.
+
+ The grep is re-run over a file holding the exact line the real test
+ forbids, so the assertion has been observed both ways. Written out again
+ rather than factored into a shared helper: a helper broken the same way
+ would make both tests pass together, which is the failure mode a self-test
+ exists to exclude.
+ """
+ bad = tmp_path / "papertrace" / "bad.py"
+ bad.parent.mkdir(parents=True)
+ bad.write_text("rec.verified = bool(res.resolved)\n", encoding="utf-8")
+
+ offenders = []
+ for path in sorted(bad.parent.rglob("*.py")):
+ for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+ code = line.split("#", 1)[0]
+ if "=" not in code or "==" in code:
+ continue
+ lhs, _, rhs = code.partition("=")
+ if any(g in lhs for g in _GUARDED) and any(f in rhs for f in _FORBIDDEN):
+ offenders.append(f"{path.name}:{lineno}")
+ assert offenders, "the grep would not catch the line it exists to catch"
From d39bea37d106090ea7a04633d49b087fba112a47 Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Fri, 18 Sep 2026 22:43:48 +0200
Subject: [PATCH 34/44] refs: a reading that cannot be compared abstains, and a
copied one cannot corroborate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
`_comparably_same` returns `True | False | None`, the same three-valued
vocabulary `models.titles_match` publishes — the two are siblings answering
opposite questions. A two-valued `False` for "nothing to compare" made a
Crossref deposit of bare DOIs dispute every label it voted on while both
parses agreed perfectly, and a run then printed an audit with no verdicts in
it.
`_label_state` is the one place the rule lives now: a cannot-tell voter is
excluded from the comparison rather than counted as dissent, which can leave
one reading standing and print `single`. A label NO voter says anything
comparable about stays `disputed` — it is carried, and nothing establishes
what it names.
`DERIVED_READINGS` excludes the model's reading from crediting an agreement.
`reflist.propose` may only copy from the two extractions that vote here in
their own right, so its agreement is one text read twice. It still votes and
can still dispute.
`stamp_seen_in` matches through the same comparator, so `seen_in` and
`labels_disputed` can no longer contradict each other on one manifest; it
credits a chosen entry to the reading it came from by identity and merges
rather than overwrites, so `[]` keeps one meaning.
---
src/papertrace/refs.py | 213 +++++++++++++++++++++++--------
tests/test_label_agreement.py | 205 ++++++++++++++++++++++++++++-
tests/test_reference_readings.py | 30 +++--
3 files changed, 377 insertions(+), 71 deletions(-)
diff --git a/src/papertrace/refs.py b/src/papertrace/refs.py
index d8291a2..1873cb5 100644
--- a/src/papertrace/refs.py
+++ b/src/papertrace/refs.py
@@ -746,34 +746,63 @@ def _first_divergence(a: list[RefEntry], b: list[RefEntry]) -> int | None:
LABEL_AGREEMENT = ("agreed", "single", "disputed", "absent")
-def _comparably_same(x: RefEntry, y: RefEntry) -> bool:
- """`_same_work` without its benefit of the doubt — agreement needs evidence.
-
- The two functions answer opposite questions and so must break ties in
- opposite directions. `_same_work` asks *"is there evidence these differ?"*,
- which is right for `_first_divergence`: nothing comparable must not
- manufacture a divergence, so it returns True. A per-label vote asks *"is
- there evidence these agree?"*, and nothing comparable must not manufacture
- agreement either.
-
- Measured before this existed: `_same_work` gives the benefit of the doubt
- when **either** side yields no title tokens, so a garbled DOI-less entry
- paired with `Fujita S (2023) Characterization of brain volume changes…`
- came back True, and the label was reported `agreed`. Two readings naming
- demonstrably unrelated papers corroborating each other is the laundering of
- ignorance this codebase exists to refuse — and `agreed` is the one state
- that lets a verdict through.
-
- `_same_work` itself is deliberately untouched: `reconcile` computes
- `contested` from it and needs the old direction.
+def _comparably_same(x: RefEntry, y: RefEntry) -> bool | None:
+ """Do these two readings name the same paper — or is there no way to tell?
+
+ Three answers, `True | False | None`, and `None` is the load-bearing one:
+ it means *cannot tell*, and it is a third answer rather than a collapsed
+ `False` for a measured reason. This is the same convention
+ `models.titles_match` publishes, for the same reason in the mirror
+ direction — the two are siblings answering opposite questions with one
+ three-valued vocabulary, so neither has to launder ignorance into an
+ answer its caller will act on.
+
+ `_same_work` is the other sibling and is deliberately untouched. It asks
+ *"is there evidence these differ?"* and answers True when there is nothing
+ comparable, which is right for `_first_divergence`: silence must not
+ manufacture a divergence. This asks *"is there evidence these agree?"*, so
+ silence must not manufacture agreement either — but it must not manufacture
+ DISAGREEMENT either, which is what returning `False` here did.
+
+ Measured, both ways round. With `_same_work` as the comparator, a garbled
+ DOI-less entry paired with `Fujita S (2023) Characterization of brain
+ volume changes…` came back True and the label was reported `agreed` — two
+ readings naming demonstrably unrelated papers corroborating each other.
+ With a two-valued `False` here, a Crossref deposit of bare DOIs (the
+ documented `_reference_raw` fallback, 49 of 52 references on the paper
+ that prompted this) disputed every label it voted on, while the parse and
+ the flat reading agreed perfectly — an audit with no verdicts in it.
+ `None` is what neither of those can say.
"""
if x.doi and y.doi:
return x.doi.lower() == y.doi.lower()
if not _title_tokens(x.raw) or not _title_tokens(y.raw):
- return False # nothing to compare is not agreement
+ return None # nothing to compare: not agreement, and not disagreement
return _same_work(x, y)
+def _says_something_comparable(e: RefEntry) -> bool:
+ """Does this entry carry anything a comparison could ever have used?
+
+ Per ENTRY, not per pair. A voter that carries a DOI has said something
+ comparable even where no other reading printed one; a voter with neither a
+ DOI nor a single title token has said nothing any reading could ever agree
+ or disagree with, and `label_agreement` treats a label ALL of whose voters
+ are like that as `disputed` rather than `single`.
+ """
+ return bool(e.doi) or bool(_title_tokens(e.raw))
+
+
+# Readings whose entries are COPIED, value by value, out of other readings
+# that vote here in their own right. `reflist.propose` may only reproduce what
+# text A or text B already printed — so the model agreeing with one of them is
+# one text read twice, not two readings agreeing, and naming it as
+# corroboration would make `numbering_corroborated` true on the strength of a
+# reading echoing another voter. It still votes: a DISAGREEMENT from it is
+# real, and disagreeing is the only power the safety property grants it.
+DERIVED_READINGS = ("llm",)
+
+
def label_agreement(
candidates: dict[str, list[RefEntry]],
body: set[str],
@@ -805,22 +834,77 @@ def label_agreement(
majority (invariant 3): "a non-unique match is refused, never ranked" is
this codebase's own rule for exactly this shape of choice, applied here to
readings instead of provided files.
+
+ A voter that **cannot be compared** abstains rather than dissenting — see
+ `_label_state` for the whole rule and for the one case where it does not
+ apply.
"""
- result: dict[str, str] = {}
- for label in body:
- voters, duplicated = _label_voters(candidates, label)
- if duplicated:
- result[label] = "disputed"
- elif not voters:
- result[label] = "absent"
- elif len(voters) == 1:
- result[label] = "single"
- else:
- agree = all(
- _comparably_same(x, y) for (_, x), (_, y) in itertools.combinations(voters, 2)
- )
- result[label] = "agreed" if agree else "disputed"
- return result
+ return {label: _label_state(candidates, label)[0] for label in body}
+
+
+def _label_state(
+ candidates: dict[str, list[RefEntry]], label: str
+) -> tuple[str, list[str]]:
+ """One label's agreement state, and the readings that earned an `agreed`.
+
+ The single place the rule lives: `label_agreement` publishes the state and
+ `corroborating_readings` intersects the names, so the two can never drift
+ into two different definitions of what agreeing means.
+
+ The comparison is pairwise through `_comparably_same`, which has three
+ answers:
+
+ * any pair `False` — two readings actively name different papers — is
+ `disputed`, with no majority and no tie-break.
+ * a pair that is `None` is not a vote against anything. The voter is
+ EXCLUDED from the comparison rather than counted as dissent: a Crossref
+ deposit of bare DOIs can be compared with nothing, and counting that as
+ disagreement withheld every verdict in a run whose two parses agreed
+ perfectly.
+ * excluding it can leave one reading standing, which is `single` — no
+ corroboration claimed, no verdict withheld. That is the state a
+ one-reading run has always been in, and it is the honest one: one
+ reading spoke and nothing agreed or disagreed with it.
+
+ The exception, and it is one character away from the rule above: where NO
+ voter said anything comparable at all (`_says_something_comparable` false
+ for every one of them) the label is `disputed`, not `single`. `absent`
+ would be false — readings do carry it; `single` would be false — none of
+ them spoke comparably; `agreed` is the laundering this function exists to
+ refuse. Nothing establishes that these name one paper, and withholding is
+ the honest answer to that.
+
+ So `disputed` carries two causes — the readings contradict each other, and
+ nothing could be compared. Both withhold a verdict, which is right, but
+ **no prose may say the readings named different papers on the strength of
+ this state alone**; the report's wording covers both.
+
+ The second returned value is the readings credited with the agreement,
+ which is not simply the voters: `DERIVED_READINGS` are excluded, and a
+ label needs two CREDITED readings to reach `agreed`.
+ """
+ voters, duplicated = _label_voters(candidates, label)
+ if duplicated:
+ return "disputed", []
+ if not voters:
+ return "absent", []
+ if len(voters) == 1:
+ # nothing was compared because there was nobody to compare with — a
+ # different fact from "compared and found uncomparable" below
+ return "single", []
+ concurring: set[str] = set()
+ for (name_x, x), (name_y, y) in itertools.combinations(voters, 2):
+ same = _comparably_same(x, y)
+ if same is False:
+ return "disputed", []
+ if same:
+ concurring |= {name_x, name_y}
+ if not concurring:
+ if not any(_says_something_comparable(e) for _, e in voters):
+ return "disputed", [] # see this docstring's "exception"
+ return "single", []
+ credited = sorted(n for n in concurring if n not in DERIVED_READINGS)
+ return ("agreed", credited) if len(credited) >= 2 else ("single", [])
def _label_voters(
@@ -855,23 +939,22 @@ def corroborating_readings(candidates: dict[str, list[RefEntry]], body: set[str]
an entry — is not one that agreed on all of them, and a reading whose only
carrier of a label is `boundary_ambiguous` cast no vote on it at all. Both
were measured to overstate `corroborating_readings` in exactly this
- function's absence. This recomputes each label's voters with
- `_label_voters` — the same test `label_agreement` uses — and intersects
- their names across every cited label, so a reading is named only when it
+ function's absence. This runs `_label_state` — the one place the rule
+ lives, the same call `label_agreement` makes — and intersects the readings
+ it credits across every cited label, so a reading is named only when it
voted, and agreed, everywhere it was asked to.
+
+ A reading that could not be COMPARED on a label is not credited with it
+ either, and neither is a `DERIVED_READINGS` one: the model's reading is
+ copied out of the other voters' own texts, so naming it here would print
+ "two readings agree" over one text read twice.
"""
names: set[str] | None = None
for label in body:
- voters, duplicated = _label_voters(candidates, label)
- if duplicated or len(voters) < 2:
- return []
- agree = all(
- _comparably_same(x, y) for (_, x), (_, y) in itertools.combinations(voters, 2)
- )
- if not agree:
+ state, credited = _label_state(candidates, label)
+ if state != "agreed":
return []
- label_names = {name for name, _ in voters}
- names = label_names if names is None else (names & label_names)
+ names = set(credited) if names is None else (names & set(credited))
return sorted(names) if names else []
@@ -882,18 +965,36 @@ def stamp_seen_in(chosen: list[RefEntry], candidates: dict[str, list[RefEntry]])
by one reading only is the interesting case — especially `["llm"]`, which
means no deterministic reading found it at all.
- Matched on the label AND `_same_work`: a reading that carries [12] naming a
- different paper has not corroborated this entry, and stamping its name here
- on the strength of the shared numeral would turn a disagreement into
- provenance. That is the same mistake `_covers` makes about extent, and it is
- the reason this is not simply `label in {e.num for e in cand}`.
+ Matched on the label AND `_comparably_same`, the SAME comparator the
+ agreement axis uses. Stamping on the shared numeral alone would turn a
+ disagreement into provenance — the extent-for-content substitution
+ `_covers` is documented as blind to — and matching through `_same_work`
+ did it anyway: that function gives the benefit of the doubt when either
+ side yields no title tokens, so one manifest published both "two readings
+ contributed this entry" and "the readings do not agree about this label"
+ for the same label in the same run. Only a `True` stamps; `None` — nothing
+ to compare — is not evidence that a reading carried this work.
+
+ Two rules keep `[]` meaning one thing (no reading was established as
+ carrying this work) rather than doubling as "never computed":
+
+ * a chosen entry that IS one of the readings' own objects is credited to
+ that reading by identity, whatever the comparator can or cannot see in
+ it. A garbled entry the parse demonstrably produced must not come back
+ with an empty provenance.
+ * an entry that already carries a `seen_in` keeps it. `cli` stamps after
+ the escalation, and an entry substituted by `reflist.resolve_disputed`
+ records which printed text its values were copied from — a fact this
+ pass cannot recompute, because the readings that disputed the label
+ carry no entry matching the resolved one.
"""
for e in chosen:
- e.seen_in = sorted(
+ names = {
name
for name, cand in candidates.items()
- if any(o.num == e.num and _same_work(e, o) for o in cand)
- )
+ if any(o.num == e.num and (o is e or _comparably_same(e, o) is True) for o in cand)
+ }
+ e.seen_in = sorted(names | set(e.seen_in))
def reconcile(
diff --git a/tests/test_label_agreement.py b/tests/test_label_agreement.py
index 35bb5ec..fec1159 100644
--- a/tests/test_label_agreement.py
+++ b/tests/test_label_agreement.py
@@ -334,6 +334,11 @@ def test_two_uncomparable_entries_are_disputed_rather_than_agreed():
must not manufacture a divergence out of silence. A per-label vote asks the
opposite question, so silence must not manufacture agreement either.
Measured before `_comparably_same` existed: these two came back `agreed`.
+
+ NEITHER entry carries a DOI or a single title token, so nothing comparable
+ was said by anybody — see
+ `test_a_label_every_voter_is_mute_about_is_disputed_not_single` for the
+ ruling that keeps this `disputed` now that a cannot-tell voter abstains.
"""
a = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
b = RefEntry(num="7", raw="Q. Z. 2020. 88(1):4-7.")
@@ -344,14 +349,154 @@ def test_one_garbled_reading_cannot_corroborate_a_legible_one():
"""Broader than it looks: the benefit of the doubt applied when **either**
side yielded no tokens, not only when both did. So a real reference paired
with an unreadable one was `agreed`, and `agreed` is the single state that
- lets a verdict through untouched."""
+ lets a verdict through untouched.
+
+ RE-RULED at the whole-branch review: the answer is `single`, not
+ `disputed`. The intent of this test — a garbled reading must not
+ CORROBORATE a legible one — is preserved by abstention, which is what
+ `single` records: one reading spoke, nothing agreed with it, nothing
+ contradicted it. Counting the garbled reading as dissent instead withheld
+ the verdict on a label nothing had said anything against, and on a
+ bare-DOI Crossref deposit it withheld every verdict in the run.
+ """
good = RefEntry(
num="7",
raw="Fujita S (2023) Characterization of brain volume changes in aging individuals. "
"JAMA Netw Open.",
)
garbled = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
- assert label_agreement({"parsed": [good], "llm": [garbled]}, {"7"}) == {"7": "disputed"}
+ assert label_agreement({"parsed": [good], "llm": [garbled]}, {"7"}) == {"7": "single"}
+ assert corroborating_readings({"parsed": [good], "llm": [garbled]}, {"7"}) == []
+
+
+# --- "cannot compare" is a third answer, and it abstains --------------------
+
+
+def _bare_doi(num: str, doi: str) -> RefEntry:
+ """A Crossref deposit of a DOI and nothing else — `_reference_raw`'s
+ documented fallback, and 49 of 52 references on the Wiley paper that
+ prompted this. It yields no title tokens at all, permanently, by the
+ deposit's shape."""
+ return RefEntry(num=num, raw=f"https://doi.org/{doi}", doi=doi)
+
+
+def _printed(num: str) -> RefEntry:
+ """One entry as a reference list that prints no DOIs carries it — normal
+ for many journals, and the other half of Critical 1's shape: the deposit
+ has a DOI and nothing else, the page has everything but."""
+ return _entry(
+ num,
+ f"Fujita S, Mori S. Characterization of brain volume changes, entry {num}. "
+ "JAMA Netw Open. 2023;6(6).",
+ )
+
+
+def test_comparably_same_has_a_third_answer_for_nothing_to_compare():
+ """`None`, the same three-valued vocabulary `models.titles_match`
+ publishes for the same reason. `False` here is a claim about the two
+ papers being different, and a bare-DOI deposit is not evidence of that."""
+ assert _comparably_same(_bare_doi("7", "10.1002/hep.31884"), _printed("7")) is None
+ assert _comparably_same(_work("7", "10.1000/x7"), _work("7", "10.1000/x7")) is True
+ assert _comparably_same(_work("7", "10.1000/x7"), _work("7", "10.1000/x9")) is False
+
+
+def test_a_bare_doi_deposit_does_not_withhold_a_label_the_parses_agree_on():
+ """Critical 1 of the whole-branch review, reproduced.
+
+ A publisher deposits bare DOIs; the printed list prints none. The deposit
+ can be compared with neither parse, but the two parses agree with each
+ other perfectly. Counting the uncomparable voter as dissent made every
+ cited label `disputed`, every source withheld and every claim
+ `unchecked` — an audit with no verdicts in it, on a paper where nothing
+ was wrong."""
+ candidates = {
+ "crossref": [_bare_doi("7", "10.1002/hep.31884")],
+ "parsed": [_printed("7")],
+ "pymupdf": [_printed("7")],
+ }
+ assert label_agreement(candidates, {"7"}) == {"7": "agreed"}
+ # and the voter nobody could compare is not named as having agreed
+ assert corroborating_readings(candidates, {"7"}) == ["parsed", "pymupdf"]
+
+
+def test_dropping_the_uncomparable_voter_can_leave_one_and_that_prints():
+ """The second-order effect of the abstention rule, pinned deliberately.
+
+ On a `--backend pymupdf` run there is no flat reading, so the deposit and
+ the parse are the only two voters and the deposit cannot be compared.
+ One comparable voter remains, which is `single`: no corroboration is
+ claimed and no verdict is withheld. That is the state a one-reading run
+ has always been in, and it is what the pre-branch behaviour was — the
+ laundering this branch removed is `agreed`, which is what would claim the
+ deposit backed the parse."""
+ candidates = {
+ "crossref": [_bare_doi("7", "10.1002/hep.31884")],
+ "parsed": [_printed("7")],
+ }
+ assert label_agreement(candidates, {"7"}) == {"7": "single"}
+ assert corroborating_readings(candidates, {"7"}) == []
+
+
+def test_a_label_every_voter_is_mute_about_is_disputed_not_single():
+ """The cell the abstention rule does not cover, ruled at the review.
+
+ Two readings carry [7] and NEITHER says anything a comparison could use.
+ `absent` is false (two readings carry it), `single` is false (no reading
+ spoke comparably), `agreed` is the laundering. `disputed` is the true one:
+ nothing establishes that these name one paper."""
+ a = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
+ b = RefEntry(num="7", raw="Q. Z. 2020. 88(1):4-7.")
+ assert label_agreement({"parsed": [a], "pymupdf": [b]}, {"7"}) == {"7": "disputed"}
+
+
+def test_a_label_no_voter_carries_at_all_is_absent_not_disputed():
+ """One character apart from the cell above and opposite in meaning: zero
+ voters is `absent` (nobody read it), zero COMPARABLE voters with the label
+ carried is `disputed` (it was read and nothing could be compared)."""
+ a = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
+ b = RefEntry(num="7", raw="Q. Z. 2020. 88(1):4-7.")
+ assert label_agreement({"parsed": [a], "pymupdf": [b]}, {"9"}) == {"9": "absent"}
+
+
+# --- the model's reading may dispute, and that is all it may do -------------
+
+
+def test_the_model_reading_never_corroborates_what_it_copied_out_of():
+ """Major 1. `reflist.propose` may only COPY from the two extractions, and
+ both of those are voters in their own right — so the model agreeing with
+ one of them is one text read twice, not two readings agreeing. Here the
+ flat parse lost [7] entirely and the model echoed it back out of text A:
+ counting that as a second reading turned `numbering_corroborated` True and
+ printed "two readings of the reference list agree" over one text."""
+ e = _work("7", "10.1000/x7")
+ candidates = {"parsed": [e], "pymupdf": [], "llm": [_work("7", "10.1000/x7")]}
+ assert label_agreement(candidates, {"7"}) == {"7": "single"}
+ assert corroborating_readings(candidates, {"7"}) == []
+
+
+def test_the_model_reading_can_still_dispute_a_label():
+ """The other half of Major 1's ruling, and the only power the safety
+ property grants the model reading: it may disagree. Excluding it from
+ corroboration must not also excuse it from the comparison."""
+ candidates = {
+ "parsed": [_work("7", "10.1000/x7")],
+ "pymupdf": [_work("7", "10.1000/x7")],
+ "llm": [_work("7", "10.1000/x9")],
+ }
+ assert label_agreement(candidates, {"7"}) == {"7": "disputed"}
+
+
+def test_the_model_reading_is_not_named_beside_two_real_readings_that_agree():
+ """Three voters all agreeing: the two that read the page are named, the
+ one that copied out of them is not. The printed count is then 2, which is
+ what `NUMBERING_CORROBORATION_TOKEN` asserts."""
+ candidates = {
+ "parsed": [_work("7", "10.1000/x7")],
+ "pymupdf": [_work("7", "10.1000/x7")],
+ "llm": [_work("7", "10.1000/x7")],
+ }
+ assert label_agreement(candidates, {"7"}) == {"7": "agreed"}
+ assert corroborating_readings(candidates, {"7"}) == ["parsed", "pymupdf"]
def test_two_doi_less_readings_that_do_share_a_title_still_agree():
@@ -387,7 +532,10 @@ def test_same_work_keeps_its_own_direction_for_its_own_callers():
a = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
b = RefEntry(num="7", raw="Q. Z. 2020. 88(1):4-7.")
assert _same_work(a, b) is True
- assert _comparably_same(a, b) is False
+ # `None`, not `False`: the two functions are siblings answering opposite
+ # questions, and each says "cannot tell" in its own caller's safe
+ # direction — `_same_work` as True, `_comparably_same` as a third answer
+ assert _comparably_same(a, b) is None
# --- seen_in: which readings also carried a chosen entry's work ------------
@@ -420,6 +568,45 @@ def test_a_reading_naming_a_different_paper_under_the_same_label_is_not_provenan
assert chosen[0].seen_in == ["parsed"]
+def test_seen_in_never_names_a_reading_label_agreement_says_disagrees():
+ """Major 3. `stamp_seen_in` matched through the lenient `_same_work`,
+ which gives the benefit of the doubt when either side yields no title
+ tokens — so one manifest published, for one label in one run, both "two
+ readings contributed this entry" (`seen_in`) and "the readings do not
+ agree about it" (`labels_disputed`). Two published fields contradicting
+ each other. The comparator here is now the same one the agreement axis
+ uses."""
+ chosen = [RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")]
+ other = RefEntry(num="7", raw="Q. Z. 2020. 88(1):4-7.")
+ candidates = {"parsed": chosen, "pymupdf": [other]}
+ assert label_agreement(candidates, {"7"}) == {"7": "disputed"}
+ stamp_seen_in(chosen, candidates)
+ assert chosen[0].seen_in == ["parsed"]
+
+
+def test_a_chosen_entry_is_always_credited_to_the_reading_it_came_from():
+ """Identity, not comparison: the chosen list IS one of the readings'
+ lists, so an entry nothing could be compared against still records where
+ it came from. Without this, a garbled entry's `seen_in` would be `[]` —
+ which the schema reads as "no reading was established as carrying this" —
+ for a reading that demonstrably carried it."""
+ garbled = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
+ chosen = [garbled]
+ stamp_seen_in(chosen, {"parsed": [garbled], "pymupdf": []})
+ assert chosen[0].seen_in == ["parsed"]
+
+
+def test_stamping_twice_never_erases_a_provenance_an_earlier_pass_recorded():
+ """`cli` stamps AFTER the escalation, so an entry substituted by
+ `reflist.resolve_disputed` already carries the reading its values were
+ copied from. That name is a fact this pass cannot recompute — the
+ readings that disputed the label carry no entry matching the resolved
+ one — so it is merged, never overwritten."""
+ resolved = RefEntry(num="7", raw="A resolved title.", seen_in=["pymupdf"])
+ stamp_seen_in([resolved], {"parsed": [], "pymupdf": []})
+ assert resolved.seen_in == ["pymupdf"]
+
+
# --- corroborating_readings: named only where a reading actually voted -----
# (N3 of the Task 6 re-review: `any(e.num in body for e in cand)` — the filter
# that shipped in round 1 — named a reading that carried only SOME cited
@@ -447,7 +634,13 @@ def test_corroborating_readings_excludes_a_reading_whose_only_carrier_is_boundar
"""A `boundary_ambiguous` carrier casts no vote at all (invariant 5 of
`label_agreement`) — sharing that fact with `corroborating_readings` is
the point of factoring `_label_voters` out, rather than a coincidence two
- unrelated filters happen to agree on."""
+ unrelated filters happen to agree on.
+
+ The two readings that DO vote here are `crossref` and `pymupdf`. This
+ fixture used to name `llm` as the second voter, which since Major 1 is
+ excluded from corroboration for a different reason entirely — so the test
+ would have gone green through a rule it was never about. Two independent
+ exclusions must not be tested by one fixture."""
ambiguous_1 = _work("1", "10.1000/x1")
ambiguous_1.boundary_ambiguous = True
ambiguous_2 = _work("2", "10.1000/x2")
@@ -456,10 +649,10 @@ def test_corroborating_readings_excludes_a_reading_whose_only_carrier_is_boundar
candidates = {
"parsed": [ambiguous_1, ambiguous_2],
"crossref": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
- "llm": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
+ "pymupdf": [_work("1", "10.1000/x1"), _work("2", "10.1000/x2")],
}
assert label_agreement(candidates, body) == {"1": "agreed", "2": "agreed"}
- assert corroborating_readings(candidates, body) == ["crossref", "llm"]
+ assert corroborating_readings(candidates, body) == ["crossref", "pymupdf"]
def test_corroborating_readings_is_empty_unless_every_cited_label_agreed():
diff --git a/tests/test_reference_readings.py b/tests/test_reference_readings.py
index 27c04d1..94a9306 100644
--- a/tests/test_reference_readings.py
+++ b/tests/test_reference_readings.py
@@ -492,12 +492,18 @@ def test_no_model_reply_can_set_numbering_verified(tmp_path, offline, monkeypatc
has three entries against two cited labels and fails `_covers` on its own —
the only way to make the two cases distinguishable.
- `corroborating_readings` names three, not two: `offline`'s default
+ `corroborating_readings` names `parsed` and `pymupdf`: `offline`'s default
`needs_flat=True` means the real, local flat-text reading of this same PDF
also runs and also carries labels [1] and [2] (identical text to the
parse, since the real backend here is pymupdf too) — so it votes exactly
- where "parsed" does, and Major 1's fix (name only readings that actually
- carried a cited label) correctly counts it alongside "llm" and "parsed".
+ where "parsed" does.
+
+ `llm` is NOT among them, and the model here is agreeing entry for entry:
+ that is the whole-branch review's Major 1. `reflist.propose` may only copy
+ from the two extractions, and both of those vote here in their own right,
+ so the model's agreement is one text read twice. It can still dispute; it
+ cannot corroborate. The count printed on the console — 2 — is therefore
+ the number of readings that actually read the page.
"""
agrees = (
[RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
@@ -516,7 +522,7 @@ def test_no_model_reply_can_set_numbering_verified(tmp_path, offline, monkeypatc
payload = json.loads((case / "refs_manifest.json").read_text())
assert payload["numbering_verified"] is False
assert payload["numbering_corroborated"] is True
- assert payload["corroborating_readings"] == ["llm", "parsed", "pymupdf"]
+ assert payload["corroborating_readings"] == ["parsed", "pymupdf"]
def test_a_successful_reading_that_named_no_model_still_discloses_the_attempt(
@@ -870,13 +876,19 @@ def test_a_realistic_split_vote_prints_neither_a_bad_count_nor_a_false_token(
):
"""NF1 of the second Task 6 re-review, reproduced end to end — not merely
constructed. `parsed` carries a `boundary_ambiguous` entry at one cited
- label (so it casts no vote there — the 0.7.0 numbering case); the mocked
- `llm` reading skips a different cited label entirely. Every cited label
- still ends up `agreed` (by the readings that DO carry it), so
+ label (so it casts no vote there — the 0.7.0 numbering case); the
+ `crossref` deposit skips a different cited label entirely. Every cited
+ label still ends up `agreed` (by the readings that DO carry it), so
`numbering_corroborated` is `True` — but no single reading voted on both,
so `corroborating_readings` cannot name two. Before the fix this printed
`✓ 1 readings … agree … (pymupdf)` — ungrammatical, and asserting an
agreement the list does not contain.
+
+ The third reading is the deposit and not the model's: since the
+ whole-branch review's Major 1 a `DERIVED_READINGS` reading is never
+ credited with an agreement, so an `llm` third voter would make [1]
+ `single` for a reason this test is not about, and the split vote NF1 is
+ about would stop being reproduced at all.
"""
ambiguous_1 = RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
title="A first paper", year="2020", boundary_ambiguous=True)
@@ -912,8 +924,8 @@ def _fake_propose(reading_a, reading_b, *, label_a, label_b, model=None):
title="A first paper", year="2020"),
RefEntry(num="2", raw="Beta B. A second paper. J Fixture. 2021;2:10-19.",
title="A second paper", year="2021")],
- "llm": [RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
- title="A first paper", year="2020")],
+ "crossref": [RefEntry(num="1", raw="Alpha A. A first paper. J Fixture. 2020;1:1-9.",
+ title="A first paper", year="2020")],
}
body = {"1", "2"}
agreement = label_agreement(others, body)
From 3a5af219b0e072a717cd50b2ee25bdbc873c66ba Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Fri, 18 Sep 2026 22:44:32 +0200
Subject: [PATCH 35/44] check, report: a retrieval gap survives a withholding,
and two causes reach disputed
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
`c.unjudged_refs` is decided above every branch of the claim loop, so the
withheld path can no longer `continue` past it: a claim citing a withheld [3]
and a paywalled [4] reported [4] in none of the three accounts a reader has.
It stays empty in exactly one state — a claim where nothing was judged,
withheld or provided — where the `not_retrieved` verdict is already the whole
report.
`_claim_pairing` splits a disputed label on the manifest entry's own status,
not on the emptiness of `withheld_refs`: a label nobody retrieved gets its own
token and its own sentence instead of "the source fetched under that label",
and a `results.json` written before `withheld_refs` existed still reads as the
fetched case.
`label_agreement` returns `disputed` for two causes and no field tells them
apart, so no surface asserts the contradicting one: the note, both
disclosures, the console, the disagreement file and both schema descriptions
name the disjunction.
---
schemas/refs_manifest.schema.json | 2 +-
schemas/results.schema.json | 2 +-
src/papertrace/check.py | 35 ++++++--
src/papertrace/cli.py | 11 ++-
src/papertrace/disclosures.py | 112 ++++++++++++++++++-----
tests/test_escalation.py | 5 +-
tests/test_withholding.py | 145 ++++++++++++++++++++++++++++++
7 files changed, 275 insertions(+), 37 deletions(-)
diff --git a/schemas/refs_manifest.schema.json b/schemas/refs_manifest.schema.json
index 02044ad..df22a53 100644
--- a/schemas/refs_manifest.schema.json
+++ b/schemas/refs_manifest.schema.json
@@ -246,7 +246,7 @@
},
"labels_disputed": {
"type": "array",
- "description": "Citation labels where two or more readings of the bibliography actively named different papers, so the verdict on any claim citing them was withheld rather than printed. Absent means never computed — NOT 'none disputed' — the same three-state discipline as `table_warnings`.",
+ "description": "Citation labels where two or more readings of the bibliography did not agree on which paper the label names, so the verdict on any claim citing them was withheld rather than printed. Two causes reach this list and no field tells them apart: the readings named different papers, or nothing in them could be compared — so a reader must not take an entry here as evidence of a contradiction. Absent means never computed — NOT 'none disputed' — the same three-state discipline as `table_warnings`.",
"items": {
"type": "string"
}
diff --git a/schemas/results.schema.json b/schemas/results.schema.json
index 3166ef6..e3602e2 100644
--- a/schemas/results.schema.json
+++ b/schemas/results.schema.json
@@ -203,7 +203,7 @@
"items": {
"type": "string"
},
- "description": "Cited labels whose source WAS retrieved but withheld from judgement because two or more readings of the reference list disagree about which paper that label names. Distinct from `unjudged_refs`: an entry here means the source was fetched and read, not that it could not be reached. Absent, or an empty array, means nothing was withheld for this claim."
+ "description": "Cited labels whose source WAS retrieved but withheld from judgement because two or more readings of the reference list did not agree on which paper that label names (either they named different papers, or nothing in them could be compared). Distinct from `unjudged_refs`: an entry here means the source was fetched and read, not that it could not be reached. Absent, or an empty array, means nothing was withheld for this claim."
},
"anchor_located": {
"type": [
diff --git a/src/papertrace/check.py b/src/papertrace/check.py
index 279ca60..59abe61 100644
--- a/src/papertrace/check.py
+++ b/src/papertrace/check.py
@@ -883,17 +883,40 @@ def check_claims(
avail = [(r, e) for r, e in avail_all if r not in disputed]
c.withheld_refs = sorted({r for r, _ in withheld}, key=lambda r: int(r))
own = manifest.manuscript_supplements if c.own_supplement else []
+ # Decided HERE, above every branch, so no path can `continue` past it.
+ # What is left after the retrieval filter could NOT be obtained — the
+ # only remaining reason a cited source goes unopened. Computed from
+ # `avail_all`, not `avail`: a withheld label WAS obtained, so it
+ # belongs in `withheld_refs` above, never here.
+ #
+ # The one state it stays empty in is the last clause: a claim where
+ # NOTHING was judged, withheld or provided is `not_retrieved`, and the
+ # verdict is already the whole report — repeating every cited label
+ # here would report the same gap twice, under a field whose published
+ # meaning is the CO-cited ones a surviving verdict did not rest on.
+ # When this assignment lived below the withheld branch's `continue`
+ # instead, a claim citing a withheld label and a paywalled one put the
+ # paywalled one in none of the three accounts a reader has — judged,
+ # withheld, unjudged — so a retrieval gap was erased by a withholding.
+ avail_all_refs = {r for r, _ in avail_all}
+ if avail or own or withheld:
+ c.unjudged_refs = [r for r in c.refs if r not in avail_all_refs]
if not avail and not own:
if withheld:
# NOT not_retrieved: every one of these sources WAS fetched and
# read. not_retrieved would falsely claim the source could not
- # be obtained; the true reason is that two readings of the
- # bibliography disagree about which paper this label names, so
+ # be obtained; the true reason is that the readings of the
+ # bibliography did not agree on which paper this label names, so
# no verdict can safely name the paper it was fetched for.
c.verdict = "unchecked"
labels = f"[{'], ['.join(c.withheld_refs)}]"
+ # "do not agree", never "name different papers": `disputed`
+ # has two causes — the readings contradict each other, and
+ # nothing in them could be compared — and no published field
+ # tells them apart, so a note asserting the first would be
+ # false whenever the second happened.
c.note = (
- f"withheld: the reference list's readings disagree about {labels}, "
+ f"withheld: the reference list's readings do not agree about {labels}, "
"so the source retrieved under that label may not be the paper the "
"manuscript cites — check the retrieval manifest before relying on "
"this claim"
@@ -952,12 +975,6 @@ def check_claims(
SourceJudgement(source_slug=s.slug, ref="", kind="own_supplement")
)
by_slug.setdefault(s.slug, []).append(c)
- # what is left here could NOT be obtained — the only remaining reason a
- # cited source goes unopened. Computed from `avail_all`, not `avail`: a
- # withheld label WAS obtained, so it belongs in `withheld_refs` above,
- # never here — this is retrieval failure only.
- avail_all_refs = {r for r, _ in avail_all}
- c.unjudged_refs = [r for r in c.refs if r not in avail_all_refs]
for slug, group in by_slug.items():
try:
diff --git a/src/papertrace/cli.py b/src/papertrace/cli.py
index 727fc6d..b0e6e83 100644
--- a/src/papertrace/cli.py
+++ b/src/papertrace/cli.py
@@ -546,8 +546,9 @@ def _write_disagreement(
lines = [
"# Reference numbering — where the readings disagree",
"",
- "One section per citation label whose readings of the bibliography name "
- "different papers. Each reading's structured fields sit above the verbatim "
+ "One section per citation label whose readings of the bibliography did not "
+ "agree on one paper — either they name different papers, or nothing in them "
+ "could be compared. Each reading's structured fields sit above the verbatim "
"text it was read from, so the disagreement can be settled by eye against "
"the printed list.",
"",
@@ -1261,8 +1262,10 @@ def _refs_pipeline(
)
if rec.labels_disputed:
console.print(
- f"[yellow]⚠ readings disagree at [{'], ['.join(rec.labels_disputed)}][/yellow] — "
- "verdicts on claims citing those labels are withheld unless resolved below"
+ f"[yellow]⚠ the readings do not agree at "
+ f"[{'], ['.join(rec.labels_disputed)}][/yellow] — either they name different "
+ "papers there or nothing in them could be compared; verdicts on claims citing "
+ "those labels are withheld unless resolved below"
)
stamp_seen_in(entries, others)
diff --git a/src/papertrace/disclosures.py b/src/papertrace/disclosures.py
index fffd2c1..3f74927 100644
--- a/src/papertrace/disclosures.py
+++ b/src/papertrace/disclosures.py
@@ -59,6 +59,12 @@
# ANCHOR_UNKNOWN_TOKEN are one Disclosure key with one token per state actually
# reached.
CLAIM_PAIRING_WITHHELD_TOKEN = "verdict withheld — the reference list readings disagree"
+# A DIFFERENT fact from the one above, and it needs its own words: the
+# readings disagree AND nothing was ever fetched under the label, so there is
+# no source that "may not be the paper the manuscript cites" — there is no
+# source. `check.py` keeps such a label out of `withheld_refs` for this
+# reason; this token is how the report keeps the distinction it computed.
+CLAIM_PAIRING_UNRETRIEVED_TOKEN = "the readings disagree, and no source was retrieved for it"
CLAIM_PAIRING_RESOLVED_TOKEN = "resolved by a model reading of both texts, accepted by you"
CLAIM_PAIRING_CHOSEN_TOKEN = "you chose which reading of the reference list to use"
CLAIM_PAIRING_CORROBORATED_TOKEN = "agreed by every reading of the reference list"
@@ -555,10 +561,12 @@ def _labels_disputed(manifest) -> Disclosure | None:
token=LABELS_DISPUTED_TOKEN,
text=(
f"{named} are {LABELS_DISPUTED_TOKEN}: two or more readings of the "
- "reference list name different papers for each of these labels, so "
- "every claim citing one had that source withheld from judgement "
- "rather than risk a verdict about the wrong paper. See each claim's "
- "own note for which of its citations this affected."
+ "reference list did not agree on which paper each of these labels "
+ "names — they either named different papers, or nothing in them "
+ "could be compared — so every claim citing one had that source "
+ "withheld from judgement rather than risk a verdict about the wrong "
+ "paper. See each claim's own note for which of its citations this "
+ "affected."
),
short=f"{named} {LABELS_DISPUTED_TOKEN}",
)
@@ -1226,33 +1234,73 @@ def _no_quote(claim) -> Disclosure:
)
+def _was_obtained(manifest, label: str) -> bool:
+ """Did this manifest actually retrieve a judgeable source for this label?
+
+ The same predicate `check.py`'s retrieval filter applies before it
+ partitions the withheld from the judged, restated here rather than
+ inferred from the emptiness of `withheld_refs` — an empty list is also
+ what a `results.json` written before that field existed carries.
+ """
+ e = next((x for x in getattr(manifest, "entries", []) if x.num == label), None)
+ return bool(e and e.status in ("retrieved", "provided") and e.slug)
+
+
def _claim_pairing(claim, manifest=None) -> Disclosure | None:
"""Which pairing state this claim's cited labels are in, if not a clean one.
- Four states, four tokens — the same one-key-many-tokens shape `anchor`
- uses. `withheld` reads only `claim.withheld_refs`, so it fires with no
- `manifest` at all (as it always has); the other three read the manifest's
- escalation fields and are silent without one. `corroborated` is the only
- `info`-level state, and `claim_disclosures` gates it on the claim already
- carrying a `warn`: reassurance repeated on every clean claim is noise a
- reader stops reading.
+ Five states, five tokens — the same one-key-many-tokens shape `anchor`
+ uses. `withheld` reads `claim.withheld_refs` and fires with no `manifest`
+ at all (as it always has); the other four read the manifest's escalation
+ fields and are silent without one. `corroborated` is the only `info`-level
+ state, and `claim_disclosures` gates it on the claim already carrying a
+ `warn`: reassurance repeated on every clean claim is noise a reader stops
+ reading.
+
+ A disputed label is in exactly one of two of those states, and the split
+ is the manifest entry's own status: withheld (a source was retrieved and
+ set aside) or unretrieved (there is no source at all). Both withhold a
+ verdict; only one of them has a fetched paper to warn about.
"""
cited = set(claim.refs)
- withheld = sorted(cited & set(claim.withheld_refs), key=int)
disputed = sorted(cited & set(getattr(manifest, "labels_disputed", None) or []), key=int)
resolved = sorted(cited & set(getattr(manifest, "labels_resolved", None) or []), key=int)
-
- if withheld or disputed:
- labels = _label_group(withheld or disputed)
- one = len(withheld or disputed) == 1
+ # A disputed label splits in two, and `check.py` already computed the
+ # split once: a label whose source WAS obtained is in `withheld_refs`; one
+ # that was never retrieved is deliberately kept out of it
+ # (`check.py`'s retrieval filter, and the test that pins it). Reading only
+ # `labels_disputed` when `withheld_refs` is empty undid that care and told
+ # a reader "the source fetched under that label" about a source nobody
+ # fetched. The entry's own status is what decides, NOT the emptiness of
+ # `withheld_refs`: a `results.json` written before that field existed
+ # carries `[]` for every claim, and choosing the never-retrieved wording
+ # from that emptiness would be the same mistake in the other direction.
+ unretrieved = [r for r in disputed if not _was_obtained(manifest, r)]
+ withheld = sorted(set(claim.withheld_refs) | (set(disputed) - set(unretrieved)), key=int)
+ withheld = [r for r in withheld if r in cited]
+
+ if withheld:
+ labels = _label_group(withheld)
+ one = len(withheld) == 1
text = (
f"This claim cites {labels}, and the {CLAIM_PAIRING_WITHHELD_TOKEN} about "
- f"{'that label' if one else 'those labels'}: two readings of the "
- "bibliography name different papers for it, so the source fetched under "
- f"that label may not be the paper the manuscript actually cites. No "
- f"verdict was reached on {'it' if one else 'them'} — check the retrieval "
- "manifest before treating this claim as checked."
+ f"{'that label' if one else 'those labels'}: the readings of the "
+ "bibliography did not agree on which paper it names — they either named "
+ "different papers or left nothing that could be compared — so the source "
+ "retrieved under that label may not be the paper the manuscript actually "
+ f"cites. No verdict was reached on {'it' if one else 'them'} — check the "
+ "retrieval manifest before treating this claim as checked."
)
+ if unretrieved:
+ # named separately, never folded into the sentence above: nothing
+ # was fetched under these, so "the source retrieved under that
+ # label" is not true of them and the reader must not read one
+ # sentence as covering both
+ text += (
+ f" It also cites {_label_group(unretrieved)}, where the readings did "
+ "not agree either and the source was never retrieved, so nothing was "
+ "judged under that label at all."
+ )
if resolved:
# This claim cites BOTH a still-disputed label and one that WAS
# resolved — the withheld state is the more urgent finding and
@@ -1273,6 +1321,28 @@ def _claim_pairing(claim, manifest=None) -> Disclosure | None:
short=f"{labels} {CLAIM_PAIRING_WITHHELD_TOKEN}",
)
+ if unretrieved:
+ # Nothing was withheld here, because nothing was obtained to withhold.
+ # Its own token and its own sentence: the withheld one asserts a
+ # source was retrieved under the label, which is the one thing that
+ # did not happen.
+ labels = _label_group(unretrieved)
+ one = len(unretrieved) == 1
+ return Disclosure(
+ key="claim_pairing",
+ level="warn",
+ token=CLAIM_PAIRING_UNRETRIEVED_TOKEN,
+ text=(
+ f"This claim cites {labels}, and {CLAIM_PAIRING_UNRETRIEVED_TOKEN}: the "
+ "readings of the bibliography did not agree on which paper "
+ f"{'it names' if one else 'they name'}, and the source was never "
+ "retrieved either, so nothing was judged under "
+ f"{'that label' if one else 'those labels'} in either direction. The "
+ "disagreement is recorded, not resolved."
+ ),
+ short=f"{labels} {CLAIM_PAIRING_UNRETRIEVED_TOKEN}",
+ )
+
if resolved and manifest is not None and manifest.numbering_choice == "llm_resolved":
labels = _label_group(resolved)
return Disclosure(
diff --git a/tests/test_escalation.py b/tests/test_escalation.py
index 40a13ce..8a4a826 100644
--- a/tests/test_escalation.py
+++ b/tests/test_escalation.py
@@ -436,7 +436,10 @@ def test_a_run_never_says_confirmed_beside_a_withheld_label(disputed, capsys):
assert manifest.numbering_verified is True
assert manifest.labels_disputed == ["2"]
- assert "readings disagree at [2]" in out
+ # "do not agree", not "disagree": `disputed` also covers "nothing in the
+ # readings could be compared", and the console may not assert the
+ # contradicting cause for both — see `refs._label_state`
+ assert "the readings do not agree at [2]" in out
assert "numbering confirmed" not in out, "confirmed printed beside a withheld label"
assert "accounts for every cited label" in out
diff --git a/tests/test_withholding.py b/tests/test_withholding.py
index 252806d..7ae7c79 100644
--- a/tests/test_withholding.py
+++ b/tests/test_withholding.py
@@ -181,6 +181,151 @@ def test_withheld_refs_round_trips_and_validates_against_the_schema(tmp_path):
jsonschema.Draft202012Validator(schema).validate(json.loads(path.read_text()))
+def test_a_co_cited_label_that_could_not_be_retrieved_survives_a_withholding(tmp_path):
+ """Major 2 of the whole-branch review, and a regression against `main`.
+
+ A claim cites a disputed-and-retrieved [3] and a paywalled [4]. Nothing
+ survives to judge, so the claim is `unchecked` for the withholding — and
+ on the way out of that branch `unjudged_refs` was never filled, so [4]
+ was in none of the three accounts a reader has (judged, withheld,
+ unjudged) and appeared in no note. A retrieval gap erased by a
+ withholding is the cardinal rule's own example of what may not happen.
+ """
+ _write_source(tmp_path, "three-2019")
+ manifest = _manifest(
+ _entry("3", "three-2019"),
+ _entry("4", "four-2020", status="paywalled"),
+ )
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["3", "4"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"3"})
+
+ assert claims[0].verdict == "unchecked"
+ assert claims[0].withheld_refs == ["3"]
+ assert claims[0].unjudged_refs == ["4"]
+
+
+def test_an_unretrievable_co_citation_is_reported_where_a_verdict_survives_too(
+ tmp_path, monkeypatch
+):
+ """The same accounting on the path that DOES keep a verdict, so the fix
+ is not a special case bolted onto the one branch the review named: [3]
+ withheld, [4] paywalled, [5] judged."""
+ _write_source(tmp_path, "five-2021")
+ manifest = _manifest(
+ _entry("3", "three-2019"),
+ _entry("4", "four-2020", status="paywalled"),
+ _entry("5", "five-2021"),
+ )
+ monkeypatch.setattr(check_mod, "_ask", lambda prompt, model=None: _reply(1, "supported"))
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["3", "4", "5"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"3"})
+
+ assert claims[0].verdict == "supported"
+ assert claims[0].withheld_refs == ["3"]
+ assert claims[0].unjudged_refs == ["4"]
+
+
+# --- what the prose may say about a label whose readings did not agree -----
+
+
+def test_the_withholding_note_does_not_assert_the_readings_named_different_papers(
+ tmp_path,
+):
+ """`label_agreement` returns `disputed` for two causes — the readings
+ actively name different papers, and nothing in them could be compared —
+ and no published field tells the two apart. A note asserting the first is
+ therefore false whenever the second happened, which is the same mistake
+ as describing a source that was never fetched as one that was."""
+ _write_source(tmp_path, "six-2019")
+ manifest = _manifest(_entry("6", "six-2019"))
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["6"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"6"})
+
+ assert "different papers" not in claims[0].note
+ assert "do not agree" in claims[0].note
+
+
+def test_the_run_level_disputed_disclosure_covers_both_causes():
+ """The run-level roll-up says the same thing as the note above, so it
+ inherits the same constraint."""
+ from papertrace.disclosures import _labels_disputed
+
+ d = _labels_disputed(RefManifest(manuscript="m.pdf", labels_disputed=["6"]))
+
+ assert d is not None
+ # the disjunction, not the assertion: naming both causes is honest,
+ # asserting the contradicting one is not
+ assert "did not agree on which paper" in d.text
+ assert "nothing in them could be compared" in d.text
+ assert "name different papers for each of these labels" not in d.text
+
+
+# --- a disputed label nobody retrieved is not a source that was fetched ----
+
+
+def test_a_disputed_label_nobody_retrieved_is_not_described_as_fetched():
+ """Major 5. `check.py` keeps such a label out of `withheld_refs` on
+ purpose (`test_a_disputed_label_never_retrieved_stays_not_retrieved`);
+ the disclosure layer undid that care by falling back to
+ `labels_disputed`, and printed "the source fetched under that label" for
+ a source nobody ever fetched."""
+ from papertrace.disclosures import _claim_pairing
+
+ manifest = _manifest(_entry("6", "six-2019", status="paywalled"))
+ manifest.labels_disputed = ["6"]
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["6"],
+ verdict="not_retrieved")
+
+ d = _claim_pairing(claim, manifest)
+
+ assert d is not None
+ assert "fetched" not in d.text
+ assert "never retrieved" in d.text
+
+
+def test_a_retrieved_disputed_label_still_reads_as_a_source_that_was_fetched():
+ """The other side of Major 5's split, and the load-path guard on it: a
+ `results.json` written before `withheld_refs` existed carries `[]` for
+ every claim, so the new branch must not be chosen from that emptiness
+ alone. The manifest's own entry status is what decides, and here the
+ source really was retrieved."""
+ from papertrace.disclosures import _claim_pairing
+
+ manifest = _manifest(_entry("6", "six-2019"))
+ manifest.labels_disputed = ["6"]
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["6"]) # no withheld_refs
+
+ d = _claim_pairing(claim, manifest)
+
+ assert d is not None
+ assert "the source retrieved under that label" in d.text
+
+
+def test_a_claim_citing_both_a_withheld_and_an_unretrieved_disputed_label_names_both():
+ """One claim, both causes. The withheld label keeps the token and the
+ level — it is the more urgent finding, because a paper WAS fetched under
+ it — and the label nobody retrieved is named rather than folded into the
+ same sentence."""
+ from papertrace.disclosures import _claim_pairing
+
+ manifest = _manifest(
+ _entry("6", "six-2019"),
+ _entry("7", "seven-2020", status="paywalled"),
+ )
+ manifest.labels_disputed = ["6", "7"]
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["6", "7"],
+ withheld_refs=["6"])
+
+ d = _claim_pairing(claim, manifest)
+
+ assert d is not None
+ assert "[6]" in d.text and "[7]" in d.text
+ assert "never retrieved" in d.text
+
+
def test_any_iterable_of_labels_withholds_every_claim_not_just_the_first(tmp_path, monkeypatch):
"""`disputed` is typed `set[str] | None`, and a set is what the CLI passes.
From 5e99af6921753ec332f14a4a988f0542ebc7c075 Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Fri, 18 Sep 2026 23:00:43 +0200
Subject: [PATCH 36/44] refs, report: the resolution call sees what it rules
against, and says who made it
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
`resolve_disputed` is passed `disputing` — what each reading says at each
disputed label, including the deposit and the model's own reading, which have
no printed span and were therefore invisible to a call adjudicating between
them. It is context: `_found` still verifies every returned value against the
printed extraction alone. The prompt renders one block per extraction it
actually has, so a pymupdf-backend run no longer shows an empty `READING B`.
Five `resolution_*` manifest fields carry that call's own provenance. A run
whose backend skips the reading and then escalates recorded
`reflist_outcome: not_attempted` and nothing else, so the report said no model
read the reference list on a run where one was asked, verified field by field
and allowed to change which papers are judged. The `reflist` disclosure now
names it in every branch, and fires on its own when it was the only call.
The note, the run-level resolution and the per-claim disclosure name the
extractions the model was shown instead of asserting "both texts".
Also: `numbering_corroborated` is `bool | None`, so "never computed" is a
value and not a comment; `stamp_seen_in` runs after the escalation, so option
3 no longer publishes an empty provenance for a run that computed one; option
3's note says the reading was adopted whole; the resolution reply's entry
count is taken before de-duplication; a mid-branch `reflist_attempted` keeps
its disclosure without claiming an outcome nobody recorded; and both calls
refuse an oversized bibliography before spending anything rather than
clipping it into a reading quietly short.
---
schemas/refs_manifest.schema.json | 30 +++-
src/papertrace/cli.py | 92 +++++++++--
src/papertrace/disclosures.py | 139 ++++++++++++++--
src/papertrace/models.py | 70 +++++++-
src/papertrace/reflist.py | 128 +++++++++++++--
src/papertrace/refs.py | 12 ++
tests/test_disclosure_parity.py | 46 ++++++
tests/test_escalation.py | 256 ++++++++++++++++++++++++++++--
tests/test_label_agreement.py | 5 +-
tests/test_reference_readings.py | 208 +++++++++++++++++++++++-
tests/test_reflist.py | 94 +++++++++++
11 files changed, 1017 insertions(+), 63 deletions(-)
diff --git a/schemas/refs_manifest.schema.json b/schemas/refs_manifest.schema.json
index df22a53..16addff 100644
--- a/schemas/refs_manifest.schema.json
+++ b/schemas/refs_manifest.schema.json
@@ -234,12 +234,12 @@
}
},
"numbering_corroborated": {
- "type": "boolean",
- "description": "True when every label the manuscript cites was agreed by two or more independent readings of the bibliography. A second, independent axis from `numbering_verified`, which is about matching the body's labels, not about readings agreeing with each other — a manifest can be verified and uncorroborated, or corroborated and unverified. Absent means never computed, not 'not corroborated'."
+ "type": ["boolean", "null"],
+ "description": "True when every label the manuscript cites was agreed by two or more independent readings of the bibliography, false when that was measured and did not hold. A second, independent axis from `numbering_verified`, which is about matching the body's labels, not about readings agreeing with each other — a manifest can be verified and uncorroborated, or corroborated and unverified. `null` or absent means NEVER COMPUTED, and is not the same as `false`: as a plain boolean this field could not express the third state its own description claimed, so every consumer had to stay silent on `false` to avoid asserting a measurement nobody made."
},
"corroborating_readings": {
"type": "array",
- "description": "Which named readings (e.g. 'parsed', 'pymupdf', 'llm') fed `numbering_corroborated`, named rather than merely counted. Absent means never computed.",
+ "description": "Which named readings (e.g. 'parsed', 'pymupdf', 'crossref') fed `numbering_corroborated`, named rather than merely counted. Never includes a DERIVED reading: the model reading copies its values out of the extractions that vote here in their own right, so naming it would print two agreeing readings over one text read twice. Absent means never computed.",
"items": {
"type": "string"
}
@@ -308,6 +308,30 @@
"reflist_entries_proposed": {
"type": ["integer", "null"],
"description": "How many entries the model's reply proposed, before verification, whenever `reflist_outcome` is 'read' — including 0. `null` (or absent) means NEVER RECORDED — either no reading happened, or this manifest was written before this field existed — and is NOT the same as `0`, which means a reading happened and measured zero proposed entries. A manifest that recorded `reflist_model` or `reflist_fields_discarded` without this field is exactly the case `null` exists for: without the distinction, a model that proposed nothing was told apart from one whose count was simply never recorded, which this field alone can settle."
+ },
+ "resolution_outcome": {
+ "type": "string",
+ "description": "Which of `reflist.REFLIST_OUTCOMES` the RESOLUTION call reached — the second model call this feature can make, asked by a person to settle the labels in `labels_disputed`. A separate fact from `reflist_outcome`, which describes the call that reads the whole list: a run can make either, both or neither, and on a pymupdf-backend run the first is never attempted while the second still can be. Absent or '' means NEVER COMPUTED — not 'not_attempted'."
+ },
+ "resolution_failure": {
+ "type": "string",
+ "description": "Why the resolution call produced nothing, when `resolution_outcome` is 'failed'. '' when it answered, or when no such call was ever made — `resolution_outcome` is what tells those apart. Absent means never computed."
+ },
+ "resolution_model": {
+ "type": "string",
+ "description": "The model that answered the resolution call, when the subprocess's own reply reported a name. '' means EITHER no such call was made OR one was made and answered without naming a model; `resolution_outcome` is what distinguishes them. Absent means never computed."
+ },
+ "resolution_fields_discarded": {
+ "type": "array",
+ "items": { "type": "string" },
+ "description": "Which fields of the resolution reply were not found verbatim in the printed text(s) the call was shown, and were discarded rather than trusted. Absent, or an empty array on a manifest whose `resolution_outcome` is '', means never computed — NOT 'nothing was discarded'.",
+ "default": []
+ },
+ "resolution_readings": {
+ "type": "array",
+ "items": { "type": "string" },
+ "description": "Which printed extractions of the bibliography the resolution call was actually SHOWN, by reading name — not which readings disagreed. The report names these instead of asserting the call saw 'both texts', which is false on any run with one printed extraction. Absent or empty means never recorded.",
+ "default": []
}
}
}
diff --git a/src/papertrace/cli.py b/src/papertrace/cli.py
index b0e6e83..737aa46 100644
--- a/src/papertrace/cli.py
+++ b/src/papertrace/cli.py
@@ -788,6 +788,7 @@ def _escalate_disputed(
under a report line saying a person accepted it: the original wrong-paper
bug, reached through the one path a human authorised.
"""
+ from .disclosures import _readings_phrase # one wording for the report and the note
from .refs import _label_list, _unique_slugs # top-level in `refs.py`; not visible from
# `_refs_pipeline`'s own local import, since this is a different function's frame
@@ -877,10 +878,15 @@ def _escalate_disputed(
rec.unverified_from = None
rec.ledger = {}
rec.source = pick
+ # says what actually happened: `list(candidates[pick])` replaces EVERY
+ # entry, not only the disputed ones, so "you chose X for the 1 label in
+ # dispute" understated the change — `verified`, `contested` and
+ # `ledger` are cleared above for exactly that reason
rec.note = (
- f"you chose {_READING_LABEL.get(pick, pick)} for the {n} label"
- f"{'' if n == 1 else 's'} in dispute. That is your reading, not a check of "
- "it — neither its content nor its extent has been confirmed"
+ f"you chose {_READING_LABEL.get(pick, pick)} and it was adopted whole — "
+ f"every entry, not only the {n} label{'' if n == 1 else 's'} in dispute. "
+ "That is your reading, not a check of it — neither its content nor its "
+ "extent has been confirmed"
)
return list(candidates[pick])
@@ -891,7 +897,20 @@ def _escalate_disputed(
# which can hold readings (crossref, llm) never shown to this call at all.
names = tuple(sorted(texts))
try:
- res = resolve_disputed(labels, names, tuple(texts[k] for k in names), model=model)
+ res = resolve_disputed(
+ labels, names, tuple(texts[k] for k in names),
+ # what the dispute is BETWEEN, including the readings with no
+ # printed span of their own. Without it the call adjudicated a
+ # parse-versus-deposit disagreement having seen only the parse,
+ # while the note and the per-claim disclosure both said it was
+ # shown both. Context only — `reflist._found` still verifies
+ # every returned value against the printed text alone.
+ disputing={
+ name: [e for e in cand if e.num in set(labels)]
+ for name, cand in candidates.items()
+ },
+ model=model,
+ )
except Exception as e:
# The broad except IS the point: the refs stage has already done
# useful work and must not go down with a failed subprocess call.
@@ -908,12 +927,42 @@ def _escalate_disputed(
f"{type(e).__name__}: {str(e)[:200]}[/yellow]"
)
rec.choice, rec.chosen_by = "withheld", "user"
+ # a call that was made and did not return is the third outcome, and it
+ # is persisted: without it the manifest's only record of this call is
+ # free prose, and the `reflist_*` fields — which describe a different
+ # call entirely — say no model was asked to read the list at all
+ rec.resolution_outcome = "failed"
+ rec.resolution_failure = f"{type(e).__name__}: {str(e)[:200]}"
+ rec.resolution_readings = list(names)
rec.note += (
f"; asked a model to resolve the {n} disputed label{'' if n == 1 else 's'} and "
"the call failed, so they remain withheld"
)
return entries
+ if res.provenance.outcome != "read":
+ # `resolve_disputed` declined before spending anything — the two
+ # extractions are too long to send in one call. No reply exists, so
+ # none of the reporting below applies: `_disputed_reason` would print
+ # "the model could not tell which paper it names" over a model that
+ # was never asked, and the note would credit a reading verified
+ # verbatim against a text nobody sent. Same shape as the `except`
+ # above, and recorded the same way.
+ console.print(
+ f"[yellow]⚠ the model was not asked to resolve the disputed label"
+ f"{'' if n == 1 else 's'} — {res.provenance.failure}[/yellow]"
+ )
+ rec.choice, rec.chosen_by = "withheld", "user"
+ rec.resolution_outcome = res.provenance.outcome or "not_attempted"
+ rec.resolution_failure = res.provenance.failure
+ rec.resolution_readings = list(res.provenance.readings)
+ rec.note += (
+ f"; asked a model to resolve the {n} disputed label{'' if n == 1 else 's'} "
+ f"and the call was not made — {res.provenance.failure} — so they remain "
+ "withheld"
+ )
+ return entries
+
# A resolved label absent from `entries` has no entry to substitute — the
# chosen list simply never carried that numeral. Recording it resolved
# anyway would be a label marked settled with nothing behind it, so it is
@@ -939,16 +988,25 @@ def _escalate_disputed(
parts = [f"[{lbl}] {_disputed_reason(lbl, res.provenance)}" for lbl in rec.labels_disputed]
console.print(f" [dim]{'; '.join(parts)} — verdicts on these stay withheld.[/dim]")
named_model = res.provenance.model or "a model that did not report its own name"
+ rec.resolution_outcome = res.provenance.outcome
+ rec.resolution_model = res.provenance.model
+ rec.resolution_fields_discarded = list(res.provenance.fields_discarded)
+ rec.resolution_readings = list(res.provenance.readings)
+ # Names the extractions the call was SHOWN, never "both texts": on a
+ # pymupdf-backend run there is one printed span, and on any backend the
+ # dispute can be with a reading (the deposit, the model's) that has no
+ # span at all. "both" was false on the first and misleading on the second.
+ shown = _readings_phrase(res.provenance.readings)
rec.note += (
f"; {done} of the {n} disputed label{'' if n == 1 else 's'} were resolved by "
- f"{named_model} reading of both texts, with every field verified verbatim, and "
- "you accepted that. It is a reading, not a confirmation — the numbering is "
- "still unconfirmed"
+ f"{named_model} reading {shown}, with every field verified verbatim against "
+ "it, and you accepted that. It is a reading, not a confirmation — the "
+ "numbering is still unconfirmed"
)
if res.provenance.fields_discarded:
rec.note += (
f"; {len(res.provenance.fields_discarded)} field(s) of the resolution reply "
- "were not printed in either text and were discarded"
+ "were not printed there and were discarded"
)
# The resolved entry REPLACES the one in the chosen list, in place, same
# order, same length. Returning `entries` unchanged here takes the label
@@ -1267,8 +1325,6 @@ def _refs_pipeline(
"papers there or nothing in them could be compared; verdicts on claims citing "
"those labels are withheld unless resolved below"
)
- stamp_seen_in(entries, others)
-
if rec.verified and rec.labels_disputed:
# `numbering_verified` means EXTENT — the chosen list accounts for
# exactly the labels the body cites — and a disputed label is a question
@@ -1307,6 +1363,13 @@ def _refs_pipeline(
if needs_flat:
reading_texts["pymupdf"] = flat_text
entries = _escalate_disputed(case, rec, entries, others, reading_texts, model=None)
+ # AFTER the escalation, never before it: menu option 3 replaces the whole
+ # list with one nothing stamped, so a run that computed `seen_in`
+ # published `[]` for every entry — which the schema reads as "no reading
+ # was established as carrying this". `stamp_seen_in` merges rather than
+ # overwrites, so an entry the resolution substituted keeps the printed
+ # text its values were copied from.
+ stamp_seen_in(entries, others)
if provided is not None and provided.is_file():
console.print(
@@ -1379,6 +1442,15 @@ def tick(e):
reflist_entries_proposed=(
reflist_prov.entries_proposed if reflist_prov.outcome == "read" else None
),
+ # the SECOND model call, kept apart from the five fields above: a run
+ # can make either, both or neither, and folding them made a manifest
+ # say no model read the reference list on a run where one was asked,
+ # verified and allowed to change which papers are judged
+ resolution_outcome=rec.resolution_outcome,
+ resolution_failure=rec.resolution_failure,
+ resolution_model=rec.resolution_model,
+ resolution_fields_discarded=rec.resolution_fields_discarded,
+ resolution_readings=rec.resolution_readings,
)
manifest.to_json(case / "refs_manifest.json")
ok = len(manifest.retrieved)
diff --git a/src/papertrace/disclosures.py b/src/papertrace/disclosures.py
index 3f74927..a619236 100644
--- a/src/papertrace/disclosures.py
+++ b/src/papertrace/disclosures.py
@@ -65,7 +65,12 @@
# source. `check.py` keeps such a label out of `withheld_refs` for this
# reason; this token is how the report keeps the distinction it computed.
CLAIM_PAIRING_UNRETRIEVED_TOKEN = "the readings disagree, and no source was retrieved for it"
-CLAIM_PAIRING_RESOLVED_TOKEN = "resolved by a model reading of both texts, accepted by you"
+# "of the printed list", never "of both texts": the call is shown the
+# extractions that HAVE a printed span, which is one on a pymupdf-backend
+# run, and the reading it rules against (the deposit, the model's own) may
+# have none at all. A token is asserted verbatim in all four formats, so it
+# has to stay true in every one of those shapes.
+CLAIM_PAIRING_RESOLVED_TOKEN = "resolved by a model reading of the printed list, accepted by you"
CLAIM_PAIRING_CHOSEN_TOKEN = "you chose which reading of the reference list to use"
CLAIM_PAIRING_CORROBORATED_TOKEN = "agreed by every reading of the reference list"
# no apostrophe, `&`, `<` or `>` in either — same HTML-autoescape rule as
@@ -610,6 +615,10 @@ def _reflist(manifest) -> Disclosure | None:
# ever recording this count must not be told apart from one that measured
# zero. See `RefManifest.reflist_entries_proposed`'s own docstring.
entries_proposed = getattr(manifest, "reflist_entries_proposed", None)
+ # set only where `outcome` below is a GUESS rather than a record — see the
+ # `elif failure:` branch, which is the one inference here that cannot be
+ # backed by positive evidence
+ inferred = False
if not outcome:
# An older manifest from before `reflist_outcome` existed.
# `reflist_fields_discarded` used to be the ONLY slot for "why there
@@ -646,9 +655,13 @@ def _reflist(manifest) -> Disclosure | None:
# distinguish "never attempted" from "attempted and failed" (both
# states write to the same field), so "not_attempted" here is a
# guess at which of the two happened, not positive proof of it —
- # only reachable from a hand-written manifest, since a live run
- # always records `reflist_outcome` itself.
- outcome = "not_attempted"
+ # only reachable from a hand-written manifest, or from a case
+ # folder written mid-branch whose `reflist_attempted: true`
+ # `from_json` translates into exactly such a reason, since a live
+ # run always records `reflist_outcome` itself. `inferred` is what
+ # keeps the sentence below honest about which it is: "no reading
+ # was obtained" is a claim this branch has no evidence for.
+ outcome, inferred = "not_attempted", True
else:
outcome = ""
elif outcome not in REFLIST_OUTCOMES:
@@ -669,6 +682,7 @@ def _reflist(manifest) -> Disclosure | None:
),
short=f"{REFLIST_TOKEN}: unrecognised outcome {outcome!r}",
)
+ resolution = _resolution_clause(manifest)
if not outcome or (outcome == "not_attempted" and not failure):
# the second clause is the caller's own silent choice (`--no-llm-refs`,
# `--parse-only`): `outcome` is recorded as `"not_attempted"` either
@@ -676,7 +690,23 @@ def _reflist(manifest) -> Disclosure | None:
# from "asked for, and never happened" — only the absence of a reason
# can, the same distinction `_llm_reference_reading`'s `disabled_reason`
# makes when it decides whether to fill `failure` at all
- return None # no model reading was asked for, and nothing to report
+ if not resolution:
+ return None # no model reading was asked for, and nothing to report
+ # ...but the ESCALATION is a second, separate call that runs whether
+ # or not the reading was asked for, and a run that made it is not a
+ # run where no model read the reference list. Silence here was the
+ # same false negative one rung down.
+ return Disclosure(
+ key="reflist",
+ level="info",
+ token=REFLIST_TOKEN,
+ text=(
+ f"No reading of the whole list was taken on this run, but {REFLIST_TOKEN} "
+ "to settle a disagreement. "
+ + _resolution_clause(manifest, lead="Here")
+ ),
+ short=f"{REFLIST_TOKEN} — to settle a disagreement only",
+ )
ceiling = (
"A model agreeing with a parse is a second reading, not confirmation: it read "
"the same document, so a reference the layout destroyed is one it may also "
@@ -687,7 +717,20 @@ def _reflist(manifest) -> Disclosure | None:
# Identical wording to `cli._refs_pipeline`'s console line for the same
# state, so the two surfaces never describe one call two different ways.
named = model or "a model that did not report its own name"
- if outcome == "not_attempted":
+ if outcome == "not_attempted" and inferred:
+ # `outcome` here is a GUESS (see the `elif failure:` branch above), so
+ # the sentence says what is recorded and no more. "no reading was
+ # obtained" below is a claim, and this state has no evidence for it:
+ # the manifest says a reading was asked for and stops there.
+ head = (
+ f"This run asked that {REFLIST_TOKEN}, and this manifest does not record "
+ f"what came of it: {failure}. Nothing from such a reading reached the list "
+ "below, so the reference numbering rests on the readings above this and "
+ "nothing else."
+ )
+ short = f"{REFLIST_TOKEN}: asked for, outcome never recorded — {failure}"
+ level = "warn"
+ elif outcome == "not_attempted":
head = (
f"This run asked that {REFLIST_TOKEN}, and no reading was obtained: "
f"{failure}. The reference numbering therefore rests on the readings above "
@@ -781,7 +824,10 @@ def _reflist(manifest) -> Disclosure | None:
key="reflist",
level=level,
token=REFLIST_TOKEN,
- text=f"{head} {ceiling}",
+ # the OTHER call, appended in every branch: whatever became of the
+ # reading, a run that also asked a model to settle a disagreement is
+ # not a run where no model read the reference list
+ text=f"{head}{resolution} {ceiling}",
short=short,
# both kinds, numbering first: it describes the reading as a whole, and
# a truncated list should not lose the structural finding to six field
@@ -861,8 +907,18 @@ def _numbering_resolution(manifest) -> Disclosure | None:
if not resolved or choice not in ("llm_resolved", "parsed", "pymupdf"):
return None
named = f"[{'], ['.join(resolved)}]"
+ # names the model and the extractions it was SHOWN, never "both texts":
+ # the call sees the readings that have a printed span, which is one on a
+ # pymupdf-backend run, and the reading it ruled against may have no span
+ # at all. The claim was false on the first and misleading on the second.
+ model = (
+ getattr(manifest, "resolution_model", "")
+ or "a model that did not report its own name"
+ )
+ shown = _readings_phrase(getattr(manifest, "resolution_readings", []))
by = (
- "a model reading of both texts, verified field by field, which you accepted"
+ f"{model}, which read {shown} and was shown what each reading said at those "
+ "labels, verified field by field against that text, and which you accepted"
if choice == "llm_resolved"
else f"you, who chose the `{choice}` reading whole for every disputed label"
)
@@ -1234,6 +1290,59 @@ def _no_quote(claim) -> Disclosure:
)
+def _resolution_clause(manifest, *, lead: str = " Separately, later in this run") -> str:
+ """What the RESOLUTION call did, as a sentence to append — or "".
+
+ `lead` is how the sentence opens, because the same facts are appended to
+ a disclosure about the OTHER call and used as a disclosure of their own
+ when there was no other call — one producer, so the two surfaces cannot
+ describe one call two ways.
+
+ Two model calls can be made in one run and they are not the same call.
+ `reflist_*` describes the one that reads the whole bibliography;
+ `resolution_*` describes the one a person asked for, to settle the labels
+ the readings did not agree on. A `--backend pymupdf` run never makes the
+ first and can still make the second, and folding them left the report
+ saying no model read the reference list on a run where one was asked,
+ verified field by field and allowed to change which papers are judged.
+ """
+ outcome = getattr(manifest, "resolution_outcome", "") or ""
+ if outcome not in ("read", "failed"):
+ return "" # "" is never computed, and an absent call says nothing
+ if outcome == "failed":
+ why = getattr(manifest, "resolution_failure", "") or "no reason was recorded"
+ return (
+ f"{lead} a model was asked to settle the labels the readings did not agree "
+ f"on, and that call did not return: {why}."
+ )
+ named = (
+ getattr(manifest, "resolution_model", "")
+ or "a model that did not report its own name"
+ )
+ shown = _readings_phrase(getattr(manifest, "resolution_readings", []))
+ return (
+ f"{lead} {named} was asked to settle the labels the readings did not agree on, "
+ f"shown {shown}; what it settled is recorded with the numbering resolution."
+ )
+
+
+def _readings_phrase(readings) -> str:
+ """The extractions a model call was actually shown, named.
+
+ Never "both texts". The resolution call is shown the readings that have a
+ printed span of the page, which is TWO on a docling run and ONE on a
+ pymupdf one — and the reading it is ruling against can be the deposit or
+ the model's own, neither of which has a span at all. One phrase, used by
+ the note and by the report, so the two cannot describe one call two ways.
+ """
+ names = [n for n in (readings or []) if n]
+ if not names:
+ return "text this manifest does not name"
+ if len(names) == 1:
+ return f"the {names[0]} extraction"
+ return f"the {' and '.join(names)} extractions"
+
+
def _was_obtained(manifest, label: str) -> bool:
"""Did this manifest actually retrieve a judgeable source for this label?
@@ -1350,12 +1459,14 @@ def _claim_pairing(claim, manifest=None) -> Disclosure | None:
level="warn",
token=CLAIM_PAIRING_RESOLVED_TOKEN,
text=(
- f"This claim cites {labels}, {CLAIM_PAIRING_RESOLVED_TOKEN}. Two readings "
- "of the bibliography disagreed there. A model was shown both, every field "
- "of its answer was found verbatim in one of them, and you accepted the "
- "result — so this verdict rests on a reading nobody checked against the "
- "printed page, and you are who accepted it. The numbering is still "
- "recorded as unconfirmed."
+ f"This claim cites {labels}, {CLAIM_PAIRING_RESOLVED_TOKEN}. The readings "
+ "of the bibliography did not agree there. A model was shown "
+ f"{_readings_phrase(getattr(manifest, 'resolution_readings', []))} and "
+ "what each reading said at that label, every field of its answer was "
+ "found verbatim in that printed text, and you accepted the result — so "
+ "this verdict rests on a reading nobody checked against the printed "
+ "page, and you are who accepted it. The numbering is still recorded as "
+ "unconfirmed."
),
short=f"{labels} {CLAIM_PAIRING_RESOLVED_TOKEN}",
)
diff --git a/src/papertrace/models.py b/src/papertrace/models.py
index 10e9fa6..526400d 100644
--- a/src/papertrace/models.py
+++ b/src/papertrace/models.py
@@ -580,10 +580,13 @@ class RefManifest:
# `refs_total` and let `_slug_for_ref` hand it to a claim citing a number.
manuscript_supplements: list[Supplement] = field(default_factory=list)
# A second, independent axis from `numbering_verified` above: whether
- # EVERY label the body cites was agreed by >= 2 readings. Absent means
- # never computed — not "not corroborated", the same three-state
- # discipline as `numbering_ledger`.
- numbering_corroborated: bool = False
+ # EVERY label the body cites was agreed by >= 2 readings. `None` means
+ # never computed — not "not corroborated" — the same three-state
+ # discipline as `reflist_entries_proposed`, and the reason this is not a
+ # plain `bool`: the docstring claimed three states for a type that holds
+ # two, so "absent" and "measured false" were the same value and every
+ # consumer had to be silent on both to stay honest.
+ numbering_corroborated: bool | None = None
corroborating_readings: list[str] = field(default_factory=list)
# Absent means never computed, NOT "none disputed" — a manifest written
# before this feature says nothing about disputes, it does not assert
@@ -637,6 +640,30 @@ class RefManifest:
# manifest's `reflist_outcome` as `"read"` from `reflist_model` alone
# would otherwise make EVERY such manifest read as "proposed nothing".
reflist_entries_proposed: int | None = None
+ # The SECOND model call this feature can make: the one that settles
+ # labels the readings did not agree on, after a person asked for it.
+ # Kept in its own five fields rather than folded into the `reflist_*` set
+ # above, which describes the call that reads the whole list: a run can
+ # make either, both or neither, and on a `--backend pymupdf` run the
+ # first is never attempted while the second still happens. Folding them
+ # made the report say no model read the reference list on a run where one
+ # was asked, verified and allowed to change which papers are judged.
+ #
+ # Same vocabulary and the same three-state discipline as their
+ # counterparts: `""` means NEVER COMPUTED, not `"not_attempted"`.
+ resolution_outcome: str = "" # "" | one of reflist.REFLIST_OUTCOMES
+ # why the call produced nothing, when `resolution_outcome` is "failed".
+ # "" when it succeeded, or when no such call was ever made.
+ resolution_failure: str = ""
+ # "" means EITHER no such call OR one that answered without naming its
+ # model — `resolution_outcome` is what tells those apart.
+ resolution_model: str = ""
+ resolution_fields_discarded: list[str] = field(default_factory=list)
+ # Which printed extractions the call was actually SHOWN — the readings
+ # whose text went into the prompt, not the readings that disagreed. The
+ # report names these rather than asserting "both texts", which was false
+ # on every run with one printed extraction. `[]` means never recorded.
+ resolution_readings: list[str] = field(default_factory=list)
def document(self, slug: str) -> Document | None:
"""The judgeable file this slug names, article or supplement, or None."""
@@ -726,6 +753,11 @@ def to_json(self, path: Path) -> None:
"reflist_fields_discarded": self.reflist_fields_discarded,
"reflist_numbering_findings": self.reflist_numbering_findings,
"reflist_entries_proposed": self.reflist_entries_proposed,
+ "resolution_outcome": self.resolution_outcome,
+ "resolution_failure": self.resolution_failure,
+ "resolution_model": self.resolution_model,
+ "resolution_fields_discarded": self.resolution_fields_discarded,
+ "resolution_readings": self.resolution_readings,
"summary": {
"total": len(self.entries),
"available": len(self.retrieved),
@@ -758,7 +790,12 @@ def from_json(cls, path: Path) -> RefManifest:
unverified_from=data.get("unverified_from"),
numbering_contested=bool(data.get("numbering_contested", False)),
numbering_ledger=data.get("numbering_ledger", {}),
- numbering_corroborated=bool(data.get("numbering_corroborated", False)),
+ # no `bool(...)` and no default: absent and explicit `null` both
+ # read as `None` — never computed — and are never coerced to False
+ numbering_corroborated=(
+ None if data.get("numbering_corroborated") is None
+ else bool(data["numbering_corroborated"])
+ ),
corroborating_readings=data.get("corroborating_readings", []),
# absent means never computed, not "none disputed" or "none resolved"
labels_disputed=data.get("labels_disputed", []),
@@ -766,13 +803,34 @@ def from_json(cls, path: Path) -> RefManifest:
numbering_choice=data.get("numbering_choice", ""),
numbering_chosen_by=data.get("numbering_chosen_by", ""),
reflist_outcome=data.get("reflist_outcome", ""),
- reflist_failure=data.get("reflist_failure", ""),
reflist_model=data.get("reflist_model", ""),
reflist_fields_discarded=data.get("reflist_fields_discarded", []),
reflist_numbering_findings=data.get("reflist_numbering_findings", []),
+ # A case folder written mid-branch, before `reflist_outcome`
+ # replaced the boolean it carries: the key is dropped here (no
+ # field owns it), and `_reflist` then had nothing to fire on, so
+ # the required disclosure vanished for a reading that happened.
+ # Neither "read" nor "failed" is provable from a flag that was
+ # set on both paths, so neither is claimed — the reason goes in
+ # `reflist_failure`, where `_reflist`'s weakest inference picks
+ # it up and says the outcome was never recorded.
+ reflist_failure=data.get("reflist_failure", "") or (
+ "a model reading was attempted; this manifest predates "
+ "`reflist_outcome` and does not record whether the call returned"
+ if data.get("reflist_attempted") and not data.get("reflist_outcome")
+ else ""
+ ),
# `.get` with no default: absent AND explicit `null` both read as
# `None` — "never recorded" — never coerced to `0`
reflist_entries_proposed=data.get("reflist_entries_proposed"),
+ # absent means never computed for all five: a manifest written
+ # before the escalation existed asserts nothing about whether a
+ # resolution call was made
+ resolution_outcome=data.get("resolution_outcome", ""),
+ resolution_failure=data.get("resolution_failure", ""),
+ resolution_model=data.get("resolution_model", ""),
+ resolution_fields_discarded=data.get("resolution_fields_discarded", []),
+ resolution_readings=data.get("resolution_readings", []),
)
diff --git a/src/papertrace/reflist.py b/src/papertrace/reflist.py
index 2cf70f7..86bf5bf 100644
--- a/src/papertrace/reflist.py
+++ b/src/papertrace/reflist.py
@@ -177,6 +177,31 @@ class ReflistProvenance:
# string nobody enumerated — the same discipline as `LABEL_AGREEMENT`
REFLIST_OUTCOMES = ("not_attempted", "failed", "read")
+# Total characters of bibliography either call may be handed. A REFUSAL, not
+# a `_clip`: every other model input here is clipped and the cut disclosed
+# (`check.Truncations`), but a clipped bibliography produces a reading that
+# is simply missing the entries past the cut, and nothing in the provenance
+# could distinguish that from a page that never printed them — a reading
+# quietly short is the one failure this module's verification cannot catch.
+# A refusal costs nothing, happens before the subprocess, and says so. Two
+# ~250-reference extractions fit inside it with room to spare; anything past
+# it is a text no context window was going to hold either.
+REFLIST_CHAR_LIMIT = 400_000
+
+
+def _too_long(texts: tuple[str, ...]) -> str:
+ """Why this call was not made, or "" — measured before any money is spent."""
+ total = sum(len(t) for t in texts)
+ if total <= REFLIST_CHAR_LIMIT:
+ return ""
+ return (
+ f"the extractions of the bibliography are too long to send in one call "
+ f"({total:,} characters against a {REFLIST_CHAR_LIMIT:,} limit), so no model "
+ "was asked — they are refused whole rather than cut, because a reading "
+ "missing whatever fell past the cut cannot be told from a page that never "
+ "printed it"
+ )
+
# order matters only for `raw` below, which is re-assembled in printed order
_VERIFIED_FIELDS = ("authors", "year", "title", "journal", "doi")
@@ -223,6 +248,11 @@ def propose(
"model reading was not attempted — there was no second reading to check it against"
)
return [], prov
+ if why := _too_long(readings):
+ # same slot and same `outcome` as the guard above, for the same
+ # reason: no subprocess call happens on this path either
+ prov.failure = why
+ return [], prov
# labels before texts: a label is a short fixed string ("pymupdf",
# "docling 2.8.0") and cannot contain a placeholder, while page text
@@ -408,12 +438,13 @@ def _numeral_gaps(nums: list[str]) -> list[str]:
# (`cli._write_disagreement`) and asked for a model's opinion on it
# ---------------------------------------------------------------------------
-RESOLVE_PROMPT = """You are shown two readings of one printed bibliography, and a list of
-citation labels the two readings disagree about — they name different papers at
-that label.
+RESOLVE_PROMPT = """You are shown <> of one printed bibliography, a list of
+citation labels whose readings did not agree — they either name different
+papers at that label, or one of them says nothing that can be compared — and
+what each reading says at those labels.
-For each label, decide which paper the label actually names, using ONLY the two
-texts below. Reply with a JSON array, one object per label you can answer:
+For each label, decide which paper the label actually names, using ONLY the
+printed text below. Reply with a JSON array, one object per label you can answer:
[{"num": "6",
"title": "the paper's title, copied character for character from one of the texts",
@@ -429,19 +460,29 @@ def _numeral_gaps(nums: list[str]) -> list[str]:
other, so a guess here is the error this tool exists to prevent. Answering some
labels and abstaining on others is expected.
-Every value you give is checked against the two texts verbatim. A value that is
-not printed in one of them is discarded, and a title that is not printed leaves
-the label unresolved — so do not correct, expand, complete or tidy anything.
+Every value you give is checked against the printed text verbatim. A value that
+is not printed there is discarded, and a title that is not printed leaves the
+label unresolved — so do not correct, expand, complete or tidy anything.
LABELS IN DISPUTE: <>
---- READING A ---
-<>
-
---- READING B ---
-<>
+WHAT EACH READING SAYS AT THOSE LABELS — this is the disagreement you are being
+asked to settle. It is CONTEXT ONLY: nothing here counts as printed text, and a
+value copied from this list rather than from the extraction(s) below is
+discarded.
+<>
+<>
"""
+# How the printed extractions are laid out in the prompt. One block per text,
+# lettered, and the letters are what a reply's `seen_in` names — so a run with
+# ONE extraction renders one block rather than an empty `--- READING B ---`,
+# which told the model a second extraction existed and was empty. `propose`
+# refuses outright below two texts; this call cannot, because the printed page
+# is the only authority it needs and one extraction of it is still that page.
+_READING_BLOCK = "--- READING {letter} ({name}) ---\n{text}\n"
+_READING_LETTER_BY_INDEX = ("A", "B")
+
@dataclass
class Resolution:
@@ -486,11 +527,35 @@ def _parse_array(raw: str) -> list[dict] | None:
_RESOLVE_FIELDS = ("authors", "year", "journal")
+def _render_disputing(
+ labels: list[str], disputing: dict[str, list[RefEntry]] | None
+) -> str:
+ """Each reading's entry for each disputed label, as prompt context.
+
+ Named readings, not letters: `crossref` and `llm` have no printed span of
+ the page at all, so they can never be one of the lettered extractions
+ below — and they are exactly the readings a dispute is most often
+ between. A reading carrying nothing for a label says so, because "this
+ reading has no entry here" is itself half of many disagreements.
+ """
+ if not disputing:
+ return " (the readings' own entries were not supplied to this call)\n"
+ out: list[str] = []
+ for label in labels:
+ out.append(f" [{label}]")
+ for name in sorted(disputing):
+ hits = [e.raw for e in disputing[name] if e.num == label]
+ shown = " || ".join(hits) if hits else "— no entry for this label —"
+ out.append(f" {name}: {shown}")
+ return "\n".join(out) + "\n"
+
+
def resolve_disputed(
labels: list[str],
names: tuple[str, ...],
texts: tuple[str, ...],
*,
+ disputing: dict[str, list[RefEntry]] | None = None,
model: str | None = None,
) -> Resolution:
"""Ask the model which paper each disputed label names, and believe none of it on its word.
@@ -512,6 +577,16 @@ def resolve_disputed(
3. Partial resolution is the normal outcome, not a degraded one. Nothing
downstream may require all-or-nothing.
+ `disputing` is what each READING says at each disputed label, including
+ the readings that have no printed span of their own (`crossref`, `llm`).
+ Without it this call adjudicated a disagreement it had never been shown:
+ a parse-versus-deposit dispute — the case the whole feature exists for —
+ reached a model that had seen only the parse, and it answered with the
+ parse's own entry, which then verified verbatim and was substituted as a
+ settled label. It is context and nothing more: `_found` still checks every
+ returned value against `texts` alone, so a title only the deposit carries
+ cannot be believed.
+
`names[i]` MUST be the reading `texts[i]` actually came from — passed in
explicitly, the way `propose`'s `label_a`/`label_b` are, rather than derived
from a separate dict of candidates. A caller's candidates dict can hold more
@@ -526,11 +601,26 @@ def resolve_disputed(
prov = ReflistProvenance(readings=list(names))
if not labels:
return Resolution(provenance=prov)
+ if why := _too_long(texts):
+ # every label stays exactly where it was, and the reason says the
+ # call never happened — `cli._disputed_reason` must not print "the
+ # model could not tell" over a model that was never asked
+ prov.failure = why
+ return Resolution(still_disputed=list(labels), provenance=prov)
+ blocks = "\n".join(
+ _READING_BLOCK.format(letter=letter, name=name, text=text)
+ for letter, name, text in zip(_READING_LETTER_BY_INDEX, names, texts, strict=False)
+ )
+ how_many = (
+ "one extraction" if len(texts) < 2
+ else f"{len(texts)} extractions"
+ )
prompt = (
RESOLVE_PROMPT.replace("<>", ", ".join(f"[{n}]" for n in labels))
- .replace("<>", texts[0] if texts else "")
- .replace("<>", texts[1] if len(texts) > 1 else "")
+ .replace("<>", how_many)
+ .replace("<>", _render_disputing(labels, disputing))
+ .replace("<>", blocks)
)
with ask.for_site(ask.SITE_REFS):
raw = ask._ask(prompt, model)
@@ -550,8 +640,12 @@ def resolve_disputed(
)
return Resolution(still_disputed=list(labels), provenance=prov)
+ # counted from `items`, before the de-duplication below — the same point
+ # `propose` counts at. A reply carrying two objects for "6" proposed two
+ # entries, and recording one would under-report exactly the reply a
+ # reader most needs to know was malformed.
+ prov.entries_proposed = len(items)
by_num = {str(o.get("num", "")).strip(): o for o in items}
- prov.entries_proposed = len(by_num)
resolved: dict[str, RefEntry] = {}
unresolved: list[str] = []
for label in labels:
@@ -614,7 +708,7 @@ def resolve_disputed(
e.title = kept.get("title")
e.year = kept.get("year")
e.reason = (
- "a disputed label, resolved by a model reading of both texts and "
+ "a disputed label, resolved by a model reading of the printed list and "
"accepted by a person — a reading, not a confirmed numbering"
)
e.seen_in = seen_in
diff --git a/src/papertrace/refs.py b/src/papertrace/refs.py
index 1873cb5..006f22f 100644
--- a/src/papertrace/refs.py
+++ b/src/papertrace/refs.py
@@ -625,6 +625,18 @@ class Reconciliation:
# honest: a person consenting to proceed is an input, never evidence that
# the numbering was verified.
chosen_by: str = "" # "" | default | user
+ # What the RESOLUTION call — `reflist.resolve_disputed`, made only when a
+ # person asked for it — cost and produced. Carried here so
+ # `_refs_pipeline` can persist it beside the reflist call's own
+ # provenance: a model call that was made, paid for, verified field by
+ # field and allowed to change which papers are judged existed nowhere
+ # structured, while the `reflist_*` fields said no model read the list.
+ # "" / [] mean never computed, never "it produced nothing".
+ resolution_outcome: str = ""
+ resolution_failure: str = ""
+ resolution_model: str = ""
+ resolution_fields_discarded: list[str] = field(default_factory=list)
+ resolution_readings: list[str] = field(default_factory=list)
def _covers(body: set[str], entries: list[RefEntry]) -> bool:
diff --git a/tests/test_disclosure_parity.py b/tests/test_disclosure_parity.py
index 6b838b4..085b755 100644
--- a/tests/test_disclosure_parity.py
+++ b/tests/test_disclosure_parity.py
@@ -470,6 +470,52 @@ def test_a_corroborated_claim_pairing_disclosure_appears_in_every_format(tmp_pat
assert d.token in body, f"claim_pairing (corroborated) token missing from {name}"
+def test_a_disputed_but_unretrieved_claim_pairing_appears_in_every_format(tmp_path):
+ """The fifth `claim_pairing` state (whole-branch review, Major 5): the
+ readings did not agree AND nothing was ever fetched under the label, so
+ the withheld token's "the source retrieved under that label" is the one
+ thing that did not happen. New token, same key — the allow-lists already
+ carry it, and this is what makes the new state's parity provable."""
+ from papertrace.models import RefEntry, RefManifest
+
+ manifest = RefManifest(
+ manuscript="m.pdf",
+ entries=[RefEntry(num="6", raw="ref 6", status="paywalled")],
+ labels_disputed=["6"],
+ )
+ claim = _claim(refs=["6"], verdict="not_retrieved")
+ results = RunResults(manuscript="m.pdf", converter="pymupdf", claims=[claim])
+ write_reports(results, manifest, tmp_path, png=False)
+ rendered = {name: (tmp_path / name).read_text() for name in FORMATS}
+
+ d = next(x for x in claim_disclosures(claim, manifest) if x.key == "claim_pairing")
+ for name, body in rendered.items():
+ assert d.token in body, f"claim_pairing (unretrieved) token missing from {name}"
+
+
+def test_a_resolution_only_reflist_disclosure_appears_in_every_format(tmp_path):
+ """Major 4's new shape: no reading of the whole list was taken, but a
+ model was asked to settle a disagreement. The `reflist` key already has a
+ block in every format; this is the state that had no fixture."""
+ from papertrace.models import RefManifest
+
+ manifest = RefManifest(
+ manuscript="m.pdf",
+ labels_resolved=["7"],
+ numbering_choice="llm_resolved",
+ resolution_outcome="read",
+ resolution_model="claude-opus-5",
+ resolution_readings=["parsed"],
+ )
+ results = RunResults(manuscript="m.pdf", converter="pymupdf", claims=[_claim(refs=["7"])])
+ write_reports(results, manifest, tmp_path, png=False)
+
+ d = next(x for x in run_disclosures(results, manifest) if x.key == "reflist")
+ for name in FORMATS:
+ body = (tmp_path / name).read_text()
+ assert d.token in body, f"reflist (resolution only) token missing from {name}"
+
+
# --- the HTML reports actually escape what they interpolate ------------------
diff --git a/tests/test_escalation.py b/tests/test_escalation.py
index 8a4a826..cf80b7a 100644
--- a/tests/test_escalation.py
+++ b/tests/test_escalation.py
@@ -171,7 +171,7 @@ def test_the_default_answer_asks_the_model_and_records_the_user_as_choosing(
resolved_entry = _entry("2", "Beta B. Second paper. 2021. doi:10.1000/beta")
monkeypatch.setattr(
reflist_mod, "resolve_disputed",
- lambda labels, cands, texts, model=None: reflist_mod.Resolution(
+ lambda labels, cands, texts, disputing=None, model=None: reflist_mod.Resolution(
resolved={"2": resolved_entry}, still_disputed=[],
provenance=reflist_mod.ReflistProvenance(model="claude-opus-5", outcome="read"),
),
@@ -380,9 +380,9 @@ def test_a_resolved_label_judges_against_the_resolved_entry(tmp_path, monkeypatc
"https://doi.org/10.1148/x13")
monkeypatch.setattr(
reflist_mod, "resolve_disputed",
- lambda labels, cands, texts, model=None: reflist_mod.Resolution(
+ lambda labels, cands, texts, disputing=None, model=None: reflist_mod.Resolution(
resolved={"13": resolved}, still_disputed=[],
- provenance=reflist_mod.ReflistProvenance(model="claude-opus-5"),
+ provenance=reflist_mod.ReflistProvenance(model="claude-opus-5", outcome="read"),
),
)
@@ -500,8 +500,12 @@ def ask(prompt_text, **kw):
monkeypatch.setattr(cli.Prompt, "ask", staticmethod(ask))
monkeypatch.setattr(
reflist_mod, "resolve_disputed",
- lambda labels, names, texts, model=None: reflist_mod.Resolution(
- still_disputed=list(labels), provenance=reflist_mod.ReflistProvenance(),
+ lambda labels, names, texts, disputing=None, model=None: reflist_mod.Resolution(
+ still_disputed=list(labels),
+ # a double must satisfy the SHAPE its consumer reads: a Resolution
+ # that came back from a real call records `outcome="read"`, and
+ # `_escalate_disputed` now branches on it
+ provenance=reflist_mod.ReflistProvenance(outcome="read"),
),
)
dup_a = _entry("2", "Alpha A. First. 2020.")
@@ -533,9 +537,9 @@ def test_a_resolved_label_absent_from_entries_stays_disputed(tmp_path, monkeypat
real = _entry("2", "Beta B. The real one resolved. 2021.")
monkeypatch.setattr(
reflist_mod, "resolve_disputed",
- lambda labels, names, texts, model=None: reflist_mod.Resolution(
+ lambda labels, names, texts, disputing=None, model=None: reflist_mod.Resolution(
resolved={"2": real, "99": orphan}, still_disputed=[],
- provenance=reflist_mod.ReflistProvenance(model="claude-opus-5"),
+ provenance=reflist_mod.ReflistProvenance(model="claude-opus-5", outcome="read"),
),
)
chosen = _entry("2", "Beta B. Original. 2021.")
@@ -556,11 +560,11 @@ def test_the_console_names_the_real_reason_a_label_stayed_disputed(tmp_path, mon
three is a plausible-looking cause standing in for the honest one."""
monkeypatch.setattr(cli, "_interactive", lambda: True)
monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
- prov = reflist_mod.ReflistProvenance(model="claude-opus-5")
+ prov = reflist_mod.ReflistProvenance(model="claude-opus-5", outcome="read")
prov.fields_discarded.append("[9].title")
monkeypatch.setattr(
reflist_mod, "resolve_disputed",
- lambda labels, names, texts, model=None: reflist_mod.Resolution(
+ lambda labels, names, texts, disputing=None, model=None: reflist_mod.Resolution(
still_disputed=["9"], provenance=prov,
),
)
@@ -612,9 +616,9 @@ def test_a_resolved_entrys_slug_collision_is_disambiguated(tmp_path, monkeypatch
resolved_same_slug = _entry("2", "Smith J. The resolved paper. 2020.")
monkeypatch.setattr(
reflist_mod, "resolve_disputed",
- lambda labels, names, texts, model=None: reflist_mod.Resolution(
+ lambda labels, names, texts, disputing=None, model=None: reflist_mod.Resolution(
resolved={"2": resolved_same_slug}, still_disputed=[],
- provenance=reflist_mod.ReflistProvenance(model="claude-opus-5"),
+ provenance=reflist_mod.ReflistProvenance(model="claude-opus-5", outcome="read"),
),
)
rec = _rec(labels_disputed=["2"])
@@ -624,3 +628,233 @@ def test_a_resolved_entrys_slug_collision_is_disambiguated(tmp_path, monkeypatch
)
slugs = [e.slug for e in out]
assert len(slugs) == len(set(slugs)), f"colliding slugs would share a download path: {slugs}"
+
+
+# --- whole-branch review, Critical 2: the call sees what the report says it saw
+
+
+REFS_TEXT = (
+ "References\n"
+ "[1] Alpha A. First paper. 2020.\n"
+ "[2] Beta B. Second paper. 2021.\n"
+)
+
+
+def _capture_ask(monkeypatch, reply: str) -> dict:
+ """Intercept the one seam `resolve_disputed` calls, and keep the prompt."""
+ import papertrace.ask as ask_mod
+
+ seen: dict = {}
+
+ def fake_ask(prompt, model=None):
+ seen["prompt"] = prompt
+ return reply
+
+ monkeypatch.setattr(ask_mod, "_ask", fake_ask)
+ monkeypatch.setattr(ask_mod, "model_for", lambda site: "claude-opus-5")
+ return seen
+
+
+def test_the_resolution_call_is_shown_the_reading_it_is_ruling_against(tmp_path, monkeypatch):
+ """Critical 2. `reading_texts` holds only the readings with a printed
+ span, so a dispute between the parse and the DEPOSIT — the case this
+ whole feature was built for — was handed to a model that had never seen
+ the deposit's entry, while `rec.note` and the per-claim disclosure both
+ said it was shown both.
+
+ The disputing entries travel as CONTEXT, not as a text to copy from:
+ `_found` still verifies every returned value against the printed
+ extraction alone, so a title the deposit carries and the page does not
+ still cannot be believed."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ seen = _capture_ask(monkeypatch, json.dumps([{"num": "2", "title": "Second paper"}]))
+ parsed = [_entry("1", "Alpha A. First paper. 2020."),
+ _entry("2", "Beta B. Second paper. 2021.")]
+ deposit = [_entry("2", "Gamma G. A wholly different paper. 2019.")]
+ rec = _rec(labels_disputed=["2"])
+
+ cli._escalate_disputed(
+ tmp_path, rec, parsed,
+ {"parsed": parsed, "crossref": deposit},
+ {"parsed": REFS_TEXT},
+ )
+
+ assert "Gamma G. A wholly different paper" in seen["prompt"], seen["prompt"]
+ assert "crossref" in seen["prompt"]
+
+
+def test_one_printed_extraction_is_never_rendered_as_an_empty_second_reading(
+ tmp_path, monkeypatch
+):
+ """`--backend pymupdf` leaves exactly one printed span, and the prompt
+ rendered `--- READING B ---` followed by nothing — telling the model an
+ extraction existed and was empty. `propose` refuses outright on fewer
+ than two texts; this call cannot, because the printed page is still the
+ only authority it needs, so it says how many it has instead."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ seen = _capture_ask(monkeypatch, json.dumps([{"num": "2", "title": "Second paper"}]))
+ parsed = [_entry("2", "Beta B. Second paper. 2021.")]
+ rec = _rec(labels_disputed=["2"])
+
+ cli._escalate_disputed(
+ tmp_path, rec, parsed, {"parsed": parsed}, {"parsed": REFS_TEXT},
+ )
+
+ prompt = seen["prompt"]
+ assert "READING B" not in prompt, prompt
+ assert "one extraction" in prompt
+
+
+def test_the_note_names_the_readings_the_model_was_actually_shown(tmp_path, monkeypatch):
+ """The other half of Critical 2: the report said "a model reading of both
+ texts" on a run where the model saw one. It names them now."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ _capture_ask(monkeypatch, json.dumps([{"num": "2", "title": "Second paper"}]))
+ parsed = [_entry("2", "Beta B. Second paper. 2021.")]
+ rec = _rec(labels_disputed=["2"])
+
+ cli._escalate_disputed(
+ tmp_path, rec, parsed, {"parsed": parsed}, {"parsed": REFS_TEXT},
+ )
+
+ assert "both texts" not in rec.note
+ assert "the parsed extraction" in rec.note
+
+
+# --- whole-branch review, Major 4: the resolution call's own provenance
+
+
+def test_the_resolution_calls_provenance_reaches_the_manifest(disputed, monkeypatch):
+ """Major 4. A model call was made, paid for, verified field by field and
+ allowed to change which papers are judged — and its model name, outcome
+ and discarded fields existed nowhere structured, only inside a prose
+ note. Meanwhile the `reflist_*` fields, which describe a DIFFERENT call,
+ recorded that no model read the reference list at all."""
+ import jsonschema
+
+ pdf, case, answers = disputed
+ answers.append("1")
+ _capture_ask(monkeypatch, json.dumps([{"num": "2", "title": "Second paper"}]))
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.resolution_outcome == "read"
+ assert manifest.resolution_model == "claude-opus-5"
+ assert manifest.resolution_readings == ["parsed", "pymupdf"]
+ assert manifest.resolution_failure == ""
+ # and it is a published field, not a local convenience
+ schema = json.loads(
+ (Path(__file__).resolve().parent.parent / "schemas" / "refs_manifest.schema.json")
+ .read_text()
+ )
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ jsonschema.Draft202012Validator(schema).validate(payload)
+ assert payload["resolution_outcome"] == "read"
+
+
+def test_a_resolution_call_that_did_not_return_is_recorded_as_one_that_happened(
+ disputed, monkeypatch
+):
+ """The third state, which a boolean could not hold: the call was made and
+ raised. `resolution_outcome` says `failed` and carries why, so the run is
+ never described as one where no model was asked."""
+ pdf, case, answers = disputed
+ answers.append("1")
+
+ def boom(*a, **k):
+ raise RuntimeError("claude -p timed out after 600s")
+
+ monkeypatch.setattr(reflist_mod, "resolve_disputed", boom)
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.resolution_outcome == "failed"
+ assert "timed out" in manifest.resolution_failure
+ assert manifest.resolution_model == ""
+
+
+def test_a_manifest_with_no_resolution_call_says_nothing_about_one(disputed):
+ """"" everywhere, and `[]` for the readings — never computed, which is
+ what a run that never escalated actually knows."""
+ pdf, case, answers = disputed
+ answers.append("2")
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.resolution_outcome == ""
+ assert manifest.resolution_model == ""
+ assert manifest.resolution_readings == []
+ assert manifest.resolution_fields_discarded == []
+
+
+# --- whole-branch review, minors 2 and 3
+
+
+def test_seen_in_survives_a_whole_reading_substitution(disputed):
+ """Minor 2. `stamp_seen_in` ran before the escalation, and option 3
+ replaces every entry with one from a reading that was never stamped — so
+ the manifest published `seen_in: []`, which the schema reads as "no
+ reading was established as carrying this", for a run that established
+ it."""
+ pdf, case, answers = disputed
+ answers += ["3", "pymupdf"]
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.numbering_choice == "pymupdf"
+ assert all(e.seen_in for e in manifest.entries), [e.seen_in for e in manifest.entries]
+
+
+def test_adopting_a_reading_whole_is_described_as_adopting_it_whole(disputed):
+ """Minor 3. Option 3 returns `list(candidates[pick])` — every entry, not
+ only the disputed ones — while the note said "you chose X for the 1 label
+ in dispute"."""
+ pdf, case, answers = disputed
+ answers += ["3", "pymupdf"]
+
+ manifest = _run_refs(pdf, case)
+
+ assert "whole" in manifest.numbering_note
+ assert manifest.numbering_note.startswith("you chose")
+
+
+def test_a_resolution_call_that_was_never_made_is_not_a_model_that_could_not_tell(
+ tmp_path, monkeypatch, capsys
+):
+ """`resolve_disputed` can decline before spending anything — the two
+ extractions are too long to send. Every label stays disputed, and the
+ console must not print `_disputed_reason`'s "the model could not tell
+ which paper it names" over a call nobody made: that is a plausible-looking
+ cause standing in for the honest one, which is the failure this codebase
+ exists to refuse."""
+ monkeypatch.setattr(cli, "_interactive", lambda: True)
+ monkeypatch.setattr(cli.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
+ monkeypatch.setattr(
+ reflist_mod, "resolve_disputed",
+ lambda labels, names, texts, disputing=None, model=None: reflist_mod.Resolution(
+ still_disputed=list(labels),
+ provenance=reflist_mod.ReflistProvenance(
+ readings=list(names),
+ failure="the extractions of the bibliography are too long to send",
+ ),
+ ),
+ )
+ two = _entry("2", "Beta B. Second paper. 2021.")
+ rec = _rec(labels_disputed=["2"], note="a note")
+ capsys.readouterr()
+
+ cli._escalate_disputed(tmp_path, rec, [two], {"parsed": [two]}, {"parsed": REFS_TEXT})
+
+ out = " ".join(capsys.readouterr().out.split())
+ assert "could not tell" not in out
+ assert "too long to send" in out
+ assert rec.labels_disputed == ["2"]
+ assert rec.labels_resolved == []
+ assert rec.choice == "withheld"
+ assert rec.resolution_outcome == "not_attempted"
+ assert "too long to send" in rec.resolution_failure
+ # and the note must not claim a reading that never happened
+ assert "verified verbatim" not in rec.note
diff --git a/tests/test_label_agreement.py b/tests/test_label_agreement.py
index fec1159..bea77fa 100644
--- a/tests/test_label_agreement.py
+++ b/tests/test_label_agreement.py
@@ -294,7 +294,10 @@ def test_the_six_new_reconciliation_fields_round_trip_through_a_manifest(tmp_pat
old = {"manuscript": "m.pdf", "entries": [{"num": "1", "raw": "x", "status": "paywalled"}]}
p.write_text(json.dumps(old))
older = RefManifest.from_json(p)
- assert older.numbering_corroborated is False
+ # `None`, not `False`: the field's own description has always said absent
+ # means never computed, and as a plain boolean it could not say so — the
+ # same three-state discipline `reflist_entries_proposed` is written under
+ assert older.numbering_corroborated is None
assert older.corroborating_readings == []
assert older.labels_disputed == []
assert older.labels_resolved == []
diff --git a/tests/test_reference_readings.py b/tests/test_reference_readings.py
index 94a9306..ce9a2d9 100644
--- a/tests/test_reference_readings.py
+++ b/tests/test_reference_readings.py
@@ -1241,7 +1241,9 @@ def test_the_new_manifest_fields_round_trip_and_validate(tmp_path):
del payload[key]
path.write_text(json.dumps(payload))
old = RefManifest.from_json(path)
- assert old.numbering_corroborated is False
+ # `None` — never computed — which is what "absent means never computed"
+ # requires a field to be able to say, and a plain boolean could not
+ assert old.numbering_corroborated is None
assert old.labels_disputed == []
@@ -1393,3 +1395,207 @@ def test_a_numbering_finding_from_the_model_reading_reaches_the_manifest(tmp_pat
assert not any(
"twice" in f for f in manifest.reflist_fields_discarded
), manifest.reflist_fields_discarded
+
+
+# --- whole-branch review, Major 4: the SECOND model call has provenance too
+
+
+def _resolved_manifest(**kw) -> RefManifest:
+ """A manifest from a run that escalated: a disputed label was settled by
+ the resolution call, which is a different call from the reflist reading."""
+ base = dict(
+ manuscript="p.pdf",
+ labels_resolved=["2"],
+ numbering_choice="llm_resolved",
+ numbering_chosen_by="user",
+ resolution_outcome="read",
+ resolution_model="claude-opus-5",
+ resolution_readings=["parsed"],
+ )
+ base.update(kw)
+ return RefManifest(**base)
+
+
+def test_a_run_whose_only_model_call_was_the_resolution_does_not_deny_it():
+ """Major 4. On `--backend pymupdf` the reflist reading is deliberately
+ skipped, so the `reflist` disclosure said "This run asked that the
+ reference list was also read by a model, and no reading was obtained" —
+ directly beside a numbering resolution saying a model settled two labels
+ and the user accepted it. A model call was made, paid for, verified and
+ allowed to change which papers are judged."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = _resolved_manifest(
+ reflist_outcome="not_attempted",
+ reflist_failure="this run's backend is pymupdf, so a second flat reading of the "
+ "bibliography would be the same text",
+ )
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+
+ assert len(fired) == 1, fired
+ assert "claude-opus-5" in fired[0].text
+ assert "settle the labels" in fired[0].text
+
+
+def test_a_silent_disable_still_discloses_a_resolution_call_that_happened():
+ """`--no-llm-refs` records `not_attempted` with no reason and the
+ disclosure stays silent — correctly, for the reading. But the escalation
+ is a separate, user-consented call that runs anyway, and silence about it
+ is the same false negative one rung down."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = _resolved_manifest(reflist_outcome="not_attempted", reflist_failure="")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+
+ assert len(fired) == 1, fired
+ assert "claude-opus-5" in fired[0].text
+ assert "the parsed extraction" in fired[0].text
+
+
+def test_a_resolution_call_that_failed_is_disclosed_though_it_settled_nothing():
+ """`_numbering_resolution` needs a resolved label to fire, so a
+ resolution call that raised would otherwise reach no structured surface
+ at all — the manifest would record a run in which no model was asked."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(
+ manuscript="p.pdf",
+ labels_disputed=["2"],
+ numbering_choice="withheld",
+ resolution_outcome="failed",
+ resolution_failure="RuntimeError: claude -p timed out after 600s",
+ resolution_readings=["parsed", "pymupdf"],
+ )
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+
+ assert len(fired) == 1, fired
+ assert "did not return" in fired[0].text
+ assert "timed out" in fired[0].text
+
+
+def test_a_manifest_with_no_resolution_call_gains_no_sentence_about_one():
+ """`""` is never computed, and an absent call says nothing."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_model="claude-opus-5")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+
+ assert len(fired) == 1, fired
+ assert "settle the labels" not in fired[0].text
+
+
+def test_the_numbering_resolution_names_the_model_and_what_it_was_shown():
+ """Critical 2's report half: "a model reading of both texts" was printed
+ on runs where the model saw one text, and on runs where the reading it
+ ruled against had no text at all."""
+ from papertrace.disclosures import _numbering_resolution
+
+ d = _numbering_resolution(_resolved_manifest())
+
+ assert d is not None
+ assert "both texts" not in d.text
+ assert "claude-opus-5" in d.text
+ assert "the parsed extraction" in d.text
+
+
+def test_the_resolved_claim_disclosure_does_not_claim_the_model_saw_both():
+ """The per-claim counterpart of the sentence above."""
+ from papertrace.disclosures import _claim_pairing
+ from papertrace.models import ClaimResult
+
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["2"])
+ d = _claim_pairing(claim, _resolved_manifest())
+
+ assert d is not None
+ assert "shown both" not in d.text
+ assert "the parsed extraction" in d.text
+
+
+def test_a_resolution_whose_manifest_recorded_no_readings_says_so():
+ """`[]` is never recorded, not "it was shown nothing" — and the sentence
+ must not silently read as the latter."""
+ from papertrace.disclosures import _numbering_resolution
+
+ d = _numbering_resolution(_resolved_manifest(resolution_readings=[]))
+
+ assert d is not None
+ assert "does not name" in d.text
+
+
+# --- whole-branch review, minors 1 and 6
+
+
+def test_numbering_corroborated_keeps_never_computed_apart_from_measured_false(tmp_path):
+ """Minor 1. The dataclass and the schema both documented three states in
+ a `bool` that can hold two, so "absent means never computed" was
+ unexpressible. `None` is what the branch's own standard
+ (`reflist_entries_proposed`) already uses for exactly this."""
+ import jsonschema
+
+ old = {"manuscript": "m.pdf", "entries": []}
+ p = tmp_path / "old.json"
+ p.write_text(json.dumps(old))
+ assert RefManifest.from_json(p).numbering_corroborated is None
+
+ measured = tmp_path / "new.json"
+ RefManifest(manuscript="m.pdf", numbering_corroborated=False).to_json(measured)
+ assert RefManifest.from_json(measured).numbering_corroborated is False
+
+ schema = json.loads(SCHEMA_PATH.read_text())
+ for path in (measured,):
+ jsonschema.Draft202012Validator(schema).validate(json.loads(path.read_text()))
+ RefManifest(manuscript="m.pdf", numbering_corroborated=None).to_json(measured)
+ jsonschema.Draft202012Validator(schema).validate(json.loads(measured.read_text()))
+
+
+def test_a_mid_branch_manifest_recording_only_that_a_reading_was_attempted(tmp_path):
+ """Minor 6. A case folder written between `b74ba01` and `67ab473` carries
+ `reflist_attempted: true` and none of the fields that replaced it. The
+ key is dropped at load, so `_reflist` fell through every normalisation
+ branch and returned `None` — the required disclosure vanished for a
+ reading that actually happened, the exact false negative `67ab473` was
+ written to fix.
+
+ Neither `read` nor `failed` is provable from that flag, so neither is
+ claimed: the outcome is recorded as never computed and the report says
+ what it knows."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ p = tmp_path / "old.json"
+ p.write_text(json.dumps({"manuscript": "m.pdf", "entries": [], "reflist_attempted": True}))
+ manifest = RefManifest.from_json(p)
+
+ fired = [d for d in run_disclosures(RunResults(manuscript="m.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "does not record what came of it" in fired[0].text
+
+
+def test_a_mid_branch_manifest_with_a_model_name_still_settles_as_read(tmp_path):
+ """The stronger inference still wins: a recorded model name is positive
+ proof a reply was obtained, so the legacy flag adds nothing and must not
+ downgrade it."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ p = tmp_path / "old.json"
+ p.write_text(json.dumps({
+ "manuscript": "m.pdf", "entries": [],
+ "reflist_attempted": True, "reflist_model": "claude-opus-5",
+ }))
+ manifest = RefManifest.from_json(p)
+
+ fired = [d for d in run_disclosures(RunResults(manuscript="m.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "claude-opus-5" in fired[0].text
+ assert "does not record what came of it" not in fired[0].text
diff --git a/tests/test_reflist.py b/tests/test_reflist.py
index d9a9ef0..3df8c27 100644
--- a/tests/test_reflist.py
+++ b/tests/test_reflist.py
@@ -581,3 +581,97 @@ def test_no_labels_makes_no_model_call_at_all(monkeypatch):
assert calls == []
assert res.resolved == {}
assert res.still_disputed == []
+
+
+# --- whole-branch review, Critical 2 and minor 7 ---------------------------
+
+
+def test_the_resolution_prompt_shows_what_each_reading_says_at_the_label(monkeypatch):
+ """Critical 2. The disagreement being adjudicated has to be IN the prompt,
+ including the readings with no printed span of their own — the deposit and
+ the model's reading, which are exactly the ones a dispute is often with."""
+ from papertrace.models import RefEntry
+
+ calls = _resolve_reply(monkeypatch, json.dumps([{"num": "6", "cannot_tell": True}]))
+ reflist.resolve_disputed(
+ ["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B),
+ disputing={"crossref": [RefEntry(num="6", raw="Deposited: a third paper entirely")]},
+ )
+
+ prompt = calls[0]["prompt"]
+ assert "Deposited: a third paper entirely" in prompt
+ assert "crossref" in prompt
+
+
+def test_a_value_printed_only_in_the_disputing_entry_is_still_discarded(monkeypatch):
+ """The context block is context. Widening verification to it would let a
+ title the deposit carries and the page does not be believed — which is the
+ whole permission structure of this module inverted."""
+ from papertrace.models import RefEntry
+
+ _resolve_reply(monkeypatch, json.dumps([{"num": "6", "title": "A third paper entirely"}]))
+ res = reflist.resolve_disputed(
+ ["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B),
+ disputing={"crossref": [RefEntry(num="6", raw="Deposited: A third paper entirely")]},
+ )
+
+ assert res.resolved == {}
+ assert res.still_disputed == ["6"]
+ assert "[6].title" in res.provenance.fields_discarded
+
+
+def test_one_printed_extraction_renders_one_block_not_an_empty_second(monkeypatch):
+ calls = _resolve_reply(monkeypatch, json.dumps([{"num": "6", "cannot_tell": True}]))
+ reflist.resolve_disputed(["6"], ("pymupdf",), (_RESOLVE_A,))
+
+ prompt = calls[0]["prompt"]
+ assert "READING A (pymupdf)" in prompt
+ assert "READING B" not in prompt
+
+
+def test_a_reply_repeating_a_numeral_is_counted_once_per_object(monkeypatch):
+ """Minor 5. `entries_proposed` was measured after de-duplication, so a
+ reply with two objects for [6] — the malformed shape a reader most needs
+ to know about — was recorded as one proposed entry. `propose` counts
+ before de-duplication; so does this now."""
+ _resolve_reply(monkeypatch, json.dumps([
+ {"num": "6", "cannot_tell": True},
+ {"num": "6", "cannot_tell": True},
+ ]))
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (_RESOLVE_A, _RESOLVE_B))
+
+ assert res.provenance.entries_proposed == 2
+
+
+def test_an_oversized_bibliography_is_refused_before_the_call_not_after(monkeypatch):
+ """Minor 7. Both extractions were interpolated whole, with no limit and no
+ truncation record — the one model input in this codebase that could
+ plausibly exceed a context window. Refused rather than clipped: a clipped
+ reading proposes a list missing the entries past the cut, and nothing in
+ the provenance could say which. A refusal costs nothing and states itself.
+ """
+ calls = _reply(monkeypatch, "[]")
+ big = "6. A reference. 2020.\n" * (reflist.REFLIST_CHAR_LIMIT // 10)
+
+ entries, prov = reflist.propose(big, big, label_a="docling 2.8.0", label_b="pymupdf")
+
+ assert calls == []
+ assert entries == []
+ assert prov.outcome == "not_attempted"
+ assert "too long" in prov.failure
+
+
+def test_an_oversized_resolution_input_leaves_every_label_disputed(monkeypatch):
+ """The same guard on the other call, and the state it must leave behind:
+ every label exactly where it was, with a reason that is not "the model
+ could not tell" — nothing was asked."""
+ calls = _resolve_reply(monkeypatch, "[]")
+ big = "6. A reference. 2020.\n" * (reflist.REFLIST_CHAR_LIMIT // 10)
+
+ res = reflist.resolve_disputed(["6"], _RESOLVE_NAMES, (big, big))
+
+ assert calls == []
+ assert res.resolved == {}
+ assert res.still_disputed == ["6"]
+ assert res.provenance.outcome == "not_attempted"
+ assert "too long" in res.provenance.failure
From 8e8b7f54c9ea2a15359ed718cdead13a55c2a836 Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Fri, 18 Sep 2026 23:03:42 +0200
Subject: [PATCH 37/44] report: two calls in one paragraph must not contradict
each other
Found by enumerating the states rather than the branches: the not-obtained,
failed and unrecognised heads end "the reference numbering rests on the
readings above it and nothing else", and the resolution sentence appended
after them describes a call that substitutes an entry. The claim is dropped
where a resolution happened and kept where none did, since it is
load-bearing there.
`_also_resolved` is one producer for the sentence both dispute states owe, so
a claim citing an unretrieved disputed label and a resolved one no longer
loses the resolved caveat to the state added later.
---
src/papertrace/disclosures.py | 58 +++++++++++++++++++++-----------
tests/test_reference_readings.py | 38 +++++++++++++++++++++
tests/test_withholding.py | 19 +++++++++++
3 files changed, 96 insertions(+), 19 deletions(-)
diff --git a/src/papertrace/disclosures.py b/src/papertrace/disclosures.py
index a619236..f6c4ce4 100644
--- a/src/papertrace/disclosures.py
+++ b/src/papertrace/disclosures.py
@@ -670,6 +670,7 @@ def _reflist(manifest) -> Disclosure | None:
# the alternative (falling through to the success branch below) would
# assert the token as a reading that was taken and used, which is the
# one claim an unrecognised state can least afford to make.
+ unknown = _resolution_clause(manifest)
return Disclosure(
key="reflist",
level="warn",
@@ -677,8 +678,13 @@ def _reflist(manifest) -> Disclosure | None:
text=(
f"This run recorded that {REFLIST_TOKEN}, with an outcome "
f"({outcome!r}) this build of papertrace does not recognise. "
- "Treat the reference numbering as resting on the readings above this "
- "and nothing else, the same as an outcome that was never obtained."
+ "Treat the reference numbering as resting on the readings above this"
+ + (
+ ", the same as an outcome that was never obtained."
+ if unknown else
+ " and nothing else, the same as an outcome that was never obtained."
+ )
+ + unknown
),
short=f"{REFLIST_TOKEN}: unrecognised outcome {outcome!r}",
)
@@ -717,6 +723,12 @@ def _reflist(manifest) -> Disclosure | None:
# Identical wording to `cli._refs_pipeline`'s console line for the same
# state, so the two surfaces never describe one call two different ways.
named = model or "a model that did not report its own name"
+ # "and nothing else" is a claim about what the numbering rests on, and it
+ # is FALSE on a run that also made the resolution call: that call
+ # substitutes an entry. Dropped where one happened, kept where none did —
+ # it is load-bearing there, and the sentence would otherwise contradict
+ # the clause appended two lines later, inside one paragraph.
+ only = "" if resolution else " and nothing else"
if outcome == "not_attempted" and inferred:
# `outcome` here is a GUESS (see the `elif failure:` branch above), so
# the sentence says what is recorded and no more. "no reading was
@@ -725,8 +737,7 @@ def _reflist(manifest) -> Disclosure | None:
head = (
f"This run asked that {REFLIST_TOKEN}, and this manifest does not record "
f"what came of it: {failure}. Nothing from such a reading reached the list "
- "below, so the reference numbering rests on the readings above this and "
- "nothing else."
+ f"below, so the reference numbering rests on the readings above this{only}."
)
short = f"{REFLIST_TOKEN}: asked for, outcome never recorded — {failure}"
level = "warn"
@@ -734,7 +745,7 @@ def _reflist(manifest) -> Disclosure | None:
head = (
f"This run asked that {REFLIST_TOKEN}, and no reading was obtained: "
f"{failure}. The reference numbering therefore rests on the readings above "
- f"it and nothing else."
+ f"it{only}."
)
short = f"{REFLIST_TOKEN}: asked for, not obtained — {failure}"
level = "warn" # a structural gap, not routine information
@@ -745,8 +756,7 @@ def _reflist(manifest) -> Disclosure | None:
head = (
f"This run asked that {REFLIST_TOKEN}, and the call did not return an "
f"answer: {failure}. Nothing it might have proposed reached the list "
- f"below, and the reference numbering rests on the readings above this "
- f"and nothing else."
+ f"below, and the reference numbering rests on the readings above this{only}."
)
short = f"{REFLIST_TOKEN}: the call failed — {failure}"
level = "warn" # a call that did not return is not an aside
@@ -1355,6 +1365,26 @@ def _was_obtained(manifest, label: str) -> bool:
return bool(e and e.status in ("retrieved", "provided") and e.slug)
+def _also_resolved(resolved: list[str]) -> str:
+ """The sentence a claim owes about a label that WAS settled, or "".
+
+ A claim can cite a label still in dispute and one a resolution was
+ accepted for. The dispute keeps the token and the level — it is the more
+ urgent finding — but a reader who stops there must not be left thinking
+ every citation on this claim shares the same fate (m6 of the Task 7
+ review): the other label's verdict rests on a resolution, which is a
+ different caveat and still a live one. One producer, because BOTH dispute
+ states owe it and the second one was added later.
+ """
+ if not resolved:
+ return ""
+ return (
+ f" This claim also cites {_label_group(resolved)}, where the readings did not "
+ "agree either but a resolution was accepted for it — see that label's own "
+ "reason in the retrieval manifest."
+ )
+
+
def _claim_pairing(claim, manifest=None) -> Disclosure | None:
"""Which pairing state this claim's cited labels are in, if not a clean one.
@@ -1410,18 +1440,7 @@ def _claim_pairing(claim, manifest=None) -> Disclosure | None:
"not agree either and the source was never retrieved, so nothing was "
"judged under that label at all."
)
- if resolved:
- # This claim cites BOTH a still-disputed label and one that WAS
- # resolved — the withheld state is the more urgent finding and
- # keeps the token/level, but a reader who stops here must not be
- # left thinking every citation on this claim shares the same fate
- # (m6 of the Task 7 review): the other label's verdict rests on a
- # resolution, not a withholding, and that is a different caveat.
- text += (
- f" This claim also cites {_label_group(resolved)}, where the readings "
- "disagreed too but a resolution was accepted for it — see that label's "
- "own reason in the retrieval manifest."
- )
+ text += _also_resolved(resolved)
return Disclosure(
key="claim_pairing",
level="warn",
@@ -1448,6 +1467,7 @@ def _claim_pairing(claim, manifest=None) -> Disclosure | None:
"retrieved either, so nothing was judged under "
f"{'that label' if one else 'those labels'} in either direction. The "
"disagreement is recorded, not resolved."
+ + _also_resolved(resolved)
),
short=f"{labels} {CLAIM_PAIRING_UNRETRIEVED_TOKEN}",
)
diff --git a/tests/test_reference_readings.py b/tests/test_reference_readings.py
index ce9a2d9..2463c85 100644
--- a/tests/test_reference_readings.py
+++ b/tests/test_reference_readings.py
@@ -1599,3 +1599,41 @@ def test_a_mid_branch_manifest_with_a_model_name_still_settles_as_read(tmp_path)
assert len(fired) == 1, fired
assert "claude-opus-5" in fired[0].text
assert "does not record what came of it" not in fired[0].text
+
+
+def test_a_run_with_a_resolution_never_says_the_numbering_rests_on_nothing_else():
+ """Found by enumerating the states rather than the branches: the
+ not-obtained and failed heads end "the reference numbering rests on the
+ readings above it and nothing else", which the appended resolution
+ sentence then contradicts in the same paragraph — a resolution
+ substitutes an entry, so it is precisely something else the numbering
+ rests on."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ for reflist_state in (
+ dict(reflist_outcome="not_attempted", reflist_failure="backend is pymupdf"),
+ dict(reflist_outcome="failed", reflist_failure="RuntimeError: timed out"),
+ dict(reflist_outcome="wat"),
+ ):
+ manifest = _resolved_manifest(**reflist_state)
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, (reflist_state, fired)
+ assert "nothing else" not in fired[0].text, reflist_state
+ assert "claude-opus-5" in fired[0].text, reflist_state
+
+
+def test_a_run_with_no_resolution_keeps_the_nothing_else_claim():
+ """The other side: without a resolution the claim is true and load-bearing
+ — it is what tells a reader the numbering rests on the deterministic
+ readings alone."""
+ from papertrace.disclosures import run_disclosures
+ from papertrace.models import RunResults
+
+ manifest = RefManifest(manuscript="p.pdf", reflist_outcome="failed",
+ reflist_failure="RuntimeError: timed out")
+ fired = [d for d in run_disclosures(RunResults(manuscript="p.pdf"), manifest)
+ if d.key == "reflist"]
+ assert len(fired) == 1, fired
+ assert "nothing else" in fired[0].text
diff --git a/tests/test_withholding.py b/tests/test_withholding.py
index 7ae7c79..1aeae79 100644
--- a/tests/test_withholding.py
+++ b/tests/test_withholding.py
@@ -348,3 +348,22 @@ def test_any_iterable_of_labels_withholds_every_claim_not_just_the_first(tmp_pat
assert [c.verdict for c in claims] == ["unchecked", "unchecked"]
assert [c.withheld_refs for c in claims] == [["6"], ["6"]]
+
+
+def test_a_claim_citing_an_unretrieved_disputed_label_and_a_resolved_one_names_both():
+ """The new fifth state must not swallow the resolved caveat the withheld
+ state already carries: a verdict resting on an accepted resolution is a
+ different, and still live, caveat."""
+ from papertrace.disclosures import _claim_pairing
+
+ manifest = _manifest(_entry("6", "six-2019", status="paywalled"))
+ manifest.labels_disputed = ["6"]
+ manifest.labels_resolved = ["7"]
+ manifest.numbering_choice = "llm_resolved"
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["6", "7"])
+
+ d = _claim_pairing(claim, manifest)
+
+ assert d is not None
+ assert "never retrieved" in d.text
+ assert "[7]" in d.text
From 36658b0055e392a4a4c076f9048d43a3b1e38f82 Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Fri, 18 Sep 2026 23:06:06 +0200
Subject: [PATCH 38/44] report: a resolution from an earlier commit claims only
what it recorded
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
`resolution_readings == []` is never recorded, and on this branch that
identifies a manifest written before these fields existed — whose call was
also not shown what each reading said at the disputed label. Asserting
either half of this build's fuller account over that run is the report
describing behaviour the run did not have. `_resolution_sentence` is the one
producer for both surfaces.
---
src/papertrace/disclosures.py | 57 +++++++++++++++++++++++++-------
tests/test_reference_readings.py | 31 +++++++++++++++--
2 files changed, 74 insertions(+), 14 deletions(-)
diff --git a/src/papertrace/disclosures.py b/src/papertrace/disclosures.py
index f6c4ce4..912158a 100644
--- a/src/papertrace/disclosures.py
+++ b/src/papertrace/disclosures.py
@@ -925,10 +925,23 @@ def _numbering_resolution(manifest) -> Disclosure | None:
getattr(manifest, "resolution_model", "")
or "a model that did not report its own name"
)
- shown = _readings_phrase(getattr(manifest, "resolution_readings", []))
+ # `[]` is never recorded, and a manifest written by an earlier commit of
+ # this branch is exactly that case — its resolution call was ALSO not
+ # shown what each reading said, so neither half of the fuller sentence is
+ # true of it. Claiming either would be a report describing this build's
+ # behaviour over another build's run.
+ readings = list(getattr(manifest, "resolution_readings", []) or [])
+ by_model = (
+ f"{model}, which read {_readings_phrase(readings)} and was shown what each "
+ "reading said at those labels, verified field by field against that text, and "
+ "which you accepted"
+ if readings
+ else f"{model}, verified field by field against the printed list — this "
+ "manifest does not record which extractions it was shown — which you "
+ "accepted"
+ )
by = (
- f"{model}, which read {shown} and was shown what each reading said at those "
- "labels, verified field by field against that text, and which you accepted"
+ by_model
if choice == "llm_resolved"
else f"you, who chose the `{choice}` reading whole for every disputed label"
)
@@ -1365,6 +1378,27 @@ def _was_obtained(manifest, label: str) -> bool:
return bool(e and e.status in ("retrieved", "provided") and e.slug)
+def _resolution_sentence(manifest) -> str:
+ """What the resolution call was shown, claimed no more strongly than recorded.
+
+ `resolution_readings == []` means never recorded, which on this branch
+ identifies a manifest written by an earlier commit — whose call was also
+ not shown what each reading said. Asserting either half over that run
+ would be this build describing another build's behaviour.
+ """
+ readings = list(getattr(manifest, "resolution_readings", []) or [])
+ if not readings:
+ return (
+ "A model was shown the printed list — this manifest does not record which "
+ "extractions — and every field of its answer was found verbatim there."
+ )
+ return (
+ f"A model was shown {_readings_phrase(readings)} and what each reading said at "
+ "that label, and every field of its answer was found verbatim in that printed "
+ "text."
+ )
+
+
def _also_resolved(resolved: list[str]) -> str:
"""The sentence a claim owes about a label that WAS settled, or "".
@@ -1480,13 +1514,11 @@ def _claim_pairing(claim, manifest=None) -> Disclosure | None:
token=CLAIM_PAIRING_RESOLVED_TOKEN,
text=(
f"This claim cites {labels}, {CLAIM_PAIRING_RESOLVED_TOKEN}. The readings "
- "of the bibliography did not agree there. A model was shown "
- f"{_readings_phrase(getattr(manifest, 'resolution_readings', []))} and "
- "what each reading said at that label, every field of its answer was "
- "found verbatim in that printed text, and you accepted the result — so "
- "this verdict rests on a reading nobody checked against the printed "
- "page, and you are who accepted it. The numbering is still recorded as "
- "unconfirmed."
+ "of the bibliography did not agree there. "
+ + _resolution_sentence(manifest)
+ + " You accepted the result — so this verdict rests on a reading nobody "
+ "checked against the printed page, and you are who accepted it. The "
+ "numbering is still recorded as unconfirmed."
),
short=f"{labels} {CLAIM_PAIRING_RESOLVED_TOKEN}",
)
@@ -1498,8 +1530,9 @@ def _claim_pairing(claim, manifest=None) -> Disclosure | None:
level="warn",
token=CLAIM_PAIRING_CHOSEN_TOKEN,
text=(
- f"This claim cites {labels}. Two readings of the bibliography disagreed "
- f"there and {CLAIM_PAIRING_CHOSEN_TOKEN} — the `{manifest.numbering_choice}` "
+ f"This claim cites {labels}. The readings of the bibliography did not "
+ f"agree there and {CLAIM_PAIRING_CHOSEN_TOKEN} — the "
+ f"`{manifest.numbering_choice}` "
"one, for every disputed label. That is an assertion about which reading "
"is right, not a check of it, and this verdict is about whichever paper "
"that reading names."
diff --git a/tests/test_reference_readings.py b/tests/test_reference_readings.py
index 2463c85..057a1f3 100644
--- a/tests/test_reference_readings.py
+++ b/tests/test_reference_readings.py
@@ -1521,13 +1521,15 @@ def test_the_resolved_claim_disclosure_does_not_claim_the_model_saw_both():
def test_a_resolution_whose_manifest_recorded_no_readings_says_so():
"""`[]` is never recorded, not "it was shown nothing" — and the sentence
- must not silently read as the latter."""
+ must not silently read as the latter, nor assert the fuller account of
+ what this build's call is shown."""
from papertrace.disclosures import _numbering_resolution
d = _numbering_resolution(_resolved_manifest(resolution_readings=[]))
assert d is not None
- assert "does not name" in d.text
+ assert "does not record which extractions" in d.text
+ assert "what each reading said" not in d.text
# --- whole-branch review, minors 1 and 6
@@ -1637,3 +1639,28 @@ def test_a_run_with_no_resolution_keeps_the_nothing_else_claim():
if d.key == "reflist"]
assert len(fired) == 1, fired
assert "nothing else" in fired[0].text
+
+
+def test_a_resolution_from_an_earlier_commit_claims_only_what_it_recorded():
+ """The load axis, which is where four of this branch's defects lived: a
+ manifest written by an earlier commit of this same branch resolved a
+ label and recorded none of the `resolution_*` fields — because that call
+ was ALSO not shown the reading it ruled against. So the report may not
+ assert, for that manifest, either which extractions the model saw or that
+ it was shown what each reading said; both are true only of a run this
+ build made."""
+ from papertrace.disclosures import _claim_pairing, _numbering_resolution
+ from papertrace.models import ClaimResult
+
+ old = RefManifest(manuscript="p.pdf", labels_resolved=["2"],
+ numbering_choice="llm_resolved", numbering_chosen_by="user")
+
+ run = _numbering_resolution(old)
+ assert run is not None
+ assert "what each reading said" not in run.text
+ assert "does not record" in run.text
+
+ claim = _claim_pairing(ClaimResult(id=1, claim="x", location="R", refs=["2"]), old)
+ assert claim is not None
+ assert "what each reading said" not in claim.text
+ assert "does not record" in claim.text
From d09fcdb3a1f49dc99a61d9f3e0de80cf99afbeba Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Fri, 18 Sep 2026 23:21:46 +0200
Subject: [PATCH 39/44] models: a DOI is not a title, and which of two causes
put a label in dispute
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
`_title_tokens` strips DOIs the way it already strips URLs, for the reason
its own docstring gives: `10.1148/radiol.…` yielded `radiol` and
`10.1001/jamanetworkopen.…` yielded `jamanetworkopen`, and those identifier
substrings were compared against real title words. Only a DOI with no
five-letter run reached the abstention added in round 1, so the common
publisher shape still disputed every label a bare-DOI deposit voted on.
Two consumers move, both toward honesty and both measured in the report:
`_same_work` stops reporting a false divergence at a DOI-only deposit, and
`_title_check_text` stops carrying a token no first page prints in its
denominator — which on a short reference whose slug DID match can now reach
`mismatch`, correctly, where a spurious match used to hold it at
unverifiable.
`labels_uncomparable` is a published subset of `labels_disputed`: the labels
withheld because nothing could be compared rather than because the readings
named different papers. `labels_disputed` stays the only list that drives
withholding, the subset is read off the same `_label_state` call so the two
cannot contradict each other, and it is re-intersected once after the
escalation so a resolved label cannot survive in it. `None` is never
computed and `[]` is a measurement. Every surface — the run-level
disclosure, both claim-level dispute states, the withholding note and the
console — names the cause through one producer, `disclosures.dispute_causes`,
and falls back to the disjunction only where nothing measured the split.
`_label_key` moves down to `refs`, the layer below both its users.
---
schemas/refs_manifest.schema.json | 5 ++
src/papertrace/check.py | 17 +++--
src/papertrace/cli.py | 34 ++++++++--
src/papertrace/disclosures.py | 65 +++++++++++++-----
src/papertrace/models.py | 39 ++++++++++-
src/papertrace/reflist.py | 7 +-
src/papertrace/refs.py | 78 +++++++++++++++++-----
tests/test_escalation.py | 53 +++++++++++++++
tests/test_label_agreement.py | 107 ++++++++++++++++++++++++++++++
tests/test_reference_readings.py | 67 +++++++++++++++++++
tests/test_refs.py | 60 +++++++++++++++++
tests/test_withholding.py | 68 ++++++++++++++++++-
12 files changed, 545 insertions(+), 55 deletions(-)
diff --git a/schemas/refs_manifest.schema.json b/schemas/refs_manifest.schema.json
index 16addff..b7b651c 100644
--- a/schemas/refs_manifest.schema.json
+++ b/schemas/refs_manifest.schema.json
@@ -251,6 +251,11 @@
"type": "string"
}
},
+ "labels_uncomparable": {
+ "type": ["array", "null"],
+ "items": { "type": "string" },
+ "description": "A SUBSET of `labels_disputed`: the labels whose dispute was that nothing in the readings could be compared — a Crossref deposit of bare DOIs against a printed list carrying none — rather than that the readings named different papers. Both causes withhold a verdict and `labels_disputed` remains the single list that drives withholding; this one exists because the two warrant different reader actions. Every entry here MUST also appear in `labels_disputed`. `null` or absent means NEVER COMPUTED, and is not the same as `[]`, which means the split was measured and every dispute was a contradiction."
+ },
"labels_resolved": {
"type": "array",
"description": "Citation labels that were in `labels_disputed` and were then settled — by a targeted model call the user accepted, or by the user choosing one reading whole. Absent means never computed, not 'none resolved'.",
diff --git a/src/papertrace/check.py b/src/papertrace/check.py
index 59abe61..650366c 100644
--- a/src/papertrace/check.py
+++ b/src/papertrace/check.py
@@ -910,13 +910,18 @@ def check_claims(
# no verdict can safely name the paper it was fetched for.
c.verdict = "unchecked"
labels = f"[{'], ['.join(c.withheld_refs)}]"
- # "do not agree", never "name different papers": `disputed`
- # has two causes — the readings contradict each other, and
- # nothing in them could be compared — and no published field
- # tells them apart, so a note asserting the first would be
- # false whenever the second happened.
+ # WHICH cause, when the manifest recorded the split, and the
+ # disjunction when it did not: `disputed` covers both "the
+ # readings named different papers" and "nothing in them could
+ # be compared", and asserting the first over the second is the
+ # plausible-looking value this codebase exists to refuse.
+ # Phrased by `disclosures.dispute_causes`, the one producer
+ # every surface uses, so the note and the report cannot
+ # describe one label two ways.
+ from .disclosures import dispute_causes
+
c.note = (
- f"withheld: the reference list's readings do not agree about {labels}, "
+ f"withheld: at {labels}, {dispute_causes(manifest, c.withheld_refs)}, "
"so the source retrieved under that label may not be the paper the "
"manuscript cites — check the retrieval manifest before relying on "
"this claim"
diff --git a/src/papertrace/cli.py b/src/papertrace/cli.py
index 737aa46..b14ea40 100644
--- a/src/papertrace/cli.py
+++ b/src/papertrace/cli.py
@@ -548,7 +548,8 @@ def _write_disagreement(
"",
"One section per citation label whose readings of the bibliography did not "
"agree on one paper — either they name different papers, or nothing in them "
- "could be compared. Each reading's structured fields sit above the verbatim "
+ "could be compared; the manifest's `labels_uncomparable` says which, per "
+ "label. Each reading's structured fields sit above the verbatim "
"text it was read from, so the disagreement can be settled by eye against "
"the printed list.",
"",
@@ -1044,12 +1045,13 @@ def _refs_pipeline(
the value. This function is what they actually call; `refs()` is a thin CLI
adapter over it.
"""
+ from .disclosures import dispute_causes # one wording for console and report
from .ingest import ingest_pdf, references_span, references_span_flat
from .models import SourceMap, citation_labels, is_references_heading, paper_title
- # `_label_key` is shared with `reflist.propose`'s own numbering-findings
- # sort, rather than a third inline copy of the same tuple key — a
- # divergence between them would sort the same labels two ways in one report
+ # `_label_key` lives in `refs`, the layer below both users, rather than as
+ # a third inline copy of the same tuple key — a divergence between them
+ # would sort the same labels two ways in one report
from .reflist import _label_key
from .refs import (
_client,
@@ -1063,6 +1065,7 @@ def _refs_pipeline(
reconcile,
resolve_all,
stamp_seen_in,
+ uncomparable_labels,
unused_provided,
)
@@ -1283,6 +1286,12 @@ def _refs_pipeline(
(label for label, state in agreement.items() if state == "disputed"),
key=_label_key,
)
+ # WHICH of the two causes, per label. A subset by construction —
+ # `uncomparable_labels` reads it off the same `_label_state` call the line
+ # above does — and a real list even when empty, because `[]` here is a
+ # measurement ("every dispute was a contradiction") and `None` is the
+ # never-computed state a live run must never publish.
+ rec.labels_uncomparable = uncomparable_labels(others, body_labels)
# every cited label, or it is not corroboration. One disputed label is not
# "mostly corroborated", and `single` is not agreement — it is one reading
rec.corroborated = bool(body_labels) and all(
@@ -1321,8 +1330,10 @@ def _refs_pipeline(
if rec.labels_disputed:
console.print(
f"[yellow]⚠ the readings do not agree at "
- f"[{'], ['.join(rec.labels_disputed)}][/yellow] — either they name different "
- "papers there or nothing in them could be compared; verdicts on claims citing "
+ f"[{'], ['.join(rec.labels_disputed)}][/yellow] — "
+ # the same producer the report uses, so the console and the
+ # written report never describe one label two different ways
+ f"{dispute_causes(rec, rec.labels_disputed)}; verdicts on claims citing "
"those labels are withheld unless resolved below"
)
if rec.verified and rec.labels_disputed:
@@ -1427,6 +1438,17 @@ def tick(e):
numbering_corroborated=rec.corroborated,
corroborating_readings=rec.corroborating_readings,
labels_disputed=rec.labels_disputed,
+ # Re-intersected HERE, after every mutation: `_escalate_disputed` can
+ # take a label OUT of `labels_disputed` (a resolution accepted, or a
+ # reading adopted whole), and a subset computed before it would then
+ # name a label this manifest no longer disputes. Two published fields
+ # contradicting each other on one manifest is Major 3 of the
+ # whole-branch review; one intersection, in one place, is what stops
+ # it recurring here.
+ labels_uncomparable=(
+ None if rec.labels_uncomparable is None
+ else [x for x in rec.labels_uncomparable if x in set(rec.labels_disputed)]
+ ),
labels_resolved=rec.labels_resolved,
numbering_choice=rec.choice,
numbering_chosen_by=rec.chosen_by,
diff --git a/src/papertrace/disclosures.py b/src/papertrace/disclosures.py
index 912158a..3877b08 100644
--- a/src/papertrace/disclosures.py
+++ b/src/papertrace/disclosures.py
@@ -565,13 +565,10 @@ def _labels_disputed(manifest) -> Disclosure | None:
level="warn",
token=LABELS_DISPUTED_TOKEN,
text=(
- f"{named} are {LABELS_DISPUTED_TOKEN}: two or more readings of the "
- "reference list did not agree on which paper each of these labels "
- "names — they either named different papers, or nothing in them "
- "could be compared — so every claim citing one had that source "
- "withheld from judgement rather than risk a verdict about the wrong "
- "paper. See each claim's own note for which of its citations this "
- "affected."
+ f"{named} are {LABELS_DISPUTED_TOKEN}: {dispute_causes(manifest, labels)}. "
+ "Every claim citing one had that source withheld from judgement rather "
+ "than risk a verdict about the wrong paper. See each claim's own note "
+ "for which of its citations this affected."
),
short=f"{named} {LABELS_DISPUTED_TOKEN}",
)
@@ -1313,6 +1310,43 @@ def _no_quote(claim) -> Disclosure:
)
+def dispute_causes(manifest, labels: list[str]) -> str:
+ """Why these labels are in dispute, in the reader's terms.
+
+ One producer for every surface, because `labels_disputed` carries two
+ causes and each warrants a different action: a contradiction means one
+ reading is wrong and somebody should look; nothing comparable means no
+ conflict is known and the withholding is precautionary — on a bare-DOI
+ deposit against a list that prints no DOIs, which is common, it is
+ benign.
+
+ `labels_uncomparable is None` means the split was NEVER COMPUTED — every
+ manifest written before that field — and the honest answer there is the
+ disjunction, not a cause picked for it. `[]` is a measurement: every
+ dispute was a contradiction.
+ """
+ unc = getattr(manifest, "labels_uncomparable", None)
+ if unc is None:
+ return (
+ "the readings did not agree on which paper each of them names — they "
+ "either named different papers, or nothing in them could be compared"
+ )
+ mute = [x for x in labels if x in set(unc)]
+ clashed = [x for x in labels if x not in set(unc)]
+ if mute and clashed:
+ return (
+ f"the readings named different papers at {_label_group(clashed)}, and at "
+ f"{_label_group(mute)} nothing in them could be compared"
+ )
+ if mute:
+ return (
+ "nothing in them could be compared — no reading is known to contradict "
+ "another here, and the verdict is withheld because nothing establishes "
+ "what the label names"
+ )
+ return "the readings named different papers there"
+
+
def _resolution_clause(manifest, *, lead: str = " Separately, later in this run") -> str:
"""What the RESOLUTION call did, as a sentence to append — or "".
@@ -1457,12 +1491,12 @@ def _claim_pairing(claim, manifest=None) -> Disclosure | None:
one = len(withheld) == 1
text = (
f"This claim cites {labels}, and the {CLAIM_PAIRING_WITHHELD_TOKEN} about "
- f"{'that label' if one else 'those labels'}: the readings of the "
- "bibliography did not agree on which paper it names — they either named "
- "different papers or left nothing that could be compared — so the source "
- "retrieved under that label may not be the paper the manuscript actually "
- f"cites. No verdict was reached on {'it' if one else 'them'} — check the "
- "retrieval manifest before treating this claim as checked."
+ f"{'that label' if one else 'those labels'}: "
+ f"{dispute_causes(manifest, withheld)}. So the source retrieved under "
+ f"{'that label' if one else 'those labels'} may not be the paper the "
+ f"manuscript actually cites, and no verdict was reached on "
+ f"{'it' if one else 'them'} — check the retrieval manifest before "
+ "treating this claim as checked."
)
if unretrieved:
# named separately, never folded into the sentence above: nothing
@@ -1495,9 +1529,8 @@ def _claim_pairing(claim, manifest=None) -> Disclosure | None:
level="warn",
token=CLAIM_PAIRING_UNRETRIEVED_TOKEN,
text=(
- f"This claim cites {labels}, and {CLAIM_PAIRING_UNRETRIEVED_TOKEN}: the "
- "readings of the bibliography did not agree on which paper "
- f"{'it names' if one else 'they name'}, and the source was never "
+ f"This claim cites {labels}, and {CLAIM_PAIRING_UNRETRIEVED_TOKEN}: "
+ f"{dispute_causes(manifest, unretrieved)}, and the source was never "
"retrieved either, so nothing was judged under "
f"{'that label' if one else 'those labels'} in either direction. The "
"disagreement is recorded, not resolved."
diff --git a/src/papertrace/models.py b/src/papertrace/models.py
index 526400d..7aab5a0 100644
--- a/src/papertrace/models.py
+++ b/src/papertrace/models.py
@@ -73,6 +73,19 @@ def _declared_title_is_usable(title: str) -> bool:
)
_URL_RE = re.compile(r"(?:https?://|www\.)\S+", re.I)
+# A DOI is the same class of thing as the URL above and fails the same way.
+# `_reference_raw`'s documented fallback for a DOI-only Crossref deposit is
+# the DOI string itself, and most publisher DOIs embed a journal or platform
+# slug: `10.1148/radiol.…` yields `radiol`, `10.1001/jamanetworkopen.…` yields
+# `jamanetworkopen`. Those are substrings of an identifier, not words from a
+# title, and they were being compared against real title words — a deposit
+# came back as naming a DIFFERENT paper from the page, which disputed every
+# label it voted on. From the other side they inflate the denominator of
+# `_title_check_text` with a token no first page carries ("JAMA Network Open"
+# is printed with spaces), which pushes a correct retrieval toward mismatch.
+# Kept as its own pattern rather than folded into `_URL_RE`: a bare DOI is not
+# a URL, and `https://doi.org/10.…` is already covered by that one.
+_DOI_IN_TEXT_RE = re.compile(r"\b10\.\d{4,9}/\S+", re.I)
# What NFKD cannot decompose, because these are distinct letters rather than a
@@ -117,7 +130,7 @@ def _fold(text: str) -> str:
def _title_tokens(raw: str) -> set[str]:
- """The reference's own distinctive words — URLs removed first.
+ """The reference's own distinctive words — URLs and DOIs removed first.
A URL is not part of a title, and a *tracking parameter* least of all:
`?utm_source=chatgpt.com` on a cited news page contributed `chatgpt` and
@@ -126,10 +139,15 @@ def _title_tokens(raw: str) -> set[str]:
inflating the denominator with `firstmedical`, `assuranceprogram` and
`publications` — words no first page will carry, so they dilute the ratio
the check is measured on.
+
+ A DOI is stripped for exactly those two reasons — see `_DOI_IN_TEXT_RE`.
+ It is the same class of thing: an identifier, whose substrings are not
+ words anybody wrote as a title.
"""
# folded, not merely lowercased: `[a-z]{5,}` over raw text drops `Späth`
# entirely and truncates `Cristóbal` to `crist`
- return set(re.findall(r"[a-z]{5,}", _fold(_URL_RE.sub(" ", raw)))) - _TITLE_STOPWORDS
+ stripped = _DOI_IN_TEXT_RE.sub(" ", _URL_RE.sub(" ", raw))
+ return set(re.findall(r"[a-z]{5,}", _fold(stripped))) - _TITLE_STOPWORDS
# Four distinct words, not three. The observed false positive cleared the 0.35
@@ -592,6 +610,19 @@ class RefManifest:
# before this feature says nothing about disputes, it does not assert
# there were none.
labels_disputed: list[str] = field(default_factory=list)
+ # A documented SUBSET of `labels_disputed`: the labels whose dispute was
+ # that nothing in the readings could be compared, rather than that they
+ # named different papers. `labels_disputed` stays the single list that
+ # drives withholding, so behaviour is unchanged and the two cannot
+ # contradict each other — `refs.uncomparable_labels` reads this off the
+ # same call that decides the state.
+ #
+ # Three states, not two: `None` is NEVER COMPUTED — every manifest written
+ # before this field, including one from an earlier commit of this branch —
+ # and `[]` is "computed, and every dispute was a contradiction". One empty
+ # list could not mean both, and the report picks a cause only when one was
+ # actually measured.
+ labels_uncomparable: list[str] | None = None
labels_resolved: list[str] = field(default_factory=list)
# How a disputed numbering was left; "" means the interactive escalation
# never ran. Never set from a model's own say-so alone.
@@ -744,6 +775,7 @@ def to_json(self, path: Path) -> None:
"numbering_corroborated": self.numbering_corroborated,
"corroborating_readings": self.corroborating_readings,
"labels_disputed": self.labels_disputed,
+ "labels_uncomparable": self.labels_uncomparable,
"labels_resolved": self.labels_resolved,
"numbering_choice": self.numbering_choice,
"numbering_chosen_by": self.numbering_chosen_by,
@@ -799,6 +831,9 @@ def from_json(cls, path: Path) -> RefManifest:
corroborating_readings=data.get("corroborating_readings", []),
# absent means never computed, not "none disputed" or "none resolved"
labels_disputed=data.get("labels_disputed", []),
+ # `.get` with no default: absent AND explicit `null` both read as
+ # `None` — never computed — never coerced to an empty list
+ labels_uncomparable=data.get("labels_uncomparable"),
labels_resolved=data.get("labels_resolved", []),
numbering_choice=data.get("numbering_choice", ""),
numbering_chosen_by=data.get("numbering_chosen_by", ""),
diff --git a/src/papertrace/reflist.py b/src/papertrace/reflist.py
index 86bf5bf..81e1907 100644
--- a/src/papertrace/reflist.py
+++ b/src/papertrace/reflist.py
@@ -15,7 +15,7 @@
from . import ask
from .models import RefEntry, _fold
-from .refs import DOI_RE, _entry
+from .refs import DOI_RE, _entry, _label_key
REFLIST_PROMPT = """You are the reference-list reading step of a peer-review fact-checker.
@@ -414,11 +414,6 @@ def propose(
return out, prov
-def _label_key(label: str) -> tuple[int, int, str]:
- """Numeric labels in numeric order, anything else after them, by name."""
- return (0, int(label), "") if label.isdigit() else (1, 0, label)
-
-
def _numeral_gaps(nums: list[str]) -> list[str]:
"""Numerals `1..max` that this reading carries no entry for.
diff --git a/src/papertrace/refs.py b/src/papertrace/refs.py
index 006f22f..f106bbb 100644
--- a/src/papertrace/refs.py
+++ b/src/papertrace/refs.py
@@ -611,10 +611,14 @@ class Reconciliation:
# plan — nothing here may redefine it or be read as a substitute for it.
corroborated: bool = False
corroborating_readings: list[str] = field(default_factory=list)
- # Citation labels where two or more readings actively named different
- # papers — the set `label_agreement` marked `disputed`, and the set whose
- # verdicts a downstream withholding filter must drop.
+ # Citation labels the readings did not agree on — the set
+ # `label_agreement` marked `disputed`, and the set whose verdicts a
+ # downstream withholding filter must drop.
labels_disputed: list[str] = field(default_factory=list)
+ # A subset of the line above: those disputed because nothing in the
+ # readings could be COMPARED, rather than because they named different
+ # papers. `None` means the split was never computed.
+ labels_uncomparable: list[str] | None = None
# Labels that WERE in `labels_disputed` and were then settled — by a
# targeted model call the user accepted, or by choosing one reading whole.
labels_resolved: list[str] = field(default_factory=list)
@@ -757,6 +761,12 @@ def _first_divergence(a: list[RefEntry], b: list[RefEntry]) -> int | None:
# than trusting a bare string a typo could silently narrow to three states.
LABEL_AGREEMENT = ("agreed", "single", "disputed", "absent")
+# Why a label is `disputed`, for the one consumer that needs to tell them
+# apart. `""` is every non-disputed state. Not persisted as a per-label map:
+# the manifest publishes `labels_uncomparable`, one subset, because that is
+# the distinction that changes what a reader should do.
+DISPUTE_CAUSES = ("duplicate", "contradiction", "uncomparable")
+
def _comparably_same(x: RefEntry, y: RefEntry) -> bool | None:
"""Do these two readings name the same paper — or is there no way to tell?
@@ -854,10 +864,40 @@ def label_agreement(
return {label: _label_state(candidates, label)[0] for label in body}
+def _label_key(label: str) -> tuple[int, int, str]:
+ """Numeric labels in numeric order, anything else after them, by name.
+
+ Lives here, the lower layer, because three call sites need it —
+ `uncomparable_labels` below, `reflist.propose`'s numbering findings and
+ `cli`'s `labels_disputed` — and three inline copies of one tuple key is
+ how the same labels come to be sorted two ways in one report.
+ """
+ return (0, int(label), "") if label.isdigit() else (1, 0, label)
+
+
+def uncomparable_labels(candidates: dict[str, list[RefEntry]], body: set[str]) -> list[str]:
+ """The disputed labels whose cause was that nothing could be COMPARED.
+
+ A documented subset of what `label_agreement` calls `disputed`, and a
+ subset by construction: both are read off `_label_state`, so the pair
+ cannot contradict each other the way `seen_in` and `labels_disputed` once
+ did. It changes nothing about what is withheld — `labels_disputed` stays
+ the single list that drives withholding — and exists because the two
+ causes warrant different reader actions. "The readings named different
+ papers" means one of them is wrong and somebody should look; "nothing in
+ them could be compared" means no conflict is known and the withholding is
+ precautionary, which on a bare-DOI deposit is both common and benign.
+ """
+ return sorted(
+ (label for label in body if _label_state(candidates, label)[2] == "uncomparable"),
+ key=_label_key,
+ )
+
+
def _label_state(
candidates: dict[str, list[RefEntry]], label: str
-) -> tuple[str, list[str]]:
- """One label's agreement state, and the readings that earned an `agreed`.
+) -> tuple[str, list[str], str]:
+ """One label's agreement state, the readings that earned an `agreed`, and why.
The single place the rule lives: `label_agreement` publishes the state and
`corroborating_readings` intersects the names, so the two can never drift
@@ -886,10 +926,14 @@ def _label_state(
refuse. Nothing establishes that these name one paper, and withholding is
the honest answer to that.
- So `disputed` carries two causes — the readings contradict each other, and
- nothing could be compared. Both withhold a verdict, which is right, but
- **no prose may say the readings named different papers on the strength of
- this state alone**; the report's wording covers both.
+ So `disputed` carries three causes — a duplicate, a contradiction, and
+ nothing comparable — and the THIRD returned value is which of them, from
+ `DISPUTE_CAUSES`. They warrant different reader actions: a contradiction
+ means one reading is wrong and somebody should look; nothing comparable
+ means no conflict is known and the withholding is precautionary, which on
+ a bare-DOI deposit is the benign and common case. `uncomparable_labels`
+ is the published subset built from this, and it is a subset BY
+ CONSTRUCTION because it is read off the same call that decides the state.
The second returned value is the readings credited with the agreement,
which is not simply the voters: `DERIVED_READINGS` are excluded, and a
@@ -897,26 +941,26 @@ def _label_state(
"""
voters, duplicated = _label_voters(candidates, label)
if duplicated:
- return "disputed", []
+ return "disputed", [], "duplicate"
if not voters:
- return "absent", []
+ return "absent", [], ""
if len(voters) == 1:
# nothing was compared because there was nobody to compare with — a
# different fact from "compared and found uncomparable" below
- return "single", []
+ return "single", [], ""
concurring: set[str] = set()
for (name_x, x), (name_y, y) in itertools.combinations(voters, 2):
same = _comparably_same(x, y)
if same is False:
- return "disputed", []
+ return "disputed", [], "contradiction"
if same:
concurring |= {name_x, name_y}
if not concurring:
if not any(_says_something_comparable(e) for _, e in voters):
- return "disputed", [] # see this docstring's "exception"
- return "single", []
+ return "disputed", [], "uncomparable" # see this docstring's "exception"
+ return "single", [], ""
credited = sorted(n for n in concurring if n not in DERIVED_READINGS)
- return ("agreed", credited) if len(credited) >= 2 else ("single", [])
+ return ("agreed", credited, "") if len(credited) >= 2 else ("single", [], "")
def _label_voters(
@@ -963,7 +1007,7 @@ def corroborating_readings(candidates: dict[str, list[RefEntry]], body: set[str]
"""
names: set[str] | None = None
for label in body:
- state, credited = _label_state(candidates, label)
+ state, credited, _ = _label_state(candidates, label)
if state != "agreed":
return []
names = set(credited) if names is None else (names & set(credited))
diff --git a/tests/test_escalation.py b/tests/test_escalation.py
index cf80b7a..ae16c3d 100644
--- a/tests/test_escalation.py
+++ b/tests/test_escalation.py
@@ -858,3 +858,56 @@ def test_a_resolution_call_that_was_never_made_is_not_a_model_that_could_not_tel
assert "too long to send" in rec.resolution_failure
# and the note must not claim a reading that never happened
assert "verified verbatim" not in rec.note
+
+
+# --- fix round 2: labels_uncomparable stays a subset through the escalation
+
+
+def test_the_manifest_never_publishes_an_uncomparable_label_it_stopped_disputing(
+ disputed, monkeypatch
+):
+ """The subset invariant, through the one path that can break it: the
+ escalation takes a label OUT of `labels_disputed`, and a
+ `labels_uncomparable` computed before it would then name a label the
+ manifest no longer disputes. Two published fields disagreeing on one
+ manifest is Major 3 of this review, and this is where it would recur."""
+ pdf, case, answers = disputed
+ answers.append("1")
+ _capture_ask(monkeypatch, json.dumps([{"num": "2", "title": "Second paper"}]))
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.labels_resolved == ["2"]
+ assert manifest.labels_disputed == []
+ assert manifest.labels_uncomparable == []
+ assert set(manifest.labels_uncomparable) <= set(manifest.labels_disputed)
+
+
+def test_a_run_that_computed_the_split_publishes_it_even_when_empty(disputed):
+ """`[]` is a measurement — every dispute here was a contradiction — and a
+ live run must never leave the field at its never-computed default."""
+ pdf, case, answers = disputed
+ answers.append("2")
+
+ manifest = _run_refs(pdf, case)
+
+ assert manifest.labels_disputed == ["2"]
+ assert manifest.labels_uncomparable == []
+ assert manifest.labels_uncomparable is not None
+
+
+def test_the_console_names_the_cause_it_measured(disputed, capsys):
+ """The console is a surface like any other, and it has the split in hand
+ — `rec.labels_uncomparable` is computed two lines above the line that
+ prints. On this fixture the two readings name genuinely different papers
+ at [2], so the contradiction wording is the true one."""
+ pdf, case, answers = disputed
+ answers.append("2")
+ capsys.readouterr()
+
+ _run_refs(pdf, case)
+
+ out = " ".join(capsys.readouterr().out.split())
+ assert "the readings do not agree at [2]" in out
+ assert "named different papers there" in out
+ assert "nothing in them could be compared" not in out
diff --git a/tests/test_label_agreement.py b/tests/test_label_agreement.py
index bea77fa..cb6545d 100644
--- a/tests/test_label_agreement.py
+++ b/tests/test_label_agreement.py
@@ -38,6 +38,7 @@
corroborating_readings,
label_agreement,
stamp_seen_in,
+ uncomparable_labels,
)
ROOT = Path(__file__).resolve().parent.parent
@@ -668,3 +669,109 @@ def test_corroborating_readings_is_empty_unless_every_cited_label_agreed():
"llm": [_work("1", "10.1000/x1"), _work("2", "10.1000/x9")],
}
assert corroborating_readings(candidates, body) == []
+
+
+# --- fix round 2: a DOI-only deposit abstains on EVERY DOI shape -----------
+
+
+def test_a_doi_only_deposit_abstains_whatever_its_doi_tokenises_to():
+ """Critical 1, reopened and closed properly.
+
+ `_reference_raw`'s fallback is the DOI string itself — bare, not a URL —
+ and `[a-z]{5,}` lifts a journal slug out of most publisher DOIs. Only the
+ third shape here has no five-letter run, and it was the only one the
+ first fix reached: the other two came back `False` and disputed every
+ label they voted on, which is the audit-with-no-verdicts outcome the
+ finding named.
+ """
+ page = _printed("7")
+ for doi in ("10.1148/radiol.2019181432", # tokenised to {'radiol'}
+ "10.1001/jamanetworkopen.2023.18153", # {'jamanetworkopen'}
+ "10.1038/s41591-019-0673-2", # set()
+ "10.1016/j.neuroimage.2020.117161", # {'neuroimage'}
+ "10.1093/bioinformatics/btaa123"): # {'bioinformatics'}
+ deposit = RefEntry(num="7", raw=doi, doi=doi)
+ assert _comparably_same(deposit, page) is None, doi
+ assert label_agreement({"crossref": [deposit], "parsed": [page]}, {"7"}) == {
+ "7": "single"
+ }, doi
+ assert label_agreement(
+ {"crossref": [deposit], "parsed": [page], "pymupdf": [_printed("7")]}, {"7"}
+ ) == {"7": "agreed"}, doi
+
+
+def test_two_doi_only_deposits_still_compare_by_doi_and_never_reach_the_mute_cell():
+ """The ordering the `_comparably_same` branches rest on, re-checked under
+ the corrected tokeniser: with DOIs stripped, two deposits have no tokens
+ on either side — but they never get that far, because the DOI comparison
+ comes first. Only an entry with NEITHER a DOI nor a title word is mute."""
+ a = RefEntry(num="7", raw="10.1148/radiol.2019181432", doi="10.1148/radiol.2019181432")
+ b_same = RefEntry(num="7", raw="10.1148/radiol.2019181432", doi="10.1148/radiol.2019181432")
+ b_other = RefEntry(num="7", raw="10.1038/nature12373", doi="10.1038/nature12373")
+
+ assert _comparably_same(a, b_same) is True
+ assert _comparably_same(a, b_other) is False
+ assert label_agreement({"crossref": [a], "parsed": [b_same]}, {"7"}) == {"7": "agreed"}
+ assert label_agreement({"crossref": [a], "parsed": [b_other]}, {"7"}) == {"7": "disputed"}
+ # and the mute cell still needs an entry carrying neither
+ mute = RefEntry(num="7", raw="A. B. 2020. 14(3):1-9.")
+ assert label_agreement(
+ {"parsed": [mute], "pymupdf": [RefEntry(num="7", raw="Q. Z. 2020. 88(1):4-7.")]}, {"7"}
+ ) == {"7": "disputed"}
+
+
+# --- fix round 2: which of the two causes put a label in dispute -----------
+
+
+def test_uncomparable_labels_names_only_the_labels_nothing_could_compare():
+ """`disputed` has two causes and they warrant different reader actions:
+ a contradiction means one reading is wrong and a human should look;
+ nothing comparable means no conflict is known and the withholding is
+ precautionary. [1] is a contradiction, [2] is uncomparable, [3] agrees."""
+ body = {"1", "2", "3"}
+ candidates = {
+ "parsed": [_work("1", "10.1000/x1"),
+ RefEntry(num="2", raw="A. B. 2020. 14(3):1-9."),
+ _work("3", "10.1000/x3")],
+ "pymupdf": [_work("1", "10.1000/x9"),
+ RefEntry(num="2", raw="Q. Z. 2020. 88(1):4-7."),
+ _work("3", "10.1000/x3")],
+ }
+ assert label_agreement(candidates, body) == {
+ "1": "disputed", "2": "disputed", "3": "agreed",
+ }
+ assert uncomparable_labels(candidates, body) == ["2"]
+
+
+def test_uncomparable_labels_is_always_a_subset_of_the_disputed_ones():
+ """The invariant, pinned rather than assumed. Major 3 on this branch was
+ exactly two published fields disagreeing on one manifest, and this pair
+ is the next candidate: both come out of `_label_state`, which is the one
+ place the rule lives, and a label can only be uncomparable by way of
+ being disputed."""
+ body = {"1", "2", "3", "4"}
+ candidates = {
+ "parsed": [_work("1", "10.1000/x1"),
+ RefEntry(num="2", raw="A. B. 2020. 14(3):1-9."),
+ _work("3", "10.1000/x3"), _work("4", "10.1000/x4")],
+ "pymupdf": [_work("1", "10.1000/x9"),
+ RefEntry(num="2", raw="Q. Z. 2020. 88(1):4-7."),
+ _work("3", "10.1000/x3"), _work("4", "10.1000/x4"),
+ _work("4", "10.1000/x44")],
+ }
+ agreement = label_agreement(candidates, body)
+ disputed = {lbl for lbl, state in agreement.items() if state == "disputed"}
+ assert disputed == {"1", "2", "4"}, agreement
+ assert set(uncomparable_labels(candidates, body)) <= disputed
+
+
+def test_a_duplicate_is_disputed_for_its_own_reason_not_an_uncomparable_one():
+ """A reading carrying the label twice is judgeable — two papers, one of
+ which would really be downloaded — which is a different finding from
+ nothing being comparable, and must not be filed under it."""
+ candidates = {
+ "parsed": [_work("7", "10.1000/x7"), _work("7", "10.1000/x77")],
+ "pymupdf": [_work("7", "10.1000/x7")],
+ }
+ assert label_agreement(candidates, {"7"}) == {"7": "disputed"}
+ assert uncomparable_labels(candidates, {"7"}) == []
diff --git a/tests/test_reference_readings.py b/tests/test_reference_readings.py
index 057a1f3..297bfe8 100644
--- a/tests/test_reference_readings.py
+++ b/tests/test_reference_readings.py
@@ -1664,3 +1664,70 @@ def test_a_resolution_from_an_earlier_commit_claims_only_what_it_recorded():
assert claim is not None
assert "what each reading said" not in claim.text
assert "does not record" in claim.text
+
+
+# --- fix round 2: labels_uncomparable, a documented subset of labels_disputed
+
+
+def test_labels_uncomparable_round_trips_and_validates(tmp_path):
+ """Gate 2: additive, not required, and three-valued. `None` is never
+ computed and `[]` is "computed, every dispute was a contradiction" —
+ a single empty list could not say both, which is `table_warnings`'
+ rule and the one this branch has now broken four times."""
+ import jsonschema
+
+ m = RefManifest(manuscript="p.pdf", labels_disputed=["6", "7"],
+ labels_uncomparable=["7"])
+ path = tmp_path / "refs_manifest.json"
+ m.to_json(path)
+ schema = json.loads(SCHEMA_PATH.read_text())
+ jsonschema.Draft202012Validator(schema).validate(json.loads(path.read_text()))
+ assert "labels_uncomparable" not in schema.get("required", [])
+
+ assert RefManifest.from_json(path).labels_uncomparable == ["7"]
+
+ payload = json.loads(path.read_text())
+ del payload["labels_uncomparable"]
+ path.write_text(json.dumps(payload))
+ assert RefManifest.from_json(path).labels_uncomparable is None
+
+ m2 = RefManifest(manuscript="p.pdf", labels_disputed=["6"], labels_uncomparable=[])
+ m2.to_json(path)
+ jsonschema.Draft202012Validator(schema).validate(json.loads(path.read_text()))
+ assert RefManifest.from_json(path).labels_uncomparable == []
+
+
+def test_the_run_level_disclosure_says_which_cause_applies():
+ """The disjunction was honest but weak, and the two causes warrant
+ different reader actions."""
+ from papertrace.disclosures import _labels_disputed
+
+ contradiction = _labels_disputed(
+ RefManifest(manuscript="p", labels_disputed=["6"], labels_uncomparable=[])
+ )
+ assert "named different papers" in contradiction.text
+ assert "could be compared" not in contradiction.text
+
+ uncomparable = _labels_disputed(
+ RefManifest(manuscript="p", labels_disputed=["6"], labels_uncomparable=["6"])
+ )
+ assert "nothing in them could be compared" in uncomparable.text
+ assert "named different papers" not in uncomparable.text
+
+ mixed = _labels_disputed(
+ RefManifest(manuscript="p", labels_disputed=["6", "7"], labels_uncomparable=["7"])
+ )
+ assert "[6]" in mixed.text and "[7]" in mixed.text
+ assert "named different papers" in mixed.text
+ assert "could be compared" in mixed.text
+
+
+def test_a_manifest_that_never_split_the_causes_keeps_the_disjunction():
+ """`None` is never computed — every manifest written before this field,
+ including one from an earlier commit of this branch — and the report may
+ not pick a cause for it."""
+ from papertrace.disclosures import _labels_disputed
+
+ d = _labels_disputed(RefManifest(manuscript="p", labels_disputed=["6"]))
+ assert "either" in d.text
+ assert "named different papers" in d.text and "could be compared" in d.text
diff --git a/tests/test_refs.py b/tests/test_refs.py
index 3944bcd..7a45b91 100644
--- a/tests/test_refs.py
+++ b/tests/test_refs.py
@@ -803,6 +803,66 @@ def test_a_tracking_parameter_is_not_part_of_a_title():
assert {"launches", "practice", "artificial"} <= tokens, "the title's own words survive"
+def test_a_doi_is_not_part_of_a_title_either():
+ """The same class of thing as the tracking parameter above, and the same
+ failure. `_reference_raw`'s documented fallback for a DOI-only deposit is
+ the DOI string itself, and most publisher DOIs embed a journal or platform
+ slug — `radiol`, `jamanetworkopen`, `neuroimage`, `bioinformatics` — which
+ `[a-z]{5,}` happily lifts out and then compares against real title words.
+ An identifier's substring is not a word from a title.
+
+ Three shapes, because only the third had no five-letter run and so was the
+ only one the earlier fix reached."""
+ from papertrace.refs import _title_tokens
+
+ for doi in ("10.1148/radiol.2019181432",
+ "10.1001/jamanetworkopen.2023.18153",
+ "10.1038/s41591-019-0673-2",
+ "10.1016/j.neuroimage.2020.117161",
+ "10.1093/bioinformatics/btaa123"):
+ assert _title_tokens(doi) == set(), doi
+ # and written the way a reference prints it, prefix and all
+ assert _title_tokens(f"doi:{doi}") == set(), doi
+
+ # the reference's own words survive a trailing DOI, and the DOI's do not
+ tokens = _title_tokens(
+ "Fujita S. Characterization of brain volume changes in aging individuals. "
+ "JAMA Netw Open. 2023. doi:10.1001/jamanetworkopen.2023.18153"
+ )
+ assert {"characterization", "changes", "individuals"} <= tokens
+ assert "jamanetworkopen" not in tokens
+
+
+def test_a_trailing_doi_no_longer_inflates_the_title_checks_denominator():
+ """The URL strip's own stated reason, applied to the identifier that
+ replaced it: a token no first page will carry sits in the denominator and
+ pushes a correct retrieval toward `mismatch`. `jamanetworkopen` is printed
+ on no page — the page says "JAMA Network Open", with spaces."""
+ from papertrace.refs import _title_check_text
+
+ raw = ("Fujita S, Mori S, Onda K. Characterization of brain volume changes in aging "
+ "individuals. JAMA Netw Open. 2023;6(6):e2318153. "
+ "doi:10.1001/jamanetworkopen.2023.18153")
+ page = ("Characterization of Brain Volume Changes in Aging Individuals\n"
+ "Fujita S, Mori S, Onda K. JAMA Network Open. 2023;6(6):e2318153\n")
+
+ state, detail = _title_check_text(raw, page)
+ assert state == "verified", detail
+ assert "6/6" in detail, detail
+
+
+def test_a_doi_only_reference_has_nothing_to_match_on_rather_than_one_word():
+ """`unverifiable` either way, but for the honest reason: a single journal
+ slug lifted out of an identifier is not "one distinctive word" of the
+ reference, and calling it that invites a future ratio to believe it."""
+ from papertrace.refs import _title_check_text
+
+ state, detail = _title_check_text("10.1148/radiol.2019181432",
+ "Radiology 2019 imaging study of the chest")
+ assert state == "unverifiable"
+ assert "no distinctive words" in detail, detail
+
+
def test_three_generic_domain_words_are_not_an_identity_check():
"""The last line of defence, in case a URL-only reference reaches it by some
other route: with the URL stripped the ACR news page still scores 3/7 =
diff --git a/tests/test_withholding.py b/tests/test_withholding.py
index 1aeae79..061d397 100644
--- a/tests/test_withholding.py
+++ b/tests/test_withholding.py
@@ -244,8 +244,12 @@ def test_the_withholding_note_does_not_assert_the_readings_named_different_paper
check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"6"})
- assert "different papers" not in claims[0].note
- assert "do not agree" in claims[0].note
+ # the disjunction, not the assertion — this manifest never recorded which
+ # cause applied (`labels_uncomparable is None`), so neither may the note
+ assert "did not agree on which paper" in claims[0].note
+ assert "either named different papers, or nothing in them could be compared" in (
+ claims[0].note
+ )
def test_the_run_level_disputed_disclosure_covers_both_causes():
@@ -367,3 +371,63 @@ def test_a_claim_citing_an_unretrieved_disputed_label_and_a_resolved_one_names_b
assert d is not None
assert "never retrieved" in d.text
assert "[7]" in d.text
+
+
+# --- fix round 2: which cause, per claim and in the note ------------------
+
+
+def test_the_claim_disclosure_says_which_cause_applies():
+ """Per claim, the same split the run-level roll-up makes."""
+ from papertrace.disclosures import _claim_pairing
+
+ m = _manifest(_entry("6", "six-2019"))
+ m.labels_disputed = ["6"]
+ m.labels_uncomparable = ["6"]
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["6"], withheld_refs=["6"])
+ d = _claim_pairing(claim, m)
+ assert "nothing in them could be compared" in d.text
+ assert "named different papers" not in d.text
+
+ m.labels_uncomparable = []
+ d = _claim_pairing(claim, m)
+ assert "named different papers" in d.text
+ assert "could be compared" not in d.text
+
+
+def test_an_unretrieved_disputed_claim_says_which_cause_too():
+ from papertrace.disclosures import _claim_pairing
+
+ m = _manifest(_entry("6", "six-2019", status="paywalled"))
+ m.labels_disputed = ["6"]
+ m.labels_uncomparable = ["6"]
+ claim = ClaimResult(id=1, claim="x", location="Results", refs=["6"])
+ d = _claim_pairing(claim, m)
+ assert "never retrieved" in d.text
+ assert "nothing in them could be compared" in d.text
+
+
+def test_the_withholding_note_says_which_cause_when_the_manifest_split_them(tmp_path):
+ """`check_claims` has the manifest, so the note can say which rather than
+ naming the disjunction."""
+ _write_source(tmp_path, "six-2019")
+ manifest = _manifest(_entry("6", "six-2019"))
+ manifest.labels_disputed = ["6"]
+ manifest.labels_uncomparable = ["6"]
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["6"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"6"})
+
+ assert "nothing in them could be compared" in claims[0].note
+ assert "different papers" not in claims[0].note
+
+
+def test_the_withholding_note_keeps_the_disjunction_when_nothing_split_them(tmp_path):
+ """`None` is never computed, and the note may not pick a cause for it."""
+ _write_source(tmp_path, "six-2019")
+ manifest = _manifest(_entry("6", "six-2019"))
+ manifest.labels_disputed = ["6"]
+ claims = [ClaimResult(id=1, claim="x", location="Results", refs=["6"])]
+
+ check_claims(claims, manifest, tmp_path, backend="pymupdf", disputed={"6"})
+
+ assert "either" in claims[0].note
From 2e9e1874eac5f5c610320f61e18e6212f0961b57 Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Fri, 18 Sep 2026 23:35:18 +0200
Subject: [PATCH 40/44] docs: the voter-only property is true of the readings,
not of every model call
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The `--llm-refs` reading is a voter and reaches neither `reconcile`'s arguments
nor `resolve_all`. Three files generalised that to "no model output can cause a
source to be resolved, downloaded or judged", which the escalation's option 1
falsifies: `_escalate_disputed` substitutes entries `reflist.resolve_disputed`
built from a model's reply, `resolve_all` downloads them and `check.py` judges
them. The substitution is mandatory — returning the list untouched un-disputes
the label while leaving the entry the resolution ruled against as the paper
judged. The exception is now stated where a reader meets it, with its gate: an
interactive run, the disagreement on disk before anything is asked,
`numbering_chosen_by: "user"`, only the resolved labels substituted, and no
answer setting `numbering_verified`.
Constraint 1 cited a module that tests constraint 5; it now cites
`test_a_model_naming_different_papers_changes_no_resolved_entry`. The
escalation's stated precondition ("nothing accounted for the labels AND labels
in dispute") does not exist — `_escalate_disputed` returns early on an empty
`labels_disputed` and nothing else. A disagreement withholds the source
always, the claim only when no other source answered.
Also: `_comparably_same`, not `_same_work`, and why; the third value and the
three dispute causes; `labels_uncomparable` as a published subset;
`numbering_corroborated` as `bool | None`; the DOI strip in `_title_tokens`;
`reflist.py` rather than `refs.py` as the seam's second caller;
`reconcile`'s two candidate readings rather than its arity; the whole reading
discarded by one unverifiable title; the corroboration bullet's other branch;
`CI` and a piped stdout as non-interactive; no integer for the seam's patch
sites, which moved again today. Two wire-format rules earned here go into the
cardinal-rule section.
---
CHANGELOG.md | 126 +++++++++++++++++++++++----
CLAUDE.md | 110 +++++++++++++++++++-----
README.md | 44 +++++++---
docs/adr/0003-llm-reference-list.md | 128 +++++++++++++++++++++-------
evals/DESIGN.md | 20 ++++-
5 files changed, 345 insertions(+), 83 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0cd3b98..9578242 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -127,29 +127,119 @@ on both `refs` and `run`.
Agreement is then computed **per printed citation label** — joined on the
numeral the page carries, never on position — and reported as `agreed`,
-`single`, `disputed` or `absent`. Where two readings name different papers at a
-label, verdicts on claims citing it are **withheld**: the claim is reported
-`unchecked` with the label named, not `not_retrieved`, because the source was
-obtained and read. `single` deliberately prints, with the caveat it already
-carried. `numbering_corroborated` is a new, independent axis beside
-`numbering_verified`, which keeps its exact meaning.
-
-Where nothing accounted for the body's labels and labels are in dispute, an
-interactive run writes the full disagreement to
+`single`, `disputed` or `absent`. A disputed label's source is **withheld**:
+dropped before any model call and named in `ClaimResult.withheld_refs`. A
+claim left with no other source is then reported `unchecked` with the label
+named, not `not_retrieved`, because the source was obtained and read; a claim
+that also cites an undisputed source is still judged on that one, with the
+withheld label named beside the verdict. `single` deliberately prints, with
+the caveat it already carried. `numbering_corroborated` is a new, independent
+axis beside `numbering_verified`, which keeps its exact meaning.
+
+Where the readings are in dispute at any label, an interactive run writes the
+full disagreement to
`case/out/reference_disagreement.md` — every reading's fields and the verbatim
text each was read from — **prints the path, and only then asks** what to do
-about it. Four options: resolve the disputed labels with a model, withhold
-verdicts on all of them, adopt one reading whole, or abort. Whatever is
+about it. There is no second condition: a run whose numbering passed the
+extent check can still have a disputed label, because extent and content are
+different questions. Four options: resolve the disputed labels with a model,
+withhold verdicts on all of them, adopt one reading whole, or abort. Whatever is
answered is recorded as `numbering_chosen_by: "user"`; a person consenting to
proceed is an input, not evidence, and nothing a user answers can set
-`numbering_verified`. A non-interactive run withholds the disputed labels and
-asks nothing.
+`numbering_verified`. A run that is not interactive — no terminal, output
+piped, or `CI` set — withholds the disputed labels and asks nothing.
+
+**Choosing *resolve* is the one place a model's answer reaches retrieval, and
+it is gated on that answer.** A second model call is shown the printed
+extractions and the entries that disagree, every field it returns is verified
+verbatim against the printed text, and each label it settles has its entry
+**substituted** into the list that is then downloaded and judged. Substituting
+is mandatory rather than optional: leaving the list untouched would take the
+label out of dispute while leaving the entry the resolution ruled *against* as
+the paper judged — the original wrong-paper bug, reached through the one path
+a person authorised. The call's own provenance is persisted
+(`resolution_outcome`, `resolution_model`, `resolution_fields_discarded`,
+`resolution_readings`) and reported, including which extractions it was shown
+— never "both texts", since a pymupdf-backend run has one printed span.
Partial resolution is a normal, representable, reported outcome:
`labels_resolved` and `labels_disputed` are both non-empty on a run where some
labels were settled and some were not, and "cannot tell" from the resolution
call is a correct answer that keeps a label withheld.
+### Fixed — a reading nothing could be compared with was counted as disagreeing
+
+The comparison behind agreement is three-valued now (`True | False | None`),
+the same vocabulary `titles_match` already published in the mirror direction.
+A voter it cannot compare **abstains** instead of dissenting, which can leave
+one reading standing and print `single`. Collapsing that to "disagrees" made a
+Crossref deposit of bare DOIs dispute every label it voted on while the two
+parses agreed perfectly — an audit with no verdicts in it. The one exception
+is a label where *no* reading said anything comparable at all: that stays
+disputed, because nothing establishes what the label names.
+
+A bare DOI is the documented fallback when a deposit carries no reference
+strings, and `_title_tokens` was reading the publisher slug inside it as a
+title word — `10.1148/radiol.2019181432` contributed `radiol`, and a spread of
+real DOIs showed that is common rather than exotic. DOIs are now stripped the
+way URLs already were, for the reason the URL strip was written for: an
+identifier's substrings are not words anybody wrote as a title. The same strip
+stops `jamanetworkopen` sitting in the title check's denominator against a page
+that prints "JAMA Network Open".
+
+### Added — `labels_uncomparable`, so a report can say which cause withheld a label
+
+`disputed` carries two causes a reader should act on differently: the readings
+named different papers (one of them is wrong, somebody should look), or
+nothing in them could be compared (no conflict is known, the withholding is
+precautionary). `RefManifest.labels_uncomparable` publishes the second as a
+**documented subset** of `labels_disputed` — a subset by construction, both
+read off one call — and every surface that used to name the disjunction now
+names the cause where one was measured. Behaviour is unchanged:
+`labels_disputed` remains the single list that drives withholding.
+
+Three states, not two. `null` is never computed — including every manifest
+written before the field existed — and `[]` is "computed, and every dispute
+was a contradiction". One empty list cannot mean both.
+
+### Changed — `numbering_corroborated` is `boolean | null`
+
+Its description had claimed three states for a type that held two, so "never
+computed" and "measured, not corroborated" were the same value. The type moved
+rather than the meaning: `true` and `false` mean exactly what they did, and
+`from_json` no longer coerces an absent field to `false`. Every consumer
+already treated falsy uniformly.
+
+### Fixed — a retrieval gap erased by a withholding
+
+`ClaimResult.unjudged_refs` is decided above every branch of the claim loop
+now. A claim citing one withheld label and one co-cited source that could not
+be retrieved put the unretrievable one in none of the three accounts a reader
+has — judged, withheld, unjudged. The one path it is deliberately empty on is
+a claim where nothing at all was obtained: the `not_retrieved` verdict is
+already the whole report there, and the field's published meaning is the
+*co*-cited labels a surviving verdict did not rest on.
+
+A disputed label whose source was never retrieved is also described as that,
+rather than as withheld: the report splits on the manifest entry's own status,
+the same predicate the retrieval filter applies, so it no longer asserts a
+fetch that did not happen.
+
+### Fixed — `seen_in` could name a reading the same manifest said disagreed
+
+`stamp_seen_in` matches through the agreement comparator, so only a positive
+comparison stamps and the two published fields can no longer contradict each
+other on one manifest. It runs *after* the interactive escalation, because
+adopting a reading whole replaces every entry and a stamp computed before that
+published `[]` for all of them; it merges rather than overwrites, so an entry
+the resolution substituted keeps the printed text its values were copied from.
+`[]` now means one thing: no reading was established as carrying this work.
+
+The model's reading is excluded from `corroborating_readings` entirely. It may
+only copy values out of the two extractions, and both of those vote in their
+own right, so crediting it counts one text twice. It still votes, and a
+disagreement from it is still real.
+
### Added — `ask.py`, the one seam, with the model recorded per call site
Every model call went through one `_ask` in `check.py`, which was a convention
@@ -196,10 +286,12 @@ because three of its references are cited only in the supplement.
`--llm-refs` defaults on, so a fresh demo run now makes one extra model call
and, on the docling backend the demo uses, emits the `reflist` disclosure in
all four formats. The committed showcase in `examples/demo/output/` was
-generated before this feature existed and shows none of it. That is staleness
-in the committed artefact, not a defect in the feature — the showcase needs a
-fresh end-to-end run (network, a logged-in `claude` CLI) to pick it up, and
-that run is a branch-level step, not part of this change.
+generated before this feature existed and shows none of it — nor the report
+wording this release changed around disputes, resolutions and corroboration.
+That is staleness in the committed artefact, not a defect in the feature: the
+showcase needs a fresh end-to-end run (network, a logged-in `claude` CLI) to
+pick it up, and that run is **deferred by the user**, not skipped silently. It
+is a branch-level step and must happen before this version is released.
## [0.6.0] — 2026-09-13 (beta)
diff --git a/CLAUDE.md b/CLAUDE.md
index 0b6b99e..bd5e6c6 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -27,6 +27,26 @@ Every subsystem encodes this, and most past bugs have been breaches of it:
- A flat-text ingest fallback stamps `converter: pymupdf` into the source map so
the fidelity loss appears in the report.
+Two rules about the wire format follow from it, both learned the expensive way
+on 0.7.0:
+
+- **A published field's meaning is never widened to fix a misleading
+ sentence.** When `numbering_verified` (extent: the chosen list accounts for
+ exactly the cited labels) read as "confirmed" beside a line withholding
+ verdicts for a disputed label (content), the fix was the *sentence*, not the
+ flag — four report formats already read that boolean, and stretching it to
+ cover content would have silently changed what every one of them asserts.
+ Add a second axis (`numbering_corroborated`, `labels_uncomparable`) or
+ reword; do not redefine.
+- **A zero or an empty value must not mean both "measured zero" and "never
+ measured".** `labels_uncomparable` is `None` when nothing computed it and
+ `[]` when every dispute was measured and every one was a contradiction;
+ `numbering_corroborated` is `bool | None` for the same reason, after its
+ docstring claimed three states for a type that held two. The report picks a
+ cause only where one was actually measured, and `evals/DESIGN.md` says the
+ same thing about a rate: *"Absent is not zero, and the two must never render
+ the same way."*
+
When you add a feature, ask what it does when it fails. If the answer is "falls
back to something reasonable", that is a bug in this codebase.
@@ -147,26 +167,70 @@ ingest → refs → scout → check → highlight → report
**Agreement is per printed label, and disagreement is refused rather than
ranked** (0.7.0). `reconcile` arbitrates between the Crossref deposit and the
run backend's parse, unchanged; a flat-text pymupdf reading and a model
- reading (`reflist.py`, `--llm-refs`) are **voters only** and reach neither
- `reconcile`'s arguments nor `resolve_all`, so no model output can cause a
- source to be resolved, downloaded or judged — only a verdict to be withheld.
+ reading (`reflist.py`, `--llm-refs`) are **voters only** — neither reaches
+ `reconcile`'s arguments nor `resolve_all`, so on their own they cannot cause
+ a source to be resolved, downloaded or judged: they withhold a verdict, and
+ (through `stamp_seen_in`) name themselves on an entry they also carried.
+ **Do not restate that as "no model output can": one path escapes it and is
+ gated on a person.** `_escalate_disputed` menu option 1 makes a *second*
+ model call (`reflist.resolve_disputed`) and substitutes each resolved entry
+ into the chosen list, which `resolve_all` downloads and `check.py` judges.
+ The substitution is mandatory, not optional: returning `entries` untouched
+ un-disputes the label while leaving the entry the resolution ruled *against*
+ as the paper judged, which is the original wrong-paper bug reached through
+ the one path a user authorised. What keeps it honest is the gate, and the
+ gate is what may not be relaxed — an interactive run, the full disagreement
+ on disk before anything is asked, `numbering_chosen_by: "user"`, only the
+ labels actually resolved substituted, and no answer setting
+ `numbering_verified`.
`label_agreement` joins on `e.num`, the printed numeral, never on position:
`_first_divergence` zips, and would compare docling's 6th entry against
- pymupdf's 6th and report divergence for the wrong reason. Any pair failing
- `_same_work` is `disputed` — no majority vote, because *"a non-unique match is
- refused, never ranked"* — and `single` deliberately **prints**, since a
- pymupdf-backend run whose model candidate was discarded has one reading, every
- label would be `single`, and the audit would report nothing at all. A
- `boundary_ambiguous` entry does not speak for its label and neither does a
+ pymupdf's 6th and report divergence for the wrong reason. Any pair
+ `_comparably_same` answers `False` for makes the label `disputed` — no majority vote,
+ because *"a non-unique match is refused, never ranked"* — and `single`
+ deliberately **prints**, since a pymupdf-backend run whose model candidate
+ was discarded has one reading, every label would be `single`, and the audit
+ would report nothing at all. **`_same_work` is deliberately not reused here.**
+ It returns True when either side has no comparable title — right for
+ `_first_divergence`, where silence must not manufacture a divergence — and
+ here it once reported two demonstrably unrelated papers as `agreed`, which is
+ the one state that lets a verdict through. `_comparably_same` asks the mirror
+ question and is three-valued: `None` is *cannot tell*, and a voter it answers
+ `None` for **abstains** rather than dissenting. Collapsing that to `False`
+ made a Crossref deposit of bare DOIs dispute every label it voted on while
+ the two parses agreed perfectly — an audit with no verdicts in it. The one
+ exception is a label where *no* voter said anything comparable at all
+ (`_says_something_comparable`): that is `disputed`, because nothing
+ establishes what the label names. So `disputed` has three causes — duplicate,
+ contradiction, nothing comparable — `_label_state` returns which, and
+ `labels_uncomparable` publishes the third as a subset of `labels_disputed`
+ **by construction**, both read off the same call. Never compute the subset
+ from a second pass, and re-intersect it after every mutation: the escalation
+ can take a label out of `labels_disputed`, and a subset computed before it
+ then names a label the manifest no longer disputes.
+ A `boundary_ambiguous` entry does not speak for its label and neither does a
reading carrying that label twice: the first has said it cannot stand behind
- the label, and for the second, which of the two it means is the question.
+ the label, and for the second, which of the two it means is the question. The
+ model's reading may not **corroborate** at all (`DERIVED_READINGS`):
+ `reflist.propose` may only copy values out of the two extractions, both of
+ which vote in their own right, so crediting it is one text counted twice.
Every field of the model's reply must be found **verbatim** in one of the two
- texts it was shown or it is discarded, and an unverifiable title discards the
- answer — nothing invented may name a paper. `numbering_corroborated` is a
- second, independent axis: `numbering_verified` keeps its exact meaning and
- stays `False` through every model reply and every interactive choice, which
+ texts it was shown or it is discarded, and one entry's unverifiable title
+ discards the model's **whole reading**, never just that entry — nothing
+ invented may name a paper. `numbering_corroborated` is a second, independent
+ axis: `numbering_verified` keeps its exact meaning and stays `False` through
+ every model reply and every interactive choice, which
`tests/test_numbering_invariant.py` parametrises over and which is the one
thing in this feature that may not be relaxed for convenience.
+ `models._title_tokens` strips DOIs the way it strips URLs, and for the reason
+ the URL strip's own docstring gives: an identifier's substrings are not words
+ anybody wrote as a title. A DOI-only Crossref deposit (`_reference_raw`'s
+ documented fallback, and the majority of the deposit on the paper that
+ prompted this) was contributing `radiol` and `jamanetworkopen` as title
+ words, which disputed every label it voted on and pushed `_title_check_text`
+ toward `mismatch` on correct retrievals. Stripping it is global, not scoped
+ to the comparator: `_same_work` and `_title_check_text` both move, both
+ toward honesty, and both movements were measured before the strip landed.
- **`ask.py`** — the **only** file in `src/` that runs a subprocess, and the
only place this codebase shells out to a model (`claude -p --safe-mode
--tools ""` in a private scratch cwd; inherits the user's Claude Code login,
@@ -174,23 +238,27 @@ ingest → refs → scout → check → highlight → report
until `refs` needed a reading of the bibliography too — the rule was
protecting the seam, not the module, and `tests/test_ask.py` greps `src/` and
asserts exactly one file, which makes it enforceable rather than
- conventional. Two callers, `check.py` and `refs.py` (through `reflist.py`),
- and no third without that test going red. The model is recorded **per call
+ conventional. Two callers, `check.py` and `reflist.py` (driven by the `refs`
+ stage in `cli.py` — `refs.py` itself imports neither), and no third without
+ that test going red. The model is recorded **per call
site** (`for_site`, `model_for`, `SITE_CHECK`, `SITE_REFS`), not in one
global: `_LAST_MODEL` was overwritten by every call, so a run whose judging
made zero calls — every cited source `not_retrieved`, nothing to judge —
printed the reference-list model as the `Checker:` of verdicts it never saw.
- `_ask`'s signature is frozen at `(prompt, model=None)`; sixty-two test sites
- patch it with a two-argument lambda, which is why the site travels out of
- band in a context manager instead of as a third parameter.
+ `_ask`'s signature is frozen at `(prompt, model=None)`; every offline test
+ that patches the seam does so with a two-argument lambda, which is why the
+ site travels out of band in a context manager instead of as a third
+ parameter. **No integer here**: the count moves with every test added or
+ removed, and `ask.py` and `reflist.py` each still carry a stale one in a
+ comment.
- **`check.py`** — every prompt and every verdict rule, and no subprocess of
its own. Two prompts: `EXTRACT_PROMPT` then `CHECK_PROMPT`, one call per
**document** so context stays small — an article, each of its supplements,
and each of the audited paper's own are separate calls with separate
verdicts. Call the bare name `_ask(...)`, imported `from .ask import _ask`, so
`monkeypatch.setattr(check_mod, "_ask", …)` still intercepts; rewriting a
- call site as `ask._ask(...)` bypasses every patch and turns sixty-two offline
- tests into live paid calls. `_ask_with_retry` wraps the seam and reads
+ call site as `ask._ask(...)` bypasses every patch and turns the offline suite
+ into live paid calls. `_ask_with_retry` wraps the seam and reads
`ASK_ATTEMPTS` rather than hardcoding one retry — the wizard prints a
worst-case bill derived from that constant. Also holds `coverage_audit()`,
which is deliberately **mechanical and prompt-independent** — a regex
diff --git a/README.md b/README.md
index ad75f87..23fa671 100644
--- a/README.md
+++ b/README.md
@@ -103,7 +103,9 @@ went uncited?**
authority over neither (`--no-llm-refs` turns it off) — and the manuscript's
own `[N]` markers arbitrate between them. Only the parse and the deposit can
be *adopted* as the list; the other two are readings that can disagree, and a
- disagreement is all they can do. Every field of the model's reply is checked
+ disagreement is all they can do on their own — the one thing that can change
+ which paper is fetched is your own answer at the prompt described below, and
+ the report records that you gave it. Every field of the model's reply is checked
against the text it was shown and discarded if it is not printed there, so
nothing it invented can name a paper. A reading is used only if it accounts
for exactly the labels the body cites. When neither does, the audit continues,
@@ -129,17 +131,32 @@ went uncited?**
disagree is a label nothing can say the cited paper for — and a confident
verdict there is about a paper the manuscript may never have cited. Agreement
is computed for each printed label separately, on the numeral the page
- carries. Any disagreement withholds: the claim is reported `⚠ not checked`
+ carries. Any disagreement withholds **that source**: it is dropped before any
+ model call, and a claim left with no other source is reported `⚠ not checked`
with the label named, **not** `⊘ not retrieved`, because that source was
obtained and read — what is in doubt is whether it is the paper the label
- names. Where nothing accounted for the body's labels and labels are in
- dispute, an interactive run writes the whole disagreement to
+ names. A claim that also cites a source nobody disputes is still judged on
+ that source, and the withheld label is named beside the verdict. Two
+ different things put a label in dispute — the readings named different
+ papers, or nothing in them could be compared (a Crossref deposit of bare
+ DOIs, say) — and the report says which, per label, rather than leaving you
+ the disjunction: the first means one reading is wrong and somebody should
+ look, the second means no conflict is known and the withholding is
+ precautionary. Where the readings are in dispute at any label, an interactive
+ run writes the whole disagreement to
`case/out/reference_disagreement.md` — every reading's fields and the
verbatim printed text each came from — prints the path, and only then asks
- whether to resolve them, withhold them, adopt one reading, or stop. Nothing
- you answer marks the numbering as confirmed; the report records which choice
- was made and that you made it. A run that is not attached to a terminal
- withholds the disputed labels and asks nothing.
+ whether to resolve them, withhold them, adopt one reading, or stop.
+ *Resolve* is the one answer that changes which papers get fetched: a second
+ model call is shown the printed list and the entries that disagree, every
+ field of its answer is checked verbatim against the printed text, and each
+ label it settles has its entry substituted into the list that is then
+ downloaded and judged. Labels it cannot settle stay disputed and withheld —
+ partial resolution is the normal outcome. Nothing you answer marks the
+ numbering as confirmed; the report records which choice was made, that you
+ made it, which model was asked and which extractions it was shown. A run that
+ is not interactive — no terminal, output piped, or `CI` set — withholds the
+ disputed labels and asks nothing.
- Report every citation **occurrence** — each bracketed marker at its own place
in the text — that no extracted claim reached, so a second sentence citing an
already-checked reference is not silently counted as covered. It also
@@ -196,9 +213,14 @@ went uncited?**
**PaperTrace does not**
- **Confirm** a reference numbering by agreement between its readings. Where
- they agree, the report says the numbering was *corroborated*, and says which
- readings corroborated it — never that it was verified. They all read the same
- document: a reference the page layout destroyed is one every reading may have
+ every cited label was agreed by at least two readings, and at least two
+ readings agreed on *all* of them, the report says the numbering was
+ *corroborated* and names those readings — never that it was verified. Where
+ every label was agreed but no single reading spans them all, it says that
+ instead, in as many words: that is short of corroboration, not a stronger
+ version of it. The model's own reading never counts toward either, because it
+ may only copy values out of the two extractions that already vote. They all
+ read the same document: a reference the page layout destroyed is one every reading may have
missed, and a model agreeing with the parse read the same damaged text the
parse did. *"Verified"* still means one thing only — the manuscript's own
`[N]` markers accounted for exactly the labels a candidate list carried — and
diff --git a/docs/adr/0003-llm-reference-list.md b/docs/adr/0003-llm-reference-list.md
index e330740..8b8e77b 100644
--- a/docs/adr/0003-llm-reference-list.md
+++ b/docs/adr/0003-llm-reference-list.md
@@ -24,8 +24,8 @@ A manuscript under review had 22 printed references parse as 18. Docling
renders a hanging-indent numeral column as a GFM table, so five reference rows
arrived as `| 6. | Author A (2019) … |` and were glued onto the entry above
them as wrapped continuations. Every claim citing `[6]` or above was judged
-against a different paper than the one cited — the citation label is the join
-key, so the verdicts were confident and wrong.
+against a different paper than the one cited — six substantive verdicts naming
+papers the manuscript never cited.
The verdicts were not merely wrong, they were **confident**. Gluing five
references onto one entry left `_entry` scraping the next reference's DOI onto
@@ -54,19 +54,40 @@ with no vote of its own — behind `--llm-refs/--no-llm-refs`, default on.**
The reconciliation now weighs four readings of one bibliography: the Crossref
deposit, the run backend's parse, a flat-text (pymupdf) parse of the same PDF,
and a structured list proposed by a model. `reconcile()` itself is unchanged
-and still takes exactly two arguments — the deposit and the parse — because
-only those two can be *adopted* as the list a source is resolved and downloaded
-against. The flat-text and model readings never reach `reconcile`'s arguments
-or `resolve_all`; they are folded into a second, orthogonal computation,
-`label_agreement`, whose only power is to withhold a verdict. Five constraints
-define the shape, and each is the answer to a way this could have gone wrong:
-
-1. **The model's reading can only subtract verdicts.** `reconcile` still
- chooses between the deposit and the parse. The flat-text and model readings
- are voters only, so no model output can cause a source to be resolved,
- downloaded or judged — it can only cause a verdict to be withheld. This is
- the safety property the design rests on and
- `tests/test_numbering_invariant.py` tests it directly.
+and still weighs exactly two candidate readings — the deposit and the parse —
+because only those two can be *adopted* as the list a source is resolved and
+downloaded against. The flat-text and model readings never reach `reconcile`'s
+arguments or `resolve_all`; they are folded into a second, orthogonal
+computation, `label_agreement`, whose only power is to withhold a verdict. One
+gated path does more than that, and constraint 1 states it rather than leaving
+it to be discovered. Five constraints define the shape, and each is the answer
+to a way this could have gone wrong:
+
+1. **The model's reading can only subtract verdicts — unless a person is shown
+ the disagreement and asks for more.** `reconcile` still chooses between the
+ deposit and the parse. The `--llm-refs` reading and the flat-text reading
+ are voters only: neither reaches `reconcile`'s arguments or `resolve_all`,
+ so on their own they cannot cause a source to be resolved, downloaded or
+ judged — they can withhold a verdict, and (through `stamp_seen_in`) name
+ themselves on an entry they also carried. That is the safety property the
+ design rests on, and
+ `tests/test_reference_readings.py::test_a_model_naming_different_papers_changes_no_resolved_entry`
+ tests it directly, as an equality plus a capturing stub over `resolve_all`.
+
+ **One path escapes it, and it is the path this feature spends money to
+ reach.** Menu option 1 of the escalation asks a *second* model call
+ (`reflist.resolve_disputed`) to settle the disputed labels, and
+ `cli._escalate_disputed` substitutes each resolved entry into the chosen
+ list, which `resolve_all` then downloads and `check.py` then judges. The
+ substitution is mandatory rather than optional, and the reason sits in a
+ comment at the substitution itself: returning the list untouched would take
+ the label out of `labels_disputed` — so `check` judges it again — against the very entry
+ the resolution ruled *against*, which is the original wrong-paper bug
+ reached through the one path a person authorised. The gate on it is the
+ person: the run is interactive, the full disagreement is on disk before
+ anything is asked, the choice is recorded as `numbering_chosen_by: "user"`,
+ only the labels actually resolved are substituted (the rest stay disputed
+ and withheld), and no answer sets `numbering_verified`.
2. **Every field of the model's reply is found verbatim in one of the two
texts it was shown, or discarded.** A discarded field costs a recorded gap;
an accepted field is one a reader can grep out of the source text. Nothing
@@ -84,13 +105,37 @@ define the shape, and each is the answer to a way this could have gone wrong:
this codebase and is rejected again here.
4. **Any disagreement is `disputed`. No majority vote, no ranking.** The
codebase's own rule: *"a non-unique match is refused, never ranked."* A
- disputed label is dropped from the sources `check.py` judges, and the claim
- lands `unchecked` with a note — explicitly not `not_retrieved`, which would
- claim a retrieval failure that did not happen.
+ disputed label is dropped from the sources `check.py` judges before any
+ model call is made, and it is named in `ClaimResult.withheld_refs`. A claim
+ left with *no other* source then lands `unchecked` with the label named —
+ explicitly not `not_retrieved`, which would claim a retrieval failure that
+ did not happen. A claim that also cites a source nobody disputes is still
+ judged on that source, and the withheld label is named beside the verdict:
+ the *source* is always withheld, the *claim* is `unchecked` only when
+ nothing else answered for it.
+
+ The comparison behind `disputed` is three-valued
+ (`refs._comparably_same → True | False | None`), so *"cannot compare"* is
+ not *"the readings disagree"*: a voter whose comparison is `None` abstains
+ and is excluded rather than counted as dissent, which can leave one reading
+ standing and print `single`. The one exception is a label where **no** voter
+ said anything comparable at all — that is `disputed`, because nothing
+ establishes what the label names. `disputed` therefore carries three causes
+ — a duplicate, a contradiction, and nothing comparable — and
+ `RefManifest.labels_uncomparable` publishes the third as a documented
+ subset of `labels_disputed`, so a report can say *which* cause withheld a
+ label instead of naming a disjunction. It is a subset by construction: both
+ lists are read off the same `refs._label_state` call.
5. **`numbering_verified` keeps its exact current meaning and stays `False`
through every model reply and every interactive choice.** Corroboration is
a **second, independent axis** (`numbering_corroborated`), reported beside
it. No published boolean is redefined and no model path can flip one.
+ `tests/test_numbering_invariant.py` is the module that tests this one, over
+ every reply shape and every interactive choice, from both starting values.
+ The model's own reading may not *corroborate* at all (`refs.DERIVED_READINGS`):
+ `reflist.propose` may only copy values out of the two extractions, and both
+ of those vote in their own right, so the model agreeing with one of them is
+ one text read twice. It still votes, and a disagreement from it is real.
Four alternatives were weighed and rejected:
@@ -101,9 +146,11 @@ Four alternatives were weighed and rejected:
numeral is what the converter destroyed. It also has no deterministic
verifier, so a false "match" would overwrite a correct
`numbering_verified = False` with a plausible confirmation — the one thing
- CLAUDE.md forbids. It is moot besides: the real docling table block has
- `regions=1` for all five glued references, so no per-entry rectangle exists
- to crop.
+ CLAUDE.md forbids. It is moot besides: on the manuscript that prompted this,
+ the docling table block was measured at `regions=1` for all five glued
+ references (recorded in the design spec named above, not reproducible from
+ this repository — gate 3 forbids committing the manuscript), so no per-entry
+ rectangle exists to crop.
- **The LLM as arbiter.** The body's own `[N]` markers are the only reading
definitionally right about what the paper cites. Replacing a free, offline,
definitionally-correct arbiter with a probabilistic one is a strict loss.
@@ -111,13 +158,17 @@ Four alternatives were weighed and rejected:
entirely on some entries, so the moment one list is short by even one field,
a `num` from one reading would be stapled to another reading's `doi` at the
wrong position.
-- **Images through the seam.** `claude -p` has no image channel. Granting it
- one would dismantle `--tools ""` and the private scratch cwd, and would
- falsify the README's statement that the judge never receives a page image.
+- **Images through the seam.** Nothing but the prompt crosses it: `ask._ask`
+ runs `claude -p --output-format json --safe-mode --tools ""` with an optional
+ `--model`, the prompt on stdin, and no file, path or attachment argument
+ anywhere in `src/`. Opening an image channel would mean dismantling
+ `--tools ""` and the private scratch cwd, and would falsify the README's
+ statement that the judge never receives a page image.
`ask.py` is extracted as part of this decision. CLAUDE.md's rule *"`check.py`
is the only module that calls a model"* was protecting the seam, not the
-module; with `refs` (through `reflist.py`) calling it too, the rule becomes
+module; with the `refs` stage (through `reflist.py`) calling it too, the rule
+becomes
*"one file in `src/` shells out, and a test says which"* — enforceable rather
than conventional (`tests/test_ask.py::test_only_ask_py_shells_out_to_a_model`).
It also fixes a live misattribution: the model that answered was one module
@@ -145,12 +196,25 @@ specified as a sibling task in `evals/DESIGN.md` and is not part of this ADR.
expensive one: the cost and the safety of this join are two different
metrics in `evals/DESIGN.md`'s sibling pairing task, reported separately and
never blended.
-- Where nothing accounted for the body's labels and labels are in dispute, an
- interactive run writes the full disagreement to
- `case/out/reference_disagreement.md` **before** it asks anything, and
- records `numbering_chosen_by: "user"` for whatever the user answers. A
- person consenting to proceed is an input, not evidence. A non-interactive
- run withholds the disputed labels and asks nothing.
+- Where the readings are in dispute at any label, an interactive run writes the
+ full disagreement to `case/out/reference_disagreement.md` **before** it asks
+ anything, and records `numbering_chosen_by: "user"` for whatever the user
+ answers. There is no second condition: `_escalate_disputed` returns early on
+ an empty `labels_disputed` and on nothing else, and a run whose numbering
+ passed the extent check can still have a disputed label — extent and content
+ are different questions and both can be true at once. A person consenting to
+ proceed is an input, not evidence. A non-interactive run withholds the
+ disputed labels and asks nothing.
+- Choosing *resolve* changes which paper is fetched. That is the one place a
+ model's answer reaches retrieval, it is gated on a person, and it is
+ disclosed as such: the run records `resolution_outcome`,
+ `resolution_model`, `resolution_fields_discarded` and `resolution_readings`
+ — the extractions the call was shown, named, never "both texts", since a
+ pymupdf-backend run has one printed span and the reading being ruled against
+ may have none. A resolution is a reading, not a confirmation.
- ADR 0002's decision is unchanged. GROBID is still not adopted and still not
benchmarked; this ADR takes the escalation ADR 0002 named, not the one it
- refused.
+ refused. One housekeeping note for whoever follows the link: 0002's Related
+ list points at a README section called *"Two independent readings"*, which
+ this change rewrote — the numbering bullet under **PaperTrace does** is where
+ that material now lives.
diff --git a/evals/DESIGN.md b/evals/DESIGN.md
index 35bac6b..5e9e175 100644
--- a/evals/DESIGN.md
+++ b/evals/DESIGN.md
@@ -52,12 +52,23 @@ denominator — deleting it would quietly make the set easier.
### A second unit, specified elsewhere: (citation label, printed reference entry)
+What follows is a **pointer plus the constraints that would otherwise be
+rediscovered wrongly** — not a specification this harness implements. The
+sibling plan owns the design; the paragraphs below exist so that plan cannot
+quietly reuse `align.py`, key a gold set on position, or blend two error
+directions. Nothing here is implemented, and the Status note at the top of
+this document governs every word of it.
+
The unit above is `(manuscript claim, cited source)` and it grades a model's
judgement. It cannot grade the join that *chooses* the source. That join broke
on a real manuscript — 22 printed references read as 18, and six substantive
verdicts naming papers the manuscript never cited — and every one of them
carried `title_check: verified`, because the glued reference string really did
-contain the downloaded paper's words alongside the cited paper's.
+contain the downloaded paper's words alongside the cited paper's. (That count
+is an incident on one manuscript, recorded in
+`docs/superpowers/specs/2026-09-16-llm-reference-list-design.md`; the
+manuscript is not in this repository and gate 3 forbids committing it. It is
+not a measurement of anything.)
So there is a second task, never folded into `judgment_accuracy`, whose unit is
`(citation label, printed reference entry)`. Deliberately **not**
@@ -68,7 +79,12 @@ keyed on position breaks whenever parsing changes.
Its two error directions are reported **separately and never blended**: labels
the parse got wrong and the tool confirmed anyway (the safety metric), and
-labels the parse got right and the tool refused (the cost). Superscript papers
+labels the parse got right and the tool refused (the cost). The cost direction
+has two causes that must not be blended either — the readings contradicted
+each other, or nothing in them could be compared — because the tool now
+publishes which (`labels_uncomparable`, a subset of `labels_disputed`) and a
+precautionary withholding on a bare-DOI deposit is a different finding from a
+real contradiction. Superscript papers
have no arbiter and cannot be scored at all; they are excluded with a reason and
counted, never deleted.
From 1a2bacd0bfff0f3549f908f7ec5e9d6c9218aebb Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Sat, 19 Sep 2026 09:44:05 +0200
Subject: [PATCH 41/44] docs: drop a count that drifts, and repair a link that
stopped naming anything
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The seam comment claimed sixty-two monkeypatch sites. That was raw grep hits on
`_ask`; the real figure measured 46 at review time and 48 a day later, because
it grows with every test that touches the seam. Three values for one claim is
the tell — the number is a maintenance cost pretending to be evidence, so the
comment now states the constraint without one.
ADR 0002 cross-referenced README §"Two independent readings" by title. 0.7.0
took the count from two readings to four and the section was renamed, so the
reference did not break loudly, it simply stopped naming anything. It now
points at the section that exists and forward to 0003, which is the
authorisation 0002 explicitly withheld.
---
docs/adr/0002-no-grobid.md | 9 +++++++--
src/papertrace/ask.py | 10 +++++++---
2 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/docs/adr/0002-no-grobid.md b/docs/adr/0002-no-grobid.md
index 543e608..6d8746c 100644
--- a/docs/adr/0002-no-grobid.md
+++ b/docs/adr/0002-no-grobid.md
@@ -2,8 +2,13 @@
- **Status:** accepted
- **Date:** 2026-09-05
-- **Related:** ADR [0001](0001-no-gold-benchmark.md), `src/papertrace/refs.py`,
- README §"Two independent readings"
+- **Related:** ADR [0001](0001-no-gold-benchmark.md),
+ ADR [0003](0003-llm-reference-list.md) — which is the authorisation this ADR
+ withheld, granted once the trigger below was met with evidence —
+ `src/papertrace/refs.py`, README §"What it does — and what it does not"
+ (this pointed at §"Two independent readings" until 0.7.0 took the count from
+ two readings to four and the section was renamed; the link did not break, it
+ simply stopped naming anything)
## Context
diff --git a/src/papertrace/ask.py b/src/papertrace/ask.py
index 8ef36e2..3a3bd39 100644
--- a/src/papertrace/ask.py
+++ b/src/papertrace/ask.py
@@ -35,9 +35,13 @@
SITE_REFS = "refs"
# which site the next `_ask` belongs to, and what model answered at each.
-# `_ask`'s signature stays `(prompt, model=None)` deliberately: sixty-two test
-# sites monkeypatch it with a two-argument lambda, and a third parameter would
-# break every one of them in order to record something the fakes never produce.
+# `_ask`'s signature stays `(prompt, model=None)` deliberately: the test suite
+# monkeypatches it with a two-argument lambda in dozens of places, and a third
+# parameter would break every one of them in order to record something the
+# fakes never produce. No count is given on purpose — it was written as 62,
+# measured at 46, then 48, because it grows with every test that touches the
+# seam, and a number that drifts with unrelated work is a maintenance cost
+# pretending to be evidence.
_SITE = "unknown"
_MODELS: dict[str, str] = {}
From 9918c177abb154634fd2d89bf52a3e9f7f857d03 Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Sat, 19 Sep 2026 09:45:36 +0200
Subject: [PATCH 42/44] tests: the numbering invariant, asserted on the
artifact the pipeline wrote
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The reply-shape half of this module built `candidates` and called `reconcile`
by hand, so "the model's reading never becomes one of `reconcile`'s arguments"
was enforced by the test's own source text. Handing the `llm` reading to
`reconcile` in `_refs_pipeline` — the design's central prohibition — left all
29 cells green. So did disabling `reflist._found`, and so did widening the
manifest line to `rec.verified or rec.corroborated`.
Every cell now drives `cli._refs_pipeline` with `ask._ask` patched per reply
shape and reads `refs_manifest.json`. `reflist.propose`, `resolve_disputed`,
`reconcile`, `label_agreement` and the manifest construction all run for real;
the disputed label is produced by the pipeline rather than assigned by the
test, and the raising-resolver cell raises at the seam instead of stubbing the
resolver. Each shape carries what it is allowed to have moved, because an
invariant asserted alone cannot tell "the reply changed nothing" from "the
reply never ran".
The grep splits on the first `=` that is not a comparison rather than
discarding any line containing one, and `corroborated` joins `_FORBIDDEN`; the
self-test plants all three spellings and requires all three caught.
---
tests/test_numbering_invariant.py | 691 ++++++++++++++++++------------
1 file changed, 425 insertions(+), 266 deletions(-)
diff --git a/tests/test_numbering_invariant.py b/tests/test_numbering_invariant.py
index a89874e..61fff28 100644
--- a/tests/test_numbering_invariant.py
+++ b/tests/test_numbering_invariant.py
@@ -1,330 +1,478 @@
-"""One flag staying False, across every reply shape and every interactive choice.
-
-`numbering_verified` means exactly one thing: the manuscript's own `[N]` markers
-accounted for exactly the labels a candidate reading carried. A model reading of
-the bibliography is not that, and neither is a person answering a prompt — so
-this module parametrises over every shape a model reply can take and every
-choice the escalation offers, and asserts the flag is unmoved by every one of
-them.
+"""One flag unmoved, through the real pipeline, read back off the artifact.
+
+`numbering_verified` has a published meaning (`schemas/refs_manifest.schema.json`):
+the chosen list accounts for exactly the labels the body cites, and no entry
+refuses its own label. It is a test of EXTENT. A model's reading of the
+bibliography is not that, and neither is a person answering a prompt — so this
+module runs `cli._refs_pipeline` over every shape a model reply can take and
+every choice the escalation offers, and asserts against the
+`refs_manifest.json` each run actually wrote.
+
+The seam on one side, the artifact on the other, and the production wiring in
+between. That shape is the whole point of the rewrite: the previous version
+built `candidates` and called `reconcile` by hand, so "the model's reading
+never becomes one of `reconcile`'s arguments" was enforced by the test's own
+source text rather than by the product's. Measured — handing the `llm` reading
+to `reconcile` in `_refs_pipeline`, the design's central prohibition, left all
+29 of its cells green. Never re-create the pipeline's wiring here; a cell that
+does can only ever confirm its own re-creation.
"Unmoved" is not "always False". `_covers` is an extent check, and a fixture
whose extent legitimately matches gets `verified = True` — asserting `False`
-there would assert an impossible value (the parametrisation task's own brief
-made exactly this mistake, in the same shape three assertions in
-`tests/test_escalation.py` had to be corrected for already: see the report for
-this task). So the reply-shape test below runs every reply against BOTH a
-fixture that legitimately verifies and one that legitimately does not, and
-checks that no reply — however it is shaped — moves either one off the value
-`_covers` alone determined.
-
-The companion test asserts `numbering_corroborated` CAN be True in the same
-run. The two are independent axes on purpose; a second axis that moves only
-when the first does is not a second axis, and the false-alarm corroboration
-exists to silence would still be firing.
+there asserts an impossible value (three assertions in
+`tests/test_escalation.py` had to be corrected for exactly that). So every
+reply and every choice is run against BOTH a fixture that legitimately
+verifies and one that legitimately does not, and the assertion is that neither
+moves off the value the extent check alone produced.
+
+A reply IS allowed to do one thing: withhold. So each shape also carries what
+it is allowed to have moved — `labels_disputed` and `numbering_corroborated` —
+because an invariant asserted alone cannot tell "the reply changed nothing"
+from "the reply never reached the code". `reflist_outcome == "read"` is
+asserted for the same reason, on every cell.
"""
import json
+import re
import sys
from pathlib import Path
+import pymupdf
import pytest
+import typer
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
from papertrace import ask as ask_mod # noqa: E402
from papertrace import cli as cli_mod # noqa: E402
from papertrace import reflist as reflist_mod # noqa: E402
-from papertrace.models import RefEntry, RefManifest # noqa: E402
-from papertrace.refs import ( # noqa: E402,E501
- LABEL_AGREEMENT,
- corroborating_readings,
- label_agreement,
- reconcile,
-)
-
-# Reading A and reading B of one bibliography, as text. Every fixture DOI has a
-# 4-9 digit registrant: `10.1/a` does not match `DOI_RE` and a plan already lost
-# a round to that.
-_TEXT_A = (
- "1. Alpha A. First paper on registration. 2020. doi:10.1000/alpha\n"
- "2. Beta B. Second paper on segmentation. 2021. doi:10.1000/beta\n"
-)
-_TEXT_B = (
- "1. Alpha A. First paper on registration. 2020. doi:10.1000/alpha\n"
- "2. Gamma G. A wholly different paper. 2019. doi:10.0002/b\n"
-)
-
-_BODY = {"1", "2"}
-
-
-def _reading(pairs: list[tuple[str, str, str | None]]) -> list[RefEntry]:
- return [RefEntry(num=n, raw=raw, doi=doi) for n, raw, doi in pairs]
-
-
-def _parsed() -> list[RefEntry]:
- return _reading([
- ("1", "Alpha A. First paper on registration. 2020.", "10.1000/alpha"),
- ("2", "Beta B. Second paper on segmentation. 2021.", "10.1000/beta"),
- ])
-
-
-def _flat() -> list[RefEntry]:
- return _reading([
- ("1", "Alpha A. First paper on registration. 2020.", "10.1000/alpha"),
- ("2", "Gamma G. A wholly different paper. 2019.", "10.0002/b"),
- ])
-
-
-# Every shape a reply can take, named by what is wrong with it. `agrees` is in
-# here deliberately: the dangerous case is not the malformed reply — that fails
-# loudly — it is the well-formed one that confirms the parse, because that is
-# the reply a reader is most tempted to promote to a check.
-REPLIES = {
- "agrees_with_the_parse": json.dumps([
- {"num": "1", "title": "First paper on registration", "doi": "10.1000/alpha"},
- {"num": "2", "title": "Second paper on segmentation", "doi": "10.1000/beta"},
- ]),
- "agrees_with_nothing": json.dumps([
- {"num": "1", "title": "A title printed in neither reading"},
- {"num": "2", "title": "Nor is this one"},
- ]),
- "abstains": json.dumps([{"num": "1", "cannot_tell": True},
- {"num": "2", "cannot_tell": True}]),
- "malformed_json": "here is what I think: [1] is Alpha, probably",
- "empty_array": "[]",
- "title_printed_nowhere": json.dumps([
- {"num": "2", "title": "Characterization of Brain Volume Changes",
- "doi": "10.1000/beta"},
- ]),
- "truncated_doi": json.dumps([
- {"num": "2", "title": "Second paper on segmentation", "doi": "10.1038/s41591-"},
- ]),
- "a_label_nobody_cites": json.dumps([
- {"num": "99", "title": "Second paper on segmentation"},
- ]),
+from papertrace import refs as refs_mod # noqa: E402
+
+SCHEMA_PATH = Path(__file__).resolve().parent.parent / "schemas" / "refs_manifest.schema.json"
+
+
+def _paper(path: Path, *, extra_uncited: bool = False) -> Path:
+ """A one-page PDF citing [1] and [2], with both references printed.
+
+ `extra_uncited=True` prints a THIRD, uncited [3]: the parse then carries
+ three entries against two cited labels, `_covers` requires equality, and
+ `verified` is correctly False. Without that flag `_covers` is satisfied by
+ the parse alone and `verified` is correctly True. The two together are the
+ only way a cell can show a reply moved the flag in EITHER direction.
+
+ Written out again rather than imported from `tests/test_reference_readings.py`:
+ `tests/` has no `conftest.py` by house rule and no test module imports
+ another, so the alternative is not a shared fixture but a hidden
+ dependency between two files that are meant to fail independently.
+ """
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "A Fixture Imaging Study", fontsize=16)
+ page.insert_text((72, 140), "Body text citing [1] and also [2] here.", fontsize=11)
+ page.insert_text((72, 200), "References", fontsize=14)
+ page.insert_text((72, 230), "[1] Alpha A. A first paper. J Fixture. 2020;1:1-9.", fontsize=11)
+ page.insert_text((72, 250), "[2] Beta B. A second paper. J Fixture. 2021;2:10-19.", fontsize=11)
+ if extra_uncited:
+ page.insert_text((72, 270), "[3] Gamma G. A third, uncited paper. J Fixture. 2019;3:1-5.",
+ fontsize=11)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ doc.save(path)
+ doc.close()
+ return path
+
+
+# What the printed list carries, per extent: the numerals, and the opening of
+# each entry's `raw`. A model-composed entry is re-assembled from verified
+# FIELDS (`reflist.propose`: authors, year, title, journal, doi joined by ". "),
+# so it can never reproduce the printed string — which is what makes these two
+# a check that the manifest's entries came off the page and not out of a reply.
+_PRINTED = {
+ "covers": (["1", "2"], ["Alpha A.", "Beta B."]),
+ "extra_reference": (["1", "2", "3"], ["Alpha A.", "Beta B.", "Gamma G."]),
}
-# Two extents, both legitimate. "covers" is `_parsed()` unchanged: it carries
-# exactly the body's two labels, so `_covers` passes and `verified` is
-# correctly True. "extra_reference" adds one entry the body never cites (the
-# 22-vs-19 shape: a reference cited only in a supplement) — `_covers` requires
-# `len(entries) == max(body)` exactly, so this one fails it and `verified` is
-# correctly False. Both are computed once, by the real `reconcile`, rather than
-# asserted from outside — a hand-picked expectation here would be exactly the
-# mistake this module exists to catch elsewhere.
-_EXTRA = RefEntry(num="3", raw="Delta D. Cited only in the supplement. 2022.")
+# The marker that tells the two model calls apart inside one `ask._ask` double.
+# Asserted to be real, and to be unique, by its own test below — a double that
+# mis-routes would answer the resolution call with a reading reply and every
+# cell would still be green.
+_RESOLVE_MARKER = "LABELS IN DISPUTE:"
+
+
+def _seam(reading: str, resolution: str = "[]"):
+ """One `ask._ask` standing in for both calls the refs stage can make.
+
+ `reflist.py` has no module-level `_ask` of its own — it does `from . import
+ ask` and calls `ask._ask` qualified (`reflist.py:267-269` says so), so a
+ monkeypatch on `reflist._ask` sets an attribute nothing reads and lets the
+ real `claude -p` subprocess run. This module patches `ask._ask` and nothing
+ else, which is also what keeps `reflist.propose` and `reflist.resolve_disputed`
+ themselves REAL: every verbatim check they perform is exercised by these
+ cells rather than stubbed past.
+ """
+
+ def _ask(prompt, model=None):
+ return resolution if _RESOLVE_MARKER in prompt else reading
+
+ return _ask
-def _extent_parsed(extent: str) -> list[RefEntry]:
- return _parsed() + [_EXTRA] if extent == "extra_reference" else _parsed()
+def _wire_offline(monkeypatch, *, reading: str, resolution: str = "[]", ask=None) -> None:
+ """No network and no model call: retrieval is a no-op, Crossref answers
+ nothing, and `claude` is treated as present regardless of the host.
+
+ That last patch is load-bearing even though the seam below is what answers:
+ `_llm_reference_reading` checks `ask.claude_available()` BEFORE calling
+ `propose`, so on a machine with no `claude` CLI on PATH — CI, but not a
+ developer's laptop — every cell would silently take the "not attempted"
+ branch and the reply would never be read at all.
+
+ `_needs_flat_reading` is forced rather than the backend changed. These
+ tests ingest with `backend="pymupdf"` for speed (docling downloads ~500 MB
+ on first use), and on a genuine pymupdf run `_refs_pipeline` skips the
+ model call by design — so without this patch there is no reply to test.
+ Force the PREDICATE, not the backend.
+ """
+ monkeypatch.setattr(refs_mod, "resolve_all",
+ lambda entries, dest, email, provided_dir=None, progress=None,
+ taken=None: entries)
+ monkeypatch.setattr(refs_mod, "crossref_deposit",
+ lambda client, doi, email: refs_mod.CrossrefDeposit(
+ absent="no deposit, in a test"))
+ monkeypatch.setattr(cli_mod, "_detected_doi", lambda m: None)
+ monkeypatch.setattr(ask_mod, "claude_available", lambda: True)
+ monkeypatch.setattr(cli_mod, "_needs_flat_reading", lambda smap: True)
+ monkeypatch.setattr(ask_mod, "_ask", ask or _seam(reading, resolution))
+
+
+def _run(tmp_path: Path, monkeypatch, *, extent: str, reading: str,
+ resolution: str = "[]", ask=None) -> Path:
+ """Run the real refs stage on the fixture paper; return the case folder.
+
+ The case folder, not the manifest: menu choice 4 aborts before a manifest
+ is written, and "no manifest exists" is that cell's assertion.
+ """
+ _wire_offline(monkeypatch, reading=reading, resolution=resolution, ask=ask)
+ pdf = _paper(tmp_path / "paper.pdf", extra_uncited=(extent == "extra_reference"))
+ case = tmp_path / "case"
+ cli_mod._refs_pipeline(manuscript=pdf, case=case, provided=None,
+ email="test@example.org", parse_only=False,
+ backend="pymupdf", llm_refs=True)
+ return case
+
+
+def _manifest(case: Path) -> dict:
+ """The written artifact, schema-validated on the way out — Gate 2 for free."""
+ import jsonschema
+
+ payload = json.loads((case / "refs_manifest.json").read_text())
+ jsonschema.validate(payload, json.loads(SCHEMA_PATH.read_text()))
+ return payload
+
+
+# Every shape a reply can take, named by what is wrong with it, with what that
+# shape is ALLOWED to have moved. `agrees_with_the_parse` is in here
+# deliberately: the dangerous reply is not the malformed one — that fails
+# loudly — it is the well-formed one confirming the parse, because that is the
+# reply a reader is most tempted to promote to a check.
+#
+# `disputed` and `corroborated` are the only two things a reading may move, and
+# they are here rather than asserted uniformly because an invariant asserted
+# alone cannot tell "the reply changed nothing" from "the reply never ran". The
+# values were observed, not predicted: a shape whose entries all fail
+# verification never becomes a voter at all, so `parsed` and `pymupdf` carry
+# the corroboration by themselves.
+REPLIES = {
+ "agrees_with_the_parse": {
+ "reply": json.dumps([
+ {"num": "1", "title": "A first paper", "year": "2020",
+ "journal": "J Fixture", "reading": "AB"},
+ {"num": "2", "title": "A second paper", "year": "2021",
+ "journal": "J Fixture", "reading": "AB"},
+ ]),
+ "disputed": [], "corroborated": True,
+ },
+ # the one shape `reflist._found` exists for: a title nobody printed. The
+ # whole reading must be refused, so this reply casts no vote — if the
+ # verbatim check were disabled it would dispute [2] instead.
+ "names_a_paper_the_page_does_not_print": {
+ "reply": json.dumps([
+ {"num": "1", "title": "A first paper", "year": "2020"},
+ {"num": "2", "title": "Characterization of Brain Volume Changes", "year": "2021"},
+ ]),
+ "disputed": [], "corroborated": True,
+ },
+ "abstains": {
+ "reply": json.dumps([{"num": "1", "cannot_tell": True},
+ {"num": "2", "cannot_tell": True}]),
+ "disputed": [], "corroborated": True,
+ },
+ "malformed_json": {
+ "reply": "here is what I think: [1] is Alpha, probably",
+ "disputed": [], "corroborated": True,
+ },
+ "empty_array": {"reply": "[]", "disputed": [], "corroborated": True},
+ "numeral_only_no_fields": {
+ "reply": json.dumps([{"num": "1"}, {"num": "2"}]),
+ "disputed": [], "corroborated": True,
+ },
+ # `DOI_RE` accepts a trailing hyphen and the prefix really is printed, so
+ # only `_usable_doi`'s last-character rule tells half a DOI from a whole one
+ "truncated_doi": {
+ "reply": json.dumps([{"num": "2", "title": "A second paper",
+ "doi": "10.1038/s41591-"}]),
+ "disputed": [], "corroborated": True,
+ },
+ "a_label_nobody_cites": {
+ "reply": json.dumps([{"num": "99", "title": "A second paper"}]),
+ "disputed": [], "corroborated": True,
+ },
+ # the one shape that legitimately withholds: a reading carrying one label
+ # twice cannot say which paper it names, which is `label_agreement`'s own
+ # rule, and a withheld verdict is the ONLY thing a reply is allowed to cause
+ "duplicate_numeral": {
+ "reply": json.dumps([{"num": "1", "title": "A first paper"},
+ {"num": "1", "title": "A second paper"}]),
+ "disputed": ["1"], "corroborated": False,
+ },
+}
@pytest.mark.parametrize("extent", ["covers", "extra_reference"])
@pytest.mark.parametrize("shape", sorted(REPLIES))
-def test_no_model_reply_shape_can_set_numbering_verified(shape, extent, monkeypatch):
- """Parametrised over every model reply shape, on both a verified-True and a
- verified-False fixture.
-
- Most of this design's safety lives in one flag staying at whatever
- `reconcile`'s extent check put it at. The model's reading is a voter: it
- never becomes one of `reconcile`'s arguments (`reconcile(_BODY, None,
- parsed)` below never mentions `proposed`), so no reply can cause a source
- to be resolved, downloaded or judged, and no reply can move `verified` off
- the value the extent check alone produced — in either direction. The
- `covers` cell proves a reply cannot manufacture a False from a legitimate
- True; the `extra_reference` cell proves a reply cannot manufacture a True
- from a legitimate False, which is the direction that would actually be
- dangerous.
-
- Patches `ask._ask`, not `reflist._ask` — `reflist.py` has no such name of
- its own; it imports the whole `ask` module and calls `ask._ask` qualified.
- A monkeypatch on the wrong attribute sets an attribute nothing reads and
- lets the real subprocess seam run, which is exactly the live-model-call
- this suite's offline gate exists to forbid.
+def test_no_model_reply_shape_can_set_numbering_verified(shape, extent, tmp_path, monkeypatch):
+ """Every reply shape, through `_refs_pipeline`, asserted on the manifest.
+
+ The model's reading is a voter and nothing else: it never becomes one of
+ `reconcile`'s arguments, so no reply can cause a source to be resolved,
+ downloaded or judged, and no reply can move `numbering_verified` off the
+ value the extent check alone produced. The `covers` cell proves a reply
+ cannot manufacture a False from a legitimate True; the `extra_reference`
+ cell proves it cannot manufacture a True from a legitimate False, which is
+ the direction that would actually be dangerous.
+
+ `expected` is hardcoded from the extent rather than derived. Deriving it
+ from a second `reconcile` call would be tautological — it would assert the
+ extent check agrees with itself — and `_covers`' rule ("as many entries as
+ the highest cited label") is exactly what the two fixtures were built to
+ straddle.
"""
- monkeypatch.setattr(ask_mod, "_ask", lambda prompt, model=None: REPLIES[shape])
- proposed, prov = reflist_mod.propose(_TEXT_A, _TEXT_B, label_a="parsed", label_b="pymupdf")
- assert prov.outcome == "read" # the call happened; what came back is the point
-
- parsed = _extent_parsed(extent)
- candidates = {"parsed": parsed, "pymupdf": _flat(), "llm": proposed}
- states = label_agreement(candidates, _BODY)
-
- # the parse and the deposit are what reconcile arbitrates between; `llm` is
- # not passed, and a task that passes it has broken the design
- entries, rec = reconcile(_BODY, None, parsed)
- manifest = RefManifest(
- manuscript="p.pdf",
- entries=entries,
- numbering_verified=rec.verified,
- labels_disputed=[k for k, v in states.items() if v == "disputed"],
- )
+ case = _run(tmp_path, monkeypatch, extent=extent, reading=REPLIES[shape]["reply"])
+ payload = _manifest(case)
+
+ # the reply reached the code. Without this the cells below are satisfied by
+ # a run in which no model was ever asked, which is not what they claim.
+ assert payload["reflist_outcome"] == "read", f"{shape}: the reply never reached propose"
expected = extent == "covers"
- assert rec.verified is expected, f"{shape}/{extent} moved rec.verified"
- assert manifest.numbering_verified is expected, f"{shape}/{extent} moved the manifest"
- assert all(v in LABEL_AGREEMENT for v in states.values())
+ assert payload["numbering_verified"] is expected, f"{shape}/{extent} moved numbering_verified"
+
+ # withholding is the one thing a reply may cause, and only on a cause it
+ # demonstrated — a reply whose fields are not printed casts no vote at all
+ assert payload["labels_disputed"] == REPLIES[shape]["disputed"], shape
+ assert payload["numbering_corroborated"] is REPLIES[shape]["corroborated"], shape
+ assert "llm" not in payload["corroborating_readings"], (
+ "a model reading may only copy from the two extractions, which vote in "
+ "their own right — it can dispute, it cannot corroborate"
+ )
+
+ # the resolved list is the PRINTED one, entry for entry
+ nums, openings = _PRINTED[extent]
+ assert [e["num"] for e in payload["entries"]] == nums, shape
+ assert [e["raw"][:len(o)] for e, o in zip(payload["entries"], openings, strict=True)] == \
+ openings, f"{shape}: an entry in the manifest did not come off the page"
-@pytest.mark.parametrize("initial_verified", [True, False], ids=["was_verified", "was_unverified"])
+@pytest.mark.parametrize("extent", ["covers", "extra_reference"])
@pytest.mark.parametrize("choice", ["1", "2", "3", "4"])
-def test_no_interactive_choice_can_set_numbering_verified(choice, initial_verified, monkeypatch, tmp_path):
- """Parametrised over all four menu choices, including abort, on both a
- verified-True and a verified-False starting point.
+def test_no_interactive_choice_can_set_numbering_verified(choice, extent, tmp_path, monkeypatch):
+ """Every menu choice, through `_refs_pipeline`, asserted on the manifest.
+
+ The dispute is REAL: the `duplicate_numeral` reply makes `label_agreement`
+ refuse to let the model reading speak for [1], which is what puts the label
+ in `labels_disputed` and what makes `_refs_pipeline` escalate at all. No
+ `rec` is built by hand and no `labels_disputed` is assigned by the test.
`chosen_by: "user"` is what makes the escalation honest — a person
- consenting to proceed is an input, not evidence. So every branch of the
- menu must leave `rec.verified` no more true than `reconcile` left it: for
- choices 1, 2 and 4 that means exactly unchanged (nothing in their code
- paths touches `rec.verified` at all), and this is the test that says so for
- the branch a reviewer would be most tempted to "improve" — option 3, where
- the user positively asserts which reading is right. Option 3 is the one
- exception, and by design: it substitutes a whole different list than the
- one `reconcile` measured, so it explicitly resets `verified` to the literal
- `False` — never derived from the answer, never derived from which reading
- was picked — because the extent of the newly adopted list was never
- checked. That reset can only ever produce `False`; it is asserted here on
- both starting points to show it never produces `True`.
+ consenting to proceed is an input, not evidence. So every branch must leave
+ the flag no more true than `reconcile` left it. Option 3 is the one branch
+ that touches it, and by design in one direction only: it substitutes a
+ whole different list than the one `reconcile` measured, so it resets
+ `verified` to the literal `False`. Asserted from both starting values to
+ show the reset can never produce `True`.
+
+ Choice 4 aborts before the manifest is written, so its assertion is that no
+ manifest exists — and that the evidence file written before the menu still
+ does, because "nothing was written" means the manifest and nothing else.
"""
- import typer
-
- candidates = {"parsed": _parsed(), "pymupdf": _flat()}
- texts = {"parsed": _TEXT_A, "pymupdf": _TEXT_B}
- # `_parsed()` legitimately verifies (see the test above); overridden here
- # because this test is about `_escalate_disputed`'s own behaviour, not
- # `reconcile`'s — a disputed label and a verified extent are independent
- # findings, so the escalation must be exercised from both starting values.
- _entries, rec = reconcile(_BODY, None, _parsed())
- rec.labels_disputed = ["2"]
- rec.verified = initial_verified
-
- answers = iter([choice, "pymupdf"])
- monkeypatch.setattr(cli_mod, "_interactive", lambda: True)
- monkeypatch.setattr(cli_mod.Prompt, "ask", staticmethod(lambda *a, **k: next(answers)))
- monkeypatch.setattr(ask_mod, "_ask", lambda prompt, model=None: REPLIES["agrees_with_the_parse"])
+ answers = iter([choice, "parsed"])
+
+ def _prompt(*args, **kwargs):
+ answer = next(answers)
+ # the real menu refuses an answer it did not offer, and a double that
+ # does not is a double more permissive than the thing it stands for —
+ # the exact shape that hid a Critical earlier in this plan
+ assert answer in kwargs.get("choices", [answer]), (
+ f"the menu did not offer {answer!r}; it offered {kwargs.get('choices')}"
+ )
+ return answer
+ monkeypatch.setattr(cli_mod, "_interactive", lambda: True)
+ monkeypatch.setattr(cli_mod.Prompt, "ask", staticmethod(_prompt))
+
+ kw = dict(extent=extent, reading=REPLIES["duplicate_numeral"]["reply"],
+ # a resolution that settles [1] on a title printed on the page,
+ # so choice 1 takes the branch that substitutes an entry rather
+ # than the abstention branch, which touches less
+ resolution=json.dumps([{"num": "1", "title": "A first paper",
+ "year": "2020", "seen_in": "A"}]))
if choice == "4":
with pytest.raises(typer.Exit):
- cli_mod._escalate_disputed(tmp_path, rec, _parsed(), candidates, texts)
- else:
- cli_mod._escalate_disputed(tmp_path, rec, _parsed(), candidates, texts)
-
- expected = False if choice == "3" else initial_verified
- assert rec.verified is expected, f"menu choice {choice} moved rec.verified"
- manifest = RefManifest(
- manuscript="p.pdf",
- numbering_verified=rec.verified,
- numbering_choice=rec.choice,
- numbering_chosen_by=rec.chosen_by,
- labels_resolved=rec.labels_resolved,
- labels_disputed=rec.labels_disputed,
- )
- assert manifest.numbering_verified is expected
- if choice != "4":
- assert manifest.numbering_chosen_by == "user"
+ _run(tmp_path, monkeypatch, **kw)
+ assert not (tmp_path / "case" / "refs_manifest.json").exists(), (
+ "an abort must leave no manifest for a later stage to trust"
+ )
+ assert (tmp_path / "case" / "out" / "reference_disagreement.md").exists(), (
+ "the evidence file is written before the menu and survives the abort"
+ )
+ return
+
+ payload = _manifest(_run(tmp_path, monkeypatch, **kw))
+
+ expected = False if choice == "3" else (extent == "covers")
+ assert payload["numbering_verified"] is expected, f"choice {choice} moved numbering_verified"
+ assert payload["numbering_chosen_by"] == "user"
+ assert payload["numbering_choice"] == {"1": "llm_resolved", "2": "withheld",
+ "3": "parsed"}[choice]
-@pytest.mark.parametrize("initial_verified", [True, False], ids=["was_verified", "was_unverified"])
-def test_a_raising_resolver_cannot_set_numbering_verified(initial_verified, monkeypatch, tmp_path):
+@pytest.mark.parametrize("extent", ["covers", "extra_reference"])
+def test_a_resolution_call_that_raises_cannot_set_numbering_verified(
+ extent, tmp_path, monkeypatch
+):
"""Option 1's `except Exception` is where a live subprocess failure
- (timeout, non-zero exit, unparseable stdout) surfaces mid-escalation — a
- failed call is a third outcome, not a licence to guess a verdict. It must
- leave `rec.verified` exactly where it found it, same as every other branch
- that does not explicitly reset it.
-
- `reflist.resolve_disputed` is patched as a module attribute, not handed to
- `cli` directly: `_escalate_disputed` does `from .reflist import
- resolve_disputed` inside its own body, so the name is looked up fresh from
- `reflist`'s namespace on every call and a monkeypatch on the module
- attribute is what a function-local import actually sees.
+ (timeout, non-zero exit, unparseable stdout) surfaces mid-escalation. A
+ failed call is a third outcome, not a licence to guess a verdict: it must
+ leave the flag exactly where it found it, and say that it failed.
+
+ The failure is injected at `ask._ask`, the seam that really raises — NOT by
+ stubbing `reflist.resolve_disputed`. `resolve_disputed` calls `ask._ask`
+ with no `try` of its own, so raising at the seam reaches the same `cli`
+ branch with the real resolver in the loop. A stub there would keep this
+ green if `resolve_disputed` ever grew its own broad `except` and returned a
+ `Resolution` carrying confabulated entries — the double outliving the code
+ it stands for, which is this plan's most expensive scar.
"""
- candidates = {"parsed": _parsed(), "pymupdf": _flat()}
- texts = {"parsed": _TEXT_A, "pymupdf": _TEXT_B}
- _entries, rec = reconcile(_BODY, None, _parsed())
- rec.labels_disputed = ["2"]
- rec.verified = initial_verified
+ def _raising(prompt, model=None):
+ if _RESOLVE_MARKER in prompt:
+ raise RuntimeError("claude -p failed: simulated for this test")
+ return REPLIES["duplicate_numeral"]["reply"]
monkeypatch.setattr(cli_mod, "_interactive", lambda: True)
monkeypatch.setattr(cli_mod.Prompt, "ask", staticmethod(lambda *a, **k: "1"))
- def _boom(*args, **kwargs):
- raise RuntimeError("claude -p failed: simulated for this test")
-
- monkeypatch.setattr(reflist_mod, "resolve_disputed", _boom)
+ payload = _manifest(_run(tmp_path, monkeypatch, extent=extent,
+ reading="unused — the seam below answers", ask=_raising))
- entries_out = cli_mod._escalate_disputed(tmp_path, rec, _parsed(), candidates, texts)
-
- assert entries_out == _parsed() # nothing substituted — the call never answered
- assert rec.verified is initial_verified, "a raising resolver moved rec.verified"
- assert rec.chosen_by == "user" # a person consented to trying, even though it failed
- manifest = RefManifest(
- manuscript="p.pdf",
- numbering_verified=rec.verified,
- numbering_chosen_by=rec.chosen_by,
+ assert payload["numbering_verified"] is (extent == "covers"), (
+ "a resolution call that raised moved numbering_verified"
)
- assert manifest.numbering_verified is initial_verified
+ assert payload["resolution_outcome"] == "failed"
+ assert payload["labels_disputed"] == ["1"], "a failed call resolves nothing"
+ assert payload["labels_resolved"] == []
+ assert payload["numbering_chosen_by"] == "user" # a person consented to trying
+ nums, openings = _PRINTED[extent]
+ assert [e["num"] for e in payload["entries"]] == nums
-def test_corroboration_can_be_true_in_the_same_run_that_keeps_verified_false():
+def test_corroboration_can_be_true_in_the_same_run_that_keeps_verified_false(
+ tmp_path, monkeypatch
+):
"""The two axes must be independently settable, or the second is pointless.
This is the 22-vs-19 case the corroboration axis exists for: every cited
label is read the same way by two readings, and `_covers` still fails
- because three of the references are cited only in the supplement. Before
- the second axis, that run could say nothing but "unconfirmed" — crying
- wolf on a numbering two independent readings agreed about entry for entry.
+ because a reference is printed that the body never cites. Before the second
+ axis, that run could say nothing but "unconfirmed" — crying wolf on a
+ numbering two independent readings agreed about entry for entry.
+
+ Kept as its own cell although the parametrisation above covers the same
+ run: the parametrisation asserts the two values against a table, and what
+ is being claimed here is the RELATION between them — that one is False
+ while the other is True in one manifest. A table cell cannot say that, and
+ a future edit narrowing the table would not notice it had stopped.
+
+ The model read and agreed here, and is still not named among the
+ corroborating readings: `reflist.propose` may only copy from the two
+ extractions, and both of those already vote in their own right, so its
+ agreement is one text read twice.
"""
- candidates = {"parsed": _parsed(), "pymupdf": _parsed()}
- states = label_agreement(candidates, _BODY)
- assert set(states.values()) == {"agreed"}
-
- # a list longer than the highest cited label: `_covers` requires equality
- longer = _parsed() + [RefEntry(num="3", raw="Delta D. Cited only in the supplement. 2022.")]
- _entries, rec = reconcile(_BODY, None, longer)
- assert rec.verified is False, "the extent check still fails — that is the premise"
-
- manifest = RefManifest(
- manuscript="p.pdf",
- numbering_verified=rec.verified,
- numbering_corroborated=all(v == "agreed" for v in states.values()),
- corroborating_readings=corroborating_readings(candidates, _BODY),
- )
- assert manifest.numbering_verified is False
- assert manifest.numbering_corroborated is True
- assert manifest.corroborating_readings == ["parsed", "pymupdf"]
+ payload = _manifest(_run(tmp_path, monkeypatch, extent="extra_reference",
+ reading=REPLIES["agrees_with_the_parse"]["reply"]))
+
+ assert payload["reflist_outcome"] == "read"
+ assert payload["numbering_verified"] is False
+ assert payload["numbering_corroborated"] is True
+ assert payload["corroborating_readings"] == ["parsed", "pymupdf"]
+
+
+def test_the_two_model_calls_are_told_apart_by_a_marker_only_one_prompt_carries():
+ """The `ask._ask` double routes on `_RESOLVE_MARKER`, so the marker has to
+ be in one prompt and only one.
+
+ If `RESOLVE_PROMPT` ever stopped carrying it, every cell above would answer
+ the resolution call with a reading reply — the resolution would be read as
+ unparseable, the escalation would take its abstention branch, and the whole
+ suite would stay green while testing a branch it does not name. A double is
+ only as honest as the thing it discriminates on.
+ """
+ assert _RESOLVE_MARKER in reflist_mod.RESOLVE_PROMPT
+ assert _RESOLVE_MARKER not in reflist_mod.REFLIST_PROMPT
_SRC = Path(__file__).resolve().parent.parent / "src" / "papertrace"
-# the two names that may only ever be set from the extent check
+# the names that may only ever be set from the extent check
_GUARDED = ("numbering_verified", "rec.verified", ".verified")
-# what must never appear on the right-hand side of one
+# what must never appear on the right-hand side of one. `corroborated` is in
+# here because corroboration is the OTHER axis: `numbering_verified=rec.verified
+# or rec.corroborated` is, literally, a model reply setting the flag — a model
+# reading is a corroboration voter.
_FORBIDDEN = ("reflist", "propose", "resolve_disputed", "Prompt.ask", "_ask(",
- "answer", "choice", "chosen_by", "res.resolved")
+ "answer", "choice", "chosen_by", "res.resolved", "corroborated")
+
+# The first `=` that is an assignment: not `==`, `!=`, `<=` or `>=`. Splitting
+# on it beats discarding any line that contains a comparison — under that older
+# rule `rec.verified = bool(res.resolved) if answer == "1" else rec.verified`,
+# the docstring's own forbidden spelling plus a ternary, was never examined.
+_ASSIGN = re.compile(r"(?])=(?!=)")
def test_no_source_file_assigns_numbering_verified_from_a_model_or_a_prompt():
"""A grep, and honest about being one.
What this proves: no source line *textually* assigns one of the guarded
- names from an expression mentioning the reflist module, a prompt answer or
- the seam. That catches the change a future task is actually likely to make
- — `rec.verified = bool(res.resolved)`, written in good faith by someone who
- read "resolved" as "checked".
+ names from an expression mentioning the reflist module, a prompt answer,
+ the seam, or the corroboration axis. That catches the changes a future task
+ is actually likely to make — `rec.verified = bool(res.resolved)`, written
+ in good faith by someone who read "resolved" as "checked", and
+ `numbering_verified=rec.verified or rec.corroborated`, written by someone
+ who read the two axes as one.
What it cannot prove: that no value *derived* from a model reply reaches
- the flag through an intermediate variable, a helper or a dict. A grep has
- no dataflow. The parametrised tests above are what cover that, by running
- the real code over every reply shape and every choice; this one is the
- cheap tripwire that fails at review time rather than at runtime, and it is
- worth having for exactly that reason and no more.
+ the flag through an intermediate variable, a helper, a dict or a
+ `dataclasses.replace`. A grep has no dataflow, and every one of those
+ spellings walks straight through it. The parametrised tests above are what
+ cover that — they run `_refs_pipeline` itself and read the manifest it
+ wrote, so a value arriving by any route at all shows up in the artifact.
+ This one is the cheap tripwire that fails at review time rather than at
+ runtime, and it is worth having for that and no more.
"""
offenders = []
for path in sorted(_SRC.rglob("*.py")):
for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
code = line.split("#", 1)[0]
- if "=" not in code or "==" in code:
+ if not (m := _ASSIGN.search(code)):
continue
- lhs, _, rhs = code.partition("=")
+ lhs, rhs = code[:m.start()], code[m.end():]
if not any(g in lhs for g in _GUARDED):
continue
if any(f in rhs for f in _FORBIDDEN):
@@ -332,26 +480,37 @@ def test_no_source_file_assigns_numbering_verified_from_a_model_or_a_prompt():
assert offenders == [], "\n".join(offenders)
-def test_the_guard_above_would_actually_catch_the_change_it_describes(tmp_path):
+def test_the_guard_above_would_actually_catch_the_changes_it_describes(tmp_path):
"""A guard test nobody has seen fail is a guard test nobody knows works.
- The grep is re-run over a file holding the exact line the real test
- forbids, so the assertion has been observed both ways. Written out again
- rather than factored into a shared helper: a helper broken the same way
- would make both tests pass together, which is the failure mode a self-test
- exists to exclude.
+ The grep is re-run over a file holding the exact lines the real test
+ forbids — including the two the previous rule missed: a ternary (skipped
+ wholesale for containing `==`) and the corroboration widening (whose
+ spelling was not in `_FORBIDDEN` at all). Every planted line must be
+ caught, so a narrowing of either constant fails here rather than silently.
+
+ Written out again rather than factored into a shared helper: a helper
+ broken the same way would make both tests pass together, which is the
+ failure mode a self-test exists to exclude.
"""
+ planted = [
+ 'rec.verified = bool(res.resolved)',
+ 'rec.verified = bool(res.resolved) if answer == "1" else rec.verified',
+ ' numbering_verified=rec.verified or rec.corroborated,',
+ ]
bad = tmp_path / "papertrace" / "bad.py"
bad.parent.mkdir(parents=True)
- bad.write_text("rec.verified = bool(res.resolved)\n", encoding="utf-8")
+ bad.write_text("\n".join(planted) + "\n", encoding="utf-8")
offenders = []
for path in sorted(bad.parent.rglob("*.py")):
for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
code = line.split("#", 1)[0]
- if "=" not in code or "==" in code:
+ if not (m := _ASSIGN.search(code)):
continue
- lhs, _, rhs = code.partition("=")
+ lhs, rhs = code[:m.start()], code[m.end():]
if any(g in lhs for g in _GUARDED) and any(f in rhs for f in _FORBIDDEN):
- offenders.append(f"{path.name}:{lineno}")
- assert offenders, "the grep would not catch the line it exists to catch"
+ offenders.append(lineno)
+ assert offenders == list(range(1, len(planted) + 1)), (
+ f"the grep caught {offenders} of {len(planted)} lines it exists to catch"
+ )
From 44559514ed077b32af5eaf5ad3c89555cd79bdef Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Sat, 19 Sep 2026 09:47:36 +0200
Subject: [PATCH 43/44] reflist: past a handful of absent numerals, the count
is the finding
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
A reply naming only [99] makes every numeral below it absent, which printed 98
consecutive integers — 410 characters — into reflist_numbering_findings, a
field the report shows a reader. The first six locate the gap; the rest only
prove the reader stopped reading. Nothing is hidden and the finding is
unchanged: the total leads the sentence.
---
src/papertrace/reflist.py | 17 ++++++++++++++++-
tests/test_reflist.py | 19 +++++++++++++++++++
2 files changed, 35 insertions(+), 1 deletion(-)
diff --git a/src/papertrace/reflist.py b/src/papertrace/reflist.py
index 81e1907..504794d 100644
--- a/src/papertrace/reflist.py
+++ b/src/papertrace/reflist.py
@@ -211,6 +211,11 @@ def _too_long(texts: tuple[str, ...]) -> str:
# not one of the answers the model was given and must be dropped, not guessed
_READING_LETTERS = ("A", "B", "AB")
+# How many absent numerals to name before summarising. Enough to locate the gap
+# — a reader who sees [4], [5], [6] knows where to look — and few enough that
+# the count stays the finding when a reply names one high label and nothing else.
+_GAPS_SHOWN = 6
+
def propose(
reading_a: str,
@@ -410,7 +415,17 @@ def propose(
if twice:
prov.numbering_findings.append("numerals proposed twice: " + ", ".join(twice))
if gaps := _numeral_gaps(nums):
- prov.numbering_findings.append("numerals absent from 1..max: " + ", ".join(gaps))
+ # Summarised past a handful, because the count is the finding and the
+ # list stops being one. A reply naming only [99] makes every numeral
+ # below it absent, which printed 98 consecutive integers — 410
+ # characters — into a field a reader is shown. The first few locate the
+ # gap; the rest only prove the reader stopped reading.
+ shown = ", ".join(gaps[:_GAPS_SHOWN])
+ rest = len(gaps) - _GAPS_SHOWN
+ prov.numbering_findings.append(
+ f"{len(gaps)} numerals absent from 1..max: {shown}"
+ + (f" and {rest} more" if rest > 0 else "")
+ )
return out, prov
diff --git a/tests/test_reflist.py b/tests/test_reflist.py
index 3df8c27..815c330 100644
--- a/tests/test_reflist.py
+++ b/tests/test_reflist.py
@@ -675,3 +675,22 @@ def test_an_oversized_resolution_input_leaves_every_label_disputed(monkeypatch):
assert res.still_disputed == ["6"]
assert res.provenance.outcome == "not_attempted"
assert "too long" in res.provenance.failure
+
+
+def test_a_long_run_of_absent_numerals_is_counted_rather_than_listed(monkeypatch):
+ """The count is the finding; past a handful the list stops being one.
+
+ A reply naming only `[99]` makes every numeral below it absent, which
+ printed 98 consecutive integers — 410 characters — into
+ `reflist_numbering_findings`, a field the report shows a reader. The first
+ few locate the gap; the rest only prove the reader stopped reading. The
+ finding itself is unchanged and nothing is hidden: the total leads.
+ """
+ text = "99. Weston AD (2019) A paper. Radiology. https://doi.org/10.1148/radiol.2019181432"
+ _reply(monkeypatch, '[{"num":"99","title":"A paper","reading":"AB"}]')
+ _, prov = reflist.propose(text, text, label_a="docling", label_b="pymupdf")
+
+ gaps = next(f for f in prov.numbering_findings if "absent" in f)
+ assert gaps.startswith("98 numerals absent"), gaps
+ assert "and 92 more" in gaps, gaps
+ assert len(gaps) < 100, f"{len(gaps)} chars in a reader-facing field: {gaps}"
From c98c095d5eb8b2f8e3fdd2839c4b9dd06bf8455a Mon Sep 17 00:00:00 2001
From: defraction <796174+defraction0@users.noreply.github.com>
Date: Sat, 19 Sep 2026 13:28:40 +0200
Subject: [PATCH 44/44] tests: patch the gate, not the predicate behind it
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
All ten CI jobs failed on two tests that pass locally. `_check_pipeline` calls
`cli._require_claude()`, which does `from .check import claude_available` — and
`check.claude_available` is a re-export, a separate module attribute from
`ask.claude_available` even though both name one function. The tests patched
`ask_mod`, so the binding this path actually reads was never touched: they
passed on a machine with the CLI installed and failed on ten that have none.
The merge caused it. Before it, `_check_pipeline` imported from `.ask` and the
patch was right; `limits` refactored the check into `_require_claude`, which
imports from `.check`, and the comment above the patch went on describing the
old import. Patching `_require_claude` itself cannot drift with a future
refactor of which module it reads.
Verified the way CI does, which is the only way this shows: with `claude` off
PATH, 1145 passed, 25 skipped.
---
tests/test_pipeline_split.py | 25 ++++++++++++++++++-------
1 file changed, 18 insertions(+), 7 deletions(-)
diff --git a/tests/test_pipeline_split.py b/tests/test_pipeline_split.py
index ed9d670..c8378db 100644
--- a/tests/test_pipeline_split.py
+++ b/tests/test_pipeline_split.py
@@ -16,7 +16,6 @@
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
-from papertrace import ask as ask_mod # noqa: E402
from papertrace import check as check_mod # noqa: E402
from papertrace import cli # noqa: E402
from papertrace.models import ClaimExtraction, RefEntry, RefManifest # noqa: E402
@@ -202,11 +201,15 @@ def test_check_pipeline_passes_the_manifests_disputed_labels_to_check_claims(
labels_disputed=["9"],
).to_json(case / "refs_manifest.json")
- # `_check_pipeline` does `from .ask import claude_available` INSIDE the
- # function body, so the name it calls is resolved fresh from the `ask`
- # module at call time — patching `check_mod.claude_available` or
- # `cli.claude_available` touches an attribute nothing ever reads.
- monkeypatch.setattr(ask_mod, "claude_available", lambda: True)
+ # Patch the GATE, not the predicate behind it. `_check_pipeline` calls
+ # `cli._require_claude()`, which does `from .check import claude_available`
+ # — and `check.claude_available` is a re-export, a *separate module
+ # attribute* from `ask.claude_available` even though both name one
+ # function. Patching `ask_mod` left the one this path reads untouched, so
+ # the test passed on a machine with the CLI installed and failed all ten
+ # CI jobs, which have none. Patching `_require_claude` itself cannot
+ # drift with a future refactor of which module it imports from.
+ monkeypatch.setattr(cli, "_require_claude", lambda: None)
# `_check_pipeline` reads the extraction through `cli._extraction_for` now
# — the `limits` feature made extraction a stage of its own so a claims
# limit can retrieve only what the selected claims cite, and stubbing
@@ -250,7 +253,15 @@ def test_check_pipeline_tolerates_a_manifest_with_nothing_disputed(tmp_path, mon
case.mkdir()
RefManifest(manuscript="m.pdf", entries=[]).to_json(case / "refs_manifest.json")
- monkeypatch.setattr(ask_mod, "claude_available", lambda: True)
+ # Patch the GATE, not the predicate behind it. `_check_pipeline` calls
+ # `cli._require_claude()`, which does `from .check import claude_available`
+ # — and `check.claude_available` is a re-export, a *separate module
+ # attribute* from `ask.claude_available` even though both name one
+ # function. Patching `ask_mod` left the one this path reads untouched, so
+ # the test passed on a machine with the CLI installed and failed all ten
+ # CI jobs, which have none. Patching `_require_claude` itself cannot
+ # drift with a future refactor of which module it imports from.
+ monkeypatch.setattr(cli, "_require_claude", lambda: None)
# `_check_pipeline` reads the extraction through `cli._extraction_for` now
# — the `limits` feature made extraction a stage of its own so a claims
# limit can retrieve only what the selected claims cite, and stubbing