feat(novelty): usage and cost block for routing --model

dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md

@@ -0,0 +1,7 @@
+### Added
+
+- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals (`llm_calls` still counts them), and `usd` is also `null` when a call returned no usage.
+
+### Changed
+
+- **`novelty analyze` cost totals are what this run spent** — the per-role cost summary now leaves calls replayed from the response cache out of `input_tokens`, `output_tokens` and `usd` (they were summed from the stored records before) and counts them in a new `replayed_calls`; `llm_calls` still counts them. `usd` is now `null` when a call returned no usage, where it was the sum of the priced calls. The block also gains `cache_read_tokens` and `cache_write_tokens`, and the Markdown report's cost line names the replayed calls when there are any.

dokimi/docs/benchmarking-and-comparison.md

@@ -754,7 +754,10 @@ that never hits.
 
 A replayed entry reports the **stored** usage record — tokens, cost and latency
 as measured when the call was really made, not `$0`/`0s`, so cost stays
-comparable across runs. The flip side: latency from a cached run is a recording,
+comparable across runs. The run's `cost` totals (`novelty analyze`, `novelty
+routing --model`) are what this run spent, so they leave replayed records out of
+the token and `usd` sums and count them in `replayed_calls`; `llm_calls` still
+counts them. The flip side: latency from a cached run is a recording,
 not a measurement. That is why this cache is opt-in, and why it should stay off
 for any run whose latency numbers you intend to quote.
 

dokimi/docs/skill-novelty.md

@@ -630,6 +630,15 @@ Two numbers, reported side by side and never merged:
   queries x repeats, `n_abstained` and `n_failed` count calls, and each `per_query` row
   carries its `repeat` index.
 
+A `--model` run also writes a `cost` block (calls, input and output tokens, cache read and
+write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the
+same totals on stdout. Cache tokens are reported separately from input tokens, and depending on
+the provider they may or may not be counted inside `input_tokens`. A run without `--model` makes
+no calls and has no `cost` block. `usd` is also `null` when a call returned no usage, and a
+call that raised is not counted. A call replayed from the response cache is counted in
+`replayed_calls` and left out of the token and `usd` totals, since this run did not pay for it;
+`llm_calls` still counts it.
+
 `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3`
 and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given`
 next to `n_queries_scored`, since a query with no content words is dropped rather than

dokimi/dokimi/commands/novelty.py

@@ -361,18 +361,22 @@ def card(ref: str) -> tuple[Candidate, str]:
         "lexical": lexical,
     }
     if args.model:
+        from dokimi.novelty.client import roll_up_cost
+
         selector = _client(args.model, args.provider, "selector", args)
         payload["model"] = dict(
             model_routing(candidates, queries, client=selector, gold=0, seed=args.seed,
                           repeats=args.repeats),
             queries_self_derived=self_derived,
         )
+        payload["cost"] = roll_up_cost([selector])
 
     out.write_text(json.dumps(payload, indent=2), encoding="utf-8")
     print(f"OK Saved routing measurement: {out}")
 
     _print_routing_summary(payload, len(competitors), self_derived)
 
+
 def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool) -> None:
     """The lexical line, the model line and their caveats, printed after a routing run."""
     # Every rate is printed with the number of queries behind it, and a self-derived one
@@ -408,11 +412,22 @@ def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool
                   f"{noise['half_range']} (a crude sampling floor)")
         print("The model figure is an upper bound on harness behaviour: a harness that "
               "truncates descriptions in its own listing selects on less text.")
+        cost = payload["cost"]
+        if not cost["llm_calls"]:
+            print("cost     0 calls")
+        else:
+            spent = ("USD not known: a call had no price or no usage" if cost["usd"] is None
+                     else "<$0.0001" if 0 < cost["usd"] < 0.00005 else f"${cost['usd']:.4f}")
+            replayed = (f", {cost['replayed_calls']} replayed from the cache and not counted"
+                        if cost["replayed_calls"] else "")
+            print(f"cost     {cost['llm_calls']} calls, {cost['input_tokens']} input / "
+                  f"{cost['output_tokens']} output tokens, cache read "
+                  f"{cost['cache_read_tokens']} / write {cost['cache_write_tokens']}, "
+                  f"{spent}{replayed}")
     if self_derived:
         print("Self-derived queries: read as triage only.")
 
 
-
 def _refuse_no_queries(args: argparse.Namespace, queries: list[str], self_derived: bool,
                        out: Path, n_candidates: int) -> NoReturn:
     """Write the ``no_queries`` record, then exit saying why no query was scored."""

dokimi/dokimi/novelty/client.py

@@ -167,13 +167,22 @@ def json_array(self, prompt: str, *, system: str | None = None) -> list[Any]:
     def cost_summary(self) -> dict[str, Any]:
         """Roll up this client's usage.
 
-        ``usd`` stays ``None`` when any call had unknown pricing, rather than
-        silently reporting the sum of the priced subset as the total.
+        ``usd`` stays ``None`` when any call this run paid for had unknown pricing or
+        returned no usage (a replayed call is not paid for, so it cannot make it unknown),
+        rather than silently reporting the sum of the priced subset as the total.
+        Calls served from the response cache are counted in ``replayed_calls`` and
+        left out of the token and dollar totals.
         """
-        input_tokens = sum(u.prompt_tokens for u in self.usage)
-        output_tokens = sum(u.completion_tokens for u in self.usage)
-        costs = [u.cost_usd for u in self.usage]
-        priced = costs and None not in costs
+        # A replayed row is the response cache's copy of an earlier call: its tokens and
+        # price were not spent by this process, so it is counted apart, as the spend cap does.
+        billed = [u for u in self.usage if not u.replayed]
+        input_tokens = sum(u.prompt_tokens for u in billed)
+        output_tokens = sum(u.completion_tokens for u in billed)
+        cache_read = sum(u.cache_read_tokens for u in billed)
+        cache_write = sum(u.cache_write_tokens for u in billed)
+        costs = [u.cost_usd for u in billed]
+        # A call that returned no usage row has no price either, so it makes the total unknown.
+        priced = self.calls > 0 and len(self.usage) == self.calls and None not in costs
         usd = round(sum(c for c in costs if c is not None), 6) if priced else None
         return {
             "model": self.model,
@@ -182,6 +191,9 @@ def cost_summary(self) -> dict[str, Any]:
             "llm_calls": self.calls,
             "input_tokens": input_tokens,
             "output_tokens": output_tokens,
+            "cache_read_tokens": cache_read,
+            "cache_write_tokens": cache_write,
+            "replayed_calls": len(self.usage) - len(billed),
             "usd": usd,
         }
 
@@ -238,6 +250,9 @@ def roll_up_cost(clients: list[LLMClient]) -> dict[str, Any]:
         "llm_calls": sum(row["llm_calls"] for row in per_role),
         "input_tokens": sum(row["input_tokens"] for row in per_role),
         "output_tokens": sum(row["output_tokens"] for row in per_role),
+        "cache_read_tokens": sum(row["cache_read_tokens"] for row in per_role),
+        "cache_write_tokens": sum(row["cache_write_tokens"] for row in per_role),
+        "replayed_calls": sum(row["replayed_calls"] for row in per_role),
         "usd": round(sum(p for p in usd_parts if p is not None), 6)
         if usd_parts and None not in usd_parts
         else None,

dokimi/dokimi/novelty/report.py

@@ -489,7 +489,10 @@ def render_novelty_markdown(artifact: dict) -> str:
         lines.append(
             f"Cost: {cost.get('llm_calls', 0)} LLM calls, "
             f"{cost.get('input_tokens', 0)} in / {cost.get('output_tokens', 0)} out, "
-            f"usd {cost.get('usd')}."
+            f"usd {cost.get('usd')}"
+            + (f", {cost['replayed_calls']} replayed from the cache and not counted"
+               if cost.get("replayed_calls") else "")
+            + "."
         )
         lines.append("")
 

dokimi/dokimi/schemas/novelty.v1.json

@@ -448,9 +448,12 @@
         "llm_calls": {"type": "integer", "minimum": 0},
         "input_tokens": {"type": "integer", "minimum": 0},
         "output_tokens": {"type": "integer", "minimum": 0},
+        "cache_read_tokens": {"type": "integer", "minimum": 0},
+        "cache_write_tokens": {"type": "integer", "minimum": 0},
+        "replayed_calls": {"type": "integer", "minimum": 0, "description": "Calls served from the response cache; left out of the token and usd totals."},
         "usd": {
           "type": ["number", "null"],
-          "description": "null when any call had unknown pricing — distinct from a genuinely free run."
+          "description": "null when any call this run paid for had unknown pricing, a call returned no usage, or no call was made — distinct from a genuinely free run. Replayed calls are left out of the sum."
         },
         "by_role": {"type": "array", "items": {"type": "object"}}
       }

dokimi/scripts/growth_baseline.json

@@ -1,6 +1,6 @@
 {
   "files": {
-    "dokimi/commands/novelty.py": {"lines": 984, "rationale": "estimate-only projection (expected_usd, arm scaling, extrapolation flag) in novelty CLI, on top of main's W-series and #173 lines; file split tracked separately"},
+    "dokimi/commands/novelty.py": {"lines": 999, "rationale": "estimate-only projection (expected_usd, arm scaling, extrapolation flag) in novelty CLI, on top of main's W-series and #173 lines, plus W1 routing cost output (#164); file split tracked separately"},
     "dokimi/harnesses/benchflow.py": {"lines": 869},
     "dokimi/harnesses/docker_solver.py": {"lines": 900, "rationale": "pre-existing when the gate was introduced"},
     "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"},

dokimi/tests/test_novelty_measure.py

@@ -221,6 +221,34 @@ def test_cost_rolls_up_per_role_and_stays_none_when_unpriced():
     assert by_role["judge"]["usd"] is None
 
 
+def test_cost_rolls_up_cache_tokens_and_replays_across_every_role():
+    def client(role, records):
+        rows = iter(records)
+        return LLMClient(model=role, role=role, transport=lambda p, s: ("x", next(rows)))
+
+    extractor = client("extractor", [
+        UsageRecord(prompt_tokens=10, completion_tokens=3, cache_read_tokens=4,
+                    cache_write_tokens=1, cost_usd=0.1),
+        UsageRecord(prompt_tokens=10, completion_tokens=50, cost_usd=0.1, replayed=True),
+    ])
+    judge = client("judge", [
+        UsageRecord(prompt_tokens=20, completion_tokens=4, cache_read_tokens=8,
+                    cache_write_tokens=4, cost_usd=0.2),
+        UsageRecord(prompt_tokens=20, cost_usd=0.2, replayed=True),
+    ])
+    for c in (extractor, judge):
+        c.text("p")
+        c.text("p")
+    total = roll_up_cost([extractor, judge])
+    # analyze rolls up several roles: a total taken from one of them would pass a
+    # single-client test, so each new field is summed over two.
+    assert total["llm_calls"] == 4
+    assert total["output_tokens"] == 7   # live rows of both roles; replayed 50 left out
+    assert total["cache_read_tokens"] == 12
+    assert total["cache_write_tokens"] == 5
+    assert total["replayed_calls"] == 2
+
+
 def test_a_failed_call_is_a_dropped_pair_not_a_graded_answer():
     def answer(prompt, system):
         if prompt.startswith("Reference material"):

dokimi/tests/test_novelty_report.py

@@ -59,3 +59,13 @@ def test_found_conflicts_lead_whatever_the_coverage():
     assert "> **Conflicts first: 2 conflicting atoms, 1 pairs.**" in markdown
     assert "Conflicts not cleared" not in markdown
     assert "| harmful_conflicting | 2 |" in markdown
+
+
+@pytest.mark.parametrize("cost, line", [
+    ({"llm_calls": 2, "input_tokens": 200, "output_tokens": 20, "usd": 0.02},
+     "Cost: 2 LLM calls, 200 in / 20 out, usd 0.02."),
+    ({"llm_calls": 3, "input_tokens": 10, "output_tokens": 1, "usd": 0.001, "replayed_calls": 1},
+     "Cost: 3 LLM calls, 10 in / 1 out, usd 0.001, 1 replayed from the cache and not counted."),
+], ids=["live", "replayed"])
+def test_the_cost_line_names_replayed_calls_only_when_there_are_some(cost, line):
+    assert line in render_novelty_markdown(_artifact(cost=cost)).splitlines()