From 09f2ce72af33fdbe597da11a889a5b1f4f4d3b83 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 21:42:55 -0700 Subject: [PATCH 01/15] feat(novelty): usage and cost block for routing --model A model-backed `novelty routing` run wrote no record of what it spent, so a routing rate could not be read against its cost. The run now writes a `cost` block into routing.json (calls, input/output tokens, cache read/write tokens, and usd, null when any call is unpriced) and prints the same totals on stdout. Cache token totals are added to the per-role cost summary, so novelty analyze reports them too. Signed-off-by: Nikolay Petrov --- .../20261021_120000_napetrov_routing-usage.md | 3 + dokimi/docs/skill-novelty.md | 4 ++ dokimi/dokimi/commands/novelty.py | 9 +++ dokimi/dokimi/novelty/client.py | 6 ++ dokimi/tests/test_novelty_routing.py | 58 +++++++++++++++++++ 5 files changed, 80 insertions(+) create mode 100644 dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md diff --git a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md new file mode 100644 index 000000000..fd101f6b9 --- /dev/null +++ b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md @@ -0,0 +1,3 @@ +### Added + +- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 5dac13bbb..62e205d7f 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -607,6 +607,10 @@ Two numbers, reported side by side and never merged: `selection_rate_among_answered` beside it; a failed call is excluded from both and counted, and one failed query does not lose the rest. +A `--model` run also writes a `cost` block (calls, input and output tokens, cache read and +write tokens, and `usd`, which is `null` when any call had no known price) and prints the +same totals on stdout. A run without `--model` makes no calls and has no `cost` block. + `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given` next to `n_queries_scored`, since a query with no content words is dropped rather than diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index adbdc9432..b68ead9f0 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -333,11 +333,14 @@ def card(ref: str) -> tuple[Candidate, str]: "lexical": lexical, } if args.model: + from dokimi.novelty.client import roll_up_cost + selector = _client(args.model, args.provider, "selector", args) payload["model"] = dict( model_routing(candidates, queries, client=selector, gold=0, seed=args.seed), queries_self_derived=self_derived, ) + payload["cost"] = roll_up_cost([selector]) out = Path(args.out) if args.out else _default_dir(args.artifact) / "routing.json" out.parent.mkdir(parents=True, exist_ok=True) @@ -357,6 +360,12 @@ def card(ref: str) -> tuple[Candidate, str]: f"{row['n_abstained']} abstained, {row['n_failed']} failed) [{row['model']}]") print("The model figure is an upper bound on harness behaviour: a harness that " "truncates descriptions in its own listing selects on less text.") + cost = payload["cost"] + spent = (f"${cost['usd']:.4f}" if cost["usd"] is not None + else "USD not known: a call had no price") + print(f"cost {cost['llm_calls']} calls, {cost['input_tokens']} input / " + f"{cost['output_tokens']} output tokens, cache read {cost['cache_read_tokens']} " + f"/ write {cost['cache_write_tokens']}, {spent} [{row['model']}]") if self_derived: print("Self-derived queries: read as triage only.") diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index b8579cc8b..55fc9713b 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -166,6 +166,8 @@ def cost_summary(self) -> dict[str, Any]: """ input_tokens = sum(u.prompt_tokens for u in self.usage) output_tokens = sum(u.completion_tokens for u in self.usage) + cache_read = sum(u.cache_read_tokens for u in self.usage) + cache_write = sum(u.cache_write_tokens for u in self.usage) costs = [u.cost_usd for u in self.usage] priced = costs and None not in costs usd = round(sum(c for c in costs if c is not None), 6) if priced else None @@ -176,6 +178,8 @@ def cost_summary(self) -> dict[str, Any]: "llm_calls": self.calls, "input_tokens": input_tokens, "output_tokens": output_tokens, + "cache_read_tokens": cache_read, + "cache_write_tokens": cache_write, "usd": usd, } @@ -230,6 +234,8 @@ def roll_up_cost(clients: list[LLMClient]) -> dict[str, Any]: "llm_calls": sum(row["llm_calls"] for row in per_role), "input_tokens": sum(row["input_tokens"] for row in per_role), "output_tokens": sum(row["output_tokens"] for row in per_role), + "cache_read_tokens": sum(row["cache_read_tokens"] for row in per_role), + "cache_write_tokens": sum(row["cache_write_tokens"] for row in per_role), "usd": round(sum(p for p in usd_parts if p is not None), 6) if usd_parts and None not in usd_parts else None, diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index cd7ea0782..0a981faec 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -473,3 +473,61 @@ def test_catalog_competitors_with_name_in_competitors_excludes_it_from_catalog(t assert names.count("perf") == 0, "perf should be excluded from catalog" assert set(names) == {"xpu", "cache"}, f"Expected xpu and cache, got {names}" + + +def _stub_provider(monkeypatch, usage): + """Replace the provider call the --model path makes; every reply picks card 1.""" + from dokimi import llm + + def stub(prompt, model, **_kwargs): + return '{"pick": 1}', usage + + monkeypatch.setattr(llm, "llm_call_with_usage", stub) + + +def _model_run(tmp_path): + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary", "tune a slow query"]), encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary with perf"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--model", "selector-m", "--out", str(out)) + args.func(args) + return json.loads(out.read_text(encoding="utf-8")) + + +def test_a_model_run_records_its_calls_tokens_and_cost(tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5, + cache_read_tokens=60, cost_usd=0.002)) + payload = _model_run(tmp_path) + assert payload["cost"]["llm_calls"] == 2 + assert payload["cost"]["input_tokens"] == 200 + assert payload["cost"]["output_tokens"] == 10 + assert payload["cost"]["cache_read_tokens"] == 120 + assert payload["cost"]["cache_write_tokens"] == 0 + assert payload["cost"]["usd"] == 0.004 + assert "$0.0040" in capsys.readouterr().out + + +def test_a_model_run_with_an_unpriced_call_reports_no_usd_figure(tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=10, completion_tokens=1)) + payload = _model_run(tmp_path) + assert payload["cost"]["usd"] is None # one unpriced call must not be read as free + assert "no price" in capsys.readouterr().out + + +def test_a_lexical_run_spends_nothing_and_has_no_cost_block(tmp_path): + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary"]), encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary with perf"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--out", str(out)) + args.func(args) + assert "cost" not in json.loads(out.read_text(encoding="utf-8")) From 14a7e93205db1ee63da557006b356063cde367d3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 21 Oct 2026 22:07:32 -0700 Subject: [PATCH 02/15] fix(novelty): declare cache token fields and tidy routing cost line Declare cache_read_tokens and cache_write_tokens (integer, minimum 0) in the novelty.v1 cost block, which routing and analyze both write. Say in skill-novelty.md that cache tokens are reported separately and may or may not be inside input_tokens, depending on the provider. The routing test stub now sets cache_write_tokens too, and asserts its total. The stdout cost line drops the trailing model tag, which the model line above already carries. Signed-off-by: Nikolay Petrov --- dokimi/docs/skill-novelty.md | 4 +++- dokimi/dokimi/commands/novelty.py | 2 +- dokimi/dokimi/schemas/novelty.v1.json | 2 ++ dokimi/tests/test_novelty_routing.py | 6 +++--- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 62e205d7f..d6b95ead1 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -609,7 +609,9 @@ Two numbers, reported side by side and never merged: A `--model` run also writes a `cost` block (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call had no known price) and prints the -same totals on stdout. A run without `--model` makes no calls and has no `cost` block. +same totals on stdout. Cache tokens are reported separately from input tokens, and depending on +the provider they may or may not be counted inside `input_tokens`. A run without `--model` makes +no calls and has no `cost` block. `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given` diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index b68ead9f0..fada3e853 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -365,7 +365,7 @@ def card(ref: str) -> tuple[Candidate, str]: else "USD not known: a call had no price") print(f"cost {cost['llm_calls']} calls, {cost['input_tokens']} input / " f"{cost['output_tokens']} output tokens, cache read {cost['cache_read_tokens']} " - f"/ write {cost['cache_write_tokens']}, {spent} [{row['model']}]") + f"/ write {cost['cache_write_tokens']}, {spent}") if self_derived: print("Self-derived queries: read as triage only.") diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index dc41beec0..c0dd76b91 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -447,6 +447,8 @@ "llm_calls": {"type": "integer", "minimum": 0}, "input_tokens": {"type": "integer", "minimum": 0}, "output_tokens": {"type": "integer", "minimum": 0}, + "cache_read_tokens": {"type": "integer", "minimum": 0}, + "cache_write_tokens": {"type": "integer", "minimum": 0}, "usd": { "type": ["number", "null"], "description": "null when any call had unknown pricing — distinct from a genuinely free run." diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index 0a981faec..312bf1f95 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -474,7 +474,6 @@ def test_catalog_competitors_with_name_in_competitors_excludes_it_from_catalog(t assert set(names) == {"xpu", "cache"}, f"Expected xpu and cache, got {names}" - def _stub_provider(monkeypatch, usage): """Replace the provider call the --model path makes; every reply picks card 1.""" from dokimi import llm @@ -501,13 +500,14 @@ def test_a_model_run_records_its_calls_tokens_and_cost(tmp_path, monkeypatch, ca from dokimi.metrics.usage import UsageRecord _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5, - cache_read_tokens=60, cost_usd=0.002)) + cache_read_tokens=60, cache_write_tokens=30, + cost_usd=0.002)) payload = _model_run(tmp_path) assert payload["cost"]["llm_calls"] == 2 assert payload["cost"]["input_tokens"] == 200 assert payload["cost"]["output_tokens"] == 10 assert payload["cost"]["cache_read_tokens"] == 120 - assert payload["cost"]["cache_write_tokens"] == 0 + assert payload["cost"]["cache_write_tokens"] == 60 assert payload["cost"]["usd"] == 0.004 assert "$0.0040" in capsys.readouterr().out From 714ce0debe9a11ace637f10af3f4592460df7e95 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 11:44:51 -0700 Subject: [PATCH 03/15] fix(novelty): routing cost line separates no calls, missing usage and cache replays Signed-off-by: Nikolay Petrov --- .../20261021_120000_napetrov_routing-usage.md | 2 +- dokimi/docs/skill-novelty.md | 4 +- dokimi/dokimi/commands/novelty.py | 16 ++++--- dokimi/dokimi/novelty/client.py | 24 ++++++---- dokimi/dokimi/schemas/novelty.v1.json | 1 + dokimi/tests/test_novelty_routing.py | 45 ++++++++++++++++++- 6 files changed, 75 insertions(+), 17 deletions(-) diff --git a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md index fd101f6b9..ec8359189 100644 --- a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md +++ b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md @@ -1,3 +1,3 @@ ### Added -- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. +- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals, and `usd` is also `null` when a call returned no usage. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index d6b95ead1..040b89638 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -611,7 +611,9 @@ A `--model` run also writes a `cost` block (calls, input and output tokens, cach write tokens, and `usd`, which is `null` when any call had no known price) and prints the same totals on stdout. Cache tokens are reported separately from input tokens, and depending on the provider they may or may not be counted inside `input_tokens`. A run without `--model` makes -no calls and has no `cost` block. +no calls and has no `cost` block. `usd` is also `null` when a call returned no usage, and a +call that raised is not counted. A call replayed from the response cache is counted in +`replayed_calls` and left out of the token and `usd` totals, since this run did not pay for it. `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given` diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index fada3e853..b3557942f 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -361,11 +361,17 @@ def card(ref: str) -> tuple[Candidate, str]: print("The model figure is an upper bound on harness behaviour: a harness that " "truncates descriptions in its own listing selects on less text.") cost = payload["cost"] - spent = (f"${cost['usd']:.4f}" if cost["usd"] is not None - else "USD not known: a call had no price") - print(f"cost {cost['llm_calls']} calls, {cost['input_tokens']} input / " - f"{cost['output_tokens']} output tokens, cache read {cost['cache_read_tokens']} " - f"/ write {cost['cache_write_tokens']}, {spent}") + if not cost["llm_calls"]: + print("cost 0 calls") + else: + spent = (f"${cost['usd']:.4f}" if cost["usd"] is not None + else "USD not known: a call had no price or no usage") + replayed = (f", {cost['replayed_calls']} replayed from the cache and not counted" + if cost["replayed_calls"] else "") + print(f"cost {cost['llm_calls']} calls, {cost['input_tokens']} input / " + f"{cost['output_tokens']} output tokens, cache read " + f"{cost['cache_read_tokens']} / write {cost['cache_write_tokens']}, " + f"{spent}{replayed}") if self_derived: print("Self-derived queries: read as triage only.") diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index 55fc9713b..cec8039cd 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -161,15 +161,21 @@ def json_array(self, prompt: str, *, system: str | None = None) -> list[Any]: def cost_summary(self) -> dict[str, Any]: """Roll up this client's usage. - ``usd`` stays ``None`` when any call had unknown pricing, rather than - silently reporting the sum of the priced subset as the total. + ``usd`` stays ``None`` when any call had unknown pricing or returned no usage, + rather than silently reporting the sum of the priced subset as the total. + Calls served from the response cache are counted in ``replayed_calls`` and + left out of the token and dollar totals. """ - input_tokens = sum(u.prompt_tokens for u in self.usage) - output_tokens = sum(u.completion_tokens for u in self.usage) - cache_read = sum(u.cache_read_tokens for u in self.usage) - cache_write = sum(u.cache_write_tokens for u in self.usage) - costs = [u.cost_usd for u in self.usage] - priced = costs and None not in costs + # A replayed row is the response cache's copy of an earlier call: its tokens and + # price were not spent by this process, so it is counted apart, as the spend cap does. + billed = [u for u in self.usage if not u.replayed] + input_tokens = sum(u.prompt_tokens for u in billed) + output_tokens = sum(u.completion_tokens for u in billed) + cache_read = sum(u.cache_read_tokens for u in billed) + cache_write = sum(u.cache_write_tokens for u in billed) + costs = [u.cost_usd for u in billed] + # A call that returned no usage row has no price either, so it makes the total unknown. + priced = self.calls > 0 and len(self.usage) == self.calls and None not in costs usd = round(sum(c for c in costs if c is not None), 6) if priced else None return { "model": self.model, @@ -180,6 +186,7 @@ def cost_summary(self) -> dict[str, Any]: "output_tokens": output_tokens, "cache_read_tokens": cache_read, "cache_write_tokens": cache_write, + "replayed_calls": len(self.usage) - len(billed), "usd": usd, } @@ -236,6 +243,7 @@ def roll_up_cost(clients: list[LLMClient]) -> dict[str, Any]: "output_tokens": sum(row["output_tokens"] for row in per_role), "cache_read_tokens": sum(row["cache_read_tokens"] for row in per_role), "cache_write_tokens": sum(row["cache_write_tokens"] for row in per_role), + "replayed_calls": sum(row["replayed_calls"] for row in per_role), "usd": round(sum(p for p in usd_parts if p is not None), 6) if usd_parts and None not in usd_parts else None, diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index c0dd76b91..f302528d9 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -449,6 +449,7 @@ "output_tokens": {"type": "integer", "minimum": 0}, "cache_read_tokens": {"type": "integer", "minimum": 0}, "cache_write_tokens": {"type": "integer", "minimum": 0}, + "replayed_calls": {"type": "integer", "minimum": 0, "description": "Calls served from the response cache; left out of the token and usd totals."}, "usd": { "type": ["number", "null"], "description": "null when any call had unknown pricing — distinct from a genuinely free run." diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index 312bf1f95..13b5c657b 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -509,7 +509,8 @@ def test_a_model_run_records_its_calls_tokens_and_cost(tmp_path, monkeypatch, ca assert payload["cost"]["cache_read_tokens"] == 120 assert payload["cost"]["cache_write_tokens"] == 60 assert payload["cost"]["usd"] == 0.004 - assert "$0.0040" in capsys.readouterr().out + assert ("cost 2 calls, 200 input / 10 output tokens, cache read 120 / write 60, " + "$0.0040") in capsys.readouterr().out def test_a_model_run_with_an_unpriced_call_reports_no_usd_figure(tmp_path, monkeypatch, capsys): @@ -518,7 +519,7 @@ def test_a_model_run_with_an_unpriced_call_reports_no_usd_figure(tmp_path, monke _stub_provider(monkeypatch, UsageRecord(prompt_tokens=10, completion_tokens=1)) payload = _model_run(tmp_path) assert payload["cost"]["usd"] is None # one unpriced call must not be read as free - assert "no price" in capsys.readouterr().out + assert "USD not known: a call had no price or no usage" in capsys.readouterr().out def test_a_lexical_run_spends_nothing_and_has_no_cost_block(tmp_path): @@ -531,3 +532,43 @@ def test_a_lexical_run_spends_nothing_and_has_no_cost_block(tmp_path): "--queries", str(queries), "--out", str(out)) args.func(args) assert "cost" not in json.loads(out.read_text(encoding="utf-8")) + + +def test_a_model_run_whose_every_call_fails_prints_no_price_blame(tmp_path, monkeypatch, capsys): + from dokimi import llm + + def failing(prompt, model, **_kwargs): + raise RuntimeError("provider refused") + + monkeypatch.setattr(llm, "llm_call_with_usage", failing) + payload = _model_run(tmp_path) + assert payload["cost"]["llm_calls"] == 0 + out = capsys.readouterr().out + assert "cost 0 calls\n" in out + assert "USD not known" not in out + + +def test_a_call_with_no_usage_makes_the_total_unknown_not_the_priced_subset( + tmp_path, monkeypatch, capsys): + from dokimi import llm + from dokimi.metrics.usage import UsageRecord + + replies = iter([None, UsageRecord(prompt_tokens=10, completion_tokens=1, cost_usd=0.001)]) + monkeypatch.setattr(llm, "llm_call_with_usage", + lambda prompt, model, **_kw: ('{"pick": 1}', next(replies))) + payload = _model_run(tmp_path) + assert payload["cost"]["llm_calls"] == 2 + assert payload["cost"]["usd"] is None + assert "$0.0010" not in capsys.readouterr().out + + +def test_replayed_calls_are_counted_apart_and_not_billed(tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5, + cost_usd=0.002, replayed=True)) + payload = _model_run(tmp_path) + cost = payload["cost"] + assert (cost["llm_calls"], cost["replayed_calls"]) == (2, 2) + assert (cost["input_tokens"], cost["output_tokens"], cost["usd"]) == (0, 0, 0.0) + assert "2 replayed from the cache and not counted" in capsys.readouterr().out From 54eb79c101ad87147363028351024d782e1fe50c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 14:08:29 -0700 Subject: [PATCH 04/15] refactor(novelty): extract the routing summary; chore(growth): raise commands/novelty.py for W1 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/novelty.py | 7 ++++++- dokimi/scripts/growth_baseline.json | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index b3557942f..1d2716fd9 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -347,10 +347,15 @@ def card(ref: str) -> tuple[Candidate, str]: out.write_text(json.dumps(payload, indent=2), encoding="utf-8") print(f"OK Saved routing measurement: {out}") + _print_routing_summary(payload, len(competitors), self_derived) + + +def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool) -> None: + """The lexical line, the model line and its caveats, printed after a routing run.""" lexical = payload["lexical"] print(f"lexical R@1 {lexical['recall_at_1']} R@3 {lexical['recall_at_3']} " f"MRR {lexical['mrr']} over {lexical['n_queries_scored']} of " - f"{lexical['n_queries_given']} queries vs {len(competitors)} competitors" + f"{lexical['n_queries_given']} queries vs {n_competitors} competitors" + (" (R@3 needs more than 3 candidates to mean anything)" if lexical["recall_at_3"] is None else "")) if "model" in payload: diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 2b0f5fe8c..aefe430e3 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 827}, + "dokimi/commands/novelty.py": {"lines": 847, "rationale": "W1 adds routing cost output; file split tracked separately to avoid conflicts across the W-series"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 927, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1112}, From f6df38a838e030e65c6f3bb7b1075504e5051e5f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 22 Oct 2026 14:08:30 -0700 Subject: [PATCH 05/15] chore(growth): lower dokimi/commands/novelty.py::cmd_novelty_routing baseline after the extraction Signed-off-by: Nikolay Petrov --- dokimi/scripts/growth_baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index aefe430e3..4bf2a57f3 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -26,7 +26,7 @@ "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 18}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 32, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 14}, + "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 12}, "dokimi/commands/repo.py::cmd_repo_scan": {"complexity": 13}, "dokimi/commands/repo.py::cmd_repo_validate": {"complexity": 11, "rationale": "grew on main in #150 (repo validate --fail-on-warning) from 10 to 11 before the ratchet landed; split it in its own PR"}, "dokimi/commands/report.py::cmd_report_model_compare": {"complexity": 12}, From a5b007e47d6a87bd41cf51a273e5662aef0f307e Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 23 Oct 2026 02:28:07 -0700 Subject: [PATCH 06/15] test(novelty): pin roll_up_cost cache and replay totals across roles roll_up_cost sums the new cache_read/cache_write/replayed_calls fields over every role, but only single-client tests covered them, so a total taken from the first role alone (what novelty analyze would then report) passed. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_novelty_measure.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/dokimi/tests/test_novelty_measure.py b/dokimi/tests/test_novelty_measure.py index a94373a83..152b18104 100644 --- a/dokimi/tests/test_novelty_measure.py +++ b/dokimi/tests/test_novelty_measure.py @@ -221,6 +221,30 @@ def test_cost_rolls_up_per_role_and_stays_none_when_unpriced(): assert by_role["judge"]["usd"] is None +def test_cost_rolls_up_cache_tokens_and_replays_across_every_role(): + def client(role, records): + rows = iter(records) + return LLMClient(model=role, role=role, transport=lambda p, s: ("x", next(rows))) + + extractor = client("extractor", [ + UsageRecord(prompt_tokens=10, cache_read_tokens=4, cache_write_tokens=1, cost_usd=0.1), + UsageRecord(prompt_tokens=10, cost_usd=0.1, replayed=True), + ]) + judge = client("judge", [ + UsageRecord(prompt_tokens=20, cache_read_tokens=8, cache_write_tokens=4, cost_usd=0.2), + UsageRecord(prompt_tokens=20, cost_usd=0.2, replayed=True), + ]) + for c in (extractor, judge): + c.text("p") + c.text("p") + total = roll_up_cost([extractor, judge]) + # analyze rolls up several roles: a total taken from one of them would pass a + # single-client test, so each new field is summed over two. + assert total["llm_calls"] == 4 + assert total["cache_read_tokens"] == 12 + assert total["cache_write_tokens"] == 5 + assert total["replayed_calls"] == 2 + def test_a_failed_call_is_a_dropped_pair_not_a_graded_answer(): def answer(prompt, system): if prompt.startswith("Reference material"): From 54f4ca8b5d01a15f32c7cbb25286da8b32cd009c Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 23 Oct 2026 12:59:42 +0000 Subject: [PATCH 07/15] style(tests): add the second blank line before the next top-level test Signed-off-by: Anton Volkov --- dokimi/tests/test_novelty_measure.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/tests/test_novelty_measure.py b/dokimi/tests/test_novelty_measure.py index 152b18104..7191db323 100644 --- a/dokimi/tests/test_novelty_measure.py +++ b/dokimi/tests/test_novelty_measure.py @@ -245,6 +245,7 @@ def client(role, records): assert total["cache_write_tokens"] == 5 assert total["replayed_calls"] == 2 + def test_a_failed_call_is_a_dropped_pair_not_a_graded_answer(): def answer(prompt, system): if prompt.startswith("Reference material"): From 2594a80d8e2464b7d39f9913817df6fda9fa029f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 23 Oct 2026 12:41:44 -0700 Subject: [PATCH 08/15] test(novelty): pin usd None for no calls and replayed cache tokens; say 'this run paid for' Signed-off-by: Nikolay Petrov --- .../changelog.d/20261021_120000_napetrov_routing-usage.md | 2 +- dokimi/docs/skill-novelty.md | 2 +- dokimi/dokimi/novelty/client.py | 3 ++- dokimi/dokimi/schemas/novelty.v1.json | 2 +- dokimi/tests/test_novelty_routing.py | 6 +++++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md index ec8359189..77ee46b54 100644 --- a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md +++ b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md @@ -1,3 +1,3 @@ ### Added -- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals, and `usd` is also `null` when a call returned no usage. +- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals, and `usd` is also `null` when a call returned no usage. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index b1dd6df2e..e7fbe6337 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -609,7 +609,7 @@ Two numbers, reported side by side and never merged: counted, and one failed query does not lose the rest. A `--model` run also writes a `cost` block (calls, input and output tokens, cache read and -write tokens, and `usd`, which is `null` when any call had no known price) and prints the +write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the same totals on stdout. Cache tokens are reported separately from input tokens, and depending on the provider they may or may not be counted inside `input_tokens`. A run without `--model` makes no calls and has no `cost` block. `usd` is also `null` when a call returned no usage, and a diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index cec8039cd..4f53b46b0 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -161,7 +161,8 @@ def json_array(self, prompt: str, *, system: str | None = None) -> list[Any]: def cost_summary(self) -> dict[str, Any]: """Roll up this client's usage. - ``usd`` stays ``None`` when any call had unknown pricing or returned no usage, + ``usd`` stays ``None`` when any call this run paid for had unknown pricing or + returned no usage (a replayed call is not paid for, so it cannot make it unknown), rather than silently reporting the sum of the priced subset as the total. Calls served from the response cache are counted in ``replayed_calls`` and left out of the token and dollar totals. diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index f302528d9..5c7df79d0 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -452,7 +452,7 @@ "replayed_calls": {"type": "integer", "minimum": 0, "description": "Calls served from the response cache; left out of the token and usd totals."}, "usd": { "type": ["number", "null"], - "description": "null when any call had unknown pricing — distinct from a genuinely free run." + "description": "null when any call this run paid for had unknown pricing — distinct from a genuinely free run." }, "by_role": {"type": "array", "items": {"type": "object"}} } diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index b644e28fc..4c4702a2c 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -545,7 +545,8 @@ def test_a_model_run_records_its_calls_tokens_and_cost(tmp_path, monkeypatch, ca assert payload["cost"]["cache_write_tokens"] == 60 assert payload["cost"]["usd"] == 0.004 assert ("cost 2 calls, 200 input / 10 output tokens, cache read 120 / write 60, " - "$0.0040") in capsys.readouterr().out + "$0.0040") in (out := capsys.readouterr().out) + assert "replayed" not in out def test_a_model_run_with_an_unpriced_call_reports_no_usd_figure(tmp_path, monkeypatch, capsys): @@ -578,6 +579,7 @@ def failing(prompt, model, **_kwargs): monkeypatch.setattr(llm, "llm_call_with_usage", failing) payload = _model_run(tmp_path) assert payload["cost"]["llm_calls"] == 0 + assert payload["cost"]["usd"] is None # no calls: nothing known, not "free" out = capsys.readouterr().out assert "cost 0 calls\n" in out assert "USD not known" not in out @@ -601,11 +603,13 @@ def test_replayed_calls_are_counted_apart_and_not_billed(tmp_path, monkeypatch, from dokimi.metrics.usage import UsageRecord _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5, + cache_read_tokens=60, cache_write_tokens=30, cost_usd=0.002, replayed=True)) payload = _model_run(tmp_path) cost = payload["cost"] assert (cost["llm_calls"], cost["replayed_calls"]) == (2, 2) assert (cost["input_tokens"], cost["output_tokens"], cost["usd"]) == (0, 0, 0.0) + assert (cost["cache_read_tokens"], cost["cache_write_tokens"]) == (0, 0) assert "2 replayed from the cache and not counted" in capsys.readouterr().out From 6ca352cf62bac315052efd7ab77effb366183f2a Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 26 Oct 2026 13:06:35 +0000 Subject: [PATCH 09/15] test(novelty): compare the routing cost line as a whole line Signed-off-by: Anton Volkov --- dokimi/tests/test_novelty_routing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index 4c4702a2c..fd83823a3 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -544,8 +544,9 @@ def test_a_model_run_records_its_calls_tokens_and_cost(tmp_path, monkeypatch, ca assert payload["cost"]["cache_read_tokens"] == 120 assert payload["cost"]["cache_write_tokens"] == 60 assert payload["cost"]["usd"] == 0.004 + out = capsys.readouterr().out assert ("cost 2 calls, 200 input / 10 output tokens, cache read 120 / write 60, " - "$0.0040") in (out := capsys.readouterr().out) + "$0.0040") in out.splitlines() assert "replayed" not in out From 415c5cd8c00b1f69a039c53876ef38269ec6e83a Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 26 Oct 2026 13:06:54 +0000 Subject: [PATCH 10/15] docs(novelty): say llm_calls still counts replayed calls Signed-off-by: Anton Volkov --- dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md | 2 +- dokimi/docs/skill-novelty.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md index 77ee46b54..d78ef64e3 100644 --- a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md +++ b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md @@ -1,3 +1,3 @@ ### Added -- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals, and `usd` is also `null` when a call returned no usage. +- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals (`llm_calls` still counts them), and `usd` is also `null` when a call returned no usage. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index c11569ab9..a424fdd4d 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -615,7 +615,8 @@ same totals on stdout. Cache tokens are reported separately from input tokens, a the provider they may or may not be counted inside `input_tokens`. A run without `--model` makes no calls and has no `cost` block. `usd` is also `null` when a call returned no usage, and a call that raised is not counted. A call replayed from the response cache is counted in -`replayed_calls` and left out of the token and `usd` totals, since this run did not pay for it. +`replayed_calls` and left out of the token and `usd` totals, since this run did not pay for it; +`llm_calls` still counts it. `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given` From fb43f3368919e8df3289a9cefb4d04883addef64 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 26 Oct 2026 21:44:51 -0700 Subject: [PATCH 11/15] fix(novelty): name replayed calls in the analyze report; file the analyze change under Changed The analyze cost totals now leave replayed calls out (cost_summary is shared), so the Markdown cost line says how many were replayed, the benchmarking doc says the run totals are this run's spend, and the changelog fragment files the analyze part under Changed. Tests pin the whole replayed routing line, a live + replayed + usage-less mix (3 calls, 1 replayed, usd None), and the cross-role output_tokens sum. Signed-off-by: Nikolay Petrov --- .../20261021_120000_napetrov_routing-usage.md | 6 +++- dokimi/docs/benchmarking-and-comparison.md | 5 +++- dokimi/dokimi/novelty/report.py | 5 +++- dokimi/tests/test_novelty_measure.py | 9 ++++-- dokimi/tests/test_novelty_report.py | 10 +++++++ dokimi/tests/test_novelty_routing_cost.py | 29 ++++++++++++++++++- 6 files changed, 57 insertions(+), 7 deletions(-) diff --git a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md index d78ef64e3..3de299e9a 100644 --- a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md +++ b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md @@ -1,3 +1,7 @@ ### Added -- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. The cache token totals are also added to the per-role cost summary, so `novelty analyze` reports them in its `cost` block. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals (`llm_calls` still counts them), and `usd` is also `null` when a call returned no usage. +- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals (`llm_calls` still counts them), and `usd` is also `null` when a call returned no usage. + +### Changed + +- **`novelty analyze` cost totals are what this run spent** — the per-role cost summary now leaves calls replayed from the response cache out of `input_tokens`, `output_tokens` and `usd` (they were summed from the stored records before) and counts them in a new `replayed_calls`; `llm_calls` still counts them. `usd` is now `null` when a call returned no usage, where it was the sum of the priced calls. The block also gains `cache_read_tokens` and `cache_write_tokens`, and the Markdown report's cost line names the replayed calls when there are any. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index b45f3950c..31fb03f75 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -754,7 +754,10 @@ that never hits. A replayed entry reports the **stored** usage record — tokens, cost and latency as measured when the call was really made, not `$0`/`0s`, so cost stays -comparable across runs. The flip side: latency from a cached run is a recording, +comparable across runs. The run's `cost` totals (`novelty analyze`, `novelty +routing --model`) are what this run spent, so they leave replayed records out of +the token and `usd` sums and count them in `replayed_calls`; `llm_calls` still +counts them. The flip side: latency from a cached run is a recording, not a measurement. That is why this cache is opt-in, and why it should stay off for any run whose latency numbers you intend to quote. diff --git a/dokimi/dokimi/novelty/report.py b/dokimi/dokimi/novelty/report.py index 894a97fa1..57590833a 100644 --- a/dokimi/dokimi/novelty/report.py +++ b/dokimi/dokimi/novelty/report.py @@ -489,7 +489,10 @@ def render_novelty_markdown(artifact: dict) -> str: lines.append( f"Cost: {cost.get('llm_calls', 0)} LLM calls, " f"{cost.get('input_tokens', 0)} in / {cost.get('output_tokens', 0)} out, " - f"usd {cost.get('usd')}." + f"usd {cost.get('usd')}" + + (f", {cost['replayed_calls']} replayed from the cache and not counted" + if cost.get("replayed_calls") else "") + + "." ) lines.append("") diff --git a/dokimi/tests/test_novelty_measure.py b/dokimi/tests/test_novelty_measure.py index 7191db323..1aee35324 100644 --- a/dokimi/tests/test_novelty_measure.py +++ b/dokimi/tests/test_novelty_measure.py @@ -227,11 +227,13 @@ def client(role, records): return LLMClient(model=role, role=role, transport=lambda p, s: ("x", next(rows))) extractor = client("extractor", [ - UsageRecord(prompt_tokens=10, cache_read_tokens=4, cache_write_tokens=1, cost_usd=0.1), - UsageRecord(prompt_tokens=10, cost_usd=0.1, replayed=True), + UsageRecord(prompt_tokens=10, completion_tokens=3, cache_read_tokens=4, + cache_write_tokens=1, cost_usd=0.1), + UsageRecord(prompt_tokens=10, completion_tokens=50, cost_usd=0.1, replayed=True), ]) judge = client("judge", [ - UsageRecord(prompt_tokens=20, cache_read_tokens=8, cache_write_tokens=4, cost_usd=0.2), + UsageRecord(prompt_tokens=20, completion_tokens=4, cache_read_tokens=8, + cache_write_tokens=4, cost_usd=0.2), UsageRecord(prompt_tokens=20, cost_usd=0.2, replayed=True), ]) for c in (extractor, judge): @@ -241,6 +243,7 @@ def client(role, records): # analyze rolls up several roles: a total taken from one of them would pass a # single-client test, so each new field is summed over two. assert total["llm_calls"] == 4 + assert total["output_tokens"] == 7 # live rows of both roles; replayed 50 left out assert total["cache_read_tokens"] == 12 assert total["cache_write_tokens"] == 5 assert total["replayed_calls"] == 2 diff --git a/dokimi/tests/test_novelty_report.py b/dokimi/tests/test_novelty_report.py index b360b3cc7..508d1bb0b 100644 --- a/dokimi/tests/test_novelty_report.py +++ b/dokimi/tests/test_novelty_report.py @@ -59,3 +59,13 @@ def test_found_conflicts_lead_whatever_the_coverage(): assert "> **Conflicts first: 2 conflicting atoms, 1 pairs.**" in markdown assert "Conflicts not cleared" not in markdown assert "| harmful_conflicting | 2 |" in markdown + + +@pytest.mark.parametrize("cost, line", [ + ({"llm_calls": 2, "input_tokens": 200, "output_tokens": 20, "usd": 0.02}, + "Cost: 2 LLM calls, 200 in / 20 out, usd 0.02."), + ({"llm_calls": 2, "input_tokens": 0, "output_tokens": 0, "usd": 0, "replayed_calls": 2}, + "Cost: 2 LLM calls, 0 in / 0 out, usd 0, 2 replayed from the cache and not counted."), +], ids=["live", "replayed"]) +def test_the_cost_line_names_replayed_calls_only_when_there_are_some(cost, line): + assert line in render_novelty_markdown(_artifact(cost=cost)).splitlines() diff --git a/dokimi/tests/test_novelty_routing_cost.py b/dokimi/tests/test_novelty_routing_cost.py index 9537a2474..878d80d7b 100644 --- a/dokimi/tests/test_novelty_routing_cost.py +++ b/dokimi/tests/test_novelty_routing_cost.py @@ -121,4 +121,31 @@ def test_replayed_calls_are_counted_apart_and_not_billed(tmp_path, monkeypatch, assert (cost["llm_calls"], cost["replayed_calls"]) == (2, 2) assert (cost["input_tokens"], cost["output_tokens"], cost["usd"]) == (0, 0, 0.0) assert (cost["cache_read_tokens"], cost["cache_write_tokens"]) == (0, 0) - assert "2 replayed from the cache and not counted" in capsys.readouterr().out + assert ("cost 2 calls, 0 input / 0 output tokens, cache read 0 / write 0, $0.0000, " + "2 replayed from the cache and not counted") in capsys.readouterr().out.splitlines() + + +def test_a_live_a_replayed_and_a_usageless_call_are_told_apart(tmp_path, monkeypatch, capsys): + from dokimi import llm + from dokimi.metrics.usage import UsageRecord + + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary", "tune a slow query", "trace a syscall"]), + encoding="utf-8") + replies = iter([UsageRecord(prompt_tokens=10, completion_tokens=1, cost_usd=0.001), + UsageRecord(prompt_tokens=10, completion_tokens=1, cost_usd=0.001, + replayed=True), + None]) + monkeypatch.setattr(llm, "llm_call_with_usage", + lambda prompt, model, **_kw: ('{"pick": 1}', next(replies))) + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary with perf"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--model", "selector-m", "--out", str(out)) + args.func(args) + cost = json.loads(out.read_text(encoding="utf-8"))["cost"] + assert (cost["llm_calls"], cost["replayed_calls"], cost["usd"]) == (3, 1, None) + assert ("cost 3 calls, 10 input / 1 output tokens, cache read 0 / write 0, " + "USD not known: a call had no price or no usage, " + "1 replayed from the cache and not counted") in capsys.readouterr().out.splitlines() From 5db94ee94b1cc599b6529f20c8ae4b09b4754580 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 26 Oct 2026 21:54:34 -0700 Subject: [PATCH 12/15] test(novelty): SPDX header on the routing cost test file Signed-off-by: Nikolay Petrov --- dokimi/tests/test_novelty_routing_cost.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dokimi/tests/test_novelty_routing_cost.py b/dokimi/tests/test_novelty_routing_cost.py index 878d80d7b..6dbac233d 100644 --- a/dokimi/tests/test_novelty_routing_cost.py +++ b/dokimi/tests/test_novelty_routing_cost.py @@ -1,3 +1,6 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + """The usage and cost block ``novelty routing --model`` writes and prints.""" from __future__ import annotations From 9993768242bbcdc27ec3285390fc348645caa30a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 26 Oct 2026 21:56:41 -0700 Subject: [PATCH 13/15] docs(schema): usd is also null for a call with no usage or no calls Signed-off-by: Nikolay Petrov --- dokimi/dokimi/schemas/novelty.v1.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index 52b87c728..b4f737923 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -453,7 +453,7 @@ "replayed_calls": {"type": "integer", "minimum": 0, "description": "Calls served from the response cache; left out of the token and usd totals."}, "usd": { "type": ["number", "null"], - "description": "null when any call this run paid for had unknown pricing — distinct from a genuinely free run." + "description": "null when any call this run paid for had unknown pricing, a call returned no usage, or no call was made — distinct from a genuinely free run. Replayed calls are left out of the sum." }, "by_role": {"type": "array", "items": {"type": "object"}} } From c880fc426a6da8b32c0f6af9fca3ac88235fff2e Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 27 Oct 2026 11:22:54 +0000 Subject: [PATCH 14/15] test(novelty): report case with exactly one replayed call Signed-off-by: Anton Volkov --- dokimi/tests/test_novelty_report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/tests/test_novelty_report.py b/dokimi/tests/test_novelty_report.py index 508d1bb0b..24a5994ef 100644 --- a/dokimi/tests/test_novelty_report.py +++ b/dokimi/tests/test_novelty_report.py @@ -64,8 +64,8 @@ def test_found_conflicts_lead_whatever_the_coverage(): @pytest.mark.parametrize("cost, line", [ ({"llm_calls": 2, "input_tokens": 200, "output_tokens": 20, "usd": 0.02}, "Cost: 2 LLM calls, 200 in / 20 out, usd 0.02."), - ({"llm_calls": 2, "input_tokens": 0, "output_tokens": 0, "usd": 0, "replayed_calls": 2}, - "Cost: 2 LLM calls, 0 in / 0 out, usd 0, 2 replayed from the cache and not counted."), + ({"llm_calls": 3, "input_tokens": 10, "output_tokens": 1, "usd": 0.001, "replayed_calls": 1}, + "Cost: 3 LLM calls, 10 in / 1 out, usd 0.001, 1 replayed from the cache and not counted."), ], ids=["live", "replayed"]) def test_the_cost_line_names_replayed_calls_only_when_there_are_some(cost, line): assert line in render_novelty_markdown(_artifact(cost=cost)).splitlines() From 345c91b2e243850f757b356cdbdbe61dc0c38dac Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 27 Oct 2026 05:58:57 -0700 Subject: [PATCH 15/15] fix(novelty): print a sub-cent routing spend as <$0.0001, not $0.0000 A two-query run on a small model costs about $0.00004, which the cost line printed as $0.0000, the same figure as an all-replayed run. Use the "<$0.0001" form task_runs_report already uses for the same case. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/commands/novelty.py | 4 ++-- dokimi/tests/test_novelty_routing_cost.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 8dfeef8b8..3470383ce 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -416,8 +416,8 @@ def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool if not cost["llm_calls"]: print("cost 0 calls") else: - spent = (f"${cost['usd']:.4f}" if cost["usd"] is not None - else "USD not known: a call had no price or no usage") + spent = ("USD not known: a call had no price or no usage" if cost["usd"] is None + else "<$0.0001" if 0 < cost["usd"] < 0.00005 else f"${cost['usd']:.4f}") replayed = (f", {cost['replayed_calls']} replayed from the cache and not counted" if cost["replayed_calls"] else "") print(f"cost {cost['llm_calls']} calls, {cost['input_tokens']} input / " diff --git a/dokimi/tests/test_novelty_routing_cost.py b/dokimi/tests/test_novelty_routing_cost.py index 6dbac233d..3813e77a4 100644 --- a/dokimi/tests/test_novelty_routing_cost.py +++ b/dokimi/tests/test_novelty_routing_cost.py @@ -63,6 +63,19 @@ def test_a_model_run_records_its_calls_tokens_and_cost(tmp_path, monkeypatch, ca assert "replayed" not in out +def test_a_paid_run_under_a_hundredth_of_a_cent_is_not_printed_as_free( + tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + # Two short routing calls on a small model cost about this much; $0.0000 reads as free. + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=50, completion_tokens=1, + cost_usd=0.00001)) + payload = _model_run(tmp_path) + assert payload["cost"]["usd"] == 0.00002 + assert ("cost 2 calls, 100 input / 2 output tokens, cache read 0 / write 0, " + "<$0.0001") in capsys.readouterr().out.splitlines() + + def test_a_model_run_with_an_unpriced_call_reports_no_usd_figure(tmp_path, monkeypatch, capsys): from dokimi.metrics.usage import UsageRecord