diff --git a/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md new file mode 100644 index 000000000..3de299e9a --- /dev/null +++ b/dokimi/changelog.d/20261021_120000_napetrov_routing-usage.md @@ -0,0 +1,7 @@ +### Added + +- **Usage and cost for `novelty routing --model`** — a model run now writes a `cost` block into `routing.json` (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the same totals on stdout, so a routing measurement states what it cost. Calls replayed from the response cache are counted in `replayed_calls` and left out of the token and `usd` totals (`llm_calls` still counts them), and `usd` is also `null` when a call returned no usage. + +### Changed + +- **`novelty analyze` cost totals are what this run spent** — the per-role cost summary now leaves calls replayed from the response cache out of `input_tokens`, `output_tokens` and `usd` (they were summed from the stored records before) and counts them in a new `replayed_calls`; `llm_calls` still counts them. `usd` is now `null` when a call returned no usage, where it was the sum of the priced calls. The block also gains `cache_read_tokens` and `cache_write_tokens`, and the Markdown report's cost line names the replayed calls when there are any. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index b45f3950c..31fb03f75 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -754,7 +754,10 @@ that never hits. A replayed entry reports the **stored** usage record — tokens, cost and latency as measured when the call was really made, not `$0`/`0s`, so cost stays -comparable across runs. The flip side: latency from a cached run is a recording, +comparable across runs. The run's `cost` totals (`novelty analyze`, `novelty +routing --model`) are what this run spent, so they leave replayed records out of +the token and `usd` sums and count them in `replayed_calls`; `llm_calls` still +counts them. The flip side: latency from a cached run is a recording, not a measurement. That is why this cache is opt-in, and why it should stay off for any run whose latency numbers you intend to quote. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 3788ed974..7b761ef90 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -630,6 +630,15 @@ Two numbers, reported side by side and never merged: queries x repeats, `n_abstained` and `n_failed` count calls, and each `per_query` row carries its `repeat` index. +A `--model` run also writes a `cost` block (calls, input and output tokens, cache read and +write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the +same totals on stdout. Cache tokens are reported separately from input tokens, and depending on +the provider they may or may not be counted inside `input_tokens`. A run without `--model` makes +no calls and has no `cost` block. `usd` is also `null` when a call returned no usage, and a +call that raised is not counted. A call replayed from the response cache is counted in +`replayed_calls` and left out of the token and `usd` totals, since this run did not pay for it; +`llm_calls` still counts it. + `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given` next to `n_queries_scored`, since a query with no content words is dropped rather than diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 1ed02cae4..3470383ce 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -361,18 +361,22 @@ def card(ref: str) -> tuple[Candidate, str]: "lexical": lexical, } if args.model: + from dokimi.novelty.client import roll_up_cost + selector = _client(args.model, args.provider, "selector", args) payload["model"] = dict( model_routing(candidates, queries, client=selector, gold=0, seed=args.seed, repeats=args.repeats), queries_self_derived=self_derived, ) + payload["cost"] = roll_up_cost([selector]) out.write_text(json.dumps(payload, indent=2), encoding="utf-8") print(f"OK Saved routing measurement: {out}") _print_routing_summary(payload, len(competitors), self_derived) + def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool) -> None: """The lexical line, the model line and their caveats, printed after a routing run.""" # Every rate is printed with the number of queries behind it, and a self-derived one @@ -408,11 +412,22 @@ def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool f"{noise['half_range']} (a crude sampling floor)") print("The model figure is an upper bound on harness behaviour: a harness that " "truncates descriptions in its own listing selects on less text.") + cost = payload["cost"] + if not cost["llm_calls"]: + print("cost 0 calls") + else: + spent = ("USD not known: a call had no price or no usage" if cost["usd"] is None + else "<$0.0001" if 0 < cost["usd"] < 0.00005 else f"${cost['usd']:.4f}") + replayed = (f", {cost['replayed_calls']} replayed from the cache and not counted" + if cost["replayed_calls"] else "") + print(f"cost {cost['llm_calls']} calls, {cost['input_tokens']} input / " + f"{cost['output_tokens']} output tokens, cache read " + f"{cost['cache_read_tokens']} / write {cost['cache_write_tokens']}, " + f"{spent}{replayed}") if self_derived: print("Self-derived queries: read as triage only.") - def _refuse_no_queries(args: argparse.Namespace, queries: list[str], self_derived: bool, out: Path, n_candidates: int) -> NoReturn: """Write the ``no_queries`` record, then exit saying why no query was scored.""" diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index ae65133a1..386467ee4 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -167,13 +167,22 @@ def json_array(self, prompt: str, *, system: str | None = None) -> list[Any]: def cost_summary(self) -> dict[str, Any]: """Roll up this client's usage. - ``usd`` stays ``None`` when any call had unknown pricing, rather than - silently reporting the sum of the priced subset as the total. + ``usd`` stays ``None`` when any call this run paid for had unknown pricing or + returned no usage (a replayed call is not paid for, so it cannot make it unknown), + rather than silently reporting the sum of the priced subset as the total. + Calls served from the response cache are counted in ``replayed_calls`` and + left out of the token and dollar totals. """ - input_tokens = sum(u.prompt_tokens for u in self.usage) - output_tokens = sum(u.completion_tokens for u in self.usage) - costs = [u.cost_usd for u in self.usage] - priced = costs and None not in costs + # A replayed row is the response cache's copy of an earlier call: its tokens and + # price were not spent by this process, so it is counted apart, as the spend cap does. + billed = [u for u in self.usage if not u.replayed] + input_tokens = sum(u.prompt_tokens for u in billed) + output_tokens = sum(u.completion_tokens for u in billed) + cache_read = sum(u.cache_read_tokens for u in billed) + cache_write = sum(u.cache_write_tokens for u in billed) + costs = [u.cost_usd for u in billed] + # A call that returned no usage row has no price either, so it makes the total unknown. + priced = self.calls > 0 and len(self.usage) == self.calls and None not in costs usd = round(sum(c for c in costs if c is not None), 6) if priced else None return { "model": self.model, @@ -182,6 +191,9 @@ def cost_summary(self) -> dict[str, Any]: "llm_calls": self.calls, "input_tokens": input_tokens, "output_tokens": output_tokens, + "cache_read_tokens": cache_read, + "cache_write_tokens": cache_write, + "replayed_calls": len(self.usage) - len(billed), "usd": usd, } @@ -238,6 +250,9 @@ def roll_up_cost(clients: list[LLMClient]) -> dict[str, Any]: "llm_calls": sum(row["llm_calls"] for row in per_role), "input_tokens": sum(row["input_tokens"] for row in per_role), "output_tokens": sum(row["output_tokens"] for row in per_role), + "cache_read_tokens": sum(row["cache_read_tokens"] for row in per_role), + "cache_write_tokens": sum(row["cache_write_tokens"] for row in per_role), + "replayed_calls": sum(row["replayed_calls"] for row in per_role), "usd": round(sum(p for p in usd_parts if p is not None), 6) if usd_parts and None not in usd_parts else None, diff --git a/dokimi/dokimi/novelty/report.py b/dokimi/dokimi/novelty/report.py index 894a97fa1..57590833a 100644 --- a/dokimi/dokimi/novelty/report.py +++ b/dokimi/dokimi/novelty/report.py @@ -489,7 +489,10 @@ def render_novelty_markdown(artifact: dict) -> str: lines.append( f"Cost: {cost.get('llm_calls', 0)} LLM calls, " f"{cost.get('input_tokens', 0)} in / {cost.get('output_tokens', 0)} out, " - f"usd {cost.get('usd')}." + f"usd {cost.get('usd')}" + + (f", {cost['replayed_calls']} replayed from the cache and not counted" + if cost.get("replayed_calls") else "") + + "." ) lines.append("") diff --git a/dokimi/dokimi/schemas/novelty.v1.json b/dokimi/dokimi/schemas/novelty.v1.json index 47358cad5..b4f737923 100644 --- a/dokimi/dokimi/schemas/novelty.v1.json +++ b/dokimi/dokimi/schemas/novelty.v1.json @@ -448,9 +448,12 @@ "llm_calls": {"type": "integer", "minimum": 0}, "input_tokens": {"type": "integer", "minimum": 0}, "output_tokens": {"type": "integer", "minimum": 0}, + "cache_read_tokens": {"type": "integer", "minimum": 0}, + "cache_write_tokens": {"type": "integer", "minimum": 0}, + "replayed_calls": {"type": "integer", "minimum": 0, "description": "Calls served from the response cache; left out of the token and usd totals."}, "usd": { "type": ["number", "null"], - "description": "null when any call had unknown pricing — distinct from a genuinely free run." + "description": "null when any call this run paid for had unknown pricing, a call returned no usage, or no call was made — distinct from a genuinely free run. Replayed calls are left out of the sum." }, "by_role": {"type": "array", "items": {"type": "object"}} } diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 72aeb637b..2dd85332a 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 984, "rationale": "estimate-only projection (expected_usd, arm scaling, extrapolation flag) in novelty CLI, on top of main's W-series and #173 lines; file split tracked separately"}, + "dokimi/commands/novelty.py": {"lines": 999, "rationale": "estimate-only projection (expected_usd, arm scaling, extrapolation flag) in novelty CLI, on top of main's W-series and #173 lines, plus W1 routing cost output (#164); file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 900, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, diff --git a/dokimi/tests/test_novelty_measure.py b/dokimi/tests/test_novelty_measure.py index a94373a83..1aee35324 100644 --- a/dokimi/tests/test_novelty_measure.py +++ b/dokimi/tests/test_novelty_measure.py @@ -221,6 +221,34 @@ def test_cost_rolls_up_per_role_and_stays_none_when_unpriced(): assert by_role["judge"]["usd"] is None +def test_cost_rolls_up_cache_tokens_and_replays_across_every_role(): + def client(role, records): + rows = iter(records) + return LLMClient(model=role, role=role, transport=lambda p, s: ("x", next(rows))) + + extractor = client("extractor", [ + UsageRecord(prompt_tokens=10, completion_tokens=3, cache_read_tokens=4, + cache_write_tokens=1, cost_usd=0.1), + UsageRecord(prompt_tokens=10, completion_tokens=50, cost_usd=0.1, replayed=True), + ]) + judge = client("judge", [ + UsageRecord(prompt_tokens=20, completion_tokens=4, cache_read_tokens=8, + cache_write_tokens=4, cost_usd=0.2), + UsageRecord(prompt_tokens=20, cost_usd=0.2, replayed=True), + ]) + for c in (extractor, judge): + c.text("p") + c.text("p") + total = roll_up_cost([extractor, judge]) + # analyze rolls up several roles: a total taken from one of them would pass a + # single-client test, so each new field is summed over two. + assert total["llm_calls"] == 4 + assert total["output_tokens"] == 7 # live rows of both roles; replayed 50 left out + assert total["cache_read_tokens"] == 12 + assert total["cache_write_tokens"] == 5 + assert total["replayed_calls"] == 2 + + def test_a_failed_call_is_a_dropped_pair_not_a_graded_answer(): def answer(prompt, system): if prompt.startswith("Reference material"): diff --git a/dokimi/tests/test_novelty_report.py b/dokimi/tests/test_novelty_report.py index b360b3cc7..24a5994ef 100644 --- a/dokimi/tests/test_novelty_report.py +++ b/dokimi/tests/test_novelty_report.py @@ -59,3 +59,13 @@ def test_found_conflicts_lead_whatever_the_coverage(): assert "> **Conflicts first: 2 conflicting atoms, 1 pairs.**" in markdown assert "Conflicts not cleared" not in markdown assert "| harmful_conflicting | 2 |" in markdown + + +@pytest.mark.parametrize("cost, line", [ + ({"llm_calls": 2, "input_tokens": 200, "output_tokens": 20, "usd": 0.02}, + "Cost: 2 LLM calls, 200 in / 20 out, usd 0.02."), + ({"llm_calls": 3, "input_tokens": 10, "output_tokens": 1, "usd": 0.001, "replayed_calls": 1}, + "Cost: 3 LLM calls, 10 in / 1 out, usd 0.001, 1 replayed from the cache and not counted."), +], ids=["live", "replayed"]) +def test_the_cost_line_names_replayed_calls_only_when_there_are_some(cost, line): + assert line in render_novelty_markdown(_artifact(cost=cost)).splitlines() diff --git a/dokimi/tests/test_novelty_routing_cost.py b/dokimi/tests/test_novelty_routing_cost.py new file mode 100644 index 000000000..3813e77a4 --- /dev/null +++ b/dokimi/tests/test_novelty_routing_cost.py @@ -0,0 +1,167 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The usage and cost block ``novelty routing --model`` writes and prints.""" + +from __future__ import annotations + +import json + +from dokimi.cli import build_parser + + +def _parse(*argv): + return build_parser().parse_args(list(argv)) + + +def _skill(tmp_path, name, description, body="# B\n\n## A section here\n"): + root = tmp_path / name + root.mkdir() + (root / "SKILL.md").write_text( + f"---\nname: {name}\ndescription: {description}\n---\n\n{body}", encoding="utf-8") + return f"skill:{root}" + + +def _stub_provider(monkeypatch, usage): + """Replace the provider call the --model path makes; every reply picks card 1.""" + from dokimi import llm + + def stub(prompt, model, **_kwargs): + return '{"pick": 1}', usage + + monkeypatch.setattr(llm, "llm_call_with_usage", stub) + + +def _model_run(tmp_path): + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary", "tune a slow query"]), encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary with perf"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--model", "selector-m", "--out", str(out)) + args.func(args) + return json.loads(out.read_text(encoding="utf-8")) + + +def test_a_model_run_records_its_calls_tokens_and_cost(tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5, + cache_read_tokens=60, cache_write_tokens=30, + cost_usd=0.002)) + payload = _model_run(tmp_path) + assert payload["cost"]["llm_calls"] == 2 + assert payload["cost"]["input_tokens"] == 200 + assert payload["cost"]["output_tokens"] == 10 + assert payload["cost"]["cache_read_tokens"] == 120 + assert payload["cost"]["cache_write_tokens"] == 60 + assert payload["cost"]["usd"] == 0.004 + out = capsys.readouterr().out + assert ("cost 2 calls, 200 input / 10 output tokens, cache read 120 / write 60, " + "$0.0040") in out.splitlines() + assert "replayed" not in out + + +def test_a_paid_run_under_a_hundredth_of_a_cent_is_not_printed_as_free( + tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + # Two short routing calls on a small model cost about this much; $0.0000 reads as free. + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=50, completion_tokens=1, + cost_usd=0.00001)) + payload = _model_run(tmp_path) + assert payload["cost"]["usd"] == 0.00002 + assert ("cost 2 calls, 100 input / 2 output tokens, cache read 0 / write 0, " + "<$0.0001") in capsys.readouterr().out.splitlines() + + +def test_a_model_run_with_an_unpriced_call_reports_no_usd_figure(tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=10, completion_tokens=1)) + payload = _model_run(tmp_path) + assert payload["cost"]["usd"] is None # one unpriced call must not be read as free + assert "USD not known: a call had no price or no usage" in capsys.readouterr().out + + +def test_a_lexical_run_spends_nothing_and_has_no_cost_block(tmp_path): + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary"]), encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary with perf"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--out", str(out)) + args.func(args) + assert "cost" not in json.loads(out.read_text(encoding="utf-8")) + + +def test_a_model_run_whose_every_call_fails_prints_no_price_blame(tmp_path, monkeypatch, capsys): + from dokimi import llm + + def failing(prompt, model, **_kwargs): + raise RuntimeError("provider refused") + + monkeypatch.setattr(llm, "llm_call_with_usage", failing) + payload = _model_run(tmp_path) + assert payload["cost"]["llm_calls"] == 0 + assert payload["cost"]["usd"] is None # no calls: nothing known, not "free" + out = capsys.readouterr().out + assert "cost 0 calls\n" in out + assert "USD not known" not in out + + +def test_a_call_with_no_usage_makes_the_total_unknown_not_the_priced_subset( + tmp_path, monkeypatch, capsys): + from dokimi import llm + from dokimi.metrics.usage import UsageRecord + + replies = iter([None, UsageRecord(prompt_tokens=10, completion_tokens=1, cost_usd=0.001)]) + monkeypatch.setattr(llm, "llm_call_with_usage", + lambda prompt, model, **_kw: ('{"pick": 1}', next(replies))) + payload = _model_run(tmp_path) + assert payload["cost"]["llm_calls"] == 2 + assert payload["cost"]["usd"] is None + assert "$0.0010" not in capsys.readouterr().out + + +def test_replayed_calls_are_counted_apart_and_not_billed(tmp_path, monkeypatch, capsys): + from dokimi.metrics.usage import UsageRecord + + _stub_provider(monkeypatch, UsageRecord(prompt_tokens=100, completion_tokens=5, + cache_read_tokens=60, cache_write_tokens=30, + cost_usd=0.002, replayed=True)) + payload = _model_run(tmp_path) + cost = payload["cost"] + assert (cost["llm_calls"], cost["replayed_calls"]) == (2, 2) + assert (cost["input_tokens"], cost["output_tokens"], cost["usd"]) == (0, 0, 0.0) + assert (cost["cache_read_tokens"], cost["cache_write_tokens"]) == (0, 0) + assert ("cost 2 calls, 0 input / 0 output tokens, cache read 0 / write 0, $0.0000, " + "2 replayed from the cache and not counted") in capsys.readouterr().out.splitlines() + + +def test_a_live_a_replayed_and_a_usageless_call_are_told_apart(tmp_path, monkeypatch, capsys): + from dokimi import llm + from dokimi.metrics.usage import UsageRecord + + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary", "tune a slow query", "trace a syscall"]), + encoding="utf-8") + replies = iter([UsageRecord(prompt_tokens=10, completion_tokens=1, cost_usd=0.001), + UsageRecord(prompt_tokens=10, completion_tokens=1, cost_usd=0.001, + replayed=True), + None]) + monkeypatch.setattr(llm, "llm_call_with_usage", + lambda prompt, model, **_kw: ('{"pick": 1}', next(replies))) + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "a", "profile a binary with perf"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries", str(queries), "--model", "selector-m", "--out", str(out)) + args.func(args) + cost = json.loads(out.read_text(encoding="utf-8"))["cost"] + assert (cost["llm_calls"], cost["replayed_calls"], cost["usd"]) == (3, 1, None) + assert ("cost 3 calls, 10 input / 1 output tokens, cache read 0 / write 0, " + "USD not known: a call had no price or no usage, " + "1 replayed from the cache and not counted") in capsys.readouterr().out.splitlines()