Improve model usage accounting across providers

docs/TELEMETRY.md

@@ -50,11 +50,17 @@ any of the three drift apart.
 | usage_delta | mode | chat or task |
 | usage_delta | input_tokens | provider-reported input tokens since the preceding usage event |
 | usage_delta | output_tokens | provider-reported output tokens since the preceding usage event |
-| usage_delta | total_tokens | the sum of this event's input and output tokens |
+| usage_delta | total_tokens | the sum of this event's measured components, or the provider total with no split for total_only |
 | usage_delta | routing_provider | openrouter, openai, anthropic, google, bedrock, azure, ollama, other, or unknown |
 | usage_delta | model_family | deepseek, claude, gpt, gemini, llama, qwen, kimi, glm, other, or unknown |
-| usage_delta | usage_status | reported or missing |
+| usage_delta | usage_status | reported, total_only, mixed, estimated, partial, or missing |
 | usage_delta | accounting_source | provider or engine |
+| usage_delta | estimated_input_tokens | estimated input tokens for a missing provider count |
+| usage_delta | estimated_output_tokens | estimated observed output tokens for a missing provider count |
+| usage_delta | estimated_total_tokens | the sum of this event's estimated input and output tokens |
+| usage_delta | estimate_method | utf8_bytes_4_v1 |
+| usage_delta | estimate_scope | text_only, observed_partial, multimodal_input, or unknown |
+| usage_delta | usage_receipt | sha256 of the session and local receipt identity; groups estimates with later provider receipts |
 | session_ended | mode | chat or task |
 | session_ended | duration | a band: under 1m, 1-5m, 5-30m, 30m-2h, 2h or more |
 | session_ended | turns | a count band |
@@ -71,8 +77,14 @@ any of the three drift apart.
 
 Count bands are 0, 1, 2-5, 6-20, 21-100 and 100+. Dollar bands are 0, under
 0.01, 0.01-0.1, 0.1-1, 1-10 and 10+.
+Streaming requests ask compatible APIs for token usage with `stream_options.include_usage`.
+Connected Ollama services are classified as `ollama`, including remote runners;
+other local or custom APIs remain `other`. Counts depend on the API returning
+a usage block. Missing counts may carry separately labeled byte-based estimates;
+these never replace provider-reported fields or enter billing.
+
 Each provider usage receipt is appended locally before provider accounting returns, including
-tokens read from cache once. Missing provider receipts produce a `usage_status = missing` event without token fields; a missing receipt never invents a zero total. Routing categories follow the actual attempt, including an explicitly authorized billing-door switch and any later generation receipt. Bounded routing and model-family categories describe reported usage without sending exact model names or endpoint addresses. Hosted engines currently report aggregate turn deltas with unknown routing and family, labelled `accounting_source = engine`; those aggregates are not per-call receipts and can be lost before a turn ends. Deltas, rather
+tokens read from cache once. Missing token components carry `estimated_*` fields when text observations permit an estimate, otherwise remain absent. `usage_status` distinguishes reported, total-only, mixed, estimated, partial, and missing. A missing receipt never invents a measured zero total. Routing categories follow the actual attempt, including an explicitly authorized billing-door switch and any later generation receipt. Bounded routing and model-family categories describe reported usage without sending exact model names or endpoint addresses. Hosted engines currently report aggregate turn deltas with unknown routing and family, labelled `accounting_source = engine`; those aggregates are not per-call receipts and can be lost before a turn ends. Deltas, rather
 than repeated session totals, make the stream safe to sum while a run is open.
 
 first_run carries only the every-event properties and is sent once per
@@ -154,3 +166,33 @@ the count GitHub already shows on the release page. Sidecar files such as
 batch it would send.
 
 The release-download workflow fails visibly when its production PostHog key or GitHub token is missing; dry-run still works without a PostHog key.
+
+## Estimates and recovered receipts
+
+See [Model usage accounting](model-usage-accounting.md) for the estimator,
+limitations, privacy fields, receipt-aware PostHog query and deployment order.
+A later provider receipt supersedes each measured component's earlier estimate by `usage_receipt`;
+raw estimates and recovered measurements must not be added together. Estimates
+never change billing, response usage or token limits.
+The receipt key includes the physical attempt's identity so separate answers
+stay separate even when a compatible service reuses a generation id.
+Generation ids are trimmed before both observations and recovery. A one-sided
+recovery leaves the other component unknown; a cost-only receipt adds no token
+event. Invalid or unsafe counts and sums are omitted, and estimates require a
+receipt identity. The final status describes the components actually emitted.
+
+Native `input_tokens`/`output_tokens` aliases now supply measured counts where
+they previously stayed zero. Native cache reads and writes are included in
+prompt tokens, matching the canonical cache-inclusive prompt convention.
+Provider cost is banked unchanged; the provider billing boundary does not
+calculate a token price when cost is absent. Estimates never supply a price.
+Unmetered answers from a direct service retain the prior no-lookup behavior,
+including overflow answers; their telemetry still follows the actual attempt.
+
+Connected services request streaming usage with `stream_options.include_usage`.
+An explicit unsupported-option response on HTTP 400 or 422 retries once without
+that option; later calls to that endpoint omit it. Unrelated errors do not turn
+the option off. The default router road keeps its existing `usage.include`
+request and sends no `stream_options`.
+Independent reasoning or parameter repairs can compose with this fallback;
+each distinct repair is used at most once in the repair chain.

docs/changes/unreleased/1887-model-usage-accounting.md

@@ -0,0 +1,26 @@
+---
+kind: fixed
+title: model usage accounting covers compatible providers and separates estimates from measurements
+pr: 1887
+surface: [engine, docs]
+invalidates:
+  - "Streaming requests previously sent only the router usage opt-in. Connected compatible APIs now receive stream_options.include_usage, with a narrow fallback for endpoints that explicitly reject it on 400 or 422. The default router road keeps its original wire shape; unrelated refusals gain no new repair."
+  - "Missing provider counts previously stayed unknown even when text was observed. Missing text components now have separate versioned estimates; reported zeroes, partial counts and total-only receipts remain measurements, and media or unseen output remains unknown."
+  - "A raw stream previously closed its call log without accounting for the completed response. It now records usage through the same provider boundary as normal calls."
+  - "Estimates and later provider receipts are not additive. They share a run-scoped physical-attempt receipt hash so analytics can prefer a measurement for each component without double counting. A provider that reuses generation ids no longer collapses distinct answers into one receipt."
+  - "Native input_tokens/output_tokens aliases previously recorded zero counts in responses, call logs and ledgers. They now provide measured counts, including native cache reads and writes in the cache-inclusive prompt count; equivalent canonical counts yield the same billing and token price."
+  - "Usage-option refusal repair previously stopped after another repair, or prevented a later reasoning repair. Independent repairs now compose once each; unrelated refusals keep their existing handling."
+  - "Repeated streamed tool names and opaque replay could inflate text estimates, and metadata-only cuts could invent zero output. Names now count once per call, opaque input remains unknown, and only observed text or normal completion establishes output."
+  - "Late cost-only and one-sided receipts previously fabricated measured zeroes. Recovery now preserves component presence and trimmed receipt identity without changing its price or ledger row."
+---
+
+Deploy the companion relay change in Agent-Field/website-open-infra#82 before
+releasing the client. The prior relay rejects the new status values. Estimates
+never enter billing, response usage or token-limit enforcement; existing opt-out
+settings apply to the new count fields. The contract, limits and receipt-aware
+query are documented in docs/model-usage-accounting.md.
+The provider billing boundary copies a supplied cost and does not derive it from
+tokens when cost is absent. Unmetered direct-service answers retain the prior
+no-lookup behavior even on overflow; telemetry follows the actual attempt.
+Telemetry status is derived from emitted components, unsafe counts and sums are
+omitted, and estimates require a receipt identity.

docs/model-usage-accounting.md

@@ -0,0 +1,67 @@
+# Model usage accounting
+
+Token usage is collected at the provider attempt boundary for every compatible endpoint, including OpenRouter, Ollama, vLLM, LM Studio, llama.cpp and custom or hosted APIs. The fallback does not depend on a provider name.
+
+## Measurements and estimates
+
+- Connected services request `stream_options.include_usage` on streamed chat completions. An explicit unsupported-field 400 or 422 retries without that option and remembers the refusal for that endpoint. Other validation errors do not disable usage or earn a new repair. The default router road keeps its existing `usage.include` opt-in and sends no `stream_options`.
+- Keep provider-reported counts, including explicit zero. Normalize input/output aliases and native cache input without adding cache tokens twice.
+- Keep a provider's total-only measurement without inventing its input/output split.
+- Estimate only absent components, using `utf8_bytes_4_v1`: round observed UTF-8 bytes up at four bytes per token. Input includes the encoded text messages and tool definitions after shaping; opaque sidecars decline the input estimate. Output includes observed content, each tool name once per call, every argument fragment and visible textual reasoning. Metadata alone observes no output; a normally finished empty answer observes zero.
+- Do not estimate media from URL or base64 length. Non-text input stays unknown; output text can still be estimated. Unseen output, encrypted reasoning, hidden thinking and work inside opaque hosted engines cannot be reconstructed.
+- Mark interrupted-stream estimates `observed_partial`, never extrapolate to a completed answer. Failed or refused requests without an accepted response do not imply consumption.
+- Estimates are approximate, particularly for non-English text, tokenizers and provider chat templates. They are suitable for usage visibility, not billing, token-limit enforcement, benchmark comparisons or exact cost attribution. They never modify response usage or the spend ledger.
+
+## Wire contract
+
+`input_tokens`, `output_tokens` and `total_tokens` remain measurements. `estimated_input_tokens`, `estimated_output_tokens` and `estimated_total_tokens` are separate, optional numeric fields, emitted only for unknown measured components and with a receipt identity. Each total sums only its own present components, except a provider's total-only measurement has no split. Negative counts, counts above 2^53-1 and groups whose sum exceeds that bound remain unknown. Missing is distinct from an explicitly reported or estimated zero. The final status follows the emitted components.
+
+`usage_status` is `reported`, `total_only`, `mixed`, `estimated`, `partial` or `missing`. `estimate_method` and `estimate_scope` label estimates; scopes are `text_only`, `observed_partial`, `multimodal_input` or `unknown`.
+
+`usage_receipt` is a SHA-256 hash scoped to the run and physical attempt. When a generation id exists, its actual endpoint and id participate only locally in that hash alongside the attempt identity. Providers that reuse generation ids cannot collapse separate answers into one receipt. Recovery keeps the original attempt identity. No prompt, response, endpoint, model slug, generation id, API key or other new raw identity is sent. Existing opt-out gates apply to estimates too.
+
+## Recovered receipts and dashboards
+
+An initial estimate and a later recovered provider receipt carry the same `usage_receipt`, but different event UUIDs. Generation ids are trimmed at both boundaries, and blank ids use the attempt identity alone. For each component, prefer a reported count, even if it is zero; retain an estimate only while no measurement exists. One-sided recovery is partial; cost-only recovery adds no token event and keeps its recovered price. A measured total-only receipt supersedes both component estimates. Do not add raw estimates and later measured receipts together.
+
+Native `input_tokens`/`output_tokens` now populate response, call-log and ledger counts where those aliases previously recorded zero. Native cache reads and writes join the prompt count to match canonical cache-inclusive `prompt_tokens`. The provider billing boundary copies any supplied cost and does not price tokens when cost is absent; downstream token pricing sees equivalent native and canonical counts. No estimate is banked, priced or shown as spend. Unmetered direct-service answers, including authorized overflow, retain their prior no-lookup behavior while telemetry identifies the actual attempt.
+
+The existing measured-only series still sums `total_tokens`. The new PostHog view groups by installation, event and receipt hash (falling back to the event UUID for older builds), then selects the best available components. Legacy session-end totals remain separate. Receipt time uses the earliest observation, with a one-day lookback margin around the displayed 14-day range. The query also exposes records with unknown input/output components. This is coverage of recorded receipts, not proof that every model call or anonymous installation reported.
+
+The estimate-aware line chart is configured separately in PostHog. Its estimated series starts only after rollout; historical missing receipts cannot be reconstructed. Verify the actual project receiving `codeaf:usage_delta` before installing a query.
+
+## Deployment order
+
+1. Merge and deploy the companion change in `Agent-Field/website-open-infra`, `site/src/app/api/oss/codeaf/telemetry/`. The previous relay drops unknown status values and strips new properties.
+2. Merge the client PR into `dev`, then promote releases normally.
+3. Confirm new numeric fields arrive in PostHog. No synthetic events are sent to the production relay by the tests.
+
+The client and relay carry identical `usage-accounting.json` fixtures. Client constructors are checked against that payload and the actual relay handler is checked through a mocked PostHog capture call, including estimates, measured zeroes, partial receipts, privacy stripping, safe integer totals and late-receipt identity. A full 50-event enriched batch is checked against the relay's 64 KiB body cap.
+
+## Receipt-aware PostHog query
+
+```sql
+WITH receipts AS (
+ SELECT event, distinct_id,
+ coalesce(nullIf(toString(properties.usage_receipt), ''), toString(uuid)) AS receipt_key,
+ min(timestamp) AS receipt_time,
+ max(toFloat(properties.total_tokens)) AS measured_total,
+ max(toFloat(properties.input_tokens)) AS measured_input,
+ max(toFloat(properties.output_tokens)) AS measured_output,
+ max(toFloat(properties.estimated_input_tokens)) AS estimated_input,
+ max(toFloat(properties.estimated_output_tokens)) AS estimated_output
+ FROM events
+ WHERE event IN ('codeaf:usage_delta', 'codeaf:session_ended')
+ AND properties.channel='stable' AND coalesce(properties.usage_context,'')!='ci'
+ AND timestamp >= today()-INTERVAL 15 DAY
+ GROUP BY event, distinct_id, receipt_key
+)
+SELECT toDate(receipt_time) AS day,
+ sumIf(coalesce(measured_total,0), event='codeaf:session_ended') AS legacy_reported_tokens,
+ sumIf(if(isNull(measured_input) AND isNull(measured_output),coalesce(measured_total,0),coalesce(measured_input,0)+coalesce(measured_output,0)), event='codeaf:usage_delta') AS reported_tokens,
+ sumIf(if(isNotNull(measured_total) AND isNull(measured_input) AND isNull(measured_output),0,if(isNull(measured_input),coalesce(estimated_input,0),0)+if(isNull(measured_output),coalesce(estimated_output,0),0)),event='codeaf:usage_delta') AS estimated_tokens,
+ countIf(event='codeaf:usage_delta' AND ((isNull(measured_input) AND isNull(estimated_input)) OR (isNull(measured_output) AND isNull(estimated_output)))) AS records_with_unknown_components
+FROM receipts WHERE receipt_time >= today()-INTERVAL 14 DAY
+GROUP BY day ORDER BY day
+```
+

internal/config/config.go

@@ -988,6 +988,7 @@ func ClientConfigFor(sources modelsource.Set, model string) provider.Config {
 	effort, _ := provider.ParseEffort(level)
 	service, bare := sources.For(model)
 	configured := provider.Config{
+		SourceID:    service.Source.ID,
 		APIKey:      service.Key,
 		BaseURL:     service.Address,
 		Model:       bare,

internal/config/sources_test.go

@@ -517,7 +517,7 @@ func TestAnUnqualifiedIdResolvesExactlyAsItDidBefore(t *testing.T) {
 		for _, model := range models {
 			configured := Config{APIKey: "default-key", BaseURL: base, Model: model}.ClientConfig(model)
 			assertComparableProviderConfig(t, configured, provider.Config{
-				APIKey: "default-key", BaseURL: base, Model: model, Effort: provider.EffortNone,
+				SourceID: modelsource.DefaultID, APIKey: "default-key", BaseURL: base, Model: model, Effort: provider.EffortNone,
 			})
 		}
 	}
@@ -546,7 +546,7 @@ func TestAnUnqualifiedIdResolvesExactlyAsItDidBefore(t *testing.T) {
 		for _, model := range models {
 			configured := loaded.ClientConfig(model)
 			assertComparableProviderConfig(t, configured, provider.Config{
-				APIKey: "sk-profile-captured-1234567890", BaseURL: wantBase,
+				SourceID: modelsource.DefaultID, APIKey: "sk-profile-captured-1234567890", BaseURL: wantBase,
 				Model: model, Effort: provider.EffortNone, Timeout: DefaultTimeout,
 			})
 		}
@@ -665,3 +665,14 @@ func TestOnlyAnExplicitlyKeyOptionalServiceAcceptsBlank(t *testing.T) {
 		t.Fatalf("explicitly optional key = %+v, %v", outcome, err)
 	}
 }
+
+func TestOllamaClientConfigPreservesServiceIdentity(t *testing.T) {
+	sources := modelsource.NewSet(modelsource.Connected{
+		Source:  modelsource.Source{ID: "ollama", Written: "local-models", KeyOptional: true},
+		Address: "http://private-runner.example:9000/v1",
+	})
+	configured := ClientConfigFor(sources, "local-models/qwen3")
+	if configured.SourceID != "ollama" || configured.BaseURL != "http://private-runner.example:9000/v1" || !configured.KeyOptional || !configured.Direct {
+		t.Fatalf("Ollama identity was lost: %+v", configured)
+	}
+}