From 26048166736941e59cff13197b61a74611ae39c3 Mon Sep 17 00:00:00 2001 From: Vega Date: Wed, 19 Aug 2026 11:56:06 -0500 Subject: [PATCH 1/4] feat: record judge scores as gen_ai.evaluation.result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Judge scores land as a `gen_ai.evaluation.result` span event (and mirrored attributes) on the judge `invoke_agent` span, so conversation turn badges can render. `with_judge_evaluation` holds that span open until scoring finishes — `execute_and_track` returns after the handler has already called `span.end()`, so without the delay the event would be dropped. Existing `track(evaluation_metric_key)` behavior is unchanged. TELEMETRY-CONTRACT.md section 4a documents the event contract. Stacked on the conversation-id PR: the two halves share conversation.py and nothing else, so they review apart and merge together. O11Y-1888 Co-Authored-By: Claude --- TELEMETRY-CONTRACT.md | 16 ++ packages/client/agents.md | 2 +- .../launchdarkly_ai_server/conversation.py | 97 ++++++++++- .../src/launchdarkly_ai_server/judges.py | 155 +++++++++--------- packages/client/tests/test_conversation.py | 18 ++ tests/test_cross_handler_parity.py | 5 + 6 files changed, 214 insertions(+), 79 deletions(-) diff --git a/TELEMETRY-CONTRACT.md b/TELEMETRY-CONTRACT.md index 513b8fd..7f7d1e9 100644 --- a/TELEMETRY-CONTRACT.md +++ b/TELEMETRY-CONTRACT.md @@ -212,6 +212,22 @@ when `conversation_id(...)` is bound. --- +## 4a. Judge evaluation events + +A judge run is itself a tracked AI call (`invoke_agent` + `chat`). After the score is parsed, the +SDK writes a `gen_ai.evaluation.result` span event on that `invoke_agent` span: + +| Event attribute | Value | +|---|---| +| `gen_ai.evaluation.name` | judge config key | +| `gen_ai.evaluation.score.value` | numeric score | +| `gen_ai.evaluation.explanation` | judge reasoning, when present | + +The same keys are mirrored as span attributes. `gen_ai.evaluation.score.label` is not invented. +The existing `track(evaluationMetricKey)` call is unchanged and still feeds AI Config Monitoring. + +--- + ## 5. Finish reasons One vocabulary across all six handlers: `stop`, `length`, `content_filter`, `tool_calls`, `error`. diff --git a/packages/client/agents.md b/packages/client/agents.md index f46cda5..74c7879 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -120,7 +120,7 @@ Handlers may return any of these — the client normalizes them before emitting - Calls `handler(config, user_input, tool_handlers, variables)` - On success: emits `$ld:ai:generation:success` + token tracks - On error: emits `$ld:ai:generation:error` then re-raises -3. If `judge_configuration.judges` is present, runs each judge handler (sampled by `sampling_rate`) against the primary response and tracks `evaluation_metric_key`. +3. If `judge_configuration.judges` is present, runs each judge handler (sampled by `sampling_rate`) against the primary response, tracks `evaluation_metric_key`, and emits a `gen_ai.evaluation.result` span event on the judge's `invoke_agent` span (`gen_ai.evaluation.name` / `.score.value` / `.explanation`). 4. Returns `ProviderResponse`: `{ response: str, usage: UsageDict, track_data: TrackData, judge_results?: dict[str, JudgeResult], judge_tasks?: list[JudgeTask] }`. `judge_results` is populated when `skip_judges=False` (default) and judges ran; `judge_tasks` is populated when `skip_judges=True`. --- diff --git a/packages/client/src/launchdarkly_ai_server/conversation.py b/packages/client/src/launchdarkly_ai_server/conversation.py index 7b2218b..fbf8bbe 100644 --- a/packages/client/src/launchdarkly_ai_server/conversation.py +++ b/packages/client/src/launchdarkly_ai_server/conversation.py @@ -1,4 +1,4 @@ -"""Caller-supplied ``gen_ai.conversation.id``. +"""Caller-supplied ``gen_ai.conversation.id`` and judge evaluation span events. A dedicated OTel context key, not W3C baggage: the id must not leak onto outbound provider HTTP calls. A multi-tenant process binds a different id per request. @@ -6,8 +6,9 @@ from __future__ import annotations -from collections.abc import AsyncGenerator, Iterator -from contextlib import aclosing, contextmanager +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterator +from contextlib import aclosing, asynccontextmanager, contextmanager +from dataclasses import dataclass from typing import TYPE_CHECKING, Any from opentelemetry import context as otel_context @@ -25,6 +26,15 @@ GEN_AI_CONVERSATION_ID = "gen_ai.conversation.id" _CONV_KEY = otel_context.create_key("launchdarkly.gen_ai.conversation.id") +_EVAL_KEY = otel_context.create_key("launchdarkly.judge.evaluation") + + +@dataclass +class _JudgeEvalCapture: + name: str + released: bool = False + span: Any = None + pending_end: Callable[[], None] | None = None def _read_attribute(span: Any, key: str) -> Any: @@ -51,6 +61,50 @@ def _conversation_id_from(ctx: otel_context.Context | None) -> str | None: return value if isinstance(value, str) and value else None +def _record_evaluation( + span: Any, name: str, score: float, explanation: str | None +) -> None: + if span is None or not span.is_recording(): + return + attrs: dict[str, Any] = { + "gen_ai.evaluation.name": name, + "gen_ai.evaluation.score.value": score, + } + if explanation: + attrs["gen_ai.evaluation.explanation"] = explanation + span.add_event("gen_ai.evaluation.result", attrs) + span.set_attribute("gen_ai.evaluation.name", name) + span.set_attribute("gen_ai.evaluation.score.value", score) + if explanation: + span.set_attribute("gen_ai.evaluation.explanation", explanation) + + +def _delay_invoke_agent_end(span: Any, capture: _JudgeEvalCapture) -> None: + original_end = span.end + ended = False + + def wrapped_end(*args: Any, **kwargs: Any) -> None: + nonlocal ended + if ended: + return + if capture.released: + ended = True + original_end(*args, **kwargs) + return + capture.span = span + + def pending() -> None: + nonlocal ended + if ended: + return + ended = True + original_end(*args, **kwargs) + + capture.pending_end = pending + + span.end = wrapped_end + + @contextmanager def conversation_id(conversation: str) -> Iterator[None]: """Bind a caller-supplied conversation id for the duration of the ``with`` block. @@ -70,6 +124,9 @@ def conversation_id(conversation: str) -> Iterator[None]: otel_context.detach(token) +RecordEvaluation = Callable[[float, str | None], None] + + async def _stream_with_bound_id( generator: AsyncGenerator[Any, None], conversation: str ) -> AsyncGenerator[Any, None]: @@ -107,8 +164,31 @@ def bind_conversation_id( return _stream_with_bound_id(generator, conversation) +@asynccontextmanager +async def with_judge_evaluation(name: str) -> AsyncIterator[RecordEvaluation]: + """Hold the judge ``invoke_agent`` span open until ``record`` runs. + + ``execute_and_track`` returns after the handler has already called ``span.end()``, + so without this delay the evaluation event would be dropped. + """ + capture = _JudgeEvalCapture(name=name) + + def record(score: float, explanation: str | None = None) -> None: + if capture.span is not None: + _record_evaluation(capture.span, capture.name, score, explanation) + + token = otel_context.attach(otel_context.set_value(_EVAL_KEY, capture)) + try: + yield record + finally: + capture.released = True + if capture.pending_end is not None: + capture.pending_end() + otel_context.detach(token) + + class ConversationIdSpanProcessor(_SpanProcessorBase): - """Stamps ``gen_ai.conversation.id`` write-if-absent on every span. + """Stamps ``gen_ai.conversation.id`` write-if-absent; delays judge ``invoke_agent`` end. Structurally a ``SpanProcessor``; the base is only real to a type checker so that an api-only install (no ``[otel]`` extra) still imports this module. @@ -127,6 +207,15 @@ def on_start( if conv: set_conversation_id_if_absent(span, conv) + capture = otel_context.get_value(_EVAL_KEY, ctx) + if capture is None: + capture = otel_context.get_value(_EVAL_KEY) + if ( + isinstance(capture, _JudgeEvalCapture) + and getattr(span, "name", None) == "invoke_agent" + ): + _delay_invoke_agent_end(span, capture) + def on_end(self, span: Any) -> None: return None diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index 898f864..1404967 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -5,6 +5,7 @@ from collections.abc import Callable from typing import Any +from .conversation import with_judge_evaluation from .types import ( AiConfigRep, JudgeRunResult, @@ -154,51 +155,54 @@ async def run_judges( filter(None, [user_input, llm_response, _FORMATTING_INSTRUCTIONS]) ) - result = await execute_and_track( - config_key=judge_key, - config=effective_judge_config, - meta=judge_meta, - user_context=user_context, - handler=judge_handler, - user_input=llm_response, - tool_handlers=None, - graph_key=graph_key, - variables={ - "message_history": message_history, - "response_to_evaluate": llm_response, - }, - ) - - raw = result["response"] - judge_response = raw if isinstance(raw, str) else str(raw) - - parsed = parse_json_with_possible_fences(judge_response) - if not parsed: - raise ValueError("Invalid JSON from judge") - - score = parsed.get("score") - reasoning = parsed.get("reasoning", "") - judge_results[judge_key] = { - "usage": result["usage"], - "response": reasoning, - "score": score, - } + async with with_judge_evaluation(judge_key) as record_evaluation: + result = await execute_and_track( + config_key=judge_key, + config=effective_judge_config, + meta=judge_meta, + user_context=user_context, + handler=judge_handler, + user_input=llm_response, + tool_handlers=None, + graph_key=graph_key, + variables={ + "message_history": message_history, + "response_to_evaluate": llm_response, + }, + ) - evaluation_metric_key = ( - judge_ai_config.get("evaluationMetricKey") - if isinstance(judge_ai_config, dict) - else None - ) - if evaluation_metric_key and score is not None: - from .lifecycle import get_client - - client = get_client() - client.track( - evaluation_metric_key, - to_ld_context(client, user_context), - {**base_track_data, "judgeConfigKey": judge_key}, - score, + raw = result["response"] + judge_response = raw if isinstance(raw, str) else str(raw) + + parsed = parse_json_with_possible_fences(judge_response) + if not parsed: + raise ValueError("Invalid JSON from judge") + + score = parsed.get("score") + reasoning = parsed.get("reasoning", "") + judge_results[judge_key] = { + "usage": result["usage"], + "response": reasoning, + "score": score, + } + if score is not None: + record_evaluation(float(score), reasoning or None) + + evaluation_metric_key = ( + judge_ai_config.get("evaluationMetricKey") + if isinstance(judge_ai_config, dict) + else None ) + if evaluation_metric_key and score is not None: + from .lifecycle import get_client + + client = get_client() + client.track( + evaluation_metric_key, + to_ld_context(client, user_context), + {**base_track_data, "judgeConfigKey": judge_key}, + score, + ) except Exception as exc: logger.error("Judge '%s' failed: %s", judge_key, exc) @@ -384,39 +388,42 @@ def _matches(h: ProviderHandler) -> bool: filter(None, [task.actual_output, _FORMATTING_INSTRUCTIONS]) ) - result = await execute_and_track( - config_key=task.config_key, - config=effective_config, - meta=task.judge_meta, - user_context=task.user_context, - handler=judge_handler, - user_input=task.actual_output, - tool_handlers=None, - variables={ - **(task.variables or {}), - "message_history": message_history, - "response_to_evaluate": task.actual_output, - }, - ) + async with with_judge_evaluation(task.config_key) as record_evaluation: + result = await execute_and_track( + config_key=task.config_key, + config=effective_config, + meta=task.judge_meta, + user_context=task.user_context, + handler=judge_handler, + user_input=task.actual_output, + tool_handlers=None, + variables={ + **(task.variables or {}), + "message_history": message_history, + "response_to_evaluate": task.actual_output, + }, + ) - raw = result["response"] - judge_response = raw if isinstance(raw, str) else str(raw) - parsed = parse_json_with_possible_fences(judge_response) - if not parsed: - return None + raw = result["response"] + judge_response = raw if isinstance(raw, str) else str(raw) + parsed = parse_json_with_possible_fences(judge_response) + if not parsed: + return None - score = parsed.get("score", 0.0) - reasoning = parsed.get("reasoning", "") - raw_usage = result["usage"] + score = parsed.get("score", 0.0) + reasoning = parsed.get("reasoning", "") + if score is not None: + record_evaluation(float(score), reasoning or None) + raw_usage = result["usage"] - usage = to_usage_dict(raw_usage) + usage = to_usage_dict(raw_usage) - merged_track_data: TrackData = { - **task.parent_track_data, - **result["track_data"], - "judgeConfigKey": task.config_key, - } + merged_track_data: TrackData = { + **task.parent_track_data, + **result["track_data"], + "judgeConfigKey": task.config_key, + } - return JudgeRunResult( - score=score, response=reasoning, usage=usage, track_data=merged_track_data - ) + return JudgeRunResult( + score=score, response=reasoning, usage=usage, track_data=merged_track_data + ) diff --git a/packages/client/tests/test_conversation.py b/packages/client/tests/test_conversation.py index 3f68d48..6ab7b41 100644 --- a/packages/client/tests/test_conversation.py +++ b/packages/client/tests/test_conversation.py @@ -13,6 +13,7 @@ ConversationIdSpanProcessor, conversation_id, set_conversation_id_if_absent, + with_judge_evaluation, ) _exporter = InMemorySpanExporter() @@ -79,3 +80,20 @@ def test_writes_session_id_when_unbound(self) -> None: set_conversation_id_if_absent(span, "sess-abc") span.end() assert finished()[0].attributes[GEN_AI_CONVERSATION_ID] == "sess-abc" + + +class TestJudgeEvaluation: + async def test_puts_evaluation_event_on_invoke_agent(self) -> None: + async with with_judge_evaluation("relevance-judge") as record: + with _tracer.start_as_current_span("invoke_agent") as span: + span.set_attribute("gen_ai.operation.name", "invoke_agent") + record(0.91, "on topic") + span = next(s for s in finished() if s.name == "invoke_agent") + assert span.attributes["gen_ai.evaluation.name"] == "relevance-judge" + assert span.attributes["gen_ai.evaluation.score.value"] == 0.91 + assert span.attributes["gen_ai.evaluation.explanation"] == "on topic" + event = next(e for e in span.events if e.name == "gen_ai.evaluation.result") + assert event.attributes["gen_ai.evaluation.name"] == "relevance-judge" + assert event.attributes["gen_ai.evaluation.score.value"] == 0.91 + assert event.attributes["gen_ai.evaluation.explanation"] == "on topic" + assert "gen_ai.evaluation.score.label" not in event.attributes diff --git a/tests/test_cross_handler_parity.py b/tests/test_cross_handler_parity.py index 19ed8de..b7faa7d 100644 --- a/tests/test_cross_handler_parity.py +++ b/tests/test_cross_handler_parity.py @@ -308,6 +308,11 @@ def test_the_langchain_provider_name_is_binary_not_a_passthrough( "gen_ai.response.finish_reasons", "gen_ai.agent.name", "gen_ai.conversation.id", + # Judge evaluation event + mirrored span attributes on the judge invoke_agent span + "gen_ai.evaluation.result", + "gen_ai.evaluation.name", + "gen_ai.evaluation.score.value", + "gen_ai.evaluation.explanation", # Usage "gen_ai.usage.input_tokens", "gen_ai.usage.output_tokens", From 8fa4f6fd81543bce5a5aae2448f4eabdac3dce37 Mon Sep 17 00:00:00 2001 From: Vega Date: Wed, 19 Aug 2026 17:12:14 -0500 Subject: [PATCH 2/4] fix: stop exporting judge reasoning, validate the score, freeze the end time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the judge half. - Do not emit `gen_ai.evaluation.explanation`. The judge's reasoning is model-generated prose about the user's conversation — content — and AGENTS.md gates content attributes behind `capture_content`, a handler-factory option this layer never receives. It was exported unconditionally, including for callers who left capture off. The reasoning is still returned to the caller in `judge_results`; only the telemetry copy is withheld. - Replace `float(score)` with a non-raising finite-number guard. `float()` was a new raise site sitting ahead of the existing `client.track(evaluation_metric_key, …)` call, so a judge returning "0.9 (high)" silently killed the metric this PR claims is unchanged — and in `run_judge` it escaped uncaught, breaking that function's documented "returns None" contract. - Freeze the end time when the handler calls `end()`. Replaying a no-arg `end()` at release let the SDK stamp `time_ns()` then, inflating every judge span by the tracking and parsing work that runs in between. - Detach the judge capture even if the deferred end raises. O11Y-1888 Co-Authored-By: Claude --- TELEMETRY-CONTRACT.md | 19 +++++--- .../launchdarkly_ai_server/conversation.py | 43 ++++++++++++------- .../src/launchdarkly_ai_server/judges.py | 23 ++++++++-- packages/client/tests/test_conversation.py | 37 ++++++++++++++-- packages/client/tests/test_judges.py | 21 +++++++++ tests/test_cross_handler_parity.py | 5 ++- 6 files changed, 119 insertions(+), 29 deletions(-) diff --git a/TELEMETRY-CONTRACT.md b/TELEMETRY-CONTRACT.md index 7f7d1e9..65a8364 100644 --- a/TELEMETRY-CONTRACT.md +++ b/TELEMETRY-CONTRACT.md @@ -95,6 +95,8 @@ able to tell from the trace which path ran. | `launchdarkly.run.id` | `TrackData.runId` | `set_ld_span_attributes` | | `launchdarkly.graph.key` | `TrackData.graphKey`, only when present | `set_ld_span_attributes` | | `launchdarkly.stream.abandoned` | `True`, only when abandoned | `end_span_once` | +| `gen_ai.evaluation.name` | judge config key, judge roots only | `with_judge_evaluation`, see section 4a | +| `gen_ai.evaluation.score.value` | numeric score, judge roots only | `with_judge_evaluation`, see section 4a | The root also carries one span event, `feature_flag`, with these event attributes: @@ -220,11 +222,18 @@ SDK writes a `gen_ai.evaluation.result` span event on that `invoke_agent` span: | Event attribute | Value | |---|---| | `gen_ai.evaluation.name` | judge config key | -| `gen_ai.evaluation.score.value` | numeric score | -| `gen_ai.evaluation.explanation` | judge reasoning, when present | - -The same keys are mirrored as span attributes. `gen_ai.evaluation.score.label` is not invented. -The existing `track(evaluationMetricKey)` call is unchanged and still feeds AI Config Monitoring. +| `gen_ai.evaluation.score.value` | numeric score, only when the judge returned a finite number | + +The same keys are mirrored as span attributes, so section 2 lists them too. +`gen_ai.evaluation.score.label` is not invented. +The existing `track(evaluationMetricKey)` call is unchanged and still feeds AI Config Monitoring — +a judge that returns a non-numeric score emits no evaluation event but still tracks the metric. + +`gen_ai.evaluation.explanation` is deliberately **not** emitted. The judge's reasoning is +model-generated prose about the user's conversation — content, under section 7 — and content +attributes require `captureContent` / `capture_content`, a handler-factory option this layer does +not receive. The reasoning is still returned to the caller in `judgeResults` / `judge_results`; +only the telemetry copy is withheld. Exporting it needs its own opt-in. --- diff --git a/packages/client/src/launchdarkly_ai_server/conversation.py b/packages/client/src/launchdarkly_ai_server/conversation.py index 67142ed..ce03f4a 100644 --- a/packages/client/src/launchdarkly_ai_server/conversation.py +++ b/packages/client/src/launchdarkly_ai_server/conversation.py @@ -9,6 +9,7 @@ from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterator from contextlib import aclosing, asynccontextmanager, contextmanager from dataclasses import dataclass +from time import time_ns from typing import TYPE_CHECKING, Any from opentelemetry import context as otel_context @@ -36,6 +37,7 @@ class _JudgeEvalCapture: span: Any = None pending_end: Callable[[], None] | None = None + # Every tracer this SDK creates is named "@launchdarkly/ai-". The processor is registered # on the *global* provider, so without this gate it stamps a caller-supplied id onto every span in # the process — Postgres queries, inbound HTTP server spans, and the outbound provider call itself. @@ -77,22 +79,24 @@ def _conversation_id_from(ctx: otel_context.Context | None) -> str | None: return value if isinstance(value, str) and value else None -def _record_evaluation( - span: Any, name: str, score: float, explanation: str | None -) -> None: +def _record_evaluation(span: Any, name: str, score: float) -> None: + """Write the judge score as a ``gen_ai.evaluation.result`` event plus mirrored attributes. + + The judge's free-text reasoning is deliberately NOT exported. It is model prose about the + user's conversation, i.e. content, and AGENTS.md restricts content attributes to callers who + pass ``capture_content=True`` — a handler-factory option this layer has no access to. The + reasoning is still returned to the caller in ``judge_results[key].response``; only the + telemetry copy is dropped. Exporting it needs its own opt-in. + """ if span is None or not span.is_recording(): return attrs: dict[str, Any] = { "gen_ai.evaluation.name": name, "gen_ai.evaluation.score.value": score, } - if explanation: - attrs["gen_ai.evaluation.explanation"] = explanation span.add_event("gen_ai.evaluation.result", attrs) - span.set_attribute("gen_ai.evaluation.name", name) - span.set_attribute("gen_ai.evaluation.score.value", score) - if explanation: - span.set_attribute("gen_ai.evaluation.explanation", explanation) + for key, value in attrs.items(): + span.set_attribute(key, value) def _delay_invoke_agent_end(span: Any, capture: _JudgeEvalCapture) -> None: @@ -108,6 +112,11 @@ def wrapped_end(*args: Any, **kwargs: Any) -> None: original_end(*args, **kwargs) return capture.span = span + # Freeze the end time at the handler's call. Replaying a no-arg end() later would let the + # SDK stamp time_ns() at release, inflating the judge span by the tracking and parsing + # work that runs between the handler ending the span and the score being recorded. + if not args and "end_time" not in kwargs: + kwargs = {**kwargs, "end_time": time_ns()} def pending() -> None: nonlocal ended @@ -142,7 +151,7 @@ def conversation_id(conversation: str | None) -> Iterator[None]: otel_context.detach(token) -RecordEvaluation = Callable[[float, str | None], None] +RecordEvaluation = Callable[[float], None] async def _stream_with_bound_id( @@ -196,18 +205,22 @@ async def with_judge_evaluation(name: str) -> AsyncIterator[RecordEvaluation]: """ capture = _JudgeEvalCapture(name=name) - def record(score: float, explanation: str | None = None) -> None: + def record(score: float) -> None: if capture.span is not None: - _record_evaluation(capture.span, capture.name, score, explanation) + _record_evaluation(capture.span, capture.name, score) token = otel_context.attach(otel_context.set_value(_EVAL_KEY, capture)) try: yield record finally: capture.released = True - if capture.pending_end is not None: - capture.pending_end() - otel_context.detach(token) + try: + if capture.pending_end is not None: + capture.pending_end() + finally: + # Detach even if ending the span raises (a user span processor's on_end can throw); + # otherwise the judge capture stays bound in this task's context for good. + otel_context.detach(token) class ConversationIdSpanProcessor(_SpanProcessorBase): diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index 1404967..5b73b11 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -3,6 +3,7 @@ import logging import random from collections.abc import Callable +from math import isfinite from typing import Any from .conversation import with_judge_evaluation @@ -48,6 +49,18 @@ def _provider_matches(handler: ProviderHandler, provider: str | None) -> bool: ) +def _numeric_score(score: Any) -> float | None: + """Return ``score`` as a float only when it already is a finite number. + + Never raises. A judge that returns ``"0.9 (high)"`` or ``None`` must not take down the + evaluation metric track that follows, and must not put a string where semconv defines a double. + """ + if isinstance(score, bool) or not isinstance(score, (int, float)): + return None + value = float(score) + return value if isfinite(value) else None + + async def run_judges( *, config: AiConfigRep, @@ -185,8 +198,9 @@ async def run_judges( "response": reasoning, "score": score, } - if score is not None: - record_evaluation(float(score), reasoning or None) + numeric_score = _numeric_score(score) + if numeric_score is not None: + record_evaluation(numeric_score) evaluation_metric_key = ( judge_ai_config.get("evaluationMetricKey") @@ -412,8 +426,9 @@ def _matches(h: ProviderHandler) -> bool: score = parsed.get("score", 0.0) reasoning = parsed.get("reasoning", "") - if score is not None: - record_evaluation(float(score), reasoning or None) + numeric_score = _numeric_score(score) + if numeric_score is not None: + record_evaluation(numeric_score) raw_usage = result["usage"] usage = to_usage_dict(raw_usage) diff --git a/packages/client/tests/test_conversation.py b/packages/client/tests/test_conversation.py index 4f3aac2..a3a86bc 100644 --- a/packages/client/tests/test_conversation.py +++ b/packages/client/tests/test_conversation.py @@ -1,6 +1,8 @@ from __future__ import annotations +import asyncio from collections.abc import Iterator +from time import time_ns import pytest from opentelemetry import trace @@ -87,17 +89,46 @@ async def test_puts_evaluation_event_on_invoke_agent(self) -> None: async with with_judge_evaluation("relevance-judge") as record: with _tracer.start_as_current_span("invoke_agent") as span: span.set_attribute("gen_ai.operation.name", "invoke_agent") - record(0.91, "on topic") + record(0.91) span = next(s for s in finished() if s.name == "invoke_agent") assert span.attributes["gen_ai.evaluation.name"] == "relevance-judge" assert span.attributes["gen_ai.evaluation.score.value"] == 0.91 - assert span.attributes["gen_ai.evaluation.explanation"] == "on topic" event = next(e for e in span.events if e.name == "gen_ai.evaluation.result") assert event.attributes["gen_ai.evaluation.name"] == "relevance-judge" assert event.attributes["gen_ai.evaluation.score.value"] == 0.91 - assert event.attributes["gen_ai.evaluation.explanation"] == "on topic" assert "gen_ai.evaluation.score.label" not in event.attributes + async def test_does_not_export_the_judge_explanation(self) -> None: + """Judge reasoning is model prose about the user's conversation, i.e. content. + + AGENTS.md gates content attributes behind ``capture_content=True``, which this layer + cannot see, so the reasoning must not reach telemetry at all. + """ + async with with_judge_evaluation("relevance-judge") as record: + with _tracer.start_as_current_span("invoke_agent"): + pass + record(0.2) + span = next(s for s in finished() if s.name == "invoke_agent") + event = next(e for e in span.events if e.name == "gen_ai.evaluation.result") + assert not any("explanation" in k for k in span.attributes) + assert not any("explanation" in k for k in (event.attributes or {})) + + async def test_end_time_is_the_handler_call_not_the_release(self) -> None: + """The deferred end must not stamp flush time, or the judge span absorbs tracking work.""" + async with with_judge_evaluation("slow-judge") as record: + with _tracer.start_as_current_span("invoke_agent"): + pass + ended_at = time_ns() + await asyncio.sleep( + 0.05 + ) # stands in for tracking + parsing after the handler ends + record(0.5) + span = next(s for s in finished() if s.name == "invoke_agent") + assert span.end_time is not None + # Allow scheduling slack, but nothing close to the 50ms of post-end work. + assert span.end_time - ended_at < 20_000_000 + + class TestProcessorScope: """The processor is registered on the *global* provider, so it sees every span in the process. diff --git a/packages/client/tests/test_judges.py b/packages/client/tests/test_judges.py index 55c2246..65a802d 100644 --- a/packages/client/tests/test_judges.py +++ b/packages/client/tests/test_judges.py @@ -337,3 +337,24 @@ async def test_returns_empty_dict_when_judges_array_is_empty( base_track_data={}, ) assert result == {} + + +class TestScoreGuard: + """`float(score)` used to sit ahead of the evaluation-metric track, so a junk score killed it.""" + + def test_rejects_non_numeric_scores_without_raising(self) -> None: + from launchdarkly_ai_server.judges import _numeric_score + + for junk in ("0.9 (high)", "85%", None, {"v": 1}, [], True, False): + assert _numeric_score(junk) is None + + def test_accepts_finite_numbers(self) -> None: + from math import inf, nan + + from launchdarkly_ai_server.judges import _numeric_score + + assert _numeric_score(0.9) == 0.9 + assert _numeric_score(1) == 1.0 + assert _numeric_score(0) == 0.0 + assert _numeric_score(inf) is None + assert _numeric_score(nan) is None diff --git a/tests/test_cross_handler_parity.py b/tests/test_cross_handler_parity.py index b7faa7d..ad66f25 100644 --- a/tests/test_cross_handler_parity.py +++ b/tests/test_cross_handler_parity.py @@ -308,11 +308,12 @@ def test_the_langchain_provider_name_is_binary_not_a_passthrough( "gen_ai.response.finish_reasons", "gen_ai.agent.name", "gen_ai.conversation.id", - # Judge evaluation event + mirrored span attributes on the judge invoke_agent span + # Judge evaluation event + mirrored span attributes on the judge invoke_agent span. + # No `explanation`: it is model prose about the user's conversation, and content attributes + # need capture_content, which this layer does not receive. See TELEMETRY-CONTRACT.md 4a. "gen_ai.evaluation.result", "gen_ai.evaluation.name", "gen_ai.evaluation.score.value", - "gen_ai.evaluation.explanation", # Usage "gen_ai.usage.input_tokens", "gen_ai.usage.output_tokens", From e600df99386a2dc54e5f452d7e27dfa2aaf5c224 Mon Sep 17 00:00:00 2001 From: Chris Schmitz Date: Thu, 20 Aug 2026 10:29:03 -0500 Subject: [PATCH 3/4] docs: add a multi-turn conversation + judge example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One run that exercises everything O11Y-1888 touches: three turns bound to a single conversation id, each with inline judge evaluation. Prints the id so it can be opened directly in the Conversations view. It is also the manual check for the content decision — the judge's reasoning is printed from `judge_results` (the caller's copy) and should appear nowhere in the exported telemetry. O11Y-1888 Co-Authored-By: Claude --- examples/conversation.py | 87 ++++++++++++++++++++++++++++++++++++++++ main.py | 1 + 2 files changed, 88 insertions(+) create mode 100644 examples/conversation.py diff --git a/examples/conversation.py b/examples/conversation.py new file mode 100644 index 0000000..278afd9 --- /dev/null +++ b/examples/conversation.py @@ -0,0 +1,87 @@ +""" +Example: a multi-turn conversation grouped under one ``gen_ai.conversation.id``, with inline +judge evaluation on every turn. + +This is the end-to-end check for O11Y-1888. Run it, then open the printed conversation id in +LaunchDarkly's Conversations view and confirm: + + 1. One conversation, three turns — not three conversations. Every span of every turn carries + the same id: root, ``chat``, ``execute_tool``, and the judge's own ``invoke_agent``. + 2. Each turn shows a score badge, sourced from the ``gen_ai.evaluation.result`` span event on + the judge span. + 3. No judge reasoning anywhere in the telemetry. The score and the judge's config key are + exported; the explanation is not, because it is model prose about the user's conversation + and content attributes require ``capture_content``. The reasoning IS printed below, straight + from ``judge_results`` — that is the caller's copy, and it is unaffected. + +The flag key must point at an AI Config with a ``judge_configuration``, otherwise there are no +judge turns to look at. + +Usage (via main.py): + python main.py conversation "" +""" + +from __future__ import annotations + +import sys +from typing import Any + +import examples.register # noqa: F401 – side-effect: populate global_registry +from examples.utils import new_context, new_conversation_id +from launchdarkly_ai_server import config, conversation_id, global_registry + +FOLLOW_UPS = [ + "Can you give me a concrete example of that?", + "What is the most common mistake teams make with it?", +] + + +async def run(key: str, user_input: str) -> None: + conversation = new_conversation_id("conversation-example") + ctx = new_context() + history: list[dict[str, Any]] = [] + + print(f"[conversation] {conversation}", file=sys.stderr) + + turns = [user_input or "What is a feature flag?", *FOLLOW_UPS] + + for index, prompt in enumerate(turns, start=1): + # One binding per turn, same id every time — that is what makes them one conversation + # rather than three. Re-binding per turn is the realistic shape: each turn is usually a + # separate inbound request that looks the id up from its own thread/session. + with conversation_id(conversation): + response = await config( + key=key, + registry=global_registry, + ).invoke(prompt, ctx, None, history) + + text = ( + response.response + if isinstance(response.response, str) + else str(response.response) + ) + print(f"\n─── turn {index} ───\n> {prompt}\n{text}") + + judge_results = response.judge_results or {} + for judge_key, result in judge_results.items(): + # `response` here is the judge's reasoning. It reaches the caller and is deliberately + # absent from the span — see the module docstring. + score = getattr(result, "score", None) + reasoning = getattr(result, "response", None) + print( + f"[judge] {judge_key} score={score} reasoning={reasoning}", + file=sys.stderr, + ) + if not judge_results: + print( + "[judge] no judges ran — does this AI Config have a judge_configuration?", + file=sys.stderr, + ) + + history.append({"role": "user", "content": prompt}) + history.append({"role": "assistant", "content": text}) + + print( + f"\n[conversation] done — open {conversation} in the Conversations view", + file=sys.stderr, + ) diff --git a/main.py b/main.py index 2f957fb..6befa5e 100644 --- a/main.py +++ b/main.py @@ -43,6 +43,7 @@ "agent": "examples.agent", "streaming": "examples.streaming", "graph": "examples.graph_example", + "conversation": "examples.conversation", "history": "examples.history", "judge": "examples.judge_example", "claude-agents": "examples.claude_agents_example", From 8a66365059386b637d43c96c56b659867decb047 Mon Sep 17 00:00:00 2001 From: Chris Schmitz Date: Thu, 20 Aug 2026 15:58:15 -0500 Subject: [PATCH 4/4] feat: gate the judge explanation on capture_content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The judge's reasoning was removed outright on PII grounds. That was the right instinct in the wrong shape: the conversation view reads `gen_ai.evaluation.explanation` for its badge tooltip and evaluation banner, so dropping it left a field the UI is built to display permanently blank. Gate it instead, like every other content attribute: - `ProviderHandler` now carries `capture_content`, set by `create_handler` alongside `provides_for`. The client core can apply the handler's own content decision to content it writes on the handler's behalf, without reaching into the factory's closure. - All six handler factories pass their flag through. - `run_judges` / `run_judge` forward the reasoning only when the judge's own handler captures content. Capture on: the explanation reaches the span and the UI renders it. Capture off: absent, exactly as before this commit — no ungated leak. O11Y-1888 Co-Authored-By: Claude --- .../launchdarkly_ai_claude_agents/handler.py | 7 ++++- .../handler.py | 7 ++++- .../launchdarkly_ai_server/conversation.py | 20 +++++++------ .../src/launchdarkly_ai_server/judges.py | 10 +++++-- .../src/launchdarkly_ai_server/types.py | 8 ++++++ .../src/launchdarkly_ai_server/utils.py | 8 +++++- packages/client/tests/test_conversation.py | 28 +++++++++++++++++++ .../handler.py | 7 ++++- .../handler.py | 7 ++++- .../launchdarkly_ai_openai_agents/handler.py | 7 ++++- .../handler.py | 7 ++++- tests/test_cross_handler_parity.py | 5 ++-- 12 files changed, 101 insertions(+), 20 deletions(-) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py index cc8bf5e..182d31e 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py @@ -617,7 +617,12 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler(("Anthropic", "agent"), _call_impl, _stream_impl) # type: ignore[arg-type] + return create_handler( + ("Anthropic", "agent"), + _call_impl, # type: ignore[arg-type] + _stream_impl, # type: ignore[arg-type] + capture_content=capture_content, + ) async def _stream_gen( diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py index 7bde917..254a01e 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py @@ -383,7 +383,12 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler(("Anthropic", "messages"), _call_impl, _stream_impl) # type: ignore[arg-type] + return create_handler( + ("Anthropic", "messages"), + _call_impl, # type: ignore[arg-type] + _stream_impl, # type: ignore[arg-type] + capture_content=capture_content, + ) async def _stream_gen( diff --git a/packages/client/src/launchdarkly_ai_server/conversation.py b/packages/client/src/launchdarkly_ai_server/conversation.py index ce03f4a..38feb42 100644 --- a/packages/client/src/launchdarkly_ai_server/conversation.py +++ b/packages/client/src/launchdarkly_ai_server/conversation.py @@ -79,14 +79,14 @@ def _conversation_id_from(ctx: otel_context.Context | None) -> str | None: return value if isinstance(value, str) and value else None -def _record_evaluation(span: Any, name: str, score: float) -> None: +def _record_evaluation( + span: Any, name: str, score: float, explanation: str | None = None +) -> None: """Write the judge score as a ``gen_ai.evaluation.result`` event plus mirrored attributes. - The judge's free-text reasoning is deliberately NOT exported. It is model prose about the - user's conversation, i.e. content, and AGENTS.md restricts content attributes to callers who - pass ``capture_content=True`` — a handler-factory option this layer has no access to. The - reasoning is still returned to the caller in ``judge_results[key].response``; only the - telemetry copy is dropped. Exporting it needs its own opt-in. + ``explanation`` is passed only when the judge's own handler captures content — it is model + prose about the user's conversation, so it follows the same gate as every other content + attribute. The reasoning always reaches the caller in ``judge_results``. """ if span is None or not span.is_recording(): return @@ -94,6 +94,8 @@ def _record_evaluation(span: Any, name: str, score: float) -> None: "gen_ai.evaluation.name": name, "gen_ai.evaluation.score.value": score, } + if explanation: + attrs["gen_ai.evaluation.explanation"] = explanation span.add_event("gen_ai.evaluation.result", attrs) for key, value in attrs.items(): span.set_attribute(key, value) @@ -151,7 +153,7 @@ def conversation_id(conversation: str | None) -> Iterator[None]: otel_context.detach(token) -RecordEvaluation = Callable[[float], None] +RecordEvaluation = Callable[..., None] async def _stream_with_bound_id( @@ -205,9 +207,9 @@ async def with_judge_evaluation(name: str) -> AsyncIterator[RecordEvaluation]: """ capture = _JudgeEvalCapture(name=name) - def record(score: float) -> None: + def record(score: float, explanation: str | None = None) -> None: if capture.span is not None: - _record_evaluation(capture.span, capture.name, score) + _record_evaluation(capture.span, capture.name, score, explanation) token = otel_context.attach(otel_context.set_value(_EVAL_KEY, capture)) try: diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index 5b73b11..4481299 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -200,7 +200,10 @@ async def run_judges( } numeric_score = _numeric_score(score) if numeric_score is not None: - record_evaluation(numeric_score) + record_evaluation( + numeric_score, + reasoning if judge_handler.capture_content else None, + ) evaluation_metric_key = ( judge_ai_config.get("evaluationMetricKey") @@ -428,7 +431,10 @@ def _matches(h: ProviderHandler) -> bool: reasoning = parsed.get("reasoning", "") numeric_score = _numeric_score(score) if numeric_score is not None: - record_evaluation(numeric_score) + record_evaluation( + numeric_score, + reasoning if judge_handler.capture_content else None, + ) raw_usage = result["usage"] usage = to_usage_dict(raw_usage) diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 57f8134..f350536 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -146,19 +146,27 @@ class ProviderHandler: - ``__call__`` — blocking invocation - ``stream`` — optional async-generator streaming (may be ``None``) - ``provides_for`` — ``(provider_name, mode)`` tuple or ``None`` + - ``capture_content`` — whether this handler was built with content capture on + + ``capture_content`` is declared here so the client core can apply the handler's own content + decision to content it writes on the handler's behalf — notably the judge's reasoning — + without reaching into the factory's closure. """ provides_for: tuple[str, Literal["agent", "messages"]] | None + capture_content: bool def __init__( self, fn: _HandlerFn, provides_for: tuple[str, Literal["agent", "messages"]] | None = None, stream_fn: _StreamFn | None = None, + capture_content: bool = False, ) -> None: self._fn = fn self.provides_for = provides_for self._stream_fn = stream_fn + self.capture_content = capture_content async def __call__( self, diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index 0eb2c21..676934d 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -21,12 +21,18 @@ def create_handler( provides_for: tuple[str, Literal["agent", "messages"]], fn: _HandlerFn, stream_fn: _StreamFn | None = None, + capture_content: bool = False, ) -> ProviderHandler: """ Wraps a plain async callable in a :class:`ProviderHandler` with the given ``provides_for`` metadata and optional streaming implementation. """ - return ProviderHandler(fn=fn, provides_for=provides_for, stream_fn=stream_fn) + return ProviderHandler( + fn=fn, + provides_for=provides_for, + stream_fn=stream_fn, + capture_content=capture_content, + ) def collapse_messages_to_instructions(config: AiConfigRep) -> AiConfigRep: diff --git a/packages/client/tests/test_conversation.py b/packages/client/tests/test_conversation.py index a3a86bc..e4e0d5d 100644 --- a/packages/client/tests/test_conversation.py +++ b/packages/client/tests/test_conversation.py @@ -155,3 +155,31 @@ def test_does_not_stamp_third_party_instrumentation_spans(self) -> None: assert (span.attributes or {}).get(GEN_AI_CONVERSATION_ID) is None, ( span.name ) + + +class TestJudgeExplanationGating: + """The judge's reasoning follows the same content gate as prompts and completions.""" + + async def test_writes_explanation_when_supplied(self) -> None: + async with with_judge_evaluation("relevance-judge") as record: + with _tracer.start_as_current_span("invoke_agent"): + pass + record(0.8, "on topic and complete") + span = next(s for s in finished() if s.name == "invoke_agent") + assert ( + span.attributes["gen_ai.evaluation.explanation"] == "on topic and complete" + ) + event = next(e for e in span.events if e.name == "gen_ai.evaluation.result") + assert ( + event.attributes["gen_ai.evaluation.explanation"] == "on topic and complete" + ) + + async def test_omits_explanation_when_not_supplied(self) -> None: + async with with_judge_evaluation("relevance-judge") as record: + with _tracer.start_as_current_span("invoke_agent"): + pass + record(0.8) + span = next(s for s in finished() if s.name == "invoke_agent") + assert not any("explanation" in k for k in span.attributes) + event = next(e for e in span.events if e.name == "gen_ai.evaluation.result") + assert not any("explanation" in k for k in (event.attributes or {})) diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py index 0514458..0e13ebc 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py @@ -340,7 +340,12 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler(("*", "agent"), _call_impl, _stream_impl) # type: ignore[arg-type] + return create_handler( + ("*", "agent"), + _call_impl, # type: ignore[arg-type] + _stream_impl, # type: ignore[arg-type] + capture_content=capture_content, + ) async def _stream_gen( diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py index ce5124c..f5dcd95 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py @@ -604,7 +604,12 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler(("*", "messages"), _call_impl, _stream_impl) # type: ignore[arg-type] + return create_handler( + ("*", "messages"), + _call_impl, # type: ignore[arg-type] + _stream_impl, # type: ignore[arg-type] + capture_content=capture_content, + ) async def _stream_gen( diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py index 6c4941b..45ac311 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py @@ -509,7 +509,12 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler(("OpenAI", "agent"), _call_impl, _stream_impl) # type: ignore[arg-type] + return create_handler( + ("OpenAI", "agent"), + _call_impl, # type: ignore[arg-type] + _stream_impl, # type: ignore[arg-type] + capture_content=capture_content, + ) def _stringify_output(value: Any) -> str: diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py index f81ddfb..0d651eb 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py @@ -361,7 +361,12 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler(("OpenAI", "messages"), _call_impl, _stream_impl) # type: ignore[arg-type] + return create_handler( + ("OpenAI", "messages"), + _call_impl, # type: ignore[arg-type] + _stream_impl, # type: ignore[arg-type] + capture_content=capture_content, + ) def _final_output_messages(output: str) -> list[SpanMessage]: diff --git a/tests/test_cross_handler_parity.py b/tests/test_cross_handler_parity.py index ad66f25..ff47d17 100644 --- a/tests/test_cross_handler_parity.py +++ b/tests/test_cross_handler_parity.py @@ -309,11 +309,12 @@ def test_the_langchain_provider_name_is_binary_not_a_passthrough( "gen_ai.agent.name", "gen_ai.conversation.id", # Judge evaluation event + mirrored span attributes on the judge invoke_agent span. - # No `explanation`: it is model prose about the user's conversation, and content attributes - # need capture_content, which this layer does not receive. See TELEMETRY-CONTRACT.md 4a. + # `explanation` is gated on the judge handler's capture_content, like every other content + # attribute. See TELEMETRY-CONTRACT.md 4a. "gen_ai.evaluation.result", "gen_ai.evaluation.name", "gen_ai.evaluation.score.value", + "gen_ai.evaluation.explanation", # Usage "gen_ai.usage.input_tokens", "gen_ai.usage.output_tokens",