diff --git a/TELEMETRY-CONTRACT.md b/TELEMETRY-CONTRACT.md index 513b8fd..65a8364 100644 --- a/TELEMETRY-CONTRACT.md +++ b/TELEMETRY-CONTRACT.md @@ -95,6 +95,8 @@ able to tell from the trace which path ran. | `launchdarkly.run.id` | `TrackData.runId` | `set_ld_span_attributes` | | `launchdarkly.graph.key` | `TrackData.graphKey`, only when present | `set_ld_span_attributes` | | `launchdarkly.stream.abandoned` | `True`, only when abandoned | `end_span_once` | +| `gen_ai.evaluation.name` | judge config key, judge roots only | `with_judge_evaluation`, see section 4a | +| `gen_ai.evaluation.score.value` | numeric score, judge roots only | `with_judge_evaluation`, see section 4a | The root also carries one span event, `feature_flag`, with these event attributes: @@ -212,6 +214,29 @@ when `conversation_id(...)` is bound. --- +## 4a. Judge evaluation events + +A judge run is itself a tracked AI call (`invoke_agent` + `chat`). After the score is parsed, the +SDK writes a `gen_ai.evaluation.result` span event on that `invoke_agent` span: + +| Event attribute | Value | +|---|---| +| `gen_ai.evaluation.name` | judge config key | +| `gen_ai.evaluation.score.value` | numeric score, only when the judge returned a finite number | + +The same keys are mirrored as span attributes, so section 2 lists them too. +`gen_ai.evaluation.score.label` is not invented. +The existing `track(evaluationMetricKey)` call is unchanged and still feeds AI Config Monitoring — +a judge that returns a non-numeric score emits no evaluation event but still tracks the metric. + +`gen_ai.evaluation.explanation` is deliberately **not** emitted. The judge's reasoning is +model-generated prose about the user's conversation — content, under section 7 — and content +attributes require `captureContent` / `capture_content`, a handler-factory option this layer does +not receive. The reasoning is still returned to the caller in `judgeResults` / `judge_results`; +only the telemetry copy is withheld. Exporting it needs its own opt-in. + +--- + ## 5. Finish reasons One vocabulary across all six handlers: `stop`, `length`, `content_filter`, `tool_calls`, `error`. diff --git a/examples/conversation.py b/examples/conversation.py new file mode 100644 index 0000000..278afd9 --- /dev/null +++ b/examples/conversation.py @@ -0,0 +1,87 @@ +""" +Example: a multi-turn conversation grouped under one ``gen_ai.conversation.id``, with inline +judge evaluation on every turn. + +This is the end-to-end check for O11Y-1888. Run it, then open the printed conversation id in +LaunchDarkly's Conversations view and confirm: + + 1. One conversation, three turns — not three conversations. Every span of every turn carries + the same id: root, ``chat``, ``execute_tool``, and the judge's own ``invoke_agent``. + 2. Each turn shows a score badge, sourced from the ``gen_ai.evaluation.result`` span event on + the judge span. + 3. No judge reasoning anywhere in the telemetry. The score and the judge's config key are + exported; the explanation is not, because it is model prose about the user's conversation + and content attributes require ``capture_content``. The reasoning IS printed below, straight + from ``judge_results`` — that is the caller's copy, and it is unaffected. + +The flag key must point at an AI Config with a ``judge_configuration``, otherwise there are no +judge turns to look at. + +Usage (via main.py): + python main.py conversation "" +""" + +from __future__ import annotations + +import sys +from typing import Any + +import examples.register # noqa: F401 – side-effect: populate global_registry +from examples.utils import new_context, new_conversation_id +from launchdarkly_ai_server import config, conversation_id, global_registry + +FOLLOW_UPS = [ + "Can you give me a concrete example of that?", + "What is the most common mistake teams make with it?", +] + + +async def run(key: str, user_input: str) -> None: + conversation = new_conversation_id("conversation-example") + ctx = new_context() + history: list[dict[str, Any]] = [] + + print(f"[conversation] {conversation}", file=sys.stderr) + + turns = [user_input or "What is a feature flag?", *FOLLOW_UPS] + + for index, prompt in enumerate(turns, start=1): + # One binding per turn, same id every time — that is what makes them one conversation + # rather than three. Re-binding per turn is the realistic shape: each turn is usually a + # separate inbound request that looks the id up from its own thread/session. + with conversation_id(conversation): + response = await config( + key=key, + registry=global_registry, + ).invoke(prompt, ctx, None, history) + + text = ( + response.response + if isinstance(response.response, str) + else str(response.response) + ) + print(f"\n─── turn {index} ───\n> {prompt}\n{text}") + + judge_results = response.judge_results or {} + for judge_key, result in judge_results.items(): + # `response` here is the judge's reasoning. It reaches the caller and is deliberately + # absent from the span — see the module docstring. + score = getattr(result, "score", None) + reasoning = getattr(result, "response", None) + print( + f"[judge] {judge_key} score={score} reasoning={reasoning}", + file=sys.stderr, + ) + if not judge_results: + print( + "[judge] no judges ran — does this AI Config have a judge_configuration?", + file=sys.stderr, + ) + + history.append({"role": "user", "content": prompt}) + history.append({"role": "assistant", "content": text}) + + print( + f"\n[conversation] done — open {conversation} in the Conversations view", + file=sys.stderr, + ) diff --git a/main.py b/main.py index 2f957fb..6befa5e 100644 --- a/main.py +++ b/main.py @@ -43,6 +43,7 @@ "agent": "examples.agent", "streaming": "examples.streaming", "graph": "examples.graph_example", + "conversation": "examples.conversation", "history": "examples.history", "judge": "examples.judge_example", "claude-agents": "examples.claude_agents_example", diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py index cc8bf5e..182d31e 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py @@ -617,7 +617,12 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler(("Anthropic", "agent"), _call_impl, _stream_impl) # type: ignore[arg-type] + return create_handler( + ("Anthropic", "agent"), + _call_impl, # type: ignore[arg-type] + _stream_impl, # type: ignore[arg-type] + capture_content=capture_content, + ) async def _stream_gen( diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py index 7bde917..254a01e 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py @@ -383,7 +383,12 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler(("Anthropic", "messages"), _call_impl, _stream_impl) # type: ignore[arg-type] + return create_handler( + ("Anthropic", "messages"), + _call_impl, # type: ignore[arg-type] + _stream_impl, # type: ignore[arg-type] + capture_content=capture_content, + ) async def _stream_gen( diff --git a/packages/client/agents.md b/packages/client/agents.md index 61d99cf..9f143ac 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -120,7 +120,7 @@ Handlers may return any of these — the client normalizes them before emitting - Calls `handler(config, user_input, tool_handlers, variables)` - On success: emits `$ld:ai:generation:success` + token tracks - On error: emits `$ld:ai:generation:error` then re-raises -3. If `judge_configuration.judges` is present, runs each judge handler (sampled by `sampling_rate`) against the primary response and tracks `evaluation_metric_key`. +3. If `judge_configuration.judges` is present, runs each judge handler (sampled by `sampling_rate`) against the primary response, tracks `evaluation_metric_key`, and emits a `gen_ai.evaluation.result` span event on the judge's `invoke_agent` span (`gen_ai.evaluation.name` / `.score.value` / `.explanation`). 4. Returns `ProviderResponse`: `{ response: str, usage: UsageDict, track_data: TrackData, judge_results?: dict[str, JudgeResult], judge_tasks?: list[JudgeTask] }`. `judge_results` is populated when `skip_judges=False` (default) and judges ran; `judge_tasks` is populated when `skip_judges=True`. --- diff --git a/packages/client/src/launchdarkly_ai_server/conversation.py b/packages/client/src/launchdarkly_ai_server/conversation.py index 72dc969..38feb42 100644 --- a/packages/client/src/launchdarkly_ai_server/conversation.py +++ b/packages/client/src/launchdarkly_ai_server/conversation.py @@ -1,4 +1,4 @@ -"""Caller-supplied ``gen_ai.conversation.id``. +"""Caller-supplied ``gen_ai.conversation.id`` and judge evaluation span events. A dedicated OTel context key, not W3C baggage: the id must not leak onto outbound provider HTTP calls. A multi-tenant process binds a different id per request. @@ -6,8 +6,10 @@ from __future__ import annotations -from collections.abc import AsyncGenerator, Iterator -from contextlib import aclosing, contextmanager +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterator +from contextlib import aclosing, asynccontextmanager, contextmanager +from dataclasses import dataclass +from time import time_ns from typing import TYPE_CHECKING, Any from opentelemetry import context as otel_context @@ -25,6 +27,16 @@ GEN_AI_CONVERSATION_ID = "gen_ai.conversation.id" _CONV_KEY = otel_context.create_key("launchdarkly.gen_ai.conversation.id") +_EVAL_KEY = otel_context.create_key("launchdarkly.judge.evaluation") + + +@dataclass +class _JudgeEvalCapture: + name: str + released: bool = False + span: Any = None + pending_end: Callable[[], None] | None = None + # Every tracer this SDK creates is named "@launchdarkly/ai-". The processor is registered # on the *global* provider, so without this gate it stamps a caller-supplied id onto every span in @@ -67,6 +79,59 @@ def _conversation_id_from(ctx: otel_context.Context | None) -> str | None: return value if isinstance(value, str) and value else None +def _record_evaluation( + span: Any, name: str, score: float, explanation: str | None = None +) -> None: + """Write the judge score as a ``gen_ai.evaluation.result`` event plus mirrored attributes. + + ``explanation`` is passed only when the judge's own handler captures content — it is model + prose about the user's conversation, so it follows the same gate as every other content + attribute. The reasoning always reaches the caller in ``judge_results``. + """ + if span is None or not span.is_recording(): + return + attrs: dict[str, Any] = { + "gen_ai.evaluation.name": name, + "gen_ai.evaluation.score.value": score, + } + if explanation: + attrs["gen_ai.evaluation.explanation"] = explanation + span.add_event("gen_ai.evaluation.result", attrs) + for key, value in attrs.items(): + span.set_attribute(key, value) + + +def _delay_invoke_agent_end(span: Any, capture: _JudgeEvalCapture) -> None: + original_end = span.end + ended = False + + def wrapped_end(*args: Any, **kwargs: Any) -> None: + nonlocal ended + if ended: + return + if capture.released: + ended = True + original_end(*args, **kwargs) + return + capture.span = span + # Freeze the end time at the handler's call. Replaying a no-arg end() later would let the + # SDK stamp time_ns() at release, inflating the judge span by the tracking and parsing + # work that runs between the handler ending the span and the score being recorded. + if not args and "end_time" not in kwargs: + kwargs = {**kwargs, "end_time": time_ns()} + + def pending() -> None: + nonlocal ended + if ended: + return + ended = True + original_end(*args, **kwargs) + + capture.pending_end = pending + + span.end = wrapped_end + + @contextmanager def conversation_id(conversation: str | None) -> Iterator[None]: """Bind a caller-supplied conversation id for the duration of the ``with`` block. @@ -88,6 +153,9 @@ def conversation_id(conversation: str | None) -> Iterator[None]: otel_context.detach(token) +RecordEvaluation = Callable[..., None] + + async def _stream_with_bound_id( generator: AsyncGenerator[Any, None], conversation: str ) -> AsyncGenerator[Any, None]: @@ -130,8 +198,35 @@ def bind_conversation_id( return _stream_with_bound_id(generator, conversation) +@asynccontextmanager +async def with_judge_evaluation(name: str) -> AsyncIterator[RecordEvaluation]: + """Hold the judge ``invoke_agent`` span open until ``record`` runs. + + ``execute_and_track`` returns after the handler has already called ``span.end()``, + so without this delay the evaluation event would be dropped. + """ + capture = _JudgeEvalCapture(name=name) + + def record(score: float, explanation: str | None = None) -> None: + if capture.span is not None: + _record_evaluation(capture.span, capture.name, score, explanation) + + token = otel_context.attach(otel_context.set_value(_EVAL_KEY, capture)) + try: + yield record + finally: + capture.released = True + try: + if capture.pending_end is not None: + capture.pending_end() + finally: + # Detach even if ending the span raises (a user span processor's on_end can throw); + # otherwise the judge capture stays bound in this task's context for good. + otel_context.detach(token) + + class ConversationIdSpanProcessor(_SpanProcessorBase): - """Stamps ``gen_ai.conversation.id`` write-if-absent on every span. + """Stamps ``gen_ai.conversation.id`` write-if-absent; delays judge ``invoke_agent`` end. Structurally a ``SpanProcessor``; the base is only real to a type checker so that an api-only install (no ``[otel]`` extra) still imports this module. @@ -150,6 +245,15 @@ def on_start( if conv and _is_launchdarkly_span(span): set_conversation_id_if_absent(span, conv) + capture = otel_context.get_value(_EVAL_KEY, ctx) + if capture is None: + capture = otel_context.get_value(_EVAL_KEY) + if ( + isinstance(capture, _JudgeEvalCapture) + and getattr(span, "name", None) == "invoke_agent" + ): + _delay_invoke_agent_end(span, capture) + def on_end(self, span: Any) -> None: return None diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index 898f864..4481299 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -3,8 +3,10 @@ import logging import random from collections.abc import Callable +from math import isfinite from typing import Any +from .conversation import with_judge_evaluation from .types import ( AiConfigRep, JudgeRunResult, @@ -47,6 +49,18 @@ def _provider_matches(handler: ProviderHandler, provider: str | None) -> bool: ) +def _numeric_score(score: Any) -> float | None: + """Return ``score`` as a float only when it already is a finite number. + + Never raises. A judge that returns ``"0.9 (high)"`` or ``None`` must not take down the + evaluation metric track that follows, and must not put a string where semconv defines a double. + """ + if isinstance(score, bool) or not isinstance(score, (int, float)): + return None + value = float(score) + return value if isfinite(value) else None + + async def run_judges( *, config: AiConfigRep, @@ -154,51 +168,58 @@ async def run_judges( filter(None, [user_input, llm_response, _FORMATTING_INSTRUCTIONS]) ) - result = await execute_and_track( - config_key=judge_key, - config=effective_judge_config, - meta=judge_meta, - user_context=user_context, - handler=judge_handler, - user_input=llm_response, - tool_handlers=None, - graph_key=graph_key, - variables={ - "message_history": message_history, - "response_to_evaluate": llm_response, - }, - ) - - raw = result["response"] - judge_response = raw if isinstance(raw, str) else str(raw) - - parsed = parse_json_with_possible_fences(judge_response) - if not parsed: - raise ValueError("Invalid JSON from judge") + async with with_judge_evaluation(judge_key) as record_evaluation: + result = await execute_and_track( + config_key=judge_key, + config=effective_judge_config, + meta=judge_meta, + user_context=user_context, + handler=judge_handler, + user_input=llm_response, + tool_handlers=None, + graph_key=graph_key, + variables={ + "message_history": message_history, + "response_to_evaluate": llm_response, + }, + ) - score = parsed.get("score") - reasoning = parsed.get("reasoning", "") - judge_results[judge_key] = { - "usage": result["usage"], - "response": reasoning, - "score": score, - } + raw = result["response"] + judge_response = raw if isinstance(raw, str) else str(raw) + + parsed = parse_json_with_possible_fences(judge_response) + if not parsed: + raise ValueError("Invalid JSON from judge") + + score = parsed.get("score") + reasoning = parsed.get("reasoning", "") + judge_results[judge_key] = { + "usage": result["usage"], + "response": reasoning, + "score": score, + } + numeric_score = _numeric_score(score) + if numeric_score is not None: + record_evaluation( + numeric_score, + reasoning if judge_handler.capture_content else None, + ) - evaluation_metric_key = ( - judge_ai_config.get("evaluationMetricKey") - if isinstance(judge_ai_config, dict) - else None - ) - if evaluation_metric_key and score is not None: - from .lifecycle import get_client - - client = get_client() - client.track( - evaluation_metric_key, - to_ld_context(client, user_context), - {**base_track_data, "judgeConfigKey": judge_key}, - score, + evaluation_metric_key = ( + judge_ai_config.get("evaluationMetricKey") + if isinstance(judge_ai_config, dict) + else None ) + if evaluation_metric_key and score is not None: + from .lifecycle import get_client + + client = get_client() + client.track( + evaluation_metric_key, + to_ld_context(client, user_context), + {**base_track_data, "judgeConfigKey": judge_key}, + score, + ) except Exception as exc: logger.error("Judge '%s' failed: %s", judge_key, exc) @@ -384,39 +405,46 @@ def _matches(h: ProviderHandler) -> bool: filter(None, [task.actual_output, _FORMATTING_INSTRUCTIONS]) ) - result = await execute_and_track( - config_key=task.config_key, - config=effective_config, - meta=task.judge_meta, - user_context=task.user_context, - handler=judge_handler, - user_input=task.actual_output, - tool_handlers=None, - variables={ - **(task.variables or {}), - "message_history": message_history, - "response_to_evaluate": task.actual_output, - }, - ) - - raw = result["response"] - judge_response = raw if isinstance(raw, str) else str(raw) - parsed = parse_json_with_possible_fences(judge_response) - if not parsed: - return None + async with with_judge_evaluation(task.config_key) as record_evaluation: + result = await execute_and_track( + config_key=task.config_key, + config=effective_config, + meta=task.judge_meta, + user_context=task.user_context, + handler=judge_handler, + user_input=task.actual_output, + tool_handlers=None, + variables={ + **(task.variables or {}), + "message_history": message_history, + "response_to_evaluate": task.actual_output, + }, + ) - score = parsed.get("score", 0.0) - reasoning = parsed.get("reasoning", "") - raw_usage = result["usage"] + raw = result["response"] + judge_response = raw if isinstance(raw, str) else str(raw) + parsed = parse_json_with_possible_fences(judge_response) + if not parsed: + return None + + score = parsed.get("score", 0.0) + reasoning = parsed.get("reasoning", "") + numeric_score = _numeric_score(score) + if numeric_score is not None: + record_evaluation( + numeric_score, + reasoning if judge_handler.capture_content else None, + ) + raw_usage = result["usage"] - usage = to_usage_dict(raw_usage) + usage = to_usage_dict(raw_usage) - merged_track_data: TrackData = { - **task.parent_track_data, - **result["track_data"], - "judgeConfigKey": task.config_key, - } + merged_track_data: TrackData = { + **task.parent_track_data, + **result["track_data"], + "judgeConfigKey": task.config_key, + } - return JudgeRunResult( - score=score, response=reasoning, usage=usage, track_data=merged_track_data - ) + return JudgeRunResult( + score=score, response=reasoning, usage=usage, track_data=merged_track_data + ) diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 57f8134..f350536 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -146,19 +146,27 @@ class ProviderHandler: - ``__call__`` — blocking invocation - ``stream`` — optional async-generator streaming (may be ``None``) - ``provides_for`` — ``(provider_name, mode)`` tuple or ``None`` + - ``capture_content`` — whether this handler was built with content capture on + + ``capture_content`` is declared here so the client core can apply the handler's own content + decision to content it writes on the handler's behalf — notably the judge's reasoning — + without reaching into the factory's closure. """ provides_for: tuple[str, Literal["agent", "messages"]] | None + capture_content: bool def __init__( self, fn: _HandlerFn, provides_for: tuple[str, Literal["agent", "messages"]] | None = None, stream_fn: _StreamFn | None = None, + capture_content: bool = False, ) -> None: self._fn = fn self.provides_for = provides_for self._stream_fn = stream_fn + self.capture_content = capture_content async def __call__( self, diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index 0eb2c21..676934d 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -21,12 +21,18 @@ def create_handler( provides_for: tuple[str, Literal["agent", "messages"]], fn: _HandlerFn, stream_fn: _StreamFn | None = None, + capture_content: bool = False, ) -> ProviderHandler: """ Wraps a plain async callable in a :class:`ProviderHandler` with the given ``provides_for`` metadata and optional streaming implementation. """ - return ProviderHandler(fn=fn, provides_for=provides_for, stream_fn=stream_fn) + return ProviderHandler( + fn=fn, + provides_for=provides_for, + stream_fn=stream_fn, + capture_content=capture_content, + ) def collapse_messages_to_instructions(config: AiConfigRep) -> AiConfigRep: diff --git a/packages/client/tests/test_conversation.py b/packages/client/tests/test_conversation.py index 5bd8a57..e4e0d5d 100644 --- a/packages/client/tests/test_conversation.py +++ b/packages/client/tests/test_conversation.py @@ -1,6 +1,8 @@ from __future__ import annotations +import asyncio from collections.abc import Iterator +from time import time_ns import pytest from opentelemetry import trace @@ -13,6 +15,7 @@ ConversationIdSpanProcessor, conversation_id, set_conversation_id_if_absent, + with_judge_evaluation, ) _exporter = InMemorySpanExporter() @@ -81,6 +84,51 @@ def test_writes_session_id_when_unbound(self) -> None: assert finished()[0].attributes[GEN_AI_CONVERSATION_ID] == "sess-abc" +class TestJudgeEvaluation: + async def test_puts_evaluation_event_on_invoke_agent(self) -> None: + async with with_judge_evaluation("relevance-judge") as record: + with _tracer.start_as_current_span("invoke_agent") as span: + span.set_attribute("gen_ai.operation.name", "invoke_agent") + record(0.91) + span = next(s for s in finished() if s.name == "invoke_agent") + assert span.attributes["gen_ai.evaluation.name"] == "relevance-judge" + assert span.attributes["gen_ai.evaluation.score.value"] == 0.91 + event = next(e for e in span.events if e.name == "gen_ai.evaluation.result") + assert event.attributes["gen_ai.evaluation.name"] == "relevance-judge" + assert event.attributes["gen_ai.evaluation.score.value"] == 0.91 + assert "gen_ai.evaluation.score.label" not in event.attributes + + async def test_does_not_export_the_judge_explanation(self) -> None: + """Judge reasoning is model prose about the user's conversation, i.e. content. + + AGENTS.md gates content attributes behind ``capture_content=True``, which this layer + cannot see, so the reasoning must not reach telemetry at all. + """ + async with with_judge_evaluation("relevance-judge") as record: + with _tracer.start_as_current_span("invoke_agent"): + pass + record(0.2) + span = next(s for s in finished() if s.name == "invoke_agent") + event = next(e for e in span.events if e.name == "gen_ai.evaluation.result") + assert not any("explanation" in k for k in span.attributes) + assert not any("explanation" in k for k in (event.attributes or {})) + + async def test_end_time_is_the_handler_call_not_the_release(self) -> None: + """The deferred end must not stamp flush time, or the judge span absorbs tracking work.""" + async with with_judge_evaluation("slow-judge") as record: + with _tracer.start_as_current_span("invoke_agent"): + pass + ended_at = time_ns() + await asyncio.sleep( + 0.05 + ) # stands in for tracking + parsing after the handler ends + record(0.5) + span = next(s for s in finished() if s.name == "invoke_agent") + assert span.end_time is not None + # Allow scheduling slack, but nothing close to the 50ms of post-end work. + assert span.end_time - ended_at < 20_000_000 + + class TestProcessorScope: """The processor is registered on the *global* provider, so it sees every span in the process. @@ -107,3 +155,31 @@ def test_does_not_stamp_third_party_instrumentation_spans(self) -> None: assert (span.attributes or {}).get(GEN_AI_CONVERSATION_ID) is None, ( span.name ) + + +class TestJudgeExplanationGating: + """The judge's reasoning follows the same content gate as prompts and completions.""" + + async def test_writes_explanation_when_supplied(self) -> None: + async with with_judge_evaluation("relevance-judge") as record: + with _tracer.start_as_current_span("invoke_agent"): + pass + record(0.8, "on topic and complete") + span = next(s for s in finished() if s.name == "invoke_agent") + assert ( + span.attributes["gen_ai.evaluation.explanation"] == "on topic and complete" + ) + event = next(e for e in span.events if e.name == "gen_ai.evaluation.result") + assert ( + event.attributes["gen_ai.evaluation.explanation"] == "on topic and complete" + ) + + async def test_omits_explanation_when_not_supplied(self) -> None: + async with with_judge_evaluation("relevance-judge") as record: + with _tracer.start_as_current_span("invoke_agent"): + pass + record(0.8) + span = next(s for s in finished() if s.name == "invoke_agent") + assert not any("explanation" in k for k in span.attributes) + event = next(e for e in span.events if e.name == "gen_ai.evaluation.result") + assert not any("explanation" in k for k in (event.attributes or {})) diff --git a/packages/client/tests/test_judges.py b/packages/client/tests/test_judges.py index 55c2246..65a802d 100644 --- a/packages/client/tests/test_judges.py +++ b/packages/client/tests/test_judges.py @@ -337,3 +337,24 @@ async def test_returns_empty_dict_when_judges_array_is_empty( base_track_data={}, ) assert result == {} + + +class TestScoreGuard: + """`float(score)` used to sit ahead of the evaluation-metric track, so a junk score killed it.""" + + def test_rejects_non_numeric_scores_without_raising(self) -> None: + from launchdarkly_ai_server.judges import _numeric_score + + for junk in ("0.9 (high)", "85%", None, {"v": 1}, [], True, False): + assert _numeric_score(junk) is None + + def test_accepts_finite_numbers(self) -> None: + from math import inf, nan + + from launchdarkly_ai_server.judges import _numeric_score + + assert _numeric_score(0.9) == 0.9 + assert _numeric_score(1) == 1.0 + assert _numeric_score(0) == 0.0 + assert _numeric_score(inf) is None + assert _numeric_score(nan) is None diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py index 0514458..0e13ebc 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py @@ -340,7 +340,12 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler(("*", "agent"), _call_impl, _stream_impl) # type: ignore[arg-type] + return create_handler( + ("*", "agent"), + _call_impl, # type: ignore[arg-type] + _stream_impl, # type: ignore[arg-type] + capture_content=capture_content, + ) async def _stream_gen( diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py index ce5124c..f5dcd95 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py @@ -604,7 +604,12 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler(("*", "messages"), _call_impl, _stream_impl) # type: ignore[arg-type] + return create_handler( + ("*", "messages"), + _call_impl, # type: ignore[arg-type] + _stream_impl, # type: ignore[arg-type] + capture_content=capture_content, + ) async def _stream_gen( diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py index 6c4941b..45ac311 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py @@ -509,7 +509,12 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler(("OpenAI", "agent"), _call_impl, _stream_impl) # type: ignore[arg-type] + return create_handler( + ("OpenAI", "agent"), + _call_impl, # type: ignore[arg-type] + _stream_impl, # type: ignore[arg-type] + capture_content=capture_content, + ) def _stringify_output(value: Any) -> str: diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py index f81ddfb..0d651eb 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py @@ -361,7 +361,12 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler(("OpenAI", "messages"), _call_impl, _stream_impl) # type: ignore[arg-type] + return create_handler( + ("OpenAI", "messages"), + _call_impl, # type: ignore[arg-type] + _stream_impl, # type: ignore[arg-type] + capture_content=capture_content, + ) def _final_output_messages(output: str) -> list[SpanMessage]: diff --git a/tests/test_cross_handler_parity.py b/tests/test_cross_handler_parity.py index 19ed8de..ff47d17 100644 --- a/tests/test_cross_handler_parity.py +++ b/tests/test_cross_handler_parity.py @@ -308,6 +308,13 @@ def test_the_langchain_provider_name_is_binary_not_a_passthrough( "gen_ai.response.finish_reasons", "gen_ai.agent.name", "gen_ai.conversation.id", + # Judge evaluation event + mirrored span attributes on the judge invoke_agent span. + # `explanation` is gated on the judge handler's capture_content, like every other content + # attribute. See TELEMETRY-CONTRACT.md 4a. + "gen_ai.evaluation.result", + "gen_ai.evaluation.name", + "gen_ai.evaluation.score.value", + "gen_ai.evaluation.explanation", # Usage "gen_ai.usage.input_tokens", "gen_ai.usage.output_tokens",