From a159757a0d7bd847ee6f0303eb14213cd98ad5b4 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Thu, 20 Aug 2026 21:47:56 -0700 Subject: [PATCH 1/6] feat: gate evaluation generation result ingestion --- AGENTS.md | 7 ++ CLAUDE.md | 2 + .../evaluations/flags.py | 44 +++++++++++ .../evaluations/module.py | 13 +++- .../evaluations/runner.py | 4 + .../client/tests/test_evaluation_flags.py | 39 ++++++++++ packages/client/tests/test_evaluations_run.py | 73 ++++++++++++++++++- 7 files changed, 177 insertions(+), 5 deletions(-) create mode 100644 CLAUDE.md create mode 100644 packages/client/src/launchdarkly_ai_server/evaluations/flags.py create mode 100644 packages/client/tests/test_evaluation_flags.py diff --git a/AGENTS.md b/AGENTS.md index 69f1975..caac199 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -703,3 +703,10 @@ response = await graph( }, ).invoke(user_input, context) ``` + +## Maintaining this file + +Keep this file for knowledge useful to almost every future agent session in this project. +Do not repeat what the codebase already shows; point to the authoritative file or command instead. +Prefer rewriting or pruning existing entries over appending new ones. +When updating this file, preserve this bar for all agents and keep entries concise. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..a9d4d26 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,2 @@ + +@AGENTS.md diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/flags.py b/packages/client/src/launchdarkly_ai_server/evaluations/flags.py new file mode 100644 index 0000000..4468d88 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/evaluations/flags.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import inspect +import logging +from typing import Any, Final + +from ..utils import to_ld_context + +logger = logging.getLogger(__name__) + +ENABLE_TOOL_CALLS_IN_OFFLINE_EVALUATIONS_FLAG_KEY: Final[str] = ( + "enable-tool-calls-in-offline-evaluations" +) +"""Canonical rollout flag for tool calls in offline evaluations.""" + + +async def should_skip_generation_result_ingestion( + client: Any, + project_key: str, +) -> bool: + """Return whether the rollout flag selects the no-ingest path. + + Flag evaluation is fail-safe: false, malformed, or failed evaluations retain + the existing generation-result ingestion behavior. + """ + try: + context = to_ld_context( + client, + {"kind": "project", "key": project_key}, + ) + result = client.variation( + ENABLE_TOOL_CALLS_IN_OFFLINE_EVALUATIONS_FLAG_KEY, + context, + False, + ) + value = await result if inspect.isawaitable(result) else result + return value is True + except Exception: + logger.warning( + "Unable to evaluate %s; generation results will be ingested", + ENABLE_TOOL_CALLS_IN_OFFLINE_EVALUATIONS_FLAG_KEY, + exc_info=True, + ) + return False diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index dba85ba..9234099 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -12,6 +12,7 @@ Transport, urllib_transport, ) +from .flags import should_skip_generation_result_ingestion from .runner import EvalHandler, EvaluationsRunner, ToolImplementation, _segment from .types import EvalRunResult, GenerationConfig @@ -69,8 +70,12 @@ async def run( timeout=timeout, ) run_tools = dict(tools or {}) + skip_generation_result_ingestion = False if self._sdk_key: - await init_client({"sdkKey": self._sdk_key}) + client = await init_client({"sdkKey": self._sdk_key}) + skip_generation_result_ingestion = ( + await should_skip_generation_result_ingestion(client, project_key) + ) # Tool verification is deliberately first: a typo must not create records. resolved_tools = self._runner._resolve_tools(project_key, run_tools) @@ -91,7 +96,11 @@ async def run( concurrency, ) self._runner._ingest_results( - project_key, evaluation.id, evaluation_run.id, results + project_key, + evaluation.id, + evaluation_run.id, + results, + skip_generation_result_ingestion=skip_generation_result_ingestion, ) completed = await self._runner._poll_run( project_key, evaluation.id, evaluation_run.id, timeout diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 26c68e3..4006e90 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -401,7 +401,11 @@ def _ingest_results( evaluation_id: str, run_id: str, results: list[dict[str, Any]], + *, + skip_generation_result_ingestion: bool = False, ) -> None: + if skip_generation_result_ingestion: + return path = ( f"projects/{_segment(project_key)}/evaluations/{_segment(evaluation_id)}" f"/runs/{_segment(run_id)}/generation-results" diff --git a/packages/client/tests/test_evaluation_flags.py b/packages/client/tests/test_evaluation_flags.py new file mode 100644 index 0000000..fd60538 --- /dev/null +++ b/packages/client/tests/test_evaluation_flags.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from launchdarkly_ai_server.evaluations.flags import ( + ENABLE_TOOL_CALLS_IN_OFFLINE_EVALUATIONS_FLAG_KEY, + should_skip_generation_result_ingestion, +) + + +@pytest.mark.asyncio +async def test_enabled_flag_selects_generation_result_ingestion_skip() -> None: + client = MagicMock() + client.variation = AsyncMock(return_value=True) + + assert await should_skip_generation_result_ingestion(client, "project-key") is True + client.variation.assert_awaited_once_with( + ENABLE_TOOL_CALLS_IN_OFFLINE_EVALUATIONS_FLAG_KEY, + {"kind": "project", "key": "project-key"}, + False, + ) + + +@pytest.mark.asyncio +async def test_disabled_flag_preserves_generation_result_ingestion() -> None: + client = MagicMock() + client.variation = AsyncMock(return_value=False) + + assert await should_skip_generation_result_ingestion(client, "project-key") is False + + +@pytest.mark.asyncio +async def test_flag_evaluation_error_preserves_generation_result_ingestion() -> None: + client = MagicMock() + client.variation = AsyncMock(side_effect=RuntimeError("delivery unavailable")) + + assert await should_skip_generation_result_ingestion(client, "project-key") is False diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 1d25be9..e646f46 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -3,6 +3,7 @@ import json from collections.abc import Callable from typing import Any +from unittest.mock import AsyncMock, MagicMock import pytest @@ -86,9 +87,10 @@ def lookup_order(order_id: str) -> str: @pytest.mark.asyncio -async def test_run_calls_private_operations_in_order_and_returns_server_verdict() -> ( - None -): +async def test_run_calls_private_operations_in_order_and_returns_server_verdict( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("LD_SDK_KEY", raising=False) transport = SequencedTransport( [ response( @@ -189,6 +191,7 @@ async def test_run_calls_private_operations_in_order_and_returns_server_verdict( ] ) evals = init_evaluations(api_token="token", transport=transport) + assert evals.sdk_key is None result = await evals.run( project_key="proj", @@ -253,6 +256,70 @@ async def test_run_calls_private_operations_in_order_and_returns_server_verdict( assert ingested[0]["variables"]["expected_output"] == "Found A19" +@pytest.mark.asyncio +async def test_enabled_rollout_flag_skips_generation_result_ingestion( + monkeypatch: pytest.MonkeyPatch, +) -> None: + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [{"rowIndex": 3, "input": "hello", "variables": {}}], + total=1, + ), + ), + response(201, {"id": "evaluation-id", "name": "eval-key"}), + response( + 201, + { + "id": "run-id", + "evaluationId": "evaluation-id", + "state": "PENDING", + }, + ), + response( + 200, + { + "id": "run-id", + "evaluationId": "evaluation-id", + "state": "COMPLETE", + "verdict": "passed", + }, + ), + response(200, {"statusCounts": {"total": 1, "passed": 1}}), + ] + ) + client = MagicMock() + client.variation = AsyncMock(return_value=True) + + async def fake_init_client(options: dict[str, Any]) -> MagicMock: + assert options == {"sdkKey": "sdk-key"} + return client + + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.init_client", fake_init_client + ) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "generated"} + + result = await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert result.passed is True + assert not any( + request["url"].endswith("/generation-results") for request in transport.requests + ) + + @pytest.mark.asyncio async def test_run_rejects_instructions_and_messages_before_network_io() -> None: transport = SequencedTransport([]) From 7311f1668ee1553792d38c4db7eb36dedf3c1600 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Thu, 20 Aug 2026 21:47:54 -0700 Subject: [PATCH 2/6] feat: add deterministic evaluation scorers --- .../evaluations/scorers.py | 211 ++++++++++++++++++ .../client/tests/test_evaluation_scorers.py | 160 +++++++++++++ 2 files changed, 371 insertions(+) create mode 100644 packages/client/src/launchdarkly_ai_server/evaluations/scorers.py create mode 100644 packages/client/tests/test_evaluation_scorers.py diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/scorers.py b/packages/client/src/launchdarkly_ai_server/evaluations/scorers.py new file mode 100644 index 0000000..d6f88da --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/evaluations/scorers.py @@ -0,0 +1,211 @@ +"""Deterministic function scorers for client-side evaluations.""" + +from __future__ import annotations + +import inspect +import math +import time +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from types import MappingProxyType +from typing import Any, Literal + +ScoreValue = bool | int | float +ScorerFunction = Callable[["ScorerRow", str | None], ScoreValue | Awaitable[ScoreValue]] +ScorerStatus = Literal["COMPLETE", "ERROR"] +ScorerErrorCode = Literal["invalid_score", "scorer_error"] + + +@dataclass(frozen=True, slots=True) +class ScorerRow: + """Complete rendered dataset-row context passed to a scorer function.""" + + row_index: int + input: str | None + expected_output: str | None + variables: Mapping[str, Any] + metadata: Mapping[str, Any] | None + + def __post_init__(self) -> None: + if type(self.row_index) is not int or self.row_index < 0: + raise ValueError("row_index must be a non-negative integer") + if self.input is not None and not isinstance(self.input, str): + raise TypeError("input must be a string or None") + if self.expected_output is not None and not isinstance( + self.expected_output, str + ): + raise TypeError("expected_output must be a string or None") + object.__setattr__( + self, "variables", _validated_mapping(self.variables, name="variables") + ) + if self.metadata is not None: + object.__setattr__( + self, "metadata", _validated_mapping(self.metadata, name="metadata") + ) + + +@dataclass(frozen=True, slots=True) +class ScorerError: + """Structured scorer failure details for later evaluation-results ingest.""" + + code: ScorerErrorCode + message: str + exception_type: str + + +@dataclass(frozen=True, slots=True) +class ScorerResult: + """The normalized outcome and execution metadata for one row and scorer.""" + + scorer_name: str + row_index: int + score: float | None + started_at: datetime + evaluated_at: datetime + latency_ms: float + status: ScorerStatus + error: ScorerError | None = None + + def __post_init__(self) -> None: + if self.status not in {"COMPLETE", "ERROR"}: + raise ValueError(f"unknown scorer result status: {self.status!r}") + if self.status == "COMPLETE": + if self.score is None or self.error is not None: + raise ValueError( + "a COMPLETE scorer result requires a score and no error" + ) + elif self.score is not None or self.error is None: + raise ValueError("an ERROR scorer result requires an error and no score") + + +@dataclass(frozen=True, slots=True) +class Scorer: + """A named deterministic scorer with an async execution method. + + The scorer function follows the Phase 3 protocol ``fn(row, output)`` and may + be synchronous or asynchronous. ``execute`` converts function failures and + invalid return values into typed error results so evaluation orchestration + can continue processing the remaining rows. + """ + + name: str + fn: ScorerFunction + threshold: float = 1.0 + + def __post_init__(self) -> None: + if not isinstance(self.name, str) or not self.name.strip(): + raise ValueError("scorer name must not be blank") + if not callable(self.fn): + raise TypeError("scorer fn must be callable") + if isinstance(self.threshold, bool) or not isinstance( + self.threshold, (int, float) + ): + raise TypeError("scorer threshold must be numeric") + normalized_threshold = float(self.threshold) + if ( + not math.isfinite(normalized_threshold) + or not 0 <= normalized_threshold <= 1 + ): + raise ValueError("scorer threshold must be between 0 and 1") + object.__setattr__(self, "threshold", normalized_threshold) + + async def execute(self, row: ScorerRow, output: str | None) -> ScorerResult: + """Run this scorer for one generation and return a normalized result.""" + if not isinstance(row, ScorerRow): + raise TypeError("row must be a ScorerRow") + if output is not None and not isinstance(output, str): + raise TypeError("output must be a string or None") + + started_at = datetime.now(UTC) + started_clock = time.perf_counter() + try: + value = self.fn(row, output) + if inspect.isawaitable(value): + value = await value + score = _normalize_score(value) + except _InvalidScore as error: + return self._error_result( + row=row, + started_at=started_at, + started_clock=started_clock, + code="invalid_score", + error=error, + ) + except Exception as error: + return self._error_result( + row=row, + started_at=started_at, + started_clock=started_clock, + code="scorer_error", + error=error, + ) + + evaluated_at = datetime.now(UTC) + return ScorerResult( + scorer_name=self.name, + row_index=row.row_index, + score=score, + started_at=started_at, + evaluated_at=evaluated_at, + latency_ms=_elapsed_ms(started_clock), + status="COMPLETE", + ) + + def _error_result( + self, + *, + row: ScorerRow, + started_at: datetime, + started_clock: float, + code: ScorerErrorCode, + error: Exception, + ) -> ScorerResult: + evaluated_at = datetime.now(UTC) + message = str(error) or type(error).__name__ + return ScorerResult( + scorer_name=self.name, + row_index=row.row_index, + score=None, + started_at=started_at, + evaluated_at=evaluated_at, + latency_ms=_elapsed_ms(started_clock), + status="ERROR", + error=ScorerError( + code=code, + message=message, + exception_type=type(error).__name__, + ), + ) + + +class _InvalidScore(ValueError): + pass + + +def _validated_mapping(value: object, *, name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise TypeError(f"{name} must be a mapping") + if any(not isinstance(key, str) for key in value): + raise TypeError(f"{name} keys must be strings") + return MappingProxyType(dict(value)) + + +def _normalize_score(value: object) -> float: + if isinstance(value, bool): + return float(value) + if not isinstance(value, (int, float)): + raise _InvalidScore( + "scorer must return bool or a numeric score between 0 and 1; " + f"got {type(value).__name__}" + ) + score = float(value) + if not math.isfinite(score) or not 0 <= score <= 1: + raise _InvalidScore( + f"scorer must return a finite numeric score between 0 and 1; got {value!r}" + ) + return score + + +def _elapsed_ms(started_clock: float) -> float: + return round((time.perf_counter() - started_clock) * 1000, 3) diff --git a/packages/client/tests/test_evaluation_scorers.py b/packages/client/tests/test_evaluation_scorers.py new file mode 100644 index 0000000..2974e52 --- /dev/null +++ b/packages/client/tests/test_evaluation_scorers.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +from typing import Any, cast + +import pytest + +from launchdarkly_ai_server.evaluations.scorers import ( + Scorer, + ScorerRow, + ScoreValue, +) + + +def scorer_row() -> ScorerRow: + return ScorerRow( + row_index=7, + input="Rendered order A19", + expected_output="Refund A19", + variables={"order_id": "A19", "input": "Rendered order A19"}, + metadata={"suite": "refunds", "priority": 1}, + ) + + +@pytest.mark.asyncio +async def test_sync_scorer_receives_generation_output_and_row_context() -> None: + received: dict[str, Any] = {} + + def score(row: ScorerRow, output: str | None) -> float: + received.update( + { + "row_index": row.row_index, + "input": row.input, + "expected_output": row.expected_output, + "variables": dict(row.variables), + "metadata": dict(row.metadata or {}), + "output": output, + } + ) + return 0.75 + + result = await Scorer(name="refund-exists", fn=score).execute( + scorer_row(), "Refund created" + ) + + assert received == { + "row_index": 7, + "input": "Rendered order A19", + "expected_output": "Refund A19", + "variables": {"order_id": "A19", "input": "Rendered order A19"}, + "metadata": {"suite": "refunds", "priority": 1}, + "output": "Refund created", + } + assert result.scorer_name == "refund-exists" + assert result.row_index == 7 + assert result.score == 0.75 + assert result.status == "COMPLETE" + assert result.error is None + assert result.started_at.tzinfo is not None + assert result.evaluated_at >= result.started_at + assert result.latency_ms >= 0 + + +@pytest.mark.asyncio +async def test_async_scorer_is_awaited() -> None: + called = False + + async def score(row: ScorerRow, output: str | None) -> float: + nonlocal called + called = True + assert row.metadata == {"suite": "refunds", "priority": 1} + assert output == "done" + return 0.4 + + result = await Scorer(name="async-check", fn=score).execute(scorer_row(), "done") + + assert called is True + assert result.status == "COMPLETE" + assert result.score == 0.4 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("raw_score", "normalized"), + [(True, 1.0), (False, 0.0), (0, 0.0), (1, 1.0), (0.625, 0.625)], +) +async def test_bool_and_numeric_scores_are_normalized( + raw_score: ScoreValue, normalized: float +) -> None: + def score(row: ScorerRow, output: str | None) -> ScoreValue: + del row, output + return raw_score + + result = await Scorer(name="normalized", fn=score).execute(scorer_row(), "ok") + + assert result.status == "COMPLETE" + assert result.score == normalized + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "raw_score", [None, "1", -0.01, 1.01, float("nan"), float("inf")] +) +async def test_invalid_scorer_results_are_clear_error_results( + raw_score: object, +) -> None: + def score(row: ScorerRow, output: str | None) -> ScoreValue: + del row, output + return cast(ScoreValue, raw_score) + + result = await Scorer(name="bad-result", fn=score).execute(scorer_row(), "ok") + + assert result.scorer_name == "bad-result" + assert result.row_index == 7 + assert result.status == "ERROR" + assert result.score is None + assert result.error is not None + assert result.error.code == "invalid_score" + assert "between 0 and 1" in result.error.message + assert result.started_at.tzinfo is not None + assert result.evaluated_at >= result.started_at + assert result.latency_ms >= 0 + + +@pytest.mark.asyncio +async def test_scorer_exception_is_preserved_as_error_result() -> None: + def score(row: ScorerRow, output: str | None) -> float: + del row, output + raise RuntimeError("database unavailable") + + result = await Scorer(name="db-check", fn=score).execute(scorer_row(), "ok") + + assert result.status == "ERROR" + assert result.score is None + assert result.error is not None + assert result.error.code == "scorer_error" + assert result.error.exception_type == "RuntimeError" + assert result.error.message == "database unavailable" + + +def test_scorer_and_row_dtos_validate_strictly() -> None: + with pytest.raises(ValueError, match="name"): + Scorer(name=" ", fn=lambda row, output: True) + with pytest.raises(ValueError, match="between 0 and 1"): + Scorer(name="check", fn=lambda row, output: True, threshold=1.1) + with pytest.raises(ValueError, match="row_index"): + ScorerRow( + row_index=-1, + input=None, + expected_output=None, + variables={}, + metadata=None, + ) + with pytest.raises(TypeError, match="metadata keys"): + ScorerRow( + row_index=0, + input=None, + expected_output=None, + variables={}, + metadata=cast(dict[str, Any], {1: "invalid"}), + ) From b12cfae33e9d815dc0fcdde441a780682898d726 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Thu, 20 Aug 2026 21:49:46 -0700 Subject: [PATCH 3/6] feat: add offline LaunchDarkly judge foundation --- packages/client/pyproject.toml | 2 +- .../evaluations/judges.py | 412 ++++++++++++++++++ .../client/tests/test_evaluation_judges.py | 386 ++++++++++++++++ uv.lock | 2 + 4 files changed, 801 insertions(+), 1 deletion(-) create mode 100644 packages/client/src/launchdarkly_ai_server/evaluations/judges.py create mode 100644 packages/client/tests/test_evaluation_judges.py diff --git a/packages/client/pyproject.toml b/packages/client/pyproject.toml index 9ea3ce7..ee8f995 100644 --- a/packages/client/pyproject.toml +++ b/packages/client/pyproject.toml @@ -2,7 +2,7 @@ name = "launchdarkly-ai-server" version = "0.1.3" requires-python = ">=3.12" -dependencies = ["opentelemetry-api>=1.25"] +dependencies = ["opentelemetry-api>=1.25", "pydantic>=2"] description = "LaunchDarkly AI SDK core client for Python" readme = "README.md" license = "Apache-2.0" diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/judges.py b/packages/client/src/launchdarkly_ai_server/evaluations/judges.py new file mode 100644 index 0000000..203e6e7 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/evaluations/judges.py @@ -0,0 +1,412 @@ +from __future__ import annotations + +import math +from collections.abc import Awaitable, Callable, Mapping, Sequence +from typing import Any, Literal, Protocol + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from ..judges import _FORMATTING_INSTRUCTIONS +from ..lifecycle import extract_variation, init_client +from ..types import AiConfigRep, LDContext, ProviderHandler, VariationMeta +from ..utils import ( + collapse_messages_to_instructions, + normalize_mode, + parse_json_with_possible_fences, + parse_template, + parse_usage, +) +from .api import EvaluationsError + + +class JudgeReference(BaseModel): + """A reference to a LaunchDarkly judge config and its evaluation thresholds.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + key: str + threshold: float = Field(default=0.5, ge=0.0, le=1.0) + pass_rate_threshold: float = Field(default=1.0, ge=0.0, le=1.0) + ground_truth_context: str | None = None + + @field_validator("key") + @classmethod + def _key_must_not_be_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("judge key must not be blank") + return value + + def to_criterion(self) -> dict[str, Any]: + """Build the existing evaluation criteria wire representation.""" + options: dict[str, Any] = { + "threshold": self.threshold, + "passRateThreshold": self.pass_rate_threshold, + } + if self.ground_truth_context is not None: + options["groundTruthContext"] = self.ground_truth_context + return {"criterionType": self.key, "options": options} + + +class Judge(JudgeReference): + """A reference to any customer or LaunchDarkly judge config.""" + + +class Accuracy(JudgeReference): + key: Literal["$ld:ai:judge:accuracy"] = "$ld:ai:judge:accuracy" + + +class AnswerRelevancy(JudgeReference): + key: Literal["$ld:ai:judge:relevance"] = "$ld:ai:judge:relevance" + + +class Likeness(JudgeReference): + key: Literal["$ld:ai:judge:likeness"] = "$ld:ai:judge:likeness" + ground_truth_context: str | None = "{{expected_output}}" + + +class Bias(JudgeReference): + key: Literal["$ld:ai:judge:bias"] = "$ld:ai:judge:bias" + threshold: float = Field(default=0.3, ge=0.0, le=1.0) + + +class Toxicity(JudgeReference): + key: Literal["$ld:ai:judge:toxicity"] = "$ld:ai:judge:toxicity" + + +class Misinformation(JudgeReference): + key: Literal["$ld:ai:judge:misinformation"] = "$ld:ai:judge:misinformation" + threshold: float = Field(default=0.3, ge=0.0, le=1.0) + ground_truth_context: str | None = "{{expected_output}}" + + +class JudgeIdentity(BaseModel): + """Pinned judge identity retained for later evaluation-results ingest.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + key: str + variation_key: str + version: int + provider: str + model: str + mode: Literal["agent", "messages"] + is_inverted: bool = False + + +class JudgeUsage(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + input: int = 0 + output: int = 0 + total: int = 0 + + +JudgeErrorCode = Literal[ + "rate_limit_exhausted", "judge_timeout", "judge_parse_error", "judge_error" +] + + +class JudgeEvaluationError(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + code: JudgeErrorCode + message: str + + +class JudgeEvaluationResult(BaseModel): + """One offline score, including its row and pinned judge identity.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + row_index: int + judge: JudgeIdentity + status: Literal["complete", "error"] + score: float | None = Field(default=None, ge=0.0, le=1.0) + reasoning: str | None = None + usage: JudgeUsage = Field(default_factory=JudgeUsage) + error: JudgeEvaluationError | None = None + + +class EvaluationMethod(Protocol): + """Seam for evaluating one generation with its stable dataset-row context.""" + + async def evaluate( + self, + generation_output: str, + *, + row_index: int, + rendered_input: str | None, + expected_output: str | None, + variables: Mapping[str, Any], + metadata: Mapping[str, Any] | None, + ) -> JudgeEvaluationResult: ... + + +JudgeVariationResolver = Callable[[str, LDContext], Awaitable[dict[str, Any]]] +JudgeClientInitializer = Callable[[dict[str, Any]], Awaitable[Any]] + + +def _required_mapping(value: Any, description: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise EvaluationsError(f"Resolved judge has invalid {description}") + return value + + +def _required_non_blank_string(value: Any, description: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise EvaluationsError(f"Resolved judge has no {description}") + return value + + +def _select_handler( + handlers: Sequence[ProviderHandler], provider: str, mode: str +) -> tuple[ProviderHandler, bool] | None: + exact = next( + (handler for handler in handlers if handler.provides_for == (provider, mode)), + None, + ) + wildcard = next( + (handler for handler in handlers if handler.provides_for == ("*", mode)), + None, + ) + selected = exact if exact is not None else wildcard + if selected is not None: + return selected, False + + if mode == "messages": + exact_agent = next( + ( + handler + for handler in handlers + if handler.provides_for == (provider, "agent") + ), + None, + ) + wildcard_agent = next( + (handler for handler in handlers if handler.provides_for == ("*", "agent")), + None, + ) + selected_agent = exact_agent if exact_agent is not None else wildcard_agent + if selected_agent is not None: + return selected_agent, True + return None + + +def _usage(raw: Any) -> JudgeUsage: + normalized = parse_usage(dict(raw) if isinstance(raw, Mapping) else {}) + return JudgeUsage( + input=normalized["input"], + output=normalized["output"], + total=normalized["total"], + ) + + +def _error_code(error: Exception) -> JudgeErrorCode: + if isinstance(error, TimeoutError): + return "judge_timeout" + if ( + getattr(error, "status", None) == 429 + or getattr(error, "status_code", None) == 429 + ): + return "rate_limit_exhausted" + return "judge_error" + + +class LaunchDarklyJudgeEvaluation: + """Resolved, metric-free evaluation method backed by one LD judge config.""" + + def __init__( + self, + *, + reference: JudgeReference, + config: AiConfigRep, + identity: JudgeIdentity, + handler: ProviderHandler, + collapse_messages: bool, + ) -> None: + self.reference = reference + self.identity = identity + self._config = ( + collapse_messages_to_instructions(config) if collapse_messages else config + ) + self._handler = handler + + async def evaluate( + self, + generation_output: str, + *, + row_index: int, + rendered_input: str | None, + expected_output: str | None, + variables: Mapping[str, Any], + metadata: Mapping[str, Any] | None, + ) -> JudgeEvaluationResult: + """Evaluate a generation without emitting online evaluation metrics.""" + stable_variables: dict[str, Any] = { + **variables, + "row_index": row_index, + "input": rendered_input, + "expected_output": expected_output, + "metadata": dict(metadata) if metadata is not None else None, + "response_to_evaluate": generation_output, + } + history_parts = [rendered_input, generation_output, _FORMATTING_INSTRUCTIONS] + stable_variables["message_history"] = "\n\n".join( + part for part in history_parts if part + ) + if self.reference.ground_truth_context is not None: + stable_variables["ground_truth_context"] = parse_template( + self.reference.ground_truth_context, stable_variables + ) + + try: + response = await self._handler( + self._config, + generation_output, + None, + stable_variables, + None, + ) + if not isinstance(response, Mapping): + raise TypeError("judge handler result must be a mapping") + usage = _usage(response.get("usage")) + output = response.get("output") + parsed = parse_json_with_possible_fences( + output if isinstance(output, str) else str(output or "") + ) + if not isinstance(parsed, Mapping): + return self._parse_error( + row_index, usage, "Judge returned invalid JSON" + ) + score = parsed.get("score") + reasoning = parsed.get("reasoning") + if ( + isinstance(score, bool) + or not isinstance(score, int | float) + or not math.isfinite(float(score)) + or not 0.0 <= float(score) <= 1.0 + or not isinstance(reasoning, str) + ): + return self._parse_error( + row_index, + usage, + "Judge response must contain a score from 0 to 1 and string reasoning", + ) + return JudgeEvaluationResult( + row_index=row_index, + judge=self.identity, + status="complete", + score=float(score), + reasoning=reasoning, + usage=usage, + ) + except Exception as error: + return JudgeEvaluationResult( + row_index=row_index, + judge=self.identity, + status="error", + error=JudgeEvaluationError( + code=_error_code(error), message=f"Judge invocation failed: {error}" + ), + ) + + def _parse_error( + self, row_index: int, usage: JudgeUsage, message: str + ) -> JudgeEvaluationResult: + return JudgeEvaluationResult( + row_index=row_index, + judge=self.identity, + status="error", + usage=usage, + error=JudgeEvaluationError(code="judge_parse_error", message=message), + ) + + +async def resolve_launchdarkly_judges( + references: Sequence[JudgeReference], + handlers: Sequence[ProviderHandler], + *, + sdk_key: str | None, + context: LDContext | None = None, + resolver: JudgeVariationResolver = extract_variation, + initialize_client: JudgeClientInitializer = init_client, +) -> list[LaunchDarklyJudgeEvaluation]: + """Resolve all judges before evaluation/run records are created. + + The integration layer should call this during preflight. Missing credentials, + unknown/disabled judge keys, invalid variation metadata, and incompatible + handlers are hard failures, so no partially configured offline run is started. + """ + if any(not isinstance(reference, JudgeReference) for reference in references): + raise EvaluationsError( + "judges must contain typed JudgeReference objects, not strings or mappings" + ) + if not references: + return [] + if not sdk_key or not sdk_key.strip(): + raise EvaluationsError( + "LaunchDarkly judging requires an SDK key. Set LD_SDK_KEY or pass " + "sdk_key to init_evaluations()." + ) + + await initialize_client({"sdkKey": sdk_key}) + resolution_context = context or { + "kind": "user", + "key": "offline-evaluation-judge-resolution", + } + evaluations: list[LaunchDarklyJudgeEvaluation] = [] + for reference in references: + try: + variation = await resolver(reference.key, resolution_context) + except Exception as error: + raise EvaluationsError( + f"LaunchDarkly judge {reference.key!r} was not found or is unavailable; " + "create or enable it in the LaunchDarkly UI before starting the run" + ) from error + + config = dict(_required_mapping(variation.get("config"), "config")) + meta: VariationMeta = dict( + _required_mapping(variation.get("meta"), "variation metadata") + ) + provider = _required_non_blank_string( + _required_mapping(config.get("provider"), "provider").get("name"), + "provider name", + ) + model = _required_non_blank_string( + _required_mapping(config.get("model"), "model").get("name"), + "model name", + ) + variation_key = _required_non_blank_string( + meta.get("variationKey"), "variation key" + ) + version = meta.get("version") + if isinstance(version, bool) or not isinstance(version, int): + raise EvaluationsError( + f"Resolved judge {reference.key!r} has no integer version" + ) + mode = normalize_mode(meta.get("mode")) + selected = _select_handler(handlers, provider, mode) + if selected is None: + raise EvaluationsError( + f"No handler can execute LaunchDarkly judge {reference.key!r} " + f"for provider {provider!r} in {mode!r} mode" + ) + handler, collapse_messages = selected + evaluations.append( + LaunchDarklyJudgeEvaluation( + reference=reference, + config=config, + identity=JudgeIdentity( + key=reference.key, + variation_key=variation_key, + version=version, + provider=provider, + model=model, + mode=mode, + is_inverted=bool(config.get("isInverted", False)), + ), + handler=handler, + collapse_messages=collapse_messages, + ) + ) + return evaluations diff --git a/packages/client/tests/test_evaluation_judges.py b/packages/client/tests/test_evaluation_judges.py new file mode 100644 index 0000000..7092ca2 --- /dev/null +++ b/packages/client/tests/test_evaluation_judges.py @@ -0,0 +1,386 @@ +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence +from typing import Any, Literal, cast +from unittest.mock import AsyncMock + +import pytest +from pydantic import ValidationError + +from launchdarkly_ai_server.evaluations.api import EvaluationsError +from launchdarkly_ai_server.evaluations.judges import ( + Accuracy, + AnswerRelevancy, + Bias, + Judge, + JudgeReference, + Likeness, + Misinformation, + Toxicity, + resolve_launchdarkly_judges, +) +from launchdarkly_ai_server.types import ProviderHandler +from launchdarkly_ai_server.utils import create_handler + + +def judge_variation( + *, + key: str = "served-variation", + version: int = 12, + provider: str = "OpenAI", + mode: str = "messages", + inverted: bool = False, +) -> dict[str, Any]: + return { + "config": { + "provider": {"name": provider}, + "model": {"name": "judge-model"}, + "instructions": "Evaluate {{response_to_evaluate}}", + "isInverted": inverted, + "evaluationMetricKey": "must-not-be-emitted-offline", + }, + "meta": { + "enabled": True, + "variationKey": key, + "version": version, + "mode": mode, + }, + } + + +def handler( + fn: Any, + *, + provider: str = "OpenAI", + mode: Literal["agent", "messages"] = "messages", +) -> ProviderHandler: + return create_handler((provider, mode), fn) + + +def test_judge_reference_defaults_and_criteria_wire_shape() -> None: + assert Accuracy().to_criterion() == { + "criterionType": "$ld:ai:judge:accuracy", + "options": {"threshold": 0.5, "passRateThreshold": 1.0}, + } + assert AnswerRelevancy().key == "$ld:ai:judge:relevance" + assert Toxicity().threshold == 0.5 + assert Bias().threshold == 0.3 + assert Likeness().ground_truth_context == "{{expected_output}}" + assert Misinformation().to_criterion()["options"] == { + "threshold": 0.3, + "passRateThreshold": 1.0, + "groundTruthContext": "{{expected_output}}", + } + + +def test_judge_references_forbid_typos_and_invalid_thresholds() -> None: + with pytest.raises(ValidationError, match="extra_forbidden"): + Accuracy(threshhold=0.7) # type: ignore[call-arg] + with pytest.raises(ValidationError, match="less_than_equal"): + Judge(key="security", threshold=1.1) + with pytest.raises(ValidationError, match="judge key must not be blank"): + Judge(key=" ") + with pytest.raises(ValidationError, match="literal_error"): + Accuracy(key="different") # type: ignore[arg-type] + + +@pytest.mark.asyncio +async def test_missing_sdk_key_fails_before_initialization_or_resolution() -> None: + initialize = AsyncMock() + resolver = AsyncMock(return_value=judge_variation()) + + with pytest.raises(EvaluationsError, match="LD_SDK_KEY"): + await resolve_launchdarkly_judges( + [Accuracy()], + [], + sdk_key=" ", + resolver=resolver, + initialize_client=initialize, + ) + + initialize.assert_not_awaited() + resolver.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_only_typed_judge_references_are_accepted() -> None: + raw_references = cast(Sequence[JudgeReference], ["security-judge"]) + + with pytest.raises(EvaluationsError, match="typed JudgeReference"): + await resolve_launchdarkly_judges( + raw_references, + [], + sdk_key="sdk-key", + initialize_client=AsyncMock(), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("reference", "expected_key"), + [ + (Judge(key="security-judge"), "security-judge"), + (Accuracy(), "$ld:ai:judge:accuracy"), + ], +) +async def test_unknown_judge_fails_clearly_during_preflight( + reference: JudgeReference, expected_key: str +) -> None: + async def missing(key: str, context: dict[str, Any]) -> dict[str, Any]: + del key, context + raise RuntimeError("variation returned None") + + with pytest.raises( + EvaluationsError, match=rf"{re.escape(expected_key)}.*LaunchDarkly UI" + ): + await resolve_launchdarkly_judges( + [reference], + [], + sdk_key="sdk-key", + resolver=missing, + initialize_client=AsyncMock(), + ) + + +@pytest.mark.asyncio +async def test_resolved_evaluation_preserves_context_score_and_judge_identity() -> None: + received: dict[str, Any] = {} + + async def judge_handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Any] | None, + variables: dict[str, Any] | None, + history: list[dict[str, Any]] | None, + ) -> dict[str, Any]: + received.update( + config=config, + user_input=user_input, + tool_handlers=tool_handlers, + variables=variables, + history=history, + ) + return { + "output": '```json\n{"score": 0.82, "reasoning": "matches"}\n```', + "usage": {"input_tokens": 31, "output_tokens": 7}, + } + + resolver = AsyncMock( + return_value=judge_variation(key="variation-abc", version=27, inverted=True) + ) + methods = await resolve_launchdarkly_judges( + [ + Judge( + key="security-judge", + threshold=0.7, + ground_truth_context="Known: {{expected_output}} / {{account}}", + ) + ], + [handler(judge_handler)], + sdk_key="sdk-key", + resolver=resolver, + initialize_client=AsyncMock(), + ) + + result = await methods[0].evaluate( + "generated answer", + row_index=41, + rendered_input="Where is order A19?", + expected_output="Order A19 shipped", + variables={"account": "enterprise", "input": "unrendered"}, + metadata={"suite": "orders", "case_id": "stable-41"}, + ) + + resolver.assert_awaited_once_with( + "security-judge", + {"kind": "user", "key": "offline-evaluation-judge-resolution"}, + ) + assert result.status == "complete" + assert result.row_index == 41 + assert result.score == 0.82 + assert result.reasoning == "matches" + assert result.usage.model_dump() == {"input": 31, "output": 7, "total": 38} + assert result.judge.model_dump() == { + "key": "security-judge", + "variation_key": "variation-abc", + "version": 27, + "provider": "OpenAI", + "model": "judge-model", + "mode": "messages", + "is_inverted": True, + } + + assert received["user_input"] == "generated answer" + assert received["tool_handlers"] is None + assert received["history"] is None + judge_variables = cast(Mapping[str, Any], received["variables"]) + assert judge_variables["row_index"] == 41 + assert judge_variables["input"] == "Where is order A19?" + assert judge_variables["expected_output"] == "Order A19 shipped" + assert judge_variables["account"] == "enterprise" + assert judge_variables["metadata"] == { + "suite": "orders", + "case_id": "stable-41", + } + assert judge_variables["response_to_evaluate"] == "generated answer" + assert judge_variables["ground_truth_context"] == ( + "Known: Order A19 shipped / enterprise" + ) + assert "Where is order A19?" in judge_variables["message_history"] + assert "generated answer" in judge_variables["message_history"] + + +@pytest.mark.asyncio +async def test_offline_evaluation_never_emits_online_metric_events() -> None: + tracked: list[tuple[Any, ...]] = [] + + class FakeClient: + def track(self, *args: Any) -> None: + tracked.append(args) + + client = FakeClient() + + async def initialize(options: dict[str, Any]) -> FakeClient: + assert options == {"sdkKey": "sdk-key"} + return client + + async def judge_handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Any] | None, + variables: dict[str, Any] | None, + history: list[dict[str, Any]] | None, + ) -> dict[str, Any]: + del config, user_input, tool_handlers, variables, history + return {"output": '{"score": 1, "reasoning": "safe"}', "usage": {}} + + methods = await resolve_launchdarkly_judges( + [Judge(key="security-judge")], + [handler(judge_handler)], + sdk_key="sdk-key", + resolver=AsyncMock(return_value=judge_variation()), + initialize_client=initialize, + ) + result = await methods[0].evaluate( + "answer", + row_index=0, + rendered_input="question", + expected_output=None, + variables={}, + metadata=None, + ) + + assert result.status == "complete" + assert tracked == [] + + +@pytest.mark.asyncio +async def test_unparseable_judge_response_becomes_diagnosable_error_result() -> None: + async def judge_handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Any] | None, + variables: dict[str, Any] | None, + history: list[dict[str, Any]] | None, + ) -> dict[str, Any]: + del config, user_input, tool_handlers, variables, history + return { + "output": "not json", + "usage": {"inputTokens": 4, "outputTokens": 2}, + } + + methods = await resolve_launchdarkly_judges( + [Accuracy()], + [handler(judge_handler)], + sdk_key="sdk-key", + resolver=AsyncMock(return_value=judge_variation()), + initialize_client=AsyncMock(), + ) + result = await methods[0].evaluate( + "answer", + row_index=3, + rendered_input="question", + expected_output="expected", + variables={}, + metadata=None, + ) + + assert result.status == "error" + assert result.score is None + assert result.reasoning is None + assert result.usage.total == 6 + assert result.error is not None + assert result.error.code == "judge_parse_error" + assert "invalid JSON" in result.error.message + + +@pytest.mark.asyncio +async def test_messages_judge_uses_agent_fallback_with_collapsed_prompt() -> None: + received_configs: list[dict[str, Any]] = [] + + async def agent_handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Any] | None, + variables: dict[str, Any] | None, + history: list[dict[str, Any]] | None, + ) -> dict[str, Any]: + del user_input, tool_handlers, variables, history + received_configs.append(config) + return {"output": '{"score": 0.5, "reasoning": "ok"}', "usage": {}} + + variation = judge_variation() + variation["config"].pop("instructions") + variation["config"]["messages"] = [ + {"role": "system", "content": "Apply the rubric."}, + {"role": "user", "content": "Score the response."}, + ] + methods = await resolve_launchdarkly_judges( + [Accuracy()], + [handler(agent_handler, mode="agent")], + sdk_key="sdk-key", + resolver=AsyncMock(return_value=variation), + initialize_client=AsyncMock(), + ) + + await methods[0].evaluate( + "answer", + row_index=1, + rendered_input="question", + expected_output=None, + variables={}, + metadata=None, + ) + + assert received_configs == [ + { + **variation["config"], + "instructions": "Apply the rubric.\n\nScore the response.", + "messages": [], + } + ] + + +@pytest.mark.asyncio +async def test_missing_compatible_handler_fails_during_resolution() -> None: + async def unused_handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Any] | None, + variables: dict[str, Any] | None, + history: list[dict[str, Any]] | None, + ) -> dict[str, Any]: + del config, user_input, tool_handlers, variables, history + return {} + + with pytest.raises(EvaluationsError, match=r"No handler.*Anthropic"): + await resolve_launchdarkly_judges( + [Accuracy()], + [handler(unused_handler, provider="OpenAI")], + sdk_key="sdk-key", + resolver=AsyncMock( + return_value=judge_variation(provider="Anthropic", mode="messages") + ), + initialize_client=AsyncMock(), + ) diff --git a/uv.lock b/uv.lock index 7d93a3c..fa21dbc 100644 --- a/uv.lock +++ b/uv.lock @@ -922,6 +922,7 @@ version = "0.1.3" source = { editable = "packages/client" } dependencies = [ { name = "opentelemetry-api" }, + { name = "pydantic" }, ] [package.optional-dependencies] @@ -935,6 +936,7 @@ requires-dist = [ { name = "opentelemetry-api", specifier = ">=1.25" }, { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otel'", specifier = ">=1.25" }, { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.25" }, + { name = "pydantic", specifier = ">=2" }, ] provides-extras = ["otel"] From 1f0f19b622ba52a64b961b2e7c7f6bdf99e74644 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Thu, 20 Aug 2026 22:03:36 -0700 Subject: [PATCH 4/6] feat: wire offline judges into evaluation runs --- packages/ai/README.md | 9 +- packages/client/README.md | 17 +- packages/client/agents.md | 4 +- .../src/launchdarkly_ai_server/__init__.py | 40 +++ .../evaluations/__init__.py | 38 +++ .../evaluations/module.py | 75 +++++- .../evaluations/runner.py | 87 ++++-- .../evaluations/types.py | 11 +- packages/client/tests/test_evaluations_run.py | 250 +++++++++++++++++- 9 files changed, 499 insertions(+), 32 deletions(-) diff --git a/packages/ai/README.md b/packages/ai/README.md index e1539fd..140a1a8 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -57,7 +57,7 @@ Never raises. Returns `{"enabled": bool, "config": dict | None, "meta": dict | N `init_evaluations` and the evaluations result types are also re-exported: ```python -from launchdarkly_ai_python import init_evaluations +from launchdarkly_ai_python import Accuracy, Scorer, init_evaluations evals = init_evaluations() result = await evals.run( @@ -66,10 +66,15 @@ result = await evals.run( dataset="golden-dataset", handler=my_handler, generation={"provider": "OpenAI", "model": "gpt-4o"}, + judges=[ + Accuracy(), + Scorer(name="exact-match", fn=lambda row, output: output == row.expected_output), + ], ) +print(result.evaluation_results) ``` -`LD_API_TOKEN` is required. Use `LD_API_BASE_URI` for staging or local management API traffic; it is separate from the SDK delivery setting `LD_BASE_URI`. See the [core evaluations guide](../client/README.md#run-an-evaluation-from-code). +`LD_API_TOKEN` is required, and LaunchDarkly judges also require `LD_SDK_KEY`; deterministic `Scorer` values do not. Use `LD_API_BASE_URI` for staging or local management API traffic; it is separate from the SDK delivery setting `LD_BASE_URI`. See the [core evaluations guide](../client/README.md#run-an-evaluation-from-code). --- diff --git a/packages/client/README.md b/packages/client/README.md index 0aecb10..d2cfe50 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -44,18 +44,18 @@ No code changes are required — `init_client()` detects the packages at runtime ### Run an evaluation from code -The generation-only evaluations harness reads an LD-hosted dataset, creates a new evaluation and client-source run, invokes your handler once per row, uploads the generations, and returns LaunchDarkly's stored verdict. Evaluation keys must be unique because every call creates a new evaluation with `POST`. +The evaluations harness reads an LD-hosted dataset, creates a new evaluation and client-source run, invokes your handler once per row, optionally runs typed LaunchDarkly judges and deterministic scorers in the same worker, uploads the generations, and returns LaunchDarkly's stored verdict. Evaluation keys must be unique because every call creates a new evaluation with `POST`. ```python import asyncio import sys from launchdarkly_ai_openai_messages import create_openai_messages_handler -from launchdarkly_ai_server import init_evaluations +from launchdarkly_ai_server import Accuracy, Judge, Scorer, init_evaluations async def main() -> int: - evals = init_evaluations() # LD_API_TOKEN required; LD_SDK_KEY optional + evals = init_evaluations() # LD_API_TOKEN + LD_SDK_KEY for LD judges result = await evals.run( project_key="my-project", key="support-qa-2026-08-20", @@ -66,8 +66,17 @@ async def main() -> int: "model": "gpt-4o", "instructions": "You are a support agent.", }, + judges=[ + Accuracy(), + Judge(key="security-judge", threshold=0.7), + Scorer( + name="exact-match", + fn=lambda row, output: output == row.expected_output, + ), + ], ) print(result.url, result.summary) + print(result.evaluation_results) return 0 if result.passed else 1 @@ -76,6 +85,8 @@ sys.exit(asyncio.run(main())) `project_key` is supplied per run rather than during initialization. `generation.instructions` is shorthand for one system message; use `generation.messages` instead for a full message list, but do not supply both. The harness never retries a handler invocation because doing so could repeat tool side effects. Its retries apply only to LaunchDarkly management API requests. +`judges` accepts only typed `JudgeReference` values (`Accuracy`, `AnswerRelevancy`, `Likeness`, `Bias`, `Toxicity`, `Misinformation`, or `Judge`) and `Scorer` values. LaunchDarkly judges require `LD_SDK_KEY` and a generation handler built with `create_handler()` so the resolved judge model can be routed safely. Scorers may be synchronous or asynchronous and receive `(row, generation_output)`; `row` includes the rendered row index, input, expected output, variables, and metadata. Results are returned in `EvalRunResult.evaluation_results`. This judging foundation does not yet submit those local scores to LaunchDarkly's evaluation-results endpoint, so `result.passed` remains the stored generation-run verdict. + The client uses **lazy initialization**: importing the package does not connect to LaunchDarkly. The singleton is created automatically on the first API call that needs it (`config().invoke()`, `graph().invoke()`, `resolve_graph()`, etc.), as long as `LD_SDK_KEY` is set in the environment. Call `init_client()` explicitly when you want to: diff --git a/packages/client/agents.md b/packages/client/agents.md index d2ca04f..3f48a46 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -126,9 +126,9 @@ Handlers may return any of these — the client normalizes them before emitting ## SDK-run evaluations -`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK flag delivery and may point at a relay proxy. `LD_SDK_KEY` is optional for generation-only runs and enables the normal handler observability path. +`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK flag delivery and may point at a relay proxy. `LD_SDK_KEY` is optional for generation-only or deterministic-scorer runs, and required when `judges` contains a LaunchDarkly judge reference. -`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST`, so its key must be unique. The harness directly invokes the supplied handler once per row, never retries it, batches generation ingest, and trusts only the server's stored verdict. +`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST`, so its key must be unique. The harness directly invokes the supplied handler once per row, never retries it, then runs typed `JudgeReference` / `Scorer` values with the generated output and full rendered row context. Generation results are batch-ingested; local evaluation outcomes are returned in `EvalRunResult.evaluation_results`, while `passed` remains the server's stored generation-run verdict until evaluation-results ingest lands. ## OTel Setup diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index e3856fe..233357f 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -17,12 +17,32 @@ to_semconv_finish_reason, ) from .evaluations import ( + Accuracy, + AnswerRelevancy, + Bias, EvalRunResult, + EvaluationMethod, EvaluationsError, EvaluationsModule, GenerationConfig, + Judge, + JudgeEvaluationError, + JudgeEvaluationResult, + JudgeIdentity, + JudgeReference, + JudgeUsage, + LaunchDarklyJudgeEvaluation, + Likeness, + Misinformation, RunSummary, + Scorer, + ScorerError, + ScorerResult, + ScorerRow, + ScoreValue, + Toxicity, init_evaluations, + resolve_launchdarkly_judges, ) from .graph import GraphInstance, graph, resolve_graph from .judges import build_judge_tasks, run_judge, run_judges @@ -159,12 +179,32 @@ "to_semconv_finish_reason", "VariationMeta", # evaluations + "Accuracy", + "AnswerRelevancy", + "Bias", "EvalRunResult", + "EvaluationMethod", "EvaluationsError", "EvaluationsModule", "GenerationConfig", + "Judge", + "JudgeEvaluationError", + "JudgeEvaluationResult", + "JudgeIdentity", + "JudgeReference", + "JudgeUsage", + "LaunchDarklyJudgeEvaluation", + "Likeness", + "Misinformation", "RunSummary", + "ScoreValue", + "Scorer", + "ScorerError", + "ScorerResult", + "ScorerRow", + "Toxicity", "init_evaluations", + "resolve_launchdarkly_judges", # utils "create_handler", "make_track_data", diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py index 6516f4a..9f69692 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/__init__.py @@ -9,21 +9,59 @@ Transport, urllib_transport, ) +from .judges import ( + Accuracy, + AnswerRelevancy, + Bias, + EvaluationMethod, + Judge, + JudgeEvaluationError, + JudgeEvaluationResult, + JudgeIdentity, + JudgeReference, + JudgeUsage, + LaunchDarklyJudgeEvaluation, + Likeness, + Misinformation, + Toxicity, + resolve_launchdarkly_judges, +) from .module import EvaluationsModule, init_evaluations +from .scorers import Scorer, ScorerError, ScorerResult, ScorerRow, ScoreValue from .types import EvalRunResult, GenerationConfig, RunSummary, Usage __all__ = [ "DEFAULT_BASE_URI", + "Accuracy", + "AnswerRelevancy", + "Bias", "EvalRunResult", + "EvaluationMethod", "EvaluationsError", "EvaluationsModule", "GenerationConfig", "HttpResponse", + "Judge", + "JudgeEvaluationError", + "JudgeEvaluationResult", + "JudgeIdentity", + "JudgeReference", + "JudgeUsage", "LDApiClient", "LDApiError", + "LaunchDarklyJudgeEvaluation", + "Likeness", + "Misinformation", "RunSummary", + "ScoreValue", + "Scorer", + "ScorerError", + "ScorerResult", + "ScorerRow", + "Toxicity", "Transport", "Usage", "init_evaluations", + "resolve_launchdarkly_judges", "urllib_transport", ] diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 9234099..ea9d927 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -2,9 +2,11 @@ import logging import os -from collections.abc import Mapping +from collections.abc import Mapping, Sequence +from typing import cast from ..lifecycle import init_client +from ..types import ProviderHandler from .api import ( DEFAULT_BASE_URI, EvaluationsError, @@ -13,7 +15,19 @@ urllib_transport, ) from .flags import should_skip_generation_result_ingestion -from .runner import EvalHandler, EvaluationsRunner, ToolImplementation, _segment +from .judges import ( + JudgeReference, + LaunchDarklyJudgeEvaluation, + resolve_launchdarkly_judges, +) +from .runner import ( + EvalHandler, + EvaluationsRunner, + OfflineEvaluation, + ToolImplementation, + _segment, +) +from .scorers import Scorer from .types import EvalRunResult, GenerationConfig logger = logging.getLogger(__name__) @@ -51,13 +65,15 @@ async def run( handler: EvalHandler, generation: GenerationConfig, tools: Mapping[str, ToolImplementation] | None = None, + judges: Sequence[JudgeReference | Scorer] | None = None, concurrency: int = 10, timeout: float = 300.0, ) -> EvalRunResult: """ - Create and run a generation-only evaluation in the caller's process. + Create and run an evaluation in the caller's process. - The returned verdict is computed by LaunchDarkly. A CI script can exit + Typed judges and scorers run after each successful generation. The + returned verdict is computed by LaunchDarkly. A CI script can exit with ``0 if result.passed else 1`` after awaiting this method. """ self._validate_run_args( @@ -70,12 +86,17 @@ async def run( timeout=timeout, ) run_tools = dict(tools or {}) + requested_evaluations = list(judges or []) + self._validate_evaluations(requested_evaluations) skip_generation_result_ingestion = False if self._sdk_key: client = await init_client({"sdkKey": self._sdk_key}) skip_generation_result_ingestion = ( await should_skip_generation_result_ingestion(client, project_key) ) + evaluation_methods = await self._resolve_evaluations( + requested_evaluations, handler + ) # Tool verification is deliberately first: a typo must not create records. resolved_tools = self._runner._resolve_tools(project_key, run_tools) @@ -88,12 +109,13 @@ async def run( project_key, evaluation.id, len(rows), dataset_ref.id ) config = self._runner._build_handler_config(generation, resolved_tools) - results = await self._runner._run_rows( + results, evaluation_results = await self._runner._run_rows( rows, handler, config, run_tools, concurrency, + evaluation_methods, ) self._runner._ingest_results( project_key, @@ -117,8 +139,51 @@ async def run( url=url, run_id=evaluation_run.id, summary=summary, + evaluation_results=evaluation_results, ) + def _validate_evaluations( + self, evaluations: Sequence[JudgeReference | Scorer] + ) -> None: + if any( + not isinstance(evaluation, (JudgeReference, Scorer)) + for evaluation in evaluations + ): + raise EvaluationsError( + "judges must contain typed JudgeReference or Scorer objects" + ) + + async def _resolve_evaluations( + self, + evaluations: Sequence[JudgeReference | Scorer], + generation_handler: EvalHandler, + ) -> list[OfflineEvaluation]: + judge_references = [ + evaluation + for evaluation in evaluations + if isinstance(evaluation, JudgeReference) + ] + resolved_judges: list[LaunchDarklyJudgeEvaluation] = [] + if judge_references: + if not hasattr(generation_handler, "provides_for"): + raise EvaluationsError( + "LaunchDarkly judges require a ProviderHandler created with " + "create_handler()" + ) + resolved_judges = await resolve_launchdarkly_judges( + judge_references, + [cast(ProviderHandler, generation_handler)], + sdk_key=self._sdk_key, + ) + + judge_iterator = iter(resolved_judges) + return [ + next(judge_iterator) + if isinstance(evaluation, JudgeReference) + else evaluation + for evaluation in evaluations + ] + @staticmethod def _validate_run_args( *, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 4006e90..44c2ee8 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -11,6 +11,8 @@ from ..types import NativeTool from ..utils import parse_template from .api import EvaluationsError, LDApiClient, LDApiError +from .judges import JudgeEvaluationResult, LaunchDarklyJudgeEvaluation +from .scorers import Scorer, ScorerResult, ScorerRow from .types import ( DatasetRef, DatasetRow, @@ -27,6 +29,8 @@ EvalHandler = Callable[..., Awaitable[dict[str, Any]]] ToolImplementation = Callable[..., Any] | NativeTool +OfflineEvaluation = LaunchDarklyJudgeEvaluation | Scorer +OfflineEvaluationResult = JudgeEvaluationResult | ScorerResult def _segment(value: str) -> str: @@ -345,10 +349,14 @@ async def _run_rows( config: dict[str, Any], tool_handlers: dict[str, ToolImplementation], concurrency: int, - ) -> list[dict[str, Any]]: + evaluations: list[OfflineEvaluation] | None = None, + ) -> tuple[list[dict[str, Any]], list[OfflineEvaluationResult]]: controller = ConcurrencyController(concurrency) + evaluation_methods = evaluations or [] - async def invoke(row: DatasetRow) -> dict[str, Any]: + async def invoke( + row: DatasetRow, + ) -> tuple[dict[str, Any], list[OfflineEvaluationResult]]: await controller.acquire(config["provider"]["name"]) started = datetime.now(UTC) started_clock = time.perf_counter() @@ -374,26 +382,73 @@ async def invoke(row: DatasetRow) -> dict[str, Any]: usage = result.get("usage") if isinstance(usage, Mapping): payload["output"]["usage"] = dict(usage) + generation_output = result.get("output") + evaluation_results: list[OfflineEvaluationResult] = [] + for evaluation in evaluation_methods: + if isinstance(evaluation, Scorer): + evaluation_results.append( + await evaluation.execute( + ScorerRow( + row_index=row.row_index, + input=row.input, + expected_output=row.expected_output, + variables=row.variables, + metadata=row.metadata, + ), + generation_output + if isinstance(generation_output, str) + else None, + ) + ) + else: + evaluation_results.append( + await evaluation.evaluate( + generation_output + if isinstance(generation_output, str) + else "", + row_index=row.row_index, + rendered_input=row.input, + expected_output=row.expected_output, + variables=row.variables, + metadata=row.metadata, + ) + ) controller.record_success(config["provider"]["name"]) - return payload + return payload, evaluation_results except Exception as error: completed = datetime.now(UTC) - return { - "row_index": row.row_index, - "input": row.input, - "expected_output": row.expected_output, - "variables": row.variables, - "metadata": row.metadata, - "started_at": started.isoformat().replace("+00:00", "Z"), - "generated_at": completed.isoformat().replace("+00:00", "Z"), - "latency_ms": round((time.perf_counter() - started_clock) * 1000), - "status": "ERROR", - "error": {"code": 5001, "message": f"handler raised: {error}"}, - } + return ( + { + "row_index": row.row_index, + "input": row.input, + "expected_output": row.expected_output, + "variables": row.variables, + "metadata": row.metadata, + "started_at": started.isoformat().replace("+00:00", "Z"), + "generated_at": completed.isoformat().replace("+00:00", "Z"), + "latency_ms": round( + (time.perf_counter() - started_clock) * 1000 + ), + "status": "ERROR", + "error": { + "code": 5001, + "message": f"handler raised: {error}", + }, + }, + [], + ) finally: controller.release() - return list(await asyncio.gather(*(invoke(row) for row in rows))) + row_results = await asyncio.gather(*(invoke(row) for row in rows)) + return ( + [generation for generation, _ in row_results], + [ + evaluation + for _, evaluations_for_row in row_results + for evaluation in evaluations_for_row + ], + ) def _ingest_results( self, diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index dda010a..281126e 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -2,7 +2,11 @@ from collections.abc import Mapping from dataclasses import dataclass, field -from typing import Any, TypedDict +from typing import TYPE_CHECKING, Any, TypedDict + +if TYPE_CHECKING: + from .judges import JudgeEvaluationResult + from .scorers import ScorerResult @dataclass @@ -111,9 +115,12 @@ def from_wire(cls, data: Mapping[str, Any] | None) -> RunSummary: @dataclass class EvalRunResult: - """The verdict of an evaluation run, as computed and stored by LaunchDarkly.""" + """The server verdict and local judge/scorer results for an evaluation run.""" passed: bool url: str run_id: str summary: RunSummary + evaluation_results: list[JudgeEvaluationResult | ScorerResult] = field( + default_factory=list + ) diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index e646f46..2f65444 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1,8 +1,8 @@ from __future__ import annotations import json -from collections.abc import Callable -from typing import Any +from collections.abc import Callable, Mapping, Sequence +from typing import Any, cast from unittest.mock import AsyncMock, MagicMock import pytest @@ -10,8 +10,18 @@ from launchdarkly_ai_server.evaluations import ( EvaluationsError, HttpResponse, + Judge, + JudgeEvaluationResult, + JudgeIdentity, + JudgeReference, + LaunchDarklyJudgeEvaluation, + Scorer, + ScorerResult, + ScorerRow, init_evaluations, ) +from launchdarkly_ai_server.types import ProviderHandler +from launchdarkly_ai_server.utils import create_handler class SequencedTransport: @@ -320,6 +330,242 @@ async def handler(*args: object) -> dict[str, Any]: ) +@pytest.mark.asyncio +async def test_run_executes_scorer_with_generation_and_complete_row_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("LD_SDK_KEY", raising=False) + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [ + { + "rowIndex": 8, + "input": "Order {{order_id}}", + "expectedOutput": "Found {{order_id}}", + "variables": {"order_id": "A19"}, + "metadata": {"suite": "orders"}, + } + ], + total=1, + ), + ), + response(201, {"id": "evaluation-id", "name": "eval-key"}), + response( + 201, + { + "id": "run-id", + "evaluationId": "evaluation-id", + "state": "PENDING", + }, + ), + response(202, {}), + response( + 200, + { + "id": "run-id", + "evaluationId": "evaluation-id", + "state": "COMPLETE", + "verdict": "passed", + }, + ), + response(200, {"statusCounts": {"total": 1, "passed": 1}}), + ] + ) + received: dict[str, Any] = {} + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "Found A19"} + + def score(row: ScorerRow, output: str | None) -> bool: + received.update( + row_index=row.row_index, + input=row.input, + expected_output=row.expected_output, + variables=dict(row.variables), + metadata=dict(row.metadata or {}), + output=output, + ) + return True + + result = await init_evaluations(api_token="token", transport=transport).run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + judges=[Scorer(name="exact-match", fn=score)], + ) + + assert received == { + "row_index": 8, + "input": "Order A19", + "expected_output": "Found A19", + "variables": { + "order_id": "A19", + "input": "Order A19", + "expected_output": "Found A19", + }, + "metadata": {"suite": "orders"}, + "output": "Found A19", + } + assert len(result.evaluation_results) == 1 + scorer_result = result.evaluation_results[0] + assert isinstance(scorer_result, ScorerResult) + assert scorer_result.score == 1.0 + + +@pytest.mark.asyncio +async def test_run_resolves_and_executes_launchdarkly_judge( + monkeypatch: pytest.MonkeyPatch, +) -> None: + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [ + { + "rowIndex": 5, + "input": "Question", + "expectedOutput": "Expected", + "variables": {"account": "enterprise"}, + "metadata": {"suite": "judge"}, + } + ], + total=1, + ), + ), + response(201, {"id": "evaluation-id", "name": "eval-key"}), + response( + 201, + { + "id": "run-id", + "evaluationId": "evaluation-id", + "state": "PENDING", + }, + ), + response(202, {}), + response( + 200, + { + "id": "run-id", + "evaluationId": "evaluation-id", + "state": "COMPLETE", + "verdict": "passed", + }, + ), + response(200, {"statusCounts": {"total": 1, "passed": 1}}), + ] + ) + received: dict[str, Any] = {} + + async def generation_handler(*args: object) -> dict[str, Any]: + return {"output": "Generated answer"} + + async def judge_handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]] | None, + variables: dict[str, Any] | None, + history: list[dict[str, Any]] | None, + ) -> dict[str, Any]: + del config, tool_handlers, history + received.update(user_input=user_input, variables=variables) + return {"output": '{"score": 0.9, "reasoning": "good"}'} + + reference = Judge(key="security-judge") + resolved = LaunchDarklyJudgeEvaluation( + reference=reference, + config={ + "provider": {"name": "OpenAI"}, + "model": {"name": "judge-model"}, + "instructions": "Judge the response", + }, + identity=JudgeIdentity( + key="security-judge", + variation_key="variation-key", + version=3, + provider="OpenAI", + model="judge-model", + mode="messages", + ), + handler=create_handler(("OpenAI", "messages"), judge_handler), + collapse_messages=False, + ) + generation = create_handler(("OpenAI", "messages"), generation_handler) + flag_client = MagicMock() + flag_client.variation = AsyncMock(return_value=False) + + async def fake_init_client(options: dict[str, Any]) -> MagicMock: + assert options == {"sdkKey": "sdk-key"} + return flag_client + + async def fake_resolve( + references: Sequence[JudgeReference], + handlers: Sequence[ProviderHandler], + *, + sdk_key: str | None, + ) -> list[LaunchDarklyJudgeEvaluation]: + assert references == [reference] + assert handlers == [generation] + assert sdk_key == "sdk-key" + return [resolved] + + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.init_client", fake_init_client + ) + monkeypatch.setattr( + "launchdarkly_ai_server.evaluations.module.resolve_launchdarkly_judges", + fake_resolve, + ) + + result = await init_evaluations( + api_token="token", sdk_key="sdk-key", transport=transport + ).run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=generation, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + judges=[reference], + ) + + judge_result = result.evaluation_results[0] + assert isinstance(judge_result, JudgeEvaluationResult) + assert judge_result.score == 0.9 + assert received["user_input"] == "Generated answer" + judge_variables = cast(Mapping[str, Any], received["variables"]) + assert judge_variables["row_index"] == 5 + assert judge_variables["input"] == "Question" + assert judge_variables["expected_output"] == "Expected" + assert judge_variables["account"] == "enterprise" + assert judge_variables["metadata"] == {"suite": "judge"} + assert judge_variables["response_to_evaluate"] == "Generated answer" + + +@pytest.mark.asyncio +async def test_run_rejects_untyped_judges_before_network_io() -> None: + transport = SequencedTransport([]) + evals = init_evaluations(api_token="token", transport=transport) + + with pytest.raises(EvaluationsError, match="typed JudgeReference or Scorer"): + await evals.run( + project_key="proj", + key="eval-key", + dataset="golden", + handler=successful_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + judges=["accuracy"], # type: ignore[list-item] + ) + + assert transport.requests == [] + + @pytest.mark.asyncio async def test_run_rejects_instructions_and_messages_before_network_io() -> None: transport = SequencedTransport([]) From c234f04ad7e71c2a333dcf52c8c79dd5fbaa4729 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Thu, 20 Aug 2026 22:16:11 -0700 Subject: [PATCH 5/6] no-mistakes(document): docs: refresh evaluations module description; lint clean --- packages/client/agents.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index 3f48a46..6a6b03e 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,7 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | | `src/launchdarkly_ai_server/registry.py` | `Registry`, `global_registry`, `compose`, `resolve_handlers`, `resolve_tools` | | `src/launchdarkly_ai_server/judges.py` | `run_judges`, `build_judge_tasks`, `run_judge` | -| `src/launchdarkly_ai_server/evaluations/` | `init_evaluations`, the private management API operations, and generation-only `EvaluationsModule.run()` orchestration | +| `src/launchdarkly_ai_server/evaluations/` | `init_evaluations`, the private management API operations, offline judge/scorer resolution, and `EvaluationsModule.run()` orchestration | | `src/launchdarkly_ai_server/__init__.py` | Public barrel — the only surface handler packages import from | --- From e2321085b4913747462ce078cb3fa62a10fed729 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Fri, 21 Aug 2026 10:26:06 -0700 Subject: [PATCH 6/6] fix: gate evaluation batch ingest with dedicated flag --- .../evaluations/flags.py | 20 ++++++++-------- .../evaluations/module.py | 10 ++++---- .../evaluations/runner.py | 4 ++-- .../client/tests/test_evaluation_flags.py | 24 ++++++++++++------- packages/client/tests/test_evaluations_run.py | 6 ++--- 5 files changed, 35 insertions(+), 29 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/flags.py b/packages/client/src/launchdarkly_ai_server/evaluations/flags.py index 4468d88..ee1a138 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/flags.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/flags.py @@ -8,20 +8,20 @@ logger = logging.getLogger(__name__) -ENABLE_TOOL_CALLS_IN_OFFLINE_EVALUATIONS_FLAG_KEY: Final[str] = ( - "enable-tool-calls-in-offline-evaluations" +ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY: Final[str] = ( + "enable-batch-ingest-in-evals-from-code" ) -"""Canonical rollout flag for tool calls in offline evaluations.""" +"""Canonical rollout flag for generation-result batch ingestion.""" -async def should_skip_generation_result_ingestion( +async def is_generation_result_batch_ingest_enabled( client: Any, project_key: str, ) -> bool: - """Return whether the rollout flag selects the no-ingest path. + """Return whether the rollout flag enables generation-result batch ingest. - Flag evaluation is fail-safe: false, malformed, or failed evaluations retain - the existing generation-result ingestion behavior. + Flag evaluation is fail-safe: false, malformed, or failed evaluations disable + the gated batch-ingest path. """ try: context = to_ld_context( @@ -29,7 +29,7 @@ async def should_skip_generation_result_ingestion( {"kind": "project", "key": project_key}, ) result = client.variation( - ENABLE_TOOL_CALLS_IN_OFFLINE_EVALUATIONS_FLAG_KEY, + ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY, context, False, ) @@ -37,8 +37,8 @@ async def should_skip_generation_result_ingestion( return value is True except Exception: logger.warning( - "Unable to evaluate %s; generation results will be ingested", - ENABLE_TOOL_CALLS_IN_OFFLINE_EVALUATIONS_FLAG_KEY, + "Unable to evaluate %s; generation results will not be batch ingested", + ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY, exc_info=True, ) return False diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index ea9d927..f8a1c23 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -14,7 +14,7 @@ Transport, urllib_transport, ) -from .flags import should_skip_generation_result_ingestion +from .flags import is_generation_result_batch_ingest_enabled from .judges import ( JudgeReference, LaunchDarklyJudgeEvaluation, @@ -88,11 +88,11 @@ async def run( run_tools = dict(tools or {}) requested_evaluations = list(judges or []) self._validate_evaluations(requested_evaluations) - skip_generation_result_ingestion = False + batch_ingest_enabled = True if self._sdk_key: client = await init_client({"sdkKey": self._sdk_key}) - skip_generation_result_ingestion = ( - await should_skip_generation_result_ingestion(client, project_key) + batch_ingest_enabled = await is_generation_result_batch_ingest_enabled( + client, project_key ) evaluation_methods = await self._resolve_evaluations( requested_evaluations, handler @@ -122,7 +122,7 @@ async def run( evaluation.id, evaluation_run.id, results, - skip_generation_result_ingestion=skip_generation_result_ingestion, + batch_ingest_enabled=batch_ingest_enabled, ) completed = await self._runner._poll_run( project_key, evaluation.id, evaluation_run.id, timeout diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 44c2ee8..4df791e 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -457,9 +457,9 @@ def _ingest_results( run_id: str, results: list[dict[str, Any]], *, - skip_generation_result_ingestion: bool = False, + batch_ingest_enabled: bool = True, ) -> None: - if skip_generation_result_ingestion: + if not batch_ingest_enabled: return path = ( f"projects/{_segment(project_key)}/evaluations/{_segment(evaluation_id)}" diff --git a/packages/client/tests/test_evaluation_flags.py b/packages/client/tests/test_evaluation_flags.py index fd60538..9113b62 100644 --- a/packages/client/tests/test_evaluation_flags.py +++ b/packages/client/tests/test_evaluation_flags.py @@ -5,35 +5,41 @@ import pytest from launchdarkly_ai_server.evaluations.flags import ( - ENABLE_TOOL_CALLS_IN_OFFLINE_EVALUATIONS_FLAG_KEY, - should_skip_generation_result_ingestion, + ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY, + is_generation_result_batch_ingest_enabled, ) @pytest.mark.asyncio -async def test_enabled_flag_selects_generation_result_ingestion_skip() -> None: +async def test_enabled_flag_enables_generation_result_batch_ingest() -> None: client = MagicMock() client.variation = AsyncMock(return_value=True) - assert await should_skip_generation_result_ingestion(client, "project-key") is True + assert ( + await is_generation_result_batch_ingest_enabled(client, "project-key") is True + ) client.variation.assert_awaited_once_with( - ENABLE_TOOL_CALLS_IN_OFFLINE_EVALUATIONS_FLAG_KEY, + ENABLE_BATCH_INGEST_IN_EVALS_FROM_CODE_FLAG_KEY, {"kind": "project", "key": "project-key"}, False, ) @pytest.mark.asyncio -async def test_disabled_flag_preserves_generation_result_ingestion() -> None: +async def test_disabled_flag_disables_generation_result_batch_ingest() -> None: client = MagicMock() client.variation = AsyncMock(return_value=False) - assert await should_skip_generation_result_ingestion(client, "project-key") is False + assert ( + await is_generation_result_batch_ingest_enabled(client, "project-key") is False + ) @pytest.mark.asyncio -async def test_flag_evaluation_error_preserves_generation_result_ingestion() -> None: +async def test_flag_evaluation_error_disables_generation_result_batch_ingest() -> None: client = MagicMock() client.variation = AsyncMock(side_effect=RuntimeError("delivery unavailable")) - assert await should_skip_generation_result_ingestion(client, "project-key") is False + assert ( + await is_generation_result_batch_ingest_enabled(client, "project-key") is False + ) diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 2f65444..f065d0f 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -267,7 +267,7 @@ async def test_run_calls_private_operations_in_order_and_returns_server_verdict( @pytest.mark.asyncio -async def test_enabled_rollout_flag_skips_generation_result_ingestion( +async def test_disabled_batch_ingest_flag_skips_generation_result_ingestion( monkeypatch: pytest.MonkeyPatch, ) -> None: transport = SequencedTransport( @@ -302,7 +302,7 @@ async def test_enabled_rollout_flag_skips_generation_result_ingestion( ] ) client = MagicMock() - client.variation = AsyncMock(return_value=True) + client.variation = AsyncMock(return_value=False) async def fake_init_client(options: dict[str, Any]) -> MagicMock: assert options == {"sdkKey": "sdk-key"} @@ -499,7 +499,7 @@ async def judge_handler( ) generation = create_handler(("OpenAI", "messages"), generation_handler) flag_client = MagicMock() - flag_client.variation = AsyncMock(return_value=False) + flag_client.variation = AsyncMock(return_value=True) async def fake_init_client(options: dict[str, Any]) -> MagicMock: assert options == {"sdkKey": "sdk-key"}