From 746613e47af6f34e87a9e2b85e9b1b5b1f9ea69f Mon Sep 17 00:00:00 2001 From: hylin Date: Fri, 14 Aug 2026 14:18:07 +0800 Subject: [PATCH 1/2] feat(evaluation): add trajectory improvement metrics --- sforge/harness/judge_server.py | 5 ++ sforge/harness/trajectory_metrics.py | 121 +++++++++++++++++++++++++++ tests/test_trajectory_metrics.py | 109 ++++++++++++++++++++++++ 3 files changed, 235 insertions(+) create mode 100644 sforge/harness/trajectory_metrics.py create mode 100644 tests/test_trajectory_metrics.py diff --git a/sforge/harness/judge_server.py b/sforge/harness/judge_server.py index 0135830..e687878 100644 --- a/sforge/harness/judge_server.py +++ b/sforge/harness/judge_server.py @@ -39,6 +39,7 @@ from sforge.harness.selection import select_best from sforge.harness.benchmark import load_benchmark from sforge.harness.task_spec import TaskSpec, load_all_tasks +from sforge.harness.trajectory_metrics import compute_trajectory_metrics logger = logging.getLogger("sforge.judge_server") @@ -373,6 +374,9 @@ def _record_submission(self, run_id: str, submission_id: str, task_id: str, roun entry["total_tests"] = report_dict.get("total_tests", 0) entry["valid"] = report_dict.get("valid", True) entry["summary"] = report_dict.get("summary") + entry["submitted_at"] = report_dict.get("submitted_at", 0.0) + entry["runtime_seconds"] = report_dict.get("runtime_seconds", 0.0) + entry["timed_out"] = report_dict.get("timed_out", False) if error: entry["error"] = error history_key = f"{run_id}/{task_id}" @@ -410,6 +414,7 @@ def get_run_history(self, run_id: str, task_id: str | None = None) -> dict: "best_round": best["best_round"], "agent_submissions": sum(1 for e in sub_entries if (e.get("round") or "").startswith("agent-")), "auto_submissions": sum(1 for e in sub_entries if (e.get("round") or "").startswith("auto-")), + "trajectory_metrics": compute_trajectory_metrics(entries, direction), "entries": entries, } diff --git a/sforge/harness/trajectory_metrics.py b/sforge/harness/trajectory_metrics.py new file mode 100644 index 0000000..4b7de27 --- /dev/null +++ b/sforge/harness/trajectory_metrics.py @@ -0,0 +1,121 @@ +# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Process metrics for iterative agent evaluation trajectories.""" + +from __future__ import annotations + +import math +from typing import Any + + +def _finite_float(value: Any) -> float | None: + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if math.isfinite(number) else None + + +def _stable_number(value: float) -> float: + return round(value, 12) + + +def compute_trajectory_metrics( + entries: list[dict[str, Any]], + score_direction: str = "maximize", +) -> dict[str, Any]: + """Summarize score improvement across completed, valid submissions.""" + submissions = [ + entry + for entry in entries + if entry.get("type") == "submission" + and entry.get("status") == "completed" + and entry.get("valid", True) + ] + + use_score = any( + _finite_float(entry.get("score")) is not None for entry in submissions + ) + value_field = "score" if use_score else "pass_rate" + direction = score_direction if use_score else "maximize" + + points: list[tuple[float, float | None]] = [] + for entry in submissions: + value = _finite_float(entry.get(value_field)) + if value is None: + continue + points.append((value, _finite_float(entry.get("submitted_at")))) + + if not points: + return { + "value_field": value_field, + "direction": direction, + "submission_count": 0, + "initial_value": None, + "final_value": None, + "best_value": None, + "total_improvement": 0.0, + "improving_submissions": 0, + "non_improving_submissions": 0, + "first_improvement_seconds": None, + "time_to_best_seconds": None, + } + + better = ( + (lambda value, best: value < best) + if direction == "minimize" + else (lambda value, best: value > best) + ) + initial_value = points[0][0] + best_value = initial_value + best_index = 0 + first_improvement_index: int | None = None + improving_submissions = 0 + + for index, (value, _) in enumerate(points[1:], start=1): + if better(value, best_value): + best_value = value + best_index = index + improving_submissions += 1 + if first_improvement_index is None: + first_improvement_index = index + + timestamps = [timestamp for _, timestamp in points if timestamp is not None] + start_time = min(timestamps) if timestamps else None + + def elapsed(index: int | None) -> float | None: + if index is None or start_time is None: + return None + timestamp = points[index][1] + return max(timestamp - start_time, 0.0) if timestamp is not None else None + + total_improvement = ( + initial_value - best_value + if direction == "minimize" + else best_value - initial_value + ) + return { + "value_field": value_field, + "direction": direction, + "submission_count": len(points), + "initial_value": initial_value, + "final_value": points[-1][0], + "best_value": best_value, + "total_improvement": _stable_number(total_improvement), + "improving_submissions": improving_submissions, + "non_improving_submissions": len(points) - improving_submissions - 1, + "first_improvement_seconds": elapsed(first_improvement_index), + "time_to_best_seconds": elapsed(best_index), + } diff --git a/tests/test_trajectory_metrics.py b/tests/test_trajectory_metrics.py new file mode 100644 index 0000000..58b24ed --- /dev/null +++ b/tests/test_trajectory_metrics.py @@ -0,0 +1,109 @@ +# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sforge.harness.trajectory_metrics import compute_trajectory_metrics + + +def submission( + *, + pass_rate=0.0, + score=None, + submitted_at=None, + status="completed", + valid=True, +): + return { + "type": "submission", + "status": status, + "valid": valid, + "pass_rate": pass_rate, + "score": score, + "submitted_at": submitted_at, + } + + +def test_pass_rate_trajectory_reports_improvement_and_timing(): + metrics = compute_trajectory_metrics( + [ + submission(pass_rate=0.2, submitted_at=100), + submission(pass_rate=0.2, submitted_at=130), + submission(pass_rate=0.5, submitted_at=160), + submission(pass_rate=0.4, submitted_at=190), + submission(pass_rate=0.7, submitted_at=220), + ] + ) + + assert metrics == { + "value_field": "pass_rate", + "direction": "maximize", + "submission_count": 5, + "initial_value": 0.2, + "final_value": 0.7, + "best_value": 0.7, + "total_improvement": 0.5, + "improving_submissions": 2, + "non_improving_submissions": 2, + "first_improvement_seconds": 60.0, + "time_to_best_seconds": 120.0, + } + + +def test_score_trajectory_respects_minimize_direction(): + metrics = compute_trajectory_metrics( + [ + submission(score=10, submitted_at=20), + submission(score=8, submitted_at=30), + submission(score=9, submitted_at=40), + submission(score=6, submitted_at=50), + ], + score_direction="minimize", + ) + + assert metrics["value_field"] == "score" + assert metrics["direction"] == "minimize" + assert metrics["initial_value"] == 10 + assert metrics["final_value"] == 6 + assert metrics["best_value"] == 6 + assert metrics["total_improvement"] == 4 + assert metrics["improving_submissions"] == 2 + assert metrics["non_improving_submissions"] == 1 + assert metrics["first_improvement_seconds"] == 10 + assert metrics["time_to_best_seconds"] == 30 + + +def test_invalid_and_failed_submissions_are_excluded(): + metrics = compute_trajectory_metrics( + [ + submission(pass_rate=0.1, submitted_at=10), + submission(pass_rate=1.0, submitted_at=20, valid=False), + submission(pass_rate=1.0, submitted_at=30, status="error"), + ] + ) + + assert metrics["submission_count"] == 1 + assert metrics["best_value"] == 0.1 + assert metrics["total_improvement"] == 0.0 + assert metrics["improving_submissions"] == 0 + assert metrics["non_improving_submissions"] == 0 + assert metrics["first_improvement_seconds"] is None + assert metrics["time_to_best_seconds"] == 0 + + +def test_empty_trajectory_returns_stable_shape(): + metrics = compute_trajectory_metrics([]) + + assert metrics["submission_count"] == 0 + assert metrics["initial_value"] is None + assert metrics["best_value"] is None + assert metrics["first_improvement_seconds"] is None From 16d2edbbd5315e747370b413705586f35874c90d Mon Sep 17 00:00:00 2001 From: hylin Date: Sun, 16 Aug 2026 23:01:41 +0800 Subject: [PATCH 2/2] fix(evaluation): harden trajectory metrics --- sforge/harness/judge_server.py | 1 + sforge/harness/trajectory_metrics.py | 22 +++++-- tests/test_trajectory_metrics.py | 95 ++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 6 deletions(-) diff --git a/sforge/harness/judge_server.py b/sforge/harness/judge_server.py index e687878..99b6e63 100644 --- a/sforge/harness/judge_server.py +++ b/sforge/harness/judge_server.py @@ -854,6 +854,7 @@ def history(token: str = Query(...), admin_secret: str = Query("")) -> dict: "best_round": best["best_round"], "agent_submissions": sum(1 for e in agent_entries if e.get("type") == "submission"), "auto_submissions": 0, + "trajectory_metrics": compute_trajectory_metrics(agent_entries, direction), "entries": agent_entries, } diff --git a/sforge/harness/trajectory_metrics.py b/sforge/harness/trajectory_metrics.py index 4b7de27..0dce576 100644 --- a/sforge/harness/trajectory_metrics.py +++ b/sforge/harness/trajectory_metrics.py @@ -28,8 +28,9 @@ def _finite_float(value: Any) -> float | None: return number if math.isfinite(number) else None -def _stable_number(value: float) -> float: - return round(value, 12) +def _stable_difference(left: float, right: float) -> float | None: + difference = left - right + return round(difference, 12) if math.isfinite(difference) else None def compute_trajectory_metrics( @@ -58,6 +59,12 @@ def compute_trajectory_metrics( continue points.append((value, _finite_float(entry.get("submitted_at")))) + # Judge workers may finish out of order. Reports carry the evaluation start + # time, so use it when every point has one; otherwise retain insertion order + # rather than inventing a position for an undated submission. + if points and all(timestamp is not None for _, timestamp in points): + points.sort(key=lambda point: point[1] if point[1] is not None else 0.0) + if not points: return { "value_field": value_field, @@ -99,12 +106,15 @@ def elapsed(index: int | None) -> float | None: if index is None or start_time is None: return None timestamp = points[index][1] - return max(timestamp - start_time, 0.0) if timestamp is not None else None + if timestamp is None: + return None + difference = timestamp - start_time + return max(difference, 0.0) if math.isfinite(difference) else None total_improvement = ( - initial_value - best_value + _stable_difference(initial_value, best_value) if direction == "minimize" - else best_value - initial_value + else _stable_difference(best_value, initial_value) ) return { "value_field": value_field, @@ -113,7 +123,7 @@ def elapsed(index: int | None) -> float | None: "initial_value": initial_value, "final_value": points[-1][0], "best_value": best_value, - "total_improvement": _stable_number(total_improvement), + "total_improvement": total_improvement, "improving_submissions": improving_submissions, "non_improving_submissions": len(points) - improving_submissions - 1, "first_improvement_seconds": elapsed(first_improvement_index), diff --git a/tests/test_trajectory_metrics.py b/tests/test_trajectory_metrics.py index 58b24ed..7407423 100644 --- a/tests/test_trajectory_metrics.py +++ b/tests/test_trajectory_metrics.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +import threading +from types import SimpleNamespace + +from fastapi.testclient import TestClient + +from sforge.harness import judge_server from sforge.harness.trajectory_metrics import compute_trajectory_metrics @@ -107,3 +114,91 @@ def test_empty_trajectory_returns_stable_shape(): assert metrics["initial_value"] is None assert metrics["best_value"] is None assert metrics["first_improvement_seconds"] is None + + +def test_trajectory_is_ordered_by_submission_time(): + metrics = compute_trajectory_metrics( + [ + submission(pass_rate=0.8, submitted_at=20), + submission(pass_rate=0.2, submitted_at=10), + submission(pass_rate=0.5, submitted_at=15), + ] + ) + + assert metrics["initial_value"] == 0.2 + assert metrics["final_value"] == 0.8 + assert metrics["best_value"] == 0.8 + assert metrics["improving_submissions"] == 2 + assert metrics["first_improvement_seconds"] == 5 + assert metrics["time_to_best_seconds"] == 10 + + +def test_non_finite_and_missing_rewards_are_excluded(): + metrics = compute_trajectory_metrics( + [ + submission(score=1, submitted_at=10), + submission(score=float("nan"), submitted_at=20), + submission(score=float("inf"), submitted_at=30), + submission(score=None, submitted_at=40), + submission(score=1, submitted_at=50), + ] + ) + + assert metrics["value_field"] == "score" + assert metrics["submission_count"] == 2 + assert metrics["improving_submissions"] == 0 + assert metrics["non_improving_submissions"] == 1 + json.dumps(metrics, allow_nan=False) + + +def test_overflowed_differences_remain_json_serializable(): + metrics = compute_trajectory_metrics( + [ + submission(pass_rate=-1e308, submitted_at=-1e308), + submission(pass_rate=1e308, submitted_at=1e308), + ] + ) + + assert metrics["total_improvement"] is None + assert metrics["first_improvement_seconds"] is None + assert metrics["time_to_best_seconds"] is None + json.dumps(metrics, allow_nan=False) + + +def test_agent_history_includes_metrics_for_visible_entries(monkeypatch): + def fake_init(self, config): + self.tasks = { + "task": SimpleNamespace( + judge=SimpleNamespace( + selection="pass_rate_first", + score_direction="maximize", + ) + ) + } + self.tokens = {"token": {"run_id": "run", "task_id": "task"}} + self.run_history = { + "run/task": [ + submission(pass_rate=0.2, submitted_at=10) + | {"round": "agent-1", "task_id": "task"}, + submission(pass_rate=0.9, submitted_at=20) + | {"round": "auto-1", "task_id": "task"}, + submission(pass_rate=0.5, submitted_at=30) + | {"round": "agent-2", "task_id": "task"}, + ] + } + self._history_lock = threading.Lock() + self._tokens_lock = threading.Lock() + + monkeypatch.setattr(judge_server.JudgeState, "__init__", fake_init) + monkeypatch.setattr(judge_server.JudgeState, "load_tasks", lambda self: None) + + response = TestClient(judge_server.create_app(object())).get( + "/api/v1/history", + params={"token": "token"}, + ) + + assert response.status_code == 200 + body = response.json() + assert body["trajectory_metrics"]["submission_count"] == 2 + assert body["trajectory_metrics"]["best_value"] == 0.5 + assert body["auto_submissions"] == 0