diff --git a/.gitignore b/.gitignore index ad83136b..5d52c6ee 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,6 @@ refs/ # Derived pyright config for the CE036 contract engine (tests/lint/pyright_config.py) .pyright-tests.json + +# Local evalboard dataset for `next dev` (EVALBOARD_LOCAL_RUNS_DIR) +evalboard/.local-runs/ diff --git a/evalboard/app/_components/col-help.tsx b/evalboard/app/_components/col-help.tsx index 7fb5a812..309424fd 100644 --- a/evalboard/app/_components/col-help.tsx +++ b/evalboard/app/_components/col-help.tsx @@ -1,131 +1,10 @@ -"use client"; - -import { useEffect, useState } from "react"; - -// Shared column-help bubble. A small ⓘ next to a column header opens a static, -// selectable popover explaining what the number is, what drives it up, and how -// to bring it down. Used by the run-page task grid (TaskGrid) and the task-page -// message timeline so the two stay consistent. -export type ColHelp = { - title: string; - body: string; - causes?: string; // common causes of high values - fix?: string; // potential fixes -}; - -// Token-column help shared verbatim across pages. Wording is page-neutral (no -// "this task" / "this call") so it reads correctly on both the per-task grid -// and the per-message timeline. +// Column definitions, surfaced as native `title` tooltips on table headers. +// Shared by the run-page task grid and the task-page message timeline so the two +// describe the same number the same way. Wording is page-neutral (no "this task" +// / "this call") because both surfaces read the same string. export const TOKEN_COLUMN_HELP = { - input: { - title: "Input tokens (uncached)", - body: "Fresh prompt input billed at the full input rate — the slice that was neither written to nor read from the prompt cache (input_tokens minus cache-creation and cache-read).", - causes: "new content added to the prompt that wasn't cacheable yet — the latest user/tool message, anything past the cached prefix.", - fix: "this is usually small and unavoidable; the cache columns (cache-write / cache-read) are where prompt-input cost concentrates.", - }, - output: { - title: "Output tokens", - body: "Text, code, tool arguments and reasoning the model generated.", - causes: "verbose final answers, large file rewrites, heavy reasoning.", - fix: "ask for concise output, scope edits to smaller diffs, cap max_output_tokens / max_turns.", - }, - cw: { - title: "Cache-write tokens", - body: "Context newly written into the prompt cache (cache_creation_input_tokens).", - causes: "the cached prefix keeps changing — new files read mid-run, a growing transcript — so it's re-written instead of reused.", - fix: "keep stable content (system prompt, skills, instructions) at the front of the prompt; don't inject volatile content early; reuse sessions.", - }, - cr: { - title: "Cache-read tokens", - body: "Cached input re-billed on every later call (cache_read_input_tokens). Usually the dominant cost line.", - causes: "large context (big files, long transcript, many skills/tools) replayed on every call.", - fix: "put less in context (smaller file reads, fewer files), shorten the run (fewer turns), trim system/skill payloads, compact long transcripts.", - }, -} satisfies Record; - -export function HelpPopover({ - help, - align, -}: { - help: ColHelp; - align: "left" | "right"; -}) { - return ( -
e.stopPropagation()} - > -
{help.title}
-

{help.body}

- {help.causes && ( -

- - Common causes: - {" "} - {help.causes} -

- )} - {help.fix && ( -

- Reduce by:{" "} - {help.fix} -

- )} -
- ); -} - -// Self-contained ⓘ icon + popover with its own open state (click to toggle, -// click-outside / Escape to close). Drop-in for headers that aren't managed by -// a parent's shared open-state — e.g. the message-timeline header, which is a -// server component. The click-outside check keys off the `[data-col-help]` -// wrapper, so clicking a different icon closes this one (one open at a time). -export function ColHelpIcon({ - help, - align = "right", -}: { - help: ColHelp; - align?: "left" | "right"; -}) { - const [open, setOpen] = useState(false); - useEffect(() => { - if (!open) return; - const onDown = (e: MouseEvent) => { - const el = e.target as Element | null; - if (!el?.closest("[data-col-help]")) setOpen(false); - }; - const onKey = (e: KeyboardEvent) => { - if (e.key === "Escape") setOpen(false); - }; - document.addEventListener("mousedown", onDown); - document.addEventListener("keydown", onKey); - return () => { - document.removeEventListener("mousedown", onDown); - document.removeEventListener("keydown", onKey); - }; - }, [open]); - return ( - - - {open && } - - ); -} + input: "Input tokens (uncached): fresh prompt input billed at the full input rate — the slice that was neither written to nor read from the prompt cache.", + output: "Output tokens: text, code, tool arguments and reasoning the model generated.", + cw: "Cache-write tokens: context newly written into the prompt cache (cache_creation_input_tokens).", + cr: "Cache-read tokens: cached input re-billed on every later call (cache_read_input_tokens). Usually the dominant cost line.", +} satisfies Record; diff --git a/evalboard/app/_overview/__tests__/efficiency-charts.test.tsx b/evalboard/app/_overview/__tests__/efficiency-charts.test.tsx new file mode 100644 index 00000000..0e33607a --- /dev/null +++ b/evalboard/app/_overview/__tests__/efficiency-charts.test.tsx @@ -0,0 +1,80 @@ +import { describe, expect, test } from "vitest"; +import { fireEvent, render, screen } from "@testing-library/react"; +import type { RunPoint } from "@/lib/overview"; +import { EfficiencyCharts } from "../efficiency-charts"; + +function point(overrides: Partial = {}): RunPoint { + return { + runId: "2026-08-18_04-51-58", + timestamp: Date.UTC(2026, 7, 18), + harness: "codex", + successRate: 96, + turnBudgetRate: 87, + withinExpectedTimeRate: 78, + timePerPassedTask: 192, + ...overrides, + }; +} + +function renderCharts(scoped = false) { + return render( + , + ); +} + +describe("EfficiencyCharts", () => { + test("opens on the wall-clock metric", () => { + renderCharts(); + expect(screen.getByRole("heading")).toHaveTextContent( + "Time per Passed Task", + ); + expect( + screen.getByRole("tab", { name: "Time" }), + ).toHaveAttribute("aria-selected", "true"); + }); + + test("the Turns tab still reaches the retired turn-budget chart", () => { + // The turn budget is kept visible while the derived expected-time line + // is being watched; retiring it is deleting this tab. + renderCharts(); + fireEvent.click(screen.getByRole("tab", { name: "Turns" })); + expect(screen.getByRole("heading")).toHaveTextContent( + "Within Expected Turns (%)", + ); + expect(screen.getByText(/1.5× their expected turns/)).toBeInTheDocument(); + expect( + screen.getByRole("tab", { name: "Turns" }), + ).toHaveAttribute("aria-selected", "true"); + expect(screen.getByRole("tab", { name: "Time" })).toHaveAttribute( + "aria-selected", + "false", + ); + }); + + test("switching back restores the wall-clock blurb", () => { + renderCharts(); + fireEvent.click(screen.getByRole("tab", { name: "Turns" })); + fireEvent.click(screen.getByRole("tab", { name: "Time" })); + expect( + screen.getByText(/the number that passed/), + ).toBeInTheDocument(); + }); + + test("says so when the numbers are filter-scoped", () => { + renderCharts(true); + expect( + screen.getByText(/scoped to the active filter/), + ).toBeInTheDocument(); + }); + + test("no filter note when the window is unscoped", () => { + renderCharts(); + expect(screen.queryByText(/scoped to the active filter/)).toBeNull(); + }); +}); diff --git a/evalboard/app/_overview/__tests__/harness-legend.test.tsx b/evalboard/app/_overview/__tests__/harness-legend.test.tsx index f55698f4..35e33a64 100644 --- a/evalboard/app/_overview/__tests__/harness-legend.test.tsx +++ b/evalboard/app/_overview/__tests__/harness-legend.test.tsx @@ -37,6 +37,44 @@ describe("HarnessLegend", () => { describe("HarnessTooltip", () => { const s = series("claude-code", "codex"); + test("formats the value with the chart's own formatter", () => { + render( + `${Math.round(v)}s`} + payload={[{ dataKey: s[1].dataKey, value: 192.3 }]} + />, + ); + expect(screen.getByText("192s per passed task")).toBeInTheDocument(); + }); + + test("adds the secondary line for the hovered point only", () => { + // The within-expected rate belongs to one run, so it rides the hover on + // that run's point instead of being summarized over the whole chart. + render( + + harness === "codex" ? "78% within 2× expected" : null + } + payload={[ + { dataKey: s[0].dataKey, value: 300 }, + { dataKey: s[1].dataKey, value: 192 }, + ]} + />, + ); + expect(screen.getByText("78% within 2× expected")).toBeInTheDocument(); + expect(screen.queryAllByText(/within/)).toHaveLength(1); + }); + test("shows only the harnesses that actually ran at the hovered x", () => { // Recharts hands over every series, including the ones with no value // here. Listing those would invent runs that never happened. diff --git a/evalboard/app/_overview/__tests__/harness-series.test.ts b/evalboard/app/_overview/__tests__/harness-series.test.ts index 399bbd02..b6e742b6 100644 --- a/evalboard/app/_overview/__tests__/harness-series.test.ts +++ b/evalboard/app/_overview/__tests__/harness-series.test.ts @@ -12,14 +12,16 @@ function point( harness: string, timestamp: number, successRate: number | null, - turnBudgetRate: number | null = null, + withinExpectedTimeRate: number | null = null, ): RunPoint { return { runId: `run-${timestamp}`, timestamp, harness, successRate, - turnBudgetRate, + turnBudgetRate: null, + withinExpectedTimeRate, + timePerPassedTask: null, }; } @@ -99,7 +101,7 @@ describe("pivotByHarness", () => { timestamp: 100, [key]: 90, }); - expect(pivotByHarness(pts, ["codex"], "turnBudgetRate").rows[0]).toEqual( + expect(pivotByHarness(pts, ["codex"], "withinExpectedTimeRate").rows[0]).toEqual( { timestamp: 100, [key]: 40 }, ); }); @@ -111,7 +113,7 @@ describe("pivotByHarness", () => { const { rows } = pivotByHarness( [point("codex", 100, 90, null)], ["codex"], - "turnBudgetRate", + "withinExpectedTimeRate", ); expect(rows).toEqual([]); }); diff --git a/evalboard/app/_overview/efficiency-charts.tsx b/evalboard/app/_overview/efficiency-charts.tsx new file mode 100644 index 00000000..002e74b9 --- /dev/null +++ b/evalboard/app/_overview/efficiency-charts.tsx @@ -0,0 +1,104 @@ +"use client"; + +// The overview's efficiency section, tabbed. +// +// Two signals run side by side for now. "Time" is the derived wall-clock line +// (lib/timing.ts): expected seconds per task per harness, computed from that +// task's own passing history by the eval runner and stamped into run.json. +// "Turns" is the hand-written predecessor it is meant to replace, kept visible +// while the new number is being watched rather than trusted. +// +// The tab strip is the seam: retiring the turn budget is deleting the "turns" +// entry from TABS and the TurnBudgetChart import, with no other edit to this +// page. Tab state is local — deliberately not a search param, so switching +// charts never re-renders the server page or perturbs a shared link. + +import { useState } from "react"; +import type { RunPoint } from "@/lib/overview"; +import { TimePerPassedTaskChart } from "./wall-clock-chart"; +import { TurnBudgetChart } from "./turn-budget-chart"; + +type TabKey = "time" | "turns"; + +interface ChartProps { + data: RunPoint[]; + harnesses: string[]; + windowStart: number; + windowEnd: number; +} + +const TABS: Array<{ + key: TabKey; + label: string; + heading: string; + // Hover text on the heading: what the number actually is. + title: string; + // Shown under the heading. `scoped` appends the active-filter note. + blurb: (scoped: boolean) => string; + render: (props: ChartProps) => React.ReactNode; +}> = [ + { + key: "time", + label: "Time", + heading: "Time per Passed Task", + title: "Total wall clock ÷ tasks passed. Every task's seconds count, failures included; only passes count in the denominator, so a run that fails more reads slower.", + blurb: (scoped) => + "Seconds of every task that ran ÷ the number that passed · hover a point for the share within 2× expected" + + (scoped ? " · scoped to the active filter" : ""), + render: (props) => , + }, + { + key: "turns", + label: "Turns", + heading: "Within Expected Turns (%)", + title: "Share of tasks carrying an expected_turns budget whose visible turns stayed within 1.5× it. A budgeted task that failed counts as over budget.", + blurb: (scoped) => + "% of budgeted tasks that stayed within 1.5× their expected turns (a budgeted task that failed counts as over budget) · runs with no budgeted task are omitted rather than plotted at 0" + + (scoped ? " · scoped to the active filter" : ""), + render: (props) => , + }, +]; + +export function EfficiencyCharts({ + scoped = false, + ...props +}: ChartProps & { scoped?: boolean }) { + const [active, setActive] = useState(TABS[0].key); + const tab = TABS.find((t) => t.key === active) ?? TABS[0]; + return ( +
+
+

+ {tab.heading} +

+ + {TABS.map((t) => ( + + ))} + +
+

{tab.blurb(scoped)}

+ {tab.render(props)} +
+ ); +} diff --git a/evalboard/app/_overview/harness-legend.tsx b/evalboard/app/_overview/harness-legend.tsx index 750c600e..8a9aa880 100644 --- a/evalboard/app/_overview/harness-legend.tsx +++ b/evalboard/app/_overview/harness-legend.tsx @@ -57,6 +57,8 @@ export function HarnessTooltip({ series, suffix, emptyText, + format = (v) => `${v.toFixed(1)}%`, + secondary, }: { active?: boolean; payload?: TooltipEntry[]; @@ -67,6 +69,12 @@ export function HarnessTooltip({ suffix: string; // Shown when the hovered run produced no value for this metric at all. emptyText: string; + // How to render the value. Defaults to a percentage; a seconds chart passes + // its own duration formatter. + format?: (value: number) => string; + // Optional second line for a row, e.g. a companion rate the chart doesn't + // plot. Returning null omits it for that point. + secondary?: (harness: string, timestamp: number) => string | null; }) { if (!active || !payload?.length) return null; const byKey = new Map(series.map((s) => [s.dataKey, s])); @@ -86,22 +94,27 @@ export function HarnessTooltip({ ) : ( rows.map((e) => { const s = byKey.get(String(e.dataKey))!; + const sub = secondary?.(s.harness, ms) ?? null; return ( -
- - - {harnessShortLabel(s.harness)} - - - {(e.value as number).toFixed(1)}% {suffix} - +
+
+ + + {harnessShortLabel(s.harness)} + + + {format(e.value as number)} {suffix} + +
+ {sub && ( +
+ {sub} +
+ )}
); }) diff --git a/evalboard/app/_overview/harness-series.ts b/evalboard/app/_overview/harness-series.ts index 1b50f111..3f50f67b 100644 --- a/evalboard/app/_overview/harness-series.ts +++ b/evalboard/app/_overview/harness-series.ts @@ -10,9 +10,13 @@ import { harnessColor } from "@/lib/harness"; import type { RunPoint } from "@/lib/overview"; -// Which per-run rate to plot. Both metrics live on the same RunPoint, so the -// two overview charts share this module and differ only by this key. -export type HarnessMetric = "successRate" | "turnBudgetRate"; +// Which per-run metric to plot. All of them live on the same RunPoint, so the +// overview charts share this module and differ only by this key. +export type HarnessMetric = + | "successRate" + | "turnBudgetRate" + | "withinExpectedTimeRate" + | "timePerPassedTask"; export interface HarnessSeries { harness: string; diff --git a/evalboard/app/_overview/wall-clock-chart.tsx b/evalboard/app/_overview/wall-clock-chart.tsx new file mode 100644 index 00000000..f1859cbd --- /dev/null +++ b/evalboard/app/_overview/wall-clock-chart.tsx @@ -0,0 +1,138 @@ +"use client"; + +import { + CartesianGrid, + Line, + LineChart, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from "recharts"; +import type { RunPoint } from "@/lib/overview"; +import { fmtTaskSeconds } from "@/lib/timing"; +import { pivotByHarness } from "./harness-series"; +import { HarnessLegend, HarnessTooltip } from "./harness-legend"; + +// MM-DD tick label on the axis; full date+time appears in the tooltip. +function shortLabel(ms: number): string { + const d = new Date(ms); + const m = String(d.getUTCMonth() + 1).padStart(2, "0"); + const day = String(d.getUTCDate()).padStart(2, "0"); + return `${m}-${day}`; +} + +// Sibling of DailySuccessChart: same axes/styling and the same per-harness +// series split, plotting seconds per passed task instead of the success rate. +// Driven by the same windowed RunPoint[] so the shared window and harness +// selectors control both. Per-harness lines matter more here than on any other +// chart: codex runs the suite in roughly a third of claude-code's wall clock, +// so one blended line would plot the schedule rather than the speed. +export function TimePerPassedTaskChart({ + data, + harnesses, + windowStart, + windowEnd, +}: { + data: RunPoint[]; + harnesses: string[]; + windowStart: number; + windowEnd: number; +}) { + const { rows, series } = pivotByHarness( + data, + harnesses, + "timePerPassedTask", + ); + // The within-expected rate belongs to one run, so it reads as a hover detail + // on that run's point rather than as a headline over a multi-run chart. + const withinByPoint = new Map( + data + .filter((p) => p.withinExpectedTimeRate != null) + .map((p) => [ + `${p.harness}|${p.timestamp}`, + p.withinExpectedTimeRate as number, + ]), + ); + return ( +
+
+ + UTC + + + + + + fmtTaskSeconds(v)} + tick={{ fontSize: 11, fill: "#6b7280" }} + tickLine={false} + axisLine={false} + width={48} + /> + { + const r = withinByPoint.get( + `${harness}|${ms}`, + ); + return r == null + ? null + : `${r.toFixed(0)}% within 2× expected`; + }} + /> + } + cursor={{ + stroke: "#e5e7eb", + strokeDasharray: "3 3", + }} + /> + {series.map((s) => ( + + ))} + + +
+ +
+ ); +} diff --git a/evalboard/app/page.tsx b/evalboard/app/page.tsx index 98b83580..8537c1ac 100644 --- a/evalboard/app/page.tsx +++ b/evalboard/app/page.tsx @@ -11,7 +11,7 @@ import { fmtDuration, fmtRunTime, fmtTimestamp } from "@/lib/format"; import { passClass } from "@/lib/pass-rate"; import { type Window } from "@/lib/reviews-types"; import { DailySuccessChart } from "./_overview/daily-chart"; -import { TurnBudgetChart } from "./_overview/turn-budget-chart"; +import { EfficiencyCharts } from "./_overview/efficiency-charts"; import { WindowSummary } from "./_overview/window-summary"; import { ChipLegend, MergedTagRail } from "./_overview/tag-rail"; import { TableScroll } from "./_components/scroll-table"; @@ -282,24 +282,13 @@ export default async function Page({ windowStart={overview.windowStart} windowEnd={overview.windowEnd} /> -
-

- Within Expected Turns (%) -

-

- % of budgeted tasks that stayed within 1.5× their - expected turns (a budgeted task that failed counts as - over budget) · runs with no budgeted task are omitted - rather than plotted at 0 - {activeTag || q ? " · scoped to the active filter" : ""} -

- -
+
diff --git a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx index 21d96a97..cf5ade72 100644 --- a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx +++ b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx @@ -105,16 +105,15 @@ describe("MessageTimelineSection — table layout", () => { expect(screen.getByText("$0.0123")).toBeInTheDocument(); }); - test("Cost header has an ⓘ help bubble explaining per-message cost", () => { + test("Cost header explains per-message cost in its title, no ⓘ bubble", () => { render(); - const trigger = screen.getByRole("button", { - name: /What is Per-message cost/i, - }); - expect(screen.queryByRole("tooltip")).toBeNull(); - fireEvent.click(trigger); - const card = screen.getByRole("tooltip"); - expect(card).toHaveTextContent("Per-message cost"); - expect(card).toHaveTextContent("authoritative SDK number"); + expect(screen.getByText("Cost")).toHaveAttribute( + "title", + expect.stringContaining("authoritative SDK number"), + ); + expect( + screen.queryByRole("button", { name: /What is/i }), + ).toBeNull(); }); test("unpriced message shows an em-dash for cost, not $0.00", () => { diff --git a/evalboard/app/runs/[id]/[...task]/__tests__/task-stats.test.tsx b/evalboard/app/runs/[id]/[...task]/__tests__/task-stats.test.tsx new file mode 100644 index 00000000..62bb77cc --- /dev/null +++ b/evalboard/app/runs/[id]/[...task]/__tests__/task-stats.test.tsx @@ -0,0 +1,84 @@ +import { describe, expect, test } from "vitest"; +import { render, screen } from "@testing-library/react"; +import { DurationStat, ExpectedTimeStat } from "../task-stats"; + +function renderDuration(seconds: number | null, expected: number | null) { + return render( +
+ +
, + ); +} + +describe("DurationStat", () => { + test("red ratio beside the time when > 2x expected (ratio 2.4)", () => { + renderDuration(240, 100); + const dd = screen.getByText("4m00s"); + expect(dd.tagName).toBe("DD"); + const ratio = screen.getByText("2.4×"); + expect(ratio.className).toContain("text-rose-700"); + expect(ratio.className).not.toContain("bg-"); + expect(dd).toHaveAttribute( + "title", + "2.40x expected · expected time: 1m40s", + ); + }); + + test("yellow ratio between 1.5x and 2x expected (ratio 1.8)", () => { + renderDuration(180, 100); + expect(screen.getByText("1.8×").className).toContain("text-amber-700"); + }); + + test("green ratio at or under 1.5x expected (ratio 1.2)", () => { + renderDuration(120, 100); + expect(screen.getByText("1.2×").className).toContain( + "text-emerald-700", + ); + }); + + test("the time itself is never tinted, so the ratio carries the signal", () => { + renderDuration(240, 100); + const dd = screen.getByText("4m00s"); + expect(dd.className).toContain("text-gray-900"); + expect(dd.className).not.toMatch(/text-(rose|amber|emerald)-/); + }); + + test("an unscored task shows no ratio and says why", () => { + renderDuration(120, null); + const dd = screen.getByText("2m00s"); + expect(dd.className).toContain("text-gray-900"); + expect(screen.queryByText(/×$/)).toBeNull(); + expect(dd).toHaveAttribute( + "title", + "no expected time yet (needs a passing run on this harness)", + ); + }); + + test("both null renders em dash with default text", () => { + renderDuration(null, null); + const dd = screen.getByText("—"); + expect(dd.tagName).toBe("DD"); + expect(dd.className).toContain("text-gray-900"); + }); +}); + +describe("ExpectedTimeStat", () => { + test("renders the derived line when the task is scored", () => { + render( +
+ +
, + ); + expect(screen.getByText("Expected time")).toBeInTheDocument(); + expect(screen.getByText("1m44s")).toBeInTheDocument(); + }); + + test("renders em dash when the task is unscored", () => { + render( +
+ +
, + ); + expect(screen.getByText("—")).toBeInTheDocument(); + }); +}); diff --git a/evalboard/app/runs/[id]/[...task]/_sections.tsx b/evalboard/app/runs/[id]/[...task]/_sections.tsx index 12cfea2e..617da165 100644 --- a/evalboard/app/runs/[id]/[...task]/_sections.tsx +++ b/evalboard/app/runs/[id]/[...task]/_sections.tsx @@ -20,11 +20,7 @@ import { } from "@/lib/thinkingSim"; import { fmtCompact, fmtUsd } from "@/lib/format"; import { tokenBucketUsd, type TokenKind } from "@/lib/pricing"; -import { - type ColHelp, - ColHelpIcon, - TOKEN_COLUMN_HELP, -} from "@/app/_components/col-help"; +import { TOKEN_COLUMN_HELP } from "@/app/_components/col-help"; import { type Unit, UnitToggle } from "@/app/_components/unit-toggle"; import { withSource } from "@/app/_lib/source-param"; import { TableScroll } from "@/app/_components/scroll-table"; @@ -279,25 +275,11 @@ const MSG_GRID = "grid items-center gap-2 px-2 py-1 " + "grid-cols-[1.5rem_2.5rem_3.5rem_3.5rem_minmax(0,1fr)_3.5rem_3.5rem_3.5rem_3.5rem_4.5rem]"; -// Per-message Cost help (grid-specific: this is a rate-derived per-call figure, -// not the SDK's cumulative per-turn cost). Token-column help is shared via -// TOKEN_COLUMN_HELP so the timeline and the run grid stay consistent. -const MESSAGE_COST_HELP: ColHelp = { - title: "Per-message cost", - body: "This message's recorded tokens priced at list rates — the cost of this single API call. The SDK reports only a cumulative per-turn figure, so these need not sum exactly to the task's total cost (the authoritative SDK number) shown above. Blank when the model is unpriced or no per-message tokens were recorded.", -}; - -// A right-aligned message-timeline header cell with an ⓘ help bubble. Label sits -// at the right edge with the icon to its left (flex-row-reverse), matching the -// run-grid headers. -function MsgHeadHelp({ label, help }: { label: string; help: ColHelp }) { - return ( - - {label} - - - ); -} +// Per-message Cost tooltip (grid-specific: this is a rate-derived per-call +// figure, not the SDK's cumulative per-turn cost). Token-column copy is shared +// via TOKEN_COLUMN_HELP so the timeline and the run grid stay consistent. +const MESSAGE_COST_HELP = + "Per-message cost: this message's recorded tokens priced at list rates — the cost of this single API call. The SDK reports only a cumulative per-turn figure, so these need not sum exactly to the task's total cost (the authoritative SDK number) shown above. Blank when the model is unpriced or no per-message tokens were recorded."; function messageKind(blockTypes: MessageEvent["blockTypes"]): string { const set = new Set(blockTypes); @@ -496,32 +478,20 @@ export function MessageTimelineSection({ Gen Exec Content - - + + In - - + + Cache R - - + + Cache W - - + + Out - - + + Cost
    diff --git a/evalboard/app/runs/[id]/[...task]/page.tsx b/evalboard/app/runs/[id]/[...task]/page.tsx index e95bd7d8..0bf02fa2 100644 --- a/evalboard/app/runs/[id]/[...task]/page.tsx +++ b/evalboard/app/runs/[id]/[...task]/page.tsx @@ -17,6 +17,7 @@ import { ChipButton } from "../chips"; import { VersionChip } from "@/app/_components/version-list"; import { isInternal } from "@/lib/edition"; import { displayedTurns } from "@/lib/turns"; +import { DurationStat, ExpectedTimeStat } from "./task-stats"; import { ExpectedTurnsStat, TurnsStat } from "./turns-stat"; import { ArtifactsSection, @@ -183,7 +184,7 @@ export default async function TaskPage({ )}
-
+
Score @@ -192,16 +193,10 @@ export default async function TaskPage({ {task.weightedScore?.toFixed(2) ?? "—"}
-
-
- Duration -
-
- {task.durationSeconds - ? `${task.durationSeconds.toFixed(1)}s` - : "—"} -
-
+
Cost @@ -228,6 +223,7 @@ export default async function TaskPage({ expectedTurns={task.expectedTurns} /> +
Tokens diff --git a/evalboard/app/runs/[id]/[...task]/task-stats.tsx b/evalboard/app/runs/[id]/[...task]/task-stats.tsx new file mode 100644 index 00000000..4f56a722 --- /dev/null +++ b/evalboard/app/runs/[id]/[...task]/task-stats.tsx @@ -0,0 +1,69 @@ +import { + expectedTimeTitle, + fmtTaskSeconds, + fmtTimeRatio, + fmtTimeRatioCell, + timeCellClasses, + timeRatio, + tintForTimeRatio, +} from "@/lib/timing"; + +// Duration against the time this task is expected to need. The ratio is printed +// beside the time rather than left to a hover, so the color is never the only +// signal that a task ran slow. +export function DurationStat({ + durationSeconds, + expectedSeconds, +}: { + durationSeconds: number | null; + expectedSeconds: number | null; +}) { + const ratio = timeRatio(durationSeconds, expectedSeconds); + return ( +
+
+ Duration +
+
+ {fmtTaskSeconds(durationSeconds)} + {ratio != null && ( + + {fmtTimeRatioCell(ratio)} + + )} +
+
+ ); +} + +// The derived line itself, so a reader can see what the tint was measured +// against without hovering. Never hand-written: the eval runner derives it per +// harness from that task's own passing history and stamps it into run.json. +export function ExpectedTimeStat({ + expectedSeconds, +}: { + expectedSeconds: number | null; +}) { + return ( +
+
+ Expected time +
+
+ {expectedSeconds != null ? fmtTaskSeconds(expectedSeconds) : "—"} +
+
+ ); +} diff --git a/evalboard/app/runs/[id]/__tests__/run-view.render.test.tsx b/evalboard/app/runs/[id]/__tests__/run-view.render.test.tsx index 8fc74b96..4b12f178 100644 --- a/evalboard/app/runs/[id]/__tests__/run-view.render.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/run-view.render.test.tsx @@ -26,6 +26,7 @@ function row( actualCommands: null, totalTurns: null, expectedTurns: null, + expectedSeconds: null, hasFinalReply: false, inputTokens: null, outputTokens: null, diff --git a/evalboard/app/runs/[id]/__tests__/run-view.test.ts b/evalboard/app/runs/[id]/__tests__/run-view.test.ts index 41f15373..22a266bd 100644 --- a/evalboard/app/runs/[id]/__tests__/run-view.test.ts +++ b/evalboard/app/runs/[id]/__tests__/run-view.test.ts @@ -16,6 +16,7 @@ function row( actualCommands: null, totalTurns: null, expectedTurns: null, + expectedSeconds: null, hasFinalReply: false, inputTokens: null, outputTokens: null, diff --git a/evalboard/app/runs/[id]/__tests__/source-hrefs.test.tsx b/evalboard/app/runs/[id]/__tests__/source-hrefs.test.tsx index 03b3cc5a..c35143bd 100644 --- a/evalboard/app/runs/[id]/__tests__/source-hrefs.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/source-hrefs.test.tsx @@ -18,6 +18,7 @@ function row( actualCommands: null, totalTurns: null, expectedTurns: null, + expectedSeconds: null, hasFinalReply: false, inputTokens: null, outputTokens: null, diff --git a/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx b/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx index 26db92cd..965d9724 100644 --- a/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx @@ -6,7 +6,7 @@ import { TaskGrid } from "../task-grid"; function row( taskId: string, actualCommands: number | null, - expectedTurns: number | null, + expectedSeconds: number | null, extra: Partial = {}, ): TaskResultSummary { return { @@ -14,11 +14,12 @@ function row( replicateIndex: null, status: "SUCCESS", weightedScore: 1.0, - durationSeconds: 1.0, + durationSeconds: 100, totalCostUsd: 0.1, actualCommands, totalTurns: null, - expectedTurns, + expectedTurns: null, + expectedSeconds, hasFinalReply: false, inputTokens: null, outputTokens: null, @@ -38,7 +39,7 @@ function revealTokens(): void { fireEvent.click(screen.getByRole("button", { name: /show tokens/i })); } -function turnsCellFor(taskId: string): HTMLElement { +function cellFor(taskId: string, index: number): HTMLElement { // Scope to the desktop : below md the grid also renders each task as // a card (same link/values), so an unscoped query would match twice. const table = screen.getByRole("table"); @@ -47,10 +48,14 @@ function turnsCellFor(taskId: string): HTMLElement { }); const tr = link.closest("tr")!; const cells = within(tr).getAllByRole("cell"); - // Layout: Task, Status, Score, Duration, Cost, Turns, Out, Cache+, Cache↺ - return cells[5]!; + return cells[index]!; } +// Layout: Task, Status, Score, Duration, vs Exp, Cost, Turns, then the tokens +const durationCellFor = (taskId: string) => cellFor(taskId, 3); +const vsExpCellFor = (taskId: string) => cellFor(taskId, 4); +const turnsCellFor = (taskId: string) => cellFor(taskId, 6); + describe("TaskGrid — mature rows", () => { test("opens a popover linking to the run where it last executed", () => { render( @@ -117,15 +122,97 @@ describe("TaskGrid — mature rows", () => { }); }); +describe("TaskGrid — vs Expected column", () => { + // row() fixes durationSeconds at 100s, so expectedSeconds sets the ratio. + const ratioRows = [ + row("over", 3, 40), // ratio 2.5 → red (> 2) + row("mid", 3, 56), // ratio 1.79 → 1.8× → yellow (1.5 < r ≤ 2) + row("under", 3, 250), // ratio 0.4 → green (≤ 1.5) + row("unscored", 3, null), // no line yet → em dash, no tint + ]; + + test("prints the ratio so it is readable without hovering", () => { + render( + , + ); + expect(vsExpCellFor("over")).toHaveTextContent("2.5×"); + expect(vsExpCellFor("mid")).toHaveTextContent("1.8×"); + expect(vsExpCellFor("under")).toHaveTextContent("0.4×"); + expect(vsExpCellFor("unscored")).toHaveTextContent("—"); + }); + + test("colorizes the ratio per bucket (no background)", () => { + render( + , + ); + + const overCell = vsExpCellFor("over"); + expect(overCell.className).toContain("text-rose-700"); + expect(overCell.className).not.toContain("bg-"); + expect(overCell).toHaveAttribute("title", "expected time: 0m40s"); + + expect(vsExpCellFor("mid").className).toContain("text-amber-700"); + expect(vsExpCellFor("under").className).toContain("text-emerald-700"); + + const unscoredCell = vsExpCellFor("unscored"); + expect(unscoredCell.className).toContain("text-gray-900"); + expect(unscoredCell.className).not.toMatch( + /text-(rose|amber|emerald)-/, + ); + expect(unscoredCell).toHaveAttribute( + "title", + "no expected time yet (needs a passing run on this harness)", + ); + }); + + test("Duration stays untinted, so length is never mistaken for slowness", () => { + render( + , + ); + for (const id of ["over", "mid", "under", "unscored"]) { + expect(durationCellFor(id).className).not.toMatch( + /text-(rose|amber|emerald)-/, + ); + } + }); + + test("sorts by ratio, which is a different order than by duration", () => { + render( + , + ); + const order = () => + within(screen.getByRole("table")) + .getAllByRole("row") + .slice(1) + .map((tr) => within(tr).getAllByRole("cell")[0].textContent); + + fireEvent.click(screen.getByRole("button", { name: /^Duration$/ })); + expect(order()[0]).toMatch(/long/i); + + fireEvent.click(screen.getByRole("button", { name: /^vs Expected$/ })); + expect(order()[0]).toMatch(/short/i); + }); +}); + describe("TaskGrid — Turns column", () => { + // The turn budget still tints its own column, beside the wall-clock ratio. + // Both signals are shown while the derived expected-time line is watched. test("colorizes the digits per ratio bucket (no background)", () => { render( 1.5) - row("mid", 7, 5), // ratio 1.4 → yellow (1.25 < r ≤ 1.5) - row("under", 4, 10), // ratio 0.4 → green (≤ 1.25) + row("over", 10, null, { expectedTurns: 5 }), // ratio 2.0 → red (> 1.5) + row("mid", 7, null, { expectedTurns: 5 }), // ratio 1.4 → yellow (1.25 < r ≤ 1.5) + row("under", 4, null, { expectedTurns: 10 }), // ratio 0.4 → green (≤ 1.25) row("notarget", 7, null), // black-ish default ]} />, @@ -135,16 +222,10 @@ describe("TaskGrid — Turns column", () => { expect(overCell).toHaveTextContent("10"); expect(overCell.className).toContain("text-rose-700"); expect(overCell.className).not.toContain("bg-"); - expect(overCell).toHaveAttribute( - "title", - "expected_turns target: 5", - ); - - const midCell = turnsCellFor("mid"); - expect(midCell.className).toContain("text-amber-700"); + expect(overCell).toHaveAttribute("title", "expected_turns target: 5"); - const underCell = turnsCellFor("under"); - expect(underCell.className).toContain("text-emerald-700"); + expect(turnsCellFor("mid").className).toContain("text-amber-700"); + expect(turnsCellFor("under").className).toContain("text-emerald-700"); const noTargetCell = turnsCellFor("notarget"); expect(noTargetCell).toHaveTextContent("7"); @@ -167,8 +248,6 @@ describe("TaskGrid — Turns column", () => { test("token columns are collapsed by default, revealed by the toggle", () => { render(); - // Read the sort toggle (first button) per header — token columns also - // carry an ⓘ help button, so the bare th textContent isn't the label. const labels = () => screen .getAllByRole("columnheader") @@ -183,6 +262,7 @@ describe("TaskGrid — Turns column", () => { "Status", "Score", "Duration", + "vs Expected", "Cost", "Turns", ]); @@ -193,6 +273,7 @@ describe("TaskGrid — Turns column", () => { "Status", "Score", "Duration", + "vs Expected", "Cost", "Turns", "In", @@ -203,38 +284,32 @@ describe("TaskGrid — Turns column", () => { }); }); -describe("TaskGrid — column help popover", () => { - test("ⓘ toggles a static help card; Escape closes it", () => { - render(); - revealTokens(); // ⓘ help buttons live on the token columns - const trigger = screen.getByRole("button", { - name: /What is Cache R/i, - }); - - expect(screen.queryByRole("tooltip")).toBeNull(); - - fireEvent.click(trigger); - const card = screen.getByRole("tooltip"); - expect(card).toHaveTextContent("Cache-read tokens"); - expect(card).toHaveTextContent("Common causes:"); - expect(card).toHaveTextContent("Reduce by:"); - - fireEvent.keyDown(document, { key: "Escape" }); - expect(screen.queryByRole("tooltip")).toBeNull(); - }); - - test("opening one column's help closes another's", () => { +describe("TaskGrid — column tooltips", () => { + test("definitions ride on the header title, not an ⓘ popover", () => { render(); - revealTokens(); // ⓘ help buttons live on the token columns - fireEvent.click(screen.getByRole("button", { name: /What is Out/i })); - expect(screen.getByRole("tooltip")).toHaveTextContent("Output tokens"); + revealTokens(); + const header = (label: string) => + screen + .getAllByRole("columnheader") + .find((h) => h.textContent?.trim().startsWith(label))!; - fireEvent.click( - screen.getByRole("button", { name: /What is Cache R/i }), + expect(header("Cache R")).toHaveAttribute( + "title", + expect.stringContaining("Cache-read tokens"), + ); + expect(header("vs Expected")).toHaveAttribute( + "title", + expect.stringContaining("Duration ÷"), ); - const card = screen.getByRole("tooltip"); - expect(card).toHaveTextContent("Cache-read tokens"); - expect(card).not.toHaveTextContent("Output tokens"); + expect(header("Turns")).toHaveAttribute( + "title", + expect.stringContaining("expected_turns"), + ); + // No ⓘ buttons anywhere: each header carries its sort toggle and nothing else. + for (const h of screen.getAllByRole("columnheader")) { + expect(within(h).getAllByRole("button")).toHaveLength(1); + } + expect(screen.queryByRole("tooltip")).toBeNull(); }); }); diff --git a/evalboard/app/runs/[id]/task-grid.tsx b/evalboard/app/runs/[id]/task-grid.tsx index ef922a70..390fdff2 100644 --- a/evalboard/app/runs/[id]/task-grid.tsx +++ b/evalboard/app/runs/[id]/task-grid.tsx @@ -22,20 +22,24 @@ import { turnRatio, turnsCellClasses, } from "@/lib/turns"; +import { + expectedTimeTitle, + fmtTimeRatioCell, + timeCellClasses, + timeRatio, + tintForTimeRatio, +} from "@/lib/timing"; import { ChipButton } from "./chips"; import { withSource } from "@/app/_lib/source-param"; import { TableScroll } from "@/app/_components/scroll-table"; -import { - type ColHelp, - HelpPopover, - TOKEN_COLUMN_HELP, -} from "@/app/_components/col-help"; +import { TOKEN_COLUMN_HELP } from "@/app/_components/col-help"; type SortKey = | "task" | "status" | "score" | "duration" + | "vsExp" | "cost" | "turns" | "input" @@ -43,17 +47,13 @@ type SortKey = | "cw" | "cr"; -// Per-column help shown from an ⓘ next to the header. Token-column copy is -// shared with the message timeline via TOKEN_COLUMN_HELP; Cost is grid-specific -// (the authoritative SDK total for the task). -const COLUMN_HELP: Partial> = { +// Header tooltips. Token-column copy is shared with the message timeline via +// TOKEN_COLUMN_HELP; the rest is grid-specific. +const COLUMN_HELP: Partial> = { ...TOKEN_COLUMN_HELP, - cost: { - title: "Cost (USD)", - body: "Total billed cost for this task, reported by the SDK (summed across turns).", - causes: "long runs, large context replayed each turn, verbose output, or an expensive model.", - fix: "fewer turns, less context, more concise output; use a cheaper model where acceptable.", - }, + turns: "Visible turns: one per tool call plus one for the final reply. Tinted against the task's hand-written expected_turns budget (yellow past 1.25×, red past 1.5×); untinted when the task declares none.", + vsExp: "Duration ÷ the time this task is expected to need. The expected time is derived per task, per harness by the eval runner (its fastest passing run, or p10 once there are ten) and stamped into the run — never hand-written. Past 2× counts as slow; a task its harness has never passed shows —.", + cost: "Total billed cost for this task, reported by the SDK (summed across turns).", }; // A mature task that was skipped this run has no detail page in THIS run, but it @@ -278,6 +278,7 @@ const DEFAULT_DIR: Record = { status: "asc", score: "desc", duration: "desc", + vsExp: "desc", cost: "desc", turns: "desc", input: "desc", @@ -306,6 +307,11 @@ function compare( (a.durationSeconds ?? -Infinity) - (b.durationSeconds ?? -Infinity) ); + case "vsExp": + return ( + (timeRatio(a.durationSeconds, a.expectedSeconds) ?? -Infinity) - + (timeRatio(b.durationSeconds, b.expectedSeconds) ?? -Infinity) + ); case "cost": return ( (a.totalCostUsd ?? -Infinity) - (b.totalCostUsd ?? -Infinity) @@ -343,6 +349,7 @@ const COLUMNS: Array<{ { key: "status", header: "Status" }, { key: "score", header: "Score", align: "right" }, { key: "duration", header: "Duration", align: "right" }, + { key: "vsExp", header: "vs Expected", align: "right" }, { key: "cost", header: "Cost", align: "right" }, { key: "turns", header: "Turns", align: "right" }, { key: "input", header: "In", align: "right" }, @@ -429,17 +436,30 @@ function Stat({ label, value, valueClass = "text-gray-800", + sub, + subClass = "text-gray-500", + title, }: { label: string; value: string; valueClass?: string; + // Second line under the value, e.g. a duration's ratio to its expected time. + sub?: string; + subClass?: string; + // Hover text for the value, e.g. what a tinted ratio was measured against. + title?: string; }) { return ( -
+
{label}
{value}
+ {sub && ( +
+ {sub} +
+ )}
); } @@ -484,26 +504,6 @@ export function TaskGrid({ // the token detail is one click away via the toolbar toggle. const [showTokens, setShowTokens] = useState(false); - // Which column's help popover is open (one at a time). Dismissed by a click - // outside any popover/trigger or by Escape. - const [openHelp, setOpenHelp] = useState(null); - useEffect(() => { - if (openHelp == null) return; - const onDown = (e: MouseEvent) => { - const el = e.target as Element | null; - if (!el?.closest("[data-col-help]")) setOpenHelp(null); - }; - const onKey = (e: KeyboardEvent) => { - if (e.key === "Escape") setOpenHelp(null); - }; - document.addEventListener("mousedown", onDown); - document.addEventListener("keydown", onKey); - return () => { - document.removeEventListener("mousedown", onDown); - document.removeEventListener("keydown", onKey); - }; - }, [openHelp]); - // How many rows share each taskId — i.e. the replicate count for that task. // Drives whether a row shows its replicate badge + ?r link (only when >1, so // single-run tasks aren't cluttered with a "#0"). @@ -623,70 +623,25 @@ export function TaskGrid({ ? "ascending" : "descending" : "none"; - const help = COLUMN_HELP[col.key]; return (
); })} @@ -695,6 +650,15 @@ export function TaskGrid({ {sorted.map((t) => { const review = reviewsByTask?.get(t.taskId); + // Two efficiency signals, side by side while the + // wall-clock one is being watched. Seconds live in the + // vs Expected cell rather than on Duration: a long task + // is not a slow one. + const timeRatioValue = timeRatio( + t.durationSeconds, + t.expectedSeconds, + ); + const timeTint = tintForTimeRatio(timeRatioValue); // Color off the same visible-events count the cell // displays — not SDK num_turns (totalTurns), which the // visible-turns refactor dropped from the display and @@ -750,6 +714,12 @@ export function TaskGrid({ + @@ -841,6 +811,11 @@ export function TaskGrid({
{sorted.map((t) => { const review = reviewsByTask?.get(t.taskId); + const timeRatioValue = timeRatio( + t.durationSeconds, + t.expectedSeconds, + ); + const timeTint = tintForTimeRatio(timeRatioValue); const turnsTint = tintForRatio( turnRatio( displayedTurns(t.actualCommands, t.hasFinalReply), @@ -894,6 +869,13 @@ export function TaskGrid({ Duration +
@@ -276,51 +289,77 @@ function HistoryTable({ : (e.status ?? "—")} - + - {e.matureSkipped ? ( - // Not executed — no turns to compare to budget. - - ) : ( - (() => { - const tint = tintForRatio( - turnRatio( - e.totalTurns, - e.expectedTurns, - ), - ); - return ( - - ); - })() - )} +
- onSort(col.key)} + className="inline-flex items-center gap-1 hover:text-gray-900" > - - {help && ( - - - {openHelp === col.key && ( - // All help columns sit on - // the right side of the - // table; open leftward so - // the card stays inside the - // overflow-hidden container. - - )} - - )} - + {col.header} + + {arrow} + +
{fmtTableDuration(t.durationSeconds)} + {fmtTimeRatioCell(timeRatioValue)} + {fmtCost(t.totalCostUsd)} + vs Exp + Cost + {e.matureSkipped ? "—" : fmtDuration(e.durationSeconds)} + {e.matureSkipped + ? "—" + : fmtTimeRatioCell( + timeRatio( + e.durationSeconds, + e.expectedSeconds, + ), + )} + {e.matureSkipped ? "—" : fmtUsd(e.totalCostUsd)} - — - - {fmtTurnsCount( - displayedTurns( - e.actualCommands, - e.hasFinalReply, - ), - )} - + {e.matureSkipped + ? "—" + : fmtTurnsCount( + displayedTurns( + e.actualCommands, + e.hasFinalReply, + ), + )} + {e.matureSkipped ? "—" diff --git a/evalboard/app/watchlist/__tests__/watchlist-view.test.tsx b/evalboard/app/watchlist/__tests__/watchlist-view.test.tsx index c19c6c7a..90c15c32 100644 --- a/evalboard/app/watchlist/__tests__/watchlist-view.test.tsx +++ b/evalboard/app/watchlist/__tests__/watchlist-view.test.tsx @@ -36,6 +36,7 @@ function task(o: Partial): RunOverviewTask { actualCommands: null, totalTurns: null, expectedTurns: null, + expectedSeconds: null, visibleTurns: null, hasFinalReply: false, ...o, diff --git a/evalboard/app/watchlist/watchlist-view.tsx b/evalboard/app/watchlist/watchlist-view.tsx index 973d23c8..22ee3ce9 100644 --- a/evalboard/app/watchlist/watchlist-view.tsx +++ b/evalboard/app/watchlist/watchlist-view.tsx @@ -5,6 +5,7 @@ // client JS. import type { ReactNode } from "react"; import Link from "next/link"; +import { fmtTaskSeconds, TIME_BUDGET_TOLERANCE } from "@/lib/timing"; import { humanizeTaskId } from "@/lib/format"; import { passBarClass, passClassRatio } from "@/lib/pass-rate"; import { HarnessSelector } from "@/app/_components/harness-selector"; @@ -38,6 +39,7 @@ const CAP = { streaks: 8, volatility: 6, turnOverage: 8, + timeOverage: 8, } as const; // Renders the first `cap` rows, then — if there are more — a native @@ -550,6 +552,43 @@ export function WatchlistView({ /> )} + + + {data.timeOverage.length === 0 ? ( + All within expected time + ) : ( + + a.avgTimeRatio.toFixed(1) === b.avgTimeRatio.toFixed(1) + } + render={(r) => ( +
+ + {r.skill} + + + {fmtTaskSeconds(r.avgSeconds)} /{" "} + {fmtTaskSeconds(r.avgExpectedSeconds)}{" "} + expected + + 1 + TIME_BUDGET_TOLERANCE ? "bg-red-50 text-red-700 border-red-200" : "bg-amber-50 text-amber-700 border-amber-200"}`} + > + {r.avgTimeRatio.toFixed(1)}× + +
+ )} + /> + )} +

diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index c518ee2d..52507dca 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -9,7 +9,9 @@ import { summarizeListing, taskCarriesRepoTag, taskMatchesTag, + timePerPassedTaskForTasks, turnBudgetRateForTasks, + withinExpectedTimeRateForTasks, type PerRun, type RunListingRow, } from "../overview"; @@ -28,6 +30,7 @@ function task(overrides: Partial): RunOverviewTask { actualCommands: null, totalTurns: null, expectedTurns: null, + expectedSeconds: null, visibleTurns: null, hasFinalReply: false, ...overrides, @@ -206,6 +209,155 @@ describe("turnBudgetRateForTasks", () => { }); }); +describe("withinExpectedTimeRateForTasks", () => { + test("null when no task in scope is scored", () => { + // Nothing carries a derived expected time, so nothing is eligible and the + // chart shows a gap rather than a point. + expect( + withinExpectedTimeRateForTasks([task({ durationSeconds: 120 })]), + ).toBeNull(); + }); + + test("100% when every scored pass is within its line", () => { + expect( + withinExpectedTimeRateForTasks([ + task({ expectedSeconds: 100, durationSeconds: 70 }), + task({ expectedSeconds: 100, durationSeconds: 150 }), // exactly 1.5× + ]), + ).toBe(100); + }); + + test("computes the within-expected share", () => { + expect( + withinExpectedTimeRateForTasks([ + task({ expectedSeconds: 100, durationSeconds: 120 }), // within + task({ expectedSeconds: 100, durationSeconds: 260 }), // over + ]), + ).toBe(50); + }); + + test("excludes unscored tasks from the denominator", () => { + expect( + withinExpectedTimeRateForTasks([ + task({ expectedSeconds: 100, durationSeconds: 120 }), + task({ durationSeconds: 9999 }), // unscored → ignored + ]), + ).toBe(100); + }); + + test("failures are excluded, however long or short they ran", () => { + // The departure from the retired turn budget, which counted a budgeted + // failure as over budget. A 2-second crash did not blow a time budget, + // and counting a slow failure here would let a pass→timeout regression + // read as an efficiency gain once the slow pass stopped counting. + expect( + withinExpectedTimeRateForTasks([ + task({ expectedSeconds: 100, durationSeconds: 120 }), // within + task({ + status: "FAILURE", + expectedSeconds: 100, + durationSeconds: 2, + }), + task({ + status: "TIMEOUT", + expectedSeconds: 100, + durationSeconds: 1200, + }), + ]), + ).toBe(100); + }); + + test("a scored pass with no duration is excluded rather than counted over", () => { + expect( + withinExpectedTimeRateForTasks([ + task({ expectedSeconds: 100, durationSeconds: 120 }), + task({ expectedSeconds: 100, durationSeconds: null }), + ]), + ).toBe(100); + }); + + test("null when a run only has failures, even scored ones", () => { + expect( + withinExpectedTimeRateForTasks([ + task({ status: "FAILURE", expectedSeconds: 100, durationSeconds: 10 }), + task({ status: "ERROR", expectedSeconds: 100, durationSeconds: 3 }), + ]), + ).toBeNull(); + }); + + test("only reflects the tasks passed in (scoping contract)", () => { + expect( + withinExpectedTimeRateForTasks([ + task({ expectedSeconds: 80, durationSeconds: 80 }), + ]), + ).toBe(100); + }); +}); + +describe("timePerPassedTaskForTasks", () => { + test("mature-skipped passes leave both sides of the ratio", () => { + // Regression: counting carried-forward passes in the denominator only + // divided real seconds by tasks that never ran. A codex nightly whose + // runner block said 3m12s rendered as 1m17s on the front page. + const executed = [ + task({ durationSeconds: 100 }), + task({ durationSeconds: 300 }), + ]; + const carried = [ + task({ durationSeconds: 0, matureSkipped: true }), + task({ durationSeconds: 0, matureSkipped: true }), + ]; + expect(timePerPassedTaskForTasks([...executed, ...carried])).toBe( + timePerPassedTaskForTasks(executed), + ); + }); + + test("a mature-skipped pass is not counted as within expected", () => { + expect( + withinExpectedTimeRateForTasks([ + task({ expectedSeconds: 100, durationSeconds: 260 }), // over + task({ + expectedSeconds: 100, + durationSeconds: 0, + matureSkipped: true, + }), + ]), + ).toBe(0); + }); + + test("divides all seconds that ran by the number that passed", () => { + expect( + timePerPassedTaskForTasks([ + task({ durationSeconds: 100 }), + task({ durationSeconds: 300 }), + ]), + ).toBe(200); + }); + + test("seconds burned failing stay in the numerator", () => { + // A run that spends its time failing is a worse run, and the headline + // says so: 400 seconds over the single pass. + expect( + timePerPassedTaskForTasks([ + task({ durationSeconds: 100 }), + task({ status: "FAILURE", durationSeconds: 300 }), + ]), + ).toBe(400); + }); + + test("null when nothing passed", () => { + expect( + timePerPassedTaskForTasks([ + task({ status: "FAILURE", durationSeconds: 300 }), + ]), + ).toBeNull(); + }); + + test("null when no duration was recorded", () => { + expect(timePerPassedTaskForTasks([task({})])).toBeNull(); + }); +}); + describe("collectPipelineRuns", () => { // A usable run: pipeline-cadence with a non-empty overview. function run(id: string, adhoc = false): PerRun { diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts index c0031d37..991419bb 100644 --- a/evalboard/lib/__tests__/runs.test.ts +++ b/evalboard/lib/__tests__/runs.test.ts @@ -81,6 +81,18 @@ describe("toTaskRow", () => { const row = toTaskRow({ task_id: "x", expected_turns: null }); expect(row.expectedTurns).toBeNull(); }); + + test("propagates the stamped expected_seconds", () => { + const row = toTaskRow({ task_id: "x", expected_seconds: 104.5 }); + expect(row.expectedSeconds).toBe(104.5); + }); + + test("a run predating the stamp reads as unscored, not on target", () => { + expect(toTaskRow({ task_id: "x" }).expectedSeconds).toBeNull(); + expect( + toTaskRow({ task_id: "x", expected_seconds: null }).expectedSeconds, + ).toBeNull(); + }); }); describe("aggregateSubAgentUsage", () => { diff --git a/evalboard/lib/__tests__/timing.test.ts b/evalboard/lib/__tests__/timing.test.ts new file mode 100644 index 00000000..2bcf8877 --- /dev/null +++ b/evalboard/lib/__tests__/timing.test.ts @@ -0,0 +1,146 @@ +import { afterEach, describe, expect, test, vi } from "vitest"; +import { + TIME_BUDGET_TOLERANCE, + expectedTimeTitle, + fmtTaskSeconds, + fmtTimeRatio, + getTimeRatioThresholds, + timeRatio, + tintForTimeRatio, + withinExpectedTime, +} from "../timing"; + +describe("timeRatio", () => { + test("computes actual / expected", () => { + expect(timeRatio(160, 100)).toBe(1.6); + }); + + test("null when the duration is missing", () => { + expect(timeRatio(null, 100)).toBeNull(); + }); + + test("null when the task is unscored", () => { + expect(timeRatio(160, null)).toBeNull(); + }); + + test("null when expected is zero (defensive)", () => { + expect(timeRatio(160, 0)).toBeNull(); + }); +}); + +describe("tintForTimeRatio (defaults: yellow=1.5, red=2)", () => { + const t = { yellow: 1.5, red: 2 }; + + test("green well under the line", () => { + expect(tintForTimeRatio(0.5, t)).toBe("green"); + }); + + test("green at exactly 1.5 (yellow boundary)", () => { + expect(tintForTimeRatio(1.5, t)).toBe("green"); + }); + + test("yellow between the thresholds", () => { + expect(tintForTimeRatio(1.8, t)).toBe("yellow"); + }); + + test("yellow at exactly 2 (red boundary)", () => { + expect(tintForTimeRatio(2, t)).toBe("yellow"); + }); + + test("red past the red threshold", () => { + expect(tintForTimeRatio(2.5, t)).toBe("red"); + }); + + test("an unscored task is untinted, not green", () => { + expect(tintForTimeRatio(null)).toBeNull(); + }); +}); + +describe("getTimeRatioThresholds", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + test("returns defaults when env unset", () => { + vi.stubEnv("EVALBOARD_TIME_YELLOW_RATIO", ""); + vi.stubEnv("EVALBOARD_TIME_RED_RATIO", ""); + expect(getTimeRatioThresholds()).toEqual({ yellow: 1.5, red: 2 }); + }); + + test("honours env overrides", () => { + vi.stubEnv("EVALBOARD_TIME_YELLOW_RATIO", "1.1"); + vi.stubEnv("EVALBOARD_TIME_RED_RATIO", "1.75"); + expect(getTimeRatioThresholds()).toEqual({ yellow: 1.1, red: 1.75 }); + }); + + test("falls back to defaults on non-numeric env", () => { + vi.stubEnv("EVALBOARD_TIME_YELLOW_RATIO", "not-a-number"); + vi.stubEnv("EVALBOARD_TIME_RED_RATIO", "high"); + expect(getTimeRatioThresholds()).toEqual({ yellow: 1.5, red: 2 }); + }); + + test("falls back to defaults on zero / negative env", () => { + vi.stubEnv("EVALBOARD_TIME_YELLOW_RATIO", "0"); + vi.stubEnv("EVALBOARD_TIME_RED_RATIO", "-1"); + expect(getTimeRatioThresholds()).toEqual({ yellow: 1.5, red: 2 }); + }); +}); + +describe("withinExpectedTime (default tolerance 1 → 2× expected)", () => { + test("default tolerance matches the runner's", () => { + expect(TIME_BUDGET_TOLERANCE).toBe(1); + }); + + test("within at exactly 2× expected", () => { + expect(withinExpectedTime(200, 100)).toBe(true); + }); + + test("over just past 2× expected", () => { + expect(withinExpectedTime(201, 100)).toBe(false); + }); + + test("within well under expected", () => { + expect(withinExpectedTime(70, 100)).toBe(true); + }); + + test("null when there is no duration", () => { + expect(withinExpectedTime(null, 100)).toBeNull(); + }); + + test("null for an unscored task — never a verdict", () => { + expect(withinExpectedTime(120, null)).toBeNull(); + }); + + test("honours a custom tolerance", () => { + expect(withinExpectedTime(100, 100, 0)).toBe(true); + expect(withinExpectedTime(101, 100, 0)).toBe(false); + }); +}); + +describe("fmtTaskSeconds", () => { + test("keeps seconds under an hour", () => { + expect(fmtTaskSeconds(194)).toBe("3m14s"); + expect(fmtTaskSeconds(720)).toBe("12m00s"); + expect(fmtTaskSeconds(57)).toBe("0m57s"); + }); + + test("switches to hours and minutes past the hour", () => { + expect(fmtTaskSeconds(3720)).toBe("1h02m"); + }); + + test("renders em dash when null", () => { + expect(fmtTaskSeconds(null)).toBe("—"); + }); +}); + +describe("fmtTimeRatio / expectedTimeTitle", () => { + test("ratio reads as a multiple of expected", () => { + expect(fmtTimeRatio(2.632)).toBe("2.63x expected"); + expect(fmtTimeRatio(null)).toBe("—"); + }); + + test("an unscored task explains why it has no line", () => { + expect(expectedTimeTitle(100)).toBe("expected time: 1m40s"); + expect(expectedTimeTitle(null)).toContain("no expected time yet"); + }); +}); diff --git a/evalboard/lib/__tests__/trends.test.ts b/evalboard/lib/__tests__/trends.test.ts index 1d6e0d16..1a039d95 100644 --- a/evalboard/lib/__tests__/trends.test.ts +++ b/evalboard/lib/__tests__/trends.test.ts @@ -14,6 +14,7 @@ function task(overrides: Partial): RunOverviewTask { actualCommands: null, totalTurns: null, expectedTurns: null, + expectedSeconds: null, visibleTurns: null, hasFinalReply: false, ...overrides, @@ -178,7 +179,7 @@ describe("historyForTaskInner", () => { taskId: "t1", status: "SUCCESS", totalTurns: 12, - expectedTurns: 5, + expectedSeconds: 100, }), ]), perRun("r2", [ @@ -186,7 +187,7 @@ describe("historyForTaskInner", () => { taskId: "t1", status: "FAILED", totalTurns: 3, - expectedTurns: 5, + expectedSeconds: 100, }), ]), ]); @@ -195,9 +196,9 @@ describe("historyForTaskInner", () => { // Sorted newest-first by runId. expect(entries[0].runId).toBe("r2"); expect(entries[0].totalTurns).toBe(3); - expect(entries[0].expectedTurns).toBe(5); + expect(entries[0].expectedSeconds).toBe(100); expect(entries[1].totalTurns).toBe(12); - expect(entries[1].expectedTurns).toBe(5); + expect(entries[1].expectedSeconds).toBe(100); }); test("legacy rows fall through as null", async () => { @@ -206,7 +207,7 @@ describe("historyForTaskInner", () => { ]); const entries = await historyForTaskInner("t1", 10); expect(entries[0].totalTurns).toBeNull(); - expect(entries[0].expectedTurns).toBeNull(); + expect(entries[0].expectedSeconds).toBeNull(); }); test("flags mature-skipped entries; normal rows are false", async () => { diff --git a/evalboard/lib/__tests__/watchlist.test.ts b/evalboard/lib/__tests__/watchlist.test.ts index 9bd00496..c1614d39 100644 --- a/evalboard/lib/__tests__/watchlist.test.ts +++ b/evalboard/lib/__tests__/watchlist.test.ts @@ -15,6 +15,7 @@ function task(overrides: Partial): RunOverviewTask { actualCommands: null, totalTurns: null, expectedTurns: null, + expectedSeconds: null, visibleTurns: null, hasFinalReply: false, ...overrides, @@ -210,6 +211,72 @@ describe("turn overage", () => { }); }); +describe("time overage", () => { + test("ranks skills whose passing tasks run past their expected time", () => { + const data = buildWatchlist([ + perRun("2026-01-01", [ + task({ + taskId: "a", + skill: "slow", + status: "SUCCESS", + durationSeconds: 180, + expectedSeconds: 120, + }), + task({ + taskId: "b", + skill: "ok", + status: "SUCCESS", + durationSeconds: 60, + expectedSeconds: 120, + }), + ]), + ]); + expect(data.timeOverage).toEqual([ + { + skill: "slow", + avgTimeRatio: 1.5, + avgSeconds: 180, + avgExpectedSeconds: 120, + }, + ]); + }); + + test("an unscored task contributes nothing", () => { + const data = buildWatchlist([ + perRun("2026-01-01", [ + task({ + taskId: "a", + skill: "unscored", + status: "SUCCESS", + durationSeconds: 999, + expectedSeconds: null, + }), + ]), + ]); + expect(data.timeOverage).toEqual([]); + }); + + test("time overage stays out of the attention score", () => { + // The 50/30/20 fail/regression/turn split is unchanged while both + // efficiency signals run, so a slow-but-passing skill with no turn + // budget scores 0 and never reaches the attention list. + const runs = Array.from({ length: 2 }, (_, i) => + perRun(`2026-01-0${2 - i}`, [ + task({ + taskId: "a", + skill: "slow", + status: "SUCCESS", + durationSeconds: 240, + expectedSeconds: 120, + }), + ]), + ); + const data = buildWatchlist(runs); + expect(data.topAttention).toEqual([]); + expect(data.timeOverage[0].avgTimeRatio).toBe(2); + }); +}); + describe("empty window", () => { test("no runs -> every panel empty, no throw", () => { const data = buildWatchlist([]); @@ -221,6 +288,7 @@ describe("empty window", () => { streaks: [], volatility: [], turnOverage: [], + timeOverage: [], }); }); diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index 925d1f8b..3a159df0 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -13,6 +13,7 @@ import { } from "./runs"; import { listRunIdsInWindow, readRunReviewIndex, parseRunIdDate } from "./reviews"; import { DEFAULT_SOURCE, type Source } from "./sources"; +import { withinExpectedTime } from "./timing"; import { withinTurnBudget } from "./turns"; import { humanizeTaskId } from "./format"; import { mapWithConcurrency } from "./concurrency"; @@ -38,7 +39,20 @@ export interface RunPoint { // of outcome, so the rate is stable under tag/q filtering. null when no // task in scope carries a budget at all (the chart shows a gap rather than // a failure-driven 0%). Same tag/q scoping as successRate. + // + // Superseded by the two wall-clock metrics below and kept only while both + // signals run side by side; the chart behind the "Turns" tab is the only + // reader left, so retiring the turn budget is that tab plus this field. turnBudgetRate: number | null; + // % of scored passing tasks that came in within 2× their expected wall clock. + // Only tasks that passed AND carry a derived expected_seconds are eligible, + // so the rate is stable under tag/q filtering. null when no task in scope is + // scored (the chart shows a gap rather than a failure-driven 0%). + withinExpectedTimeRate: number | null; + // Seconds of every task that ran over the number that passed, for this run's + // scoped task set. Failures stay in the numerator on purpose. null when + // nothing in scope passed or no duration was recorded. + timePerPassedTask: number | null; } // The % of budgeted tasks whose visible turns stayed within 1.5× their @@ -59,7 +73,8 @@ export interface RunPoint { // NOTE: this headline aggregate folds failure into the metric (a budgeted // failure counts as over budget) and so diverges from the per-task "Turns" // cell tint (turns.ts::turnRatio), which is a pure efficiency signal blind to -// pass/fail. See turnRatio's comment. +// pass/fail. See turnRatio's comment. The wall-clock rate below does not make +// that trade: it scores passes only. export function turnBudgetRateForTasks(tasks: RunOverviewTask[]): number | null { let eligible = 0; let withinBudget = 0; @@ -80,6 +95,52 @@ export function turnBudgetRateForTasks(tasks: RunOverviewTask[]): number | null return eligible > 0 ? (withinBudget / eligible) * 100 : null; } +// The % of scored passing tasks that came in within 2× their expected wall +// clock; null when no task in scope is scored. +// +// Passing tasks only, unlike the turn budget above: a task that crashed +// in 10 seconds did not blow a time budget, and counting it would make a +// pass-to-timeout regression read as a gain. Failure is the pass rate's job. +// +// Unscored tasks (a harness that has never passed the task, or a run predating +// the stamp) are excluded, so the rate does not shift when filtering changes +// which co-scoped tasks happen to be scored. Exported for unit testing. +export function withinExpectedTimeRateForTasks( + tasks: RunOverviewTask[], +): number | null { + let eligible = 0; + let within = 0; + for (const t of tasks) { + if (t.status !== "SUCCESS" || t.matureSkipped) continue; + const verdict = withinExpectedTime(t.durationSeconds, t.expectedSeconds); + if (verdict === null) continue; // unscored, or no duration → can't judge + eligible += 1; + if (verdict) within += 1; + } + // Nothing scored in scope → nothing to report (not a failure-driven 0%). + return eligible > 0 ? (within / eligible) * 100 : null; +} + +// Seconds of every task that ran, over the number that passed. Mirrors the +// runner's headline (timing.py::time_per_passed_task) so a filtered front-page +// view and the block stamped into run.json compute the same thing on the same +// rows. The Slack rollup does not report this yet — the metric is being watched +// on the dashboard first. +// +// Mature-skipped rows leave BOTH sides. They are carried-forward passes with no +// duration, so counting them only in the denominator divides real seconds by a +// task count that never ran: on a codex nightly that reported 3m12s, including +// them read 1m17s. +export function timePerPassedTaskForTasks( + tasks: RunOverviewTask[], +): number | null { + const executed = tasks.filter((t) => !t.matureSkipped); + const passed = executed.filter((t) => t.status === "SUCCESS").length; + if (!passed) return null; + const total = executed.reduce((a, t) => a + (t.durationSeconds ?? 0), 0); + return total > 0 ? total / passed : null; +} + export interface TagCount { tag: string; count: number; @@ -665,6 +726,10 @@ export async function getOverview( harness: runHarness, successRate: (row.tasksSucceeded / row.tasksRun) * 100, turnBudgetRate: turnBudgetRateForTasks(scoped.tasks), + withinExpectedTimeRate: withinExpectedTimeRateForTasks( + scoped.tasks, + ), + timePerPassedTask: timePerPassedTaskForTasks(scoped.tasks), }); } runPoints.sort((a, b) => a.timestamp - b.timestamp); diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 7aa72f63..0126e110 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -86,6 +86,11 @@ export interface TaskResultSummary { actualCommands: number | null; totalTurns: number | null; expectedTurns: number | null; + // Wall clock this task is expected to need, derived per harness from run + // history and stamped into run.json by the eval runner. Null = unscored + // (too little history, or a run predating the stamp) — never "on target". + // Runs alongside expectedTurns while the turn budget is still reported. + expectedSeconds: number | null; // True when the agent's final iteration emitted a text reply // (i.e. ResultMessage.result was non-empty). Lets grid/trends // Turns cells inflate by +1 on legacy runs that lack total_turns. @@ -400,6 +405,10 @@ interface RawTaskResult { // null-fallback through the cell helpers in lib/turns.ts. total_turns?: number; expected_turns?: number | null; + // Derived expected wall clock for this task, stamped by the eval runner + // (see eval_runner/skills/timing.py). Absent on unscored tasks and on every + // run predating the stamp, which read as unscored through lib/timing.ts. + expected_seconds?: number | null; // Documented visible-turn count (tool calls + final reply) — the canonical // metric the "within expected turns" chart compares against expected_turns. // Absent on runs predating this field; visibleTurnsFromRaw() then reconstructs @@ -453,6 +462,23 @@ interface RawRunJson { // run.json (run_id/activation/run.json), which the dashboard finalizes with // compute_activation_rollup. The top-level skills run.json never carries it. activation?: RawActivation; + // Run-level wall-clock rollup stamped by the eval runner alongside the + // per-task expected_seconds. Absent on runs predating it. + timing?: RawTiming; +} + +// The `timing` block from run.json. `tolerance` is recorded by the runner so the +// dashboard can never disagree with the run it is describing about what counted +// as "within expected". +interface RawTiming { + harness?: string; + pool_runs?: number; + time_per_passed_task?: number | null; + scored_tasks?: number; + unscored_tasks?: number; + within_expected_time?: number; + within_expected_rate?: number | null; + tolerance?: number; } interface RawActivation { @@ -736,6 +762,7 @@ export function toTaskRow(t: RawTaskResult): TaskResultSummary { actualCommands: t.actual_commands ?? null, totalTurns: t.total_turns ?? null, expectedTurns: t.expected_turns ?? null, + expectedSeconds: t.expected_seconds ?? null, hasFinalReply: t.has_final_reply ?? false, inputTokens: t.input_tokens ?? null, outputTokens: t.output_tokens ?? null, @@ -921,6 +948,7 @@ export interface RunOverviewTask { actualCommands: number | null; totalTurns: number | null; expectedTurns: number | null; + expectedSeconds: number | null; visibleTurns: number | null; hasFinalReply: boolean; // True when the nightly skipped this mature task and carried it forward as a @@ -950,6 +978,11 @@ export interface RunOverview { // from the date-shaped id; ad-hoc ids carry no date, so the ad-hoc listing // orders by this instead. Optional so test factories predating it stay valid. startedAt?: string | null; + // Seconds of every task that ran, over the number that passed — the headline + // efficiency number, as stamped by the runner. Failures are in the numerator + // on purpose: a run that spends an hour failing is a worse run. Null when the + // run predates stamping (the front page then falls back to the task rows). + timePerPassedTask?: number | null; } // Visible-turn count for a task row: the persisted `visible_turns` field when @@ -1043,6 +1076,7 @@ export async function readRunOverview( actualCommands: t.actual_commands ?? null, totalTurns: t.total_turns ?? null, expectedTurns: t.expected_turns ?? null, + expectedSeconds: t.expected_seconds ?? null, visibleTurns: visibleTurnsFromRaw(t), hasFinalReply: t.has_final_reply ?? false, matureSkipped: t.mature_skipped ?? false, @@ -1069,6 +1103,7 @@ export async function readRunOverview( componentShas: extractComponentShas(data.environment_info), ...extractRunConfig(data), startedAt: data.start_time ?? null, + timePerPassedTask: data.timing?.time_per_passed_task ?? null, }; } diff --git a/evalboard/lib/timing.ts b/evalboard/lib/timing.ts new file mode 100644 index 00000000..31031c5c --- /dev/null +++ b/evalboard/lib/timing.ts @@ -0,0 +1,121 @@ +// Wall-clock efficiency: how a task's duration compares to the time it is +// expected to take. +// +// `expected_seconds` is derived per task, per harness by the eval runner (p10 of +// past successful durations) and stamped into run.json. The dashboard reads that +// stamp rather than deriving its own, so a number here always matches the line the +// run was actually scored against. +// +// A task with no `expected_seconds` is *unscored*, not "within budget": too young +// for history, or a run that predates stamping. Every helper returns null there. + +export interface TimeRatioThresholds { + yellow: number; + red: number; +} + +export function getTimeRatioThresholds(): TimeRatioThresholds { + const parse = (raw: string | undefined, fallback: number) => { + if (!raw) return fallback; + const n = Number(raw); + return Number.isFinite(n) && n > 0 ? n : fallback; + }; + return { + yellow: parse(process.env.EVALBOARD_TIME_YELLOW_RATIO, 1.5), + red: parse(process.env.EVALBOARD_TIME_RED_RATIO, 2), + }; +} + +export type TimeTint = "green" | "yellow" | "red" | null; + +// Pure time-efficiency ratio (seconds ÷ expected_seconds), used to tint per-task +// duration cells. Blind to pass/fail on purpose: the cell answers "did this take +// longer than it should?", which holds either way. The aggregates diverge +// deliberately and score passes only (withinExpectedTime, overview.ts). +export function timeRatio( + durationSeconds: number | null, + expectedSeconds: number | null, +): number | null { + if ( + durationSeconds == null || + expectedSeconds == null || + expectedSeconds <= 0 + ) { + return null; + } + return durationSeconds / expectedSeconds; +} + +export function tintForTimeRatio( + ratio: number | null, + t: TimeRatioThresholds = getTimeRatioThresholds(), +): TimeTint { + if (ratio == null) return null; + if (ratio > t.red) return "red"; + if (ratio > t.yellow) return "yellow"; + return "green"; +} + +export function timeCellClasses(tint: TimeTint): string { + switch (tint) { + case "green": + return "text-emerald-700"; + case "yellow": + return "text-amber-700"; + case "red": + return "text-rose-700"; + default: + return "text-gray-900"; + } +} + +// A task counts as within its expected time while it stays inside +// (1 + tolerance) × expected, so anything past 2× its line reads slow. Mirrors +// `timing.TOLERANCE` on the runner side, which records the value it used in each +// run's `timing` block. Red tints at the same 2×, so the cell and the rollup agree. +export const TIME_BUDGET_TOLERANCE = 1; + +// Whether a task came in at or under (1 + tolerance) × its expected time. +// Null when the task is not scoreable: no duration, or no positive +// `expected_seconds`. +export function withinExpectedTime( + durationSeconds: number | null, + expectedSeconds: number | null, + tolerance: number = TIME_BUDGET_TOLERANCE, +): boolean | null { + const ratio = timeRatio(durationSeconds, expectedSeconds); + if (ratio == null) return null; + return ratio <= 1 + tolerance; +} + +// Per-task wall clock, to the second: `3m14s`, `1h02m` past the hour. Seconds +// are the point of the metric, so they are never rounded away below an hour. +export function fmtTaskSeconds(seconds: number | null): string { + if (seconds == null) return "—"; + const s = Math.round(seconds); + const h = Math.floor(s / 3600); + const m = Math.floor((s % 3600) / 60); + const sec = s % 60; + return h + ? `${h}h${String(m).padStart(2, "0")}m` + : `${m}m${String(sec).padStart(2, "0")}s`; +} + +export function fmtTimeRatio(ratio: number | null): string { + return ratio == null ? "—" : `${ratio.toFixed(2)}x expected`; +} + +// Ratio as a table cell: `1.8×`, or an em dash for an unscored task. One decimal, +// not two: the baseline is a min over a handful of runs (p10 over ten), and a +// task's own night-to-night spread is wider than the digit a second decimal adds. +export function fmtTimeRatioCell(ratio: number | null): string { + return ratio == null ? "—" : `${ratio.toFixed(1)}×`; +} + +// Title text for a duration cell: what the task was measured against, or why it +// was not measured at all. +export function expectedTimeTitle(expectedSeconds: number | null): string { + return expectedSeconds != null + ? `expected time: ${fmtTaskSeconds(expectedSeconds)}` + : "no expected time yet (needs a passing run on this harness)"; +} diff --git a/evalboard/lib/trends.ts b/evalboard/lib/trends.ts index 2469874f..24128a8e 100644 --- a/evalboard/lib/trends.ts +++ b/evalboard/lib/trends.ts @@ -60,6 +60,7 @@ export interface TaskHistoryEntry { actualCommands: number | null; totalTurns: number | null; expectedTurns: number | null; + expectedSeconds: number | null; hasFinalReply: boolean; componentShas: ComponentSha[]; failureTags: string[]; @@ -307,6 +308,7 @@ export async function historyForTaskInner( actualCommands: t.actualCommands, totalTurns: t.totalTurns, expectedTurns: t.expectedTurns, + expectedSeconds: t.expectedSeconds, hasFinalReply: t.hasFinalReply, componentShas: overview.componentShas, failureTags: reviewTagsByTask[taskId] ?? [], diff --git a/evalboard/lib/watchlist.ts b/evalboard/lib/watchlist.ts index d9dafbdc..ae215881 100644 --- a/evalboard/lib/watchlist.ts +++ b/evalboard/lib/watchlist.ts @@ -11,6 +11,7 @@ import type { PerRun } from "./overview"; import type { RunOverviewTask } from "./runs"; +import { timeRatio } from "./timing"; import { turnRatio } from "./turns"; export const FAIL_WEIGHT = 50; @@ -62,6 +63,12 @@ export interface TurnOverageRow { avgTurns: number; avgExpected: number; } +export interface TimeOverageRow { + skill: string; + avgTimeRatio: number; + avgSeconds: number; + avgExpectedSeconds: number; +} export interface WatchlistData { windowSize: number; topAttention: AttentionRow[]; @@ -70,6 +77,13 @@ export interface WatchlistData { streaks: StreakRow[]; volatility: VolatilityRow[]; turnOverage: TurnOverageRow[]; + // Wall-clock sibling of turnOverage, reported alongside it while both + // efficiency signals run. Deliberately NOT folded into the attention score: + // the 50/30/20 fail/regression/turn split stays put so the ranking does not + // move while the derived expected-time line is still being watched. When the + // turn budget goes, attention() switches its third term to this ratio and + // this becomes the only overage list. + timeOverage: TimeOverageRow[]; } // Runs newest-first, dropping any with a null overview. @@ -361,6 +375,45 @@ export function turnOverage(runs: LoadedRun[]): TurnOverageRow[] { ); } +// Skills whose passing tasks run past the wall clock they are expected to need. +// Same shape as turnOverage, over `expected_seconds` (derived per task per +// harness by the eval runner) instead of a hand-written turn budget. +export function timeOverage(runs: LoadedRun[]): TimeOverageRow[] { + const ratios = new Map(); + const seconds = new Map(); + const expected = new Map(); + const push = (m: Map, k: string, v: number) => { + const arr = m.get(k); + if (arr) arr.push(v); + else m.set(k, [v]); + }; + for (const run of runs) { + for (const t of run.tasks) { + if (!t.skill) continue; + const r = timeRatio(t.durationSeconds, t.expectedSeconds); + if (r == null) continue; + push(ratios, t.skill, r); + push(seconds, t.skill, t.durationSeconds!); + push(expected, t.skill, t.expectedSeconds!); + } + } + const rows: TimeOverageRow[] = []; + for (const [skill, rs] of ratios) { + const avgTimeRatio = mean(rs); + if (avgTimeRatio <= 1) continue; + rows.push({ + skill, + avgTimeRatio, + avgSeconds: mean(seconds.get(skill)!), + avgExpectedSeconds: mean(expected.get(skill)!), + }); + } + return rows.sort( + (a, b) => + b.avgTimeRatio - a.avgTimeRatio || a.skill.localeCompare(b.skill), + ); +} + export function buildWatchlist(perRun: PerRun[]): WatchlistData { const runs = runsNewestFirst(perRun); return { @@ -371,5 +424,6 @@ export function buildWatchlist(perRun: PerRun[]): WatchlistData { streaks: streaks(runs), volatility: volatility(runs), turnOverage: turnOverage(runs), + timeOverage: timeOverage(runs), }; } diff --git a/evalboard/vitest.setup.ts b/evalboard/vitest.setup.ts index f149f27a..3a4b10d6 100644 --- a/evalboard/vitest.setup.ts +++ b/evalboard/vitest.setup.ts @@ -1 +1,22 @@ import "@testing-library/jest-dom/vitest"; + +// jsdom has no ResizeObserver, and recharts' ResponsiveContainer constructs one +// on mount — without this stub any test that renders a chart throws. jsdom also +// reports every element as 0x0, which makes recharts log a "width(0) and +// height(0)" warning and skip drawing, so the stub reports a fixed size instead. +// Chart tests assert on headings, legends and tab state, never on plotted +// geometry, so the exact numbers only need to be non-zero. +if (!("ResizeObserver" in globalThis)) { + const RECT = { width: 800, height: 300, top: 0, left: 0, bottom: 300, right: 800, x: 0, y: 0 }; + globalThis.ResizeObserver = class { + constructor(private cb: ResizeObserverCallback) {} + observe(target: Element) { + this.cb( + [{ target, contentRect: RECT } as unknown as ResizeObserverEntry], + this as unknown as ResizeObserver, + ); + } + unobserve() {} + disconnect() {} + } as unknown as typeof ResizeObserver; +}