diff --git a/docs/content/en/guide/learn/reports-and-comparisons.md b/docs/content/en/guide/learn/reports-and-comparisons.md index 15ae4c6..8bf4e17 100644 --- a/docs/content/en/guide/learn/reports-and-comparisons.md +++ b/docs/content/en/guide/learn/reports-and-comparisons.md @@ -95,6 +95,14 @@ vieval report attempts --config vieval.config.ts `attempts` lists the attempts in a report root, newest first: per experiment and attempt, the workspaces, the passed, failed, and no-result case counts, and the completion rate. "←" marks the attempt that `vieval run --resume` continues without an id; notes show which attempt an attempt resumed, interrupts, and runs killed before they wrote a summary. Pass a report root, or `--config` to list the reports next to it; filter with `--experiment` and `--workspace`, and choose `--format table|json`. +## Share reports + +```bash [Terminal] +vieval report export .vieval/reports --output report-site +``` + +Deploy `report-site` to a static host, then share the URL. + ## Compare two existing report sets The report comparison command requires a left baseline and a right candidate: diff --git a/docs/content/zh-hans/guide/learn/reports-and-comparisons.md b/docs/content/zh-hans/guide/learn/reports-and-comparisons.md index d5b81d9..1012acd 100644 --- a/docs/content/zh-hans/guide/learn/reports-and-comparisons.md +++ b/docs/content/zh-hans/guide/learn/reports-and-comparisons.md @@ -117,6 +117,14 @@ vieval report attempts --config vieval.config.ts `attempts` 按时间从新到旧列出报告目录中的尝试:每个实验和尝试一行,显示涉及的工作区、通过、失败和没有结果的用例数,以及执行成功率。"←" 标记 `vieval run --resume` 不指定标识时会继续的那次尝试;备注列显示该尝试继续自哪次尝试、是否被中断,以及没来得及写入摘要就被终止的运行。可以传入报告目录,或用 `--config` 列出配置文件旁边的报告;用 `--experiment` 和 `--workspace` 筛选,用 `--format table|json` 选择输出格式。 +## 分享报告 + +```bash [终端] +vieval report export .vieval/reports --output report-site +``` + +将 `report-site` 部署到静态托管服务后,即可分享链接。 + ## 比较两组已有报告 比较命令的第一个目录是基线,第二个目录是候选方案: diff --git a/packages/ui/README.md b/packages/ui/README.md index e835204..fd4517f 100644 --- a/packages/ui/README.md +++ b/packages/ui/README.md @@ -50,6 +50,16 @@ vieval viewer reports-a reports-b --evidence-root /data/episodes --stale-after 3 | `--evidence-root ` | — | Extra directory that `*.evidence` metric paths may point into (repeatable). The `.vieval` directory above each report root is always allowed. | | `--stale-after ` | `60` | A run without an end event whose last event is older is shown as `stale`. | +## Share a static report + +```sh +vieval report export .vieval/reports --output report-site +``` + +Export requires Node.js 22.22 or later and a new output directory. Serve that directory over HTTP and share the URL of +a run, case, or comparison. Repeat `--run ` to select runs. Attachments are included by default. Add +`--include-evidence` to publish linked external evidence and `--evidence-root ` to allow an additional directory. + ## Programmatic use ```ts diff --git a/packages/ui/client/src/api.ts b/packages/ui/client/src/api.ts index 73efb91..6d9c83c 100644 --- a/packages/ui/client/src/api.ts +++ b/packages/ui/client/src/api.ts @@ -1,8 +1,6 @@ import type { - ArtifactDescriptor, CaseDetail, ComparisonResponse, - EvidenceLink, ExternalLogsResponse, FileTreeResponse, ImagePreview, @@ -18,22 +16,17 @@ import type { RunSnapshot, ScoresResponse, ScreenshotTimeline, - TaskNode, TimelineResponse, - TreeResponse, } from '../../src/shared/contract' -/** - * Error of one viewer API request. - * - * `status` is the HTTP status, or `0` when the server could not be reached at all. - */ -export class ApiError extends Error { - constructor(readonly status: number, message: string) { - super(message) - this.name = 'ApiError' - } -} +import { ApiError } from './apiError' +import { openPublishedReport } from './publishedReport' + +export { ApiError } from './apiError' + +/** Whether the Viewer opened a static report publication. */ +export const isPublishedReport = document.querySelector('meta[name="vieval-report"][content="static"]') != null +const published = isPublishedReport ? openPublishedReport(new URL('./', document.baseURI)) : undefined /** Base of every API route; the UI is served by the same server, so it is relative to the page. */ const base = new URL('api/', document.baseURI) @@ -73,11 +66,9 @@ function url(path: string, query: Record = /** * Typed client of the `@vieval/ui` HTTP API (contract in `API.md` and `src/shared/contract.ts`). */ -export const api = { - artifacts: (runKey: string, ref: Partial) => get(runPath(runKey, '/artifacts'), ref), +const live = { case: (runKey: string, ref: CaseRef) => get(runPath(runKey, '/case'), { ...ref }), compare: (view: 'attempts' | 'experiments' | 'runs', query: Record) => get(`compare/${view}`, query), - evidence: (runKey: string, ref: Partial) => get(runPath(runKey, '/evidence'), ref), evidenceTimeline: (runKey: string, linkId: string) => get(runPath(runKey, `/evidence/${encodeURIComponent(linkId)}/timeline`)), evidenceTree: (runKey: string, linkId: string) => get(runPath(runKey, `/evidence/${encodeURIComponent(linkId)}/tree`)), externalLogs: (runKey: string, ref: CaseRef, offset: number, limit: number) => get(runPath(runKey, '/case/external-logs'), { ...ref, limit, offset }), @@ -95,11 +86,12 @@ export const api = { runs: () => get('runs'), scores: (runKey: string) => get(runPath(runKey, '/scores')), snapshot: (runKey: string) => get(runPath(runKey, '/snapshot')), - task: (runKey: string, taskId: string) => get(runPath(runKey, '/task'), { taskId }), timeline: (runKey: string, ref: Partial) => get(runPath(runKey, '/timeline'), ref), - tree: () => get('tree'), } +/** Read operations of the Viewer, backed by the server or a static publication. */ +export const api: typeof live = published ?? live + /** * Where one previewable file is read from: a run artifact or a file of an evidence directory. * @@ -118,6 +110,8 @@ export interface FileLocation { * placeholder then. */ export function imagePreview(location: FileLocation): Promise { + if (published != null) + return Promise.reject(new ApiError(415, 'Image placeholders are generated by the live Viewer.')) const route = location.linkId == null ? runPath(location.runKey, '/artifacts/preview') : runPath(location.runKey, `/evidence/${encodeURIComponent(location.linkId)}/preview`) @@ -126,6 +120,8 @@ export function imagePreview(location: FileLocation): Promise { /** Reads `limit` complete lines of a JSONL file from line `offset`. */ export function jsonlPage(location: FileLocation, offset: number, limit: number): Promise { + if (published != null) + return published.jsonl(location, offset, limit) const route = location.linkId == null ? runPath(location.runKey, '/artifacts/jsonl') : runPath(location.runKey, `/evidence/${encodeURIComponent(location.linkId)}/jsonl`) @@ -139,6 +135,8 @@ export function openRunStream(runKey: string, after: number): EventSource { /** URL of the file's bytes; `download` adds `Content-Disposition: attachment`. */ export function rawUrl(location: FileLocation, download = false): string { + if (published != null) + return published.rawUrl(location) const route = location.linkId == null ? runPath(location.runKey, '/artifacts/raw') : runPath(location.runKey, `/evidence/${encodeURIComponent(location.linkId)}/raw`) @@ -165,9 +163,10 @@ export async function readText(location: FileLocation, maxBytes: number): Promis const body = await response.json().catch(() => undefined) as undefined | { message?: string } throw new ApiError(response.status, body?.message ?? `${response.status} ${response.statusText}`) } - const text = await response.text() + const bytes = new Uint8Array(await response.arrayBuffer()) + const text = new TextDecoder().decode(bytes.subarray(0, maxBytes)) // `Content-Range: bytes 0-1023/52341` carries the full size of a partial response. const total = Number(response.headers.get('content-range')?.split('/')[1] ?? Number.NaN) - const size = Number.isFinite(total) ? total : text.length - return { text, total: size, truncated: response.status === 206 && size > maxBytes } + const size = Number.isFinite(total) ? total : bytes.length + return { text, total: size, truncated: size > maxBytes } } diff --git a/packages/ui/client/src/apiError.ts b/packages/ui/client/src/apiError.ts new file mode 100644 index 0000000..33c000c --- /dev/null +++ b/packages/ui/client/src/apiError.ts @@ -0,0 +1,7 @@ +/** A Viewer request failure, with 0 for an unreachable data source. */ +export class ApiError extends Error { + constructor(readonly status: number, message: string) { + super(message) + this.name = 'ApiError' + } +} diff --git a/packages/ui/client/src/composables/useCatalog.ts b/packages/ui/client/src/composables/useCatalog.ts index 62d75ad..b27c0e0 100644 --- a/packages/ui/client/src/composables/useCatalog.ts +++ b/packages/ui/client/src/composables/useCatalog.ts @@ -7,7 +7,7 @@ import { defineQuery, useQuery } from '@pinia/colada' import { useDocumentVisibility, useIntervalFn } from '@vueuse/core' import { computed, watch } from 'vue' -import { api } from '../api' +import { api, isPublishedReport } from '../api' import { mergeMatrices } from '../labels' import { headlineValue } from '../scores' @@ -124,9 +124,11 @@ export function sumCounts(runs: readonly RunListItem[]): CaseCounts { */ export const useCatalog = defineQuery((): Catalog => { const query = useQuery({ key: RUNS_KEY, query: () => api.runs() }) - const polling = useIntervalFn(() => void query.refetch(), REFRESH_MS) + const polling = useIntervalFn(() => void query.refetch(), REFRESH_MS, { immediate: !isPublishedReport }) // A hidden tab does not poll; it re-reads the runs once it is shown again. watch(useDocumentVisibility(), (visibility) => { + if (isPublishedReport) + return if (visibility === 'hidden') { polling.pause() return diff --git a/packages/ui/client/src/publishedReport.ts b/packages/ui/client/src/publishedReport.ts new file mode 100644 index 0000000..692467a --- /dev/null +++ b/packages/ui/client/src/publishedReport.ts @@ -0,0 +1,196 @@ +import type { ExternalLogsResponse, JsonlPage, RunListItem } from '../../src/shared/contract' +import type { PublishedCatalog, PublishedComparison, PublishedRun } from '../../src/shared/publishedReport' +import type { CaseRef, FileLocation } from './api' + +import { buildMatrix, compareAttempts, compareExperiments, compareRuns, inScope } from '../../src/shared/compare' +import { buildLogs } from '../../src/shared/logs' +import { CATALOG_PATH, fileKey, runDataPath, telemetryPath } from '../../src/shared/publishedReport' +import { ApiError } from './apiError' + +/** + * Reads a static Viewer publication using paths relative to the site. + * + * Use when: + * - the Viewer is served alongside a Nitro-generated catalog and run files + * + * Returns: + * - the read operations used by the Viewer, with data cached for the life of the page + */ +export function openPublishedReport(base: URL) { + const cache = new Map>() + const runs = new Map() + const lines = new Map>() + async function read(path: string): Promise { + let pending = cache.get(path) + if (pending == null) { + pending = fetch(new URL(path, base)).then(async (response) => { + if (!response.ok) + throw new ApiError(response.status, 'This resource was not included in the published report.') + return response.json() + }).catch((error) => { + cache.delete(path) + throw error + }) + cache.set(path, pending) + } + return await pending as T + } + + async function catalog(): Promise { + const data = await read(CATALOG_PATH) + if (data.version !== 1) + throw new ApiError(422, 'This report needs the Viewer that was published with it.') + return data.runs + } + + function indexOf(items: RunListItem[], runKey: string): number { + const index = items.findIndex(item => item.runKey === runKey) + if (index < 0) + throw new ApiError(404, 'This run was not included in the published report.') + return index + } + + async function run(runKey: string): Promise { + const data = await read(runDataPath(indexOf(await catalog(), runKey), 'run')) + runs.set(runKey, data) + return data + } + + async function task(runKey: string, taskId: string) { + const data = await run(runKey) + if (!Object.hasOwn(data.tasks, taskId)) + throw new ApiError(404, 'Task not found.') + return data.tasks[taskId] + } + + async function caseData(runKey: string, ref: CaseRef) { + const { cases } = await task(runKey, ref.taskId) + if (!Object.hasOwn(cases, ref.caseId)) + throw new ApiError(404, 'Case not found.') + return cases[ref.caseId] + } + + async function comparison(item: RunListItem, index: number) { + const data = await read(runDataPath(index, 'comparison')) + return { ...data, item } + } + + async function comparisonsInScope(scope: Parameters[1]) { + return Promise.all((await catalog()).flatMap((item, index) => inScope(item, scope) ? [comparison(item, index)] : [])) + } + + function rawUrl(location: FileLocation): string { + const path = runs.get(location.runKey)?.files[fileKey(location.path, location.linkId)] + if (path == null) + throw new ApiError(404, 'This file was not included in the published report.') + return new URL(path, base).href + } + + return { + async case(runKey: string, ref: CaseRef) { + return (await caseData(runKey, ref)).detail + }, + async compare(view: 'attempts' | 'experiments' | 'runs', query: Record) { + if (view === 'runs') { + const keys = (query.runs ?? '').split(',').map(key => key.trim()).filter(Boolean) + if (keys.length < 2) + throw new ApiError(400, 'Choose at least two runs to compare.') + const items = await catalog() + const indexes = keys.map(key => indexOf(items, key)) + return compareRuns(await Promise.all(indexes.map(index => comparison(items[index], index)))) + } + const scope = { experiment: query.experiment ?? '', project: query.project ?? '', workspace: query.workspace ?? '' } + const filter = view === 'attempts' ? scope : { project: scope.project, workspace: scope.workspace } + const data = await comparisonsInScope(filter) + return view === 'attempts' ? compareAttempts(data, scope) : compareExperiments(data, scope) + }, + async evidenceTimeline(runKey: string, linkId: string) { + return (await run(runKey)).evidenceViews[linkId]?.timeline ?? { frames: [] } + }, + async evidenceTree(runKey: string, linkId: string) { + const view = (await run(runKey)).evidenceViews[linkId] + if (view == null) + throw new ApiError(404, 'This evidence directory was not included in the published report.') + return view.tree + }, + async externalLogs(runKey: string, ref: CaseRef, offset: number, limit: number): Promise { + const data = (await caseData(runKey, ref)).externalLogs ?? { correlation: { keys: [], podNames: [], window: {} }, files: [], lines: [], nextOffset: null, total: 0 } + const end = offset + limit + return { ...data, lines: data.lines.slice(offset, end), nextOffset: end < data.lines.length ? end : null } + }, + async jsonl(location: FileLocation, offset: number, limit: number): Promise { + const url = rawUrl(location) + let pending = lines.get(url) + if (pending == null) { + pending = fetch(url).then(async (response) => { + if (!response.ok) + throw new ApiError(response.status, 'Cannot read the published file.') + const rows = (await response.text()).split('\n') + const partialTail = rows.pop() !== '' + return { + partialTail, + rows: rows.map((row) => { + if (row.trim().length === 0) + return undefined + try { + return JSON.parse(row) as unknown + } + catch { + return { $error: 'Invalid JSON', $raw: row } + } + }), + } + }).catch((error) => { + lines.delete(url) + throw error + }) + lines.set(url, pending) + } + const data = await pending + return { limit, offset, partialTail: data.partialTail, rows: data.rows.slice(offset, offset + limit).filter(row => row !== undefined), totalLines: data.rows.length } + }, + async logs(runKey: string, query: Parameters[1]) { + return buildLogs((await run(runKey)).logs, query) + }, + async matrix(scope: Parameters[1]) { + return buildMatrix(await comparisonsInScope(scope)) + }, + async metrics(runKey: string, ref: Partial = {}) { + return ref.taskId == null ? (await run(runKey)).metrics : (await task(runKey, ref.taskId)).metrics + }, + async otelLogs(runKey: string) { + return (await run(runKey)).otel.logs + }, + async otelMetrics(runKey: string) { + return (await run(runKey)).otel.metrics + }, + async otelRaw(runKey: string, signal: 'logs' | 'metrics' | 'traces') { + return read(telemetryPath(indexOf(await catalog(), runKey), signal)) + }, + async otelTraces(runKey: string) { + return (await run(runKey)).otel.traces + }, + async progress(runKey: string) { + return (await run(runKey)).progress + }, + rawUrl, + async run(runKey: string) { + const items = await catalog() + return items[indexOf(items, runKey)] + }, + runs: catalog, + async scores(runKey: string) { + return (await run(runKey)).scores + }, + async snapshot(runKey: string) { + return (await run(runKey)).snapshot + }, + async timeline(runKey: string, ref: Partial) { + if (ref.taskId == null) + return (await run(runKey)).timeline + return ref.caseId == null + ? (await task(runKey, ref.taskId)).timeline + : (await caseData(runKey, { caseId: ref.caseId, taskId: ref.taskId })).timeline + }, + } +} diff --git a/packages/ui/package.json b/packages/ui/package.json index 22c2eac..b934378 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -54,6 +54,8 @@ "@napi-rs/image": "catalog:", "es-toolkit": "catalog:", "h3": "catalog:", + "nitro": "catalog:", + "rollup": "catalog:", "srvx": "catalog:", "thumbhash": "catalog:", "tinyglobby": "catalog:", diff --git a/packages/ui/src/index.ts b/packages/ui/src/index.ts index 656b576..a53a9c8 100644 --- a/packages/ui/src/index.ts +++ b/packages/ui/src/index.ts @@ -12,6 +12,7 @@ import { createViewerApp } from './server/app' import { createEvidenceService } from './server/evidence' import { createRunStore } from './server/store' +export { exportReport } from './publishReport' export type { CreateViewerAppOptions } from './server/app' export { createViewerApp } from './server/app' export { createEventTail, parseEventLine } from './server/event-tail' diff --git a/packages/ui/src/publishReport.test.ts b/packages/ui/src/publishReport.test.ts new file mode 100644 index 0000000..ff82aad --- /dev/null +++ b/packages/ui/src/publishReport.test.ts @@ -0,0 +1,105 @@ +import type { CaseDetail } from './shared/contract' +import type { PublishedCatalog, PublishedComparison, PublishedRun } from './shared/publishedReport' + +import { appendFile, mkdir, readFile, rename, symlink, writeFile } from 'node:fs/promises' +import { basename, join } from 'node:path' + +import { afterEach, describe, expect, it } from 'vitest' + +import { cleanupTemporaryDirectories, copyOsworldFixture, createTemporaryDirectory, createTestApp, OSWORLD_RUNS, OSWORLD_TASK_ID, readRunningContinuation } from '../test/helpers' +import { exportReport } from './publishReport' +import { normalizePathSegment } from './server/views' +import { compareRuns } from './shared/compare' +import { CATALOG_PATH, fileKey, runDataPath, telemetryPath } from './shared/publishedReport' + +afterEach(cleanupTemporaryDirectories) + +/** @example vieval report export .vieval/reports --output report-site */ +describe('static report publication', () => { + // https://github.com/vieval-dev/vieval/issues/44 + /** @example expect(report.tasks[taskId].cases[caseId].detail.scoreRecords).toHaveLength(1) */ + it('issue #44 publishes case evidence and comparable run data independently of the source', async () => { + const { reportRoot, vievalDirectory } = await copyOsworldFixture() + const directory = await createTemporaryDirectory() + const output = join(directory, 'site') + const archiveKey = basename(OSWORLD_RUNS.archived) + const candidateKey = basename(OSWORLD_RUNS.running) + const caseId = '1:chrome%2F06fe7178-4491-4589-810f-2e2bc9502122' + const artifacts = join(reportRoot, OSWORLD_RUNS.archived, 'artifacts') + const artifactDir = join(artifacts, normalizePathSegment(OSWORLD_TASK_ID), normalizePathSegment(caseId)) + await mkdir(artifactDir, { recursive: true }) + await writeFile(join(artifactDir, 'preview.html'), 'Recorded output') + await writeFile(join(artifactDir, 'diagram.svg'), '') + await writeFile(join(artifactDir, 'recording.opus'), 'recorded audio') + await writeFile(join(artifactDir, 'trace.unknown'), 'unrecognized format') + const privateFile = join(directory, 'outside.txt') + await writeFile(privateFile, 'outside the artifact directory') + await symlink(privateFile, join(artifactDir, 'outside.txt')) + const candidateOtlp = join(reportRoot, OSWORLD_RUNS.running, 'otlp') + await mkdir(candidateOtlp) + await symlink(privateFile, join(candidateOtlp, 'logs.json')) + + // Finish the second fixture so comparisons exercise two different sets of results. + const continuation = await readRunningContinuation(vievalDirectory) + const last = JSON.parse(continuation.at(-1)!) as { seq: number } + const ended = { ...last, data: { failedTasks: 0, passedTasks: 0, skippedTasks: 0, totalTasks: 1 }, event: 'run.ended', seq: last.seq + 1, time: '2026-10-11T20:00:00.000Z' } + await appendFile(join(reportRoot, OSWORLD_RUNS.running, 'events.jsonl'), `${continuation.join('\n')}\n${JSON.stringify(ended)}\n`) + const live = createTestApp({ reportRoots: [reportRoot], vievalDirectory }) + const expected = await live.json(`/api/compare/runs?runs=${archiveKey},${candidateKey}`) + const caseQuery = new URLSearchParams({ caseId, taskId: OSWORLD_TASK_ID }) + const localDetail = await live.json(`/api/runs/${archiveKey}/case?${caseQuery}`) + const localTimeline = await live.json(`/api/runs/${archiveKey}/timeline?${caseQuery}`) + live.store.close() + + const result = await exportReport({ includeEvidence: true, output, reportRoots: [reportRoot] }) + expect(result.runKeys).toEqual([candidateKey, archiveKey]) + await rename(vievalDirectory, join(directory, 'original-reports')) + + const catalog = JSON.parse(await readFile(join(output, CATALOG_PATH), 'utf8')) as PublishedCatalog + const archiveIndex = catalog.runs.findIndex(run => run.runKey === archiveKey) + const candidateIndex = catalog.runs.findIndex(run => run.runKey === candidateKey) + await expect(readFile(join(output, telemetryPath(candidateIndex, 'logs')))).rejects.toMatchObject({ code: 'ENOENT' }) + expect(catalog.runs[archiveIndex].reportRoot).toBeNull() + expect(await readFile(join(output, 'index.html'), 'utf8')).toContain('name="vieval-report" content="static"') + const report = JSON.parse(await readFile(join(output, runDataPath(archiveIndex, 'run')), 'utf8')) as PublishedRun + const { detail, externalLogs, timeline } = report.tasks[OSWORLD_TASK_ID].cases[caseId] + // Publishing changes file references; the report data comes from the same builders as the Viewer API. + expect(detail).toEqual({ ...localDetail, artifacts: detail.artifacts, evidence: detail.evidence }) + expect(timeline).toEqual(localTimeline) + expect(detail.input).toMatchObject({ task: { id: '06fe7178-4491-4589-810f-2e2bc9502122' } }) + expect(detail.scoreRecords[0].record).toEqual({ name: 'osworld.success', value: 0 }) + expect(detail.artifacts.map(artifact => [ + artifact.name, + artifact.previewKind, + report.files[fileKey(artifact.path)].split('.').at(-1), + ]).sort()).toEqual([ + ['diagram.svg', 'binary', 'bin'], + ['preview.html', 'binary', 'bin'], + ['recording.opus', 'audio', 'opus'], + ['trace.unknown', 'binary', 'bin'], + ]) + expect(externalLogs?.lines).toHaveLength(24) + const [link] = detail.evidence + const frame = report.evidenceViews[link.id].timeline?.frames[0] + expect(frame?.contentType).toBe('image/webp') + expect((await readFile(join(output, report.files[fileKey(frame!.path, link.id)]))).length).toBeGreaterThan(0) + + const comparisonRuns = await Promise.all([archiveKey, candidateKey].map(async (runKey) => { + const index = catalog.runs.findIndex(run => run.runKey === runKey) + const comparison = JSON.parse(await readFile(join(output, runDataPath(index, 'comparison')), 'utf8')) as PublishedComparison + return { ...comparison, item: catalog.runs[index] } + })) + expect(JSON.parse(JSON.stringify(compareRuns(comparisonRuns)))).toEqual(expected) + }) + + /** @example expect(exportReport({ output, reportRoots: [reportRoot] })).rejects.toThrow(/already exists/) */ + it('preserves an existing destination and refuses unfinished runs', async () => { + const { reportRoot } = await copyOsworldFixture() + const output = await createTemporaryDirectory() + const marker = join(output, 'keep.txt') + await writeFile(marker, 'keep') + await expect(exportReport({ output, reportRoots: [reportRoot] })).rejects.toThrow(/already exists/) + expect(await readFile(marker, 'utf8')).toBe('keep') + await expect(exportReport({ output: join(output, 'site'), reportRoots: [reportRoot], runKeys: [basename(OSWORLD_RUNS.running)] })).rejects.toThrow(/not a supported completed report/) + }) +}) diff --git a/packages/ui/src/publishReport.ts b/packages/ui/src/publishReport.ts new file mode 100644 index 0000000..cdd094a --- /dev/null +++ b/packages/ui/src/publishReport.ts @@ -0,0 +1,141 @@ +import type { ExportReportOptions, ExportReportResult } from 'vieval/cli' + +import type { ReportPrerenderOptions } from './reportPrerender' +import type { PublishedCatalog, PublishedRun } from './shared/publishedReport' + +import { cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { isAllowedPath, isInside } from './server/files' +import { createRunStore, toRunListItem } from './server/store' +import { CATALOG_PATH, runDataPath, TELEMETRY_SIGNALS, telemetryPath } from './shared/publishedReport' + +/** + * Builds a static Viewer site for completed reports. + * + * Use when: + * - a publisher needs to deploy reports to a static HTTP host + * + * Expects: + * - the Viewer is built and the destination does not exist + * + * Returns: + * - the absolute output path, run keys, and count of omitted files + * + * Call stack: + * + * vieval report export + * -> {@link exportReport} + * -> Nitro prerender + * -> reportPrerender + */ +export async function exportReport(options: ExportReportOptions): Promise { + const output = resolve(options.output) + const reportRoots = options.reportRoots.map(root => resolve(root)) + const evidenceRoots = (options.evidenceRoots ?? []).map(root => resolve(root)) + const dist = fileURLToPath(new URL('../dist/', import.meta.url)) + const clientDir = join(dist, 'client') + const destination = await isAllowedPath([], output) + if (destination.exists) + throw new Error(`Export output "${output}" already exists.`) + for (const source of [...reportRoots, ...evidenceRoots, clientDir]) { + const root = await isAllowedPath([], source) + if (!root.exists) + throw new Error(`Export source "${source}" does not exist.`) + if (isInside(root.realPath, destination.realPath) || isInside(destination.realPath, root.realPath)) + throw new Error('Export output must not overlap report, evidence, or Viewer sources.') + } + + const at = Date.now() + let runs: PublishedCatalog['runs'] + const store = createRunStore({ reportRoots }) + try { + const completed = (await store.list()).filter((run) => { + const state = run.state(at) + return state === 'ended' || state === 'interrupted' + }) + const selected = options.runKeys?.length + ? [...new Set(options.runKeys)].map((key) => { + const run = completed.find(item => item.runKey === key) + if (run == null) + throw new Error(`Run "${key}" is not a supported completed report in the selected roots.`) + return run + }) + : completed + if (selected.length === 0) + throw new Error('No supported completed reports to export.') + runs = selected.map(run => ({ ...toRunListItem(run, at), reportPath: null, reportRoot: null })) + .sort((left, right) => (right.startedAt ?? '').localeCompare(left.startedAt ?? '')) + } + finally { + store.close() + } + + // Nitro owns a scratch build. An exclusive mkdir claims the user's destination only after the build succeeds. + const scratch = await mkdtemp(join(tmpdir(), 'vieval-export-')) + try { + const publicDir = join(scratch, 'site') + const inputs: ReportPrerenderOptions = { + at, + catalog: { runs, version: 1 }, + clientDir, + evidenceRoots, + includeEvidence: options.includeEvidence === true, + output: destination.realPath, + publicDir, + reportRoots, + } + // The worker gets explicit build inputs without reading the caller's project config or environment variables. + const serverEntry = join(scratch, 'server.mjs') + const handler = join(dist, 'reportPrerender.mjs') + await writeFile(serverEntry, `import { createReportPrerender } from ${JSON.stringify(handler)}\nexport default createReportPrerender(${JSON.stringify(inputs)})\n`) + const routes = ['/index.html', `/${CATALOG_PATH}`, ...runs.flatMap((_, index) => [ + `/${runDataPath(index, 'run')}`, + `/${runDataPath(index, 'comparison')}`, + ...TELEMETRY_SIGNALS.map(signal => `/${telemetryPath(index, signal)}`), + ])] + const { copyPublicAssets, createNitro, prepare, prerender } = await import('nitro/builder') + const nitro = await createNitro({ + builder: 'rollup', + // Resolve the shipped builder from this package, since the Nitro project lives in a temporary directory. + deps: { install: false, paths: { rollup: import.meta.resolve('rollup/package.json') } }, + output: { dir: join(scratch, 'output'), publicDir }, + prerender: { crawlLinks: false, failOnError: true, ignoreStatusCodes: [404], routes }, + preset: 'static', + publicAssets: [{ dir: clientDir, ignore: [join(clientDir, 'index.html')] }], + rootDir: scratch, + serverEntry, + }, { dotenv: false }) + try { + await prepare(nitro) + await copyPublicAssets(nitro) + await mkdir(join(publicDir, 'files'), { recursive: true }) + await prerender(nitro) + } + finally { + await nitro.close() + } + + let omittedFiles = 0 + for (const index of runs.keys()) { + const run = JSON.parse(await readFile(join(publicDir, runDataPath(index, 'run')), 'utf8')) as PublishedRun + omittedFiles += run.omitted.length + } + await mkdir(dirname(output), { recursive: true }) + await mkdir(output) + try { + for (const name of await readdir(publicDir)) + await cp(join(publicDir, name), join(output, name), { errorOnExist: true, force: false, recursive: true }) + } + catch (error) { + await rm(output, { force: true, recursive: true }) + throw error + } + return { omittedFiles, output, runKeys: runs.map(run => run.runKey) } + } + finally { + await rm(scratch, { force: true, recursive: true }) + } +} diff --git a/packages/ui/src/reportPrerender.ts b/packages/ui/src/reportPrerender.ts new file mode 100644 index 0000000..1393287 --- /dev/null +++ b/packages/ui/src/reportPrerender.ts @@ -0,0 +1,102 @@ +import type { PublishedCatalog, PublishedComparison } from './shared/publishedReport' + +import { readFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' + +import { H3 } from 'h3' + +import { projectComparisonRun } from './server/comparisonRun' +import { createEvidenceService } from './server/evidence' +import { isAllowedPath } from './server/files' +import { publishRun } from './server/publishRun' +import { createRunStore } from './server/store' +import { CATALOG_PATH, runDataPath, TELEMETRY_SIGNALS, telemetryPath } from './shared/publishedReport' + +/** Build-time inputs sent to the isolated Nitro prerenderer. */ +export interface ReportPrerenderOptions { + /** Fixed timestamp used to compute the report. */ + at: number + /** Selected runs, in publication order. */ + catalog: PublishedCatalog + /** Built Viewer directory. */ + clientDir: string + /** Additional allowed evidence roots. */ + evidenceRoots: string[] + /** Whether the reader may access linked evidence. */ + includeEvidence: boolean + /** Canonical destination, checked against linked evidence. */ + output: string + /** Scratch public output directory used by Nitro. */ + publicDir: string + /** Input report roots. */ + reportRoots: string[] +} + +/** + * Creates the H3 application that Nitro prerenders into a static publication. + * + * Use when: + * - Nitro loads the generated build entry for a selected set of runs + * + * Expects: + * - the Vue build and the selected runs remain available for the duration of prerendering + * + * Returns: + * - routes for the Viewer shell, report data, and optional raw telemetry + */ +export function createReportPrerender(options: ReportPrerenderOptions): H3 { + const store = createRunStore({ reportRoots: options.reportRoots }) + const reportDirectories = options.reportRoots.map(root => dirname(root)) + const evidence = createEvidenceService({ + evidenceRoots: [...reportDirectories, ...options.evidenceRoots], + logDirectories: options.includeEvidence ? reportDirectories.map(directory => join(directory, 'logs')) : [], + }) + + const app: H3 = new H3() + app.get('/index.html', async () => { + const html = await readFile(join(options.clientDir, 'index.html'), 'utf8') + return new Response(html.replace(/]*)?>/i, '$&'), { + headers: { 'content-type': 'text/html; charset=utf-8' }, + }) + }) + app.get(`/${CATALOG_PATH}`, () => options.catalog) + + for (const [index, item] of options.catalog.runs.entries()) { + // Resolve each selected run once within the worker. The route paths are independent of report IDs. + let resolved: ReturnType | undefined + const handle = () => resolved ??= store.get(item.runKey) + app.get(`/${runDataPath(index, 'run')}`, async () => { + const run = await handle() + if (run == null || ['running', 'stale', 'unsupported'].includes(run.state(options.at))) + throw new Error(`Run "${item.runKey}" is no longer a supported completed report.`) + return publishRun(run, evidence, options) + }) + app.get(`/${runDataPath(index, 'comparison')}`, async (): Promise => { + const run = await handle() + if (run == null) + throw new Error(`Run "${item.runKey}" no longer exists.`) + const { item: _item, ...comparison } = projectComparisonRun(run, options.at) + return comparison + }) + for (const signal of TELEMETRY_SIGNALS) { + app.get(`/${telemetryPath(index, signal)}`, async () => { + const run = await handle() + if (run?.reportDirectory == null) + return new Response(null, { status: 404 }) + const file = await isAllowedPath([run.reportDirectory], join(run.reportDirectory, 'otlp', `${signal}.json`)) + if (!file.allowed || !file.exists) + return new Response(null, { status: 404 }) + const data = await readFile(file.realPath, 'utf8').catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') + throw error + return undefined + }) + return data == null + ? new Response(null, { status: 404 }) + : new Response(data, { headers: { 'content-type': 'application/json' } }) + }) + } + } + + return app +} diff --git a/packages/ui/src/server/app.ts b/packages/ui/src/server/app.ts index cb1f363..1ffc7fc 100644 --- a/packages/ui/src/server/app.ts +++ b/packages/ui/src/server/app.ts @@ -1,6 +1,6 @@ import type { H3Event } from 'h3' -import type { EventsResponse, RunEvent, RunListItem, TreeResponse } from '../shared/contract' +import type { EventsResponse, RunEvent, TreeResponse } from '../shared/contract' import type { EvidenceService } from './evidence' import type { RunHandle, RunStore } from './store' @@ -10,15 +10,16 @@ import { join } from 'node:path' import { errorMessageFrom } from '@moeru/std' import { getQuery, getRouterParam, H3 } from 'h3' +import { buildMatrix, compareAttempts, compareExperiments, compareRuns, inScope } from '../shared/compare' import { VIEWER_API_VERSION } from '../shared/contract' -import { buildTaskNodes } from '../shared/reduce' -import { buildMatrix, compareAttempts, compareExperiments, compareRuns } from './compare' +import { buildLogs } from '../shared/logs' +import { projectComparisonRun } from './comparisonRun' import { ViewerError } from './errors' import { contentTypeFromName, listTree, readJsonlPage, resolveInsideRoot, serveFile } from './files' import { buildOtelLogs, buildOtelMetrics, buildOtelTraces } from './otel' import { createImagePreviews } from './preview' import { toRunListItem, UNSUPPORTED_RUN_MESSAGE } from './store' -import { buildCaseDetail, buildLogs, buildMetrics, buildProgress, buildScores, buildSnapshot, buildTask, buildTimeline, findCase, listArtifacts, MAX_LOG_LIMIT } from './views' +import { buildCaseDetail, buildMetrics, buildProgress, buildScores, buildSnapshot, buildTask, buildTimeline, findCase, listArtifacts, MAX_LOG_LIMIT } from './views' /** Most events one `/events` page returns. */ const MAX_EVENTS_LIMIT = 5000 @@ -219,7 +220,7 @@ export function createViewerApp(options: CreateViewerAppOptions): H3 { } const handle = await runFrom(event) scopeFrom(event, handle) - return buildLogs(handle, { + return buildLogs(handle.reducer().logs, { caseId: query.caseId, limit: intParam(query.limit, 'limit', 500, 1, MAX_LOG_LIMIT), offset: intParam(query.offset, 'offset', 0, 0), @@ -339,26 +340,22 @@ export function createViewerApp(options: CreateViewerAppOptions): H3 { app.get('/api/matrix', async (event) => { const query = getQuery(event) as Record - const at = now() - const runs = (await store.list()).filter(handle => handle.unsupportedVersion() == null).map((handle) => { - // Built once and shared by the list item and the matrix: building task nodes walks every case. - const tasks = buildTaskNodes(handle.reducer(), () => 0) - return { item: toRunListItem(handle, at, tasks), scoreDefinitions: handle.reducer().scoreDefinitions, tasks } - }) - return buildMatrix(runs.filter(run => inScope(run.item, query))) + return buildMatrix(await comparisonsInScope(query)) }) app.get('/api/compare/experiments', async (event) => { const query = getQuery(event) as Record - return compareExperiments(await store.list(), { project: required(query.project, 'project'), workspace: required(query.workspace, 'workspace') }, now()) + const scope = { project: required(query.project, 'project'), workspace: required(query.workspace, 'workspace') } + return compareExperiments(await comparisonsInScope(scope), scope) }) app.get('/api/compare/attempts', async (event) => { const query = getQuery(event) as Record - return compareAttempts(await store.list(), { + const scope = { experiment: required(query.experiment, 'experiment'), project: required(query.project, 'project'), workspace: required(query.workspace, 'workspace'), - }, now()) + } + return compareAttempts(await comparisonsInScope(scope), scope) }) app.get('/api/compare/runs', async (event) => { @@ -374,13 +371,31 @@ export function createViewerApp(options: CreateViewerAppOptions): H3 { assertSupported(handle) return handle })) - return compareRuns(handles, now()) + const at = now() + return compareRuns(handles.map(handle => projectComparisonRun(handle, at))) }) app.all('/api/**', () => { throw new ViewerError(404, 'Unknown API route.') }) + async function comparisonsInScope(scope: { experiment?: string, project?: string, workspace?: string }) { + const at = now() + return (await store.list()).flatMap((handle) => { + if (handle.unsupportedVersion() != null) + return [] + const reducer = handle.reducer() + const identity = reducer.identity() + if (identity != null) { + return inScope({ ...identity, projectIds: [...reducer.projects.keys()], projectSegment: handle.projectSegment }, scope) + ? [projectComparisonRun(handle, at)] + : [] + } + const run = projectComparisonRun(handle, at) + return inScope(run.item, scope) ? [run] : [] + }) + } + app.get('/**', async (event) => { const directory = options.clientDirectory if (directory != null && existsSync(join(directory, 'index.html'))) { @@ -438,16 +453,6 @@ function hostnameOf(header: null | string): string | undefined { return header.split(':')[0].toLowerCase() } -/** - * Whether a run belongs to the hierarchy scope of the optional `workspace`, `project`, and `experiment` filters. - * A project matches the report path segment or any project the run executed (multi-project runs). - */ -function inScope(item: RunListItem, query: Record): boolean { - return (query.workspace == null || item.workspaceId === query.workspace) - && (query.project == null || item.projectSegment === query.project || item.projectIds.includes(query.project)) - && (query.experiment == null || item.experimentId === query.experiment) -} - function intParam(value: string | undefined, name: string, fallback: number, min: number, max = Number.MAX_SAFE_INTEGER): number { if (value == null || value === '') { return fallback diff --git a/packages/ui/src/server/compare.test.ts b/packages/ui/src/server/compare.test.ts index 829c5c9..f22e643 100644 --- a/packages/ui/src/server/compare.test.ts +++ b/packages/ui/src/server/compare.test.ts @@ -1,11 +1,12 @@ -import type { MetricDefinition, RunListItem, ScoreDefinition, TaskNode } from '../shared/contract' -import type { RunReducer } from '../shared/reduce' +import type { ComparisonRun } from '../shared/compare' +import type { MetricDefinition, RunListItem, ScoreDefinition } from '../shared/contract' import { describe, expect, it } from 'vitest' import { createEnvelopeFactory, createReducerHandle } from '../../test/helpers' -import { buildTaskNodes, createRunReducer } from '../shared/reduce' -import { buildMatrix, compareRuns } from './compare' +import { buildMatrix, compareRuns } from '../shared/compare' +import { createRunReducer } from '../shared/reduce' +import { projectComparisonRun } from './comparisonRun' const TASK = 'evals%2Fdemo::default::run=tts%3Dofficial::eval=asr%3Dwhisper-1' const TASK_WITH_JUDGE = 'evals%2Fdemo::default::run=tts%3Dofficial::eval=judge%3Dgpt-5.5' @@ -21,15 +22,8 @@ interface CaseResult { state: 'errored' | 'failed' | 'passed' } -interface TestRun { - item: RunListItem - reducer: RunReducer - scoreDefinitions: ReadonlyMap - tasks: TaskNode[] -} - /** One ended run of {@link TASK}: the given cases (named `-`) settle, out of `total` expected ones. */ -function runOf(runKey: string, startedAt: string, total: number, results: readonly CaseResult[], options: { definitions?: ScoreDefinition[], metricDefinitions?: MetricDefinition[], prefix?: string, taskId?: string, taskIds?: string[] } = {}): TestRun { +function runOf(runKey: string, startedAt: string, total: number, results: readonly CaseResult[], options: { definitions?: ScoreDefinition[], metricDefinitions?: MetricDefinition[], prefix?: string, taskId?: string, taskIds?: string[] } = {}): ComparisonRun { const { prefix = 'case', taskId = TASK } = options const envelope = createEnvelopeFactory(runKey) const reducer = createRunReducer() @@ -76,7 +70,8 @@ function runOf(runKey: string, startedAt: string, total: number, results: readon state: 'ended', workspaceId: 'ws-a', } - return { item, reducer, scoreDefinitions: reducer.scoreDefinitions, tasks: buildTaskNodes(reducer, () => 0) } + const run = projectComparisonRun(createReducerHandle(reducer), Date.parse(startedAt)) + return { ...run, item: { ...item, headline: run.item.headline } } } const scored = (count: number, value: number): CaseResult[] => Array.from({ length: count }, () => ({ scores: { [SCORE]: value }, state: 'passed' })) @@ -257,7 +252,7 @@ describe('compareRuns', () => { { scores: { [SCORE]: 0 }, state: 'failed' }, ], { definitions: [{ name: SCORE, primary: true }] }) - const comparison = compareRuns([createReducerHandle(left.reducer), createReducerHandle(right.reducer)], Date.parse('2026-10-20T00:00:00.000Z')) + const comparison = compareRuns([left, right]) expect(comparison.primary).toBe(SCORE) expect(comparison.rows.map(row => [row.caseName, row.changed])).toEqual([['case-0', false], ['case-1', true], ['case-2', false]]) expect(comparison.rows[0].cells['run-a']).toEqual({ counted: { 'demo.other': 0, [SCORE]: 1 }, durationMs: 3000, metrics: {}, roundCount: 1, scores: { unscored: {}, values: { 'demo.other': 0, [SCORE]: 1 } }, state: 'passed' }) @@ -273,7 +268,7 @@ describe('compareRuns', () => { const left = runOf('run-a', '2026-10-19T06:36:00.000Z', 1, scored(1, 1), { definitions: [{ grader: { kind: 'model', model: 'judge-a' }, name: SCORE, primary: true }] }) const right = runOf('run-b', '2026-10-19T07:00:00.000Z', 1, scored(1, 1), { definitions: [{ grader: { kind: 'model', model: 'judge-b' }, name: SCORE, primary: true }] }) - const comparison = compareRuns([createReducerHandle(left.reducer), createReducerHandle(right.reducer)], Date.parse('2026-10-20T00:00:00.000Z')) + const comparison = compareRuns([left, right]) expect(comparison.columns.map(column => column.scoreGraders)).toEqual([ { [SCORE]: [{ kind: 'model', model: 'judge-a' }] }, { [SCORE]: [{ kind: 'model', model: 'judge-b' }] }, @@ -288,7 +283,7 @@ describe('compareRuns', () => { const left = runOf('run-a', '2026-10-19T06:36:00.000Z', 1, [{ scores: { 'demo.a': 1 }, state: 'passed' }]) const right = runOf('run-b', '2026-10-19T07:00:00.000Z', 1, [{ scores: { 'demo.b': 0 }, state: 'passed' }]) - const comparison = compareRuns([createReducerHandle(left.reducer), createReducerHandle(right.reducer)], Date.parse('2026-10-20T00:00:00.000Z')) + const comparison = compareRuns([left, right]) expect(comparison.primary).toBeUndefined() expect(comparison.rows.map(row => row.changed)).toEqual([false]) }) @@ -313,7 +308,7 @@ describe('compareRuns', () => { const left = runOf('run-a', '2026-10-19T06:36:00.000Z', 1, scored(1, 1), { definitions: [{ name: SCORE, primary: true }], taskIds }) const right = runOf('run-b', '2026-10-19T07:00:00.000Z', 1, scored(1, 0.5), { definitions: [{ name: SCORE, primary: true }], taskIds }) - const comparison = compareRuns([createReducerHandle(left.reducer), createReducerHandle(right.reducer)], Date.parse('2026-10-20T00:00:00.000Z')) + const comparison = compareRuns([left, right]) expect(comparison.rows.map(row => [row.caseName, row.variant])).toEqual([['case-0', 'context=home'], ['case-0', 'context=none']]) expect(comparison.rows.map(row => row.cells['run-b']?.counted[SCORE])).toEqual([0.5, 0.5]) expect(comparison.summary.rows).toBe(2) @@ -337,7 +332,7 @@ describe('compareRuns', () => { const left = runOf('run-a', '2026-10-19T06:36:00.000Z', 1, scored(1, 1), { definitions: [{ name: SCORE, primary: true }], taskIds: shared }) const right = runOf('run-b', '2026-10-19T07:00:00.000Z', 1, scored(1, 0.5), { definitions: [{ name: SCORE, primary: true }], taskIds: withPrompt }) - const comparison = compareRuns([createReducerHandle(left.reducer), createReducerHandle(right.reducer)], Date.parse('2026-10-20T00:00:00.000Z')) + const comparison = compareRuns([left, right]) expect(comparison.rows.map(row => row.variant)).toEqual(['context=home', 'context=none']) expect(comparison.rows.map(row => [row.cells['run-a']?.pooled, row.cells['run-b']?.pooled])).toEqual([[undefined, 2], [undefined, 2]]) expect(comparison.rows.map(row => row.cells['run-b']?.counted[SCORE])).toEqual([0.5, 0.5]) @@ -352,7 +347,7 @@ describe('compareRuns', () => { metricDefinitions: [{ better: 'lower', name: 'demo.latency_ms', unit: 'ms' }], }) - const comparison = compareRuns([createReducerHandle(run.reducer)], Date.parse('2026-10-20T00:00:00.000Z')) + const comparison = compareRuns([run]) expect(comparison.rows[0].cells['run-a']?.metrics).toEqual({ 'demo.latency_ms': 120 }) expect(comparison.metricDefinitions).toEqual([{ better: 'lower', name: 'demo.latency_ms', unit: 'ms' }]) }) @@ -364,7 +359,7 @@ describe('compareRuns', () => { it('names the workspace and project of each compared run, so runs of several workspaces can be told apart', () => { const run = runOf('run-a', '2026-10-19T06:36:00.000Z', 1, scored(1, 1)) - const comparison = compareRuns([createReducerHandle(run.reducer)], Date.parse('2026-10-20T00:00:00.000Z')) + const comparison = compareRuns([run]) expect(comparison.columns[0].runs[0]?.project).toBe('demo') expect(comparison.columns[0].runs[0]?.workspaceId).toBe('ws-a') }) diff --git a/packages/ui/src/server/comparisonRun.ts b/packages/ui/src/server/comparisonRun.ts new file mode 100644 index 0000000..d3e68c0 --- /dev/null +++ b/packages/ui/src/server/comparisonRun.ts @@ -0,0 +1,62 @@ +import type { ComparisonRun } from '../shared/compare' +import type { ReducedCase } from '../shared/reduce' +import type { RunHandle } from './store' + +import { buildTaskNodes } from '../shared/reduce' +import { toRunListItem } from './store' + +/** + * Projects a run's reduced state into the data used by matrix and case comparisons. + * + * Use when: + * - a live request or a static export needs to compare this run + * + * Returns: + * - serializable task and case results with the definitions needed to score them + */ +export function projectComparisonRun(handle: RunHandle, now: number): ComparisonRun { + const reducer = handle.reducer() + const tasks = buildTaskNodes(reducer, () => 0) + return { + item: toRunListItem(handle, now, tasks), + metricDefinitions: [...reducer.metricDefinitions.values()], + scoreDefinitions: [...reducer.scoreDefinitions.values()], + tasks: tasks.map(({ caseCounts, cases, caseTotal, displayName, entryId, id, matrix, scoreGraders, scoring, state }) => ({ + caseCounts, + cases: cases.map(({ durationMs, identity, key, name, rounds, scores, state }) => { + const round = rounds.at(-1) + return { + durationMs, + identity, + latest: round == null ? undefined : { scores: round.scores, state: round.state }, + metrics: numericMetrics(reducer.cases.get(key), round?.roundIndex), + name, + roundCount: rounds.length, + scores, + state, + } + }), + caseTotal, + displayName, + entryId, + id, + matrix, + scoreDefinitions: [...reducer.tasks.get(id)?.scoreDefinitions.values() ?? []], + scoreGraders, + scoring, + state, + })), + } +} + +/** Last finite number each metric recorded for the case in the given round (or without a round). */ +function numericMetrics(reduced: ReducedCase | undefined, roundIndex: number | undefined): Record { + const values: Record = {} + for (const entry of reduced?.metrics ?? []) { + if (entry.roundIndex != null && roundIndex != null && entry.roundIndex !== roundIndex) + continue + if (typeof entry.value === 'number' && Number.isFinite(entry.value)) + values[entry.name] = entry.value + } + return values +} diff --git a/packages/ui/src/server/files.ts b/packages/ui/src/server/files.ts index cc8abbb..c3210e2 100644 --- a/packages/ui/src/server/files.ts +++ b/packages/ui/src/server/files.ts @@ -12,19 +12,20 @@ import * as v from 'valibot' import { ViewerError } from './errors' -const CONTENT_TYPES: Record = { +// Static overrides keep browser-active files and ambiguous extensions such as .ts out of inline serving. +const CONTENT_TYPES: Record = { '.avif': 'image/avif', '.bmp': 'image/bmp', - '.css': 'text/css; charset=utf-8', + '.css': { staticExtension: '.bin', type: 'text/css; charset=utf-8' }, '.csv': 'text/csv; charset=utf-8', '.flac': 'audio/flac', '.gif': 'image/gif', - '.htm': 'text/html; charset=utf-8', - '.html': 'text/html; charset=utf-8', + '.htm': { staticExtension: '.bin', type: 'text/html; charset=utf-8' }, + '.html': { staticExtension: '.bin', type: 'text/html; charset=utf-8' }, '.ico': 'image/x-icon', '.jpeg': 'image/jpeg', '.jpg': 'image/jpeg', - '.js': 'text/javascript; charset=utf-8', + '.js': { staticExtension: '.bin', type: 'text/javascript; charset=utf-8' }, '.json': 'application/json', '.jsonl': 'application/jsonl', '.log': 'text/plain; charset=utf-8', @@ -40,15 +41,15 @@ const CONTENT_TYPES: Record = { '.opus': 'audio/opus', '.pdf': 'application/pdf', '.png': 'image/png', - '.svg': 'image/svg+xml', + '.svg': { staticExtension: '.bin', type: 'image/svg+xml' }, '.toml': 'application/toml', - '.ts': 'text/plain; charset=utf-8', + '.ts': { staticExtension: '.bin', type: 'text/plain; charset=utf-8' }, '.tsv': 'text/tab-separated-values; charset=utf-8', '.txt': 'text/plain; charset=utf-8', '.wav': 'audio/wav', '.webm': 'video/webm', '.webp': 'image/webp', - '.xml': 'application/xml', + '.xml': { staticExtension: '.bin', type: 'application/xml' }, '.yaml': 'application/yaml', '.yml': 'application/yaml', } @@ -112,7 +113,8 @@ export function classifyPreview(contentType: string, name: string): PreviewKind * Content type from a file name, `application/octet-stream` when unknown. */ export function contentTypeFromName(name: string): string { - return CONTENT_TYPES[extname(name).toLowerCase()] ?? 'application/octet-stream' + const entry = CONTENT_TYPES[extname(name).toLowerCase()] + return typeof entry === 'string' ? entry : entry?.type ?? 'application/octet-stream' } /** @@ -297,6 +299,21 @@ export async function resolveInsideRoot(root: string, requestedPath: string): Pr return realTarget } +/** + * Normalizes an attachment's extension for hosts that infer the MIME type from its filename. + * + * Before: + * - "recording.OPUS", "preview.html", "trace.unknown" + * + * After: + * - ".opus", ".bin", ".bin" + */ +export function staticFileExtension(name: string): string { + const extension = extname(name).toLowerCase() + const entry = CONTENT_TYPES[extension] + return typeof entry === 'string' ? extension : entry?.staticExtension ?? '.bin' +} + /** * Content security policy of the viewer's web UI: its own scripts and API only. * diff --git a/packages/ui/src/server/publishRun.ts b/packages/ui/src/server/publishRun.ts new file mode 100644 index 0000000..3c6ca95 --- /dev/null +++ b/packages/ui/src/server/publishRun.ts @@ -0,0 +1,163 @@ +import type { EvidenceLink, ExternalLogsResponse, PreviewKind } from '../shared/contract' +import type { PublishedRun } from '../shared/publishedReport' +import type { EvidenceService } from './evidence' +import type { RunHandle } from './store' + +import { createHash } from 'node:crypto' +import { copyFile } from 'node:fs/promises' +import { basename, join } from 'node:path' + +import { fileKey } from '../shared/publishedReport' +import { ViewerError } from './errors' +import { isAllowedPath, isInside, listTree, resolveInsideRoot, staticFileExtension } from './files' +import { buildOtelLogs, buildOtelMetrics, buildOtelTraces } from './otel' +import { buildCaseDetail, buildMetrics, buildScores, buildSnapshot, buildTimeline, findCase, listArtifacts } from './views' + +/** + * Materializes a completed run and copies the files that its static Viewer can open. + * + * Use when: + * - the Nitro prerenderer publishes one selected run + * + * Expects: + * - the site's files directory exists and the evidence service knows the allowed roots + * + * Returns: + * - the report data with relative file URLs and a list of unavailable or excluded files + */ +export async function publishRun(handle: RunHandle, evidence: EvidenceService, options: { at: number, includeEvidence: boolean, output: string, publicDir: string }): Promise { + const { at, includeEvidence, output, publicDir } = options + const reducer = handle.reducer() + const snapshot = buildSnapshot(handle, evidence, at) + const { current, progress, runKey, seq, state } = snapshot + const files: PublishedRun['files'] = {} + const omitted: PublishedRun['omitted'] = [] + const evidenceViews: PublishedRun['evidenceViews'] = Object.create(null) + const tasks: PublishedRun['tasks'] = Object.create(null) + + async function publishFile(root: string, path: string, linkId?: string): Promise { + const key = fileKey(path, linkId) + if (files[key] != null) + return + try { + const source = await resolveInsideRoot(root, path) + if (isInside(output, source) || isInside(source, output)) + throw new Error('Export output must not overlap linked files.') + + const name = createHash('sha256').update(JSON.stringify([handle.runKey, key])).digest('hex') + const target = `files/${name}${staticFileExtension(path)}` + await copyFile(source, join(publicDir, target)) + files[key] = target + } + catch (error) { + const code = (error as NodeJS.ErrnoException | undefined)?.code + if (!(error instanceof ViewerError) && code !== 'ENOENT' && code !== 'EISDIR') + throw error + omitted.push({ path: linkId == null ? path : `evidence/${linkId}/${path}`, reason: error instanceof ViewerError ? error.message : 'File is unavailable.' }) + } + } + + function portablePreview(item: T): T { + return item.previewKind === 'html' || (item.previewKind === 'image' && staticFileExtension(item.path) === '.bin') + ? { ...item, previewKind: 'binary' } + : item + } + + const artifactRoot = handle.artifactsRoot() + const root = artifactRoot == null || handle.reportDirectory == null + ? undefined + : await isAllowedPath([handle.reportDirectory], artifactRoot) + const artifacts = [] + for (const artifact of await listArtifacts(handle, {})) { + if (!root?.allowed || !root.exists) { + omitted.push({ path: artifact.path, reason: 'Run has no allowed artifact directory.' }) + continue + } + await publishFile(root.realPath, artifact.path) + if (files[fileKey(artifact.path)] != null) + artifacts.push(portablePreview(artifact)) + } + + const links: EvidenceLink[] = [] + for (const source of await evidence.links(reducer)) { + const path = `evidence/${source.id}/${source.path.split(/[\\/]/).filter(Boolean).at(-1) ?? source.id}` + if (!includeEvidence || !source.allowed || !source.exists) { + const reason = includeEvidence ? source.reason ?? 'Evidence is unavailable.' : 'External evidence was not included.' + omitted.push({ path, reason }) + links.push({ ...source, allowed: false, path, reason, summary: undefined }) + continue + } + const link = await evidence.find(reducer, source.id) + const target = await isAllowedPath(evidence.roots, link.realPath) + if (!target.allowed || !target.exists) { + const reason = 'Evidence is unavailable.' + omitted.push({ path, reason }) + links.push({ ...source, allowed: false, path, reason, summary: undefined }) + continue + } + if (isInside(target.realPath, output) || isInside(output, target.realPath)) + throw new Error('Export output must not overlap linked evidence.') + links.push({ ...source, path }) + if (link.kind !== 'directory') { + omitted.push({ path, reason: 'The Viewer opens evidence directories.' }) + continue + } + const tree = await listTree(target.realPath) + for (const entry of tree.entries) { + if (entry.kind === 'file') + await publishFile(target.realPath, entry.path, link.id) + } + const timeline = link.adapter === 'osworld' ? await evidence.screenshotTimeline(link) : undefined + const entries = tree.entries.filter(entry => entry.kind !== 'file' || files[fileKey(entry.path, link.id)] != null) + evidenceViews[link.id] = { + timeline: timeline == null ? undefined : { ...timeline, frames: timeline.frames.filter(frame => files[fileKey(frame.path, link.id)] != null) }, + tree: { ...tree, entries: entries.map(portablePreview) }, + } + } + + for (const task of snapshot.tasks) { + const cases: PublishedRun['tasks'][string]['cases'] = Object.create(null) + for (const { caseId } of task.cases) { + const scope = { caseId, taskId: task.id } + const detail = await buildCaseDetail(handle, evidence, task.id, caseId) + const caseEvidence = links.filter(link => link.taskId === task.id && link.caseId === caseId) + let externalLogs: ExternalLogsResponse | undefined + if (caseEvidence.some(link => link.allowed)) { + const reduced = findCase(handle, task.id, caseId) + let offset: null | number = 0 + while (offset != null) { + const page = await evidence.externalLogs(reducer, reduced, { limit: 2000, offset }) + externalLogs ??= { ...page, files: page.files.map(file => ({ ...file, path: basename(file.path) })), lines: [], nextOffset: null } + for (const line of page.lines) + externalLogs.lines.push({ ...line, file: basename(line.file) }) + offset = page.nextOffset + } + } + cases[caseId] = { + detail: { + ...detail, + artifacts: artifacts.filter(artifact => artifact.taskId === task.id && artifact.caseId === caseId), + evidence: caseEvidence, + }, + externalLogs, + timeline: buildTimeline(handle, scope), + } + } + const scope = { taskId: task.id } + tasks[task.id] = { cases, metrics: buildMetrics(handle, scope), timeline: buildTimeline(handle, scope) } + } + + return { + evidenceViews, + files, + logs: [...reducer.logs], + metrics: buildMetrics(handle, {}), + omitted, + otel: { logs: buildOtelLogs(handle), metrics: buildOtelMetrics(handle), traces: buildOtelTraces(handle) }, + progress: { current, progress, runKey, seq, state }, + scores: buildScores(handle), + snapshot, + tasks, + timeline: buildTimeline(handle, {}), + } +} diff --git a/packages/ui/src/server/views.ts b/packages/ui/src/server/views.ts index dcbeab7..88869de 100644 --- a/packages/ui/src/server/views.ts +++ b/packages/ui/src/server/views.ts @@ -1,7 +1,6 @@ import type { ArtifactDescriptor, CaseDetail, - LogsResponse, MetricsResponse, ProgressResponse, RunSnapshot, @@ -50,20 +49,6 @@ export async function buildCaseDetail(handle: RunHandle, evidence: EvidenceServi } } -/** - * Console output (`case.output`) of a run, split into lines and paged. - */ -export function buildLogs(handle: RunHandle, query: { caseId?: string, limit: number, offset: number, round?: number, stream?: string, taskId?: string }): LogsResponse { - const lines = handle.reducer().logs.filter(line => - (query.taskId == null || line.taskId === query.taskId) - && (query.caseId == null || line.caseId === query.caseId) - && (query.stream == null || line.stream === query.stream) - && (query.round == null || line.roundIndex === query.round)) - const slice = lines.slice(query.offset, query.offset + query.limit) - const nextOffset = query.offset + slice.length - return { lines: slice, nextOffset: nextOffset < lines.length ? nextOffset : null, total: lines.length } -} - /** * Metrics of a run grouped by name: numeric series with summaries, other values as lists, and the scores beside them. * @@ -190,22 +175,18 @@ export function buildSnapshot(handle: RunHandle, evidence: EvidenceService, now: const summary = handle.summary() const tasks = buildTaskNodes(reducer, key => counts.get(key) ?? 0) return { - current: currentPosition(reducer, now), + ...buildProgress(handle, now), endedAt: run.endedAt, eventCounts: { ...reducer.eventCounts }, headline: resolveTaskHeadline(tasks), identity: reducer.identity() ?? { attemptId: 'unknown-attempt', experimentId: 'unknown-experiment', runId: handle.runKey, workspaceId: 'unknown-workspace' }, interruptSignal: run.interruptSignal ?? summary?.interruptSignal, parseErrors: handle.parseErrors(), - progress: estimateProgress(reducer, now), projects: buildProjectNodes(reducer), - runKey: handle.runKey, scoreDefinitions: [...reducer.scoreDefinitions.values()], - seq: reducer.seq(), seqGaps: reducer.seqGaps(), source: handle.source(), startedAt: run.startedAt, - state: handle.state(now), summary: summary == null ? undefined : { diff --git a/packages/ui/src/server/compare.ts b/packages/ui/src/shared/compare.ts similarity index 78% rename from packages/ui/src/server/compare.ts rename to packages/ui/src/shared/compare.ts index 67eaad9..342b697 100644 --- a/packages/ui/src/server/compare.ts +++ b/packages/ui/src/shared/compare.ts @@ -1,9 +1,36 @@ -import type { CellCoverage, ComparedRun, ComparisonCell, ComparisonResponse, MatrixCell, MatrixResponse, MetricDefinition, RunListItem, ScoreDefinition, ScoreGraders, TaskNode } from '../shared/contract' -import type { ReducedCase } from '../shared/reduce' -import type { RunHandle } from './store' +import type { CaseNode, CaseRound, CellCoverage, ComparedRun, ComparisonCell, ComparisonResponse, MatrixCell, MatrixResponse, MetricDefinition, RunListItem, ScoreDefinition, ScoreGraders, TaskNode } from './contract' -import { addScoreGrader, axisValues, buildTaskNodes, compareNatural, countedScores, countingDefinitions, pickScoreDefinitions } from '../shared/reduce' -import { toRunListItem } from './store' +import { addScoreGrader, axisValues, compareNatural, countedScores, countingDefinitions, pickScoreDefinitions } from './reduce' + +/** Case results needed to compare completed or live runs. */ +export interface ComparisonCase extends Pick { + /** Values from the case's last round. */ + latest?: Pick + /** Last numeric metrics recorded for that round. */ + metrics: Record + /** Number of rounds attempted. */ + roundCount: number +} + +/** Run data shared by live comparisons and exported reports. */ +export interface ComparisonRun { + /** Run identity and summary. */ + item: RunListItem + /** Definitions of the metrics included in the comparison. */ + metricDefinitions: MetricDefinition[] + /** Run-wide score definitions. */ + scoreDefinitions: ScoreDefinition[] + /** Tasks in runner order. */ + tasks: ComparisonTask[] +} + +/** Task results and configuration used in matrix and case comparisons. */ +export interface ComparisonTask extends Pick { + /** Case results in task order. */ + cases: ComparisonCase[] + /** Score definitions at task scope. */ + scoreDefinitions: ScoreDefinition[] +} /** One case of one task of a column, before the rows are split and pooled. */ interface CaseEntry { @@ -20,7 +47,7 @@ interface CellRun { } interface Column { - handles: RunHandle[] + handles: ComparisonRun[] id: string label: string } @@ -44,7 +71,7 @@ interface Column { * Each run is scored by its task's primary score (`score`, and `caseScores` per case by the rule of `scoreCases`), * with its completion rate next to it. */ -export function buildMatrix(runs: readonly { item: RunListItem, scoreDefinitions: ReadonlyMap, tasks: readonly TaskNode[] }[]): MatrixResponse { +export function buildMatrix(runs: readonly ComparisonRun[]): MatrixResponse { const oldestFirst = [...runs].sort((left, right) => (left.item.startedAt ?? '').localeCompare(right.item.startedAt ?? '')) // Runs of each cell, oldest first, with the cases each one ran; whether a run is `full` depends on the cases of the // cell's other runs, so it is settled after every run was read. @@ -54,9 +81,10 @@ export function buildMatrix(runs: readonly { item: RunListItem, scoreDefinitions // Newest definition of each score, since runs are read oldest first. const definitions = new Map() for (const { item, scoreDefinitions, tasks } of oldestFirst) { - for (const [name, definition] of scoreDefinitions) { - definitions.set(name, definition) + for (const definition of scoreDefinitions) { + definitions.set(definition.name, definition) } + const definitionsByName = new Map(scoreDefinitions.map(definition => [definition.name, definition])) for (const task of tasks) { if (task.matrix == null || task.entryId == null) { continue @@ -64,11 +92,11 @@ export function buildMatrix(runs: readonly { item: RunListItem, scoreDefinitions runRows.push(task.matrix.run) evalRows.push(task.matrix.eval) const primary = task.scoring.primary - const taskDefinitions = countingDefinitions(task.scoring, scoreDefinitions) + const taskDefinitions = countingDefinitions(task.scoring, definitionsByName) const result: MatrixCell['latest'] = { attemptId: item.attemptId, caseScores: task.cases.map((node) => { - const round = node.rounds.at(-1) + const round = node.latest return primary == null || round == null ? null : countedScores(round, taskDefinitions)[primary] ?? null }), caseTotal: task.caseTotal, @@ -156,38 +184,43 @@ export function buildMatrix(runs: readonly { item: RunListItem, scoreDefinitions /** * Case-level comparison of the runs (attempt labels) of one experiment, oldest first. */ -export function compareAttempts(handles: readonly RunHandle[], filter: { experiment: string, project: string, workspace: string }, now: number): ComparisonResponse { - const selected = sortByStart(handles.filter(handle => matches(handle, filter, now) && toRunListItem(handle, now).experimentId === filter.experiment), now) +export function compareAttempts(handles: readonly ComparisonRun[], filter: { experiment: string, project: string, workspace: string }): ComparisonResponse { + const selected = sortByStart(handles.filter(handle => matches(handle, filter) && handle.item.experimentId === filter.experiment)) const columns = selected.map((handle) => { - const item = toRunListItem(handle, now) - return { handles: [handle], id: handle.runKey, label: `${item.attemptId} (${item.runId})` } + const { item } = handle + return { handles: [handle], id: item.runKey, label: `${item.attemptId} (${item.runId})` } }) - return buildComparison('attempts', columns, now) + return buildComparison('attempts', columns) } /** * Case-level comparison of experiments: one column per experiment of a workspace/project, using the latest run * of each experiment (by start time). */ -export function compareExperiments(handles: readonly RunHandle[], filter: { project: string, workspace: string }, now: number): ComparisonResponse { - const selected = handles.filter(handle => matches(handle, filter, now)) - const byExperiment = Map.groupBy(selected, handle => toRunListItem(handle, now).experimentId) +export function compareExperiments(handles: readonly ComparisonRun[], filter: { project: string, workspace: string }): ComparisonResponse { + const selected = handles.filter(handle => matches(handle, filter)) + const byExperiment = new Map() + for (const run of selected) { + const group = byExperiment.get(run.item.experimentId) ?? [] + group.push(run) + byExperiment.set(run.item.experimentId, group) + } const columns = [...byExperiment].sort((left, right) => left[0].localeCompare(right[0])).map(([experimentId, list]) => ({ - handles: sortByStart(list, now), + handles: sortByStart(list), id: experimentId, label: experimentId, })) - return buildComparison('experiments', columns, now) + return buildComparison('experiments', columns) } /** * Case-level comparison of explicitly chosen runs, in the given order. */ -export function compareRuns(handles: readonly RunHandle[], now: number): ComparisonResponse { +export function compareRuns(handles: readonly ComparisonRun[]): ComparisonResponse { return buildComparison('runs', handles.map((handle) => { - const item = toRunListItem(handle, now) - return { handles: [handle], id: handle.runKey, label: `${item.experimentId} / ${item.attemptId}` } - }), now) + const { item } = handle + return { handles: [handle], id: item.runKey, label: `${item.experimentId} / ${item.attemptId}` } + })) } /** @@ -199,7 +232,7 @@ export function compareRuns(handles: readonly RunHandle[], now: number): Compari * that every column has, so the columns stay paired case by case. Tasks that differ only in an axis some column lacks * (`prompt` run by one workspace alone) are averaged into one cell, as the matrix view averages axes. */ -function buildComparison(view: ComparisonResponse['view'], columns: readonly Column[], now: number): ComparisonResponse { +function buildComparison(view: ComparisonResponse['view'], columns: readonly Column[]): ComparisonResponse { const outputColumns: ComparisonResponse['columns'] = [] const definitions = new Map() const metricDefinitions = new Map() @@ -208,7 +241,7 @@ function buildComparison(view: ComparisonResponse['view'], columns: readonly Col const entriesByColumn = new Map>() for (const column of columns) { const runs: ComparedRun[] = column.handles.map((handle) => { - const item = toRunListItem(handle, now) + const { item } = handle return { attemptId: item.attemptId, counts: item.counts, @@ -228,26 +261,26 @@ function buildComparison(view: ComparisonResponse['view'], columns: readonly Col entriesByColumn.set(column.id, entries) // Older runs first, so a newer run overwrites the cells of the same case. for (const handle of column.handles) { - const reducer = handle.reducer() - for (const [name, definition] of reducer.scoreDefinitions) { - definitions.set(name, definition) + for (const definition of handle.scoreDefinitions) { + definitions.set(definition.name, definition) } - for (const task of buildTaskNodes(reducer, () => 0)) { + const metricsByName = new Map(handle.metricDefinitions.map(definition => [definition.name, definition])) + for (const task of handle.tasks) { for (const [name, graders] of Object.entries(task.scoreGraders)) { graders.forEach(grader => addScoreGrader(scoreGraders, name, grader)) } if (task.scoring.primary != null) { primaries.add(task.scoring.primary) } - const taskDefinitions = countingDefinitions(task.scoring, reducer.tasks.get(task.id)?.scoreDefinitions ?? new Map()) + const taskDefinitions = countingDefinitions(task.scoring, new Map(task.scoreDefinitions.map(definition => [definition.name, definition]))) // A task without a matrix is told apart from the others of its run by its name. const values = task.matrix == null ? { task: task.displayName } : { ...task.matrix.run, ...task.matrix.eval } const configuration = Object.entries(values).sort(([left], [right]) => compareNatural(left, right)).map(([key, value]) => `${key}=${value}`).join('&') for (const node of task.cases) { - const round = node.rounds.at(-1) - const metrics = numericMetrics(reducer.cases.get(node.key), round?.roundIndex) + const round = node.latest + const { metrics } = node for (const name of Object.keys(metrics)) { - const definition = reducer.metricDefinitions.get(name) + const definition = metricsByName.get(name) if (definition != null) metricDefinitions.set(name, definition) } @@ -258,7 +291,7 @@ function buildComparison(view: ComparisonResponse['view'], columns: readonly Col counted: round == null ? {} : countedScores(round, taskDefinitions), ...(node.durationMs == null ? {} : { durationMs: node.durationMs }), metrics, - roundCount: node.rounds.length, + roundCount: node.roundCount, scores: node.scores, state: node.state, }, @@ -327,12 +360,19 @@ function rowAxes(columns: readonly (readonly CaseEntry[])[]): string[] { /** States of pooled cells, the one that most needs attention first: a pooled cell shows the worst of its cases. */ const STATE_SEVERITY: readonly ComparisonCell['state'][] = ['errored', 'timeout', 'failed', 'unfinished', 'running', 'requeued', 'skipped', 'passed'] +/** Whether a run belongs to a workspace, project, or experiment selected in the Viewer. */ +export function inScope(item: Pick, scope: { experiment?: string, project?: string, workspace?: string }): boolean { + return (scope.workspace == null || item.workspaceId === scope.workspace) + && (scope.project == null || item.projectSegment === scope.project || item.projectIds.includes(scope.project)) + && (scope.experiment == null || item.experimentId === scope.experiment) +} + /** * How much of its task a run measured (see `cellCoverageSchema`): cases with a result are `passed` and `failed`; * every expected case must have settled for `full`. Whether a full run ran the cell's cases or `other` ones depends on * the cell's other runs, so {@link buildMatrix} decides that. */ -function coverageOf(task: TaskNode): CellCoverage { +function coverageOf(task: ComparisonTask): CellCoverage { const { errored, failed, passed, skipped, timeout, total } = task.caseCounts if (passed + failed === 0) { return 'none' @@ -349,25 +389,8 @@ function isStrictSubset(inner: ReadonlySet, outer: ReadonlySet): return inner.size < outer.size && [...inner].every(key => outer.has(key)) } -function matches(handle: RunHandle, filter: { project: string, workspace: string }, now: number): boolean { - // A run written by an older vieval cannot be read, so it is no column. - if (handle.unsupportedVersion() != null) { - return false - } - const item = toRunListItem(handle, now) - return item.workspaceId === filter.workspace && (item.projectSegment === filter.project || item.projectIds.includes(filter.project)) -} - -/** Last finite number each metric recorded for the case in the given round (or without a round). */ -function numericMetrics(reduced: ReducedCase | undefined, roundIndex: number | undefined): Record { - const values: Record = {} - for (const entry of reduced?.metrics ?? []) { - if (entry.roundIndex != null && roundIndex != null && entry.roundIndex !== roundIndex) - continue - if (typeof entry.value === 'number' && Number.isFinite(entry.value)) - values[entry.name] = entry.value - } - return values +function matches(run: ComparisonRun, filter: { project: string, workspace: string }): boolean { + return run.item.state !== 'unsupported' && inScope(run.item, filter) } /** @@ -403,8 +426,6 @@ function primaryMean(scoring: TaskNode['scoring']): null | number { return scoring.scores.find(score => score.name === scoring.primary)?.mean ?? null } -function sortByStart(list: readonly RunHandle[], now: number): RunHandle[] { - // `toRunListItem` rebuilds every task node of a run; compute each start time once, not once per comparison. - const startedAt = new Map(list.map(handle => [handle, toRunListItem(handle, now).startedAt ?? ''])) - return [...list].sort((left, right) => startedAt.get(left)!.localeCompare(startedAt.get(right)!)) +function sortByStart(list: readonly ComparisonRun[]): ComparisonRun[] { + return [...list].sort((left, right) => (left.item.startedAt ?? '').localeCompare(right.item.startedAt ?? '')) } diff --git a/packages/ui/src/shared/logs.ts b/packages/ui/src/shared/logs.ts new file mode 100644 index 0000000..9ff40b2 --- /dev/null +++ b/packages/ui/src/shared/logs.ts @@ -0,0 +1,21 @@ +import type { LogLine, LogsResponse } from './contract' + +/** + * Filters recorded console lines and returns one page in recording order. + * + * Use when: + * - the Viewer reads a run's logs from its reducer or a published report + * + * Returns: + * - the lines, the filtered total, and the offset of the next page if available + */ +export function buildLogs(lines: readonly LogLine[], query: { caseId?: string, limit: number, offset: number, round?: number, stream?: string, taskId?: string }): LogsResponse { + const matched = lines.filter(line => + (query.taskId == null || line.taskId === query.taskId) + && (query.caseId == null || line.caseId === query.caseId) + && (query.stream == null || line.stream === query.stream) + && (query.round == null || line.roundIndex === query.round)) + const page = matched.slice(query.offset, query.offset + query.limit) + const nextOffset = query.offset + page.length + return { lines: page, nextOffset: nextOffset < matched.length ? nextOffset : null, total: matched.length } +} diff --git a/packages/ui/src/shared/publishedReport.ts b/packages/ui/src/shared/publishedReport.ts new file mode 100644 index 0000000..bd33339 --- /dev/null +++ b/packages/ui/src/shared/publishedReport.ts @@ -0,0 +1,65 @@ +import type { ComparisonRun } from './compare' +import type { CaseDetail, ExternalLogsResponse, FileTreeResponse, LogLine, MetricsResponse, OtelLogsResponse, OtelMetricsResponse, OtelTracesResponse, ProgressResponse, RunListItem, RunSnapshot, ScoresResponse, ScreenshotTimeline, TimelineResponse } from './contract' + +/** Entry point of a static site, ordered by most recent run first. */ +export interface PublishedCatalog { + /** Run summaries in file order. */ + runs: RunListItem[] + /** Format version expected by the accompanying Viewer. */ + version: 1 +} + +/** Comparison and matrix inputs published independently of case details and logs. */ +export type PublishedComparison = Omit + +/** Data needed to inspect one completed run and its cases. */ +export interface PublishedRun { + /** Directory views for included evidence, keyed by link ID. */ + evidenceViews: Record + /** Relative file URLs, keyed by file identity. */ + files: Record + /** Console lines in recorded order. */ + logs: LogLine[] + /** Unfiltered metrics for the run. */ + metrics: MetricsResponse + /** Paths and reasons for artifacts or evidence left out of the site. */ + omitted: { path: string, reason: string }[] + /** Telemetry views. Raw files remain separate. */ + otel: { logs: OtelLogsResponse, metrics: OtelMetricsResponse, traces: OtelTracesResponse } + /** Timing and score summaries. */ + progress: ProgressResponse + scores: ScoresResponse + /** Execution tree and final run state. */ + snapshot: RunSnapshot + /** Task views indexed by task ID. */ + tasks: Record + /** Retry timeline for the entire run. */ + timeline: TimelineResponse +} + +/** Inspection views of a task and its individual cases. */ +interface PublishedTask { + /** Detail views indexed by case ID. */ + cases: Record + /** Full metric series and retry timeline for the task. */ + metrics: MetricsResponse + timeline: TimelineResponse +} + +export const CATALOG_PATH = 'report/catalog.json' +export const TELEMETRY_SIGNALS = ['logs', 'metrics', 'traces'] as const + +/** Identifies one file within a run, including the evidence link when applicable. */ +export function fileKey(path: string, linkId?: string): string { + return JSON.stringify([linkId ?? null, path]) +} + +/** Locates the report or comparison data of a run by its index in the published catalog. */ +export function runDataPath(index: number, kind: 'comparison' | 'run'): string { + return `report/${kind}/${index}.json` +} + +/** Identifies the optional raw telemetry document of a published run. */ +export function telemetryPath(index: number, signal: (typeof TELEMETRY_SIGNALS)[number]): string { + return `report/otel/${index}/${signal}.json` +} diff --git a/packages/ui/tsdown.config.ts b/packages/ui/tsdown.config.ts index 9fdc9a5..addd99a 100644 --- a/packages/ui/tsdown.config.ts +++ b/packages/ui/tsdown.config.ts @@ -5,6 +5,7 @@ export default defineConfig({ dts: true, entry: { 'index': 'src/index.ts', + 'reportPrerender': 'src/reportPrerender.ts', 'shared/contract': 'src/shared/contract.ts', 'shared/reduce': 'src/shared/reduce.ts', }, diff --git a/packages/vieval/README.md b/packages/vieval/README.md index 1f5a99a..fccd331 100644 --- a/packages/vieval/README.md +++ b/packages/vieval/README.md @@ -454,6 +454,7 @@ vieval run [--config ] [--workspace ] [--project ] [--task ] [--host ] [--open] vieval viewer [reportRoot...] [--port ] [--host ] [--open] [--evidence-root ] [--stale-after ] vieval report analyze +vieval report export [reportRoot...] --output [--run ] [--include-evidence] vieval report index [--output ] [--format table|json|jsonl] vieval report cases [--where ] [--group-by ] [--format table|json|jsonl] vieval report compare [--case-key ] [--score ] [--format table|json] diff --git a/packages/vieval/src/cli/index.ts b/packages/vieval/src/cli/index.ts index e58cf21..a13cff8 100644 --- a/packages/vieval/src/cli/index.ts +++ b/packages/vieval/src/cli/index.ts @@ -7,6 +7,7 @@ import { runReportAnalyzeCli } from './report-analyze' import { runReportAttemptsCli } from './report-attempts' import { runReportCompareCli } from './report-case-compare' import { runReportCasesCli } from './report-cases' +import { runReportExportCli } from './report-export' import { runReportIndexCli } from './report-index' import { runViewerCli } from './viewer' @@ -16,7 +17,7 @@ export type { CliRunOutput, RunVievalCliOptions } from './run' export { runVievalCli } from './run' export type { ParsedViewerCliArguments, ViewerCliRuntime } from './viewer' export { parseViewerCliArguments, runViewerCli } from './viewer' -export type { RunReadyContext, RunReadyDisposer, RunReadyHook, StartViewerOptions, ViewerHost, ViewerModule } from './viewer-host' +export type { ExportReportOptions, ExportReportResult, RunReadyContext, RunReadyDisposer, RunReadyHook, StartViewerOptions, ViewerHost, ViewerModule } from './viewer-host' export { importViewer, VIEWER_PACKAGE_NAME } from './viewer-host' type Command = 'report' | 'run' | 'viewer' @@ -44,6 +45,7 @@ const topLevelHelpText = ` $ vieval report analyze .vieval/reports/my-run $ vieval report attempts --config vieval.config.ts $ vieval report index .vieval/reports --output .vieval/reports/index/runs.jsonl + $ vieval report export .vieval/reports --output report-site $ vieval viewer .vieval/reports ` @@ -119,6 +121,11 @@ export async function runTopLevelCli(argv: readonly string[]): Promise { if (parsed.command === 'report') { const reportSubcommand = parsed.commandArgv[0] + if (reportSubcommand === 'export') { + await runReportExportCli(parsed.commandArgv.slice(1)) + return + } + if (reportSubcommand === 'analyze') { await runReportAnalyzeCli(parsed.commandArgv) return @@ -144,7 +151,7 @@ export async function runTopLevelCli(argv: readonly string[]): Promise { return } - throw new Error(`Unsupported vieval report command "${reportSubcommand ?? '(none)'}". Expected "analyze", "attempts", "index", "cases", or "compare".`) + throw new Error(`Unsupported vieval report command "${reportSubcommand ?? '(none)'}". Expected "analyze", "attempts", "index", "cases", "compare", or "export".`) } if (parsed.command === 'viewer') { diff --git a/packages/vieval/src/cli/report-export.ts b/packages/vieval/src/cli/report-export.ts new file mode 100644 index 0000000..8d99f90 --- /dev/null +++ b/packages/vieval/src/cli/report-export.ts @@ -0,0 +1,65 @@ +import process from 'node:process' + +import { dirname, join, resolve } from 'node:path' + +import meow from 'meow' + +import { importViewer, VIEWER_PACKAGE_NAME } from './viewer-host' + +const help = ` + Export completed evaluations as a static Viewer site. + + Usage + $ vieval report export [reportRoot...] --output [options] + + Options + --output New directory for the website + --config, -c Resolve the default .vieval/reports next to this config + --run Exact run key to export (repeatable) + --include-evidence Copy linked external evidence + --evidence-root Extra allowed evidence directory (repeatable) + + Serve the resulting directory over HTTP to share the report. +` + +/** + * Publishes completed reports through the optional Viewer package. + * + * Call stack: + * + * runTopLevelCli (./index) + * -> {@link runReportExportCli} + * -> {@link importViewer} + * -> @vieval/ui exportReport + */ +export async function runReportExportCli(argv: readonly string[]): Promise { + const cli = meow(help, { + allowUnknownFlags: false, + argv: [...argv], + flags: { + config: { shortFlag: 'c', type: 'string' }, + evidenceRoot: { isMultiple: true, type: 'string' }, + includeEvidence: { type: 'boolean' }, + output: { type: 'string' }, + run: { isMultiple: true, type: 'string' }, + }, + importMeta: import.meta, + }) + if (!cli.flags.output) + throw new Error('Static export requires --output .') + + const cwd = process.cwd() + const project = cli.flags.config == null ? cwd : dirname(resolve(cwd, cli.flags.config)) + const viewer = await importViewer(project, VIEWER_PACKAGE_NAME, 'vieval report export') + if (viewer.exportReport == null) + throw new Error('Install a version of @vieval/ui with static report export support.') + + const result = await viewer.exportReport({ + evidenceRoots: cli.flags.evidenceRoot?.map(root => resolve(cwd, root)), + includeEvidence: cli.flags.includeEvidence, + output: resolve(cwd, cli.flags.output), + reportRoots: cli.input.length > 0 ? cli.input.map(root => resolve(cwd, root)) : [join(project, '.vieval', 'reports')], + runKeys: cli.flags.run, + }) + process.stdout.write(`Exported ${result.runKeys.length} run(s) to ${result.output}\nOmitted files: ${result.omittedFiles}\n`) +} diff --git a/packages/vieval/src/cli/viewer-host.ts b/packages/vieval/src/cli/viewer-host.ts index ede93b8..32e5db2 100644 --- a/packages/vieval/src/cli/viewer-host.ts +++ b/packages/vieval/src/cli/viewer-host.ts @@ -21,6 +21,30 @@ export const DEFAULT_VIEWER_PORT = 51204 */ export const DEFAULT_VIEWER_HOST = '127.0.0.1' +/** Selection and destination for publishing completed runs. */ +export interface ExportReportOptions { + /** Extra directories to allow for linked evidence. @default [] */ + evidenceRoots?: string[] + /** Publish external evidence linked by the selected runs. @default false */ + includeEvidence?: boolean + /** Directory to create for the static site. */ + output: string + /** Roots to search for reports. */ + reportRoots: string[] + /** Exact run keys to publish. @default All supported completed runs */ + runKeys?: string[] +} + +/** Summary of the static site written by the exporter. */ +export interface ExportReportResult { + /** Number of artifacts or evidence entries that could not be included. */ + omittedFiles: number + /** Absolute path of the static site. */ + output: string + /** Runs included in the site. */ + runKeys: string[] +} + /** * What an external host receives once a run's event bus exists, before the first event is emitted. */ @@ -135,6 +159,8 @@ export interface ViewerHost { * Module shape `@vieval/ui` must export. */ export interface ViewerModule { + /** Exports completed reports as a portable Viewer site. */ + exportReport?: (options: ExportReportOptions) => Promise startViewer: (options: StartViewerOptions) => Promise } @@ -155,10 +181,10 @@ export interface ViewerModule { * Throws: * - when the package is not installed or does not export `startViewer` */ -export async function importViewer(baseDirectory: string, packageName: string = VIEWER_PACKAGE_NAME): Promise { +export async function importViewer(baseDirectory: string, packageName: string = VIEWER_PACKAGE_NAME, command = '--with-viewer'): Promise { const loaded = await importFromProject(baseDirectory, packageName) ?? await importFromVieval(packageName) if (loaded == null) { - throw new Error(`--with-viewer requires the "${packageName}" package. Install it next to your vieval config, for example: pnpm add -D ${packageName}`) + throw new Error(`${command} requires the "${packageName}" package. Install it next to your vieval config, for example: pnpm add -D ${packageName}`) } if (!isViewerModule(loaded)) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eb5f16c..c98470d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -138,6 +138,9 @@ catalogs: nano-staged: specifier: ^1.0.2 version: 1.0.2 + nitro: + specifier: 3.0.261103-rc + version: 3.0.261103-rc oxc-minify: specifier: ^0.138.0 version: 0.138.0 @@ -458,7 +461,13 @@ importers: version: 1.43.0 h3: specifier: 'catalog:' - version: 2.0.2 + version: 2.0.2(crossws@0.4.17(srvx@1.0.5))(ocache@0.3.2) + nitro: + specifier: 'catalog:' + version: 3.0.261103-rc + rollup: + specifier: 'catalog:' + version: 4.62.2 srvx: specifier: 'catalog:' version: 1.0.5 @@ -567,7 +576,7 @@ importers: version: 1.2.0 '@moeru/eventa': specifier: 'catalog:' - version: 1.0.2(h3@2.0.2) + version: 1.0.2(h3@2.0.8(crossws@0.4.17(srvx@1.0.8))(ocache@0.3.2)) '@moeru/std': specifier: 'catalog:' version: 0.1.0-beta.17 @@ -3305,6 +3314,14 @@ packages: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} + crossws@0.4.17: + resolution: {integrity: sha512-YG2kyfafNXKmu7e6k64yrj6whrw7m5sBIsOSXOlJ/72X3TQWRQuw8DhvXFDcBk8HF77PnTSygnm7xpYCTnFpow==} + peerDependencies: + srvx: '>=0.11.5' + peerDependenciesMeta: + srvx: + optional: true + css-tree@3.2.1: resolution: {integrity: sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==} engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0} @@ -3550,6 +3567,10 @@ packages: resolution: {integrity: sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==} engines: {node: '>=0.12'} + env-runner@0.3.11: + resolution: {integrity: sha512-jKG29hxqGrL9ktunIwmt9aHbP3n995DGUY1KyXS7iGYFoD/yDdpc2CjJt6remFI3x2H1JOmBBMo0KXxjEQ2G6g==} + hasBin: true + errno@0.1.8: resolution: {integrity: sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==} hasBin: true @@ -3992,6 +4013,19 @@ packages: ocache: optional: true + h3@2.0.8: + resolution: {integrity: sha512-ENuKlF9kC6E6PrvY1PUyE0qkrnnPTIt2/v48uyEEfTVfgoiqBCE0uwwf++2XrWwT1gxjQBDLQ8PAsmZVniKvTw==} + engines: {node: '>=20.19.0'} + hasBin: true + peerDependencies: + crossws: ^0.4.12 + ocache: '>=0.3.0' + peerDependenciesMeta: + crossws: + optional: true + ocache: + optional: true + has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} @@ -4625,6 +4659,11 @@ packages: engines: {node: '>= 4.4.x'} hasBin: true + nitro@3.0.261103-rc: + resolution: {integrity: sha512-tLD7r5fWCtwnQEE4ZkckLUnx1wkWDaLY48dD+BCJtcQ+srSWAUpxJEK3YfTsgN3v+e4UgBYRzuR7zV7Sqns/zA==} + engines: {node: '>=22.22.0'} + hasBin: true + node-fetch-native@1.6.7: resolution: {integrity: sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==} @@ -4653,6 +4692,9 @@ packages: resolution: {integrity: sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==} engines: {node: '>=12.20.0'} + ocache@0.3.2: + resolution: {integrity: sha512-wu82jmgyoORtWMBZy6PvyJ8dBhtaOOhCwfRnSKBRbnSLht/RqNUs5WUzOxw5/IikJLMC9e+s1ReReZm4XbrNoQ==} + ofetch@1.5.1: resolution: {integrity: sha512-2W4oUZlVaqAPAil6FUg/difl6YhqhUR7x2eZY4bQCko22UXg3hptq9KLQdqFClV+Wu85UX7hNtdGTngi/1BxcA==} @@ -5081,6 +5123,11 @@ packages: engines: {node: '>=20.16.0'} hasBin: true + srvx@1.0.8: + resolution: {integrity: sha512-ngFS/J6St4PJfHRHGjr78xkPPtbyXtdMOLNE4x/XZunEK3OhV7aGUWMHf9WTUEy6wm1U0W4okkd82ZFFWRVQTw==} + engines: {node: '>=20.16.0'} + hasBin: true + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -6153,12 +6200,12 @@ snapshots: - supports-color - typescript - '@moeru/eventa@1.0.2(h3@2.0.2)': + '@moeru/eventa@1.0.2(h3@2.0.8(crossws@0.4.17(srvx@1.0.8))(ocache@0.3.2))': dependencies: nanoid: 6.0.2 picomatch: 4.0.4 optionalDependencies: - h3: 2.0.2 + h3: 2.0.8(crossws@0.4.17(srvx@1.0.8))(ocache@0.3.2) '@moeru/std@0.1.0-beta.17': {} @@ -8024,6 +8071,10 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 + crossws@0.4.17(srvx@1.0.8): + optionalDependencies: + srvx: 1.0.8 + css-tree@3.2.1: dependencies: mdn-data: 2.27.1 @@ -8258,6 +8309,11 @@ snapshots: entities@7.0.1: {} + env-runner@0.3.11: + dependencies: + crossws: 0.4.17(srvx@1.0.8) + srvx: 1.0.8 + errno@0.1.8: dependencies: prr: 1.0.1 @@ -8792,10 +8848,21 @@ snapshots: dependencies: duplexer: 0.1.2 - h3@2.0.2: + h3@2.0.2(crossws@0.4.17(srvx@1.0.5))(ocache@0.3.2): dependencies: rou3: 1.0.0 srvx: 1.0.5 + optionalDependencies: + crossws: 0.4.17(srvx@1.0.8) + ocache: 0.3.2 + + h3@2.0.8(crossws@0.4.17(srvx@1.0.8))(ocache@0.3.2): + dependencies: + rou3: 1.0.0 + srvx: 1.0.8 + optionalDependencies: + crossws: 0.4.17(srvx@1.0.8) + ocache: 0.3.2 has-flag@4.0.0: {} @@ -9545,6 +9612,15 @@ snapshots: sax: 1.6.1 optional: true + nitro@3.0.261103-rc: + dependencies: + crossws: 0.4.17(srvx@1.0.8) + env-runner: 0.3.11 + h3: 2.0.8(crossws@0.4.17(srvx@1.0.8))(ocache@0.3.2) + ocache: 0.3.2 + rou3: 1.0.0 + srvx: 1.0.8 + node-fetch-native@1.6.7: {} node-releases@2.0.37: {} @@ -9568,6 +9644,8 @@ snapshots: obug@2.1.3: {} + ocache@0.3.2: {} + ofetch@1.5.1: dependencies: destr: 2.0.5 @@ -10082,6 +10160,8 @@ snapshots: srvx@1.0.5: {} + srvx@1.0.8: {} + stackback@0.0.2: {} std-env@4.1.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 32408e7..2d8e054 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -71,6 +71,7 @@ catalog: h3: ^2.0.2 meow: ^14.1.0 nano-staged: ^1.0.2 + nitro: 3.0.261103-rc oxc-minify: ^0.138.0 oxlint: ^1.72.0 pinia: ^4.0.3