diff --git a/docs/content/en/guide/learn/reports-and-comparisons.md b/docs/content/en/guide/learn/reports-and-comparisons.md
index 15ae4c6..8bf4e17 100644
--- a/docs/content/en/guide/learn/reports-and-comparisons.md
+++ b/docs/content/en/guide/learn/reports-and-comparisons.md
@@ -95,6 +95,14 @@ vieval report attempts --config vieval.config.ts
`attempts` lists the attempts in a report root, newest first: per experiment and attempt, the workspaces, the passed, failed, and no-result case counts, and the completion rate. "←" marks the attempt that `vieval run --resume` continues without an id; notes show which attempt an attempt resumed, interrupts, and runs killed before they wrote a summary. Pass a report root, or `--config` to list the reports next to it; filter with `--experiment` and `--workspace`, and choose `--format table|json`.
+## Share reports
+
+```bash [Terminal]
+vieval report export .vieval/reports --output report-site
+```
+
+Deploy `report-site` to a static host, then share the URL.
+
## Compare two existing report sets
The report comparison command requires a left baseline and a right candidate:
diff --git a/docs/content/zh-hans/guide/learn/reports-and-comparisons.md b/docs/content/zh-hans/guide/learn/reports-and-comparisons.md
index d5b81d9..1012acd 100644
--- a/docs/content/zh-hans/guide/learn/reports-and-comparisons.md
+++ b/docs/content/zh-hans/guide/learn/reports-and-comparisons.md
@@ -117,6 +117,14 @@ vieval report attempts --config vieval.config.ts
`attempts` 按时间从新到旧列出报告目录中的尝试:每个实验和尝试一行,显示涉及的工作区、通过、失败和没有结果的用例数,以及执行成功率。"←" 标记 `vieval run --resume` 不指定标识时会继续的那次尝试;备注列显示该尝试继续自哪次尝试、是否被中断,以及没来得及写入摘要就被终止的运行。可以传入报告目录,或用 `--config` 列出配置文件旁边的报告;用 `--experiment` 和 `--workspace` 筛选,用 `--format table|json` 选择输出格式。
+## 分享报告
+
+```bash [终端]
+vieval report export .vieval/reports --output report-site
+```
+
+将 `report-site` 部署到静态托管服务后,即可分享链接。
+
## 比较两组已有报告
比较命令的第一个目录是基线,第二个目录是候选方案:
diff --git a/packages/ui/README.md b/packages/ui/README.md
index e835204..fd4517f 100644
--- a/packages/ui/README.md
+++ b/packages/ui/README.md
@@ -50,6 +50,16 @@ vieval viewer reports-a reports-b --evidence-root /data/episodes --stale-after 3
| `--evidence-root
` | — | Extra directory that `*.evidence` metric paths may point into (repeatable). The `.vieval` directory above each report root is always allowed. |
| `--stale-after ` | `60` | A run without an end event whose last event is older is shown as `stale`. |
+## Share a static report
+
+```sh
+vieval report export .vieval/reports --output report-site
+```
+
+Export requires Node.js 22.22 or later and a new output directory. Serve that directory over HTTP and share the URL of
+a run, case, or comparison. Repeat `--run ` to select runs. Attachments are included by default. Add
+`--include-evidence` to publish linked external evidence and `--evidence-root ` to allow an additional directory.
+
## Programmatic use
```ts
diff --git a/packages/ui/client/src/api.ts b/packages/ui/client/src/api.ts
index 73efb91..6d9c83c 100644
--- a/packages/ui/client/src/api.ts
+++ b/packages/ui/client/src/api.ts
@@ -1,8 +1,6 @@
import type {
- ArtifactDescriptor,
CaseDetail,
ComparisonResponse,
- EvidenceLink,
ExternalLogsResponse,
FileTreeResponse,
ImagePreview,
@@ -18,22 +16,17 @@ import type {
RunSnapshot,
ScoresResponse,
ScreenshotTimeline,
- TaskNode,
TimelineResponse,
- TreeResponse,
} from '../../src/shared/contract'
-/**
- * Error of one viewer API request.
- *
- * `status` is the HTTP status, or `0` when the server could not be reached at all.
- */
-export class ApiError extends Error {
- constructor(readonly status: number, message: string) {
- super(message)
- this.name = 'ApiError'
- }
-}
+import { ApiError } from './apiError'
+import { openPublishedReport } from './publishedReport'
+
+export { ApiError } from './apiError'
+
+/** Whether the Viewer opened a static report publication. */
+export const isPublishedReport = document.querySelector('meta[name="vieval-report"][content="static"]') != null
+const published = isPublishedReport ? openPublishedReport(new URL('./', document.baseURI)) : undefined
/** Base of every API route; the UI is served by the same server, so it is relative to the page. */
const base = new URL('api/', document.baseURI)
@@ -73,11 +66,9 @@ function url(path: string, query: Record =
/**
* Typed client of the `@vieval/ui` HTTP API (contract in `API.md` and `src/shared/contract.ts`).
*/
-export const api = {
- artifacts: (runKey: string, ref: Partial) => get(runPath(runKey, '/artifacts'), ref),
+const live = {
case: (runKey: string, ref: CaseRef) => get(runPath(runKey, '/case'), { ...ref }),
compare: (view: 'attempts' | 'experiments' | 'runs', query: Record) => get(`compare/${view}`, query),
- evidence: (runKey: string, ref: Partial) => get(runPath(runKey, '/evidence'), ref),
evidenceTimeline: (runKey: string, linkId: string) => get(runPath(runKey, `/evidence/${encodeURIComponent(linkId)}/timeline`)),
evidenceTree: (runKey: string, linkId: string) => get(runPath(runKey, `/evidence/${encodeURIComponent(linkId)}/tree`)),
externalLogs: (runKey: string, ref: CaseRef, offset: number, limit: number) => get(runPath(runKey, '/case/external-logs'), { ...ref, limit, offset }),
@@ -95,11 +86,12 @@ export const api = {
runs: () => get('runs'),
scores: (runKey: string) => get(runPath(runKey, '/scores')),
snapshot: (runKey: string) => get(runPath(runKey, '/snapshot')),
- task: (runKey: string, taskId: string) => get(runPath(runKey, '/task'), { taskId }),
timeline: (runKey: string, ref: Partial) => get(runPath(runKey, '/timeline'), ref),
- tree: () => get('tree'),
}
+/** Read operations of the Viewer, backed by the server or a static publication. */
+export const api: typeof live = published ?? live
+
/**
* Where one previewable file is read from: a run artifact or a file of an evidence directory.
*
@@ -118,6 +110,8 @@ export interface FileLocation {
* placeholder then.
*/
export function imagePreview(location: FileLocation): Promise {
+ if (published != null)
+ return Promise.reject(new ApiError(415, 'Image placeholders are generated by the live Viewer.'))
const route = location.linkId == null
? runPath(location.runKey, '/artifacts/preview')
: runPath(location.runKey, `/evidence/${encodeURIComponent(location.linkId)}/preview`)
@@ -126,6 +120,8 @@ export function imagePreview(location: FileLocation): Promise {
/** Reads `limit` complete lines of a JSONL file from line `offset`. */
export function jsonlPage(location: FileLocation, offset: number, limit: number): Promise {
+ if (published != null)
+ return published.jsonl(location, offset, limit)
const route = location.linkId == null
? runPath(location.runKey, '/artifacts/jsonl')
: runPath(location.runKey, `/evidence/${encodeURIComponent(location.linkId)}/jsonl`)
@@ -139,6 +135,8 @@ export function openRunStream(runKey: string, after: number): EventSource {
/** URL of the file's bytes; `download` adds `Content-Disposition: attachment`. */
export function rawUrl(location: FileLocation, download = false): string {
+ if (published != null)
+ return published.rawUrl(location)
const route = location.linkId == null
? runPath(location.runKey, '/artifacts/raw')
: runPath(location.runKey, `/evidence/${encodeURIComponent(location.linkId)}/raw`)
@@ -165,9 +163,10 @@ export async function readText(location: FileLocation, maxBytes: number): Promis
const body = await response.json().catch(() => undefined) as undefined | { message?: string }
throw new ApiError(response.status, body?.message ?? `${response.status} ${response.statusText}`)
}
- const text = await response.text()
+ const bytes = new Uint8Array(await response.arrayBuffer())
+ const text = new TextDecoder().decode(bytes.subarray(0, maxBytes))
// `Content-Range: bytes 0-1023/52341` carries the full size of a partial response.
const total = Number(response.headers.get('content-range')?.split('/')[1] ?? Number.NaN)
- const size = Number.isFinite(total) ? total : text.length
- return { text, total: size, truncated: response.status === 206 && size > maxBytes }
+ const size = Number.isFinite(total) ? total : bytes.length
+ return { text, total: size, truncated: size > maxBytes }
}
diff --git a/packages/ui/client/src/apiError.ts b/packages/ui/client/src/apiError.ts
new file mode 100644
index 0000000..33c000c
--- /dev/null
+++ b/packages/ui/client/src/apiError.ts
@@ -0,0 +1,7 @@
+/** A Viewer request failure, with 0 for an unreachable data source. */
+export class ApiError extends Error {
+ constructor(readonly status: number, message: string) {
+ super(message)
+ this.name = 'ApiError'
+ }
+}
diff --git a/packages/ui/client/src/composables/useCatalog.ts b/packages/ui/client/src/composables/useCatalog.ts
index 62d75ad..b27c0e0 100644
--- a/packages/ui/client/src/composables/useCatalog.ts
+++ b/packages/ui/client/src/composables/useCatalog.ts
@@ -7,7 +7,7 @@ import { defineQuery, useQuery } from '@pinia/colada'
import { useDocumentVisibility, useIntervalFn } from '@vueuse/core'
import { computed, watch } from 'vue'
-import { api } from '../api'
+import { api, isPublishedReport } from '../api'
import { mergeMatrices } from '../labels'
import { headlineValue } from '../scores'
@@ -124,9 +124,11 @@ export function sumCounts(runs: readonly RunListItem[]): CaseCounts {
*/
export const useCatalog = defineQuery((): Catalog => {
const query = useQuery({ key: RUNS_KEY, query: () => api.runs() })
- const polling = useIntervalFn(() => void query.refetch(), REFRESH_MS)
+ const polling = useIntervalFn(() => void query.refetch(), REFRESH_MS, { immediate: !isPublishedReport })
// A hidden tab does not poll; it re-reads the runs once it is shown again.
watch(useDocumentVisibility(), (visibility) => {
+ if (isPublishedReport)
+ return
if (visibility === 'hidden') {
polling.pause()
return
diff --git a/packages/ui/client/src/publishedReport.ts b/packages/ui/client/src/publishedReport.ts
new file mode 100644
index 0000000..692467a
--- /dev/null
+++ b/packages/ui/client/src/publishedReport.ts
@@ -0,0 +1,196 @@
+import type { ExternalLogsResponse, JsonlPage, RunListItem } from '../../src/shared/contract'
+import type { PublishedCatalog, PublishedComparison, PublishedRun } from '../../src/shared/publishedReport'
+import type { CaseRef, FileLocation } from './api'
+
+import { buildMatrix, compareAttempts, compareExperiments, compareRuns, inScope } from '../../src/shared/compare'
+import { buildLogs } from '../../src/shared/logs'
+import { CATALOG_PATH, fileKey, runDataPath, telemetryPath } from '../../src/shared/publishedReport'
+import { ApiError } from './apiError'
+
+/**
+ * Reads a static Viewer publication using paths relative to the site.
+ *
+ * Use when:
+ * - the Viewer is served alongside a Nitro-generated catalog and run files
+ *
+ * Returns:
+ * - the read operations used by the Viewer, with data cached for the life of the page
+ */
+export function openPublishedReport(base: URL) {
+ const cache = new Map>()
+ const runs = new Map()
+ const lines = new Map>()
+ async function read(path: string): Promise {
+ let pending = cache.get(path)
+ if (pending == null) {
+ pending = fetch(new URL(path, base)).then(async (response) => {
+ if (!response.ok)
+ throw new ApiError(response.status, 'This resource was not included in the published report.')
+ return response.json()
+ }).catch((error) => {
+ cache.delete(path)
+ throw error
+ })
+ cache.set(path, pending)
+ }
+ return await pending as T
+ }
+
+ async function catalog(): Promise {
+ const data = await read(CATALOG_PATH)
+ if (data.version !== 1)
+ throw new ApiError(422, 'This report needs the Viewer that was published with it.')
+ return data.runs
+ }
+
+ function indexOf(items: RunListItem[], runKey: string): number {
+ const index = items.findIndex(item => item.runKey === runKey)
+ if (index < 0)
+ throw new ApiError(404, 'This run was not included in the published report.')
+ return index
+ }
+
+ async function run(runKey: string): Promise {
+ const data = await read(runDataPath(indexOf(await catalog(), runKey), 'run'))
+ runs.set(runKey, data)
+ return data
+ }
+
+ async function task(runKey: string, taskId: string) {
+ const data = await run(runKey)
+ if (!Object.hasOwn(data.tasks, taskId))
+ throw new ApiError(404, 'Task not found.')
+ return data.tasks[taskId]
+ }
+
+ async function caseData(runKey: string, ref: CaseRef) {
+ const { cases } = await task(runKey, ref.taskId)
+ if (!Object.hasOwn(cases, ref.caseId))
+ throw new ApiError(404, 'Case not found.')
+ return cases[ref.caseId]
+ }
+
+ async function comparison(item: RunListItem, index: number) {
+ const data = await read(runDataPath(index, 'comparison'))
+ return { ...data, item }
+ }
+
+ async function comparisonsInScope(scope: Parameters[1]) {
+ return Promise.all((await catalog()).flatMap((item, index) => inScope(item, scope) ? [comparison(item, index)] : []))
+ }
+
+ function rawUrl(location: FileLocation): string {
+ const path = runs.get(location.runKey)?.files[fileKey(location.path, location.linkId)]
+ if (path == null)
+ throw new ApiError(404, 'This file was not included in the published report.')
+ return new URL(path, base).href
+ }
+
+ return {
+ async case(runKey: string, ref: CaseRef) {
+ return (await caseData(runKey, ref)).detail
+ },
+ async compare(view: 'attempts' | 'experiments' | 'runs', query: Record) {
+ if (view === 'runs') {
+ const keys = (query.runs ?? '').split(',').map(key => key.trim()).filter(Boolean)
+ if (keys.length < 2)
+ throw new ApiError(400, 'Choose at least two runs to compare.')
+ const items = await catalog()
+ const indexes = keys.map(key => indexOf(items, key))
+ return compareRuns(await Promise.all(indexes.map(index => comparison(items[index], index))))
+ }
+ const scope = { experiment: query.experiment ?? '', project: query.project ?? '', workspace: query.workspace ?? '' }
+ const filter = view === 'attempts' ? scope : { project: scope.project, workspace: scope.workspace }
+ const data = await comparisonsInScope(filter)
+ return view === 'attempts' ? compareAttempts(data, scope) : compareExperiments(data, scope)
+ },
+ async evidenceTimeline(runKey: string, linkId: string) {
+ return (await run(runKey)).evidenceViews[linkId]?.timeline ?? { frames: [] }
+ },
+ async evidenceTree(runKey: string, linkId: string) {
+ const view = (await run(runKey)).evidenceViews[linkId]
+ if (view == null)
+ throw new ApiError(404, 'This evidence directory was not included in the published report.')
+ return view.tree
+ },
+ async externalLogs(runKey: string, ref: CaseRef, offset: number, limit: number): Promise {
+ const data = (await caseData(runKey, ref)).externalLogs ?? { correlation: { keys: [], podNames: [], window: {} }, files: [], lines: [], nextOffset: null, total: 0 }
+ const end = offset + limit
+ return { ...data, lines: data.lines.slice(offset, end), nextOffset: end < data.lines.length ? end : null }
+ },
+ async jsonl(location: FileLocation, offset: number, limit: number): Promise {
+ const url = rawUrl(location)
+ let pending = lines.get(url)
+ if (pending == null) {
+ pending = fetch(url).then(async (response) => {
+ if (!response.ok)
+ throw new ApiError(response.status, 'Cannot read the published file.')
+ const rows = (await response.text()).split('\n')
+ const partialTail = rows.pop() !== ''
+ return {
+ partialTail,
+ rows: rows.map((row) => {
+ if (row.trim().length === 0)
+ return undefined
+ try {
+ return JSON.parse(row) as unknown
+ }
+ catch {
+ return { $error: 'Invalid JSON', $raw: row }
+ }
+ }),
+ }
+ }).catch((error) => {
+ lines.delete(url)
+ throw error
+ })
+ lines.set(url, pending)
+ }
+ const data = await pending
+ return { limit, offset, partialTail: data.partialTail, rows: data.rows.slice(offset, offset + limit).filter(row => row !== undefined), totalLines: data.rows.length }
+ },
+ async logs(runKey: string, query: Parameters[1]) {
+ return buildLogs((await run(runKey)).logs, query)
+ },
+ async matrix(scope: Parameters[1]) {
+ return buildMatrix(await comparisonsInScope(scope))
+ },
+ async metrics(runKey: string, ref: Partial = {}) {
+ return ref.taskId == null ? (await run(runKey)).metrics : (await task(runKey, ref.taskId)).metrics
+ },
+ async otelLogs(runKey: string) {
+ return (await run(runKey)).otel.logs
+ },
+ async otelMetrics(runKey: string) {
+ return (await run(runKey)).otel.metrics
+ },
+ async otelRaw(runKey: string, signal: 'logs' | 'metrics' | 'traces') {
+ return read(telemetryPath(indexOf(await catalog(), runKey), signal))
+ },
+ async otelTraces(runKey: string) {
+ return (await run(runKey)).otel.traces
+ },
+ async progress(runKey: string) {
+ return (await run(runKey)).progress
+ },
+ rawUrl,
+ async run(runKey: string) {
+ const items = await catalog()
+ return items[indexOf(items, runKey)]
+ },
+ runs: catalog,
+ async scores(runKey: string) {
+ return (await run(runKey)).scores
+ },
+ async snapshot(runKey: string) {
+ return (await run(runKey)).snapshot
+ },
+ async timeline(runKey: string, ref: Partial) {
+ if (ref.taskId == null)
+ return (await run(runKey)).timeline
+ return ref.caseId == null
+ ? (await task(runKey, ref.taskId)).timeline
+ : (await caseData(runKey, { caseId: ref.caseId, taskId: ref.taskId })).timeline
+ },
+ }
+}
diff --git a/packages/ui/package.json b/packages/ui/package.json
index 22c2eac..b934378 100644
--- a/packages/ui/package.json
+++ b/packages/ui/package.json
@@ -54,6 +54,8 @@
"@napi-rs/image": "catalog:",
"es-toolkit": "catalog:",
"h3": "catalog:",
+ "nitro": "catalog:",
+ "rollup": "catalog:",
"srvx": "catalog:",
"thumbhash": "catalog:",
"tinyglobby": "catalog:",
diff --git a/packages/ui/src/index.ts b/packages/ui/src/index.ts
index 656b576..a53a9c8 100644
--- a/packages/ui/src/index.ts
+++ b/packages/ui/src/index.ts
@@ -12,6 +12,7 @@ import { createViewerApp } from './server/app'
import { createEvidenceService } from './server/evidence'
import { createRunStore } from './server/store'
+export { exportReport } from './publishReport'
export type { CreateViewerAppOptions } from './server/app'
export { createViewerApp } from './server/app'
export { createEventTail, parseEventLine } from './server/event-tail'
diff --git a/packages/ui/src/publishReport.test.ts b/packages/ui/src/publishReport.test.ts
new file mode 100644
index 0000000..ff82aad
--- /dev/null
+++ b/packages/ui/src/publishReport.test.ts
@@ -0,0 +1,105 @@
+import type { CaseDetail } from './shared/contract'
+import type { PublishedCatalog, PublishedComparison, PublishedRun } from './shared/publishedReport'
+
+import { appendFile, mkdir, readFile, rename, symlink, writeFile } from 'node:fs/promises'
+import { basename, join } from 'node:path'
+
+import { afterEach, describe, expect, it } from 'vitest'
+
+import { cleanupTemporaryDirectories, copyOsworldFixture, createTemporaryDirectory, createTestApp, OSWORLD_RUNS, OSWORLD_TASK_ID, readRunningContinuation } from '../test/helpers'
+import { exportReport } from './publishReport'
+import { normalizePathSegment } from './server/views'
+import { compareRuns } from './shared/compare'
+import { CATALOG_PATH, fileKey, runDataPath, telemetryPath } from './shared/publishedReport'
+
+afterEach(cleanupTemporaryDirectories)
+
+/** @example vieval report export .vieval/reports --output report-site */
+describe('static report publication', () => {
+ // https://github.com/vieval-dev/vieval/issues/44
+ /** @example expect(report.tasks[taskId].cases[caseId].detail.scoreRecords).toHaveLength(1) */
+ it('issue #44 publishes case evidence and comparable run data independently of the source', async () => {
+ const { reportRoot, vievalDirectory } = await copyOsworldFixture()
+ const directory = await createTemporaryDirectory()
+ const output = join(directory, 'site')
+ const archiveKey = basename(OSWORLD_RUNS.archived)
+ const candidateKey = basename(OSWORLD_RUNS.running)
+ const caseId = '1:chrome%2F06fe7178-4491-4589-810f-2e2bc9502122'
+ const artifacts = join(reportRoot, OSWORLD_RUNS.archived, 'artifacts')
+ const artifactDir = join(artifacts, normalizePathSegment(OSWORLD_TASK_ID), normalizePathSegment(caseId))
+ await mkdir(artifactDir, { recursive: true })
+ await writeFile(join(artifactDir, 'preview.html'), 'Recorded output')
+ await writeFile(join(artifactDir, 'diagram.svg'), '')
+ await writeFile(join(artifactDir, 'recording.opus'), 'recorded audio')
+ await writeFile(join(artifactDir, 'trace.unknown'), 'unrecognized format')
+ const privateFile = join(directory, 'outside.txt')
+ await writeFile(privateFile, 'outside the artifact directory')
+ await symlink(privateFile, join(artifactDir, 'outside.txt'))
+ const candidateOtlp = join(reportRoot, OSWORLD_RUNS.running, 'otlp')
+ await mkdir(candidateOtlp)
+ await symlink(privateFile, join(candidateOtlp, 'logs.json'))
+
+ // Finish the second fixture so comparisons exercise two different sets of results.
+ const continuation = await readRunningContinuation(vievalDirectory)
+ const last = JSON.parse(continuation.at(-1)!) as { seq: number }
+ const ended = { ...last, data: { failedTasks: 0, passedTasks: 0, skippedTasks: 0, totalTasks: 1 }, event: 'run.ended', seq: last.seq + 1, time: '2026-10-11T20:00:00.000Z' }
+ await appendFile(join(reportRoot, OSWORLD_RUNS.running, 'events.jsonl'), `${continuation.join('\n')}\n${JSON.stringify(ended)}\n`)
+ const live = createTestApp({ reportRoots: [reportRoot], vievalDirectory })
+ const expected = await live.json(`/api/compare/runs?runs=${archiveKey},${candidateKey}`)
+ const caseQuery = new URLSearchParams({ caseId, taskId: OSWORLD_TASK_ID })
+ const localDetail = await live.json(`/api/runs/${archiveKey}/case?${caseQuery}`)
+ const localTimeline = await live.json(`/api/runs/${archiveKey}/timeline?${caseQuery}`)
+ live.store.close()
+
+ const result = await exportReport({ includeEvidence: true, output, reportRoots: [reportRoot] })
+ expect(result.runKeys).toEqual([candidateKey, archiveKey])
+ await rename(vievalDirectory, join(directory, 'original-reports'))
+
+ const catalog = JSON.parse(await readFile(join(output, CATALOG_PATH), 'utf8')) as PublishedCatalog
+ const archiveIndex = catalog.runs.findIndex(run => run.runKey === archiveKey)
+ const candidateIndex = catalog.runs.findIndex(run => run.runKey === candidateKey)
+ await expect(readFile(join(output, telemetryPath(candidateIndex, 'logs')))).rejects.toMatchObject({ code: 'ENOENT' })
+ expect(catalog.runs[archiveIndex].reportRoot).toBeNull()
+ expect(await readFile(join(output, 'index.html'), 'utf8')).toContain('name="vieval-report" content="static"')
+ const report = JSON.parse(await readFile(join(output, runDataPath(archiveIndex, 'run')), 'utf8')) as PublishedRun
+ const { detail, externalLogs, timeline } = report.tasks[OSWORLD_TASK_ID].cases[caseId]
+ // Publishing changes file references; the report data comes from the same builders as the Viewer API.
+ expect(detail).toEqual({ ...localDetail, artifacts: detail.artifacts, evidence: detail.evidence })
+ expect(timeline).toEqual(localTimeline)
+ expect(detail.input).toMatchObject({ task: { id: '06fe7178-4491-4589-810f-2e2bc9502122' } })
+ expect(detail.scoreRecords[0].record).toEqual({ name: 'osworld.success', value: 0 })
+ expect(detail.artifacts.map(artifact => [
+ artifact.name,
+ artifact.previewKind,
+ report.files[fileKey(artifact.path)].split('.').at(-1),
+ ]).sort()).toEqual([
+ ['diagram.svg', 'binary', 'bin'],
+ ['preview.html', 'binary', 'bin'],
+ ['recording.opus', 'audio', 'opus'],
+ ['trace.unknown', 'binary', 'bin'],
+ ])
+ expect(externalLogs?.lines).toHaveLength(24)
+ const [link] = detail.evidence
+ const frame = report.evidenceViews[link.id].timeline?.frames[0]
+ expect(frame?.contentType).toBe('image/webp')
+ expect((await readFile(join(output, report.files[fileKey(frame!.path, link.id)]))).length).toBeGreaterThan(0)
+
+ const comparisonRuns = await Promise.all([archiveKey, candidateKey].map(async (runKey) => {
+ const index = catalog.runs.findIndex(run => run.runKey === runKey)
+ const comparison = JSON.parse(await readFile(join(output, runDataPath(index, 'comparison')), 'utf8')) as PublishedComparison
+ return { ...comparison, item: catalog.runs[index] }
+ }))
+ expect(JSON.parse(JSON.stringify(compareRuns(comparisonRuns)))).toEqual(expected)
+ })
+
+ /** @example expect(exportReport({ output, reportRoots: [reportRoot] })).rejects.toThrow(/already exists/) */
+ it('preserves an existing destination and refuses unfinished runs', async () => {
+ const { reportRoot } = await copyOsworldFixture()
+ const output = await createTemporaryDirectory()
+ const marker = join(output, 'keep.txt')
+ await writeFile(marker, 'keep')
+ await expect(exportReport({ output, reportRoots: [reportRoot] })).rejects.toThrow(/already exists/)
+ expect(await readFile(marker, 'utf8')).toBe('keep')
+ await expect(exportReport({ output: join(output, 'site'), reportRoots: [reportRoot], runKeys: [basename(OSWORLD_RUNS.running)] })).rejects.toThrow(/not a supported completed report/)
+ })
+})
diff --git a/packages/ui/src/publishReport.ts b/packages/ui/src/publishReport.ts
new file mode 100644
index 0000000..cdd094a
--- /dev/null
+++ b/packages/ui/src/publishReport.ts
@@ -0,0 +1,141 @@
+import type { ExportReportOptions, ExportReportResult } from 'vieval/cli'
+
+import type { ReportPrerenderOptions } from './reportPrerender'
+import type { PublishedCatalog, PublishedRun } from './shared/publishedReport'
+
+import { cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { dirname, join, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import { isAllowedPath, isInside } from './server/files'
+import { createRunStore, toRunListItem } from './server/store'
+import { CATALOG_PATH, runDataPath, TELEMETRY_SIGNALS, telemetryPath } from './shared/publishedReport'
+
+/**
+ * Builds a static Viewer site for completed reports.
+ *
+ * Use when:
+ * - a publisher needs to deploy reports to a static HTTP host
+ *
+ * Expects:
+ * - the Viewer is built and the destination does not exist
+ *
+ * Returns:
+ * - the absolute output path, run keys, and count of omitted files
+ *
+ * Call stack:
+ *
+ * vieval report export
+ * -> {@link exportReport}
+ * -> Nitro prerender
+ * -> reportPrerender
+ */
+export async function exportReport(options: ExportReportOptions): Promise {
+ const output = resolve(options.output)
+ const reportRoots = options.reportRoots.map(root => resolve(root))
+ const evidenceRoots = (options.evidenceRoots ?? []).map(root => resolve(root))
+ const dist = fileURLToPath(new URL('../dist/', import.meta.url))
+ const clientDir = join(dist, 'client')
+ const destination = await isAllowedPath([], output)
+ if (destination.exists)
+ throw new Error(`Export output "${output}" already exists.`)
+ for (const source of [...reportRoots, ...evidenceRoots, clientDir]) {
+ const root = await isAllowedPath([], source)
+ if (!root.exists)
+ throw new Error(`Export source "${source}" does not exist.`)
+ if (isInside(root.realPath, destination.realPath) || isInside(destination.realPath, root.realPath))
+ throw new Error('Export output must not overlap report, evidence, or Viewer sources.')
+ }
+
+ const at = Date.now()
+ let runs: PublishedCatalog['runs']
+ const store = createRunStore({ reportRoots })
+ try {
+ const completed = (await store.list()).filter((run) => {
+ const state = run.state(at)
+ return state === 'ended' || state === 'interrupted'
+ })
+ const selected = options.runKeys?.length
+ ? [...new Set(options.runKeys)].map((key) => {
+ const run = completed.find(item => item.runKey === key)
+ if (run == null)
+ throw new Error(`Run "${key}" is not a supported completed report in the selected roots.`)
+ return run
+ })
+ : completed
+ if (selected.length === 0)
+ throw new Error('No supported completed reports to export.')
+ runs = selected.map(run => ({ ...toRunListItem(run, at), reportPath: null, reportRoot: null }))
+ .sort((left, right) => (right.startedAt ?? '').localeCompare(left.startedAt ?? ''))
+ }
+ finally {
+ store.close()
+ }
+
+ // Nitro owns a scratch build. An exclusive mkdir claims the user's destination only after the build succeeds.
+ const scratch = await mkdtemp(join(tmpdir(), 'vieval-export-'))
+ try {
+ const publicDir = join(scratch, 'site')
+ const inputs: ReportPrerenderOptions = {
+ at,
+ catalog: { runs, version: 1 },
+ clientDir,
+ evidenceRoots,
+ includeEvidence: options.includeEvidence === true,
+ output: destination.realPath,
+ publicDir,
+ reportRoots,
+ }
+ // The worker gets explicit build inputs without reading the caller's project config or environment variables.
+ const serverEntry = join(scratch, 'server.mjs')
+ const handler = join(dist, 'reportPrerender.mjs')
+ await writeFile(serverEntry, `import { createReportPrerender } from ${JSON.stringify(handler)}\nexport default createReportPrerender(${JSON.stringify(inputs)})\n`)
+ const routes = ['/index.html', `/${CATALOG_PATH}`, ...runs.flatMap((_, index) => [
+ `/${runDataPath(index, 'run')}`,
+ `/${runDataPath(index, 'comparison')}`,
+ ...TELEMETRY_SIGNALS.map(signal => `/${telemetryPath(index, signal)}`),
+ ])]
+ const { copyPublicAssets, createNitro, prepare, prerender } = await import('nitro/builder')
+ const nitro = await createNitro({
+ builder: 'rollup',
+ // Resolve the shipped builder from this package, since the Nitro project lives in a temporary directory.
+ deps: { install: false, paths: { rollup: import.meta.resolve('rollup/package.json') } },
+ output: { dir: join(scratch, 'output'), publicDir },
+ prerender: { crawlLinks: false, failOnError: true, ignoreStatusCodes: [404], routes },
+ preset: 'static',
+ publicAssets: [{ dir: clientDir, ignore: [join(clientDir, 'index.html')] }],
+ rootDir: scratch,
+ serverEntry,
+ }, { dotenv: false })
+ try {
+ await prepare(nitro)
+ await copyPublicAssets(nitro)
+ await mkdir(join(publicDir, 'files'), { recursive: true })
+ await prerender(nitro)
+ }
+ finally {
+ await nitro.close()
+ }
+
+ let omittedFiles = 0
+ for (const index of runs.keys()) {
+ const run = JSON.parse(await readFile(join(publicDir, runDataPath(index, 'run')), 'utf8')) as PublishedRun
+ omittedFiles += run.omitted.length
+ }
+ await mkdir(dirname(output), { recursive: true })
+ await mkdir(output)
+ try {
+ for (const name of await readdir(publicDir))
+ await cp(join(publicDir, name), join(output, name), { errorOnExist: true, force: false, recursive: true })
+ }
+ catch (error) {
+ await rm(output, { force: true, recursive: true })
+ throw error
+ }
+ return { omittedFiles, output, runKeys: runs.map(run => run.runKey) }
+ }
+ finally {
+ await rm(scratch, { force: true, recursive: true })
+ }
+}
diff --git a/packages/ui/src/reportPrerender.ts b/packages/ui/src/reportPrerender.ts
new file mode 100644
index 0000000..1393287
--- /dev/null
+++ b/packages/ui/src/reportPrerender.ts
@@ -0,0 +1,102 @@
+import type { PublishedCatalog, PublishedComparison } from './shared/publishedReport'
+
+import { readFile } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+
+import { H3 } from 'h3'
+
+import { projectComparisonRun } from './server/comparisonRun'
+import { createEvidenceService } from './server/evidence'
+import { isAllowedPath } from './server/files'
+import { publishRun } from './server/publishRun'
+import { createRunStore } from './server/store'
+import { CATALOG_PATH, runDataPath, TELEMETRY_SIGNALS, telemetryPath } from './shared/publishedReport'
+
+/** Build-time inputs sent to the isolated Nitro prerenderer. */
+export interface ReportPrerenderOptions {
+ /** Fixed timestamp used to compute the report. */
+ at: number
+ /** Selected runs, in publication order. */
+ catalog: PublishedCatalog
+ /** Built Viewer directory. */
+ clientDir: string
+ /** Additional allowed evidence roots. */
+ evidenceRoots: string[]
+ /** Whether the reader may access linked evidence. */
+ includeEvidence: boolean
+ /** Canonical destination, checked against linked evidence. */
+ output: string
+ /** Scratch public output directory used by Nitro. */
+ publicDir: string
+ /** Input report roots. */
+ reportRoots: string[]
+}
+
+/**
+ * Creates the H3 application that Nitro prerenders into a static publication.
+ *
+ * Use when:
+ * - Nitro loads the generated build entry for a selected set of runs
+ *
+ * Expects:
+ * - the Vue build and the selected runs remain available for the duration of prerendering
+ *
+ * Returns:
+ * - routes for the Viewer shell, report data, and optional raw telemetry
+ */
+export function createReportPrerender(options: ReportPrerenderOptions): H3 {
+ const store = createRunStore({ reportRoots: options.reportRoots })
+ const reportDirectories = options.reportRoots.map(root => dirname(root))
+ const evidence = createEvidenceService({
+ evidenceRoots: [...reportDirectories, ...options.evidenceRoots],
+ logDirectories: options.includeEvidence ? reportDirectories.map(directory => join(directory, 'logs')) : [],
+ })
+
+ const app: H3 = new H3()
+ app.get('/index.html', async () => {
+ const html = await readFile(join(options.clientDir, 'index.html'), 'utf8')
+ return new Response(html.replace(/]*)?>/i, '$&'), {
+ headers: { 'content-type': 'text/html; charset=utf-8' },
+ })
+ })
+ app.get(`/${CATALOG_PATH}`, () => options.catalog)
+
+ for (const [index, item] of options.catalog.runs.entries()) {
+ // Resolve each selected run once within the worker. The route paths are independent of report IDs.
+ let resolved: ReturnType | undefined
+ const handle = () => resolved ??= store.get(item.runKey)
+ app.get(`/${runDataPath(index, 'run')}`, async () => {
+ const run = await handle()
+ if (run == null || ['running', 'stale', 'unsupported'].includes(run.state(options.at)))
+ throw new Error(`Run "${item.runKey}" is no longer a supported completed report.`)
+ return publishRun(run, evidence, options)
+ })
+ app.get(`/${runDataPath(index, 'comparison')}`, async (): Promise => {
+ const run = await handle()
+ if (run == null)
+ throw new Error(`Run "${item.runKey}" no longer exists.`)
+ const { item: _item, ...comparison } = projectComparisonRun(run, options.at)
+ return comparison
+ })
+ for (const signal of TELEMETRY_SIGNALS) {
+ app.get(`/${telemetryPath(index, signal)}`, async () => {
+ const run = await handle()
+ if (run?.reportDirectory == null)
+ return new Response(null, { status: 404 })
+ const file = await isAllowedPath([run.reportDirectory], join(run.reportDirectory, 'otlp', `${signal}.json`))
+ if (!file.allowed || !file.exists)
+ return new Response(null, { status: 404 })
+ const data = await readFile(file.realPath, 'utf8').catch((error: NodeJS.ErrnoException) => {
+ if (error.code !== 'ENOENT')
+ throw error
+ return undefined
+ })
+ return data == null
+ ? new Response(null, { status: 404 })
+ : new Response(data, { headers: { 'content-type': 'application/json' } })
+ })
+ }
+ }
+
+ return app
+}
diff --git a/packages/ui/src/server/app.ts b/packages/ui/src/server/app.ts
index cb1f363..1ffc7fc 100644
--- a/packages/ui/src/server/app.ts
+++ b/packages/ui/src/server/app.ts
@@ -1,6 +1,6 @@
import type { H3Event } from 'h3'
-import type { EventsResponse, RunEvent, RunListItem, TreeResponse } from '../shared/contract'
+import type { EventsResponse, RunEvent, TreeResponse } from '../shared/contract'
import type { EvidenceService } from './evidence'
import type { RunHandle, RunStore } from './store'
@@ -10,15 +10,16 @@ import { join } from 'node:path'
import { errorMessageFrom } from '@moeru/std'
import { getQuery, getRouterParam, H3 } from 'h3'
+import { buildMatrix, compareAttempts, compareExperiments, compareRuns, inScope } from '../shared/compare'
import { VIEWER_API_VERSION } from '../shared/contract'
-import { buildTaskNodes } from '../shared/reduce'
-import { buildMatrix, compareAttempts, compareExperiments, compareRuns } from './compare'
+import { buildLogs } from '../shared/logs'
+import { projectComparisonRun } from './comparisonRun'
import { ViewerError } from './errors'
import { contentTypeFromName, listTree, readJsonlPage, resolveInsideRoot, serveFile } from './files'
import { buildOtelLogs, buildOtelMetrics, buildOtelTraces } from './otel'
import { createImagePreviews } from './preview'
import { toRunListItem, UNSUPPORTED_RUN_MESSAGE } from './store'
-import { buildCaseDetail, buildLogs, buildMetrics, buildProgress, buildScores, buildSnapshot, buildTask, buildTimeline, findCase, listArtifacts, MAX_LOG_LIMIT } from './views'
+import { buildCaseDetail, buildMetrics, buildProgress, buildScores, buildSnapshot, buildTask, buildTimeline, findCase, listArtifacts, MAX_LOG_LIMIT } from './views'
/** Most events one `/events` page returns. */
const MAX_EVENTS_LIMIT = 5000
@@ -219,7 +220,7 @@ export function createViewerApp(options: CreateViewerAppOptions): H3 {
}
const handle = await runFrom(event)
scopeFrom(event, handle)
- return buildLogs(handle, {
+ return buildLogs(handle.reducer().logs, {
caseId: query.caseId,
limit: intParam(query.limit, 'limit', 500, 1, MAX_LOG_LIMIT),
offset: intParam(query.offset, 'offset', 0, 0),
@@ -339,26 +340,22 @@ export function createViewerApp(options: CreateViewerAppOptions): H3 {
app.get('/api/matrix', async (event) => {
const query = getQuery(event) as Record
- const at = now()
- const runs = (await store.list()).filter(handle => handle.unsupportedVersion() == null).map((handle) => {
- // Built once and shared by the list item and the matrix: building task nodes walks every case.
- const tasks = buildTaskNodes(handle.reducer(), () => 0)
- return { item: toRunListItem(handle, at, tasks), scoreDefinitions: handle.reducer().scoreDefinitions, tasks }
- })
- return buildMatrix(runs.filter(run => inScope(run.item, query)))
+ return buildMatrix(await comparisonsInScope(query))
})
app.get('/api/compare/experiments', async (event) => {
const query = getQuery(event) as Record
- return compareExperiments(await store.list(), { project: required(query.project, 'project'), workspace: required(query.workspace, 'workspace') }, now())
+ const scope = { project: required(query.project, 'project'), workspace: required(query.workspace, 'workspace') }
+ return compareExperiments(await comparisonsInScope(scope), scope)
})
app.get('/api/compare/attempts', async (event) => {
const query = getQuery(event) as Record
- return compareAttempts(await store.list(), {
+ const scope = {
experiment: required(query.experiment, 'experiment'),
project: required(query.project, 'project'),
workspace: required(query.workspace, 'workspace'),
- }, now())
+ }
+ return compareAttempts(await comparisonsInScope(scope), scope)
})
app.get('/api/compare/runs', async (event) => {
@@ -374,13 +371,31 @@ export function createViewerApp(options: CreateViewerAppOptions): H3 {
assertSupported(handle)
return handle
}))
- return compareRuns(handles, now())
+ const at = now()
+ return compareRuns(handles.map(handle => projectComparisonRun(handle, at)))
})
app.all('/api/**', () => {
throw new ViewerError(404, 'Unknown API route.')
})
+ async function comparisonsInScope(scope: { experiment?: string, project?: string, workspace?: string }) {
+ const at = now()
+ return (await store.list()).flatMap((handle) => {
+ if (handle.unsupportedVersion() != null)
+ return []
+ const reducer = handle.reducer()
+ const identity = reducer.identity()
+ if (identity != null) {
+ return inScope({ ...identity, projectIds: [...reducer.projects.keys()], projectSegment: handle.projectSegment }, scope)
+ ? [projectComparisonRun(handle, at)]
+ : []
+ }
+ const run = projectComparisonRun(handle, at)
+ return inScope(run.item, scope) ? [run] : []
+ })
+ }
+
app.get('/**', async (event) => {
const directory = options.clientDirectory
if (directory != null && existsSync(join(directory, 'index.html'))) {
@@ -438,16 +453,6 @@ function hostnameOf(header: null | string): string | undefined {
return header.split(':')[0].toLowerCase()
}
-/**
- * Whether a run belongs to the hierarchy scope of the optional `workspace`, `project`, and `experiment` filters.
- * A project matches the report path segment or any project the run executed (multi-project runs).
- */
-function inScope(item: RunListItem, query: Record): boolean {
- return (query.workspace == null || item.workspaceId === query.workspace)
- && (query.project == null || item.projectSegment === query.project || item.projectIds.includes(query.project))
- && (query.experiment == null || item.experimentId === query.experiment)
-}
-
function intParam(value: string | undefined, name: string, fallback: number, min: number, max = Number.MAX_SAFE_INTEGER): number {
if (value == null || value === '') {
return fallback
diff --git a/packages/ui/src/server/compare.test.ts b/packages/ui/src/server/compare.test.ts
index 829c5c9..f22e643 100644
--- a/packages/ui/src/server/compare.test.ts
+++ b/packages/ui/src/server/compare.test.ts
@@ -1,11 +1,12 @@
-import type { MetricDefinition, RunListItem, ScoreDefinition, TaskNode } from '../shared/contract'
-import type { RunReducer } from '../shared/reduce'
+import type { ComparisonRun } from '../shared/compare'
+import type { MetricDefinition, RunListItem, ScoreDefinition } from '../shared/contract'
import { describe, expect, it } from 'vitest'
import { createEnvelopeFactory, createReducerHandle } from '../../test/helpers'
-import { buildTaskNodes, createRunReducer } from '../shared/reduce'
-import { buildMatrix, compareRuns } from './compare'
+import { buildMatrix, compareRuns } from '../shared/compare'
+import { createRunReducer } from '../shared/reduce'
+import { projectComparisonRun } from './comparisonRun'
const TASK = 'evals%2Fdemo::default::run=tts%3Dofficial::eval=asr%3Dwhisper-1'
const TASK_WITH_JUDGE = 'evals%2Fdemo::default::run=tts%3Dofficial::eval=judge%3Dgpt-5.5'
@@ -21,15 +22,8 @@ interface CaseResult {
state: 'errored' | 'failed' | 'passed'
}
-interface TestRun {
- item: RunListItem
- reducer: RunReducer
- scoreDefinitions: ReadonlyMap
- tasks: TaskNode[]
-}
-
/** One ended run of {@link TASK}: the given cases (named `-`) settle, out of `total` expected ones. */
-function runOf(runKey: string, startedAt: string, total: number, results: readonly CaseResult[], options: { definitions?: ScoreDefinition[], metricDefinitions?: MetricDefinition[], prefix?: string, taskId?: string, taskIds?: string[] } = {}): TestRun {
+function runOf(runKey: string, startedAt: string, total: number, results: readonly CaseResult[], options: { definitions?: ScoreDefinition[], metricDefinitions?: MetricDefinition[], prefix?: string, taskId?: string, taskIds?: string[] } = {}): ComparisonRun {
const { prefix = 'case', taskId = TASK } = options
const envelope = createEnvelopeFactory(runKey)
const reducer = createRunReducer()
@@ -76,7 +70,8 @@ function runOf(runKey: string, startedAt: string, total: number, results: readon
state: 'ended',
workspaceId: 'ws-a',
}
- return { item, reducer, scoreDefinitions: reducer.scoreDefinitions, tasks: buildTaskNodes(reducer, () => 0) }
+ const run = projectComparisonRun(createReducerHandle(reducer), Date.parse(startedAt))
+ return { ...run, item: { ...item, headline: run.item.headline } }
}
const scored = (count: number, value: number): CaseResult[] => Array.from({ length: count }, () => ({ scores: { [SCORE]: value }, state: 'passed' }))
@@ -257,7 +252,7 @@ describe('compareRuns', () => {
{ scores: { [SCORE]: 0 }, state: 'failed' },
], { definitions: [{ name: SCORE, primary: true }] })
- const comparison = compareRuns([createReducerHandle(left.reducer), createReducerHandle(right.reducer)], Date.parse('2026-10-20T00:00:00.000Z'))
+ const comparison = compareRuns([left, right])
expect(comparison.primary).toBe(SCORE)
expect(comparison.rows.map(row => [row.caseName, row.changed])).toEqual([['case-0', false], ['case-1', true], ['case-2', false]])
expect(comparison.rows[0].cells['run-a']).toEqual({ counted: { 'demo.other': 0, [SCORE]: 1 }, durationMs: 3000, metrics: {}, roundCount: 1, scores: { unscored: {}, values: { 'demo.other': 0, [SCORE]: 1 } }, state: 'passed' })
@@ -273,7 +268,7 @@ describe('compareRuns', () => {
const left = runOf('run-a', '2026-10-19T06:36:00.000Z', 1, scored(1, 1), { definitions: [{ grader: { kind: 'model', model: 'judge-a' }, name: SCORE, primary: true }] })
const right = runOf('run-b', '2026-10-19T07:00:00.000Z', 1, scored(1, 1), { definitions: [{ grader: { kind: 'model', model: 'judge-b' }, name: SCORE, primary: true }] })
- const comparison = compareRuns([createReducerHandle(left.reducer), createReducerHandle(right.reducer)], Date.parse('2026-10-20T00:00:00.000Z'))
+ const comparison = compareRuns([left, right])
expect(comparison.columns.map(column => column.scoreGraders)).toEqual([
{ [SCORE]: [{ kind: 'model', model: 'judge-a' }] },
{ [SCORE]: [{ kind: 'model', model: 'judge-b' }] },
@@ -288,7 +283,7 @@ describe('compareRuns', () => {
const left = runOf('run-a', '2026-10-19T06:36:00.000Z', 1, [{ scores: { 'demo.a': 1 }, state: 'passed' }])
const right = runOf('run-b', '2026-10-19T07:00:00.000Z', 1, [{ scores: { 'demo.b': 0 }, state: 'passed' }])
- const comparison = compareRuns([createReducerHandle(left.reducer), createReducerHandle(right.reducer)], Date.parse('2026-10-20T00:00:00.000Z'))
+ const comparison = compareRuns([left, right])
expect(comparison.primary).toBeUndefined()
expect(comparison.rows.map(row => row.changed)).toEqual([false])
})
@@ -313,7 +308,7 @@ describe('compareRuns', () => {
const left = runOf('run-a', '2026-10-19T06:36:00.000Z', 1, scored(1, 1), { definitions: [{ name: SCORE, primary: true }], taskIds })
const right = runOf('run-b', '2026-10-19T07:00:00.000Z', 1, scored(1, 0.5), { definitions: [{ name: SCORE, primary: true }], taskIds })
- const comparison = compareRuns([createReducerHandle(left.reducer), createReducerHandle(right.reducer)], Date.parse('2026-10-20T00:00:00.000Z'))
+ const comparison = compareRuns([left, right])
expect(comparison.rows.map(row => [row.caseName, row.variant])).toEqual([['case-0', 'context=home'], ['case-0', 'context=none']])
expect(comparison.rows.map(row => row.cells['run-b']?.counted[SCORE])).toEqual([0.5, 0.5])
expect(comparison.summary.rows).toBe(2)
@@ -337,7 +332,7 @@ describe('compareRuns', () => {
const left = runOf('run-a', '2026-10-19T06:36:00.000Z', 1, scored(1, 1), { definitions: [{ name: SCORE, primary: true }], taskIds: shared })
const right = runOf('run-b', '2026-10-19T07:00:00.000Z', 1, scored(1, 0.5), { definitions: [{ name: SCORE, primary: true }], taskIds: withPrompt })
- const comparison = compareRuns([createReducerHandle(left.reducer), createReducerHandle(right.reducer)], Date.parse('2026-10-20T00:00:00.000Z'))
+ const comparison = compareRuns([left, right])
expect(comparison.rows.map(row => row.variant)).toEqual(['context=home', 'context=none'])
expect(comparison.rows.map(row => [row.cells['run-a']?.pooled, row.cells['run-b']?.pooled])).toEqual([[undefined, 2], [undefined, 2]])
expect(comparison.rows.map(row => row.cells['run-b']?.counted[SCORE])).toEqual([0.5, 0.5])
@@ -352,7 +347,7 @@ describe('compareRuns', () => {
metricDefinitions: [{ better: 'lower', name: 'demo.latency_ms', unit: 'ms' }],
})
- const comparison = compareRuns([createReducerHandle(run.reducer)], Date.parse('2026-10-20T00:00:00.000Z'))
+ const comparison = compareRuns([run])
expect(comparison.rows[0].cells['run-a']?.metrics).toEqual({ 'demo.latency_ms': 120 })
expect(comparison.metricDefinitions).toEqual([{ better: 'lower', name: 'demo.latency_ms', unit: 'ms' }])
})
@@ -364,7 +359,7 @@ describe('compareRuns', () => {
it('names the workspace and project of each compared run, so runs of several workspaces can be told apart', () => {
const run = runOf('run-a', '2026-10-19T06:36:00.000Z', 1, scored(1, 1))
- const comparison = compareRuns([createReducerHandle(run.reducer)], Date.parse('2026-10-20T00:00:00.000Z'))
+ const comparison = compareRuns([run])
expect(comparison.columns[0].runs[0]?.project).toBe('demo')
expect(comparison.columns[0].runs[0]?.workspaceId).toBe('ws-a')
})
diff --git a/packages/ui/src/server/comparisonRun.ts b/packages/ui/src/server/comparisonRun.ts
new file mode 100644
index 0000000..d3e68c0
--- /dev/null
+++ b/packages/ui/src/server/comparisonRun.ts
@@ -0,0 +1,62 @@
+import type { ComparisonRun } from '../shared/compare'
+import type { ReducedCase } from '../shared/reduce'
+import type { RunHandle } from './store'
+
+import { buildTaskNodes } from '../shared/reduce'
+import { toRunListItem } from './store'
+
+/**
+ * Projects a run's reduced state into the data used by matrix and case comparisons.
+ *
+ * Use when:
+ * - a live request or a static export needs to compare this run
+ *
+ * Returns:
+ * - serializable task and case results with the definitions needed to score them
+ */
+export function projectComparisonRun(handle: RunHandle, now: number): ComparisonRun {
+ const reducer = handle.reducer()
+ const tasks = buildTaskNodes(reducer, () => 0)
+ return {
+ item: toRunListItem(handle, now, tasks),
+ metricDefinitions: [...reducer.metricDefinitions.values()],
+ scoreDefinitions: [...reducer.scoreDefinitions.values()],
+ tasks: tasks.map(({ caseCounts, cases, caseTotal, displayName, entryId, id, matrix, scoreGraders, scoring, state }) => ({
+ caseCounts,
+ cases: cases.map(({ durationMs, identity, key, name, rounds, scores, state }) => {
+ const round = rounds.at(-1)
+ return {
+ durationMs,
+ identity,
+ latest: round == null ? undefined : { scores: round.scores, state: round.state },
+ metrics: numericMetrics(reducer.cases.get(key), round?.roundIndex),
+ name,
+ roundCount: rounds.length,
+ scores,
+ state,
+ }
+ }),
+ caseTotal,
+ displayName,
+ entryId,
+ id,
+ matrix,
+ scoreDefinitions: [...reducer.tasks.get(id)?.scoreDefinitions.values() ?? []],
+ scoreGraders,
+ scoring,
+ state,
+ })),
+ }
+}
+
+/** Last finite number each metric recorded for the case in the given round (or without a round). */
+function numericMetrics(reduced: ReducedCase | undefined, roundIndex: number | undefined): Record {
+ const values: Record = {}
+ for (const entry of reduced?.metrics ?? []) {
+ if (entry.roundIndex != null && roundIndex != null && entry.roundIndex !== roundIndex)
+ continue
+ if (typeof entry.value === 'number' && Number.isFinite(entry.value))
+ values[entry.name] = entry.value
+ }
+ return values
+}
diff --git a/packages/ui/src/server/files.ts b/packages/ui/src/server/files.ts
index cc8abbb..c3210e2 100644
--- a/packages/ui/src/server/files.ts
+++ b/packages/ui/src/server/files.ts
@@ -12,19 +12,20 @@ import * as v from 'valibot'
import { ViewerError } from './errors'
-const CONTENT_TYPES: Record = {
+// Static overrides keep browser-active files and ambiguous extensions such as .ts out of inline serving.
+const CONTENT_TYPES: Record = {
'.avif': 'image/avif',
'.bmp': 'image/bmp',
- '.css': 'text/css; charset=utf-8',
+ '.css': { staticExtension: '.bin', type: 'text/css; charset=utf-8' },
'.csv': 'text/csv; charset=utf-8',
'.flac': 'audio/flac',
'.gif': 'image/gif',
- '.htm': 'text/html; charset=utf-8',
- '.html': 'text/html; charset=utf-8',
+ '.htm': { staticExtension: '.bin', type: 'text/html; charset=utf-8' },
+ '.html': { staticExtension: '.bin', type: 'text/html; charset=utf-8' },
'.ico': 'image/x-icon',
'.jpeg': 'image/jpeg',
'.jpg': 'image/jpeg',
- '.js': 'text/javascript; charset=utf-8',
+ '.js': { staticExtension: '.bin', type: 'text/javascript; charset=utf-8' },
'.json': 'application/json',
'.jsonl': 'application/jsonl',
'.log': 'text/plain; charset=utf-8',
@@ -40,15 +41,15 @@ const CONTENT_TYPES: Record = {
'.opus': 'audio/opus',
'.pdf': 'application/pdf',
'.png': 'image/png',
- '.svg': 'image/svg+xml',
+ '.svg': { staticExtension: '.bin', type: 'image/svg+xml' },
'.toml': 'application/toml',
- '.ts': 'text/plain; charset=utf-8',
+ '.ts': { staticExtension: '.bin', type: 'text/plain; charset=utf-8' },
'.tsv': 'text/tab-separated-values; charset=utf-8',
'.txt': 'text/plain; charset=utf-8',
'.wav': 'audio/wav',
'.webm': 'video/webm',
'.webp': 'image/webp',
- '.xml': 'application/xml',
+ '.xml': { staticExtension: '.bin', type: 'application/xml' },
'.yaml': 'application/yaml',
'.yml': 'application/yaml',
}
@@ -112,7 +113,8 @@ export function classifyPreview(contentType: string, name: string): PreviewKind
* Content type from a file name, `application/octet-stream` when unknown.
*/
export function contentTypeFromName(name: string): string {
- return CONTENT_TYPES[extname(name).toLowerCase()] ?? 'application/octet-stream'
+ const entry = CONTENT_TYPES[extname(name).toLowerCase()]
+ return typeof entry === 'string' ? entry : entry?.type ?? 'application/octet-stream'
}
/**
@@ -297,6 +299,21 @@ export async function resolveInsideRoot(root: string, requestedPath: string): Pr
return realTarget
}
+/**
+ * Normalizes an attachment's extension for hosts that infer the MIME type from its filename.
+ *
+ * Before:
+ * - "recording.OPUS", "preview.html", "trace.unknown"
+ *
+ * After:
+ * - ".opus", ".bin", ".bin"
+ */
+export function staticFileExtension(name: string): string {
+ const extension = extname(name).toLowerCase()
+ const entry = CONTENT_TYPES[extension]
+ return typeof entry === 'string' ? extension : entry?.staticExtension ?? '.bin'
+}
+
/**
* Content security policy of the viewer's web UI: its own scripts and API only.
*
diff --git a/packages/ui/src/server/publishRun.ts b/packages/ui/src/server/publishRun.ts
new file mode 100644
index 0000000..3c6ca95
--- /dev/null
+++ b/packages/ui/src/server/publishRun.ts
@@ -0,0 +1,163 @@
+import type { EvidenceLink, ExternalLogsResponse, PreviewKind } from '../shared/contract'
+import type { PublishedRun } from '../shared/publishedReport'
+import type { EvidenceService } from './evidence'
+import type { RunHandle } from './store'
+
+import { createHash } from 'node:crypto'
+import { copyFile } from 'node:fs/promises'
+import { basename, join } from 'node:path'
+
+import { fileKey } from '../shared/publishedReport'
+import { ViewerError } from './errors'
+import { isAllowedPath, isInside, listTree, resolveInsideRoot, staticFileExtension } from './files'
+import { buildOtelLogs, buildOtelMetrics, buildOtelTraces } from './otel'
+import { buildCaseDetail, buildMetrics, buildScores, buildSnapshot, buildTimeline, findCase, listArtifacts } from './views'
+
+/**
+ * Materializes a completed run and copies the files that its static Viewer can open.
+ *
+ * Use when:
+ * - the Nitro prerenderer publishes one selected run
+ *
+ * Expects:
+ * - the site's files directory exists and the evidence service knows the allowed roots
+ *
+ * Returns:
+ * - the report data with relative file URLs and a list of unavailable or excluded files
+ */
+export async function publishRun(handle: RunHandle, evidence: EvidenceService, options: { at: number, includeEvidence: boolean, output: string, publicDir: string }): Promise {
+ const { at, includeEvidence, output, publicDir } = options
+ const reducer = handle.reducer()
+ const snapshot = buildSnapshot(handle, evidence, at)
+ const { current, progress, runKey, seq, state } = snapshot
+ const files: PublishedRun['files'] = {}
+ const omitted: PublishedRun['omitted'] = []
+ const evidenceViews: PublishedRun['evidenceViews'] = Object.create(null)
+ const tasks: PublishedRun['tasks'] = Object.create(null)
+
+ async function publishFile(root: string, path: string, linkId?: string): Promise {
+ const key = fileKey(path, linkId)
+ if (files[key] != null)
+ return
+ try {
+ const source = await resolveInsideRoot(root, path)
+ if (isInside(output, source) || isInside(source, output))
+ throw new Error('Export output must not overlap linked files.')
+
+ const name = createHash('sha256').update(JSON.stringify([handle.runKey, key])).digest('hex')
+ const target = `files/${name}${staticFileExtension(path)}`
+ await copyFile(source, join(publicDir, target))
+ files[key] = target
+ }
+ catch (error) {
+ const code = (error as NodeJS.ErrnoException | undefined)?.code
+ if (!(error instanceof ViewerError) && code !== 'ENOENT' && code !== 'EISDIR')
+ throw error
+ omitted.push({ path: linkId == null ? path : `evidence/${linkId}/${path}`, reason: error instanceof ViewerError ? error.message : 'File is unavailable.' })
+ }
+ }
+
+ function portablePreview(item: T): T {
+ return item.previewKind === 'html' || (item.previewKind === 'image' && staticFileExtension(item.path) === '.bin')
+ ? { ...item, previewKind: 'binary' }
+ : item
+ }
+
+ const artifactRoot = handle.artifactsRoot()
+ const root = artifactRoot == null || handle.reportDirectory == null
+ ? undefined
+ : await isAllowedPath([handle.reportDirectory], artifactRoot)
+ const artifacts = []
+ for (const artifact of await listArtifacts(handle, {})) {
+ if (!root?.allowed || !root.exists) {
+ omitted.push({ path: artifact.path, reason: 'Run has no allowed artifact directory.' })
+ continue
+ }
+ await publishFile(root.realPath, artifact.path)
+ if (files[fileKey(artifact.path)] != null)
+ artifacts.push(portablePreview(artifact))
+ }
+
+ const links: EvidenceLink[] = []
+ for (const source of await evidence.links(reducer)) {
+ const path = `evidence/${source.id}/${source.path.split(/[\\/]/).filter(Boolean).at(-1) ?? source.id}`
+ if (!includeEvidence || !source.allowed || !source.exists) {
+ const reason = includeEvidence ? source.reason ?? 'Evidence is unavailable.' : 'External evidence was not included.'
+ omitted.push({ path, reason })
+ links.push({ ...source, allowed: false, path, reason, summary: undefined })
+ continue
+ }
+ const link = await evidence.find(reducer, source.id)
+ const target = await isAllowedPath(evidence.roots, link.realPath)
+ if (!target.allowed || !target.exists) {
+ const reason = 'Evidence is unavailable.'
+ omitted.push({ path, reason })
+ links.push({ ...source, allowed: false, path, reason, summary: undefined })
+ continue
+ }
+ if (isInside(target.realPath, output) || isInside(output, target.realPath))
+ throw new Error('Export output must not overlap linked evidence.')
+ links.push({ ...source, path })
+ if (link.kind !== 'directory') {
+ omitted.push({ path, reason: 'The Viewer opens evidence directories.' })
+ continue
+ }
+ const tree = await listTree(target.realPath)
+ for (const entry of tree.entries) {
+ if (entry.kind === 'file')
+ await publishFile(target.realPath, entry.path, link.id)
+ }
+ const timeline = link.adapter === 'osworld' ? await evidence.screenshotTimeline(link) : undefined
+ const entries = tree.entries.filter(entry => entry.kind !== 'file' || files[fileKey(entry.path, link.id)] != null)
+ evidenceViews[link.id] = {
+ timeline: timeline == null ? undefined : { ...timeline, frames: timeline.frames.filter(frame => files[fileKey(frame.path, link.id)] != null) },
+ tree: { ...tree, entries: entries.map(portablePreview) },
+ }
+ }
+
+ for (const task of snapshot.tasks) {
+ const cases: PublishedRun['tasks'][string]['cases'] = Object.create(null)
+ for (const { caseId } of task.cases) {
+ const scope = { caseId, taskId: task.id }
+ const detail = await buildCaseDetail(handle, evidence, task.id, caseId)
+ const caseEvidence = links.filter(link => link.taskId === task.id && link.caseId === caseId)
+ let externalLogs: ExternalLogsResponse | undefined
+ if (caseEvidence.some(link => link.allowed)) {
+ const reduced = findCase(handle, task.id, caseId)
+ let offset: null | number = 0
+ while (offset != null) {
+ const page = await evidence.externalLogs(reducer, reduced, { limit: 2000, offset })
+ externalLogs ??= { ...page, files: page.files.map(file => ({ ...file, path: basename(file.path) })), lines: [], nextOffset: null }
+ for (const line of page.lines)
+ externalLogs.lines.push({ ...line, file: basename(line.file) })
+ offset = page.nextOffset
+ }
+ }
+ cases[caseId] = {
+ detail: {
+ ...detail,
+ artifacts: artifacts.filter(artifact => artifact.taskId === task.id && artifact.caseId === caseId),
+ evidence: caseEvidence,
+ },
+ externalLogs,
+ timeline: buildTimeline(handle, scope),
+ }
+ }
+ const scope = { taskId: task.id }
+ tasks[task.id] = { cases, metrics: buildMetrics(handle, scope), timeline: buildTimeline(handle, scope) }
+ }
+
+ return {
+ evidenceViews,
+ files,
+ logs: [...reducer.logs],
+ metrics: buildMetrics(handle, {}),
+ omitted,
+ otel: { logs: buildOtelLogs(handle), metrics: buildOtelMetrics(handle), traces: buildOtelTraces(handle) },
+ progress: { current, progress, runKey, seq, state },
+ scores: buildScores(handle),
+ snapshot,
+ tasks,
+ timeline: buildTimeline(handle, {}),
+ }
+}
diff --git a/packages/ui/src/server/views.ts b/packages/ui/src/server/views.ts
index dcbeab7..88869de 100644
--- a/packages/ui/src/server/views.ts
+++ b/packages/ui/src/server/views.ts
@@ -1,7 +1,6 @@
import type {
ArtifactDescriptor,
CaseDetail,
- LogsResponse,
MetricsResponse,
ProgressResponse,
RunSnapshot,
@@ -50,20 +49,6 @@ export async function buildCaseDetail(handle: RunHandle, evidence: EvidenceServi
}
}
-/**
- * Console output (`case.output`) of a run, split into lines and paged.
- */
-export function buildLogs(handle: RunHandle, query: { caseId?: string, limit: number, offset: number, round?: number, stream?: string, taskId?: string }): LogsResponse {
- const lines = handle.reducer().logs.filter(line =>
- (query.taskId == null || line.taskId === query.taskId)
- && (query.caseId == null || line.caseId === query.caseId)
- && (query.stream == null || line.stream === query.stream)
- && (query.round == null || line.roundIndex === query.round))
- const slice = lines.slice(query.offset, query.offset + query.limit)
- const nextOffset = query.offset + slice.length
- return { lines: slice, nextOffset: nextOffset < lines.length ? nextOffset : null, total: lines.length }
-}
-
/**
* Metrics of a run grouped by name: numeric series with summaries, other values as lists, and the scores beside them.
*
@@ -190,22 +175,18 @@ export function buildSnapshot(handle: RunHandle, evidence: EvidenceService, now:
const summary = handle.summary()
const tasks = buildTaskNodes(reducer, key => counts.get(key) ?? 0)
return {
- current: currentPosition(reducer, now),
+ ...buildProgress(handle, now),
endedAt: run.endedAt,
eventCounts: { ...reducer.eventCounts },
headline: resolveTaskHeadline(tasks),
identity: reducer.identity() ?? { attemptId: 'unknown-attempt', experimentId: 'unknown-experiment', runId: handle.runKey, workspaceId: 'unknown-workspace' },
interruptSignal: run.interruptSignal ?? summary?.interruptSignal,
parseErrors: handle.parseErrors(),
- progress: estimateProgress(reducer, now),
projects: buildProjectNodes(reducer),
- runKey: handle.runKey,
scoreDefinitions: [...reducer.scoreDefinitions.values()],
- seq: reducer.seq(),
seqGaps: reducer.seqGaps(),
source: handle.source(),
startedAt: run.startedAt,
- state: handle.state(now),
summary: summary == null
? undefined
: {
diff --git a/packages/ui/src/server/compare.ts b/packages/ui/src/shared/compare.ts
similarity index 78%
rename from packages/ui/src/server/compare.ts
rename to packages/ui/src/shared/compare.ts
index 67eaad9..342b697 100644
--- a/packages/ui/src/server/compare.ts
+++ b/packages/ui/src/shared/compare.ts
@@ -1,9 +1,36 @@
-import type { CellCoverage, ComparedRun, ComparisonCell, ComparisonResponse, MatrixCell, MatrixResponse, MetricDefinition, RunListItem, ScoreDefinition, ScoreGraders, TaskNode } from '../shared/contract'
-import type { ReducedCase } from '../shared/reduce'
-import type { RunHandle } from './store'
+import type { CaseNode, CaseRound, CellCoverage, ComparedRun, ComparisonCell, ComparisonResponse, MatrixCell, MatrixResponse, MetricDefinition, RunListItem, ScoreDefinition, ScoreGraders, TaskNode } from './contract'
-import { addScoreGrader, axisValues, buildTaskNodes, compareNatural, countedScores, countingDefinitions, pickScoreDefinitions } from '../shared/reduce'
-import { toRunListItem } from './store'
+import { addScoreGrader, axisValues, compareNatural, countedScores, countingDefinitions, pickScoreDefinitions } from './reduce'
+
+/** Case results needed to compare completed or live runs. */
+export interface ComparisonCase extends Pick {
+ /** Values from the case's last round. */
+ latest?: Pick
+ /** Last numeric metrics recorded for that round. */
+ metrics: Record
+ /** Number of rounds attempted. */
+ roundCount: number
+}
+
+/** Run data shared by live comparisons and exported reports. */
+export interface ComparisonRun {
+ /** Run identity and summary. */
+ item: RunListItem
+ /** Definitions of the metrics included in the comparison. */
+ metricDefinitions: MetricDefinition[]
+ /** Run-wide score definitions. */
+ scoreDefinitions: ScoreDefinition[]
+ /** Tasks in runner order. */
+ tasks: ComparisonTask[]
+}
+
+/** Task results and configuration used in matrix and case comparisons. */
+export interface ComparisonTask extends Pick {
+ /** Case results in task order. */
+ cases: ComparisonCase[]
+ /** Score definitions at task scope. */
+ scoreDefinitions: ScoreDefinition[]
+}
/** One case of one task of a column, before the rows are split and pooled. */
interface CaseEntry {
@@ -20,7 +47,7 @@ interface CellRun {
}
interface Column {
- handles: RunHandle[]
+ handles: ComparisonRun[]
id: string
label: string
}
@@ -44,7 +71,7 @@ interface Column {
* Each run is scored by its task's primary score (`score`, and `caseScores` per case by the rule of `scoreCases`),
* with its completion rate next to it.
*/
-export function buildMatrix(runs: readonly { item: RunListItem, scoreDefinitions: ReadonlyMap, tasks: readonly TaskNode[] }[]): MatrixResponse {
+export function buildMatrix(runs: readonly ComparisonRun[]): MatrixResponse {
const oldestFirst = [...runs].sort((left, right) => (left.item.startedAt ?? '').localeCompare(right.item.startedAt ?? ''))
// Runs of each cell, oldest first, with the cases each one ran; whether a run is `full` depends on the cases of the
// cell's other runs, so it is settled after every run was read.
@@ -54,9 +81,10 @@ export function buildMatrix(runs: readonly { item: RunListItem, scoreDefinitions
// Newest definition of each score, since runs are read oldest first.
const definitions = new Map()
for (const { item, scoreDefinitions, tasks } of oldestFirst) {
- for (const [name, definition] of scoreDefinitions) {
- definitions.set(name, definition)
+ for (const definition of scoreDefinitions) {
+ definitions.set(definition.name, definition)
}
+ const definitionsByName = new Map(scoreDefinitions.map(definition => [definition.name, definition]))
for (const task of tasks) {
if (task.matrix == null || task.entryId == null) {
continue
@@ -64,11 +92,11 @@ export function buildMatrix(runs: readonly { item: RunListItem, scoreDefinitions
runRows.push(task.matrix.run)
evalRows.push(task.matrix.eval)
const primary = task.scoring.primary
- const taskDefinitions = countingDefinitions(task.scoring, scoreDefinitions)
+ const taskDefinitions = countingDefinitions(task.scoring, definitionsByName)
const result: MatrixCell['latest'] = {
attemptId: item.attemptId,
caseScores: task.cases.map((node) => {
- const round = node.rounds.at(-1)
+ const round = node.latest
return primary == null || round == null ? null : countedScores(round, taskDefinitions)[primary] ?? null
}),
caseTotal: task.caseTotal,
@@ -156,38 +184,43 @@ export function buildMatrix(runs: readonly { item: RunListItem, scoreDefinitions
/**
* Case-level comparison of the runs (attempt labels) of one experiment, oldest first.
*/
-export function compareAttempts(handles: readonly RunHandle[], filter: { experiment: string, project: string, workspace: string }, now: number): ComparisonResponse {
- const selected = sortByStart(handles.filter(handle => matches(handle, filter, now) && toRunListItem(handle, now).experimentId === filter.experiment), now)
+export function compareAttempts(handles: readonly ComparisonRun[], filter: { experiment: string, project: string, workspace: string }): ComparisonResponse {
+ const selected = sortByStart(handles.filter(handle => matches(handle, filter) && handle.item.experimentId === filter.experiment))
const columns = selected.map((handle) => {
- const item = toRunListItem(handle, now)
- return { handles: [handle], id: handle.runKey, label: `${item.attemptId} (${item.runId})` }
+ const { item } = handle
+ return { handles: [handle], id: item.runKey, label: `${item.attemptId} (${item.runId})` }
})
- return buildComparison('attempts', columns, now)
+ return buildComparison('attempts', columns)
}
/**
* Case-level comparison of experiments: one column per experiment of a workspace/project, using the latest run
* of each experiment (by start time).
*/
-export function compareExperiments(handles: readonly RunHandle[], filter: { project: string, workspace: string }, now: number): ComparisonResponse {
- const selected = handles.filter(handle => matches(handle, filter, now))
- const byExperiment = Map.groupBy(selected, handle => toRunListItem(handle, now).experimentId)
+export function compareExperiments(handles: readonly ComparisonRun[], filter: { project: string, workspace: string }): ComparisonResponse {
+ const selected = handles.filter(handle => matches(handle, filter))
+ const byExperiment = new Map()
+ for (const run of selected) {
+ const group = byExperiment.get(run.item.experimentId) ?? []
+ group.push(run)
+ byExperiment.set(run.item.experimentId, group)
+ }
const columns = [...byExperiment].sort((left, right) => left[0].localeCompare(right[0])).map(([experimentId, list]) => ({
- handles: sortByStart(list, now),
+ handles: sortByStart(list),
id: experimentId,
label: experimentId,
}))
- return buildComparison('experiments', columns, now)
+ return buildComparison('experiments', columns)
}
/**
* Case-level comparison of explicitly chosen runs, in the given order.
*/
-export function compareRuns(handles: readonly RunHandle[], now: number): ComparisonResponse {
+export function compareRuns(handles: readonly ComparisonRun[]): ComparisonResponse {
return buildComparison('runs', handles.map((handle) => {
- const item = toRunListItem(handle, now)
- return { handles: [handle], id: handle.runKey, label: `${item.experimentId} / ${item.attemptId}` }
- }), now)
+ const { item } = handle
+ return { handles: [handle], id: item.runKey, label: `${item.experimentId} / ${item.attemptId}` }
+ }))
}
/**
@@ -199,7 +232,7 @@ export function compareRuns(handles: readonly RunHandle[], now: number): Compari
* that every column has, so the columns stay paired case by case. Tasks that differ only in an axis some column lacks
* (`prompt` run by one workspace alone) are averaged into one cell, as the matrix view averages axes.
*/
-function buildComparison(view: ComparisonResponse['view'], columns: readonly Column[], now: number): ComparisonResponse {
+function buildComparison(view: ComparisonResponse['view'], columns: readonly Column[]): ComparisonResponse {
const outputColumns: ComparisonResponse['columns'] = []
const definitions = new Map()
const metricDefinitions = new Map()
@@ -208,7 +241,7 @@ function buildComparison(view: ComparisonResponse['view'], columns: readonly Col
const entriesByColumn = new Map>()
for (const column of columns) {
const runs: ComparedRun[] = column.handles.map((handle) => {
- const item = toRunListItem(handle, now)
+ const { item } = handle
return {
attemptId: item.attemptId,
counts: item.counts,
@@ -228,26 +261,26 @@ function buildComparison(view: ComparisonResponse['view'], columns: readonly Col
entriesByColumn.set(column.id, entries)
// Older runs first, so a newer run overwrites the cells of the same case.
for (const handle of column.handles) {
- const reducer = handle.reducer()
- for (const [name, definition] of reducer.scoreDefinitions) {
- definitions.set(name, definition)
+ for (const definition of handle.scoreDefinitions) {
+ definitions.set(definition.name, definition)
}
- for (const task of buildTaskNodes(reducer, () => 0)) {
+ const metricsByName = new Map(handle.metricDefinitions.map(definition => [definition.name, definition]))
+ for (const task of handle.tasks) {
for (const [name, graders] of Object.entries(task.scoreGraders)) {
graders.forEach(grader => addScoreGrader(scoreGraders, name, grader))
}
if (task.scoring.primary != null) {
primaries.add(task.scoring.primary)
}
- const taskDefinitions = countingDefinitions(task.scoring, reducer.tasks.get(task.id)?.scoreDefinitions ?? new Map())
+ const taskDefinitions = countingDefinitions(task.scoring, new Map(task.scoreDefinitions.map(definition => [definition.name, definition])))
// A task without a matrix is told apart from the others of its run by its name.
const values = task.matrix == null ? { task: task.displayName } : { ...task.matrix.run, ...task.matrix.eval }
const configuration = Object.entries(values).sort(([left], [right]) => compareNatural(left, right)).map(([key, value]) => `${key}=${value}`).join('&')
for (const node of task.cases) {
- const round = node.rounds.at(-1)
- const metrics = numericMetrics(reducer.cases.get(node.key), round?.roundIndex)
+ const round = node.latest
+ const { metrics } = node
for (const name of Object.keys(metrics)) {
- const definition = reducer.metricDefinitions.get(name)
+ const definition = metricsByName.get(name)
if (definition != null)
metricDefinitions.set(name, definition)
}
@@ -258,7 +291,7 @@ function buildComparison(view: ComparisonResponse['view'], columns: readonly Col
counted: round == null ? {} : countedScores(round, taskDefinitions),
...(node.durationMs == null ? {} : { durationMs: node.durationMs }),
metrics,
- roundCount: node.rounds.length,
+ roundCount: node.roundCount,
scores: node.scores,
state: node.state,
},
@@ -327,12 +360,19 @@ function rowAxes(columns: readonly (readonly CaseEntry[])[]): string[] {
/** States of pooled cells, the one that most needs attention first: a pooled cell shows the worst of its cases. */
const STATE_SEVERITY: readonly ComparisonCell['state'][] = ['errored', 'timeout', 'failed', 'unfinished', 'running', 'requeued', 'skipped', 'passed']
+/** Whether a run belongs to a workspace, project, or experiment selected in the Viewer. */
+export function inScope(item: Pick, scope: { experiment?: string, project?: string, workspace?: string }): boolean {
+ return (scope.workspace == null || item.workspaceId === scope.workspace)
+ && (scope.project == null || item.projectSegment === scope.project || item.projectIds.includes(scope.project))
+ && (scope.experiment == null || item.experimentId === scope.experiment)
+}
+
/**
* How much of its task a run measured (see `cellCoverageSchema`): cases with a result are `passed` and `failed`;
* every expected case must have settled for `full`. Whether a full run ran the cell's cases or `other` ones depends on
* the cell's other runs, so {@link buildMatrix} decides that.
*/
-function coverageOf(task: TaskNode): CellCoverage {
+function coverageOf(task: ComparisonTask): CellCoverage {
const { errored, failed, passed, skipped, timeout, total } = task.caseCounts
if (passed + failed === 0) {
return 'none'
@@ -349,25 +389,8 @@ function isStrictSubset(inner: ReadonlySet, outer: ReadonlySet):
return inner.size < outer.size && [...inner].every(key => outer.has(key))
}
-function matches(handle: RunHandle, filter: { project: string, workspace: string }, now: number): boolean {
- // A run written by an older vieval cannot be read, so it is no column.
- if (handle.unsupportedVersion() != null) {
- return false
- }
- const item = toRunListItem(handle, now)
- return item.workspaceId === filter.workspace && (item.projectSegment === filter.project || item.projectIds.includes(filter.project))
-}
-
-/** Last finite number each metric recorded for the case in the given round (or without a round). */
-function numericMetrics(reduced: ReducedCase | undefined, roundIndex: number | undefined): Record {
- const values: Record = {}
- for (const entry of reduced?.metrics ?? []) {
- if (entry.roundIndex != null && roundIndex != null && entry.roundIndex !== roundIndex)
- continue
- if (typeof entry.value === 'number' && Number.isFinite(entry.value))
- values[entry.name] = entry.value
- }
- return values
+function matches(run: ComparisonRun, filter: { project: string, workspace: string }): boolean {
+ return run.item.state !== 'unsupported' && inScope(run.item, filter)
}
/**
@@ -403,8 +426,6 @@ function primaryMean(scoring: TaskNode['scoring']): null | number {
return scoring.scores.find(score => score.name === scoring.primary)?.mean ?? null
}
-function sortByStart(list: readonly RunHandle[], now: number): RunHandle[] {
- // `toRunListItem` rebuilds every task node of a run; compute each start time once, not once per comparison.
- const startedAt = new Map(list.map(handle => [handle, toRunListItem(handle, now).startedAt ?? '']))
- return [...list].sort((left, right) => startedAt.get(left)!.localeCompare(startedAt.get(right)!))
+function sortByStart(list: readonly ComparisonRun[]): ComparisonRun[] {
+ return [...list].sort((left, right) => (left.item.startedAt ?? '').localeCompare(right.item.startedAt ?? ''))
}
diff --git a/packages/ui/src/shared/logs.ts b/packages/ui/src/shared/logs.ts
new file mode 100644
index 0000000..9ff40b2
--- /dev/null
+++ b/packages/ui/src/shared/logs.ts
@@ -0,0 +1,21 @@
+import type { LogLine, LogsResponse } from './contract'
+
+/**
+ * Filters recorded console lines and returns one page in recording order.
+ *
+ * Use when:
+ * - the Viewer reads a run's logs from its reducer or a published report
+ *
+ * Returns:
+ * - the lines, the filtered total, and the offset of the next page if available
+ */
+export function buildLogs(lines: readonly LogLine[], query: { caseId?: string, limit: number, offset: number, round?: number, stream?: string, taskId?: string }): LogsResponse {
+ const matched = lines.filter(line =>
+ (query.taskId == null || line.taskId === query.taskId)
+ && (query.caseId == null || line.caseId === query.caseId)
+ && (query.stream == null || line.stream === query.stream)
+ && (query.round == null || line.roundIndex === query.round))
+ const page = matched.slice(query.offset, query.offset + query.limit)
+ const nextOffset = query.offset + page.length
+ return { lines: page, nextOffset: nextOffset < matched.length ? nextOffset : null, total: matched.length }
+}
diff --git a/packages/ui/src/shared/publishedReport.ts b/packages/ui/src/shared/publishedReport.ts
new file mode 100644
index 0000000..bd33339
--- /dev/null
+++ b/packages/ui/src/shared/publishedReport.ts
@@ -0,0 +1,65 @@
+import type { ComparisonRun } from './compare'
+import type { CaseDetail, ExternalLogsResponse, FileTreeResponse, LogLine, MetricsResponse, OtelLogsResponse, OtelMetricsResponse, OtelTracesResponse, ProgressResponse, RunListItem, RunSnapshot, ScoresResponse, ScreenshotTimeline, TimelineResponse } from './contract'
+
+/** Entry point of a static site, ordered by most recent run first. */
+export interface PublishedCatalog {
+ /** Run summaries in file order. */
+ runs: RunListItem[]
+ /** Format version expected by the accompanying Viewer. */
+ version: 1
+}
+
+/** Comparison and matrix inputs published independently of case details and logs. */
+export type PublishedComparison = Omit
+
+/** Data needed to inspect one completed run and its cases. */
+export interface PublishedRun {
+ /** Directory views for included evidence, keyed by link ID. */
+ evidenceViews: Record
+ /** Relative file URLs, keyed by file identity. */
+ files: Record
+ /** Console lines in recorded order. */
+ logs: LogLine[]
+ /** Unfiltered metrics for the run. */
+ metrics: MetricsResponse
+ /** Paths and reasons for artifacts or evidence left out of the site. */
+ omitted: { path: string, reason: string }[]
+ /** Telemetry views. Raw files remain separate. */
+ otel: { logs: OtelLogsResponse, metrics: OtelMetricsResponse, traces: OtelTracesResponse }
+ /** Timing and score summaries. */
+ progress: ProgressResponse
+ scores: ScoresResponse
+ /** Execution tree and final run state. */
+ snapshot: RunSnapshot
+ /** Task views indexed by task ID. */
+ tasks: Record
+ /** Retry timeline for the entire run. */
+ timeline: TimelineResponse
+}
+
+/** Inspection views of a task and its individual cases. */
+interface PublishedTask {
+ /** Detail views indexed by case ID. */
+ cases: Record
+ /** Full metric series and retry timeline for the task. */
+ metrics: MetricsResponse
+ timeline: TimelineResponse
+}
+
+export const CATALOG_PATH = 'report/catalog.json'
+export const TELEMETRY_SIGNALS = ['logs', 'metrics', 'traces'] as const
+
+/** Identifies one file within a run, including the evidence link when applicable. */
+export function fileKey(path: string, linkId?: string): string {
+ return JSON.stringify([linkId ?? null, path])
+}
+
+/** Locates the report or comparison data of a run by its index in the published catalog. */
+export function runDataPath(index: number, kind: 'comparison' | 'run'): string {
+ return `report/${kind}/${index}.json`
+}
+
+/** Identifies the optional raw telemetry document of a published run. */
+export function telemetryPath(index: number, signal: (typeof TELEMETRY_SIGNALS)[number]): string {
+ return `report/otel/${index}/${signal}.json`
+}
diff --git a/packages/ui/tsdown.config.ts b/packages/ui/tsdown.config.ts
index 9fdc9a5..addd99a 100644
--- a/packages/ui/tsdown.config.ts
+++ b/packages/ui/tsdown.config.ts
@@ -5,6 +5,7 @@ export default defineConfig({
dts: true,
entry: {
'index': 'src/index.ts',
+ 'reportPrerender': 'src/reportPrerender.ts',
'shared/contract': 'src/shared/contract.ts',
'shared/reduce': 'src/shared/reduce.ts',
},
diff --git a/packages/vieval/README.md b/packages/vieval/README.md
index 1f5a99a..fccd331 100644
--- a/packages/vieval/README.md
+++ b/packages/vieval/README.md
@@ -454,6 +454,7 @@ vieval run [--config ] [--workspace ] [--project ] [--task ] [--host ] [--open]
vieval viewer [reportRoot...] [--port ] [--host ] [--open] [--evidence-root ] [--stale-after ]
vieval report analyze
+vieval report export [reportRoot...] --output [--run ] [--include-evidence]
vieval report index [--output ] [--format table|json|jsonl]
vieval report cases [--where ] [--group-by ] [--format table|json|jsonl]
vieval report compare [--case-key ] [--score ] [--format table|json]
diff --git a/packages/vieval/src/cli/index.ts b/packages/vieval/src/cli/index.ts
index e58cf21..a13cff8 100644
--- a/packages/vieval/src/cli/index.ts
+++ b/packages/vieval/src/cli/index.ts
@@ -7,6 +7,7 @@ import { runReportAnalyzeCli } from './report-analyze'
import { runReportAttemptsCli } from './report-attempts'
import { runReportCompareCli } from './report-case-compare'
import { runReportCasesCli } from './report-cases'
+import { runReportExportCli } from './report-export'
import { runReportIndexCli } from './report-index'
import { runViewerCli } from './viewer'
@@ -16,7 +17,7 @@ export type { CliRunOutput, RunVievalCliOptions } from './run'
export { runVievalCli } from './run'
export type { ParsedViewerCliArguments, ViewerCliRuntime } from './viewer'
export { parseViewerCliArguments, runViewerCli } from './viewer'
-export type { RunReadyContext, RunReadyDisposer, RunReadyHook, StartViewerOptions, ViewerHost, ViewerModule } from './viewer-host'
+export type { ExportReportOptions, ExportReportResult, RunReadyContext, RunReadyDisposer, RunReadyHook, StartViewerOptions, ViewerHost, ViewerModule } from './viewer-host'
export { importViewer, VIEWER_PACKAGE_NAME } from './viewer-host'
type Command = 'report' | 'run' | 'viewer'
@@ -44,6 +45,7 @@ const topLevelHelpText = `
$ vieval report analyze .vieval/reports/my-run
$ vieval report attempts --config vieval.config.ts
$ vieval report index .vieval/reports --output .vieval/reports/index/runs.jsonl
+ $ vieval report export .vieval/reports --output report-site
$ vieval viewer .vieval/reports
`
@@ -119,6 +121,11 @@ export async function runTopLevelCli(argv: readonly string[]): Promise {
if (parsed.command === 'report') {
const reportSubcommand = parsed.commandArgv[0]
+ if (reportSubcommand === 'export') {
+ await runReportExportCli(parsed.commandArgv.slice(1))
+ return
+ }
+
if (reportSubcommand === 'analyze') {
await runReportAnalyzeCli(parsed.commandArgv)
return
@@ -144,7 +151,7 @@ export async function runTopLevelCli(argv: readonly string[]): Promise {
return
}
- throw new Error(`Unsupported vieval report command "${reportSubcommand ?? '(none)'}". Expected "analyze", "attempts", "index", "cases", or "compare".`)
+ throw new Error(`Unsupported vieval report command "${reportSubcommand ?? '(none)'}". Expected "analyze", "attempts", "index", "cases", "compare", or "export".`)
}
if (parsed.command === 'viewer') {
diff --git a/packages/vieval/src/cli/report-export.ts b/packages/vieval/src/cli/report-export.ts
new file mode 100644
index 0000000..8d99f90
--- /dev/null
+++ b/packages/vieval/src/cli/report-export.ts
@@ -0,0 +1,65 @@
+import process from 'node:process'
+
+import { dirname, join, resolve } from 'node:path'
+
+import meow from 'meow'
+
+import { importViewer, VIEWER_PACKAGE_NAME } from './viewer-host'
+
+const help = `
+ Export completed evaluations as a static Viewer site.
+
+ Usage
+ $ vieval report export [reportRoot...] --output [options]
+
+ Options
+ --output New directory for the website
+ --config, -c Resolve the default .vieval/reports next to this config
+ --run Exact run key to export (repeatable)
+ --include-evidence Copy linked external evidence
+ --evidence-root Extra allowed evidence directory (repeatable)
+
+ Serve the resulting directory over HTTP to share the report.
+`
+
+/**
+ * Publishes completed reports through the optional Viewer package.
+ *
+ * Call stack:
+ *
+ * runTopLevelCli (./index)
+ * -> {@link runReportExportCli}
+ * -> {@link importViewer}
+ * -> @vieval/ui exportReport
+ */
+export async function runReportExportCli(argv: readonly string[]): Promise {
+ const cli = meow(help, {
+ allowUnknownFlags: false,
+ argv: [...argv],
+ flags: {
+ config: { shortFlag: 'c', type: 'string' },
+ evidenceRoot: { isMultiple: true, type: 'string' },
+ includeEvidence: { type: 'boolean' },
+ output: { type: 'string' },
+ run: { isMultiple: true, type: 'string' },
+ },
+ importMeta: import.meta,
+ })
+ if (!cli.flags.output)
+ throw new Error('Static export requires --output .')
+
+ const cwd = process.cwd()
+ const project = cli.flags.config == null ? cwd : dirname(resolve(cwd, cli.flags.config))
+ const viewer = await importViewer(project, VIEWER_PACKAGE_NAME, 'vieval report export')
+ if (viewer.exportReport == null)
+ throw new Error('Install a version of @vieval/ui with static report export support.')
+
+ const result = await viewer.exportReport({
+ evidenceRoots: cli.flags.evidenceRoot?.map(root => resolve(cwd, root)),
+ includeEvidence: cli.flags.includeEvidence,
+ output: resolve(cwd, cli.flags.output),
+ reportRoots: cli.input.length > 0 ? cli.input.map(root => resolve(cwd, root)) : [join(project, '.vieval', 'reports')],
+ runKeys: cli.flags.run,
+ })
+ process.stdout.write(`Exported ${result.runKeys.length} run(s) to ${result.output}\nOmitted files: ${result.omittedFiles}\n`)
+}
diff --git a/packages/vieval/src/cli/viewer-host.ts b/packages/vieval/src/cli/viewer-host.ts
index ede93b8..32e5db2 100644
--- a/packages/vieval/src/cli/viewer-host.ts
+++ b/packages/vieval/src/cli/viewer-host.ts
@@ -21,6 +21,30 @@ export const DEFAULT_VIEWER_PORT = 51204
*/
export const DEFAULT_VIEWER_HOST = '127.0.0.1'
+/** Selection and destination for publishing completed runs. */
+export interface ExportReportOptions {
+ /** Extra directories to allow for linked evidence. @default [] */
+ evidenceRoots?: string[]
+ /** Publish external evidence linked by the selected runs. @default false */
+ includeEvidence?: boolean
+ /** Directory to create for the static site. */
+ output: string
+ /** Roots to search for reports. */
+ reportRoots: string[]
+ /** Exact run keys to publish. @default All supported completed runs */
+ runKeys?: string[]
+}
+
+/** Summary of the static site written by the exporter. */
+export interface ExportReportResult {
+ /** Number of artifacts or evidence entries that could not be included. */
+ omittedFiles: number
+ /** Absolute path of the static site. */
+ output: string
+ /** Runs included in the site. */
+ runKeys: string[]
+}
+
/**
* What an external host receives once a run's event bus exists, before the first event is emitted.
*/
@@ -135,6 +159,8 @@ export interface ViewerHost {
* Module shape `@vieval/ui` must export.
*/
export interface ViewerModule {
+ /** Exports completed reports as a portable Viewer site. */
+ exportReport?: (options: ExportReportOptions) => Promise
startViewer: (options: StartViewerOptions) => Promise
}
@@ -155,10 +181,10 @@ export interface ViewerModule {
* Throws:
* - when the package is not installed or does not export `startViewer`
*/
-export async function importViewer(baseDirectory: string, packageName: string = VIEWER_PACKAGE_NAME): Promise {
+export async function importViewer(baseDirectory: string, packageName: string = VIEWER_PACKAGE_NAME, command = '--with-viewer'): Promise {
const loaded = await importFromProject(baseDirectory, packageName) ?? await importFromVieval(packageName)
if (loaded == null) {
- throw new Error(`--with-viewer requires the "${packageName}" package. Install it next to your vieval config, for example: pnpm add -D ${packageName}`)
+ throw new Error(`${command} requires the "${packageName}" package. Install it next to your vieval config, for example: pnpm add -D ${packageName}`)
}
if (!isViewerModule(loaded)) {
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index eb5f16c..c98470d 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -138,6 +138,9 @@ catalogs:
nano-staged:
specifier: ^1.0.2
version: 1.0.2
+ nitro:
+ specifier: 3.0.261103-rc
+ version: 3.0.261103-rc
oxc-minify:
specifier: ^0.138.0
version: 0.138.0
@@ -458,7 +461,13 @@ importers:
version: 1.43.0
h3:
specifier: 'catalog:'
- version: 2.0.2
+ version: 2.0.2(crossws@0.4.17(srvx@1.0.5))(ocache@0.3.2)
+ nitro:
+ specifier: 'catalog:'
+ version: 3.0.261103-rc
+ rollup:
+ specifier: 'catalog:'
+ version: 4.62.2
srvx:
specifier: 'catalog:'
version: 1.0.5
@@ -567,7 +576,7 @@ importers:
version: 1.2.0
'@moeru/eventa':
specifier: 'catalog:'
- version: 1.0.2(h3@2.0.2)
+ version: 1.0.2(h3@2.0.8(crossws@0.4.17(srvx@1.0.8))(ocache@0.3.2))
'@moeru/std':
specifier: 'catalog:'
version: 0.1.0-beta.17
@@ -3305,6 +3314,14 @@ packages:
resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
engines: {node: '>= 8'}
+ crossws@0.4.17:
+ resolution: {integrity: sha512-YG2kyfafNXKmu7e6k64yrj6whrw7m5sBIsOSXOlJ/72X3TQWRQuw8DhvXFDcBk8HF77PnTSygnm7xpYCTnFpow==}
+ peerDependencies:
+ srvx: '>=0.11.5'
+ peerDependenciesMeta:
+ srvx:
+ optional: true
+
css-tree@3.2.1:
resolution: {integrity: sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==}
engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0}
@@ -3550,6 +3567,10 @@ packages:
resolution: {integrity: sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==}
engines: {node: '>=0.12'}
+ env-runner@0.3.11:
+ resolution: {integrity: sha512-jKG29hxqGrL9ktunIwmt9aHbP3n995DGUY1KyXS7iGYFoD/yDdpc2CjJt6remFI3x2H1JOmBBMo0KXxjEQ2G6g==}
+ hasBin: true
+
errno@0.1.8:
resolution: {integrity: sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==}
hasBin: true
@@ -3992,6 +4013,19 @@ packages:
ocache:
optional: true
+ h3@2.0.8:
+ resolution: {integrity: sha512-ENuKlF9kC6E6PrvY1PUyE0qkrnnPTIt2/v48uyEEfTVfgoiqBCE0uwwf++2XrWwT1gxjQBDLQ8PAsmZVniKvTw==}
+ engines: {node: '>=20.19.0'}
+ hasBin: true
+ peerDependencies:
+ crossws: ^0.4.12
+ ocache: '>=0.3.0'
+ peerDependenciesMeta:
+ crossws:
+ optional: true
+ ocache:
+ optional: true
+
has-flag@4.0.0:
resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==}
engines: {node: '>=8'}
@@ -4625,6 +4659,11 @@ packages:
engines: {node: '>= 4.4.x'}
hasBin: true
+ nitro@3.0.261103-rc:
+ resolution: {integrity: sha512-tLD7r5fWCtwnQEE4ZkckLUnx1wkWDaLY48dD+BCJtcQ+srSWAUpxJEK3YfTsgN3v+e4UgBYRzuR7zV7Sqns/zA==}
+ engines: {node: '>=22.22.0'}
+ hasBin: true
+
node-fetch-native@1.6.7:
resolution: {integrity: sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==}
@@ -4653,6 +4692,9 @@ packages:
resolution: {integrity: sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==}
engines: {node: '>=12.20.0'}
+ ocache@0.3.2:
+ resolution: {integrity: sha512-wu82jmgyoORtWMBZy6PvyJ8dBhtaOOhCwfRnSKBRbnSLht/RqNUs5WUzOxw5/IikJLMC9e+s1ReReZm4XbrNoQ==}
+
ofetch@1.5.1:
resolution: {integrity: sha512-2W4oUZlVaqAPAil6FUg/difl6YhqhUR7x2eZY4bQCko22UXg3hptq9KLQdqFClV+Wu85UX7hNtdGTngi/1BxcA==}
@@ -5081,6 +5123,11 @@ packages:
engines: {node: '>=20.16.0'}
hasBin: true
+ srvx@1.0.8:
+ resolution: {integrity: sha512-ngFS/J6St4PJfHRHGjr78xkPPtbyXtdMOLNE4x/XZunEK3OhV7aGUWMHf9WTUEy6wm1U0W4okkd82ZFFWRVQTw==}
+ engines: {node: '>=20.16.0'}
+ hasBin: true
+
stackback@0.0.2:
resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==}
@@ -6153,12 +6200,12 @@ snapshots:
- supports-color
- typescript
- '@moeru/eventa@1.0.2(h3@2.0.2)':
+ '@moeru/eventa@1.0.2(h3@2.0.8(crossws@0.4.17(srvx@1.0.8))(ocache@0.3.2))':
dependencies:
nanoid: 6.0.2
picomatch: 4.0.4
optionalDependencies:
- h3: 2.0.2
+ h3: 2.0.8(crossws@0.4.17(srvx@1.0.8))(ocache@0.3.2)
'@moeru/std@0.1.0-beta.17': {}
@@ -8024,6 +8071,10 @@ snapshots:
shebang-command: 2.0.0
which: 2.0.2
+ crossws@0.4.17(srvx@1.0.8):
+ optionalDependencies:
+ srvx: 1.0.8
+
css-tree@3.2.1:
dependencies:
mdn-data: 2.27.1
@@ -8258,6 +8309,11 @@ snapshots:
entities@7.0.1: {}
+ env-runner@0.3.11:
+ dependencies:
+ crossws: 0.4.17(srvx@1.0.8)
+ srvx: 1.0.8
+
errno@0.1.8:
dependencies:
prr: 1.0.1
@@ -8792,10 +8848,21 @@ snapshots:
dependencies:
duplexer: 0.1.2
- h3@2.0.2:
+ h3@2.0.2(crossws@0.4.17(srvx@1.0.5))(ocache@0.3.2):
dependencies:
rou3: 1.0.0
srvx: 1.0.5
+ optionalDependencies:
+ crossws: 0.4.17(srvx@1.0.8)
+ ocache: 0.3.2
+
+ h3@2.0.8(crossws@0.4.17(srvx@1.0.8))(ocache@0.3.2):
+ dependencies:
+ rou3: 1.0.0
+ srvx: 1.0.8
+ optionalDependencies:
+ crossws: 0.4.17(srvx@1.0.8)
+ ocache: 0.3.2
has-flag@4.0.0: {}
@@ -9545,6 +9612,15 @@ snapshots:
sax: 1.6.1
optional: true
+ nitro@3.0.261103-rc:
+ dependencies:
+ crossws: 0.4.17(srvx@1.0.8)
+ env-runner: 0.3.11
+ h3: 2.0.8(crossws@0.4.17(srvx@1.0.8))(ocache@0.3.2)
+ ocache: 0.3.2
+ rou3: 1.0.0
+ srvx: 1.0.8
+
node-fetch-native@1.6.7: {}
node-releases@2.0.37: {}
@@ -9568,6 +9644,8 @@ snapshots:
obug@2.1.3: {}
+ ocache@0.3.2: {}
+
ofetch@1.5.1:
dependencies:
destr: 2.0.5
@@ -10082,6 +10160,8 @@ snapshots:
srvx@1.0.5: {}
+ srvx@1.0.8: {}
+
stackback@0.0.2: {}
std-env@4.1.0: {}
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 32408e7..2d8e054 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -71,6 +71,7 @@ catalog:
h3: ^2.0.2
meow: ^14.1.0
nano-staged: ^1.0.2
+ nitro: 3.0.261103-rc
oxc-minify: ^0.138.0
oxlint: ^1.72.0
pinia: ^4.0.3