Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions docs/content/en/guide/learn/reports-and-comparisons.md
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,14 @@ vieval report attempts --config vieval.config.ts

`attempts` lists the attempts in a report root, newest first: per experiment and attempt, the workspaces, the passed, failed, and no-result case counts, and the completion rate. "←" marks the attempt that `vieval run --resume` continues without an id; notes show which attempt an attempt resumed, interrupts, and runs killed before they wrote a summary. Pass a report root, or `--config` to list the reports next to it; filter with `--experiment` and `--workspace`, and choose `--format table|json`.

## Share reports

```bash [Terminal]
vieval report export .vieval/reports --output report-site
```

Deploy `report-site` to a static host, then share the URL.

## Compare two existing report sets

The report comparison command requires a left baseline and a right candidate:
Expand Down
8 changes: 8 additions & 0 deletions docs/content/zh-hans/guide/learn/reports-and-comparisons.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,14 @@ vieval report attempts --config vieval.config.ts

`attempts` 按时间从新到旧列出报告目录中的尝试:每个实验和尝试一行,显示涉及的工作区、通过、失败和没有结果的用例数,以及执行成功率。"←" 标记 `vieval run --resume` 不指定标识时会继续的那次尝试;备注列显示该尝试继续自哪次尝试、是否被中断,以及没来得及写入摘要就被终止的运行。可以传入报告目录,或用 `--config` 列出配置文件旁边的报告;用 `--experiment` 和 `--workspace` 筛选,用 `--format table|json` 选择输出格式。

## 分享报告

```bash [终端]
vieval report export .vieval/reports --output report-site
```

将 `report-site` 部署到静态托管服务后,即可分享链接。

## 比较两组已有报告

比较命令的第一个目录是基线,第二个目录是候选方案:
Expand Down
10 changes: 10 additions & 0 deletions packages/ui/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,16 @@ vieval viewer reports-a reports-b --evidence-root /data/episodes --stale-after 3
| `--evidence-root <dir>` | — | Extra directory that `*.evidence` metric paths may point into (repeatable). The `.vieval` directory above each report root is always allowed. |
| `--stale-after <min>` | `60` | A run without an end event whose last event is older is shown as `stale`. |

## Share a static report

```sh
vieval report export .vieval/reports --output report-site
```

Export requires Node.js 22.22 or later and a new output directory. Serve that directory over HTTP and share the URL of
a run, case, or comparison. Repeat `--run <runKey>` to select runs. Attachments are included by default. Add
`--include-evidence` to publish linked external evidence and `--evidence-root <dir>` to allow an additional directory.

## Programmatic use

```ts
Expand Down
45 changes: 22 additions & 23 deletions packages/ui/client/src/api.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import type {
ArtifactDescriptor,
CaseDetail,
ComparisonResponse,
EvidenceLink,
ExternalLogsResponse,
FileTreeResponse,
ImagePreview,
Expand All @@ -18,22 +16,17 @@ import type {
RunSnapshot,
ScoresResponse,
ScreenshotTimeline,
TaskNode,
TimelineResponse,
TreeResponse,
} from '../../src/shared/contract'

/**
* Error of one viewer API request.
*
* `status` is the HTTP status, or `0` when the server could not be reached at all.
*/
export class ApiError extends Error {
constructor(readonly status: number, message: string) {
super(message)
this.name = 'ApiError'
}
}
import { ApiError } from './apiError'
import { openPublishedReport } from './publishedReport'

export { ApiError } from './apiError'

/** Whether the Viewer opened a static report publication. */
export const isPublishedReport = document.querySelector('meta[name="vieval-report"][content="static"]') != null
const published = isPublishedReport ? openPublishedReport(new URL('./', document.baseURI)) : undefined

/** Base of every API route; the UI is served by the same server, so it is relative to the page. */
const base = new URL('api/', document.baseURI)
Expand Down Expand Up @@ -73,11 +66,9 @@ function url(path: string, query: Record<string, number | string | undefined> =
/**
* Typed client of the `@vieval/ui` HTTP API (contract in `API.md` and `src/shared/contract.ts`).
*/
export const api = {
artifacts: (runKey: string, ref: Partial<CaseRef>) => get<ArtifactDescriptor[]>(runPath(runKey, '/artifacts'), ref),
const live = {
case: (runKey: string, ref: CaseRef) => get<CaseDetail>(runPath(runKey, '/case'), { ...ref }),
compare: (view: 'attempts' | 'experiments' | 'runs', query: Record<string, string | undefined>) => get<ComparisonResponse>(`compare/${view}`, query),
evidence: (runKey: string, ref: Partial<CaseRef>) => get<EvidenceLink[]>(runPath(runKey, '/evidence'), ref),
evidenceTimeline: (runKey: string, linkId: string) => get<ScreenshotTimeline>(runPath(runKey, `/evidence/${encodeURIComponent(linkId)}/timeline`)),
evidenceTree: (runKey: string, linkId: string) => get<FileTreeResponse>(runPath(runKey, `/evidence/${encodeURIComponent(linkId)}/tree`)),
externalLogs: (runKey: string, ref: CaseRef, offset: number, limit: number) => get<ExternalLogsResponse>(runPath(runKey, '/case/external-logs'), { ...ref, limit, offset }),
Expand All @@ -95,11 +86,12 @@ export const api = {
runs: () => get<RunListItem[]>('runs'),
scores: (runKey: string) => get<ScoresResponse>(runPath(runKey, '/scores')),
snapshot: (runKey: string) => get<RunSnapshot>(runPath(runKey, '/snapshot')),
task: (runKey: string, taskId: string) => get<TaskNode>(runPath(runKey, '/task'), { taskId }),
timeline: (runKey: string, ref: Partial<CaseRef>) => get<TimelineResponse>(runPath(runKey, '/timeline'), ref),
tree: () => get<TreeResponse>('tree'),
}

/** Read operations of the Viewer, backed by the server or a static publication. */
export const api: typeof live = published ?? live

/**
* Where one previewable file is read from: a run artifact or a file of an evidence directory.
*
Expand All @@ -118,6 +110,8 @@ export interface FileLocation {
* placeholder then.
*/
export function imagePreview(location: FileLocation): Promise<ImagePreview> {
if (published != null)
return Promise.reject(new ApiError(415, 'Image placeholders are generated by the live Viewer.'))
const route = location.linkId == null
? runPath(location.runKey, '/artifacts/preview')
: runPath(location.runKey, `/evidence/${encodeURIComponent(location.linkId)}/preview`)
Expand All @@ -126,6 +120,8 @@ export function imagePreview(location: FileLocation): Promise<ImagePreview> {

/** Reads `limit` complete lines of a JSONL file from line `offset`. */
export function jsonlPage(location: FileLocation, offset: number, limit: number): Promise<JsonlPage> {
if (published != null)
return published.jsonl(location, offset, limit)
const route = location.linkId == null
? runPath(location.runKey, '/artifacts/jsonl')
: runPath(location.runKey, `/evidence/${encodeURIComponent(location.linkId)}/jsonl`)
Expand All @@ -139,6 +135,8 @@ export function openRunStream(runKey: string, after: number): EventSource {

/** URL of the file's bytes; `download` adds `Content-Disposition: attachment`. */
export function rawUrl(location: FileLocation, download = false): string {
if (published != null)
return published.rawUrl(location)
const route = location.linkId == null
? runPath(location.runKey, '/artifacts/raw')
: runPath(location.runKey, `/evidence/${encodeURIComponent(location.linkId)}/raw`)
Expand All @@ -165,9 +163,10 @@ export async function readText(location: FileLocation, maxBytes: number): Promis
const body = await response.json().catch(() => undefined) as undefined | { message?: string }
throw new ApiError(response.status, body?.message ?? `${response.status} ${response.statusText}`)
}
const text = await response.text()
const bytes = new Uint8Array(await response.arrayBuffer())
const text = new TextDecoder().decode(bytes.subarray(0, maxBytes))
// `Content-Range: bytes 0-1023/52341` carries the full size of a partial response.
const total = Number(response.headers.get('content-range')?.split('/')[1] ?? Number.NaN)
const size = Number.isFinite(total) ? total : text.length
return { text, total: size, truncated: response.status === 206 && size > maxBytes }
const size = Number.isFinite(total) ? total : bytes.length
return { text, total: size, truncated: size > maxBytes }
}
7 changes: 7 additions & 0 deletions packages/ui/client/src/apiError.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
/** A Viewer request failure, with 0 for an unreachable data source. */
export class ApiError extends Error {
constructor(readonly status: number, message: string) {
super(message)
this.name = 'ApiError'
}
}
6 changes: 4 additions & 2 deletions packages/ui/client/src/composables/useCatalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { defineQuery, useQuery } from '@pinia/colada'
import { useDocumentVisibility, useIntervalFn } from '@vueuse/core'
import { computed, watch } from 'vue'

import { api } from '../api'
import { api, isPublishedReport } from '../api'
import { mergeMatrices } from '../labels'
import { headlineValue } from '../scores'

Expand Down Expand Up @@ -124,9 +124,11 @@ export function sumCounts(runs: readonly RunListItem[]): CaseCounts {
*/
export const useCatalog = defineQuery((): Catalog => {
const query = useQuery({ key: RUNS_KEY, query: () => api.runs() })
const polling = useIntervalFn(() => void query.refetch(), REFRESH_MS)
const polling = useIntervalFn(() => void query.refetch(), REFRESH_MS, { immediate: !isPublishedReport })
// A hidden tab does not poll; it re-reads the runs once it is shown again.
watch(useDocumentVisibility(), (visibility) => {
if (isPublishedReport)
return
if (visibility === 'hidden') {
polling.pause()
return
Expand Down
196 changes: 196 additions & 0 deletions packages/ui/client/src/publishedReport.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
import type { ExternalLogsResponse, JsonlPage, RunListItem } from '../../src/shared/contract'
import type { PublishedCatalog, PublishedComparison, PublishedRun } from '../../src/shared/publishedReport'
import type { CaseRef, FileLocation } from './api'

import { buildMatrix, compareAttempts, compareExperiments, compareRuns, inScope } from '../../src/shared/compare'
import { buildLogs } from '../../src/shared/logs'
import { CATALOG_PATH, fileKey, runDataPath, telemetryPath } from '../../src/shared/publishedReport'
import { ApiError } from './apiError'

/**
* Reads a static Viewer publication using paths relative to the site.
*
* Use when:
* - the Viewer is served alongside a Nitro-generated catalog and run files
*
* Returns:
* - the read operations used by the Viewer, with data cached for the life of the page
*/
export function openPublishedReport(base: URL) {
const cache = new Map<string, Promise<unknown>>()
const runs = new Map<string, PublishedRun>()
const lines = new Map<string, Promise<{ partialTail: boolean, rows: unknown[] }>>()
async function read<T>(path: string): Promise<T> {
let pending = cache.get(path)
if (pending == null) {
pending = fetch(new URL(path, base)).then(async (response) => {
if (!response.ok)
throw new ApiError(response.status, 'This resource was not included in the published report.')
return response.json()
}).catch((error) => {
cache.delete(path)
throw error
})
cache.set(path, pending)
}
return await pending as T
}

async function catalog(): Promise<RunListItem[]> {
const data = await read<PublishedCatalog>(CATALOG_PATH)
if (data.version !== 1)
throw new ApiError(422, 'This report needs the Viewer that was published with it.')
return data.runs
}

function indexOf(items: RunListItem[], runKey: string): number {
const index = items.findIndex(item => item.runKey === runKey)
if (index < 0)
throw new ApiError(404, 'This run was not included in the published report.')
return index
}

async function run(runKey: string): Promise<PublishedRun> {
const data = await read<PublishedRun>(runDataPath(indexOf(await catalog(), runKey), 'run'))
runs.set(runKey, data)
return data
}

async function task(runKey: string, taskId: string) {
const data = await run(runKey)
if (!Object.hasOwn(data.tasks, taskId))
throw new ApiError(404, 'Task not found.')
return data.tasks[taskId]
}

async function caseData(runKey: string, ref: CaseRef) {
const { cases } = await task(runKey, ref.taskId)
if (!Object.hasOwn(cases, ref.caseId))
throw new ApiError(404, 'Case not found.')
return cases[ref.caseId]
}

async function comparison(item: RunListItem, index: number) {
const data = await read<PublishedComparison>(runDataPath(index, 'comparison'))
return { ...data, item }
}

async function comparisonsInScope(scope: Parameters<typeof inScope>[1]) {
return Promise.all((await catalog()).flatMap((item, index) => inScope(item, scope) ? [comparison(item, index)] : []))
}

function rawUrl(location: FileLocation): string {
const path = runs.get(location.runKey)?.files[fileKey(location.path, location.linkId)]
if (path == null)
throw new ApiError(404, 'This file was not included in the published report.')
return new URL(path, base).href
}

return {
async case(runKey: string, ref: CaseRef) {
return (await caseData(runKey, ref)).detail
},
async compare(view: 'attempts' | 'experiments' | 'runs', query: Record<string, string | undefined>) {
if (view === 'runs') {
const keys = (query.runs ?? '').split(',').map(key => key.trim()).filter(Boolean)
if (keys.length < 2)
throw new ApiError(400, 'Choose at least two runs to compare.')
const items = await catalog()
const indexes = keys.map(key => indexOf(items, key))
return compareRuns(await Promise.all(indexes.map(index => comparison(items[index], index))))
}
const scope = { experiment: query.experiment ?? '', project: query.project ?? '', workspace: query.workspace ?? '' }
const filter = view === 'attempts' ? scope : { project: scope.project, workspace: scope.workspace }
const data = await comparisonsInScope(filter)
return view === 'attempts' ? compareAttempts(data, scope) : compareExperiments(data, scope)
},
async evidenceTimeline(runKey: string, linkId: string) {
return (await run(runKey)).evidenceViews[linkId]?.timeline ?? { frames: [] }
},
async evidenceTree(runKey: string, linkId: string) {
const view = (await run(runKey)).evidenceViews[linkId]
if (view == null)
throw new ApiError(404, 'This evidence directory was not included in the published report.')
return view.tree
},
async externalLogs(runKey: string, ref: CaseRef, offset: number, limit: number): Promise<ExternalLogsResponse> {
const data = (await caseData(runKey, ref)).externalLogs ?? { correlation: { keys: [], podNames: [], window: {} }, files: [], lines: [], nextOffset: null, total: 0 }
const end = offset + limit
return { ...data, lines: data.lines.slice(offset, end), nextOffset: end < data.lines.length ? end : null }
},
async jsonl(location: FileLocation, offset: number, limit: number): Promise<JsonlPage> {
const url = rawUrl(location)
let pending = lines.get(url)
if (pending == null) {
pending = fetch(url).then(async (response) => {
if (!response.ok)
throw new ApiError(response.status, 'Cannot read the published file.')
const rows = (await response.text()).split('\n')
const partialTail = rows.pop() !== ''
return {
partialTail,
rows: rows.map((row) => {
if (row.trim().length === 0)
return undefined
try {
return JSON.parse(row) as unknown
}
catch {
return { $error: 'Invalid JSON', $raw: row }
}
}),
}
}).catch((error) => {
lines.delete(url)
throw error
})
lines.set(url, pending)
}
const data = await pending
return { limit, offset, partialTail: data.partialTail, rows: data.rows.slice(offset, offset + limit).filter(row => row !== undefined), totalLines: data.rows.length }
},
async logs(runKey: string, query: Parameters<typeof buildLogs>[1]) {
return buildLogs((await run(runKey)).logs, query)
},
async matrix(scope: Parameters<typeof inScope>[1]) {
return buildMatrix(await comparisonsInScope(scope))
},
async metrics(runKey: string, ref: Partial<CaseRef> = {}) {
return ref.taskId == null ? (await run(runKey)).metrics : (await task(runKey, ref.taskId)).metrics
},
async otelLogs(runKey: string) {
return (await run(runKey)).otel.logs
},
async otelMetrics(runKey: string) {
return (await run(runKey)).otel.metrics
},
async otelRaw(runKey: string, signal: 'logs' | 'metrics' | 'traces') {
return read<unknown>(telemetryPath(indexOf(await catalog(), runKey), signal))
},
async otelTraces(runKey: string) {
return (await run(runKey)).otel.traces
},
async progress(runKey: string) {
return (await run(runKey)).progress
},
rawUrl,
async run(runKey: string) {
const items = await catalog()
return items[indexOf(items, runKey)]
},
runs: catalog,
async scores(runKey: string) {
return (await run(runKey)).scores
},
async snapshot(runKey: string) {
return (await run(runKey)).snapshot
},
async timeline(runKey: string, ref: Partial<CaseRef>) {
if (ref.taskId == null)
return (await run(runKey)).timeline
return ref.caseId == null
? (await task(runKey, ref.taskId)).timeline
: (await caseData(runKey, { caseId: ref.caseId, taskId: ref.taskId })).timeline
},
}
}
2 changes: 2 additions & 0 deletions packages/ui/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@
"@napi-rs/image": "catalog:",
"es-toolkit": "catalog:",
"h3": "catalog:",
"nitro": "catalog:",
"rollup": "catalog:",
"srvx": "catalog:",
"thumbhash": "catalog:",
"tinyglobby": "catalog:",
Expand Down
1 change: 1 addition & 0 deletions packages/ui/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { createViewerApp } from './server/app'
import { createEvidenceService } from './server/evidence'
import { createRunStore } from './server/store'

export { exportReport } from './publishReport'
export type { CreateViewerAppOptions } from './server/app'
export { createViewerApp } from './server/app'
export { createEventTail, parseEventLine } from './server/event-tail'
Expand Down
Loading
Loading