Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,7 @@ without a linked personal account. Account signup and rig registration remain on
- Measure one request at a time. A throughput configuration — OpenVINO's
`PERFORMANCE_HINT=THROUGHPUT`, vLLM serving many sequences at once, llama.cpp with parallel slots —
reports the total across concurrent requests, which is a larger and different number from what one
person sees waiting for one answer. If that is what you measured, say so in the flags and set the
batch size.
person sees waiting for one answer. If that is what you measured, record it as `concurrency` with `decodeScope` set to `total`: it ranks on the Throughput board instead of against single requests, and several request counts of the same setup draw a scaling curve on the rig page.
- Report the runtime version. An evidence link is optional: benchmark logs, scripts, a gist, a report,
or a repository can help others check your result. Any full HTTPS URL works; no separate repository
is required. In PR JSON, use `evidenceUrl`. The submission PR is recorded separately.
Expand Down
2 changes: 1 addition & 1 deletion frontend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"og:assets": "node scripts/generate-og-assets.mjs",
"og:preview": "node scripts/og-preview.mjs",
"supabase:db:push:prod:dry-run": "supabase --workdir .. db push --project-ref ujlacyedspjfekotemmd --dry-run --skip-vault",
"results:test": "node --test scripts/tests/pr-results.test.mjs scripts/tests/catalog-sync.test.mjs scripts/tests/result-evidence.test.mjs scripts/tests/hardware-chart.test.mjs scripts/tests/supabase-fetch.test.mjs scripts/tests/rig-presentation.test.mjs scripts/tests/board-submissions.test.mjs scripts/tests/agent-api.test.mjs scripts/tests/result-integrity.test.mjs scripts/tests/result-conditions.test.mjs",
"results:test": "node --test scripts/tests/pr-results.test.mjs scripts/tests/catalog-sync.test.mjs scripts/tests/result-evidence.test.mjs scripts/tests/hardware-chart.test.mjs scripts/tests/supabase-fetch.test.mjs scripts/tests/rig-presentation.test.mjs scripts/tests/board-submissions.test.mjs scripts/tests/streams.test.mjs scripts/tests/agent-api.test.mjs scripts/tests/result-integrity.test.mjs scripts/tests/result-conditions.test.mjs",
"supabase:db:push:prod": "supabase --workdir .. db push --project-ref ujlacyedspjfekotemmd --skip-vault",
"agents:test": "node --test scripts/tests/agent-api.test.mjs scripts/tests/result-integrity.test.mjs scripts/tests/result-conditions.test.mjs",
"agents:docs": "node scripts/generate-agent-docs.mjs",
Expand Down
18 changes: 15 additions & 3 deletions frontend/scripts/tests/board-submissions.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ const results = [
run(14, 'intel-arc-pro-b60', 1, 50, { quant_id: 'q8_0', ...standard, decode_method: 'wall' }),
run(15, 'intel-arc-pro-b60', 2, 100, { quant_id: 'q8_0' }),
run(16, 'intel-arc-pro-b70', 1, 20, { quant_id: 'q8_0', ...standard, concurrency: 4, decode_scope: 'per_request' }),
// A machine total across 8 requests: a different quantity, so it ranks only under Throughput.
run(17, 'intel-arc-pro-b70', 1, 500, { quant_id: 'q8_0', ...standard, concurrency: 8, decode_scope: 'total', decode_method: 'wall' }),
]

before(async () => {
Expand Down Expand Up @@ -111,15 +113,16 @@ test('a quant outside the suggested list gets its own board once it has visible
test('setup tabs rank only results measured that way, and count every setup', async () => {
const q8 = (setup, extra = {}) => supabaseApi.board('qwen3-8-27b', 'q8_0', { kind: 'components', setup, ...extra })
const all = await q8(undefined)
assert.deepEqual(ids(all), ['15', '13', '14'], 'All results keeps everything, best per part')
assert.deepEqual(all.setupCounts, { standard: 1, speculative: 1, long: 0, concurrent: 1, all: 3 })
assert.deepEqual(ids(all), ['15', '13', '14'], 'All results keeps every per-request number, best per part; the 8-request total stays out')
assert.deepEqual(all.setupCounts, { standard: 1, speculative: 1, long: 0, concurrent: 1, throughput: 1, all: 3 })
assert.deepEqual(ids(await q8('throughput')), ['17'])
assert.deepEqual(ids(await q8('standard')), ['12'], 'speculation, wall clock and 4 requests stay out of Standard')
assert.deepEqual(ids(await q8('speculative')), ['13'])
assert.deepEqual(ids(await q8('concurrent')), ['16'])
assert.equal((await q8('long')).board.total, 0)
assert.deepEqual(ids(await q8('all')), ids(all))
// Other filters still apply inside a setup, and the counts follow them.
assert.deepEqual((await q8(undefined, { q: 'B60' })).setupCounts, { standard: 0, speculative: 0, long: 0, concurrent: 0, all: 2 })
assert.deepEqual((await q8(undefined, { q: 'B60' })).setupCounts, { standard: 0, speculative: 0, long: 0, concurrent: 0, throughput: 0, all: 2 })
})

test('a result page shows its own rank, or names the result that holds its part\'s row', async () => {
Expand All @@ -140,4 +143,13 @@ test('a result page ranks within the result\'s own setup', async () => {
// Conditions that match no setup (wall clock) or none recorded rank on All results.
assert.deepEqual(await rank('14'), { kind: 'components', setup: 'all', boardSize: 3, position: 3 })
assert.deepEqual(await rank('15'), { kind: 'components', setup: 'all', boardSize: 3, position: 1 })
assert.deepEqual(await rank('17'), { kind: 'components', setup: 'throughput', boardSize: 1, position: 1 })
})

test('a total across requests never counts as a best decode speed', async () => {
const rig = await supabaseApi.rig('1')
assert.ok(rig.results.some((result) => result.id === '17'), 'the total is still a result of the rig')
assert.ok(rig.chart.every((bar) => bar.tps < 500), 'the rig chart shows per-request rates only')
const top = await supabaseApi.topResults({ model: 'qwen3-8-27b', quant: 'q8_0' })
assert.ok(top.items.every((row) => row.result.id !== '17'))
})
73 changes: 73 additions & 0 deletions frontend/scripts/tests/streams.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import { test, before } from 'node:test'
import assert from 'node:assert/strict'
import { fileURLToPath } from 'node:url'
import { build } from 'esbuild'

// Decode rate against concurrent requests (issue #130): totals and per-request rates are separate curves, a single
// request seeds both, and nothing is drawn from a setup measured at only one request count.
let streamSeries, isTotalThroughput, SETUP_BY_ID, setupOf
before(async () => {
const bundle = await build({
stdin: {
contents: `export { streamSeries } from './lib/streams'; export { isTotalThroughput, SETUP_BY_ID, setupOf } from './lib/setups';`,
resolveDir: fileURLToPath(new URL('../../src', import.meta.url)),
},
alias: { '@': fileURLToPath(new URL('../../src', import.meta.url)) },
bundle: true, write: false, platform: 'browser', format: 'esm', logLevel: 'silent',
})
;({ streamSeries, isTotalThroughput, SETUP_BY_ID, setupOf } = await import(`data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString('base64')}`))
})

const result = (id, concurrency, decodeTps, extra = {}) => ({
id: String(id), rigId: '1', modelId: 'qwen3-8b', quant: 'int4', runtimeId: 'openvino-genai', runtimeVersion: 'x', execution: 'stock',
decodeTps, runDate: '2026-11-13', concurrency, moderation: { hidden: false }, verification: { status: 'self_reported', confirmations: 0 },
createdAt: '', updatedAt: '', ...extra,
})

test('a single request seeds both curves and totals and per-request rates stay apart', () => {
const series = streamSeries([
result(1, 1, 18.5),
result(2, 8, 136, { decodeScope: 'total' }),
result(3, 48, 227.5, { decodeScope: 'total' }),
result(4, 8, 17, { decodeScope: 'per_request' }),
result(5, 4, 12, { decodeScope: 'per_request' }),
])
assert.deepEqual(series.map((s) => [s.scope, s.points.map((p) => [p.concurrency, p.decodeTps])]), [
['per_request', [[1, 18.5], [4, 12], [8, 17]]],
['total', [[1, 18.5], [8, 136], [48, 227.5]]],
])
assert.equal(series[0].label, 'Qwen3-8B INT4')
})

test('one request count is not a curve; repeats at a count keep the best; legacy batch size and hidden results are ignored', () => {
assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, 1, 19.0)]), [])
assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, 8, 100, { decodeScope: 'total' }), result(3, 8, 130, { decodeScope: 'total' })]).map((s) => s.points.map((p) => [p.concurrency, p.decodeTps, p.resultId])), [[[1, 18.5, '1'], [8, 130, '3']]])
assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, undefined, 100, { batchSize: 8 })]), [])
assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, 8, 100, { decodeScope: 'total', moderation: { hidden: true } })]), [])
// A total at several requests with no decodeScope recorded is ambiguous and stays out.
assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, 8, 100)]), [])
})

test('setups are keyed by model, quant, runtime, build and part', () => {
const series = streamSeries([
result(1, 1, 18.5), result(2, 8, 136, { decodeScope: 'total' }),
result(3, 1, 20, { runtimeId: 'cascadia' }), result(4, 8, 150, { runtimeId: 'cascadia', decodeScope: 'total' }),
result(5, 1, 9, { componentId: 'intel-core-ultra-x7-358h', componentQuantity: 1, component: { name: 'Core Ultra X7 358H' } }),
result(6, 8, 40, { componentId: 'intel-core-ultra-x7-358h', componentQuantity: 1, component: { name: 'Core Ultra X7 358H' }, decodeScope: 'total' }),
])
assert.deepEqual(series.map((s) => [s.scope, s.runtimeId, s.label]), [
['total', 'cascadia', 'Qwen3-8B INT4'],
['total', 'openvino-genai', 'Qwen3-8B INT4'],
['total', 'openvino-genai', 'Qwen3-8B INT4 · 1× Core Ultra X7 358H'],
])
})

test('the Throughput setup takes totals across requests and nothing else', () => {
assert.equal(isTotalThroughput({ concurrency: 8, decodeScope: 'total' }), true)
assert.equal(isTotalThroughput({ concurrency: 1 }), false)
assert.equal(isTotalThroughput({ concurrency: 8, decodeScope: 'per_request' }), false)
assert.equal(isTotalThroughput({ concurrency: 8 }), false)
assert.equal(setupOf({ concurrency: 144, decodeScope: 'total', decodeMethod: 'wall' })?.id, 'throughput')
assert.equal(SETUP_BY_ID.throughput.match({ concurrency: 4, decodeScope: 'per_request', inputTokens: 512, decodeMethod: 'streamed' }), false)
assert.equal(setupOf({ concurrency: 4, decodeScope: 'per_request', inputTokens: 512, decodeMethod: 'streamed' })?.id, 'concurrent')
})
91 changes: 91 additions & 0 deletions frontend/src/components/StreamsChart.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
import { useMemo } from 'react'
import { useNavigate } from 'react-router-dom'
import { CartesianGrid, Line, LineChart, XAxis, YAxis } from 'recharts'
import { ChartContainer, ChartTooltip, type ChartConfig } from '@/components/ui/chart'
import { RuntimeDot } from '@/components/RuntimeBadge'
import type { Runtime } from '@/lib/api/types'
import type { StreamSeries } from '@/lib/streams'
import { fmtTps, pluralize } from '@/lib/format'

const SCOPE_TITLE = { total: 'Total decode tok/s across requests', per_request: 'Decode tok/s each request sees' } as const

/**
* Decode rate against concurrent requests, one line per measured setup, colored by runtime. Totals and per-request
* rates are different quantities, so each scope gets its own chart. Points link to the result that holds them.
*/
export function StreamsChart({ series, runtimes, className }: { series: StreamSeries[]; runtimes: Runtime[]; className?: string }) {
const navigate = useNavigate()
const byId = useMemo(() => Object.fromEntries(runtimes.map((r) => [r.id, r])), [runtimes])
const scopes = (['total', 'per_request'] as const).filter((scope) => series.some((s) => s.scope === scope))
if (!scopes.length) return null
return (
<div className={className}>
{scopes.map((scope) => {
const lines = series.filter((s) => s.scope === scope)
const xs = [...new Set(lines.flatMap((s) => s.points.map((p) => p.concurrency)))].sort((a, b) => a - b)
const data = xs.map((x) => Object.fromEntries([['concurrency', x], ...lines.map((s) => [s.key, s.points.find((p) => p.concurrency === x)?.decodeTps ?? null])]))
const config: ChartConfig = Object.fromEntries(lines.map((s) => [s.key, { label: s.label, color: byId[s.runtimeId]?.color ?? 'var(--muted-foreground)' }]))
const summary = lines.map((s) => `${s.label}: ${s.points.map((p) => `${fmtTps(p.decodeTps)} at ${p.concurrency}`).join(', ')}`).join('; ')
return (
<div key={scope} className="py-4" role="img" aria-label={`${SCOPE_TITLE[scope]} by concurrent requests. ${summary}`}>
<div className="mb-2 text-xs font-medium text-muted-foreground">{SCOPE_TITLE[scope]}</div>
<ChartContainer config={config} className="w-full" style={{ height: 260 }}>
<LineChart data={data} margin={{ left: 0, right: 16, top: 8, bottom: 4 }}>
<CartesianGrid strokeDasharray="3 3" stroke="var(--border)" />
<XAxis
dataKey="concurrency" type="number" domain={['dataMin', 'dataMax']} ticks={xs} tickLine={false} axisLine={false}
tick={{ fill: 'var(--muted-foreground)', fontSize: 11 }} label={{ value: 'concurrent requests', position: 'insideBottomRight', offset: -2, fill: 'var(--muted-foreground)', fontSize: 11 }}
/>
<YAxis type="number" tickLine={false} axisLine={false} width={48} tick={{ fill: 'var(--muted-foreground)', fontSize: 11 }} tickFormatter={(v: number) => fmtTps(v)} />
<ChartTooltip
cursor={{ stroke: 'var(--border)' }}
content={({ active, payload, label }) => {
if (!active || !payload?.length) return null
return (
<div className="rounded-lg border bg-popover px-3 py-2 text-xs shadow-md">
<div className="font-medium text-foreground">{pluralize(Number(label), 'concurrent request')}</div>
{payload.filter((p) => p.value != null).map((p) => {
const s = lines.find((line) => line.key === p.dataKey)
return s ? (
<div key={s.key} className="mt-1 flex items-center gap-1.5 text-muted-foreground">
<RuntimeDot runtime={byId[s.runtimeId]} /> {s.label}
<span className="ml-2 font-mono text-foreground">{fmtTps(Number(p.value))} tok/s</span>
</div>
) : null
})}
</div>
)
}}
/>
{lines.map((s) => (
<Line
key={s.key} dataKey={s.key} type="monotone" connectNulls stroke={config[s.key].color} strokeWidth={2}
activeDot={{ r: 6 }} isAnimationActive={false}
// Each point links to the result that holds it.
dot={({ cx, cy, payload, value }: { cx?: number; cy?: number; payload?: { concurrency?: number }; value?: number | null }) => {
const point = s.points.find((p) => p.concurrency === payload?.concurrency)
if (cx == null || cy == null || value == null || !point) return <g key={`${s.key}-${payload?.concurrency}`} />
return (
<circle
key={`${s.key}-${point.concurrency}`} cx={cx} cy={cy} r={4} fill={config[s.key].color} className="cursor-pointer"
onClick={() => navigate(`/results/${point.resultId}`)}
/>
)
}}
/>
))}
</LineChart>
</ChartContainer>
<div className="mt-2 flex flex-wrap gap-x-4 gap-y-1 text-xs text-muted-foreground">
{lines.map((s) => (
<span key={s.key} className="inline-flex items-center gap-1.5">
<RuntimeDot runtime={byId[s.runtimeId]} /> {s.label} · {byId[s.runtimeId]?.name ?? s.runtimeId}
</span>
))}
</div>
</div>
)
})}
</div>
)
}
4 changes: 3 additions & 1 deletion frontend/src/lib/api/hardware-chart.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import { MODEL_BY_ID, QUANT_BY_ID, RUNTIME_BY_ID } from '@/catalog'
import type { ChartBar, Result } from './types'
import { isTotalThroughput } from '@/lib/setups'

/** Keep each runtime's best run visible, even when another runtime is faster. */
/** Keep each runtime's best run visible, even when another runtime is faster. Totals across concurrent requests do not count. */
export function hardwareChart(results: Result[]): ChartBar[] {
const best = new Map<string, Result>()
for (const result of results) {
if (isTotalThroughput(result)) continue
const key = `${result.modelId}/${result.quant}/${result.runtimeId}`
const current = best.get(key)
if (!current || result.decodeTps > current.decodeTps || (result.decodeTps === current.decodeTps && result.runDate < current.runDate)) {
Expand Down
Loading
Loading