diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6744351..1346f6f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,8 +36,7 @@ without a linked personal account. Account signup and rig registration remain on - Measure one request at a time. A throughput configuration — OpenVINO's `PERFORMANCE_HINT=THROUGHPUT`, vLLM serving many sequences at once, llama.cpp with parallel slots — reports the total across concurrent requests, which is a larger and different number from what one - person sees waiting for one answer. If that is what you measured, say so in the flags and set the - batch size. + person sees waiting for one answer. If that is what you measured, record it as `concurrency` with `decodeScope` set to `total`: it ranks on the Throughput board instead of against single requests, and several request counts of the same setup draw a scaling curve on the rig page. - Report the runtime version. An evidence link is optional: benchmark logs, scripts, a gist, a report, or a repository can help others check your result. Any full HTTPS URL works; no separate repository is required. In PR JSON, use `evidenceUrl`. The submission PR is recorded separately. diff --git a/frontend/package.json b/frontend/package.json index 3399249..9efb3ed 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,7 +14,7 @@ "og:assets": "node scripts/generate-og-assets.mjs", "og:preview": "node scripts/og-preview.mjs", "supabase:db:push:prod:dry-run": "supabase --workdir .. db push --project-ref ujlacyedspjfekotemmd --dry-run --skip-vault", - "results:test": "node --test scripts/tests/pr-results.test.mjs scripts/tests/catalog-sync.test.mjs scripts/tests/result-evidence.test.mjs scripts/tests/hardware-chart.test.mjs scripts/tests/supabase-fetch.test.mjs scripts/tests/rig-presentation.test.mjs scripts/tests/board-submissions.test.mjs scripts/tests/agent-api.test.mjs scripts/tests/result-integrity.test.mjs scripts/tests/result-conditions.test.mjs", + "results:test": "node --test scripts/tests/pr-results.test.mjs scripts/tests/catalog-sync.test.mjs scripts/tests/result-evidence.test.mjs scripts/tests/hardware-chart.test.mjs scripts/tests/supabase-fetch.test.mjs scripts/tests/rig-presentation.test.mjs scripts/tests/board-submissions.test.mjs scripts/tests/streams.test.mjs scripts/tests/agent-api.test.mjs scripts/tests/result-integrity.test.mjs scripts/tests/result-conditions.test.mjs", "supabase:db:push:prod": "supabase --workdir .. db push --project-ref ujlacyedspjfekotemmd --skip-vault", "agents:test": "node --test scripts/tests/agent-api.test.mjs scripts/tests/result-integrity.test.mjs scripts/tests/result-conditions.test.mjs", "agents:docs": "node scripts/generate-agent-docs.mjs", diff --git a/frontend/scripts/tests/board-submissions.test.mjs b/frontend/scripts/tests/board-submissions.test.mjs index 3409135..de45a7b 100644 --- a/frontend/scripts/tests/board-submissions.test.mjs +++ b/frontend/scripts/tests/board-submissions.test.mjs @@ -31,6 +31,8 @@ const results = [ run(14, 'intel-arc-pro-b60', 1, 50, { quant_id: 'q8_0', ...standard, decode_method: 'wall' }), run(15, 'intel-arc-pro-b60', 2, 100, { quant_id: 'q8_0' }), run(16, 'intel-arc-pro-b70', 1, 20, { quant_id: 'q8_0', ...standard, concurrency: 4, decode_scope: 'per_request' }), + // A machine total across 8 requests: a different quantity, so it ranks only under Throughput. + run(17, 'intel-arc-pro-b70', 1, 500, { quant_id: 'q8_0', ...standard, concurrency: 8, decode_scope: 'total', decode_method: 'wall' }), ] before(async () => { @@ -111,15 +113,16 @@ test('a quant outside the suggested list gets its own board once it has visible test('setup tabs rank only results measured that way, and count every setup', async () => { const q8 = (setup, extra = {}) => supabaseApi.board('qwen3-8-27b', 'q8_0', { kind: 'components', setup, ...extra }) const all = await q8(undefined) - assert.deepEqual(ids(all), ['15', '13', '14'], 'All results keeps everything, best per part') - assert.deepEqual(all.setupCounts, { standard: 1, speculative: 1, long: 0, concurrent: 1, all: 3 }) + assert.deepEqual(ids(all), ['15', '13', '14'], 'All results keeps every per-request number, best per part; the 8-request total stays out') + assert.deepEqual(all.setupCounts, { standard: 1, speculative: 1, long: 0, concurrent: 1, throughput: 1, all: 3 }) + assert.deepEqual(ids(await q8('throughput')), ['17']) assert.deepEqual(ids(await q8('standard')), ['12'], 'speculation, wall clock and 4 requests stay out of Standard') assert.deepEqual(ids(await q8('speculative')), ['13']) assert.deepEqual(ids(await q8('concurrent')), ['16']) assert.equal((await q8('long')).board.total, 0) assert.deepEqual(ids(await q8('all')), ids(all)) // Other filters still apply inside a setup, and the counts follow them. - assert.deepEqual((await q8(undefined, { q: 'B60' })).setupCounts, { standard: 0, speculative: 0, long: 0, concurrent: 0, all: 2 }) + assert.deepEqual((await q8(undefined, { q: 'B60' })).setupCounts, { standard: 0, speculative: 0, long: 0, concurrent: 0, throughput: 0, all: 2 }) }) test('a result page shows its own rank, or names the result that holds its part\'s row', async () => { @@ -140,4 +143,13 @@ test('a result page ranks within the result\'s own setup', async () => { // Conditions that match no setup (wall clock) or none recorded rank on All results. assert.deepEqual(await rank('14'), { kind: 'components', setup: 'all', boardSize: 3, position: 3 }) assert.deepEqual(await rank('15'), { kind: 'components', setup: 'all', boardSize: 3, position: 1 }) + assert.deepEqual(await rank('17'), { kind: 'components', setup: 'throughput', boardSize: 1, position: 1 }) +}) + +test('a total across requests never counts as a best decode speed', async () => { + const rig = await supabaseApi.rig('1') + assert.ok(rig.results.some((result) => result.id === '17'), 'the total is still a result of the rig') + assert.ok(rig.chart.every((bar) => bar.tps < 500), 'the rig chart shows per-request rates only') + const top = await supabaseApi.topResults({ model: 'qwen3-8-27b', quant: 'q8_0' }) + assert.ok(top.items.every((row) => row.result.id !== '17')) }) diff --git a/frontend/scripts/tests/streams.test.mjs b/frontend/scripts/tests/streams.test.mjs new file mode 100644 index 0000000..4d7b0b3 --- /dev/null +++ b/frontend/scripts/tests/streams.test.mjs @@ -0,0 +1,73 @@ +import { test, before } from 'node:test' +import assert from 'node:assert/strict' +import { fileURLToPath } from 'node:url' +import { build } from 'esbuild' + +// Decode rate against concurrent requests (issue #130): totals and per-request rates are separate curves, a single +// request seeds both, and nothing is drawn from a setup measured at only one request count. +let streamSeries, isTotalThroughput, SETUP_BY_ID, setupOf +before(async () => { + const bundle = await build({ + stdin: { + contents: `export { streamSeries } from './lib/streams'; export { isTotalThroughput, SETUP_BY_ID, setupOf } from './lib/setups';`, + resolveDir: fileURLToPath(new URL('../../src', import.meta.url)), + }, + alias: { '@': fileURLToPath(new URL('../../src', import.meta.url)) }, + bundle: true, write: false, platform: 'browser', format: 'esm', logLevel: 'silent', + }) + ;({ streamSeries, isTotalThroughput, SETUP_BY_ID, setupOf } = await import(`data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString('base64')}`)) +}) + +const result = (id, concurrency, decodeTps, extra = {}) => ({ + id: String(id), rigId: '1', modelId: 'qwen3-8b', quant: 'int4', runtimeId: 'openvino-genai', runtimeVersion: 'x', execution: 'stock', + decodeTps, runDate: '2026-11-13', concurrency, moderation: { hidden: false }, verification: { status: 'self_reported', confirmations: 0 }, + createdAt: '', updatedAt: '', ...extra, +}) + +test('a single request seeds both curves and totals and per-request rates stay apart', () => { + const series = streamSeries([ + result(1, 1, 18.5), + result(2, 8, 136, { decodeScope: 'total' }), + result(3, 48, 227.5, { decodeScope: 'total' }), + result(4, 8, 17, { decodeScope: 'per_request' }), + result(5, 4, 12, { decodeScope: 'per_request' }), + ]) + assert.deepEqual(series.map((s) => [s.scope, s.points.map((p) => [p.concurrency, p.decodeTps])]), [ + ['per_request', [[1, 18.5], [4, 12], [8, 17]]], + ['total', [[1, 18.5], [8, 136], [48, 227.5]]], + ]) + assert.equal(series[0].label, 'Qwen3-8B INT4') +}) + +test('one request count is not a curve; repeats at a count keep the best; legacy batch size and hidden results are ignored', () => { + assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, 1, 19.0)]), []) + assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, 8, 100, { decodeScope: 'total' }), result(3, 8, 130, { decodeScope: 'total' })]).map((s) => s.points.map((p) => [p.concurrency, p.decodeTps, p.resultId])), [[[1, 18.5, '1'], [8, 130, '3']]]) + assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, undefined, 100, { batchSize: 8 })]), []) + assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, 8, 100, { decodeScope: 'total', moderation: { hidden: true } })]), []) + // A total at several requests with no decodeScope recorded is ambiguous and stays out. + assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, 8, 100)]), []) +}) + +test('setups are keyed by model, quant, runtime, build and part', () => { + const series = streamSeries([ + result(1, 1, 18.5), result(2, 8, 136, { decodeScope: 'total' }), + result(3, 1, 20, { runtimeId: 'cascadia' }), result(4, 8, 150, { runtimeId: 'cascadia', decodeScope: 'total' }), + result(5, 1, 9, { componentId: 'intel-core-ultra-x7-358h', componentQuantity: 1, component: { name: 'Core Ultra X7 358H' } }), + result(6, 8, 40, { componentId: 'intel-core-ultra-x7-358h', componentQuantity: 1, component: { name: 'Core Ultra X7 358H' }, decodeScope: 'total' }), + ]) + assert.deepEqual(series.map((s) => [s.scope, s.runtimeId, s.label]), [ + ['total', 'cascadia', 'Qwen3-8B INT4'], + ['total', 'openvino-genai', 'Qwen3-8B INT4'], + ['total', 'openvino-genai', 'Qwen3-8B INT4 · 1× Core Ultra X7 358H'], + ]) +}) + +test('the Throughput setup takes totals across requests and nothing else', () => { + assert.equal(isTotalThroughput({ concurrency: 8, decodeScope: 'total' }), true) + assert.equal(isTotalThroughput({ concurrency: 1 }), false) + assert.equal(isTotalThroughput({ concurrency: 8, decodeScope: 'per_request' }), false) + assert.equal(isTotalThroughput({ concurrency: 8 }), false) + assert.equal(setupOf({ concurrency: 144, decodeScope: 'total', decodeMethod: 'wall' })?.id, 'throughput') + assert.equal(SETUP_BY_ID.throughput.match({ concurrency: 4, decodeScope: 'per_request', inputTokens: 512, decodeMethod: 'streamed' }), false) + assert.equal(setupOf({ concurrency: 4, decodeScope: 'per_request', inputTokens: 512, decodeMethod: 'streamed' })?.id, 'concurrent') +}) diff --git a/frontend/src/components/StreamsChart.tsx b/frontend/src/components/StreamsChart.tsx new file mode 100644 index 0000000..d6d26a7 --- /dev/null +++ b/frontend/src/components/StreamsChart.tsx @@ -0,0 +1,91 @@ +import { useMemo } from 'react' +import { useNavigate } from 'react-router-dom' +import { CartesianGrid, Line, LineChart, XAxis, YAxis } from 'recharts' +import { ChartContainer, ChartTooltip, type ChartConfig } from '@/components/ui/chart' +import { RuntimeDot } from '@/components/RuntimeBadge' +import type { Runtime } from '@/lib/api/types' +import type { StreamSeries } from '@/lib/streams' +import { fmtTps, pluralize } from '@/lib/format' + +const SCOPE_TITLE = { total: 'Total decode tok/s across requests', per_request: 'Decode tok/s each request sees' } as const + +/** + * Decode rate against concurrent requests, one line per measured setup, colored by runtime. Totals and per-request + * rates are different quantities, so each scope gets its own chart. Points link to the result that holds them. + */ +export function StreamsChart({ series, runtimes, className }: { series: StreamSeries[]; runtimes: Runtime[]; className?: string }) { + const navigate = useNavigate() + const byId = useMemo(() => Object.fromEntries(runtimes.map((r) => [r.id, r])), [runtimes]) + const scopes = (['total', 'per_request'] as const).filter((scope) => series.some((s) => s.scope === scope)) + if (!scopes.length) return null + return ( +
+ {scopes.map((scope) => { + const lines = series.filter((s) => s.scope === scope) + const xs = [...new Set(lines.flatMap((s) => s.points.map((p) => p.concurrency)))].sort((a, b) => a - b) + const data = xs.map((x) => Object.fromEntries([['concurrency', x], ...lines.map((s) => [s.key, s.points.find((p) => p.concurrency === x)?.decodeTps ?? null])])) + const config: ChartConfig = Object.fromEntries(lines.map((s) => [s.key, { label: s.label, color: byId[s.runtimeId]?.color ?? 'var(--muted-foreground)' }])) + const summary = lines.map((s) => `${s.label}: ${s.points.map((p) => `${fmtTps(p.decodeTps)} at ${p.concurrency}`).join(', ')}`).join('; ') + return ( +
+
{SCOPE_TITLE[scope]}
+ + + + + fmtTps(v)} /> + { + if (!active || !payload?.length) return null + return ( +
+
{pluralize(Number(label), 'concurrent request')}
+ {payload.filter((p) => p.value != null).map((p) => { + const s = lines.find((line) => line.key === p.dataKey) + return s ? ( +
+ {s.label} + {fmtTps(Number(p.value))} tok/s +
+ ) : null + })} +
+ ) + }} + /> + {lines.map((s) => ( + { + const point = s.points.find((p) => p.concurrency === payload?.concurrency) + if (cx == null || cy == null || value == null || !point) return + return ( + navigate(`/results/${point.resultId}`)} + /> + ) + }} + /> + ))} +
+
+
+ {lines.map((s) => ( + + {s.label} · {byId[s.runtimeId]?.name ?? s.runtimeId} + + ))} +
+
+ ) + })} +
+ ) +} diff --git a/frontend/src/lib/api/hardware-chart.ts b/frontend/src/lib/api/hardware-chart.ts index dc327ff..89e25c6 100644 --- a/frontend/src/lib/api/hardware-chart.ts +++ b/frontend/src/lib/api/hardware-chart.ts @@ -1,10 +1,12 @@ import { MODEL_BY_ID, QUANT_BY_ID, RUNTIME_BY_ID } from '@/catalog' import type { ChartBar, Result } from './types' +import { isTotalThroughput } from '@/lib/setups' -/** Keep each runtime's best run visible, even when another runtime is faster. */ +/** Keep each runtime's best run visible, even when another runtime is faster. Totals across concurrent requests do not count. */ export function hardwareChart(results: Result[]): ChartBar[] { const best = new Map() for (const result of results) { + if (isTotalThroughput(result)) continue const key = `${result.modelId}/${result.quant}/${result.runtimeId}` const current = best.get(key) if (!current || result.decodeTps > current.decodeTps || (result.decodeTps === current.decodeTps && result.runDate < current.runDate)) { diff --git a/frontend/src/lib/api/supabase.ts b/frontend/src/lib/api/supabase.ts index 4335557..da8baef 100644 --- a/frontend/src/lib/api/supabase.ts +++ b/frontend/src/lib/api/supabase.ts @@ -10,7 +10,7 @@ import type { CustomRuntime, DecodeMethod, DecodeScope, Speculative, } from './types' import { ApiError, BUILD_SUMMARY_MAX, REVISION_MAX, RUNTIME_FLAGS_MAX } from './types' -import { SETUPS, SETUP_BY_ID, type SetupId } from '@/lib/setups' +import { SETUPS, SETUP_BY_ID, isTotalThroughput, type SetupId } from '@/lib/setups' import { resultRank } from '@/lib/rank' import { hardwareChart } from './hardware-chart' @@ -237,7 +237,7 @@ function view(snapshot: Snapshot) { updatedAt: row.updated_at, resultsCount: mine.length, rigsCount: new Set(mine.map((result) => stringId(result.rig_id))).size, - bestTps: mine.length ? Math.max(...mine.map((result) => numberValue(result.decode_tps))) : undefined, + bestTps: bestDecode(mine), } } const customRuntimeById = new Map(snapshot.customRuntimes.map((row) => [stringId(row.id), customRuntime(row)])) @@ -254,7 +254,7 @@ function view(snapshot: Snapshot) { owner: owner ? publicUser(owner) : undefined, components: (componentsByRig.get(id) ?? []).map((component) => ({ hardwareId: component.hardware_id, quantity: component.quantity })), resultsCount: matching.length, - bestTps: matching.length ? Math.max(...matching.map((result) => numberValue(result.decode_tps))) : undefined, + bestTps: bestDecode(matching), } } @@ -352,6 +352,15 @@ function paginate(items: T[], limit = 25, cursor?: string): Page { const unitKey = (result: Result) => result.componentId ? `${result.componentId}x${result.componentQuantity ?? 1}` : `rig:${result.rigId}` +/** A row whose `decode_tps` is a total across concurrent requests; see `isTotalThroughput`. */ +const totalRow = (row: ResultRow) => row.concurrency != null && row.concurrency > 1 && row.decode_scope === 'total' + +/** The best per-request decode rate among rows, for summaries. Totals across requests do not count. */ +function bestDecode(items: ResultRow[]): number | undefined { + const perRequest = items.filter((row) => !totalRow(row)) + return perRequest.length ? Math.max(...perRequest.map((row) => numberValue(row.decode_tps))) : undefined +} + function bestPerKey(items: Result[], key: (result: Result) => string): Result[] { const best = new Map() for (const result of items) { @@ -378,7 +387,10 @@ function boardItems(results: Result[], modelId: string, quant: string, params: P if (params.runtime?.length && !params.runtime.includes(result.runtimeId)) return false if (params.verification && result.verification.status !== params.verification) return false if (!params.includeModified && result.execution === 'modified') return false - if (params.setup && !SETUP_BY_ID[params.setup].match(result)) return false + // A named setup ranks only what matches it. Without one, totals across concurrent requests stay out: they are + // not the same quantity as a per-request rate and have their own setup. + const setup = params.setup && params.setup !== 'all' ? params.setup : undefined + if (setup ? !SETUP_BY_ID[setup].match(result) : isTotalThroughput(result)) return false if (params.vendor) { if (result.component?.vendor !== params.vendor && !result.rig?.summary.toLowerCase().includes(params.vendor.toLowerCase())) return false } @@ -404,7 +416,7 @@ const chartFromRows = (items: BoardRow[]): ChartBar[] => items.slice(0, 10).map( })) function bestChart(items: Result[], includePart = false): ChartBar[] { - const best = bestPerKey(items, (result) => `${result.modelId}/${result.quant}${includePart ? `/${unitKey(result)}` : ''}`) + const best = bestPerKey(items.filter((result) => !isTotalThroughput(result)), (result) => `${result.modelId}/${result.quant}${includePart ? `/${unitKey(result)}` : ''}`) return best.slice(0, 10).map((result) => ({ label: `${MODEL_BY_ID[result.modelId]?.name ?? result.modelId} ${QUANT_BY_ID[result.quant]?.label ?? result.quant}${includePart ? ` · ${unitLabel(unitFor(result))}` : ''}`, tps: result.decodeTps, @@ -607,7 +619,7 @@ export const supabaseApi: Api = { ...runtime, resultsCount: mine.length, buildsCount: snapshot.customRuntimes.filter((row) => row.runtime_id === runtime.id).length, - bestTps: mine.length ? Math.max(...mine.map((row) => numberValue(row.decode_tps))) : undefined, + bestTps: bestDecode(mine), } }) }, @@ -753,7 +765,7 @@ export const supabaseApi: Api = { async topResults(params = {}) { const snapshot = await loadSnapshot() const data = view(snapshot) - let results = snapshot.results.map(data.result).filter((result) => !result.moderation.hidden && result.execution !== 'modified') + let results = snapshot.results.map(data.result).filter((result) => !result.moderation.hidden && result.execution !== 'modified' && !isTotalThroughput(result)) if (params.model) results = results.filter((result) => result.modelId === params.model) if (params.quant) results = results.filter((result) => result.quant === params.quant) const best = bestPerKey(results, (result) => `${unitKey(result)}|${result.modelId}|${result.quant}`) diff --git a/frontend/src/lib/setups.ts b/frontend/src/lib/setups.ts index e6f0164..f82fa21 100644 --- a/frontend/src/lib/setups.ts +++ b/frontend/src/lib/setups.ts @@ -3,10 +3,10 @@ import type { Conditions } from '@/lib/conditions' /** * Named setups (issue #130, step 3, "option 2"). Each one is a fixed way of measuring, so results inside it can be * ranked against each other. A result only joins a setup when the conditions the setup checks are recorded and - * match; anything not recorded never matches. "All results" keeps everything and stays the default until the - * setups have filled up. + * match; anything not recorded never matches. "All results" keeps every per-request number and stays the default + * until the setups have filled up; totals across concurrent requests live only in Throughput. */ -export type SetupId = 'standard' | 'speculative' | 'long' | 'concurrent' | 'all' +export type SetupId = 'standard' | 'speculative' | 'long' | 'concurrent' | 'throughput' | 'all' export type Setup = { id: SetupId @@ -21,6 +21,13 @@ const timedAfterFirstToken = (r: Partial) => r.decodeMethod === 'str const shortPrompt = (r: Partial) => r.inputTokens != null && r.inputTokens <= 1024 const enoughOutput = (r: Partial) => r.outputTokens != null && r.outputTokens >= 128 +/** + * Several requests at once and `decodeTps` is the machine's total across them. That is a different quantity from + * what one request sees, so it only ever ranks against other totals (the Throughput setup), never on a per-request + * board, and never counts as a rig's or part's best decode speed. + */ +export const isTotalThroughput = (r: Partial) => r.concurrency != null && r.concurrency > 1 && r.decodeScope === 'total' + export const SETUPS: Setup[] = [ { id: 'standard', name: 'Standard', @@ -42,6 +49,11 @@ export const SETUPS: Setup[] = [ rules: ['prompt ≤ 1k tokens', '4 requests', 'per-request decode', 'timed after first token'], match: (r) => shortPrompt(r) && r.concurrency === 4 && r.decodeScope === 'per_request' && timedAfterFirstToken(r), }, + { + id: 'throughput', name: 'Throughput', + rules: ['≥ 2 requests', 'total across requests'], + match: isTotalThroughput, + }, { id: 'all', name: 'All results', rules: [], match: () => true }, ] diff --git a/frontend/src/lib/streams.ts b/frontend/src/lib/streams.ts new file mode 100644 index 0000000..0c6dbbd --- /dev/null +++ b/frontend/src/lib/streams.ts @@ -0,0 +1,54 @@ +import type { Result } from '@/lib/api/types' +import { MODEL_BY_ID, QUANT_BY_ID } from '@/catalog' + +/** One measured point of a series: how many requests ran at once and the decode rate recorded for it. */ +export type StreamPoint = { concurrency: number; decodeTps: number; resultId: string } + +/** + * Results of one setup (model, quant, runtime, build, part) measured at several request counts, as the two curves a + * scaling study produces: the machine's total across requests, and what each request sees. A single-request result + * is the first point of both, since one request's rate is also the total. + */ +export type StreamSeries = { + key: string + scope: 'total' | 'per_request' + label: string + modelId: string + quant: string + runtimeId: string + points: StreamPoint[] +} + +const unit = (r: Result) => (r.componentId ? `${r.componentQuantity ?? 1}× ${r.component?.name ?? r.componentId}` : '') + +/** + * Group a rig's results into series with at least two distinct request counts. Results without a recorded + * `concurrency` are left out (the legacy `batchSize` never had a fixed meaning), as are hidden ones. When a setup + * was measured more than once at the same request count, the best rate stands, as on the boards. + */ +export function streamSeries(results: Result[]): StreamSeries[] { + const groups = new Map() + for (const r of results) { + if (r.moderation.hidden || r.concurrency == null || r.concurrency < 1) continue + const scopes: StreamSeries['scope'][] = r.concurrency === 1 ? ['total', 'per_request'] : r.decodeScope === 'total' ? ['total'] : r.decodeScope === 'per_request' ? ['per_request'] : [] + for (const scope of scopes) { + const base = [r.modelId, r.quant, r.runtimeId, r.customRuntimeId ?? '', r.componentId ?? '', r.componentQuantity ?? ''].join('|') + const key = `${scope}|${base}` + let series = groups.get(key) + if (!series) { + const model = MODEL_BY_ID[r.modelId]?.name ?? r.modelId + const quant = QUANT_BY_ID[r.quant]?.label ?? r.quant + const part = unit(r) + series = { key, scope, label: `${model} ${quant}${part ? ` · ${part}` : ''}`, modelId: r.modelId, quant: r.quant, runtimeId: r.runtimeId, points: [] } + groups.set(key, series) + } + const existing = series.points.find((p) => p.concurrency === r.concurrency) + if (!existing) series.points.push({ concurrency: r.concurrency, decodeTps: r.decodeTps, resultId: r.id }) + else if (r.decodeTps > existing.decodeTps) Object.assign(existing, { decodeTps: r.decodeTps, resultId: r.id }) + } + } + return [...groups.values()] + .filter((series) => series.points.length >= 2) + .map((series) => ({ ...series, points: [...series.points].sort((a, b) => a.concurrency - b.concurrency) })) + .sort((a, b) => a.scope.localeCompare(b.scope) || a.label.localeCompare(b.label) || a.runtimeId.localeCompare(b.runtimeId)) +} diff --git a/frontend/src/pages/Board.tsx b/frontend/src/pages/Board.tsx index 1bd3c49..5a629ec 100644 --- a/frontend/src/pages/Board.tsx +++ b/frontend/src/pages/Board.tsx @@ -183,7 +183,7 @@ export default function Board() { value: s.id, label: s.name, count: board.data?.setupCounts[s.id], - hint: s.rules.length ? `Ranks only: ${s.rules.join(', ')}` : 'Every result, whatever it was measured with', + hint: s.rules.length ? `Ranks only: ${s.rules.join(', ')}` : 'Every per-request number, whatever it was measured with', }))} /> @@ -194,7 +194,7 @@ export default function Board() { {SETUP_BY_ID[setup].rules.map((rule) => {rule})} ) : ( - Every result, whatever it was measured with. A tag shows when a result also ranks in a named setup. + Every per-request number, whatever it was measured with. A tag shows when a result also ranks in a named setup; totals across concurrent requests rank under Throughput. )} diff --git a/frontend/src/pages/RigDetail.tsx b/frontend/src/pages/RigDetail.tsx index 715e030..3f61013 100644 --- a/frontend/src/pages/RigDetail.tsx +++ b/frontend/src/pages/RigDetail.tsx @@ -13,6 +13,7 @@ import { RigPhoto } from '@/components/RigPhoto' import { UserLink } from '@/components/UserLink' import { HardwareLink } from '@/components/HardwareLink' import { TpsBarChart } from '@/components/TpsBarChart' +import { StreamsChart } from '@/components/StreamsChart' import { ResultsTable } from '@/components/ResultsTable' import { keySpec } from '@/components/cards' import { EmptyBoard } from '@/components/empty' @@ -27,6 +28,8 @@ import { ApiError } from '@/lib/api/types' import { fmtDate, pluralize } from '@/lib/format' import { rigShareTarget } from '@/lib/share' import { UNIT_LABEL, nestParts } from '@/lib/hardware' +import { streamSeries } from '@/lib/streams' +import { isTotalThroughput } from '@/lib/setups' import { HARDWARE_TYPE_LABEL } from '@/catalog' import { cn } from '@/lib/utils' @@ -54,8 +57,10 @@ export default function RigDetail() { const r = rig.data const isOwner = user?.id === r.ownerId const visibleResults = r.results.filter((x) => !x.moderation.hidden) + const series = streamSeries(r.results) const visible = visibleResults.length - const bestResult = visibleResults.reduce<(typeof visibleResults)[number] | undefined>((best, x) => (!best || x.decodeTps > best.decodeTps ? x : best), undefined) + // Totals across concurrent requests are not a rig's best decode speed. + const bestResult = visibleResults.filter((x) => !isTotalThroughput(x)).reduce<(typeof visibleResults)[number] | undefined>((best, x) => (!best || x.decodeTps > best.decodeTps ? x : best), undefined) const shareTarget = rigShareTarget( r, bestResult @@ -167,6 +172,18 @@ export default function RigDetail() { ) : null} + {series.length ? ( +
+ + +

+ Each line is one model, quant and runtime on this rig measured at several request counts. Submit the same setup at another count, with + concurrency and decodeScope recorded, and it joins the line. +

+
+
+ ) : null} +
{r.results.length ? ( diff --git a/frontend/src/pages/Static.tsx b/frontend/src/pages/Static.tsx index 27ac5ae..060782b 100644 --- a/frontend/src/pages/Static.tsx +++ b/frontend/src/pages/Static.tsx @@ -19,7 +19,7 @@ export function Guidelines() {

How to measure decode tok/s

  • Generate at least 256 tokens from a short prompt and average over a few runs.
  • -
  • Measure one request at a time. A throughput configuration — OpenVINO's PERFORMANCE_HINT=THROUGHPUT, vLLM serving many sequences at once, llama.cpp with parallel slots — reports the total across concurrent requests, which is a larger and different number from what one person sees waiting for one answer. If that is what you measured, say so in the flags and set the batch size.
  • +
  • Measure one request at a time. A throughput configuration — OpenVINO's PERFORMANCE_HINT=THROUGHPUT, vLLM serving many sequences at once, llama.cpp with parallel slots — reports the total across concurrent requests, which is a larger and different number from what one person sees waiting for one answer. If that is what you measured, record it as concurrency with decodeScope set to total: it ranks on the Throughput board instead of against single requests, and several request counts of the same setup draw a scaling curve on the rig page.
  • Report the runtime version you used. An evidence link is optional: logs, scripts, a gist, a report, or a repository can help others check your result. Any full HTTPS URL works.
  • Note the flags and settings that moved the number: the backend you compiled in, flash attention, KV cache precision. The same card on the same runtime can differ twofold on these.
  • If you ran on one card out of several, submit it as a component result and set the quantity you used.
  • diff --git a/results/README.md b/results/README.md index 9d8aa5a..4429806 100644 --- a/results/README.md +++ b/results/README.md @@ -64,6 +64,11 @@ measured the same way can be compared: | `speculative` | `none`, `lossless` (output unchanged, such as MTP or a verified draft model), or `lossy`. | | `revision` | The exact engine commit or tag, for stock and custom runs alike. | +Boards rank per-request numbers. A result with `concurrency` above 1 and `decodeScope` `total` is the machine's +throughput, a different quantity from what one request sees, so it ranks only on the **Throughput** tab and never +counts as a rig's best decode speed. Measure the same model, quant and runtime at several request counts, record +`concurrency` and `decodeScope` on each, and the rig page draws decode tok/s against concurrent requests from them. + `contextLength` and `batchSize` still work but are ambiguous: some people meant the prompt, others the context window; some meant concurrent requests, others the engine batch size. Use `inputTokens` and `concurrency` instead.