diff --git a/docs/AGENTS_API_INTRO.md b/docs/AGENTS_API_INTRO.md index ec25f4b..4658c80 100644 --- a/docs/AGENTS_API_INTRO.md +++ b/docs/AGENTS_API_INTRO.md @@ -125,7 +125,12 @@ Record how the number was measured in the optional condition fields: `total`, only when `concurrency` is above 1), `decodeMethod` (`streamed`, `engine`, `llama_bench`, or `wall`), `repeats`, `promptCache`, `thinking`, and `speculative` (`none`, `lossless`, or `lossy`). Set `revision` to the engine -commit for stock runs too. Leave a field out rather than guess. Put any other +commit for stock runs too. Name the model file with `modelRepo`, `modelFile` +and `modelSha256` (lowercase hex), and the spread of the repeats with +`decodeTpsMin` and `decodeTpsMax`. When the items of a batch are one run, the +same setup measured at several input lengths or request counts, add +`"sameRun": true` beside `items`: they share a `runId` and the rig page draws +them as one curve. Leave a field out rather than guess. Put any other caveats in `runtimeFlags` and `notes`. Publishing measured aggregate throughput as a single stream measurement would misrepresent the run. diff --git a/frontend/package.json b/frontend/package.json index 9efb3ed..e052e8c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,9 +14,9 @@ "og:assets": "node scripts/generate-og-assets.mjs", "og:preview": "node scripts/og-preview.mjs", "supabase:db:push:prod:dry-run": "supabase --workdir .. db push --project-ref ujlacyedspjfekotemmd --dry-run --skip-vault", - "results:test": "node --test scripts/tests/pr-results.test.mjs scripts/tests/catalog-sync.test.mjs scripts/tests/result-evidence.test.mjs scripts/tests/hardware-chart.test.mjs scripts/tests/supabase-fetch.test.mjs scripts/tests/rig-presentation.test.mjs scripts/tests/board-submissions.test.mjs scripts/tests/streams.test.mjs scripts/tests/agent-api.test.mjs scripts/tests/result-integrity.test.mjs scripts/tests/result-conditions.test.mjs", + "results:test": "node --test scripts/tests/pr-results.test.mjs scripts/tests/catalog-sync.test.mjs scripts/tests/result-evidence.test.mjs scripts/tests/hardware-chart.test.mjs scripts/tests/supabase-fetch.test.mjs scripts/tests/rig-presentation.test.mjs scripts/tests/board-submissions.test.mjs scripts/tests/streams.test.mjs scripts/tests/agent-api.test.mjs scripts/tests/result-integrity.test.mjs scripts/tests/result-conditions.test.mjs scripts/tests/result-runs.test.mjs", "supabase:db:push:prod": "supabase --workdir .. db push --project-ref ujlacyedspjfekotemmd --skip-vault", - "agents:test": "node --test scripts/tests/agent-api.test.mjs scripts/tests/result-integrity.test.mjs scripts/tests/result-conditions.test.mjs", + "agents:test": "node --test scripts/tests/agent-api.test.mjs scripts/tests/result-integrity.test.mjs scripts/tests/result-conditions.test.mjs scripts/tests/result-runs.test.mjs", "agents:docs": "node scripts/generate-agent-docs.mjs", "agents:docs:check": "node scripts/generate-agent-docs.mjs --check", "agents:dev": "node scripts/agent-api/dev.mjs", diff --git a/frontend/public/docs/agents.md b/frontend/public/docs/agents.md index 8e3b2b3..a7bcc4d 100644 --- a/frontend/public/docs/agents.md +++ b/frontend/public/docs/agents.md @@ -125,7 +125,12 @@ Record how the number was measured in the optional condition fields: `total`, only when `concurrency` is above 1), `decodeMethod` (`streamed`, `engine`, `llama_bench`, or `wall`), `repeats`, `promptCache`, `thinking`, and `speculative` (`none`, `lossless`, or `lossy`). Set `revision` to the engine -commit for stock runs too. Leave a field out rather than guess. Put any other +commit for stock runs too. Name the model file with `modelRepo`, `modelFile` +and `modelSha256` (lowercase hex), and the spread of the repeats with +`decodeTpsMin` and `decodeTpsMax`. When the items of a batch are one run, the +same setup measured at several input lengths or request counts, add +`"sameRun": true` beside `items`: they share a `runId` and the rig page draws +them as one curve. Leave a field out rather than guess. Put any other caveats in `runtimeFlags` and `notes`. Publishing measured aggregate throughput as a single stream measurement would misrepresent the run. @@ -189,7 +194,7 @@ All paths below are relative to `/api/v1`. Every route except `/catalog` require | POST | `/custom-runtimes` | write | Create a record owned by the authenticated user. | | PATCH | `/custom-runtimes/{id}` | write | Update your record. Requires X-Expected-Version with its updatedAt value. | | DELETE | `/custom-runtimes/{id}` | write | Delete your record. Requires X-Expected-Version. Rig deletion with results also requires cascade=true. | -| POST | `/results/batch` | write | Create up to 100 results atomically. A failure rolls back the entire batch. | +| POST | `/results/batch` | write | Create up to 100 results atomically. A failure rolls back the entire batch. Set sameRun when the items are one run (one setup at several input lengths or request counts): they get a shared runId and the rig page draws them as one curve. | | GET | `/results` | read | List results. Filter by owner=me, q, or resource-specific IDs. | | GET | `/results/{id}` | read | Read one record and its current updatedAt version. | | POST | `/results` | write | Create a record owned by the authenticated user. | @@ -235,6 +240,7 @@ Unknown fields are rejected. PATCH accepts the same fields as POST, all optional | Field | Required on create | Type and limits | | --- | --- | --- | | `items` | Yes | array; 1–100 items | +| `sameRun` | No | boolean | Each item uses the POST /results fields below. @@ -269,6 +275,11 @@ Unknown fields are rejected. PATCH accepts the same fields as POST, all optional | `promptCache` | No | boolean or null | | `thinking` | No | boolean or null | | `speculative` | No | string or null: none, lossless, lossy | +| `modelRepo` | No | string or null; max 200 characters | +| `modelFile` | No | string or null; max 255 characters | +| `modelSha256` | No | string or null | +| `decodeTpsMin` | No | number or null; greater than 0; maximum 1000000000000 | +| `decodeTpsMax` | No | number or null; greater than 0; maximum 1000000000000 | | `repoUrl` | No | string or null (https-url); max 2048 characters | | `runDate` | Yes | string (date) | diff --git a/frontend/scripts/pr-results.mjs b/frontend/scripts/pr-results.mjs index 1f82b11..a9d49f7 100644 --- a/frontend/scripts/pr-results.mjs +++ b/frontend/scripts/pr-results.mjs @@ -39,7 +39,7 @@ export function prepareFile(path, status, raw, author, parseResultFile) { } const { file, problems } = parseResultFile(raw) if (status !== 'added' && !file.result) problems.push('Only new files can submit runs. Use a new filename; edit existing live results on the site.') - for (const [field, limit] of [['runtimeVersion', 80], ['runtimeFlags', 200], ['revision', 80], ['notes', 5000]]) { + for (const [field, limit] of [['runtimeVersion', 80], ['runtimeFlags', 200], ['revision', 80], ['notes', 5000], ['modelRepo', 200], ['modelFile', 255]]) { if (file[field]?.length > limit) problems.push(`${field} must be ${limit} characters or fewer.`) if (file[field] && matcher.hasMatch(file[field])) problems.push(`${field}: please remove offensive or profane language.`) } @@ -117,6 +117,9 @@ export async function processPr({ github, database, repository, number, defaultB const lines = data.map((entry) => { const registration = files.find((f) => f.path === entry.path && f.path.startsWith('custom-runtimes/')) const url = entry.id ? `${site}/${registration ? `runtimes/${registration.file.runtime}/custom` : 'results'}/${entry.id}` : null + // A file with points is one run: name how many results it makes and link each one. + if (entry.ids?.length > 1) return `- ${entry.path}: ${entry.status}, ${entry.ids.length} points (${entry.ids.map((id) => `${site}/results/${id}`).join(', ')})` + if (entry.points > 1) return `- ${entry.path}: ${entry.status}, ${entry.points} points` return `- ${entry.path}: ${entry.status}${url ? ` (${url})` : ''}` }) lines.push(merged diff --git a/frontend/scripts/tests/agent-api.test.mjs b/frontend/scripts/tests/agent-api.test.mjs index e8fd0ed..dcdbcfc 100644 --- a/frontend/scripts/tests/agent-api.test.mjs +++ b/frontend/scripts/tests/agent-api.test.mjs @@ -201,3 +201,17 @@ test('results record optional conditions, and a stock run keeps its revision',as assert.notEqual((await call('/api/v1/results',{method:'POST',body:{...resultInput(),decodeScope:'total',concurrency:1}})).status,201) okay(await call(`/api/v1/results/${created.id}`,{method:'DELETE',version:cleared.updatedAt})) }) + +test('a batch with sameRun shares one run id; details round-trip; runId is not writable',async()=>{ + const details={modelRepo:'Qwen/Qwen3-8B-GGUF',modelFile:'Qwen3-8B-Q4_K_M.gguf',modelSha256:'b'.repeat(64),decodeTpsMin:7.5,decodeTpsMax:8.1} + const run=okay(await call('/api/v1/results/batch',{method:'POST',body:{sameRun:true,items:[{...resultInput(),...details,concurrency:1},{...resultInput(),concurrency:4,decodeScope:'total',decodeTps:20}]}}),201) + assert.ok(run.items[0].runId);assert.equal(run.items[0].runId,run.items[1].runId) + for(const [k,v] of Object.entries(details)) assert.equal(run.items[0][k],v,k) + const loose=okay(await call('/api/v1/results/batch',{method:'POST',body:{items:[resultInput(),resultInput()]}}),201) + assert.equal(loose.items[0].runId,null);assert.equal(loose.items[1].runId,null) + const other=okay(await call('/api/v1/results/batch',{method:'POST',body:{sameRun:true,items:[resultInput()]}}),201) + assert.notEqual(other.items[0].runId,run.items[0].runId,'each batch is its own run') + assert.equal((await call('/api/v1/results',{method:'POST',body:{...resultInput(),runId:run.items[0].runId}})).status,400) + assert.equal((await call('/api/v1/results',{method:'POST',body:{...resultInput(),modelSha256:'B'.repeat(64)}})).status,400) + assert.notEqual((await call('/api/v1/results',{method:'POST',body:{...resultInput(),decodeTpsMin:9}})).status,201,'the slowest run cannot beat the mean') +}) diff --git a/frontend/scripts/tests/result-runs.test.mjs b/frontend/scripts/tests/result-runs.test.mjs new file mode 100644 index 0000000..f07c6cf --- /dev/null +++ b/frontend/scripts/tests/result-runs.test.mjs @@ -0,0 +1,146 @@ +import { test, before, after } from 'node:test' +import assert from 'node:assert/strict' +import { createDatabase, seedUsers, owner, other } from '../agent-api/database.mjs' +import { loadResultParser, prepareFile } from '../pr-results.mjs' + +// Runs and details (issue #130 step 4, light version): a file with points becomes one result per point sharing a +// run id; any result can name its model file and the spread of its repeats. +const repository = 'labscommunity/intelinside' +const sha = 'a'.repeat(64) +const shared = { + rig: '1', model: 'qwen3-8b', quant: 'q4_k_m', runtime: 'llamacpp', runtimeVersion: 'b6512', runDate: '2026-11-18', + outputTokens: 256, decodeMethod: 'streamed', repeats: 3, speculative: 'none', + modelRepo: 'Qwen/Qwen3-8B-GGUF', modelFile: 'Qwen3-8B-Q4_K_M.gguf', modelSha256: sha, +} +const points = [ + { inputTokens: 512, concurrency: 1, decodeTps: 20.66, decodeTpsMin: 20.5, decodeTpsMax: 20.8 }, + { inputTokens: 4096, concurrency: 1, decodeTps: 18.51 }, + { inputTokens: 512, concurrency: 4, decodeScope: 'total', decodeTps: 28.8 }, +] +const single = { ...shared, decodeTps: 34.2, inputTokens: 512, concurrency: 1 } +let db, parseResultFile +before(async () => { + parseResultFile = await loadResultParser() + db = await createDatabase() + await seedUsers(db) + await db.query(`insert into auth.identities values ('github','123',$1)`, [owner]) + await db.query(`insert into public.rigs(owner_id,name) values ($1,'Alice rig')`, [owner]) +}) +after(async () => { await db?.close() }) + +async function as(user, fn) { + await db.query("select set_config('request.jwt.claims',$1,false)", [JSON.stringify({ sub: user, role: 'authenticated' })]) + await db.exec('set role authenticated') + try { return await fn() } + finally { await db.exec("reset role; select set_config('request.jwt.claims','',false)") } +} +const insert = (columns) => as(owner, async () => { + const keys = ['submitter_id', 'rig_id', 'model_id', 'quant_id', 'runtime_id', 'runtime_version', 'decode_tps', 'run_date', ...Object.keys(columns)] + const values = [owner, 1, 'qwen3-8b', 'q4_k_m', 'llamacpp', 'b6512', 34.2, '2026-11-18', ...Object.values(columns)] + return (await db.query(`insert into public.results (${keys.join(',')}) values (${keys.map((_, i) => `$${i + 1}`).join(',')}) returning *`, values)).rows[0] +}) +const ingest = async (path, raw, { pr = 90, dryRun = false } = {}) => { + const { file } = prepareFile(path, 'added', raw, 'alice', parseResultFile) + return (await db.query('select public.ingest_pr_results($1,$2,$3,$4::jsonb,$5) as result', + [repository, pr, '123', JSON.stringify([{ path, file }]), dryRun])).rows[0].result +} + +test('the parser keeps details, lowercases the hash, and checks the spread against the rate', () => { + const { file, problems } = parseResultFile({ ...single, modelSha256: sha.toUpperCase(), decodeTpsMin: 33.9, decodeTpsMax: 34.6 }) + assert.deepEqual(problems, []) + assert.equal(file.modelSha256, sha) + assert.equal(file.modelFile, 'Qwen3-8B-Q4_K_M.gguf') + assert.equal(file.decodeTpsMax, 34.6) + assert.match(parseResultFile({ ...single, decodeTpsMin: 35 }).problems.join(' '), /decodeTpsMin.*at most/) + assert.match(parseResultFile({ ...single, decodeTpsMax: 30 }).problems.join(' '), /decodeTpsMax.*at least/) + assert.match(parseResultFile({ ...single, modelSha256: 'abc' }).problems.join(' '), /modelSha256/) + assert.match(parseResultFile({ ...single, modelFile: '' }).problems.join(' '), /modelFile/) +}) + +test('the parser reads a file with points as one full result per point', () => { + const { file, problems } = parseResultFile({ ...shared, points }) + assert.deepEqual(problems, []) + assert.equal(file.points.length, 3) + assert.equal(file.decodeTps, undefined, 'measured numbers live in the points') + assert.equal(file.modelRepo, shared.modelRepo) + assert.deepEqual(file.points[2], { decodeTps: 28.8, inputTokens: 512, outputTokens: 256, concurrency: 4, decodeScope: 'total' }) +}) + +test('the parser names point problems once when shared, by point otherwise', () => { + const shared1 = parseResultFile({ ...shared, model: 'not-a-model', points }).problems + assert.equal(shared1.filter((p) => p.includes('not-a-model')).length, 1) + assert.ok(!shared1.some((p) => p.startsWith('points[') && p.includes('not-a-model'))) + const one = parseResultFile({ ...shared, points: [points[0], { ...points[1], decodeTps: -1 }] }).problems + assert.deepEqual(one, ['points[1]: "decodeTps" must be a number above zero.']) + assert.match(parseResultFile({ ...shared, decodeTps: 3, points }).problems.join(' '), /"decodeTps" goes in each point/) + assert.match(parseResultFile({ ...shared, points: [{ ...points[0], notes: 'x' }] }).problems.join(' '), /points\[0\]: "notes" cannot vary/) + assert.match(parseResultFile({ ...shared, points: [points[0], { ...points[0] }] }).problems.join(' '), /points\[1\] repeats points\[0\]/) + assert.match(parseResultFile({ ...shared, points: [] }).problems.join(' '), /1 to 64 points/) + assert.match(parseResultFile({ ...shared, points: [{ inputTokens: 512, concurrency: 2, decodeScope: 'total', decodeTps: 9 }, { inputTokens: 512, concurrency: 1, decodeScope: 'total', decodeTps: 9 }] }).problems.join(' '), + /points\[1\]: "decodeScope" only applies/) + assert.throws(() => prepareFile('results/alice/sweep.json', 'added', { ...shared, points: 'many' }, 'alice', parseResultFile), /points/) +}) + +test('PR ingestion turns points into results sharing a run id, and a retry reports them all', async () => { + const dry = await ingest('results/alice/sweep.json', { ...shared, points }, { dryRun: true }) + assert.deepEqual(dry, [{ path: 'results/alice/sweep.json', status: 'validated', id: null, points: 3 }]) + assert.equal((await db.query('select count(*)::int n from public.results')).rows[0].n, 0, 'dry runs roll back') + + const [answer] = await ingest('results/alice/sweep.json', { ...shared, points }) + assert.equal(answer.status, 'imported') + assert.equal(answer.ids.length, 3) + assert.equal(answer.id, answer.ids[0]) + const rows = (await db.query('select * from public.results where id = any($1::bigint[]) order by id', [answer.ids])).rows + assert.equal(new Set(rows.map((r) => r.run_id)).size, 1) + assert.ok(rows[0].run_id) + assert.deepEqual(rows.map((r) => [r.input_tokens, r.concurrency, r.decode_scope, Number(r.decode_tps)]), + [[512, 1, null, 20.66], [4096, 1, null, 18.51], [512, 4, 'total', 28.8]]) + for (const r of rows) { + assert.equal(r.model_file, shared.modelFile) + assert.equal(r.model_sha256, sha) + assert.equal(r.output_tokens, 256, 'shared conditions reach every point') + } + assert.equal(Number(rows[0].decode_tps_min), 20.5) + assert.equal(rows[1].decode_tps_min, null, 'a point keeps only its own spread') + + const [again] = await ingest('results/alice/sweep.json', { ...shared, points }) + assert.equal(again.status, 'already_imported') + assert.deepEqual(again.ids, answer.ids) +}) + +test('a single-result file gets no run id but keeps its details', async () => { + const [answer] = await ingest('results/alice/single.json', { ...single, decodeTpsMin: 33.9, decodeTpsMax: 34.6 }, { pr: 91 }) + assert.equal(answer.ids, undefined) + const row = (await db.query('select * from public.results where id = $1', [answer.id])).rows[0] + assert.equal(row.run_id, null) + assert.equal(row.model_repo, shared.modelRepo) + assert.equal(Number(row.decode_tps_max), 34.6) +}) + +test('a browser writes details but can never set a run id', async () => { + const row = await insert({ model_file: 'x.gguf', model_sha256: sha, decode_tps_min: 30, decode_tps_max: 35 }) + assert.equal(row.model_file, 'x.gguf') + await assert.rejects(insert({ run_id: '00000000-0000-0000-0000-0000000000aa' }), /permission denied/) + await assert.rejects(as(owner, () => db.query("update public.results set run_id = gen_random_uuid() where id = $1", [row.id])), /permission denied/) + await assert.rejects(as(owner, () => db.query('select private.assign_result_run($1, gen_random_uuid())', [row.id])), /permission denied/) +}) + +test('the database rejects invalid details from any writer', async () => { + for (const [columns, constraint] of [ + [{ model_sha256: 'A'.repeat(64) }, 'results_model_sha256_hex'], + [{ model_sha256: 'abc' }, 'results_model_sha256_hex'], + [{ model_file: '' }, 'results_model_file_length'], + [{ model_repo: 'x'.repeat(201) }, 'results_model_repo_length'], + [{ decode_tps_min: 40 }, 'results_decode_tps_min_range'], + [{ decode_tps_min: 0 }, 'results_decode_tps_min_range'], + [{ decode_tps_max: 30 }, 'results_decode_tps_max_range'], + ]) await assert.rejects(insert(columns), new RegExp(constraint), constraint) +}) + +test('editing only a detail resets confirmations', async () => { + const { id } = await insert({ model_file: 'a.gguf' }) + await as(other, () => db.query('insert into public.result_confirmations(result_id,user_id) values ($1,$2)', [id, other])) + assert.equal((await db.query('select confirmations_count n from public.results where id=$1', [id])).rows[0].n, 1) + await as(owner, () => db.query("update public.results set model_file = 'b.gguf' where id = $1", [id])) + assert.equal((await db.query('select confirmations_count n from public.results where id=$1', [id])).rows[0].n, 0) +}) diff --git a/frontend/scripts/tests/streams.test.mjs b/frontend/scripts/tests/streams.test.mjs index 4d7b0b3..6de32a6 100644 --- a/frontend/scripts/tests/streams.test.mjs +++ b/frontend/scripts/tests/streams.test.mjs @@ -5,17 +5,17 @@ import { build } from 'esbuild' // Decode rate against concurrent requests (issue #130): totals and per-request rates are separate curves, a single // request seeds both, and nothing is drawn from a setup measured at only one request count. -let streamSeries, isTotalThroughput, SETUP_BY_ID, setupOf +let streamSeries, contextSeries, isTotalThroughput, SETUP_BY_ID, setupOf before(async () => { const bundle = await build({ stdin: { - contents: `export { streamSeries } from './lib/streams'; export { isTotalThroughput, SETUP_BY_ID, setupOf } from './lib/setups';`, + contents: `export { streamSeries, contextSeries } from './lib/streams'; export { isTotalThroughput, SETUP_BY_ID, setupOf } from './lib/setups';`, resolveDir: fileURLToPath(new URL('../../src', import.meta.url)), }, alias: { '@': fileURLToPath(new URL('../../src', import.meta.url)) }, bundle: true, write: false, platform: 'browser', format: 'esm', logLevel: 'silent', }) - ;({ streamSeries, isTotalThroughput, SETUP_BY_ID, setupOf } = await import(`data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString('base64')}`)) + ;({ streamSeries, contextSeries, isTotalThroughput, SETUP_BY_ID, setupOf } = await import(`data:text/javascript;base64,${Buffer.from(bundle.outputFiles[0].contents).toString('base64')}`)) }) const result = (id, concurrency, decodeTps, extra = {}) => ({ @@ -32,7 +32,7 @@ test('a single request seeds both curves and totals and per-request rates stay a result(4, 8, 17, { decodeScope: 'per_request' }), result(5, 4, 12, { decodeScope: 'per_request' }), ]) - assert.deepEqual(series.map((s) => [s.scope, s.points.map((p) => [p.concurrency, p.decodeTps])]), [ + assert.deepEqual(series.map((s) => [s.scope, s.points.map((p) => [p.x, p.decodeTps])]), [ ['per_request', [[1, 18.5], [4, 12], [8, 17]]], ['total', [[1, 18.5], [8, 136], [48, 227.5]]], ]) @@ -41,7 +41,7 @@ test('a single request seeds both curves and totals and per-request rates stay a test('one request count is not a curve; repeats at a count keep the best; legacy batch size and hidden results are ignored', () => { assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, 1, 19.0)]), []) - assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, 8, 100, { decodeScope: 'total' }), result(3, 8, 130, { decodeScope: 'total' })]).map((s) => s.points.map((p) => [p.concurrency, p.decodeTps, p.resultId])), [[[1, 18.5, '1'], [8, 130, '3']]]) + assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, 8, 100, { decodeScope: 'total' }), result(3, 8, 130, { decodeScope: 'total' })]).map((s) => s.points.map((p) => [p.x, p.decodeTps, p.resultId])), [[[1, 18.5, '1'], [8, 130, '3']]]) assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, undefined, 100, { batchSize: 8 })]), []) assert.deepEqual(streamSeries([result(1, 1, 18.5), result(2, 8, 100, { decodeScope: 'total', moderation: { hidden: true } })]), []) // A total at several requests with no decodeScope recorded is ambiguous and stays out. @@ -71,3 +71,20 @@ test('the Throughput setup takes totals across requests and nothing else', () => assert.equal(SETUP_BY_ID.throughput.match({ concurrency: 4, decodeScope: 'per_request', inputTokens: 512, decodeMethod: 'streamed' }), false) assert.equal(setupOf({ concurrency: 4, decodeScope: 'per_request', inputTokens: 512, decodeMethod: 'streamed' })?.id, 'concurrent') }) + +test('prompt-length curves take single requests with inputTokens, best per length, and skip speculation', () => { + const series = contextSeries([ + result(1, 1, 20.66, { inputTokens: 512 }), + result(2, 1, 18.51, { inputTokens: 4096 }), + result(3, 1, 19.9, { inputTokens: 4096 }), + result(4, 4, 15.05, { inputTokens: 16384, decodeScope: 'per_request' }), + result(5, 1, 40, { inputTokens: 16384, speculative: 'lossless' }), + result(6, undefined, 30, { inputTokens: 32768 }), + result(7, 1, 30, { contextLength: 32768 }), + result(8, 1, 12, { inputTokens: 32768, moderation: { hidden: true } }), + ]) + assert.deepEqual(series.map((s) => [s.axis, s.scope, s.points.map((p) => [p.x, p.decodeTps, p.resultId])]), + [['input', 'per_request', [[512, 20.66, '1'], [4096, 19.9, '3']]]]) + assert.deepEqual(contextSeries([result(1, 1, 20, { inputTokens: 512 })]), [], 'one length draws no line') + assert.deepEqual(contextSeries([result(1, 1, 20, { inputTokens: 512 }), result(2, 1, 18, { inputTokens: 4096, quant: 'q8_0' })]), [], 'another quant is another line') +}) diff --git a/frontend/src/agent-api/contract.ts b/frontend/src/agent-api/contract.ts index 91247f5..12fa892 100644 --- a/frontend/src/agent-api/contract.ts +++ b/frontend/src/agent-api/contract.ts @@ -32,6 +32,11 @@ export const resultSchema = object({ decodeMethod: nullable({ type: 'string', enum: ['streamed', 'engine', 'llama_bench', 'wall'], description: 'How decode was timed. wall includes prefill.' }), repeats: nullable(integer), promptCache: nullable({ type: 'boolean' }), thinking: nullable({ type: 'boolean' }), speculative: nullable({ type: 'string', enum: ['none', 'lossless', 'lossy'] }), + modelRepo: nullable({ ...text(200, 1), description: 'Where the weights came from, such as Qwen/Qwen3-8B-GGUF.' }), + modelFile: nullable({ ...text(255, 1), description: 'The model file, such as Qwen3-8B-Q4_K_M.gguf.' }), + modelSha256: nullable({ type: 'string', pattern: '^[0-9a-f]{64}$', description: 'SHA-256 of the model file, lowercase hex.' }), + decodeTpsMin: nullable({ ...positive, description: 'Slowest measured repeat; at most decodeTps.' }), + decodeTpsMax: nullable({ ...positive, description: 'Fastest measured repeat; at least decodeTps.' }), repoUrl: nullable(https), runDate: { type: 'string', format: 'date' }, }, ['rigId', 'modelId', 'quant', 'runtimeId', 'runtimeVersion', 'decodeTps', 'runDate']) const flagSchema = object({ reason: { type: 'string', enum: ['implausible', 'wrong_hardware', 'duplicate', 'spam', 'other'] }, note: nullable(text(5000)) }, ['reason']) @@ -54,7 +59,7 @@ export const routes: Route[] = [ ...resource('/rigs', 'rigs', rigSchema), ...resource('/custom-runtimes', 'custom_runtimes', runtimeSchema), // Put the static batch route before /results/{id}. - { method: 'POST', path: '/results/batch', action: 'results.batch', description: 'Create up to 100 results atomically. A failure rolls back the entire batch.', scope: 'write', schema: object({ items: { type: 'array', minItems: 1, maxItems: 100, items: resultSchema } }, ['items']) }, + { method: 'POST', path: '/results/batch', action: 'results.batch', description: 'Create up to 100 results atomically. A failure rolls back the entire batch. Set sameRun when the items are one run (one setup at several input lengths or request counts): they get a shared runId and the rig page draws them as one curve.', scope: 'write', schema: object({ items: { type: 'array', minItems: 1, maxItems: 100, items: resultSchema }, sameRun: { type: 'boolean' } }, ['items']) }, ...resource('/results', 'results', resultSchema), { method: 'PUT', path: '/results/{id}/confirmation', action: 'confirmations.set', description: 'Confirm another user’s result. Repeating this never removes a confirmation.', scope: 'community' }, { method: 'DELETE', path: '/results/{id}/confirmation', action: 'confirmations.delete', description: 'Remove your confirmation.', scope: 'community' }, @@ -103,7 +108,7 @@ export function validate(schema: Schema, value: unknown, field = 'body'): void { } export function moderateBody(body: Record) { - const fields = ['name', 'os', 'summary', 'notes', 'note', 'runtimeVersion', 'runtimeFlags', 'revision'] + const fields = ['name', 'os', 'summary', 'notes', 'note', 'runtimeVersion', 'runtimeFlags', 'revision', 'modelRepo', 'modelFile'] moderateText(fields.filter((key) => typeof body[key] === 'string').map((key) => [key, body[key] as string, 5000])) if (Array.isArray(body.items)) body.items.forEach((item) => moderateBody(item as Record)) } diff --git a/frontend/src/components/ResultCard.tsx b/frontend/src/components/ResultCard.tsx index 490253f..fdbabba 100644 --- a/frontend/src/components/ResultCard.tsx +++ b/frontend/src/components/ResultCard.tsx @@ -10,6 +10,7 @@ import { UserLink } from '@/components/UserLink' import { HardwareLink } from '@/components/HardwareLink' import { UnitLabel } from '@/components/UnitLabel' import { ConditionChips } from '@/components/ConditionChips' +import { spreadLabel } from '@/lib/conditions' import type { Model, Quant, Result, ResultRank, Runtime } from '@/lib/api/types' import { fmtDate, fmtMs, fmtTps } from '@/lib/format' import { boardPhrase, rankBoardPath, unitPhrase } from '@/lib/rank' @@ -53,6 +54,11 @@ export function ResultCard({ result: r, models, quants, runtimes, rank, eyebrow
{hasFigure ? fmtTps(r.decodeTps) : '—'} tok/s
+ {spreadLabel(r) ? ( +
+ range {spreadLabel(r)}{r.repeats ? ` over ${r.repeats} runs` : ''} +
+ ) : null} {rank?.position ? ( #{rank.position} of {rank.boardSize} {boardPhrase(rank, model?.name ?? r.modelId, quant)} @@ -79,6 +85,12 @@ export function ResultCard({ result: r, models, quants, runtimes, rank, eyebrow ) : ( — )} + {r.modelRepo || r.modelFile ? ( +
{[r.modelRepo, r.modelFile].filter(Boolean).join(' / ')}
+ ) : null} + {r.modelSha256 ? ( +
sha256 {r.modelSha256.slice(0, 12)}…
+ ) : null} {r.component ? ( diff --git a/frontend/src/components/StreamsChart.tsx b/frontend/src/components/StreamsChart.tsx index d6d26a7..52d225e 100644 --- a/frontend/src/components/StreamsChart.tsx +++ b/frontend/src/components/StreamsChart.tsx @@ -8,33 +8,49 @@ import type { StreamSeries } from '@/lib/streams' import { fmtTps, pluralize } from '@/lib/format' const SCOPE_TITLE = { total: 'Total decode tok/s across requests', per_request: 'Decode tok/s each request sees' } as const +const tokens = (n: number) => (n >= 1024 && n % 1024 === 0 ? `${n / 1024}k` : n >= 1000 && n % 1000 === 0 ? `${n / 1000}k` : n.toLocaleString('en-US')) + +/** Per axis: the chart's title, its x label, how a tick and a tooltip read, and whether x is logarithmic. */ +const AXIS = { + concurrency: { + title: (scope: StreamSeries['scope']) => SCOPE_TITLE[scope], label: 'concurrent requests', log: false, + tick: (x: number) => String(x), tooltip: (x: number) => pluralize(x, 'concurrent request'), aria: 'by concurrent requests', at: (x: number) => `at ${x}`, + }, + // Prompt grids double (512, 4k, 16k, 32k), so a log scale spaces them evenly. + input: { + title: () => 'Decode tok/s for one request, by prompt length', label: 'prompt tokens', log: true, + tick: tokens, tooltip: (x: number) => `${tokens(x)} prompt tokens`, aria: 'by prompt length', at: (x: number) => `after ${tokens(x)}`, + }, +} as const /** - * Decode rate against concurrent requests, one line per measured setup, colored by runtime. Totals and per-request - * rates are different quantities, so each scope gets its own chart. Points link to the result that holds them. + * Decode rate against concurrent requests or prompt length, one line per measured setup, colored by runtime. Totals + * and per-request rates are different quantities, so each scope gets its own chart. Points link to their result. */ export function StreamsChart({ series, runtimes, className }: { series: StreamSeries[]; runtimes: Runtime[]; className?: string }) { const navigate = useNavigate() const byId = useMemo(() => Object.fromEntries(runtimes.map((r) => [r.id, r])), [runtimes]) const scopes = (['total', 'per_request'] as const).filter((scope) => series.some((s) => s.scope === scope)) if (!scopes.length) return null + const axis = AXIS[series[0].axis] return (
{scopes.map((scope) => { const lines = series.filter((s) => s.scope === scope) - const xs = [...new Set(lines.flatMap((s) => s.points.map((p) => p.concurrency)))].sort((a, b) => a - b) - const data = xs.map((x) => Object.fromEntries([['concurrency', x], ...lines.map((s) => [s.key, s.points.find((p) => p.concurrency === x)?.decodeTps ?? null])])) + const xs = [...new Set(lines.flatMap((s) => s.points.map((p) => p.x)))].sort((a, b) => a - b) + const data = xs.map((x) => Object.fromEntries([['x', x], ...lines.map((s) => [s.key, s.points.find((p) => p.x === x)?.decodeTps ?? null])])) const config: ChartConfig = Object.fromEntries(lines.map((s) => [s.key, { label: s.label, color: byId[s.runtimeId]?.color ?? 'var(--muted-foreground)' }])) - const summary = lines.map((s) => `${s.label}: ${s.points.map((p) => `${fmtTps(p.decodeTps)} at ${p.concurrency}`).join(', ')}`).join('; ') + const summary = lines.map((s) => `${s.label}: ${s.points.map((p) => `${fmtTps(p.decodeTps)} ${axis.at(p.x)}`).join(', ')}`).join('; ') return ( -
-
{SCOPE_TITLE[scope]}
+
+
{axis.title(scope)}
axis.tick(v)} + tick={{ fill: 'var(--muted-foreground)', fontSize: 11 }} label={{ value: axis.label, position: 'insideBottomRight', offset: -2, fill: 'var(--muted-foreground)', fontSize: 11 }} /> fmtTps(v)} /> -
{pluralize(Number(label), 'concurrent request')}
+
{axis.tooltip(Number(label))}
{payload.filter((p) => p.value != null).map((p) => { const s = lines.find((line) => line.key === p.dataKey) return s ? ( @@ -62,12 +78,12 @@ export function StreamsChart({ series, runtimes, className }: { series: StreamSe key={s.key} dataKey={s.key} type="monotone" connectNulls stroke={config[s.key].color} strokeWidth={2} activeDot={{ r: 6 }} isAnimationActive={false} // Each point links to the result that holds it. - dot={({ cx, cy, payload, value }: { cx?: number; cy?: number; payload?: { concurrency?: number }; value?: number | null }) => { - const point = s.points.find((p) => p.concurrency === payload?.concurrency) - if (cx == null || cy == null || value == null || !point) return + dot={({ cx, cy, payload, value }: { cx?: number; cy?: number; payload?: { x?: number }; value?: number | null }) => { + const point = s.points.find((p) => p.x === payload?.x) + if (cx == null || cy == null || value == null || !point) return return ( navigate(`/results/${point.resultId}`)} /> ) diff --git a/frontend/src/lib/agentPrompt.ts b/frontend/src/lib/agentPrompt.ts index a8c0779..8117ed7 100644 --- a/frontend/src/lib/agentPrompt.ts +++ b/frontend/src/lib/agentPrompt.ts @@ -26,7 +26,10 @@ Read ${RESULTS_DIR}/README.md in that repo for the field list, and frontend/src/ and runtime ids. Fill the file from my run: model, quant, runtime, runtimeVersion, decodeTps and runDate are required; promptTps, ttftMs, runtimeFlags and revision (the engine commit) if the log has them. Also record how it was measured, where the log or my setup says so: inputTokens, outputTokens, concurrency (and decodeScope when -above 1), decodeMethod, repeats, promptCache, thinking and speculative. Leave out anything you would have to guess. +above 1), decodeMethod, repeats, promptCache, thinking and speculative; the model file as modelRepo, modelFile and +modelSha256; and the slowest and fastest repeat as decodeTpsMin and decodeTpsMax. If I measured the same setup at +several prompt lengths or request counts, write one file with a "points" list instead of one file per point (the +README shows how). Leave out anything you would have to guess. If I have a published log, script, gist, report, or repository for the run, include its full HTTPS URL as evidenceUrl. This is optional; omit it if unavailable. The submission PR is linked automatically. diff --git a/frontend/src/lib/api/mock.ts b/frontend/src/lib/api/mock.ts index 2b83079..0ed5bf9 100644 --- a/frontend/src/lib/api/mock.ts +++ b/frontend/src/lib/api/mock.ts @@ -8,7 +8,7 @@ import { ApiError, BUILD_SUMMARY_MAX } from './types' import { hardwareChart } from './hardware-chart' import { isEvidenceUrl } from '@/lib/evidence' import { hostIn } from '@/lib/hardware' -import { conditionErrors } from '@/lib/conditions' +import { conditionErrors, detailErrors } from '@/lib/conditions' import { SETUPS, SETUP_BY_ID, type SetupId } from '@/lib/setups' import { resultRank } from '@/lib/rank' import { boardQuants, HARDWARE_BY_ID, MODELS, MODEL_BY_ID, QUANTS, QUANT_BY_ID, RUNTIMES, RUNTIME_BY_ID, VISIBLE_HARDWARE } from '@/catalog' @@ -289,6 +289,7 @@ function validateResult(input: Partial, forCreate: boolean): Record if (input.repoUrl?.trim() && !isEvidenceUrl(input.repoUrl.trim())) errors.repoUrl = 'Enter a full HTTPS URL.' if (input.runDate && new Date(input.runDate).getTime() > Date.now() + 86400000) errors.runDate = 'Run date cannot be in the future.' Object.assign(errors, conditionErrors(input)) + Object.assign(errors, detailErrors(input, input.decodeTps)) if (input.rigId) { const rig = db.rigs.find((r) => r.id === input.rigId) if (!rig) errors.rigId = 'Unknown rig.' diff --git a/frontend/src/lib/api/supabase.ts b/frontend/src/lib/api/supabase.ts index da8baef..faf654f 100644 --- a/frontend/src/lib/api/supabase.ts +++ b/frontend/src/lib/api/supabase.ts @@ -10,6 +10,7 @@ import type { CustomRuntime, DecodeMethod, DecodeScope, Speculative, } from './types' import { ApiError, BUILD_SUMMARY_MAX, REVISION_MAX, RUNTIME_FLAGS_MAX } from './types' +import { MODEL_FILE_MAX, MODEL_REPO_MAX, normalizeSha256 } from '@/lib/conditions' import { SETUPS, SETUP_BY_ID, isTotalThroughput, type SetupId } from '@/lib/setups' import { resultRank } from '@/lib/rank' import { hardwareChart } from './hardware-chart' @@ -60,6 +61,12 @@ type ResultRow = { prompt_cache: boolean | null thinking: boolean | null speculative: Speculative | null + model_repo: string | null + model_file: string | null + model_sha256: string | null + decode_tps_min: number | string | null + decode_tps_max: number | string | null + run_id: string | null notes: string | null repo_url: string | null source_pr_url: string | null @@ -163,12 +170,15 @@ function resultPayload(input: Partial, create = false): Record = {} for (const [source, target] of map) { if (create || source in input) payload[target] = input[source] ?? null } if ('repo_url' in payload) payload.repo_url = input.repoUrl?.trim() || null + if (typeof payload.model_sha256 === 'string') payload.model_sha256 = normalizeSha256(payload.model_sha256) || null if (payload.component_id == null) payload.component_quantity = null return payload } @@ -321,6 +331,12 @@ function view(snapshot: Snapshot) { promptCache: optional(row.prompt_cache), thinking: optional(row.thinking), speculative: optional(row.speculative), + modelRepo: optional(row.model_repo), + modelFile: optional(row.model_file), + modelSha256: optional(row.model_sha256), + decodeTpsMin: row.decode_tps_min == null ? undefined : numberValue(row.decode_tps_min), + decodeTpsMax: row.decode_tps_max == null ? undefined : numberValue(row.decode_tps_max), + runId: optional(row.run_id), notes: optional(row.notes), repoUrl: optional(row.repo_url), sourcePrUrl: optional(row.source_pr_url), @@ -700,7 +716,7 @@ export const supabaseApi: Api = { }, async createResult(input) { const submitterId = await requireUserId() - moderateText([['runtimeVersion', input.runtimeVersion, 80], ['runtimeFlags', input.runtimeFlags, RUNTIME_FLAGS_MAX], ['revision', input.revision, REVISION_MAX], ['notes', input.notes, 5000]]) + moderateText([['runtimeVersion', input.runtimeVersion, 80], ['runtimeFlags', input.runtimeFlags, RUNTIME_FLAGS_MAX], ['revision', input.revision, REVISION_MAX], ['notes', input.notes, 5000], ['modelRepo', input.modelRepo, MODEL_REPO_MAX], ['modelFile', input.modelFile, MODEL_FILE_MAX]]) const created = await rows(requiredClient().from('results').insert({ ...resultPayload(input, true), submitter_id: submitterId, @@ -709,7 +725,7 @@ export const supabaseApi: Api = { }, async updateResult(id, input) { await requireUserId() - moderateText([['runtimeVersion', input.runtimeVersion, 80], ['runtimeFlags', input.runtimeFlags, RUNTIME_FLAGS_MAX], ['revision', input.revision, REVISION_MAX], ['notes', input.notes, 5000]]) + moderateText([['runtimeVersion', input.runtimeVersion, 80], ['runtimeFlags', input.runtimeFlags, RUNTIME_FLAGS_MAX], ['revision', input.revision, REVISION_MAX], ['notes', input.notes, 5000], ['modelRepo', input.modelRepo, MODEL_REPO_MAX], ['modelFile', input.modelFile, MODEL_FILE_MAX]]) const updated = await rows(requiredClient().from('results').update(resultPayload(input)).eq('id', id).select('*')) if (!updated.length) throw new ApiError('not_found', 'No such result.', 404) return supabaseApi.result(id) diff --git a/frontend/src/lib/api/types.ts b/frontend/src/lib/api/types.ts index 1b3cf55..c30cfb8 100644 --- a/frontend/src/lib/api/types.ts +++ b/frontend/src/lib/api/types.ts @@ -183,6 +183,18 @@ export type Result = { promptCache?: boolean thinking?: boolean speculative?: Speculative + // Details (issue #130 step 4). Optional; see lib/conditions.ts. + /** Where the weights came from, such as Qwen/Qwen3-8B-GGUF. */ + modelRepo?: string + /** The file within it, such as Qwen3-8B-Q4_K_M.gguf. */ + modelFile?: string + /** The file's SHA-256, lowercase hex. */ + modelSha256?: string + /** Slowest and fastest measured repeat behind decodeTps. */ + decodeTpsMin?: number + decodeTpsMax?: number + /** Shared by the results of one run: one setup measured at several input lengths or request counts. Set by the server. */ + runId?: string notes?: string /** Optional HTTPS evidence link. The legacy name is retained for API compatibility. */ repoUrl?: string @@ -199,7 +211,7 @@ export type Result = { export type ResultInput = Omit< Result, | 'id' | 'submitterId' | 'submitter' | 'rig' | 'component' | 'verification' | 'moderation' | 'createdAt' | 'updatedAt' - | 'execution' | 'customRuntime' | 'componentHost' | 'sourcePrUrl' + | 'execution' | 'customRuntime' | 'componentHost' | 'sourcePrUrl' | 'runId' > export type BoardKind = 'rigs' | 'components' diff --git a/frontend/src/lib/conditions.ts b/frontend/src/lib/conditions.ts index e126bb4..f72dabd 100644 --- a/frontend/src/lib/conditions.ts +++ b/frontend/src/lib/conditions.ts @@ -84,3 +84,42 @@ export function compactChips(r: Partial): string[] { if (r.decodeMethod === 'wall') chips.push('wall clock') return chips } + +/** + * The model file that ran and how far the repeats spread (issue #130, step 4). Optional like the conditions, but no + * setup checks them: they let a reader tell two files of the same quant apart, and a steady number from a noisy one. + */ +export type RunDetails = Pick +export const DETAIL_KEYS = ['modelRepo', 'modelFile', 'modelSha256', 'decodeTpsMin', 'decodeTpsMax'] as const +export const MODEL_REPO_MAX = 200 +export const MODEL_FILE_MAX = 255 +const SHA256 = /^[0-9a-f]{64}$/ + +/** Lowercases the hash, the one detail with two spellings. Everything else passes through. */ +export const normalizeSha256 = (value: unknown) => (typeof value === 'string' ? value.trim().toLowerCase() : value) + +/** Checks one result's details against its decode rate, keyed by field like `conditionErrors`. */ +export function detailErrors(d: Partial>, decodeTps?: unknown): Partial> { + const errors: Partial> = {} + for (const [key, max] of [['modelRepo', MODEL_REPO_MAX], ['modelFile', MODEL_FILE_MAX]] as const) { + const v = d[key] + if (v != null && !(typeof v === 'string' && v.trim() && v === v.trim() && v.length <= max)) errors[key] = `"${key}" must be text of ${max} characters or fewer.` + } + if (d.modelSha256 != null && !(typeof d.modelSha256 === 'string' && SHA256.test(d.modelSha256))) errors.modelSha256 = '"modelSha256" must be 64 lowercase hex characters.' + const rate = typeof decodeTps === 'number' && decodeTps > 0 ? decodeTps : undefined + for (const key of ['decodeTpsMin', 'decodeTpsMax'] as const) { + const v = d[key] + if (v == null) continue + if (!(typeof v === 'number' && Number.isFinite(v) && v > 0)) errors[key] = `"${key}" must be a number above zero.` + else if (rate != null && (key === 'decodeTpsMin' ? v > rate : v < rate)) errors[key] = `"${key}" must be ${key === 'decodeTpsMin' ? 'at most' : 'at least'} "decodeTps".` + } + return errors +} +export const detailProblems = (d: Partial>, decodeTps?: unknown) => Object.values(detailErrors(d, decodeTps)) as string[] + +/** The details worth a line on the result page: spread, then the file. */ +export function spreadLabel(r: Partial): string | undefined { + if (r.decodeTpsMin == null && r.decodeTpsMax == null) return undefined + const fmt = (n?: number) => (n == null ? '?' : n.toFixed(n >= 100 ? 0 : 1)) + return `${fmt(r.decodeTpsMin)}–${fmt(r.decodeTpsMax)}` +} diff --git a/frontend/src/lib/pr.ts b/frontend/src/lib/pr.ts index dffa4f3..f8af5de 100644 --- a/frontend/src/lib/pr.ts +++ b/frontend/src/lib/pr.ts @@ -2,7 +2,7 @@ import { REVISION_MAX, RUNTIME_FLAGS_MAX, type Result, type ResultInput } from ' import { REPO } from '@/lib/brand' import { HARDWARE_BY_ID, MODEL_BY_ID, QUANT_BY_ID, RUNTIME_BY_ID } from '@/catalog' import { isEvidenceUrl } from '@/lib/evidence' -import { CONDITION_KEYS, conditionProblems, type Conditions } from '@/lib/conditions' +import { CONDITION_KEYS, DETAIL_KEYS, conditionProblems, detailProblems, normalizeSha256, type Conditions, type RunDetails } from '@/lib/conditions' // PR results are validated by trusted CI and inserted into Supabase on merge. // This parser also supports the legacy browser prefill and exporting existing results as archive files. @@ -12,7 +12,17 @@ export const RESULTS_DIR = 'results' const API = 'https://api.github.com' /** A result as committed to the repo. Mirrors ResultInput with catalog ids, plus the rig by id or name. */ -export type ResultFile = Conditions & { +/** + * The fields that change from point to point of a run. A file with `points` lists these per point and keeps + * everything else once at the top, shared by every point. + */ +export const POINT_KEYS = ['decodeTps', 'decodeTpsMin', 'decodeTpsMax', 'promptTps', 'ttftMs', 'inputTokens', 'outputTokens', 'concurrency', 'decodeScope'] as const +/** Measured numbers: only ever per point in a file with points, never at the top. */ +const MEASURED_KEYS = ['decodeTps', 'decodeTpsMin', 'decodeTpsMax', 'promptTps', 'ttftMs'] as const +export const POINTS_MAX = 64 +export type ResultPoint = Partial> & { decodeTps: number } + +export type ResultFile = Conditions & RunDetails & { /** The number at the end of the rig's URL on the site, or the rig's exact name. */ rig: string /** Hardware id of the one part the run used. Leave out for a whole-rig result. */ @@ -42,6 +52,8 @@ export type ResultFile = Conditions & { evidenceUrl?: string /** The result's URL on the site once it has been submitted, so the archive entry and the live entry point at each other. */ result?: string + /** One run at several input lengths or request counts: one result per point, sharing the rest of the file. */ + points?: ResultPoint[] } export type PrRef = { number: number; url: string } @@ -69,7 +81,7 @@ const DATE = /^\d{4}-\d{2}-\d{2}$/ export const RESULT_FILE_KEYS = [ '$schema', 'rig', 'component', 'componentQuantity', 'model', 'quant', 'runtime', 'runtimeVersion', 'runtimeFlags', 'customRuntime', 'customRuntimeFile', 'revision', 'decodeTps', 'promptTps', 'ttftMs', 'contextLength', 'batchSize', - ...CONDITION_KEYS, 'runDate', 'notes', 'evidenceUrl', 'result', + ...CONDITION_KEYS, ...DETAIL_KEYS, 'runDate', 'notes', 'evidenceUrl', 'result', 'points', ] as const /** How many single-letter edits turn a into b. Small strings only. */ @@ -97,6 +109,51 @@ function unknownFieldProblem(key: string): string { * bad id still prefills the rest of the form. The CI check treats any problem as a failure. */ export function parseResultFile(raw: unknown): { file: Partial; problems: string[] } { + if (raw && typeof raw === 'object' && !Array.isArray(raw) && 'points' in raw) return parsePointsFile(raw as Record) + return parseFields(raw) +} + +/** + * A file with points is read as one full result per point: the top-level fields with that point's fields on top. + * A problem every point shares (a bad rig, say) is named once; a problem in one point names the point. + */ +function parsePointsFile(data: Record): { file: Partial; problems: string[] } { + const { points, ...shared } = data + const problems: string[] = [] + for (const key of MEASURED_KEYS) if (key in shared) problems.push(`"${key}" goes in each point, not at the top, in a file with "points".`) + if (!Array.isArray(points) || points.length < 1 || points.length > POINTS_MAX) { + // Still check the shared fields; the placeholder rate only stands in for the missing points. + const base = parseFields({ ...shared, decodeTps: 1 }) + delete base.file.decodeTps + return { file: base.file, problems: [...problems, ...base.problems, `"points" must be a list of 1 to ${POINTS_MAX} points.`] } + } + const parsed = points.map((point, i) => { + if (!point || typeof point !== 'object' || Array.isArray(point)) return { file: {} as Partial, problems: [`points[${i}] must be an object.`], own: true } + const extra = Object.keys(point).filter((key) => !(POINT_KEYS as readonly string[]).includes(key)) + const extraProblems = extra.map((key) => `points[${i}]: "${key}" cannot vary by point; put it at the top. A point holds ${POINT_KEYS.map((k) => `"${k}"`).join(', ')}.`) + const result = parseFields({ ...shared, ...(point as Record) }) + return { file: result.file, problems: [...extraProblems, ...result.problems], own: false } + }) + const counts = new Map() + for (const p of parsed) for (const problem of new Set(p.problems)) counts.set(problem, (counts.get(problem) ?? 0) + 1) + const everywhere = new Set([...counts].filter(([, n]) => n === parsed.length).map(([problem]) => problem)) + problems.push(...everywhere) + parsed.forEach((p, i) => { + for (const problem of p.problems) if (!everywhere.has(problem)) problems.push(problem.startsWith('points[') ? problem : `points[${i}]: ${problem}`) + }) + const seen = new Map() + parsed.forEach((p, i) => { + const key = [p.file.inputTokens, p.file.outputTokens, p.file.concurrency, p.file.decodeScope].join('|') + if (seen.has(key)) problems.push(`points[${i}] repeats points[${seen.get(key)}]: each point needs its own input tokens, output tokens, concurrency or scope.`) + else seen.set(key, i) + }) + const first = parsed.find((p) => !p.own)?.file ?? {} + const file: Partial = Object.fromEntries(Object.entries(first).filter(([key]) => !(POINT_KEYS as readonly string[]).includes(key))) + file.points = parsed.map((p) => Object.fromEntries(POINT_KEYS.filter((key) => p.file[key] != null).map((key) => [key, p.file[key]])) as ResultPoint) + return { file, problems } +} + +function parseFields(raw: unknown): { file: Partial; problems: string[] } { const problems: string[] = [] const file: Partial = {} if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return { file, problems: ['The file must be a JSON object.'] } @@ -156,6 +213,10 @@ export function parseResultFile(raw: unknown): { file: Partial; prob const conditionIssues = conditionProblems(conditions) problems.push(...conditionIssues) if (!conditionIssues.length) Object.assign(file, conditions) + const details = Object.fromEntries(DETAIL_KEYS.filter((key) => data[key] != null).map((key) => [key, key === 'modelSha256' ? normalizeSha256(data[key]) : data[key]])) + const detailIssues = detailProblems(details, file.decodeTps) + problems.push(...detailIssues) + if (!detailIssues.length) Object.assign(file, details) file.runDate = str('runDate', true) if (file.runDate && !DATE.test(file.runDate)) { problems.push('"runDate" must be YYYY-MM-DD.') @@ -231,6 +292,7 @@ export function resultFileFor(r: Result | ResultInput, resultUrl?: string): Resu ...(r.contextLength != null ? { contextLength: r.contextLength } : {}), ...(r.batchSize != null ? { batchSize: r.batchSize } : {}), ...Object.fromEntries(CONDITION_KEYS.filter((key) => r[key] != null).map((key) => [key, r[key]])), + ...Object.fromEntries(DETAIL_KEYS.filter((key) => r[key] != null).map((key) => [key, r[key]])), runDate: r.runDate.slice(0, 10), ...(r.notes ? { notes: r.notes } : {}), ...(r.repoUrl?.trim() ? { evidenceUrl: r.repoUrl.trim() } : {}), diff --git a/frontend/src/lib/streams.ts b/frontend/src/lib/streams.ts index 0c6dbbd..e86a430 100644 --- a/frontend/src/lib/streams.ts +++ b/frontend/src/lib/streams.ts @@ -1,16 +1,23 @@ import type { Result } from '@/lib/api/types' import { MODEL_BY_ID, QUANT_BY_ID } from '@/catalog' -/** One measured point of a series: how many requests ran at once and the decode rate recorded for it. */ -export type StreamPoint = { concurrency: number; decodeTps: number; resultId: string } +/** One measured point of a series: the x value (request count or prompt tokens), the decode rate, and its result. */ +export type StreamPoint = { x: number; decodeTps: number; resultId: string } /** - * Results of one setup (model, quant, runtime, build, part) measured at several request counts, as the two curves a - * scaling study produces: the machine's total across requests, and what each request sees. A single-request result - * is the first point of both, since one request's rate is also the total. + * What a curve is drawn against. `concurrency`: decode against simultaneous requests, as a total or per request. + * `input`: decode for one request against the prompt length it followed. + */ +export type StreamAxis = 'concurrency' | 'input' + +/** + * Results of one setup (model, quant, runtime, build, part) measured at several points of one axis. On the + * concurrency axis a scaling study produces two curves, the machine's total across requests and what each request + * sees, and a single-request result is the first point of both, since one request's rate is also the total. */ export type StreamSeries = { key: string + axis: StreamAxis scope: 'total' | 'per_request' label: string modelId: string @@ -21,34 +28,58 @@ export type StreamSeries = { const unit = (r: Result) => (r.componentId ? `${r.componentQuantity ?? 1}× ${r.component?.name ?? r.componentId}` : '') +type Placement = { scope: StreamSeries['scope']; x: number } + /** - * Group a rig's results into series with at least two distinct request counts. Results without a recorded - * `concurrency` are left out (the legacy `batchSize` never had a fixed meaning), as are hidden ones. When a setup - * was measured more than once at the same request count, the best rate stands, as on the boards. + * Groups results into series per setup with at least two distinct x values. When a setup was measured more than + * once at the same x, the best rate stands, as on the boards. Hidden results are left out. */ -export function streamSeries(results: Result[]): StreamSeries[] { +function group(results: Result[], axis: StreamAxis, place: (r: Result) => Placement[]): StreamSeries[] { const groups = new Map() for (const r of results) { - if (r.moderation.hidden || r.concurrency == null || r.concurrency < 1) continue - const scopes: StreamSeries['scope'][] = r.concurrency === 1 ? ['total', 'per_request'] : r.decodeScope === 'total' ? ['total'] : r.decodeScope === 'per_request' ? ['per_request'] : [] - for (const scope of scopes) { + if (r.moderation.hidden) continue + for (const { scope, x } of place(r)) { const base = [r.modelId, r.quant, r.runtimeId, r.customRuntimeId ?? '', r.componentId ?? '', r.componentQuantity ?? ''].join('|') - const key = `${scope}|${base}` + const key = `${axis}|${scope}|${base}` let series = groups.get(key) if (!series) { const model = MODEL_BY_ID[r.modelId]?.name ?? r.modelId const quant = QUANT_BY_ID[r.quant]?.label ?? r.quant const part = unit(r) - series = { key, scope, label: `${model} ${quant}${part ? ` · ${part}` : ''}`, modelId: r.modelId, quant: r.quant, runtimeId: r.runtimeId, points: [] } + series = { key, axis, scope, label: `${model} ${quant}${part ? ` · ${part}` : ''}`, modelId: r.modelId, quant: r.quant, runtimeId: r.runtimeId, points: [] } groups.set(key, series) } - const existing = series.points.find((p) => p.concurrency === r.concurrency) - if (!existing) series.points.push({ concurrency: r.concurrency, decodeTps: r.decodeTps, resultId: r.id }) + const existing = series.points.find((p) => p.x === x) + if (!existing) series.points.push({ x, decodeTps: r.decodeTps, resultId: r.id }) else if (r.decodeTps > existing.decodeTps) Object.assign(existing, { decodeTps: r.decodeTps, resultId: r.id }) } } return [...groups.values()] .filter((series) => series.points.length >= 2) - .map((series) => ({ ...series, points: [...series.points].sort((a, b) => a.concurrency - b.concurrency) })) + .map((series) => ({ ...series, points: [...series.points].sort((a, b) => a.x - b.x) })) .sort((a, b) => a.scope.localeCompare(b.scope) || a.label.localeCompare(b.label) || a.runtimeId.localeCompare(b.runtimeId)) } + +/** + * Decode against concurrent requests. Results without a recorded `concurrency` are left out (the legacy + * `batchSize` never had a fixed meaning). + */ +export function streamSeries(results: Result[]): StreamSeries[] { + return group(results, 'concurrency', (r) => { + if (r.concurrency == null || r.concurrency < 1) return [] + const scopes: StreamSeries['scope'][] = r.concurrency === 1 ? ['total', 'per_request'] : r.decodeScope === 'total' ? ['total'] : r.decodeScope === 'per_request' ? ['per_request'] : [] + return scopes.map((scope) => ({ scope, x: r.concurrency! })) + }) +} + +/** + * Decode for one request against prompt length: how much slower decode gets as the context fills. Only results with + * `inputTokens` and exactly one request recorded count (the legacy `contextLength` meant the prompt to some and the + * window to others). Speculative runs are left out so they never pose as a faster point on the same line. + */ +export function contextSeries(results: Result[]): StreamSeries[] { + return group(results, 'input', (r) => + r.inputTokens != null && r.concurrency === 1 && r.speculative !== 'lossless' && r.speculative !== 'lossy' + ? [{ scope: 'per_request', x: r.inputTokens }] + : []) +} diff --git a/frontend/src/mocks/seed.ts b/frontend/src/mocks/seed.ts index a849ae1..702fbb2 100644 --- a/frontend/src/mocks/seed.ts +++ b/frontend/src/mocks/seed.ts @@ -222,6 +222,22 @@ export function createSeed(): SeedDb { r.moderation = { flags: f.size, hidden: true, reasons: ['implausible'] } } + // One run at four prompt lengths (issue #130 step 4): the points share a run id, and the rig page draws the curve. + const base = results.find((r) => !r.componentId && !r.customRuntimeId && !r.moderation.hidden) + if (base) { + const runId = 'run-mock-sweep' + ;[[512, 20.66], [4096, 18.51], [16384, 15.2], [32768, 12.4]].forEach(([inputTokens, decodeTps], i) => { + const id = `res-sweep-${i}` + results.push({ + ...base, id, runId, decodeTps, decodeTpsMin: +(decodeTps * 0.99).toFixed(2), decodeTpsMax: +(decodeTps * 1.01).toFixed(2), + inputTokens, outputTokens: 256, concurrency: 1, decodeScope: undefined, decodeMethod: 'streamed', repeats: 3, + promptCache: false, thinking: false, speculative: 'none', modelFile: `${base.modelId}-${base.quant}.bin`, + verification: { status: 'self_reported', confirmations: 0 }, moderation: { flags: 0, hidden: false }, + }) + confirmations.set(id, new Set()) + }) + } + return { users, rigs, customRuntimes, results, confirmations, flags } } diff --git a/frontend/src/pages/RigDetail.tsx b/frontend/src/pages/RigDetail.tsx index 3f61013..481ff66 100644 --- a/frontend/src/pages/RigDetail.tsx +++ b/frontend/src/pages/RigDetail.tsx @@ -28,7 +28,7 @@ import { ApiError } from '@/lib/api/types' import { fmtDate, pluralize } from '@/lib/format' import { rigShareTarget } from '@/lib/share' import { UNIT_LABEL, nestParts } from '@/lib/hardware' -import { streamSeries } from '@/lib/streams' +import { contextSeries, streamSeries } from '@/lib/streams' import { isTotalThroughput } from '@/lib/setups' import { HARDWARE_TYPE_LABEL } from '@/catalog' import { cn } from '@/lib/utils' @@ -58,6 +58,7 @@ export default function RigDetail() { const isOwner = user?.id === r.ownerId const visibleResults = r.results.filter((x) => !x.moderation.hidden) const series = streamSeries(r.results) + const contextCurves = contextSeries(r.results) const visible = visibleResults.length // Totals across concurrent requests are not a rig's best decode speed. const bestResult = visibleResults.filter((x) => !isTotalThroughput(x)).reduce<(typeof visibleResults)[number] | undefined>((best, x) => (!best || x.decodeTps > best.decodeTps ? x : best), undefined) @@ -184,6 +185,19 @@ export default function RigDetail() { ) : null} + {contextCurves.length ? ( +
+ + +

+ Each line is one model, quant and runtime on this rig, one request at a time, after prompts of several lengths. Decode slows as the + context fills. Submit the same setup at another prompt length, with inputTokens and concurrency: 1 recorded, + and it joins the line. +

+
+
+ ) : null} +
{r.results.length ? ( diff --git a/frontend/src/pages/SubmitResult.tsx b/frontend/src/pages/SubmitResult.tsx index 175f641..037607c 100644 --- a/frontend/src/pages/SubmitResult.tsx +++ b/frontend/src/pages/SubmitResult.tsx @@ -31,7 +31,7 @@ import { ApiError, REVISION_MAX, RUNTIME_FLAGS_MAX, type CustomRuntime, type Res import { fmtTps, pluralize } from '@/lib/format' import { resultShareTarget } from '@/lib/share' import { cn } from '@/lib/utils' -import { DECODE_METHODS, DECODE_SCOPES, SPECULATIVE, type Conditions } from '@/lib/conditions' +import { DECODE_METHODS, DECODE_SCOPES, MODEL_FILE_MAX, MODEL_REPO_MAX, SPECULATIVE, detailErrors, normalizeSha256, type Conditions, type RunDetails } from '@/lib/conditions' import { boardPhrase, rankBoardPath, unitPhrase } from '@/lib/rank' import { HARDWARE_BY_ID, MODEL_BY_ID, QUANT_BY_ID, RUNTIME_BY_ID } from '@/catalog' @@ -64,6 +64,12 @@ type Form = { promptCache: string thinking: string speculative: string + // Details: the model file and the spread of the repeats. + modelRepo: string + modelFile: string + modelSha256: string + decodeTpsMin: string + decodeTpsMax: string repoUrl: string runDate: string notes: string @@ -80,19 +86,24 @@ const num = (s: string) => (s.trim() === '' ? undefined : Number(s)) const CONDITION_INTEGERS = ['inputTokens', 'outputTokens', 'concurrency', 'repeats'] as const function emptyConditions() { - return { inputTokens: '', outputTokens: '', concurrency: '', decodeScope: '', decodeMethod: '', repeats: '', promptCache: '', thinking: '', speculative: '' } + return { + inputTokens: '', outputTokens: '', concurrency: '', decodeScope: '', decodeMethod: '', repeats: '', promptCache: '', thinking: '', speculative: '', + modelRepo: '', modelFile: '', modelSha256: '', decodeTpsMin: '', decodeTpsMax: '', + } } /** Form strings from stored conditions, for a file, an edit, or a carried-over setup. */ -function conditionFields(c: Partial): ReturnType { +function conditionFields(c: Partial): ReturnType { const text = (v: number | boolean | string | undefined) => (v == null ? '' : String(v)) return { inputTokens: text(c.inputTokens), outputTokens: text(c.outputTokens), concurrency: text(c.concurrency), decodeScope: text(c.decodeScope), decodeMethod: text(c.decodeMethod), repeats: text(c.repeats), promptCache: text(c.promptCache), thinking: text(c.thinking), speculative: text(c.speculative), + modelRepo: text(c.modelRepo), modelFile: text(c.modelFile), modelSha256: text(c.modelSha256), + decodeTpsMin: text(c.decodeTpsMin), decodeTpsMax: text(c.decodeTpsMax), } } /** Stored conditions from the form. Per request vs total only applies with several requests, so it is dropped otherwise. */ -function conditionsFrom(f: Form): Conditions { +function conditionsFrom(f: Form): Conditions & RunDetails { const bool = (v: string) => (v === '' ? undefined : v === 'true') const concurrency = num(f.concurrency) return { @@ -102,10 +113,13 @@ function conditionsFrom(f: Form): Conditions { repeats: num(f.repeats), promptCache: bool(f.promptCache), thinking: bool(f.thinking), speculative: (f.speculative || undefined) as Conditions['speculative'], + modelRepo: f.modelRepo.trim() || undefined, modelFile: f.modelFile.trim() || undefined, + modelSha256: (normalizeSha256(f.modelSha256) as string) || undefined, + decodeTpsMin: num(f.decodeTpsMin), decodeTpsMax: num(f.decodeTpsMax), } } -/** "Submit another" keeps the setup, since a series of runs usually changes one thing at a time. */ -const conditionsCarried = (f: Form) => conditionFields(conditionsFrom(f)) +/** "Submit another" keeps the setup and the model file, since a series of runs usually changes one thing at a time. The spread belongs to one number. */ +const conditionsCarried = (f: Form) => ({ ...conditionFields(conditionsFrom(f)), decodeTpsMin: '', decodeTpsMax: '' }) // The same model, quant, runtime, and card can differ twofold on kernel choices, so the placeholder names the ones // that matter for each runtime rather than leaving people to guess what belongs here. @@ -144,6 +158,13 @@ function validate(f: Form): Record { const v = num(f[k]) if (v != null && !(Number.isInteger(v) && v >= 1)) e[k] = 'Enter a whole number of 1 or more.' } + for (const [k, message] of Object.entries(detailErrors(conditionsFrom(f), d))) { + e[k] = k === 'modelSha256' ? 'Paste the 64-character SHA-256.' : k.startsWith('decodeTps') ? (k === 'decodeTpsMin' ? 'Slowest run cannot be faster than decode tok/s.' : 'Fastest run cannot be slower than decode tok/s.') : message + } + for (const k of ['decodeTpsMin', 'decodeTpsMax'] as const) { + const v = num(f[k]) + if (v != null && !(v > 0)) e[k] = 'Enter tok/s above zero.' + } if (f.repoUrl.trim() && !isEvidenceUrl(f.repoUrl.trim())) e.repoUrl = 'Enter a full URL, starting with https://.' if (!f.runDate) e.runDate = 'Enter the run date.' else if (f.runDate > today()) e.runDate = 'Run date cannot be in the future.' @@ -178,6 +199,10 @@ function toInput(f: Form): ResultInput { /** Fills the form from a result file. Ids the catalog does not know stay blank and are reported, so the rest still lands. */ function formFromFile(file: Partial, rigs: RigSummary[], customRuntimes: CustomRuntime[], prev: Form): { form: Form; problems: string[] } { const problems: string[] = [] + if (file.points?.length) { + problems.push(`This file holds ${file.points.length} points of one run. The form shows the first; merging submits all of them.`) + file = { ...file, ...file.points[0] } + } if (file.customRuntimeFile) problems.push(`Custom runtime ${file.customRuntimeFile} resolves when this PR is merged. This preview cannot resolve an unmerged registration.`) const wanted = file.rig?.toLowerCase() const rig = wanted ? rigs.find((r) => r.id.toLowerCase() === wanted) ?? rigs.find((r) => r.name.toLowerCase() === wanted) : undefined @@ -878,6 +903,12 @@ export default function SubmitResult({ mode = 'create' }: { mode?: 'create' | 'e + + + + + + {/* Legacy and ambiguous, so new results use the conditions below. Shown only while a value is there, so editing an older result never drops it silently. */} {form.contextLength || errors.contextLength ? ( @@ -944,6 +975,17 @@ export default function SubmitResult({ mode = 'create' }: { mode?: 'create' | 'e
+
+ + + + + + +
+ + + diff --git a/results/README.md b/results/README.md index e4cb415..75a7d50 100644 --- a/results/README.md +++ b/results/README.md @@ -63,12 +63,52 @@ measured the same way can be compared: | `thinking` | `true` if reasoning mode was on. | | `speculative` | `none`, `lossless` (output unchanged, such as MTP or a verified draft model), or `lossy`. | | `revision` | The exact engine commit or tag, for stock and custom runs alike. | +| `modelRepo`, `modelFile` | Where the weights came from and which file: `Qwen/Qwen3-8B-GGUF`, `Qwen3-8B-Q4_K_M.gguf`. Two files of the same quant from different quantizers can differ in speed. | +| `modelSha256` | The model file's SHA-256 (`sha256sum`, or the hash on its download page). | +| `decodeTpsMin`, `decodeTpsMax` | The slowest and fastest measured repeat behind `decodeTps`. | Boards rank per-request numbers. A result with `concurrency` above 1 and `decodeScope` `total` is the machine's throughput, a different quantity from what one request sees, so it ranks only on the **Throughput** tab and never counts as a rig's best decode speed. Measure the same model, quant and runtime at several request counts, record `concurrency` and `decodeScope` on each, and the rig page draws decode tok/s against concurrent requests from them. +### One run, several points + +Measuring the same setup at several prompt lengths or request counts is one run. Write it as one file with a +`points` list instead of one file per point. Each point holds only what changes between points (`decodeTps`, +`decodeTpsMin`, `decodeTpsMax`, `promptTps`, `ttftMs`, `inputTokens`, `outputTokens`, `concurrency`, +`decodeScope`); everything else is written once at the top and applies to every point: + +```json +{ + "$schema": "https://raw.githubusercontent.com/labscommunity/intelinside/main/results/schema.json", + "rig": "16", + "model": "qwen3-8b", + "quant": "q4_k_m", + "runtime": "llamacpp", + "runtimeVersion": "b6512", + "revision": "b6512", + "outputTokens": 256, + "decodeMethod": "streamed", + "repeats": 3, + "promptCache": false, + "thinking": false, + "speculative": "none", + "modelRepo": "Qwen/Qwen3-8B-GGUF", + "modelFile": "Qwen3-8B-Q4_K_M.gguf", + "runDate": "2026-11-18", + "points": [ + { "inputTokens": 512, "concurrency": 1, "decodeTps": 20.66, "decodeTpsMin": 20.51, "decodeTpsMax": 20.79 }, + { "inputTokens": 4096, "concurrency": 1, "decodeTps": 18.51 }, + { "inputTokens": 512, "concurrency": 4, "decodeScope": "per_request", "decodeTps": 15.05 }, + { "inputTokens": 512, "concurrency": 4, "decodeScope": "total", "decodeTps": 28.8 } + ] +} +``` + +Each point becomes its own result, so each ranks on the board of its setup, and the results share a run id. At most +64 points per file; no two points may have the same prompt, output, request count and scope. + `contextLength` and `batchSize` still work but are ambiguous: some people meant the prompt, others the context window; some meant concurrent requests, others the engine batch size. Use `inputTokens` and `concurrency` instead. @@ -87,12 +127,15 @@ requests ranks on **4 requests** and **Throughput**; every point counts toward t For every point: 1. **Settings.** 256 output tokens, temperature 0, thinking off, no speculative decoding, prompt cache off. -2. **Repeats.** One warm-up run, discarded, then 3 measured runs. Report the mean; put the min and max in `notes`. +2. **Repeats.** One warm-up run, discarded, then 3 measured runs. Report the mean as `decodeTps` and the slowest and + fastest as `decodeTpsMin` and `decodeTpsMax`. 3. **Decode.** The streamed rate after the first token: `(outputTokens − 1) / (time of last token − time of first token)`. Report TTFT separately as `ttftMs`. At 4 requests, submit two files from the same runs: the mean per-request decode (`decodeScope: "per_request"`) and the total completion tokens over wall time (`decodeScope: "total"`). -4. **Record.** The engine commit as `revision`, the engine flags as `runtimeFlags`, and the model file (repository, - file name, SHA-256) in `notes`. +4. **Record.** The engine commit as `revision`, the engine flags as `runtimeFlags`, and the model file as + `modelRepo`, `modelFile` and `modelSha256`. + +The whole grid fits in one file: see [One run, several points](#one-run-several-points). ```json { @@ -105,7 +148,12 @@ For every point: "thinking": false, "speculative": "none", "revision": "b6512", - "notes": "min/max 33.9/34.6. Model: Qwen/Qwen3-8B-GGUF Qwen3-8B-Q4_K_M.gguf sha256 1a2b…" + "decodeTps": 34.2, + "decodeTpsMin": 33.9, + "decodeTpsMax": 34.6, + "modelRepo": "Qwen/Qwen3-8B-GGUF", + "modelFile": "Qwen3-8B-Q4_K_M.gguf", + "modelSha256": "<64 hex characters>" } ``` diff --git a/results/schema.json b/results/schema.json index 428ea0d..1313bb6 100644 --- a/results/schema.json +++ b/results/schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/labscommunity/intelinside/main/results/schema.json", "title": "Intelinside result", - "description": "One benchmark result, committed as results//.json. Ids come from the hardware, model, quant, and runtime catalog in frontend/src/catalog/. The pull request check validates the ids; this schema covers the shape.", + "description": "One benchmark result, committed as results//.json. Ids come from the hardware, model, quant, and runtime catalog in frontend/src/catalog/. The pull request check validates the ids; this schema covers the shape. A file with \"points\" is one run measured at several input lengths or request counts: each point becomes its own result, sharing every other field in the file.", "type": "object", "additionalProperties": false, "required": [ @@ -11,7 +11,6 @@ "quant", "runtime", "runtimeVersion", - "decodeTps", "runDate" ], "properties": { @@ -150,6 +149,33 @@ ], "description": "Speculative decoding: none, lossless (output unchanged, e.g. MTP or a draft model with exact verification), or lossy." }, + "modelRepo": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "Where the weights came from, for example Qwen/Qwen3-8B-GGUF." + }, + "modelFile": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The model file within it, for example Qwen3-8B-Q4_K_M.gguf. Two files of the same quant from different quantizers can differ in speed." + }, + "modelSha256": { + "type": "string", + "pattern": "^[0-9a-fA-F]{64}$", + "description": "SHA-256 of the model file." + }, + "decodeTpsMin": { + "type": "number", + "exclusiveMinimum": 0, + "description": "Slowest measured repeat, in decode tokens per second. At most decodeTps." + }, + "decodeTpsMax": { + "type": "number", + "exclusiveMinimum": 0, + "description": "Fastest measured repeat, in decode tokens per second. At least decodeTps." + }, "runDate": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$", @@ -174,6 +200,48 @@ "type": "string", "pattern": "^custom-runtimes/[a-zA-Z0-9-]+/[a-zA-Z0-9][a-zA-Z0-9._-]*\\.json$", "description": "Registration file added in this PR. Mutually exclusive with customRuntime." + }, + "points": { + "type": "array", + "minItems": 1, + "maxItems": 64, + "description": "One run at several input lengths or request counts. Each point becomes a result with the file's other fields; a point holds only the fields that change between points. With points, leave decodeTps, decodeTpsMin, decodeTpsMax, promptTps and ttftMs out of the top level.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "decodeTps" + ], + "properties": { + "decodeTps": { + "$ref": "#/properties/decodeTps" + }, + "decodeTpsMin": { + "$ref": "#/properties/decodeTpsMin" + }, + "decodeTpsMax": { + "$ref": "#/properties/decodeTpsMax" + }, + "promptTps": { + "$ref": "#/properties/promptTps" + }, + "ttftMs": { + "$ref": "#/properties/ttftMs" + }, + "inputTokens": { + "$ref": "#/properties/inputTokens" + }, + "outputTokens": { + "$ref": "#/properties/outputTokens" + }, + "concurrency": { + "$ref": "#/properties/concurrency" + }, + "decodeScope": { + "$ref": "#/properties/decodeScope" + } + } + } } }, "not": { @@ -181,5 +249,27 @@ "customRuntime", "customRuntimeFile" ] - } + }, + "oneOf": [ + { + "required": [ + "decodeTps" + ], + "not": { + "required": [ + "points" + ] + } + }, + { + "required": [ + "points" + ], + "not": { + "required": [ + "decodeTps" + ] + } + } + ] } diff --git a/supabase/migrations/20261118120000_result_runs.sql b/supabase/migrations/20261118120000_result_runs.sql new file mode 100644 index 0000000..c18d12e --- /dev/null +++ b/supabase/migrations/20261118120000_result_runs.sql @@ -0,0 +1,398 @@ +begin; + +-- Issue #130 step 4, the light version: runs on top of the results table instead of a second table. +-- A run is one setup measured at several input lengths or request counts; its results share run_id. Each result can +-- also name the model file it ran (repository, file, SHA-256) and the spread of its repeats. Every field is optional, +-- and existing results keep null. +alter table public.results + add column model_repo text, -- where the weights came from, such as Qwen/Qwen3-8B-GGUF + add column model_file text, -- the file within it, such as Qwen3-8B-Q4_K_M.gguf + add column model_sha256 text, -- the file's SHA-256, lowercase hex + add column decode_tps_min numeric, -- slowest measured repeat + add column decode_tps_max numeric, -- fastest measured repeat + add column run_id uuid, -- shared by the results of one run; set only by PR ingestion and the API + add constraint results_model_repo_length check (model_repo is null or length(model_repo) between 1 and 200), + add constraint results_model_file_length check (model_file is null or length(model_file) between 1 and 255), + add constraint results_model_sha256_hex check (model_sha256 is null or model_sha256 ~ '^[0-9a-f]{64}$'), + add constraint results_decode_tps_min_range check (decode_tps_min is null or (decode_tps_min > 0 and decode_tps_min <= decode_tps)), + add constraint results_decode_tps_max_range check (decode_tps_max is null or decode_tps_max >= decode_tps); +create index results_run_idx on public.results (run_id) where run_id is not null; + +-- The browser writes the model file and spread like any other field. run_id is not granted: only the two trusted +-- paths below set it, so nobody can attach a result to someone else's run. +grant insert (model_repo, model_file, model_sha256, decode_tps_min, decode_tps_max), + update (model_repo, model_file, model_sha256, decode_tps_min, decode_tps_max) +on public.results to authenticated; + +-- A different file or spread is a different measurement, so it resets verification like every other measured field. +drop trigger reset_result_verification_before_edit on public.results; +create trigger reset_result_verification_before_edit +before update of model_id, quant_id, runtime_id, runtime_version, rig_id, component_id, component_quantity, + decode_tps, prompt_tps, ttft_ms, context_length, batch_size, notes, repo_url, run_date, + runtime_flags, custom_runtime_id, revision, + input_tokens, output_tokens, concurrency, decode_scope, decode_method, repeats, prompt_cache, thinking, speculative, + model_repo, model_file, model_sha256, decode_tps_min, decode_tps_max +on public.results +for each row execute function private.touch_result_and_reset_verification(); + +-- A PR file with points becomes several results; its receipt keeps the run so a retry reports every id. +alter table private.pr_result_imports add column run_id uuid; + +-- The API's batch route puts its new results in one run. Only the dispatcher calls this, with an id it generated, +-- and only for the caller's own result. +create function private.assign_result_run(p_result_id bigint, p_run_id uuid) +returns void language sql security definer set search_path = '' as $$ + update public.results set run_id = p_run_id where id = p_result_id and submitter_id = auth.uid() and run_id is null; +$$; +revoke all on function private.assign_result_run(bigint, uuid) from public, anon, authenticated; +grant execute on function private.assign_result_run(bigint, uuid) to intelinside_agent; + +-- PR ingestion writes the new fields and expands points. Otherwise identical to 20261113190000. +create or replace function public.ingest_pr_results( + p_repository text, p_pr_number bigint, p_github_id text, + p_files jsonb, p_dry_run boolean default true +) returns jsonb +language plpgsql security definer set search_path = '' +as $$ +declare + owner_uuid uuid; + item jsonb; + f jsonb; + receipt private.pr_result_imports%rowtype; + registration private.pr_custom_runtime_imports%rowtype; + candidates text; + rig bigint; + build bigint; + matches integer; + result_id bigint; + answer jsonb := '[]'::jsonb; + source_pr text; + g jsonb; + run uuid; + first_id bigint; + ids jsonb; +begin + if p_repository is distinct from 'labscommunity/intelinside' + or p_pr_number is null or p_pr_number <= 0 + or p_github_id is null or p_github_id !~ '^[0-9]+$' + or p_dry_run is null then + raise exception 'Invalid PR provenance'; + end if; + if jsonb_typeof(p_files) is distinct from 'array' or jsonb_array_length(p_files) not between 1 and 100 then + raise exception 'Submit between 1 and 100 submission files'; + end if; + + select p.id into owner_uuid + from auth.identities i join public.profiles p on p.auth_user_id = i.user_id + where i.provider = 'github' and i.provider_id = p_github_id; + if owner_uuid is null then + raise exception 'No site account is linked to the PR author. Sign up on the site with the same GitHub account, then rerun this check.'; + end if; + source_pr := 'https://github.com/' || p_repository || '/pull/' || p_pr_number; + -- Serialize imports of a repository, including simultaneous retries of the same PR. + perform pg_advisory_xact_lock(hashtextextended('pr-results:' || p_repository, 0)); + -- Also serialize against UI/API writes while checking existing sources. + if exists (select 1 from jsonb_array_elements(p_files) where value ->> 'path' like 'custom-runtimes/%') then + lock table public.custom_runtimes in share row exclusive mode; + end if; + if (select count(distinct value ->> 'path') from jsonb_array_elements(p_files)) <> jsonb_array_length(p_files) then + raise exception 'Duplicate or missing submission paths'; + end if; + begin + for item in select value from jsonb_array_elements(p_files) where value ->> 'path' like 'custom-runtimes/%' loop + f := item -> 'file'; + if item ->> 'path' !~ '^custom-runtimes/[a-zA-Z0-9-]+/[a-zA-Z0-9][a-zA-Z0-9._-]*[.]json$' + or jsonb_typeof(f) is distinct from 'object' then raise exception 'Invalid registration file'; end if; + if exists (select 1 from jsonb_object_keys(f) k where k not in ('runtime','name','repoUrl','summary','notes')) then + raise exception 'Unknown registration field'; + end if; + if exists (select 1 from unnest(array['runtime','name','repoUrl','summary']) k + where jsonb_typeof(f -> k) is distinct from 'string' or length(btrim(f ->> k)) = 0 or f ->> k <> btrim(f ->> k)) + or (f ? 'notes' and (jsonb_typeof(f -> 'notes') is distinct from 'string' or length(btrim(f ->> 'notes')) = 0)) + or f ->> 'repoUrl' !~* '^https?://[^[:space:]/?#@\\]+([/?#][^[:space:]\\]*)?$' then + raise exception 'Registration fields must be trimmed nonempty text and repoUrl a full HTTP(S) URL without credentials'; + end if; + select * into registration from private.pr_custom_runtime_imports + where repository = p_repository and path = item ->> 'path'; + if found then + if registration.pr_number <> p_pr_number or registration.github_id <> p_github_id or registration.payload <> f then + raise exception 'Registration file % was already imported. Reuse its ID or edit the live registration on the site.', item ->> 'path'; + end if; + answer := answer || jsonb_build_array(jsonb_build_object('path', item ->> 'path', 'status', 'already_imported', 'id', registration.runtime_id::text)); + continue; + end if; + select string_agg(id::text, ', ' order by id) into candidates from public.custom_runtimes + where runtime_id = f ->> 'runtime' and private.pr_runtime_source(repo_url) = private.pr_runtime_source(f ->> 'repoUrl'); + if candidates is not null then + raise exception 'File %: source/base runtime already registered as IDs %. Reuse an ID or ask a maintainer to resolve the duplicate.', item ->> 'path', candidates; + end if; + insert into public.custom_runtimes(owner_id,runtime_id,name,repo_url,summary,notes,source_pr_url) + values (owner_uuid,f ->> 'runtime',f ->> 'name',f ->> 'repoUrl',f ->> 'summary',f ->> 'notes',source_pr) + returning id into build; + insert into private.pr_custom_runtime_imports(repository,path,pr_number,github_id,payload,runtime_id) + values (p_repository,item ->> 'path',p_pr_number,p_github_id,f,build); + answer := answer || jsonb_build_array(jsonb_build_object('path',item ->> 'path', + 'status',case when p_dry_run then 'validated' else 'imported' end, + 'id',case when p_dry_run then null else build::text end)); + end loop; + for item in select value from jsonb_array_elements(p_files) where value ->> 'path' not like 'custom-runtimes/%' loop + f := item -> 'file'; + if (item ->> 'path') is null or (item ->> 'path') !~ '^results/[^/]+/[^/]+[.]json$' + or jsonb_typeof(f) is distinct from 'object' then + raise exception 'Invalid result file'; + end if; + -- Exports of already-live results are archives, never an instruction to insert/update. + if nullif(f ->> 'result', '') is not null then + answer := answer || jsonb_build_array(jsonb_build_object('path', item ->> 'path', 'status', 'archive')); + continue; + end if; + select * into receipt from private.pr_result_imports + where repository = p_repository and path = item ->> 'path'; + if found then + if receipt.pr_number <> p_pr_number or receipt.github_id <> p_github_id or receipt.payload <> f then + raise exception 'File % was already imported. Edit the live result on the site; use a new filename for a new run.', item ->> 'path'; + end if; + answer := answer || jsonb_build_array(jsonb_build_object('path', item ->> 'path', 'status', 'already_imported', 'id', receipt.result_id::text) + || case when receipt.run_id is null then '{}'::jsonb else jsonb_build_object('ids', + (select coalesce(jsonb_agg(r.id::text order by r.id), '[]'::jsonb) from public.results r where r.run_id = receipt.run_id)) end); + continue; + end if; + -- A file with points is one run measured at several input lengths or request counts: one result per point, + -- sharing the file's other fields and a run id. + if f ? 'points' and (jsonb_typeof(f -> 'points') is distinct from 'array' or jsonb_array_length(f -> 'points') not between 1 and 64 + or exists (select 1 from jsonb_array_elements(f -> 'points') p where jsonb_typeof(p) is distinct from 'object')) then + raise exception 'File %: points must be a list of 1 to 64 objects', item ->> 'path'; + end if; + + select count(*), min(id) into matches, rig from public.rigs + where owner_id = owner_uuid and + (case when f ->> 'rig' ~ '^[0-9]+$' then id::text = f ->> 'rig' else name = f ->> 'rig' end); + if matches <> 1 then + raise exception 'File %: rig must identify exactly one rig owned by the PR author. Register the rig first or use its numeric ID.', item ->> 'path'; + end if; + build := null; + if f ? 'customRuntimeFile' then + if f ? 'customRuntime' or f ->> 'customRuntimeFile' !~ '^custom-runtimes/[a-zA-Z0-9-]+/[a-zA-Z0-9][a-zA-Z0-9._-]*[.]json$' + or jsonb_typeof(f -> 'customRuntimeFile') is distinct from 'string' + or not exists (select 1 from jsonb_array_elements(p_files) x where x ->> 'path' = f ->> 'customRuntimeFile') then + raise exception 'customRuntimeFile must reference a registration in this batch and cannot accompany customRuntime'; + end if; + select c.id into build from private.pr_custom_runtime_imports i join public.custom_runtimes c on c.id = i.runtime_id + where i.repository = p_repository and i.path = f ->> 'customRuntimeFile' and c.runtime_id = f ->> 'runtime'; + if build is null then raise exception 'customRuntimeFile refers to a deleted registration or mismatched runtime'; end if; + elsif nullif(f ->> 'customRuntime', '') is not null then + select count(*), min(id) into matches, build from public.custom_runtimes + where runtime_id = f ->> 'runtime' and + (case when f ->> 'customRuntime' ~ '^[0-9]+$' then id::text = f ->> 'customRuntime' else name = f ->> 'customRuntime' end); + if matches <> 1 then + raise exception 'File %: customRuntime must identify exactly one registered build of the selected runtime; use its numeric ID.', item ->> 'path'; + end if; + end if; + if length(coalesce(f ->> 'notes', '')) > 5000 then + raise exception 'Notes must be 5000 characters or fewer'; + end if; + -- A previous submission through the legacy prefill form must not be imported again. + if exists (select 1 from public.results r where (r.repo_url = source_pr or r.source_pr_url = source_pr) + and not exists (select 1 from private.pr_result_imports i where i.result_id = r.id)) then + raise exception 'This PR already has a result submitted through the site. Add its result URL to the JSON to archive it without creating a duplicate.'; + end if; + run := case when f ? 'points' then gen_random_uuid() end; + first_id := null; + ids := '[]'::jsonb; + for g in select case when f ? 'points' then (f - 'points') || p.value else f end + from jsonb_array_elements(case when f ? 'points' then f -> 'points' else '[{}]'::jsonb end) with ordinality p(value, n) order by p.n loop + insert into public.results ( + submitter_id, rig_id, component_id, component_quantity, model_id, quant_id, + runtime_id, runtime_version, runtime_flags, custom_runtime_id, revision, + decode_tps, prompt_tps, ttft_ms, context_length, batch_size, run_date, notes, repo_url, source_pr_url, + input_tokens, output_tokens, concurrency, decode_scope, decode_method, repeats, prompt_cache, thinking, speculative, + model_repo, model_file, model_sha256, decode_tps_min, decode_tps_max, run_id + ) values ( + owner_uuid, rig, g ->> 'component', + case when g ->> 'component' is not null then coalesce((g ->> 'componentQuantity')::smallint, 1) end, + g ->> 'model', g ->> 'quant', g ->> 'runtime', g ->> 'runtimeVersion', + g ->> 'runtimeFlags', build, g ->> 'revision', + (g ->> 'decodeTps')::numeric, (g ->> 'promptTps')::numeric, (g ->> 'ttftMs')::numeric, + (g ->> 'contextLength')::integer, (g ->> 'batchSize')::integer, + (g ->> 'runDate')::date, g ->> 'notes', nullif(btrim(g ->> 'evidenceUrl'), ''), source_pr, + (g ->> 'inputTokens')::integer, (g ->> 'outputTokens')::integer, (g ->> 'concurrency')::integer, + g ->> 'decodeScope', g ->> 'decodeMethod', (g ->> 'repeats')::integer, + (g ->> 'promptCache')::boolean, (g ->> 'thinking')::boolean, g ->> 'speculative', + g ->> 'modelRepo', g ->> 'modelFile', g ->> 'modelSha256', (g ->> 'decodeTpsMin')::numeric, (g ->> 'decodeTpsMax')::numeric, run + ) returning id into result_id; + first_id := coalesce(first_id, result_id); + ids := ids || to_jsonb(result_id::text); + end loop; + insert into private.pr_result_imports(repository, path, pr_number, github_id, payload, result_id, run_id) + values (p_repository, item ->> 'path', p_pr_number, p_github_id, f, first_id, run); + answer := answer || jsonb_build_array(jsonb_build_object('path', item ->> 'path', + 'status', case when p_dry_run then 'validated' else 'imported' end, + 'id', case when p_dry_run then null else first_id::text end) + || case when run is null or p_dry_run then '{}'::jsonb else jsonb_build_object('ids', ids) end + || case when run is null then '{}'::jsonb else jsonb_build_object('points', jsonb_array_length(f -> 'points')) end); + end loop; + if p_dry_run then raise exception using errcode = 'PT001', message = 'Rollback validation'; end if; + exception when sqlstate 'PT001' then null; + end; + return answer; +end; +$$; +revoke all on function public.ingest_pr_results(text, bigint, text, jsonb, boolean) from public, anon, authenticated; +grant execute on function public.ingest_pr_results(text, bigint, text, jsonb, boolean) to service_role; + +-- The HTTP API accepts the new fields, and a batch can share one run id. Otherwise identical to 20261113190000. +create or replace function private.agent_dispatch(p_action text, p_params jsonb) +returns jsonb language plpgsql security definer set search_path = '' set row_security = on as $$ +declare + actor uuid := auth.uid(); + body jsonb := coalesce(p_params->'body', '{}'); + query jsonb := coalesce(p_params->'query', '{}'); + resource text := split_part(p_action, '.', 1); + operation text := split_part(p_action, '.', 2); + record_id bigint := (p_params->>'id')::bigint; + record jsonb; current_record jsonb; item jsonb; output jsonb := '[]'; + allowed text[]; fields text; values_sql text; updates text; owner_column text; + page_limit integer := least(100, greatest(1, coalesce((query->>'limit')::integer, 25))); + entry record; + batch_run uuid; +begin + if actor is null then raise exception using errcode = '42501', message = 'Missing user identity'; end if; + if p_action = 'me' or p_action = 'users.get' then + select jsonb_build_object('id', id, 'handle', handle, 'name', name, 'avatar_url', avatar_url, 'bio', bio, 'created_at', created_at) + into record from public.profiles where (p_action = 'me' and id = actor) or (p_action = 'users.get' and handle = p_params->>'handle'); + if record is null then raise exception using errcode = 'P0002', message = 'User not found'; end if; + return record; + end if; + if p_action = 'uploads.prepare' then + record := jsonb_build_object('photo_path', actor::text || '/' || gen_random_uuid()::text || + case body->>'content_type' when 'image/png' then '.png' when 'image/jpeg' then '.jpg' when 'image/webp' then '.webp' when 'image/gif' then '.gif' else '' end); + insert into private.agent_uploads(photo_path, owner_id, content_type, size) + values(record->>'photo_path', actor, body->>'content_type', (body->>'size')::integer); + return record; + end if; + if p_action = 'uploads.complete' then + -- Storage metadata is read by a narrowly scoped helper, never by the dispatcher. + if not private.agent_upload_exists(body->>'photo_path', actor) then + raise exception using errcode = '23514', message = 'Upload is missing, belongs to another user, or does not match its declared type and size'; + end if; + update private.agent_uploads set completed_at = coalesce(completed_at, now()) where photo_path = body->>'photo_path'; + return jsonb_build_object('photo_path', body->>'photo_path'); + end if; + if p_action = 'results.batch' then + -- same_run: the items are one run (one setup at several input lengths or request counts) and share a run id. + -- The id is generated here and passed outside the body, so a caller can never choose or join someone else's run. + if coalesce((body->>'same_run')::boolean, false) then batch_run := gen_random_uuid(); end if; + for item in select jsonb_array_elements(body->'items') loop + output := output || jsonb_build_array(private.agent_dispatch('results.create', jsonb_build_object('body', item) + || case when batch_run is null then '{}'::jsonb else jsonb_build_object('run_id', batch_run) end)); + end loop; + return jsonb_build_object('items', output); + end if; + if resource in ('confirmations','flags') then + perform 1 from public.results where id = record_id and submitter_id <> actor and not hidden; + if not found and operation = 'set' then raise exception using errcode = '42501', message = 'Choose another user''s visible result'; end if; + if resource = 'confirmations' then + if operation = 'set' then + insert into public.result_confirmations(result_id,user_id) values(record_id,actor) on conflict do nothing; + else delete from public.result_confirmations where result_id = record_id and user_id = actor; end if; + else + if operation = 'set' then + insert into public.result_flags(result_id,user_id,reason,note) values(record_id,actor,body->>'reason',body->>'note') + on conflict(result_id,user_id) do update set reason = excluded.reason, note = excluded.note; + else delete from public.result_flags where result_id = record_id and user_id = actor; end if; + end if; + return jsonb_build_object('result_id', record_id::text, 'active', operation = 'set'); + end if; + if resource not in ('rigs','results','custom_runtimes') then raise exception using errcode = '22023', message = 'Unknown operation'; end if; + owner_column := case when resource = 'results' then 'submitter_id' else 'owner_id' end; + if operation = 'get' then return private.agent_record(resource,record_id); end if; + if operation = 'list' then + for entry in execute format($query$ + select id from public.%I t where + ($1->>'cursor' is null or id < ($1->>'cursor')::bigint) + and ($1->>'owner' is null or to_jsonb(t)->>%L = case when $1->>'owner' = 'me' then $2::text else $1->>'owner' end) + and ($1->>'q' is null or coalesce(to_jsonb(t)->>'name',to_jsonb(t)->>'notes','') ilike '%%' || ($1->>'q') || '%%') + and ($1->>'rig' is null or to_jsonb(t)->>'rig_id' = $1->>'rig') + and ($1->>'model' is null or to_jsonb(t)->>'model_id' = $1->>'model') + and ($1->>'runtime' is null or to_jsonb(t)->>'runtime_id' = $1->>'runtime') + and ($1->>'hardware' is null or ($3 = 'rigs' and exists(select 1 from public.rig_components c where c.rig_id=t.id and c.hardware_id=$1->>'hardware')) or ($3 = 'results' and to_jsonb(t)->>'component_id'=$1->>'hardware')) + order by id desc limit $4 + $query$,resource,owner_column) using query,actor,resource,page_limit+1 loop + output := output || jsonb_build_array(private.agent_record(resource,entry.id)); + end loop; + if jsonb_array_length(output) > page_limit then + output := output - page_limit; + return jsonb_build_object('items',output,'next_cursor',output->(page_limit-1)->>'id'); + end if; + return jsonb_build_object('items',output); + end if; + if operation in ('update','delete') then + execute format('select to_jsonb(t) from public.%I t where id=$1 and %I=$2 for update',resource,owner_column) + into current_record using record_id,actor; + if current_record is null then raise exception using errcode = 'P0002', message = 'Record not found or not owned by you'; end if; + if (p_params->>'expected_updated_at')::timestamptz is distinct from (current_record->>'updated_at')::timestamptz then + raise exception using errcode = 'PT412', message = 'Record changed. Read it again before retrying your edit'; + end if; + end if; + if operation = 'delete' then + if resource = 'rigs' and exists(select 1 from public.results where rig_id=record_id) and coalesce(query->>'cascade','false') <> 'true' then + raise exception using errcode = 'PT409', message = 'This rig has results. Set cascade=true to explicitly delete them too'; + end if; + execute format('delete from public.%I where id=$1',resource) using record_id; + return jsonb_build_object('id',record_id::text,'deleted',true); + end if; + if operation not in ('create','update') then raise exception using errcode = '22023', message = 'Unknown operation'; end if; + if resource = 'rigs' then + if operation = 'create' then + record_id := public.create_rig(btrim(body->>'name'),coalesce(btrim(body->>'os'),''),body->>'photo_path',body->>'notes',body->'components'); + else + body := current_record || body; + if not body ? 'components' then body := body || jsonb_build_object('components',private.agent_record('rigs',record_id)->'components'); end if; + perform public.update_rig(record_id,btrim(body->>'name'),btrim(body->>'os'),body->>'photo_path',body->>'notes',body->'components'); + end if; + else + if resource = 'custom_runtimes' then + allowed := array['runtime_id','name','repo_url','summary','notes']; + if operation = 'update' and body ? 'runtime_id' and body->>'runtime_id' <> current_record->>'runtime_id' + and private.agent_build_in_use(record_id) then + raise exception using errcode = 'PT409', message = 'Cannot change the base runtime of a build with results'; + end if; + else + allowed := array['rig_id','model_id','quant_id','runtime_id','runtime_version','runtime_flags','custom_runtime_id','revision','component_id','component_quantity','decode_tps','prompt_tps','ttft_ms','context_length','batch_size','notes','repo_url','run_date', + 'input_tokens','output_tokens','concurrency','decode_scope','decode_method','repeats','prompt_cache','thinking','speculative', + 'model_repo','model_file','model_sha256','decode_tps_min','decode_tps_max']; + record := coalesce(current_record,'{}') || body; + if not exists(select 1 from public.rigs where id=(record->>'rig_id')::bigint and owner_id=actor) then + raise exception using errcode = '42501', message = 'Result rig must belong to you'; + end if; + if record->>'component_id' is not null and record->>'component_quantity' is null then body := body || '{"component_quantity":1}'; end if; + if body ? 'component_id' and body->>'component_id' is null then body := body || '{"component_quantity":null}'; end if; + if record->>'custom_runtime_id' is not null and not exists(select 1 from public.custom_runtimes where id=(record->>'custom_runtime_id')::bigint and runtime_id=record->>'runtime_id') then + raise exception using errcode = '23514', message = 'Custom runtime must use the result''s base runtime'; + end if; + end if; + -- Identifier lists come exclusively from the allowlist above, never user SQL. + select string_agg(format('%I',key),','), string_agg(format('v.%I',key),','), string_agg(format('%I=v.%I',key,key),',') + into fields, values_sql, updates from jsonb_object_keys(body) key where key=any(allowed); + if fields is null then raise exception using errcode = '22023', message = 'Provide at least one field to update'; end if; + if operation = 'create' then + execute format('insert into public.%I(%I,%s) select $2,%s from jsonb_populate_record(null::public.%I,$1) v returning id',resource,owner_column,fields,values_sql,resource) + into record_id using body,actor; + else + execute format('update public.%I t set %s from jsonb_populate_record(null::public.%I,$1) v where t.id=$2 returning t.id',resource,updates,resource) + into record_id using body,record_id; + end if; + if resource = 'results' and operation = 'create' and p_params ? 'run_id' then + perform private.assign_result_run(record_id, (p_params->>'run_id')::uuid); + end if; + end if; + return private.agent_record(resource,record_id); +end; +$$; +-- Ownership is the RLS boundary. Never change this function to postgres/service_role. +grant create on schema private to intelinside_agent; +alter function private.agent_dispatch(text,jsonb) owner to intelinside_agent; +revoke create on schema private from intelinside_agent; +revoke all on function private.agent_dispatch(text,jsonb) from public, anon, authenticated; + +commit;