diff --git a/apps/data/bin/build-manifest.mts b/apps/data/bin/build-manifest.mts index 8c2ee19844..971e6bddc1 100755 --- a/apps/data/bin/build-manifest.mts +++ b/apps/data/bin/build-manifest.mts @@ -15,15 +15,18 @@ * * node bin/build-manifest.mts --out suite/results gh-pages-data suite/results * - * Writes only index.json, manifest.json and the manifest/ shards. It never + * Writes only index.json, manifest.json, the manifest/ shards and the summary/ + * files. It never * copies or re-emits the published date directories (decision D2), and never * writes an index.html. */ -import { mkdirSync, statSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { buildManifest } from "../src/manifest.ts"; import { scanRoots } from "../src/scan.ts"; -import { INDEX_FILE, MANIFEST_DIR, MANIFEST_FILE } from "../src/paths.ts"; +import { buildSummary } from "../src/summary.ts"; +import { INDEX_FILE, MANIFEST_DIR, MANIFEST_FILE, SUMMARY_DIR } from "../src/paths.ts"; +import { isDateKey } from "@benchmarks/types"; const argv = process.argv.slice(2); let out: string | undefined; @@ -107,9 +110,28 @@ for (const [name, entry] of Object.entries(manifest.entries)) { shardCount++; } +// One results summary per date (src/summary.ts), deterministic for the same +// reason as the shards. latest/ is merged rather than a run, so it gets none. +const summaryDir = join(out, SUMMARY_DIR); +mkdirSync(summaryDir, { recursive: true }); +let summaryCount = 0; +const skipped: string[] = []; +for (const listing of listings) { + if (!isDateKey(listing.name) || listing.dir === undefined) continue; + const dir = listing.dir; + const built = buildSummary(listing, (file) => readFileSync(join(dir, file), "utf8")); + skipped.push(...built.skipped.map((file) => `${listing.name}/${file}`)); + writeFileSync(join(summaryDir, `${listing.name}.json`), `${JSON.stringify(built.summary)}\n`); + summaryCount++; +} +if (skipped.length > 0) { + console.warn(`left out of the summaries, not benchmark files: ${skipped.join(", ")}`); +} + const dirCount = Object.keys(manifest.entries).length; console.log( - `wrote ${INDEX_FILE}, ${MANIFEST_FILE} and ${shardCount} ${MANIFEST_DIR}/ shards to ${out}: ` + + `wrote ${INDEX_FILE}, ${MANIFEST_FILE}, ${shardCount} ${MANIFEST_DIR}/ shards and ` + + `${summaryCount} ${SUMMARY_DIR}/ files to ${out}: ` + `${index.dates.length} dates, ${dirCount} directories, latest=${index.latest ?? "none"}, ` + `${index.fixtures.length} fixtures, ${index.variations.length} variations`, ); diff --git a/apps/data/src/client.ts b/apps/data/src/client.ts index 1e66629464..3aabc9a515 100644 --- a/apps/data/src/client.ts +++ b/apps/data/src/client.ts @@ -25,8 +25,10 @@ import { manifestShardPath, packageCountFile, processCountFile, + summaryPath, VERSIONS_FILE, } from "./paths.ts"; +import type { DateSummary } from "./summary.ts"; export type FetchLike = (url: string) => Promise<{ ok: boolean; @@ -129,6 +131,11 @@ export class DataClient { return this.#get(date, processCountFile(fixture, variation)); } + /** Every benchmark file of one date in one fetch (src/summary.ts); 404 until a deploy has written it. */ + getSummary(date: string): Promise { + return this.#get(...summaryPath(date)); + } + getChartData(date: string): Promise { return this.#get(date, CHART_DATA_FILE); } diff --git a/apps/data/src/index.ts b/apps/data/src/index.ts index e0da4af58d..0f8813f979 100644 --- a/apps/data/src/index.ts +++ b/apps/data/src/index.ts @@ -21,8 +21,11 @@ export { packageCountFile, parseDataFile, processCountFile, + SUMMARY_DIR, + summaryPath, VERSIONS_FILE, type FileKind, type ParsedDataFile, } from "./paths.ts"; export { scanRoots } from "./scan.ts"; +export { buildSummary, type DateSummary, type SummaryResult } from "./summary.ts"; diff --git a/apps/data/src/manifest.ts b/apps/data/src/manifest.ts index 83f33451e1..9ef466dbfd 100644 --- a/apps/data/src/manifest.ts +++ b/apps/data/src/manifest.ts @@ -66,6 +66,8 @@ export interface DataManifest { export interface DirectoryListing { name: string; files: string[]; + /** Where the directory is on disk, when it was scanned from one. */ + dir?: string; } const addPair = (into: VariationsByFixture, fixture: string, variation: string): void => { diff --git a/apps/data/src/paths.ts b/apps/data/src/paths.ts index 8a04263253..1852cca603 100644 --- a/apps/data/src/paths.ts +++ b/apps/data/src/paths.ts @@ -34,6 +34,11 @@ export const MANIFEST_DIR = "manifest"; */ export const manifestShardPath = (name: string): string[] => [MANIFEST_DIR, `${name}.json`]; +/** Directory holding one results summary per date (src/summary.ts), beside the shards. */ +export const SUMMARY_DIR = "summary"; + +export const summaryPath = (date: string): string[] => [SUMMARY_DIR, `${date}.json`]; + const PACKAGE_COUNT_SUFFIX = "-package-count.json"; const PROCESS_COUNT_SUFFIX = "-process-count.json"; const JSON_SUFFIX = ".json"; diff --git a/apps/data/src/scan.ts b/apps/data/src/scan.ts index 24af88cd57..5c9b96bfc4 100644 --- a/apps/data/src/scan.ts +++ b/apps/data/src/scan.ts @@ -31,7 +31,7 @@ export const scanRoots = (roots: string[]): DirectoryListing[] => { } catch { continue; } - byName.set(name, { name, files }); + byName.set(name, { name, files, dir }); } } diff --git a/apps/data/src/summary.ts b/apps/data/src/summary.ts new file mode 100644 index 0000000000..aca48b66ba --- /dev/null +++ b/apps/data/src/summary.ts @@ -0,0 +1,54 @@ +/** + * One date's benchmark results, in one file. + * + * A trend over many scenarios -- the "average" view averages every scenario + * on every fixture -- otherwise costs one fetch per benchmark file per run: + * 54 a run for package managers, about 6,500 for 120 days. This is the same + * numbers, lifted verbatim out of those files, so it costs one. + * + * Not `chart-data.json`. That file gives a failed run a time, and nothing in + * it says the run failed. Every result here keeps its `exit_codes`, so a + * consumer applies the same rule to it as to the file it came from. + */ +import type { BenchmarkResult } from "@benchmarks/types"; +import type { DirectoryListing } from "./manifest.ts"; +import { parseDataFile } from "./paths.ts"; + +/** What a trend needs of a result, exactly as the benchmark file has it. */ +export type SummaryResult = Pick; + +/** Variation, then fixture, to that file's results. */ +export type DateSummary = { + [variation: string]: { [fixture: string]: SummaryResult[] | undefined } | undefined; +}; + +/** + * Summarise one date directory. `read` returns a file's text; a file that + * does not parse as a benchmark file is reported in `skipped`, not guessed at. + */ +export const buildSummary = ( + listing: DirectoryListing, + read: (file: string) => string, +): { summary: DateSummary; skipped: string[] } => { + // A variation is whatever the file name says, `__proto__` included. + const summary: DateSummary = Object.create(null); + const skipped: string[] = []; + for (const file of listing.files) { + const parsed = parseDataFile(file); + if (parsed?.kind !== "benchmark") continue; + let results: unknown; + try { + results = (JSON.parse(read(file)) as { results?: unknown }).results; + } catch { + results = undefined; + } + if (!Array.isArray(results) || !results.every((r) => typeof r === "object" && r !== null)) { + skipped.push(file); + continue; + } + (summary[parsed.variation] ??= {})[parsed.fixture] = (results as BenchmarkResult[]).map( + ({ command, mean, stddev, exit_codes }) => ({ command, mean, stddev, exit_codes }), + ); + } + return { summary, skipped }; +}; diff --git a/apps/data/test/summary.test.ts b/apps/data/test/summary.test.ts new file mode 100644 index 0000000000..2d2504469d --- /dev/null +++ b/apps/data/test/summary.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { buildSummary } from "../src/summary.ts"; + +describe("buildSummary", () => { + it("lifts each benchmark file's results verbatim, exit codes and all", () => { + const files: { [name: string]: string } = { + "next-clean.json": JSON.stringify({ + results: [ + { command: "npm", mean: 2, stddev: 0.1, median: 2, times: [2], exit_codes: [0] }, + { command: "zpm", mean: 0, stddev: 0, median: 0, times: [0], exit_codes: [1] }, + ], + }), + "next-registry-clean.json": JSON.stringify({ results: [] }), + "next-clean-package-count.json": "{}", + "chart-data.json": "{}", + "astro-clean.json": "not json", + "vue-clean.json": JSON.stringify({ results: [null] }), + "next-__proto__.json": JSON.stringify({ results: [] }), + }; + const { summary, skipped } = buildSummary( + { name: "2026-10-19", files: Object.keys(files) }, + (file) => files[file] ?? "", + ); + expect(summary).toEqual({ + clean: { + next: [ + { command: "npm", mean: 2, stddev: 0.1, exit_codes: [0] }, + { command: "zpm", mean: 0, stddev: 0, exit_codes: [1] }, + ], + }, + "registry-clean": { next: [] }, + ["__proto__"]: { next: [] }, + }); + expect(Object.prototype).not.toHaveProperty("next"); + expect(skipped).toEqual(["astro-clean.json", "vue-clean.json"]); + }); +}); diff --git a/apps/www/package.json b/apps/www/package.json index 1f3bc686fe..e49afe7597 100644 --- a/apps/www/package.json +++ b/apps/www/package.json @@ -15,6 +15,7 @@ "test": "vitest run" }, "dependencies": { + "@base-ui/react": "1.9.0", "@benchmarks/data": "workspace:*", "@benchmarks/types": "workspace:*", "@mdx-js/loader": "^3.1.1", diff --git a/apps/www/src/app/[section]/[[...params]]/page.tsx b/apps/www/src/app/[section]/[[...params]]/page.tsx index ef2258a190..6d0b8c227e 100644 --- a/apps/www/src/app/[section]/[[...params]]/page.tsx +++ b/apps/www/src/app/[section]/[[...params]]/page.tsx @@ -1,7 +1,7 @@ import type { Metadata } from "next"; import { Fragment, Suspense } from "react"; import { notFound, redirect } from "next/navigation"; -import { ArrowRight } from "lucide-react"; +import { ArrowRight, ChartLine } from "lucide-react"; import { InlineCode, Marked } from "@/components/inline-code"; import { Button } from "@/components/ui/button"; import { TimeSeriesChart } from "@/components/charts-echarts"; @@ -13,6 +13,7 @@ import { NoData } from "@/components/no-data"; import { Callout, Figure, ReportSection, Stat } from "@/components/report"; import { DataAndSource } from "@/components/raw-results"; import { FixtureMenu } from "@/components/fixture-menu"; +import { HeadingLink } from "@/components/heading-anchor"; import { PathSkeleton } from "@/components/path-skeleton"; import { PendingLink, PendingProvider, PendingRegion } from "@/components/report-pending"; import { @@ -637,18 +638,31 @@ const Report = async ({ } action={ - run.isLatest ? undefined : ( - - ) + {run.isLatest ? null : ( + + )} + } /> {caveats.qualifier === null ? null : ( -
+
@@ -656,7 +670,7 @@ const Report = async ({ )} {report.stats === null ? null : ( -
+
{report.stats.map((stat) => ( +
{caveats.hidden.body}{" "} @@ -684,7 +698,10 @@ const Report = async ({ )} {caveats.disclosure === null ? null : ( -

+

Disclosure. {caveats.disclosure}

)} diff --git a/apps/www/src/app/globals.css b/apps/www/src/app/globals.css index cafa54dd4b..4be35969c3 100644 --- a/apps/www/src/app/globals.css +++ b/apps/www/src/app/globals.css @@ -21,6 +21,8 @@ */ @custom-variant dark (&:where([data-theme="dark"], [data-theme="dark"] *)); +/* Data mode (components/view-toggle.tsx). */ +@custom-variant data-mode (&:where([data-view="data"] *)); /* The snapshot banner is in the header while a past run is shown. */ :root:has([data-snapshot-banner]) { @@ -502,3 +504,24 @@ background-position: 0 0; } } + +/* Data mode (components/view-toggle.tsx): the charts, tables and numbers, without the prose around them. */ +[data-view="data"] [data-narrative] { + display: none; +} + +/* The actions row goes once every link in it has (the latest run's link stays). */ +[data-view="data"] [data-actions]:not(:has(a:not([data-narrative]))) { + display: none; +} + +/* The finding is the page's h1, so it stays for a screen reader. */ +[data-view="data"] [data-narrative="heading"] { + display: block; + position: absolute; + width: 1px; + height: 1px; + overflow: hidden; + clip-path: inset(50%); + white-space: nowrap; +} diff --git a/apps/www/src/app/layout.tsx b/apps/www/src/app/layout.tsx index 6503b5dec7..77af0fe2cd 100644 --- a/apps/www/src/app/layout.tsx +++ b/apps/www/src/app/layout.tsx @@ -70,6 +70,14 @@ const RootLayout = async ({ children }: { children: ReactNode }) => { className={`${sans.variable} ${mono.variable} ${pixel.variable} h-full antialiased`} suppressHydrationWarning > + + {/* The reader's Report or Data choice, on before paint (components/view-toggle.tsx). */} +