diff --git a/.cursor/rules/adding-new-fixtures.mdc b/.cursor/rules/adding-new-fixtures.mdc index 1e6319b036..b17d010848 100644 --- a/.cursor/rules/adding-new-fixtures.mdc +++ b/.cursor/rules/adding-new-fixtures.mdc @@ -1,192 +1,159 @@ +--- +description: Adding New Fixtures to Benchmarks +globs: suite/fixtures/**,suite/scripts/**,packages/types/src/**,.github/workflows/** +--- # Adding New Fixtures to Benchmarks -Comprehensive guide for adding new fixture types to the vlt benchmarks suite. +Checklist for adding a fixture to the benchmark suite. The explanation, with the +verification commands and the reasoning, is in +`docs/FIXTURES/ADDING-A-FIXTURE.md` — keep the two in step. name: adding_new_fixtures -description: Complete checklist and procedures for adding new benchmark fixtures +description: Checklist for adding a benchmark fixture, including the vlt.json step whose omission silently measures the wrong project filters: - # Match fixture-related files and directories - type: path - pattern: "^fixtures/" + pattern: "^suite/fixtures/" - type: path - pattern: "scripts/(generate-chart\\.js|process-results\\.sh|benchmark\\.sh)" + pattern: "^suite/fixtures/.*/package\\.json$" - type: path - pattern: "app/src/(types|hooks|lib|components)/" + pattern: "^suite/fixtures/.*/vlt\\.json$" - type: path - pattern: "\\.github/workflows/" - # Match package.json files in fixtures + pattern: "suite/scripts/(generate-chart\\.js|process-results\\.sh|benchmark\\.sh)" + - type: path + pattern: "^packages/types/src/" - type: path - pattern: "fixtures/.*/package\\.json$" + pattern: "\\.github/workflows/" actions: - type: suggest message: | - ## Adding a New Fixture Checklist + ## Adding a fixture: checklist - When adding a new fixture (e.g., "my-fixture"), follow these steps: + Full reasoning: `docs/FIXTURES/ADDING-A-FIXTURE.md`. - ### 1. Create Fixture Directory and Configuration - ```bash - mkdir -p fixtures/my-fixture - ``` - - Create `fixtures/my-fixture/package.json` with: - - Appropriate dependencies for your test case - - Valid npm package.json structure - - Consider the fixture's purpose (small, large, specific framework, etc.) - - ### 2. Update Scripts (Data Processing) - - **File: `scripts/generate-chart.js`** - ```javascript - // Line ~54: Add fixture to the fixtures array - const fixtures = ["next", "astro", "svelte", "vue", "large", "my-fixture", "run"]; + ### 0. `vlt.json` — DO NOT SKIP THIS + + Every fixture directory needs its own `vlt.json`: + + ```json + { "config": { "registries": { "npm": "https://registry.vlt.io/vlt-benchmarks/npm/" } } } ``` - **File: `scripts/process-results.sh`** + It points installs at the benchmark registry mirror AND makes the fixture + directory a vlt project root. + + **Without it, vlt walks up and resolves the REPOSITORY ROOT as the + project.** The install then operates on the monorepo's dependency graph + instead of the fixture's. There is no error — you get numbers, and they + measure the wrong thing. Verified: from a directory with a `package.json` + and no `vlt.json`, `vlt query ':root'` returns the `benchmarks` root + package at `location: "."`. + + If the fixture is itself a monorepo, declare its workspaces here too — + `suite/fixtures/babylon/vlt.json` is the worked example. + + ### 1. Create the fixture + ```bash - # Line ~55: Add fixture to the processing loop - for fixture in next astro svelte vue large my-fixture; do + mkdir -p suite/fixtures/ + # then: package.json AND vlt.json ``` - ### 3. Update TypeScript Types + `` becomes part of every published filename as + `{fixture}-{variation}.json`, and the parser splits on the FIRST `-`, so + the name must not contain a hyphen. Use kebab-case otherwise. - **File: `app/src/types/chart-data.ts`** - ```typescript - // Line ~13: Add to Fixture type union - export type Fixture = "next" | "astro" | "svelte" | "vue" | "large" | "my-fixture" | "run"; - ``` + ### 2. Suite enumerations — TWO occurrences in each file - ### 4. Update React Application - - **File: `app/src/hooks/use-package-count-data.ts`** - ```typescript - // Lines ~26-33: Add to fixtures array - const fixtures: Fixture[] = sortFixtures([ - "next", - "astro", - "svelte", - "vue", - "large", - "my-fixture", - "run", - ]); - ``` + | File | Occurrences | + | --- | --- | + | `suite/scripts/generate-chart.js` | **2** arrays (one includes `run`, one does not) | + | `suite/scripts/process-results.sh` | **2** loops (both exclude `run`) | - **File: `app/src/lib/utils.ts`** - ```typescript - // Line ~334: Add to sortFixtures preferredOrder - const preferredOrder: Fixture[] = ["next", "vue", "svelte", "astro", "large", "my-fixture", "run"]; - ``` + Updating only the first occurrence yields a partial integration that + still exits 0. - ### 5. Update GitHub Actions Workflow - - **File: `.github/workflows/benchmark.yaml`** - ```yaml - # Line ~8: Update default fixtures list - default: '["next", "astro", "vue", "svelte", "large", "my-fixture"]' - - # Line ~36: Update matrix fixture list - fixture: ${{ fromJson(inputs.fixtures || '["next", "astro", "vue", "svelte", "large", "my-fixture"]') }} - - # Lines ~102-108: Update JavaScript fixtures array - const fixtures = [ - "next", - "astro", - "vue", - "svelte", - "large", - "my-fixture", - "run" - ]; - ``` + ### 3. Published types - ### 6. Update Icon Support (Optional) - - **File: `app/src/lib/get-icons.ts`** - ```typescript - // Import appropriate icon or use existing one - import { MyFixtureIcon } from "@/components/icons/index"; - - // Add to frameworkMap - const frameworkMap: Partial> = { - astro: Astro, - next: Next, - svelte: Svelte, - vue: Vue, - large: Package, - "my-fixture": MyFixtureIcon, // or reuse existing icon - }; - ``` + `packages/types/src/index.ts` — add to **both** the `Fixture` union and + the `FIXTURES` array. A `satisfies` clause ties them together, so missing + one is a type error rather than a silent gap. - ### 7. Update Documentation - - **File: `README.md`** - ```markdown - ### Project Types (fixtures) - - Next.js - - Astro - - Svelte - - Vue - - Large - - My Fixture (description of what this fixture tests) - ``` + ### 4. Workflow — THREE places in `.github/workflows/benchmark.yaml` - ### 8. Verification Steps + | Line (approx.) | What | + | --- | --- | + | 8 | the `fixtures` input default | + | 98 | the literal the partial-run gate string-compares against | + | 119 | the matrix `fixture` list | - After making all changes: + Line 98 is the trap. The workflow decides a dispatch is a "partial run" — + and therefore must not deploy — by comparing the input against the + default list as a string. Change the default without changing that + literal and every dispatch is classified partial, so results silently + stop publishing while every job stays green. - 1. **Lint Check**: Run linting on modified TypeScript files - 2. **Directory Check**: Verify fixture directory exists with package.json - 3. **Build Test**: Test that the app builds successfully - 4. **Workflow Test**: Verify GitHub Actions workflow accepts the new fixture + Decide separately whether the fixture joins the reduced set that runs on + pushes to non-`main` branches. `large` and `babylon` are excluded there + because they are slow. - ### 9. Fixture Design Considerations + ### 5. Documentation - When creating the fixture's `package.json`: - - - **Small fixtures**: 5-15 dependencies (testing basic scenarios) - - **Medium fixtures**: 20-50 dependencies (realistic projects) - - **Large fixtures**: 50+ dependencies (stress testing) - - **Framework fixtures**: Use actual framework templates - - **Specific use cases**: Target particular scenarios (monorepo, TypeScript, etc.) + - `docs/FIXTURES/.md`, following the existing pages + - a row in `docs/FIXTURES/README.md` + - the fixture list in the root `README.md` - ### 10. Common Pitfalls + ### 6. What you do NOT need to touch - ⚠️ **Don't forget to update ALL locations**: - - Missing GitHub Actions updates will cause CI failures - - Missing TypeScript type updates will cause build errors - - Missing script updates will exclude fixture from processing - - Missing icon mapping will show no icon in UI + `apps/www` discovers fixtures from the generated manifest at runtime, so + the frontend needs no per-fixture change. `apps/data` validates fixture + names against `FIXTURES`, so step 3 covers it. - ⚠️ **Maintain consistent ordering**: - - Keep the same fixture order across all files - - Follow the established naming conventions - - Use kebab-case for fixture directory names + Older versions of this rule listed files under `app/src/` — those + instructions are obsolete. That app is parked at `tmp/app/`. + + ### 7. Verify + + ```bash + # resolves to itself, not the repository root + cd suite/fixtures/ && vlt install && vlt query ':root' --view=json + + # one short run, from the repo root + ./suite/bench run --fixtures= --variation=clean --runs=1 + ``` + + Then clean up before committing. A run leaves + `suite/fixtures//package-lock.json`, which DOES show as untracked, + and `suite/fixtures//node_modules/`, which is gitignored and shows + nowhere: + + ```bash + git status --porcelain suite/fixtures/ + rm -rf suite/fixtures//node_modules suite/fixtures//package-lock.json + ``` examples: - input: | - # Example: Adding a "typescript" fixture - - 1. Create fixtures/typescript/package.json - 2. Update all 7+ files mentioned above - 3. Use TypeScript icon or create new one - 4. Update README with description - - output: "Complete fixture integration following all steps" + # Adding a "remix" fixture + suite/fixtures/remix/package.json + suite/fixtures/remix/vlt.json <- without this, wrong numbers + suite/scripts/generate-chart.js <- 2 arrays + suite/scripts/process-results.sh <- 2 loops + packages/types/src/index.ts <- Fixture union + FIXTURES + .github/workflows/benchmark.yaml <- lines 8, 98, 119 + docs/FIXTURES/REMIX.md + README rows + output: "Complete fixture integration" metadata: priority: high - version: 1.0 - last_updated: "2024-09-25" + version: 2.0 + last_updated: "2026-09-30" related_files: - - "scripts/generate-chart.js" - - "scripts/process-results.sh" - - "app/src/types/chart-data.ts" - - "app/src/hooks/use-package-count-data.ts" - - "app/src/lib/utils.ts" + - "docs/FIXTURES/ADDING-A-FIXTURE.md" + - "suite/scripts/generate-chart.js" + - "suite/scripts/process-results.sh" + - "packages/types/src/index.ts" - ".github/workflows/benchmark.yaml" - - "app/src/lib/get-icons.ts" - "README.md" - \ No newline at end of file + diff --git a/.cursor/rules/keeping-docs-current.mdc b/.cursor/rules/keeping-docs-current.mdc new file mode 100644 index 0000000000..d5aeefbbc6 --- /dev/null +++ b/.cursor/rules/keeping-docs-current.mdc @@ -0,0 +1,109 @@ +--- +description: Keeping docs/ current +globs: docs/**,suite/**,apps/**,packages/**,.github/workflows/**,README.md +--- +# Keeping `docs/` current + +`docs/` is the answer to "nobody knows how to run this thing locally or how it +works in production". It only stays the answer if changes that invalidate it +update it in the same commit. + + +name: keeping_docs_current +description: Which documents a change invalidates, and the rule that documented commands must be run +filters: + - type: path + pattern: "^docs/" + - type: path + pattern: "^suite/" + - type: path + pattern: "^apps/" + - type: path + pattern: "^packages/" + - type: path + pattern: "\\.github/workflows/" + - type: path + pattern: "^README\\.md$" + +actions: + - type: suggest + message: | + ## Does this change invalidate a document? + + | If you changed | Update | + | --- | --- | + | a variation, fixture, or anything about what is measured | `docs/METHODOLOGY.md` | + | the shape of published data | `docs/METHODOLOGY.md` **and** `packages/types` | + | a fixture | its `docs/FIXTURES/.md`, the table in `docs/FIXTURES/README.md`, the root `README.md` | + | added or removed a fixture | also `docs/FIXTURES/ADDING-A-FIXTURE.md` and `.cursor/rules/adding-new-fixtures.mdc` | + | a local command, script, or env var | `docs/GET-STARTED/DEVELOPMENT.md` | + | `apps/data`'s endpoints or client methods | `docs/METHODOLOGY.md` and `docs/GET-STARTED/DEVELOPMENT.md` | + | a workflow, deploy step, or gh-pages behaviour | `docs/GET-STARTED/PRODUCTION.md` | + | **`runs-on`, or anything about the machine** | the **Environment** section of the root `README.md` — it states the hardware the published numbers were produced on, and it said `ubuntu-latest` for the whole time the suite ran on arm | + | `apps/www` — charts, tokens, components, or an honesty behaviour | `docs/FRONTEND.md` | + | workspace layout, catalogs, or required scripts | `docs/PACKAGE-CONTRACT.md` | + + ## Three rules + + ### 1. Every documented command must have been run + + Not "should work". Run it, confirm the exit code, and prefer pasting real + output over describing it. This documentation exists because the previous + instructions had drifted into being wrong while still looking plausible. + + ### 2. A silent failure is worth more words than a feature + + This repository's signature failure is a command that exits 0 having done + the wrong thing or nothing: `bench process` reprocessing stale output, + `test` passing with a type error, a fixture without `vlt.json` measuring + the repository, `NEXT_PUBLIC_*` ignoring a changed env var. When you find + another, add it to + `docs/GET-STARTED/DEVELOPMENT.md#traps-that-cost-people-an-afternoon` + with the real terminal output and the exit code. + + State the blast radius too. "CI is not exposed to this" is as important as + the trap, or a reader draws a bigger conclusion than the evidence supports. + + ### 3. Do not duplicate; link + + Each fact lives in one document. `docs/PACKAGE-CONTRACT.md` owns the + workspace contract, `docs/METHODOLOGY.md` owns the data contract, + `docs/GET-STARTED/*` own the workflows. A second copy is a second thing to + go stale. Cross-reference with a relative link and check it resolves. + + ## When a path in the frozen suite is wrong + + `suite/` is frozen: changing it changes what the benchmarks measure. Some + strings it *prints* are stale — `bench:73`, `bench:352`, + `download-latest-artifacts.sh:178-183`, `generate-chart.js:6` all name an + `app/latest` path that no longer applies. + + Do not edit them to fix documentation. Correct the path in `docs/` and + make sure the stale-strings table in + `docs/GET-STARTED/DEVELOPMENT.md#stale-printed-guidance-inside-the-suite` + lists it, so the printed text is not the only answer a reader finds. + +examples: + - input: | + # Added a client method to apps/data + apps/data/src/client.ts + docs/METHODOLOGY.md <- endpoint table + docs/GET-STARTED/DEVELOPMENT.md <- endpoint table + output: "Change and its documentation in one commit" + - input: | + # Found a command that exits 0 while doing nothing + docs/GET-STARTED/DEVELOPMENT.md <- traps section, with real output, + exit code, and blast radius + output: "Trap documented with evidence" + +metadata: + priority: high + version: 1.0 + last_updated: "2026-09-30" + related_files: + - "docs/METHODOLOGY.md" + - "docs/GET-STARTED/DEVELOPMENT.md" + - "docs/GET-STARTED/PRODUCTION.md" + - "docs/PACKAGE-CONTRACT.md" + - "docs/FIXTURES/README.md" + diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index e435d1b43f..e7f0777189 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -32,7 +32,7 @@ jobs: name: "Detect Changes" runs-on: ubuntu-24.04-arm outputs: - app_only: ${{ steps.check.outputs.app_only }} + skip_benchmarks: ${{ steps.check.outputs.skip_benchmarks }} partial_run: ${{ steps.partial.outputs.partial_run }} steps: - uses: actions/checkout@v6 @@ -43,7 +43,7 @@ jobs: run: | # For scheduled runs and manual triggers, always run benchmarks if [[ "${{ github.event_name }}" != "push" ]]; then - echo "app_only=false" >> "$GITHUB_OUTPUT" + echo "skip_benchmarks=false" >> "$GITHUB_OUTPUT" echo "Non-push event (${{ github.event_name }}), running full benchmarks" exit 0 fi @@ -52,7 +52,7 @@ jobs: CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || echo "") if [ -z "$CHANGED_FILES" ]; then - echo "app_only=false" >> "$GITHUB_OUTPUT" + echo "skip_benchmarks=false" >> "$GITHUB_OUTPUT" echo "Could not determine changed files, running full benchmarks" exit 0 fi @@ -60,24 +60,36 @@ jobs: echo "Changed files:" echo "$CHANGED_FILES" - # Check if ALL changed files are under app/ (or non-benchmark files like README, .editorconfig) - APP_ONLY=true + # skip_benchmarks=true means: nothing benchmark-relevant changed in + # this push, so there is nothing to run. It was called app_only until + # the deploy-app job was removed; with no app to deploy, the name was + # a lie but the logic is unchanged and the truth table is identical. + # + # The allowlist below is everything OUTSIDE the benchmark suite: the + # apps, packages, docs and the parked tmp/app, plus non-benchmark + # files like README and .editorconfig. + # + # It FAILS OPEN, and that is the property that makes it safe: any + # path not matched falls through to *) and runs the full matrix. A + # new top-level directory therefore costs a benchmark run rather than + # silently skipping one. Do not change that by adding a catch-all. + SKIP_BENCHMARKS=true while IFS= read -r file; do case "$file" in - app/*|README.md|.editorconfig|.cursor/*|.github/workflows/*) + apps/*|packages/*|docs/*|tmp/*|README.md|.editorconfig|.cursor/*|.github/workflows/*) # App or non-benchmark files — safe to skip benchmarks ;; *) # Benchmark-relevant file changed - APP_ONLY=false + SKIP_BENCHMARKS=false echo "Benchmark-relevant file changed: $file" break ;; esac done <<< "$CHANGED_FILES" - echo "app_only=$APP_ONLY" >> "$GITHUB_OUTPUT" - echo "App-only changes: $APP_ONLY" + echo "skip_benchmarks=$SKIP_BENCHMARKS" >> "$GITHUB_OUTPUT" + echo "Nothing benchmark-relevant changed: $SKIP_BENCHMARKS" - name: Detect partial run id: partial env: @@ -107,7 +119,7 @@ jobs: name: "Run Benchmarks" runs-on: ubuntu-24.04-arm needs: [detect-changes] - if: needs.detect-changes.outputs.app_only != 'true' + if: needs.detect-changes.outputs.skip_benchmarks != 'true' timeout-minutes: 120 environment: Benchmark strategy: @@ -139,6 +151,10 @@ jobs: node-version: "24" package-manager-cache: false - name: Install & Setup Tools + # setup.sh writes ./results/versions.json relative to cwd, and bench + # resolves results/ against its own directory. Both must agree on + # suite/, so this runs from there rather than via a path prefix. + working-directory: suite run: | bash ./scripts/setup.sh - name: Configure AWS Credentials @@ -201,7 +217,7 @@ jobs: BENCH_WARMUP: ${{ steps.tune.outputs.warmup }} BENCH_RUNS: ${{ steps.tune.outputs.runs }} run: | - ./bench run \ + ./suite/bench run \ --fixtures="${{ matrix.fixture }}" \ --variation="${{ matrix.variation }}" - name: Upload Benchmark Results @@ -209,20 +225,20 @@ jobs: uses: actions/upload-artifact@v7 with: name: results-${{ matrix.fixture }}-${{ matrix.variation }} - path: ./results/${{ matrix.fixture }}/${{ matrix.variation }}/ + path: ./suite/results/${{ matrix.fixture }}/${{ matrix.variation }}/ retention-days: 7 - name: Upload Versions Info if: ${{ !cancelled() }} uses: actions/upload-artifact@v7 with: name: versions-${{ matrix.fixture }}-${{ matrix.variation }} - path: ./results/versions.json + path: ./suite/results/versions.json retention-days: 7 process: name: "Process Results" runs-on: ubuntu-24.04-arm needs: [detect-changes, benchmark] - if: needs.detect-changes.outputs.app_only != 'true' + if: needs.detect-changes.outputs.skip_benchmarks != 'true' timeout-minutes: 10 steps: - uses: actions/checkout@v6 @@ -233,12 +249,12 @@ jobs: - name: Download Results uses: actions/download-artifact@v8 with: - path: results + path: suite/results pattern: results-* - name: Download Versions uses: actions/download-artifact@v8 with: - path: versions-temp + path: suite/versions-temp pattern: versions-* - name: Scan Raw Registry Results id: registry-raw @@ -254,10 +270,10 @@ jobs: const path = require('path') const failures = [] - for (const entry of fs.readdirSync('results').sort()) { + for (const entry of fs.readdirSync('suite/results').sort()) { const match = /^results-(.+)-(registry-[a-z]+)$/.exec(entry) if (!match) continue - const file = path.join('results', entry, 'benchmarks.json') + const file = path.join('suite/results', entry, 'benchmarks.json') if (!fs.existsSync(file)) continue const label = `${match[1]} ${match[2]}` @@ -278,8 +294,13 @@ jobs: core.setOutput('vlt_failures', failures.join('\n')) - name: Process Results run: | - ./bench process + ./suite/bench process - name: Summarize results in log + # Reads the raw per-matrix artifact directories, which live under the + # suite. The node snippet also indexes the path by position + # (p.split("/")[1]), so running from suite/ keeps it correct without + # rewriting the snippet. + working-directory: suite run: | { echo '```' @@ -297,20 +318,31 @@ jobs: done echo '```' } | tee -a "$GITHUB_STEP_SUMMARY" + - name: Checkout gh-pages data + # The data index must describe the WHOLE published tree, not just this + # run. keep_files merges, so an index generated from results/ alone + # would replace the published one with a partial view. + uses: actions/checkout@v6 + with: + ref: gh-pages + path: gh-pages-data - name: Install vlt run: | npm install -g vlt@latest - - name: Build Charts View + - name: Generate Data Index + # apps/data emits index.json, manifest.json and the manifest/ shards + # into suite/results/, the single results tree. It never re-emits the + # published date directories (decision D2) and never writes an + # index.html -- the published index.html is the live site at + # benchmarks.vlt.sh and survives via keep_files. run: | - pushd app vlt install - vlt run build - popd + vlt run --workspace=apps/data build - name: Upload Processed Results uses: actions/upload-artifact@v7 with: name: results - path: results/ + path: suite/results/ retention-days: 7 - name: Check Registry Results id: registry-check @@ -333,7 +365,7 @@ jobs: // alert fires exactly when vlt appears slower on the site. // Registry variations are the only ones where `command` is a // registry name rather than a package manager. - const dir = 'results/latest' + const dir = 'suite/results/latest' for (const entry of fs.readdirSync(dir).sort()) { const match = /^(.+)-(registry-[a-z]+)\.json$/.exec(entry) if (!match) continue @@ -386,59 +418,19 @@ jobs: needs: [detect-changes, process] permissions: contents: write - if: github.ref == 'refs/heads/main' && needs.detect-changes.outputs.app_only != 'true' && needs.detect-changes.outputs.partial_run != 'true' + if: github.ref == 'refs/heads/main' && needs.detect-changes.outputs.skip_benchmarks != 'true' && needs.detect-changes.outputs.partial_run != 'true' steps: - uses: actions/checkout@v6 - name: Download Results uses: actions/download-artifact@v8 with: name: results - path: results/ - - name: Deploy to GitHub Pages - uses: peaceiris/actions-gh-pages@v4 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: results - keep_files: true - - deploy-app: - name: "Deploy App Only" - runs-on: ubuntu-24.04-arm - needs: [detect-changes] - permissions: - contents: write - if: github.ref == 'refs/heads/main' && needs.detect-changes.outputs.app_only == 'true' - steps: - - uses: actions/checkout@v6 - - name: Install Node - uses: actions/setup-node@v6 - with: - node-version: "24" - - name: Checkout gh-pages data - uses: actions/checkout@v6 - with: - ref: gh-pages - path: gh-pages-data - - name: Install vlt - run: | - npm install -g vlt@latest - - name: Build App - run: | - pushd app - vlt install - vlt run build - popd - - name: Prepare deploy directory - run: | - # Copy the latest data files from gh-pages into the built results - cp -r gh-pages-data/latest results/latest - # Copy index.html and assets from the fresh build are already in results/ - # The deploy action with keep_files merges with existing gh-pages content + path: suite/results/ - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: results + publish_dir: suite/results keep_files: true on-failure: @@ -454,7 +446,6 @@ jobs: - benchmark - process - deploy - - deploy-app steps: - name: Slack Notify uses: rtCamp/action-slack-notify@v2.3.3 @@ -474,4 +465,3 @@ jobs: Benchmark: ${{ needs.benchmark.result }} Process: ${{ needs.process.result }} Deploy: ${{ needs.deploy.result }} - Deploy App: ${{ needs.deploy-app.result }} diff --git a/.github/workflows/reprocess.yml b/.github/workflows/reprocess.yml index 90ab0de278..646e50a12b 100644 --- a/.github/workflows/reprocess.yml +++ b/.github/workflows/reprocess.yml @@ -32,14 +32,14 @@ jobs: - name: Download Results uses: actions/download-artifact@v8 with: - path: results + path: suite/results pattern: results-* run-id: ${{ inputs.run-id }} github-token: ${{ secrets.GITHUB_TOKEN }} - name: Download Versions uses: actions/download-artifact@v8 with: - path: versions-temp + path: suite/versions-temp pattern: versions-* run-id: ${{ inputs.run-id }} github-token: ${{ secrets.GITHUB_TOKEN }} @@ -47,20 +47,23 @@ jobs: env: BENCH_DATE: ${{ inputs.date }} run: | - ./bench process + ./suite/bench process + - name: Checkout gh-pages data + uses: actions/checkout@v6 + with: + ref: gh-pages + path: gh-pages-data - name: Install vlt run: | npm install -g vlt@latest - - name: Build Charts View + - name: Generate Data Index run: | - pushd app vlt install - vlt run build - popd + vlt run --workspace=apps/data build - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: results + publish_dir: suite/results keep_files: true full_commit_message: "reprocess: run ${{ inputs.run-id }} -> ${{ inputs.date }} (dispatched via ${{ github.workflow }} run ${{ github.run_id }})" diff --git a/.gitignore b/.gitignore index 8aa57f60fc..51f582815e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,11 +13,16 @@ node_modules dist chart charts -/results/ /versions-temp/ - -# ignore latest in app for testing -/app/latest/ +/suite/results/ +/suite/versions-temp/ +/gh-pages-data/ +/tmp/app/latest/ # local agent config (kept untracked, same as vlt.io/vltpkg) .claude + +# agent stuff +.mcp.json +.openrig +CLAUDE.md diff --git a/.oxfmtrc.json b/.oxfmtrc.json new file mode 100644 index 0000000000..5557c55ba6 --- /dev/null +++ b/.oxfmtrc.json @@ -0,0 +1,19 @@ +{ + "$schema": "./node_modules/oxfmt/configuration_schema.json", + "ignorePatterns": [ + "**/node_modules/**", + "**/dist/**", + "**/.next/**", + "**/vlt-lock.json", + "**/package-lock.json", + ".github/**", + "bench", + "fixtures/**", + "scripts/**", + "suite/**", + "app/**", + "tmp/**", + "versions-temp/**", + "projects/**" + ] +} diff --git a/.oxlintrc.json b/.oxlintrc.json new file mode 100644 index 0000000000..7f83a3df19 --- /dev/null +++ b/.oxlintrc.json @@ -0,0 +1,25 @@ +{ + "$schema": "./node_modules/oxlint/configuration_schema.json", + "plugins": ["typescript", "unicorn", "oxc"], + "categories": { + "correctness": "error" + }, + "rules": {}, + "env": { + "builtin": true + }, + "ignorePatterns": [ + "**/node_modules/**", + "**/dist/**", + "**/.next/**", + ".github/**", + "bench", + "fixtures/**", + "scripts/**", + "suite/**", + "app/**", + "tmp/**", + "versions-temp/**", + "projects/**" + ] +} diff --git a/README.md b/README.md index 8bdbad9dfd..04d683dbfb 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,42 @@ This repo contains a suite of fixtures & tools to track the performance of package managers. We benchmark various Node.js package managers (npm, yarn, pnpm, berry, deno, bun, vlt, aube, upm, nx, turbo) across different project types and scenarios. +## Documentation + +| Document | For | +| ------------------------------------------ | -------------------------------------------------------- | +| [docs/METHODOLOGY.md](docs/METHODOLOGY.md) | what is measured, how, and what the published data means | +| [docs/FIXTURES/](docs/FIXTURES/README.md) | one page per fixture | + +Two things that surprise most newcomers, both documented above: `suite/results/` +is the **only** results tree — measured into, indexed in place and published from, +with no copy step anywhere — and `./suite/bench process` can report success while +taking no new measurements. + ## Environment -We currently only test the latest `linux` runner which is the most common GitHub Action environment. The [standard GitHub-hosted public runner environment](https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories) specs are below: +We currently only test a `linux` runner, the most common GitHub Actions +environment. The workflow label is read from the workflows themselves — all six +benchmark jobs in `.github/workflows/benchmark.yaml` and +`.github/workflows/reprocess.yml` set `runs-on: ubuntu-24.04-arm`: -- VM: Linux -- Processor (CPU): 4 +- Workflow label: `ubuntu-24.04-arm` +- VM: Linux (Ubuntu 24.04) +- Architecture: `arm64` +- Processor (CPU): 4 cores - Memory (RAM): 16 GB - Storage (SSD): 14 GB -- Workflow label: `ubuntu-latest` + +The label and the architecture come from the workflows. The remaining figures +are GitHub's [published specification for standard public +runners](https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories), +not measurements taken here. + +> This section said `ubuntu-latest` until 2026-10-02, with the x64 specs under +> it. The suite has been running on arm since the move to arm runners, so the +> published methodology was describing a different machine than the one +> producing the numbers. If you are comparing these results against your own, +> that difference matters more than any of the figures above. We may add Mac/Windows in the future but it will likely exponentially increase the already slow run time of the suite (**~40min**). @@ -25,12 +52,15 @@ The benchmarks measure: ### Project Types (fixtures) -- Next.js -- Astro -- Svelte -- Vue -- Large -- Babylon.js (Massive 3D engine monorepo with 86+ packages) +Seven, each with [its own page](docs/FIXTURES/README.md): + +- [Next.js](docs/FIXTURES/NEXT.md) — the default fixture; the only one with `build` variations +- [Astro](docs/FIXTURES/ASTRO.md) — smallest install, one declared dependency +- [Svelte](docs/FIXTURES/SVELTE.md) — entirely devDependencies +- [Vue](docs/FIXTURES/VUE.md) — heaviest framework starter +- [Large](docs/FIXTURES/LARGE.md) — 109 direct dependencies, no application +- [Babylon.js](docs/FIXTURES/BABYLON.md) — monorepo, 86 workspace packages +- [Run](docs/FIXTURES/RUN.md) — script execution only, no dependencies ### Package Managers @@ -87,10 +117,10 @@ Examples: ```bash # Run only npm + vlt registry benchmarks -./bench run --variation=registry-clean --fixtures=next --registries=npm,vlt +./suite/bench run --variation=registry-clean --fixtures=next --registries=npm,vlt # Run only aws registry benchmarks (requires token) -CODEARTIFACT_AUTH_TOKEN= ./bench run --variation=registry-clean --fixtures=next --registries=aws +CODEARTIFACT_AUTH_TOKEN= ./suite/bench run --variation=registry-clean --fixtures=next --registries=aws ``` Auth notes: @@ -127,61 +157,59 @@ This suite also tests the performance of basic script execution (ex. `npm run fo ```bash npm install -g npm@latest corepack@latest vlt@latest bun@latest deno@latest nx@latest turbo@latest ``` - - 1.4 Make a new `results` folder: - ```bash - mkdir -p results - ``` + - 1.4 No directory setup is needed. `./suite/bench` creates its output tree at + `suite/results/`, which is the only results tree there is. - 2. Run benchmarks: ```bash # Install and benchmark a specific project - ./bench run --fixtures= --variation= + ./suite/bench run --fixtures= --variation= # Example: Benchmark Next.js with cold cache - ./bench run --fixtures=next --variation=clean + ./suite/bench run --fixtures=next --variation=clean ``` - 3. Local bench wrapper options: ```bash # Defaults: next + clean (uses script defaults for runs/warmup) - ./bench run + ./suite/bench run # Pick fixtures, package managers, and runs - ./bench run --fixtures=next,astro --pms=vlt,pnpm --runs=3 + ./suite/bench run --fixtures=next,astro --pms=vlt,pnpm --runs=3 # Run all fixtures (except "run") - ./bench run --fixtures=all --variation=clean + ./suite/bench run --fixtures=all --variation=clean # Run multiple variations in one command - ./bench run --fixtures=svelte --variation=lockfile,registry-lockfile,run --registries=npm,vlt + ./suite/bench run --fixtures=svelte --variation=lockfile,registry-lockfile,run --registries=npm,vlt # Script-execution benchmark - ./bench run --variation=run --pms=vlt + ./suite/bench run --variation=run --pms=vlt # Clean failed runs immediately during run/chart (optional) - ./bench run --fixtures=next --variation=clean --clean + ./suite/bench run --fixtures=next --variation=clean --clean - # Generate chart data (and copy to app/latest) after a run - ./bench run --fixtures=next --runs=3 --chart + # Generate chart data (and copy to tmp/app/latest) after a run + ./suite/bench run --fixtures=next --runs=3 --chart # Run benchmarks then process outputs in one command - ./bench run --fixtures=next --runs=3 --process + ./suite/bench run --fixtures=next --runs=3 --process # Process existing results into chart data - ./bench chart --fixtures=next --variation=clean + ./suite/bench chart --fixtures=next --variation=clean ``` - By default, cleaning happens during `./bench process`. Use `--clean` on `run`/`chart` for immediate cleanup. + By default, cleaning happens during `./suite/bench process`. Use `--clean` on `run`/`chart` for immediate cleanup. - 4. Process benchmark output: ```bash # Filter failed runs and generate dated + latest outputs and chart data - ./bench process + ./suite/bench process # Or pass --process directly when running - ./bench run --fixtures=next --runs=3 --process + ./suite/bench run --fixtures=next --runs=3 --process ``` ### GitHub Actions @@ -256,35 +284,34 @@ Preferred workflow (matches CI): 2. Download artifacts from the latest successful run on `main`: ```sh - ./scripts/download-latest-artifacts.sh + ./suite/scripts/download-latest-artifacts.sh ``` This keeps artifact-style directories such as - `results/results--/` and `versions-temp/`. + `suite/results/results--/` and `suite/versions-temp/`. 3. Process downloaded artifacts: ```sh - ./bench process + ./suite/bench process ``` This cleans benchmark outputs, builds dated and latest result folders, - and generates chart data (`results//chart-data.json` and `results/latest/chart-data.json`). + and generates chart data (`suite/results//chart-data.json` and + `suite/results/latest/chart-data.json`). -4. Copy the latest chart data into the app: +4. Index and serve the data, then run the current app: ```sh - mkdir -p app/latest - cp results/latest/chart-data.json app/latest/chart-data.json + node apps/data/bin/build-manifest.mts --out suite/results suite/results + node apps/data/bin/serve.mts --root suite/results --port 8811 + vlt run --workspace=apps/www build && vlt run --workspace=apps/www start ``` -5. Run the app: - - ```sh - cd app - vlt install - vlr dev - ``` + The current frontend (`apps/www`) fetches from `apps/data` over HTTP, so + there is **no chart-data file to copy**. Steps 4 and 5 of the instructions + the download script prints are for `tmp/app/`, the parked previous app, and + no longer apply. ## License diff --git a/app/vite.config.ts b/app/vite.config.ts deleted file mode 100644 index 2bbc3dab60..0000000000 --- a/app/vite.config.ts +++ /dev/null @@ -1,17 +0,0 @@ -import path from "path"; -import { defineConfig } from "vite"; -import react from "@vitejs/plugin-react"; -import tailwindcss from "@tailwindcss/vite"; - -export default defineConfig({ - base: "/", - plugins: [react(), tailwindcss()], - resolve: { - alias: { - "@": path.resolve(__dirname, "./src"), - }, - }, - build: { - outDir: "../results/", - }, -}); diff --git a/apps/data/bin/build-manifest.mts b/apps/data/bin/build-manifest.mts new file mode 100755 index 0000000000..8c2ee19844 --- /dev/null +++ b/apps/data/bin/build-manifest.mts @@ -0,0 +1,115 @@ +#!/usr/bin/env node +/** + * Generate index.json and manifest.json for the served tree. + * + * Usage: + * node bin/build-manifest.mts --out [...] + * + * `--out` is REQUIRED and has no default. A default would point at a fixed + * directory, and a default pointing at a directory that may not exist is this + * repository's signature failure: it produces a silent, empty success. A bare + * invocation must fail loudly instead. + * + * Sources are scanned in order and later ones win, so CI layers this run's + * freshly assembled tree over a checkout of the published history: + * + * node bin/build-manifest.mts --out suite/results gh-pages-data suite/results + * + * Writes only index.json, manifest.json and the manifest/ shards. It never + * copies or re-emits the published date directories (decision D2), and never + * writes an index.html. + */ +import { mkdirSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { buildManifest } from "../src/manifest.ts"; +import { scanRoots } from "../src/scan.ts"; +import { INDEX_FILE, MANIFEST_DIR, MANIFEST_FILE } from "../src/paths.ts"; + +const argv = process.argv.slice(2); +let out: string | undefined; +const roots: string[] = []; + +for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--out") { + const value = argv[i + 1]; + if (value === undefined) { + console.error("--out requires a directory"); + process.exit(2); + } + out = value; + i++; + } else if (arg !== undefined && arg.startsWith("--out=")) { + out = arg.slice("--out=".length); + } else if (arg !== undefined) { + roots.push(arg); + } +} + +if (out === undefined) { + console.error( + "--out is required.\n" + + "Refusing to guess an output directory: a wrong guess writes an index\n" + + "describing nothing and exits 0.\n" + + " node bin/build-manifest.mts --out suite/results gh-pages-data suite/results", + ); + process.exit(2); +} + +if (roots.length === 0) roots.push(out); + +// Same gap as serve.mts had: a source that is named but absent gets skipped by +// scanRoots, and if every source is absent the result is an index describing +// nothing, written successfully, exit 0. A named source must exist. +const missing = roots.filter((r) => { + try { + return !statSync(r).isDirectory(); + } catch { + return true; + } +}); +if (missing.length > 0) { + console.error( + `source directories do not exist: ${missing.join(", ")}\n` + + "Refusing to continue: skipping them would write an index describing\n" + + "less than the real tree, and exit 0.", + ); + process.exit(2); +} + +const listings = scanRoots(roots); +const { index, manifest } = buildManifest(listings, new Date().toISOString()); + +// A tree with no dates at all is never a legitimate outcome here; it means the +// sources existed but held nothing recognisable. +if (Object.keys(manifest.entries).length === 0) { + console.error( + `no date or latest directories found under: ${roots.join(", ")}\n` + + "Refusing to write an index that describes nothing.", + ); + process.exit(2); +} + +mkdirSync(out, { recursive: true }); +writeFileSync(join(out, INDEX_FILE), `${JSON.stringify(index, null, 2)}\n`); +writeFileSync(join(out, MANIFEST_FILE), `${JSON.stringify(manifest)}\n`); + +// One shard per directory, so a consumer wanting a single date does not pull +// the whole archive. Each shard is exactly manifest.entries[name] -- no +// timestamp, no wrapper -- which keeps unchanged dates byte-identical between +// runs, so the gh-pages deploy records no diff for them. +const shardDir = join(out, MANIFEST_DIR); +mkdirSync(shardDir, { recursive: true }); +let shardCount = 0; +for (const [name, entry] of Object.entries(manifest.entries)) { + if (entry === undefined) continue; + writeFileSync(join(shardDir, `${name}.json`), `${JSON.stringify(entry, null, 2)}\n`); + shardCount++; +} + +const dirCount = Object.keys(manifest.entries).length; +console.log( + `wrote ${INDEX_FILE}, ${MANIFEST_FILE} and ${shardCount} ${MANIFEST_DIR}/ shards to ${out}: ` + + `${index.dates.length} dates, ${dirCount} directories, latest=${index.latest ?? "none"}, ` + + `${index.fixtures.length} fixtures, ${index.variations.length} variations`, +); diff --git a/apps/data/bin/serve.mts b/apps/data/bin/serve.mts new file mode 100755 index 0000000000..72f438fd53 --- /dev/null +++ b/apps/data/bin/serve.mts @@ -0,0 +1,68 @@ +#!/usr/bin/env node +/** + * Serve the results tree locally. + * + * Usage: + * node bin/serve.mts --root [--port ] + * + * `--root` is REQUIRED and has no default. Serving a directory that does not + * exist answers 404 to everything while the process reports itself healthy -- + * the same silent-empty-success this repository keeps producing. Refusing to + * guess turns that into an immediate, readable failure. + */ +import { statSync } from "node:fs"; +import { createDataServer } from "../src/serve.ts"; + +const argv = process.argv.slice(2); +let root: string | undefined; +let port = 8811; + +for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--root") { + root = argv[i + 1] ?? root; + i++; + } else if (arg === "--port") { + port = Number(argv[i + 1] ?? port); + i++; + } else if (arg === "--watch") { + // Accepted so `dev` and `start` can share this entrypoint. The tree is + // read per request, so there is nothing to rebuild on change. + } +} + +if (root === undefined) { + console.error( + "--root is required.\n" + + "Refusing to guess a directory to serve: a wrong guess answers 404 to\n" + + "every request while looking healthy.\n" + + " node bin/serve.mts --root suite/results", + ); + process.exit(2); +} + +// Requiring the argument is not enough. suite/results/ is gitignored and +// absent in a fresh checkout, so the common failure is a root that is supplied +// and spelled correctly but does not exist yet -- which serves 404 to +// everything while reporting itself healthy. Same trap, one step later. +let rootIsDir = false; +try { + rootIsDir = statSync(root).isDirectory(); +} catch { + rootIsDir = false; +} +if (!rootIsDir) { + console.error( + `--root ${root} is not a directory.\n` + + "Refusing to serve it: an absent root answers 404 to every request while\n" + + "the process looks healthy. suite/results/ is gitignored and absent in a\n" + + "fresh checkout -- generate it first:\n" + + " ./suite/bench process # from measured results\n" + + " vlt run --workspace=apps/data build", + ); + process.exit(2); +} + +createDataServer({ root, port }).listen(port, () => { + console.log(`serving ${root} on http://localhost:${port}`); +}); diff --git a/apps/data/package.json b/apps/data/package.json new file mode 100644 index 0000000000..3fcffbb820 --- /dev/null +++ b/apps/data/package.json @@ -0,0 +1,31 @@ +{ + "name": "@benchmarks/data", + "version": "0.0.0", + "private": true, + "description": "Discovery layer over the benchmark data published on gh-pages: a generated index plus a typed client. Does not re-emit the published tree.", + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "build": "node bin/build-manifest.mts --out ../../suite/results ../../gh-pages-data ../../suite/results", + "dev": "node bin/serve.mts --root ../../suite/results", + "start": "node bin/serve.mts --root ../../suite/results", + "lint:check": "oxlint bin src test", + "lint": "oxlint --fix bin src test", + "format:check": "oxfmt --check bin src test", + "format": "oxfmt bin src test", + "typecheck": "tsc --noEmit", + "test": "vitest run" + }, + "dependencies": { + "@benchmarks/types": "workspace:*" + }, + "devDependencies": { + "@types/node": "catalog:hygiene", + "oxfmt": "catalog:hygiene", + "oxlint": "catalog:hygiene", + "typescript": "catalog:hygiene", + "vitest": "catalog:testing" + } +} diff --git a/apps/data/src/client.ts b/apps/data/src/client.ts new file mode 100644 index 0000000000..1e66629464 --- /dev/null +++ b/apps/data/src/client.ts @@ -0,0 +1,140 @@ +/** + * Typed client over the served tree. + * + * This is the only thing `apps/www` should use to reach the data. Per the + * mission's interface contract it talks HTTP against a base URL and never + * reads across the workspace boundary on disk, so the same code works against + * a local server and against the gh-pages origin. + */ +import type { + BenchmarkFile, + ChartData, + Fixture, + PackageCount, + ProcessCount, + Variation, + Versions, +} from "@benchmarks/types"; +import type { DataIndex, DataManifest, DirectoryEntry } from "./manifest.ts"; +import { + benchmarkFile, + CHART_DATA_FILE, + dataUrl, + INDEX_FILE, + MANIFEST_FILE, + manifestShardPath, + packageCountFile, + processCountFile, + VERSIONS_FILE, +} from "./paths.ts"; + +export type FetchLike = (url: string) => Promise<{ + ok: boolean; + status: number; + json: () => Promise; +}>; + +export interface DataClientOptions { + /** Base URL of the served tree, e.g. `https://benchmarks.vlt.sh`. */ + baseUrl: string; + /** Injectable for tests; defaults to global fetch. */ + fetch?: FetchLike; +} + +export class DataFetchError extends Error { + readonly url: string; + readonly status: number; + + constructor(url: string, status: number) { + super(`Request for ${url} failed with status ${status}`); + this.name = "DataFetchError"; + this.url = url; + this.status = status; + } +} + +export class DataClient { + readonly baseUrl: string; + readonly #fetch: FetchLike; + + constructor(options: DataClientOptions) { + this.baseUrl = options.baseUrl; + this.#fetch = + options.fetch ?? ((url: string) => fetch(url) as unknown as ReturnType); + } + + url(...segments: string[]): string { + return dataUrl(this.baseUrl, ...segments); + } + + async #get(...segments: string[]): Promise { + const url = this.url(...segments); + const response = await this.#fetch(url); + if (!response.ok) throw new DataFetchError(url, response.status); + return (await response.json()) as T; + } + + /** The light index: which dates exist, and the observed vocabularies. */ + getIndex(): Promise { + return this.#get(INDEX_FILE); + } + + /** + * The full per-directory detail: ~842KB over the real tree. + * + * Prefer {@link getDirectoryEntry} when one directory is wanted. Reach for + * this only for genuinely cross-date work. + */ + getManifest(): Promise { + return this.#get(MANIFEST_FILE); + } + + /** + * One directory's entry, without fetching the whole manifest. + * + * `name` is a `yyyy-mm-dd` date or `latest`. The value is byte-for-byte the + * same object as `getManifest().entries[name]`, so switching between the two + * changes cost and nothing else. + */ + getDirectoryEntry(name: string): Promise { + return this.#get(...manifestShardPath(name)); + } + + getBenchmark( + date: string, + fixture: Fixture, + variation: Variation | string, + ): Promise { + return this.#get(date, benchmarkFile(fixture, variation)); + } + + getPackageCount( + date: string, + fixture: Fixture, + variation: Variation | string, + ): Promise { + return this.#get(date, packageCountFile(fixture, variation)); + } + + /** + * Note that `registry-*` variations publish no process-count file at all, so + * this rejects with a 404 for them rather than returning an empty object. + * Check the manifest's `processCounts` before calling. + */ + getProcessCount( + date: string, + fixture: Fixture, + variation: Variation | string, + ): Promise { + return this.#get(date, processCountFile(fixture, variation)); + } + + getChartData(date: string): Promise { + return this.#get(date, CHART_DATA_FILE); + } + + /** `latest/` carries no `versions.json`; this rejects with a 404 there. */ + getVersions(date: string): Promise { + return this.#get(date, VERSIONS_FILE); + } +} diff --git a/apps/data/src/index.ts b/apps/data/src/index.ts new file mode 100644 index 0000000000..e0da4af58d --- /dev/null +++ b/apps/data/src/index.ts @@ -0,0 +1,28 @@ +export { DataClient, DataFetchError, type DataClientOptions, type FetchLike } from "./client.ts"; +export { + buildDirectoryEntry, + buildManifest, + INDEX_VERSION, + type DataIndex, + type DataManifest, + type DirectoryEntry, + type DirectoryListing, + type VariationsByFixture, +} from "./manifest.ts"; +export { + benchmarkFile, + CHART_DATA_FILE, + dataUrl, + INDEX_FILE, + LATEST_DIR, + MANIFEST_DIR, + MANIFEST_FILE, + manifestShardPath, + packageCountFile, + parseDataFile, + processCountFile, + VERSIONS_FILE, + type FileKind, + type ParsedDataFile, +} from "./paths.ts"; +export { scanRoots } from "./scan.ts"; diff --git a/apps/data/src/manifest.ts b/apps/data/src/manifest.ts new file mode 100644 index 0000000000..83f33451e1 --- /dev/null +++ b/apps/data/src/manifest.ts @@ -0,0 +1,164 @@ +/** + * The generated index and manifest. + * + * Both are built from what each directory ACTUALLY contains. That is not + * pedantry: `chart-data.json` has six distinct shapes across the 352 published + * date directories and five of its seven top-level fields are optional, so an + * index derived from an assumed schema would misreport most of the archive. + * See packages/types for the survey. + */ +import { isDateKey } from "@benchmarks/types"; +import { CHART_DATA_FILE, LATEST_DIR, parseDataFile, VERSIONS_FILE } from "./paths.ts"; + +export const INDEX_VERSION = 1; + +/** Fixture name to the variations present for it, sorted. */ +export type VariationsByFixture = { [fixture: string]: string[] | undefined }; + +/** What one directory of the published tree contains. */ +export interface DirectoryEntry { + /** `yyyy-mm-dd`, or `latest`. */ + name: string; + /** True for `latest`, which is merged rather than replaced and is not a date. */ + isLatest: boolean; + /** Total files seen in the directory, including ones not classified below. */ + fileCount: number; + benchmarks: VariationsByFixture; + packageCounts: VariationsByFixture; + processCounts: VariationsByFixture; + hasChartData: boolean; + /** `latest/` carries no `versions.json`, so this is genuinely per-directory. */ + hasVersions: boolean; +} + +/** + * The light index, published at the root of the served tree. + * + * Deliberately small: navigation only. Pair-level detail lives in the + * manifest so a consumer that only needs "which dates exist" does not pay for + * the whole archive. + */ +export interface DataIndex { + version: number; + generatedAt: string; + /** Newest date directory, or null when the tree has none. */ + latest: string | null; + /** True when a `latest/` directory is present alongside the dates. */ + hasLatestDir: boolean; + /** Date directory names, oldest first. */ + dates: string[]; + /** Every fixture observed anywhere in the tree, sorted. */ + fixtures: string[]; + /** Every variation observed anywhere in the tree, sorted. */ + variations: string[]; +} + +/** The full detail, published alongside the index. */ +export interface DataManifest { + version: number; + generatedAt: string; + latest: string | null; + /** Keyed by directory name, so `latest` sits beside the dates. */ + entries: { [name: string]: DirectoryEntry | undefined }; +} + +/** One directory's raw listing, as a scanner produces it. */ +export interface DirectoryListing { + name: string; + files: string[]; +} + +const addPair = (into: VariationsByFixture, fixture: string, variation: string): void => { + const list = into[fixture]; + if (list === undefined) into[fixture] = [variation]; + else if (!list.includes(variation)) list.push(variation); +}; + +const sortLists = (byFixture: VariationsByFixture): void => { + for (const list of Object.values(byFixture)) list?.sort(); +}; + +/** Classify one directory listing. Pure -- no filesystem, no network. */ +export const buildDirectoryEntry = (listing: DirectoryListing): DirectoryEntry => { + const entry: DirectoryEntry = { + name: listing.name, + isLatest: listing.name === LATEST_DIR, + fileCount: listing.files.length, + benchmarks: {}, + packageCounts: {}, + processCounts: {}, + hasChartData: listing.files.includes(CHART_DATA_FILE), + hasVersions: listing.files.includes(VERSIONS_FILE), + }; + + for (const file of listing.files) { + if (file === CHART_DATA_FILE || file === VERSIONS_FILE) continue; + const parsed = parseDataFile(file); + if (parsed === undefined) continue; + const target = + parsed.kind === "benchmark" + ? entry.benchmarks + : parsed.kind === "package-count" + ? entry.packageCounts + : entry.processCounts; + addPair(target, parsed.fixture, parsed.variation); + } + + sortLists(entry.benchmarks); + sortLists(entry.packageCounts); + sortLists(entry.processCounts); + return entry; +}; + +/** + * Build the index and manifest from directory listings. + * + * Listings that are neither a `yyyy-mm-dd` date nor `latest` are ignored. That + * matters: the publish root also holds `assets/`, `data/`, `old/`, `raw/` and + * the raw `results--/` directories every deploy publishes + * alongside the processed data (roughly 85 of them). + */ +export const buildManifest = ( + listings: DirectoryListing[], + generatedAt: string, +): { index: DataIndex; manifest: DataManifest } => { + const entries: { [name: string]: DirectoryEntry | undefined } = {}; + const dates: string[] = []; + const fixtures = new Set(); + const variations = new Set(); + let hasLatestDir = false; + + for (const listing of listings) { + const isDate = isDateKey(listing.name); + const isLatest = listing.name === LATEST_DIR; + if (!isDate && !isLatest) continue; + + const entry = buildDirectoryEntry(listing); + entries[listing.name] = entry; + if (isDate) dates.push(listing.name); + if (isLatest) hasLatestDir = true; + + for (const byFixture of [entry.benchmarks, entry.packageCounts, entry.processCounts]) { + for (const [fixture, list] of Object.entries(byFixture)) { + fixtures.add(fixture); + for (const variation of list ?? []) variations.add(variation); + } + } + } + + dates.sort(); + const latest = dates.at(-1) ?? null; + + return { + index: { + version: INDEX_VERSION, + generatedAt, + latest, + hasLatestDir, + dates, + fixtures: [...fixtures].sort(), + variations: [...variations].sort(), + }, + manifest: { version: INDEX_VERSION, generatedAt, latest, entries }, + }; +}; diff --git a/apps/data/src/paths.ts b/apps/data/src/paths.ts new file mode 100644 index 0000000000..8a04263253 --- /dev/null +++ b/apps/data/src/paths.ts @@ -0,0 +1,113 @@ +/** + * Filename conventions of the published tree. + * + * The tree is frozen (HARD CONSTRAINT 3), so these describe what is already + * there. Nothing here invents a path. + */ +import { type Fixture, FIXTURES, type Variation } from "@benchmarks/types"; + +export const CHART_DATA_FILE = "chart-data.json"; +export const VERSIONS_FILE = "versions.json"; +export const LATEST_DIR = "latest"; + +/** Names this package publishes at the root of the served tree. */ +export const INDEX_FILE = "index.json"; +export const MANIFEST_FILE = "manifest.json"; + +/** + * Directory holding one shard per published directory. + * + * `manifest.json` answers cross-date questions and `index.json` answers "which + * dates exist", but neither answers "what is in THIS date" without pulling the + * whole archive. The shards close that hole. Measured over the real tree: + * index + one shard is 12,093 bytes against 849,397 for index + manifest. + */ +export const MANIFEST_DIR = "manifest"; + +/** + * Path of one directory's shard, relative to the tree root. + * + * `name` is a `yyyy-mm-dd` date or `latest`. Deliberately a sibling directory + * rather than a file inside each date directory: decision D2 forbids + * re-emitting the published date directories, and this keeps every new file in + * one new subtree. + */ +export const manifestShardPath = (name: string): string[] => [MANIFEST_DIR, `${name}.json`]; + +const PACKAGE_COUNT_SUFFIX = "-package-count.json"; +const PROCESS_COUNT_SUFFIX = "-process-count.json"; +const JSON_SUFFIX = ".json"; + +/** What a filename in a date directory refers to. */ +export type FileKind = "benchmark" | "package-count" | "process-count"; + +export interface ParsedDataFile { + kind: FileKind; + fixture: Fixture; + /** + * The variation, as a plain string rather than {@link Variation}. + * + * The published tree can hold variations this checkout's suite does not. + * `ci+setup` was added upstream on 2026-09-30 and published the same day, + * while this workspace was on an earlier base. A discovery layer that + * validated against the union would under-report what is actually there. + */ + variation: string; +} + +const FIXTURE_SET: ReadonlySet = new Set(FIXTURES); + +/** + * Classify a filename from a date directory, or return undefined if it is not + * one of the three per-fixture kinds. + * + * Splitting on the first `-` is safe because no fixture name contains one, + * while several variations do (`build-cache`, `registry-clean`). + */ +export const parseDataFile = (name: string): ParsedDataFile | undefined => { + let kind: FileKind; + let stem: string; + + if (name.endsWith(PACKAGE_COUNT_SUFFIX)) { + kind = "package-count"; + stem = name.slice(0, -PACKAGE_COUNT_SUFFIX.length); + } else if (name.endsWith(PROCESS_COUNT_SUFFIX)) { + kind = "process-count"; + stem = name.slice(0, -PROCESS_COUNT_SUFFIX.length); + } else if (name.endsWith(JSON_SUFFIX)) { + kind = "benchmark"; + stem = name.slice(0, -JSON_SUFFIX.length); + } else { + return undefined; + } + + const split = stem.indexOf("-"); + if (split <= 0) return undefined; + + const fixture = stem.slice(0, split); + const variation = stem.slice(split + 1); + if (!FIXTURE_SET.has(fixture) || variation.length === 0) return undefined; + + return { kind, fixture: fixture as Fixture, variation }; +}; + +export const benchmarkFile = (fixture: Fixture, variation: Variation | string): string => + `${fixture}-${variation}.json`; + +export const packageCountFile = (fixture: Fixture, variation: Variation | string): string => + `${fixture}-${variation}${PACKAGE_COUNT_SUFFIX}`; + +export const processCountFile = (fixture: Fixture, variation: Variation | string): string => + `${fixture}-${variation}${PROCESS_COUNT_SUFFIX}`; + +/** + * Join path segments into a URL relative to a base, encoding each segment. + * + * Several variations contain `+`. It is legal literally in a path segment, but + * it is encoded here so the URL is unambiguous for any consumer that treats + * the value as form-encoded. + */ +export const dataUrl = (baseUrl: string, ...segments: string[]): string => { + const base = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`; + return base + segments.map((s) => encodeURIComponent(s)).join("/"); +}; diff --git a/apps/data/src/scan.ts b/apps/data/src/scan.ts new file mode 100644 index 0000000000..24af88cd57 --- /dev/null +++ b/apps/data/src/scan.ts @@ -0,0 +1,39 @@ +/** Read directory listings off disk, for the manifest builder to classify. */ +import { readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; +import type { DirectoryListing } from "./manifest.ts"; + +/** + * List the immediate subdirectories of each root and the files in them. + * + * Later roots win on name collisions, so a caller can layer this run's freshly + * assembled output over a checkout of the published history. + */ +export const scanRoots = (roots: string[]): DirectoryListing[] => { + const byName = new Map(); + + for (const root of roots) { + let names: string[]; + try { + names = readdirSync(root); + } catch { + // A missing root is not an error: the reprocess workflow runs without a + // freshly assembled tree, and local dev often has only one of the two. + continue; + } + + for (const name of names.sort()) { + const dir = join(root, name); + let files: string[]; + try { + if (!statSync(dir).isDirectory()) continue; + files = readdirSync(dir).sort(); + } catch { + continue; + } + byName.set(name, { name, files }); + } + } + + return [...byName.values()].sort((a, b) => a.name.localeCompare(b.name)); +}; diff --git a/apps/data/src/serve.ts b/apps/data/src/serve.ts new file mode 100644 index 0000000000..ccc0b69889 --- /dev/null +++ b/apps/data/src/serve.ts @@ -0,0 +1,73 @@ +/** + * Minimal static server for the assembled tree. + * + * Deliberately dependency-free (`node:http` only) and deliberately does not + * serve a directory index. The published root's `index.html` IS the live site + * at benchmarks.vlt.sh, and this app must never be the thing that puts an + * index page there. + */ +import { createServer, type Server } from "node:http"; +import { createReadStream, statSync } from "node:fs"; +import { join, normalize, resolve, sep } from "node:path"; + +const CONTENT_TYPES: { [extension: string]: string | undefined } = { + ".json": "application/json; charset=utf-8", + ".txt": "text/plain; charset=utf-8", + ".log": "text/plain; charset=utf-8", +}; + +export interface ServeOptions { + root: string; + port: number; + host?: string; +} + +/** Resolve a request path to a file inside root, or undefined if it escapes. */ +export const resolveRequestPath = (root: string, urlPath: string): string | undefined => { + let decoded: string; + try { + decoded = decodeURIComponent(urlPath); + } catch { + return undefined; + } + if (decoded.includes("\0")) return undefined; + + const rootAbs = resolve(root); + const candidate = resolve(join(rootAbs, normalize(decoded))); + if (candidate !== rootAbs && !candidate.startsWith(rootAbs + sep)) return undefined; + return candidate; +}; + +export const createDataServer = (options: ServeOptions): Server => + createServer((request, response) => { + const urlPath = new URL(request.url ?? "/", "http://localhost").pathname; + const filePath = resolveRequestPath(options.root, urlPath); + + if (filePath === undefined) { + response.writeHead(400, { "content-type": "text/plain" }); + response.end("bad request\n"); + return; + } + + let isFile = false; + try { + isFile = statSync(filePath).isFile(); + } catch { + isFile = false; + } + + if (!isFile) { + // No directory listing and no index fallback, on purpose. + response.writeHead(404, { "content-type": "text/plain" }); + response.end("not found\n"); + return; + } + + const dot = filePath.lastIndexOf("."); + const extension = dot === -1 ? "" : filePath.slice(dot); + response.writeHead(200, { + "content-type": CONTENT_TYPES[extension] ?? "application/octet-stream", + "access-control-allow-origin": "*", + }); + createReadStream(filePath).pipe(response); + }); diff --git a/apps/data/test/client.test.ts b/apps/data/test/client.test.ts new file mode 100644 index 0000000000..a0d1d75a40 --- /dev/null +++ b/apps/data/test/client.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it, vi } from "vitest"; +import { DataClient, DataFetchError, type FetchLike } from "../src/client.ts"; + +const stub = (body: unknown, ok = true, status = 200): { fetch: FetchLike; urls: string[] } => { + const urls: string[] = []; + const fetch: FetchLike = (url) => { + urls.push(url); + return Promise.resolve({ ok, status, json: () => Promise.resolve(body) }); + }; + return { fetch, urls }; +}; + +describe("DataClient url construction", () => { + it("requests the index and manifest from the tree root", async () => { + const { fetch, urls } = stub({}); + const client = new DataClient({ baseUrl: "https://x.test", fetch }); + await client.getIndex(); + await client.getManifest(); + expect(urls).toEqual(["https://x.test/index.json", "https://x.test/manifest.json"]); + }); + + it("builds the three per-fixture paths correctly", async () => { + const { fetch, urls } = stub({}); + const client = new DataClient({ baseUrl: "https://x.test", fetch }); + await client.getBenchmark("2026-09-30", "next", "clean"); + await client.getPackageCount("2026-09-30", "astro", "registry-clean"); + await client.getProcessCount("2026-09-30", "vue", "cache"); + expect(urls).toEqual([ + "https://x.test/2026-09-30/next-clean.json", + "https://x.test/2026-09-30/astro-registry-clean-package-count.json", + "https://x.test/2026-09-30/vue-cache-process-count.json", + ]); + }); + + it("encodes + in variation names", async () => { + const { fetch, urls } = stub({}); + const client = new DataClient({ baseUrl: "https://x.test", fetch }); + await client.getBenchmark("2026-09-30", "astro", "cache+lockfile+node_modules"); + expect(urls[0]).toBe("https://x.test/2026-09-30/astro-cache%2Blockfile%2Bnode_modules.json"); + }); + + it("fetches one directory entry from its shard, not the whole manifest", async () => { + const { fetch, urls } = stub({}); + const client = new DataClient({ baseUrl: "https://x.test", fetch }); + await client.getDirectoryEntry("2026-09-30"); + await client.getDirectoryEntry("latest"); + expect(urls).toEqual([ + "https://x.test/manifest/2026-09-30.json", + "https://x.test/manifest/latest.json", + ]); + // The point of the method: it must not reach for manifest.json. + expect(urls.some((u) => u.endsWith("/manifest.json"))).toBe(false); + }); + + it("reaches chart-data and versions per date", async () => { + const { fetch, urls } = stub({}); + const client = new DataClient({ baseUrl: "https://x.test/", fetch }); + await client.getChartData("2026-09-30"); + await client.getVersions("2026-09-30"); + expect(urls).toEqual([ + "https://x.test/2026-09-30/chart-data.json", + "https://x.test/2026-09-30/versions.json", + ]); + }); +}); + +describe("DataClient errors", () => { + it("throws DataFetchError carrying the url and status", async () => { + const { fetch } = stub(undefined, false, 404); + const client = new DataClient({ baseUrl: "https://x.test", fetch }); + await expect(client.getVersions("latest")).rejects.toBeInstanceOf(DataFetchError); + await expect(client.getVersions("latest")).rejects.toMatchObject({ + status: 404, + url: "https://x.test/latest/versions.json", + }); + }); + + it("does not swallow a 404 into an empty object", async () => { + // registry-* variations have no process-count file. Callers must be able + // to tell "absent" from "present but empty". + const { fetch } = stub({}, false, 404); + const client = new DataClient({ baseUrl: "https://x.test", fetch }); + const promise = client.getProcessCount("2026-09-30", "astro", "registry-clean"); + await expect(promise).rejects.toThrow(/failed with status 404/); + }); + + it("uses global fetch when none is injected", () => { + const spy = vi.fn(); + const client = new DataClient({ baseUrl: "https://x.test" }); + expect(client.url("index.json")).toBe("https://x.test/index.json"); + expect(spy).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/data/test/manifest.test.ts b/apps/data/test/manifest.test.ts new file mode 100644 index 0000000000..a49bbe0b19 --- /dev/null +++ b/apps/data/test/manifest.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from "vitest"; +import { buildDirectoryEntry, buildManifest, INDEX_VERSION } from "../src/manifest.ts"; + +const AT = "2026-09-30T00:00:00.000Z"; + +describe("buildDirectoryEntry", () => { + it("groups variations by fixture per kind", () => { + const entry = buildDirectoryEntry({ + name: "2026-09-30", + files: [ + "astro-clean.json", + "astro-clean-package-count.json", + "astro-clean-process-count.json", + "astro-cache+lockfile.json", + "next-clean.json", + "chart-data.json", + "versions.json", + ], + }); + expect(entry.benchmarks).toEqual({ + astro: ["cache+lockfile", "clean"], + next: ["clean"], + }); + expect(entry.packageCounts).toEqual({ astro: ["clean"] }); + expect(entry.processCounts).toEqual({ astro: ["clean"] }); + expect(entry.hasChartData).toBe(true); + expect(entry.hasVersions).toBe(true); + expect(entry.isLatest).toBe(false); + expect(entry.fileCount).toBe(7); + }); + + it("records a registry variation as having no process-count", () => { + // The published tree has 0 registry-* process-count files. The manifest + // must reflect that rather than implying the triple always exists. + const entry = buildDirectoryEntry({ + name: "2026-09-30", + files: ["astro-registry-clean.json", "astro-registry-clean-package-count.json"], + }); + expect(entry.benchmarks).toEqual({ astro: ["registry-clean"] }); + expect(entry.packageCounts).toEqual({ astro: ["registry-clean"] }); + expect(entry.processCounts).toEqual({}); + }); + + it("marks latest/ and reports its missing versions.json", () => { + const entry = buildDirectoryEntry({ + name: "latest", + files: ["next-clean.json", "chart-data.json"], + }); + expect(entry.isLatest).toBe(true); + expect(entry.hasVersions).toBe(false); + expect(entry.hasChartData).toBe(true); + }); + + it("counts unclassifiable files but does not index them", () => { + const entry = buildDirectoryEntry({ + name: "2026-09-30", + files: ["next-clean.json", "npm-output-0.log", "README.md"], + }); + expect(entry.fileCount).toBe(3); + expect(entry.benchmarks).toEqual({ next: ["clean"] }); + }); +}); + +describe("buildManifest", () => { + it("ignores everything at the publish root that is not a date or latest", () => { + const { index, manifest } = buildManifest( + [ + { name: "2025-08-22", files: ["next-clean.json"] }, + { name: "2026-09-30", files: ["astro-clean.json", "chart-data.json"] }, + { name: "latest", files: ["astro-clean.json"] }, + // All of these really are at the gh-pages root. + { name: "assets", files: ["index-abc.js"] }, + { name: "data", files: ["x.json"] }, + { name: "old", files: [] }, + { name: "raw", files: [] }, + { name: "results-astro-clean", files: ["benchmarks.json"] }, + { name: "results-astro-ci+setup", files: ["benchmarks.json"] }, + ], + AT, + ); + expect(index.dates).toEqual(["2025-08-22", "2026-09-30"]); + expect(index.hasLatestDir).toBe(true); + expect(Object.keys(manifest.entries).sort()).toEqual(["2025-08-22", "2026-09-30", "latest"]); + }); + + it("reports latest as the newest DATE, never the latest/ directory", () => { + const { index } = buildManifest( + [ + { name: "2026-01-02", files: [] }, + { name: "latest", files: [] }, + { name: "2025-12-31", files: [] }, + ], + AT, + ); + expect(index.latest).toBe("2026-01-02"); + expect(index.dates).toEqual(["2025-12-31", "2026-01-02"]); + }); + + it("collects the observed vocabularies rather than assuming them", () => { + const { index } = buildManifest( + [{ name: "2026-09-30", files: ["astro-ci+setup.json", "run-run.json"] }], + AT, + ); + expect(index.fixtures).toEqual(["astro", "run"]); + expect(index.variations).toEqual(["ci+setup", "run"]); + }); + + it("handles an empty tree without inventing a latest", () => { + const { index, manifest } = buildManifest([], AT); + expect(index.latest).toBeNull(); + expect(index.dates).toEqual([]); + expect(index.hasLatestDir).toBe(false); + expect(manifest.entries).toEqual({}); + expect(index.version).toBe(INDEX_VERSION); + }); +}); diff --git a/apps/data/test/paths.test.ts b/apps/data/test/paths.test.ts new file mode 100644 index 0000000000..e16a334f91 --- /dev/null +++ b/apps/data/test/paths.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from "vitest"; +import { dataUrl, manifestShardPath, parseDataFile } from "../src/paths.ts"; + +describe("parseDataFile", () => { + it("splits a plain variation", () => { + expect(parseDataFile("next-clean.json")).toEqual({ + kind: "benchmark", + fixture: "next", + variation: "clean", + }); + }); + + it("keeps hyphens that belong to the variation", () => { + // The reason splitting on the FIRST hyphen is correct: no fixture name + // contains one, but several variations do. + expect(parseDataFile("astro-registry-clean.json")).toEqual({ + kind: "benchmark", + fixture: "astro", + variation: "registry-clean", + }); + expect(parseDataFile("next-build-cache.json")).toEqual({ + kind: "benchmark", + fixture: "next", + variation: "build-cache", + }); + }); + + it("handles variations containing +", () => { + expect(parseDataFile("astro-cache+lockfile+node_modules.json")).toEqual({ + kind: "benchmark", + fixture: "astro", + variation: "cache+lockfile+node_modules", + }); + }); + + it("distinguishes the three kinds, including on a registry variation", () => { + expect(parseDataFile("astro-registry-clean-package-count.json")).toEqual({ + kind: "package-count", + fixture: "astro", + variation: "registry-clean", + }); + expect(parseDataFile("vue-cache+lockfile-process-count.json")).toEqual({ + kind: "process-count", + fixture: "vue", + variation: "cache+lockfile", + }); + }); + + it("accepts a variation newer than this checkout's suite", () => { + // `ci+setup` was added upstream on 2026-09-30 and published the same day, + // ahead of this workspace's base at the time. Rejecting such a variation + // would under-report the tree. + expect(parseDataFile("astro-ci+setup.json")).toEqual({ + kind: "benchmark", + fixture: "astro", + variation: "ci+setup", + }); + }); + + it("rejects everything that is not a per-fixture data file", () => { + for (const name of [ + "chart-data.json", + "versions.json", + "index.json", + "manifest.json", + ".nojekyll", + "CNAME", + "index.html", + "astro.json", + "-clean.json", + "unknownfixture-clean.json", + "astro-.json", + "astro-clean.txt", + ]) { + expect(parseDataFile(name), name).toBeUndefined(); + } + }); +}); + +describe("dataUrl", () => { + it("joins segments onto a base with or without a trailing slash", () => { + expect(dataUrl("https://x.test", "2026-09-30", "next-clean.json")).toBe( + "https://x.test/2026-09-30/next-clean.json", + ); + expect(dataUrl("https://x.test/", "index.json")).toBe("https://x.test/index.json"); + }); + + it("encodes + in a variation so the URL is unambiguous", () => { + expect(dataUrl("https://x.test", "2026-09-30", "astro-cache+lockfile.json")).toBe( + "https://x.test/2026-09-30/astro-cache%2Blockfile.json", + ); + }); +}); + +describe("manifestShardPath", () => { + it("puts shards in a sibling directory, not inside the date directory", () => { + // Decision D2 forbids re-emitting the published date directories, so the + // shard must not land inside one. + expect(manifestShardPath("2026-09-30")).toEqual(["manifest", "2026-09-30.json"]); + expect(manifestShardPath("2026-09-30")[0]).not.toBe("2026-09-30"); + }); + + it("shards latest/ alongside the dates", () => { + expect(manifestShardPath("latest")).toEqual(["manifest", "latest.json"]); + }); + + it("builds a fetchable url", () => { + expect(dataUrl("https://x.test", ...manifestShardPath("2026-09-30"))).toBe( + "https://x.test/manifest/2026-09-30.json", + ); + }); +}); diff --git a/apps/data/test/served-tree.test.ts b/apps/data/test/served-tree.test.ts new file mode 100644 index 0000000000..df633fe73c --- /dev/null +++ b/apps/data/test/served-tree.test.ts @@ -0,0 +1,246 @@ +/** + * End-to-end over REAL published data: materialize a subset of gh-pages, build + * the manifest, serve it, and fetch through the client. + * + * Reads the branch through `git show` -- no network. Skips when the ref is + * absent so a shallow clone does not fail; the unit tests always run. + */ +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AddressInfo } from "node:net"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { DataClient, DataFetchError } from "../src/client.ts"; +import type { DataIndex, DataManifest } from "../src/manifest.ts"; +import { createDataServer } from "../src/serve.ts"; + +const REF = "origin/gh-pages"; + +/** + * Every git call runs from the repository root. `git ls-tree` resolves paths + * against the current prefix, and vitest runs from the package directory -- a + * trap that silently returned an empty listing at exit 0 in packages/types. + */ +const REPO_ROOT = execFileSync("git", ["rev-parse", "--show-toplevel"], { + encoding: "utf8", +}).trim(); + +const git = (...args: string[]): string => + execFileSync("git", args, { + cwd: REPO_ROOT, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + +const refExists = (): boolean => { + try { + git("rev-parse", "--verify", `${REF}^{commit}`); + return true; + } catch { + return false; + } +}; + +// One current date, the oldest date, and latest/ -- the three directories with +// materially different contents. +const DIRS = ["2026-09-30", "2025-08-22", "latest"]; + +let root = ""; +let baseUrl = ""; +let server: ReturnType | undefined; +let index: DataIndex; +let manifest: DataManifest; + +const skip = !refExists(); + +describe.skipIf(skip)("served tree, built from real gh-pages data", () => { + beforeAll(async () => { + root = mkdtempSync(join(tmpdir(), "bench-data-")); + for (const dir of DIRS) { + mkdirSync(join(root, dir), { recursive: true }); + for (const name of git("ls-tree", "--name-only", `${REF}:${dir}`) + .split("\n") + .filter(Boolean)) { + writeFileSync(join(root, dir, name), git("show", `${REF}:${dir}/${name}`)); + } + } + // Also reproduce the junk the publish root really carries, to prove the + // scanner ignores it. + mkdirSync(join(root, "assets"), { recursive: true }); + writeFileSync(join(root, "assets", "index-abc.js"), "// ui asset\n"); + mkdirSync(join(root, "results-astro-clean"), { recursive: true }); + writeFileSync(join(root, "results-astro-clean", "benchmarks.json"), "{}\n"); + writeFileSync(join(root, "index.html"), "live site\n"); + writeFileSync(join(root, "CNAME"), "benchmarks.vlt.sh\n"); + + // Run the REAL CLI rather than reimplementing what it does. An earlier + // version of this fixture built the index and shards inline, which meant a + // change to the CLI could not fail any test -- found by injecting a + // timestamp into the CLI's shards and watching the suite still pass. + execFileSync( + process.execPath, + [join(import.meta.dirname, "..", "bin", "build-manifest.mts"), "--out", root, root], + { encoding: "utf8" }, + ); + index = JSON.parse(readFileSync(join(root, "index.json"), "utf8")) as DataIndex; + manifest = JSON.parse(readFileSync(join(root, "manifest.json"), "utf8")) as DataManifest; + + server = createDataServer({ root, port: 0 }); + await new Promise((done) => server?.listen(0, "127.0.0.1", done)); + const address = server?.address() as AddressInfo; + baseUrl = `http://127.0.0.1:${address.port}`; + }); + + afterAll(() => { + server?.close(); + if (root) rmSync(root, { recursive: true, force: true }); + }); + + const client = (): DataClient => new DataClient({ baseUrl }); + + it("indexes the real dates and ignores the publish-root junk", async () => { + const served = await client().getIndex(); + expect(served.dates).toEqual(["2025-08-22", "2026-09-30"]); + expect(served.latest).toBe("2026-09-30"); + expect(served.hasLatestDir).toBe(true); + expect(Object.keys(served).sort()).toEqual([ + "dates", + "fixtures", + "generatedAt", + "hasLatestDir", + "latest", + "variations", + "version", + ]); + const entries = (await client().getManifest()).entries; + expect(Object.keys(entries).sort()).toEqual(["2025-08-22", "2026-09-30", "latest"]); + }); + + it("serves a shard equal to the manifest entry, for every published directory", async () => { + // The invariant that makes the two interchangeable: a consumer switching + // from getManifest() to getDirectoryEntry() changes cost and nothing else. + const served = await client().getManifest(); + for (const name of Object.keys(served.entries)) { + const shard = await client().getDirectoryEntry(name); + expect(shard, name).toEqual(served.entries[name]); + } + // Includes the oldest date, the newest, one with a '+' variation, and latest/. + expect(Object.keys(served.entries).sort()).toEqual(["2025-08-22", "2026-09-30", "latest"]); + const newest = await client().getDirectoryEntry("2026-09-30"); + expect(newest.benchmarks.astro).toContain("cache+lockfile+node_modules"); + const oldest = await client().getDirectoryEntry("2025-08-22"); + expect(oldest.hasChartData).toBe(true); + const latest = await client().getDirectoryEntry("latest"); + expect(latest.isLatest).toBe(true); + expect(latest.hasVersions).toBe(false); + }); + + it("carries no timestamp in a shard, so unchanged dates stay byte-identical", () => { + // If a shard embedded generatedAt, all 353 would change every nightly run + // and the gh-pages branch would grow by 353 blobs a day. + for (const entry of Object.values(manifest.entries)) { + const keys = Object.keys(entry ?? {}); + expect(keys).not.toContain("generatedAt"); + expect(keys.some((k) => /at$|time/i.test(k))).toBe(false); + } + }); + + it("a shard costs roughly one entry, not the whole archive", async () => { + // Scale-independent: a shard must be about the size of its own entry, so + // the saving grows with the number of published directories. This fixture + // has three, so no fixed ratio is assertable here -- the real-tree figure + // (842,509 bytes against a 5,205-byte shard, 161.9:1) is in the evidence. + const shardText = JSON.stringify(await client().getDirectoryEntry("2026-09-30")); + const fullText = JSON.stringify(await client().getManifest()); + const entryCount = Object.keys(manifest.entries).length; + expect(shardText.length).toBeLessThan(fullText.length); + // Within a small constant of the manifest's per-entry share. + expect(shardText.length).toBeLessThan((fullText.length / entryCount) * 3); + }); + + it("does not publish an index.html of its own", () => { + // The gh-pages root index.html IS the live site. This app writes only the + // two JSON files; the one in the fixture is the pre-existing site file. + const written = readFileSync(join(root, "index.html"), "utf8"); + expect(written).toContain("live site"); + expect(index).not.toHaveProperty("html"); + }); + + it("serves a + variation through the client, end to end", async () => { + const file = await client().getBenchmark("2026-09-30", "astro", "cache+lockfile+node_modules"); + expect(file.results.length).toBeGreaterThan(0); + const first = file.results[0]; + expect(typeof first?.command).toBe("string"); + expect(typeof first?.median).toBe("number"); + expect(first?.exit_codes).toHaveLength(first?.times.length ?? -1); + }); + + it("serves a registry-* benchmark and its REGISTRY-keyed package-count", async () => { + const file = await client().getBenchmark("2026-09-30", "astro", "registry-clean"); + expect(file.results.length).toBeGreaterThan(0); + const counts = await client().getPackageCount("2026-09-30", "astro", "registry-clean"); + expect(Object.keys(counts).sort()).toEqual([ + "aws", + "cloudsmith", + "github", + "jfrog", + "npm", + "vlt", + ]); + }); + + it("reports registry-* variations as having no process-count, and 404s for one", async () => { + const entry = manifest.entries["2026-09-30"]; + expect(entry?.benchmarks.astro).toContain("registry-clean"); + expect(entry?.processCounts.astro ?? []).not.toContain("registry-clean"); + await expect( + client().getProcessCount("2026-09-30", "astro", "registry-clean"), + ).rejects.toBeInstanceOf(DataFetchError); + }); + + it("distinguishes package-counts with minCount/maxCount from those without", async () => { + const withRange = await client().getPackageCount("2026-09-30", "astro", "cache+lockfile"); + const withoutRange = await client().getPackageCount("2026-09-30", "astro", "clean"); + expect(Object.values(withRange).some((e) => e?.minCount !== undefined)).toBe(true); + expect(Object.values(withoutRange).every((e) => e?.minCount === undefined)).toBe(true); + }); + + it("records that latest/ has no versions.json, and 404s for it", async () => { + expect(manifest.entries["latest"]?.hasVersions).toBe(false); + expect(manifest.entries["2026-09-30"]?.hasVersions).toBe(true); + await expect(client().getVersions("latest")).rejects.toMatchObject({ status: 404 }); + const versions = await client().getVersions("2026-09-30"); + expect(Object.keys(versions).length).toBeGreaterThan(0); + }); + + it("serves the newest chart-data with all seven keys", async () => { + const chart = await client().getChartData("2026-09-30"); + expect(Object.keys(chart).sort()).toEqual([ + "chartData", + "commitSha", + "date", + "perPackageCountChartData", + "registryChartData", + "registryPerPackageCountChartData", + "versions", + ]); + }); + + it("serves the OLDEST chart-data, which has only its two required keys", async () => { + const chart = await client().getChartData("2025-08-22"); + expect(Object.keys(chart).sort()).toEqual(["chartData", "perPackageCountChartData"]); + expect(chart.date).toBeUndefined(); + expect(chart.versions).toBeUndefined(); + expect(chart.registryChartData).toBeUndefined(); + }); + + it("serves JSON with a json content-type and 404s outside the tree", async () => { + const response = await fetch(`${baseUrl}/index.json`); + expect(response.headers.get("content-type")).toContain("application/json"); + expect((await fetch(`${baseUrl}/nope.json`)).status).toBe(404); + // No directory listing, and no traversal. + expect((await fetch(`${baseUrl}/2026-09-30/`)).status).toBe(404); + expect((await fetch(`${baseUrl}/../../etc/passwd`)).status).toBeGreaterThanOrEqual(400); + }); +}); diff --git a/apps/data/tsconfig.json b/apps/data/tsconfig.json new file mode 100644 index 0000000000..7b24c2fcc7 --- /dev/null +++ b/apps/data/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + // Node runs this package's TypeScript directly (native type stripping, no + // bundler -- decision D2 forbids re-emitting the published tree). Stripping + // only erases types, so syntax that needs codegen must be rejected here + // rather than at runtime. + "erasableSyntaxOnly": true, + "allowImportingTsExtensions": true, + "types": ["node"] + }, + "include": ["bin", "src", "test"] +} diff --git a/apps/data/vitest.config.mts b/apps/data/vitest.config.mts new file mode 100644 index 0000000000..8b5840acac --- /dev/null +++ b/apps/data/vitest.config.mts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["test/**/*.test.ts"], + }, +}); diff --git a/apps/www/.env.example b/apps/www/.env.example new file mode 100644 index 0000000000..6f3143c2ff --- /dev/null +++ b/apps/www/.env.example @@ -0,0 +1,21 @@ +# Base URL of the served benchmark data tree (@benchmarks/data). +# +# NEXT_PUBLIC_DATA_URL is read at BUILD time. Next constant-folds every +# NEXT_PUBLIC_* read into the bundle, so changing this value requires a +# rebuild -- `next start` will not pick it up. +# +# Local development, after `vlt run --workspace=apps/data build`: +# vlt run --workspace=apps/data start # serves results/ on :8811 +# +# Production is the gh-pages origin, which serves the same tree: +# NEXT_PUBLIC_DATA_URL=https://benchmarks.vlt.sh +NEXT_PUBLIC_DATA_URL=http://localhost:8811 + +# Optional server-side runtime override, checked first when set. +# +# Every fetch in this app happens on the server, so this one is NOT inlined and +# does repoint an already-built app at another origin. Useful for pointing a +# single build at a staging tree, and for verifying the endpoint is switchable +# without rebuilding. Leave it unset for normal use. +# +# DATA_URL=http://localhost:8899 diff --git a/apps/www/.env.local b/apps/www/.env.local new file mode 100644 index 0000000000..6f3143c2ff --- /dev/null +++ b/apps/www/.env.local @@ -0,0 +1,21 @@ +# Base URL of the served benchmark data tree (@benchmarks/data). +# +# NEXT_PUBLIC_DATA_URL is read at BUILD time. Next constant-folds every +# NEXT_PUBLIC_* read into the bundle, so changing this value requires a +# rebuild -- `next start` will not pick it up. +# +# Local development, after `vlt run --workspace=apps/data build`: +# vlt run --workspace=apps/data start # serves results/ on :8811 +# +# Production is the gh-pages origin, which serves the same tree: +# NEXT_PUBLIC_DATA_URL=https://benchmarks.vlt.sh +NEXT_PUBLIC_DATA_URL=http://localhost:8811 + +# Optional server-side runtime override, checked first when set. +# +# Every fetch in this app happens on the server, so this one is NOT inlined and +# does repoint an already-built app at another origin. Useful for pointing a +# single build at a staging tree, and for verifying the endpoint is switchable +# without rebuilding. Leave it unset for normal use. +# +# DATA_URL=http://localhost:8899 diff --git a/apps/www/.gitignore b/apps/www/.gitignore new file mode 100644 index 0000000000..f24d48d91c --- /dev/null +++ b/apps/www/.gitignore @@ -0,0 +1,3 @@ +# Next.js build output and the type shim it generates on first build. +.next/ +next-env.d.ts diff --git a/apps/www/AGENTS.md b/apps/www/AGENTS.md new file mode 100644 index 0000000000..643577dfae --- /dev/null +++ b/apps/www/AGENTS.md @@ -0,0 +1,9 @@ + + +# This is NOT the Next.js you know + +This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` (resolved from this file's directory; in monorepos the `next` package may not be visible from the repo root) before writing any code. Heed deprecation notices. + +This block is written and re-added by `next dev` — verify at `node_modules/next/dist/server/lib/generate-agent-files.js`. Removing it from a diff only re-creates the uncommitted change; committing it with your work keeps the tree clean. + + diff --git a/apps/www/components.json b/apps/www/components.json new file mode 100644 index 0000000000..3289f237ad --- /dev/null +++ b/apps/www/components.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://ui.shadcn.com/schema.json", + "style": "new-york", + "rsc": true, + "tsx": true, + "tailwind": { + "config": "", + "css": "src/app/globals.css", + "baseColor": "neutral", + "cssVariables": true, + "prefix": "" + }, + "aliases": { + "components": "@/components", + "utils": "@/lib/utils", + "ui": "@/components/ui", + "lib": "@/lib", + "hooks": "@/hooks" + }, + "iconLibrary": "lucide" +} diff --git a/apps/www/next.config.ts b/apps/www/next.config.ts new file mode 100644 index 0000000000..21c7683922 --- /dev/null +++ b/apps/www/next.config.ts @@ -0,0 +1,79 @@ +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import createMDX from "@next/mdx"; +import type { NextConfig } from "next"; + +/** + * The workspace root, two levels up from `apps/www`. + * + * Turbopack otherwise infers its filesystem root as this app's own directory + * and refuses to compile anything above it. vlt installs into a content- + * addressed store at the workspace root and links into each member, so `next` + * itself resolves through `apps/www/node_modules/next` to + * `/node_modules/.vlt/...`. Without this the build fails with + * "Could not find the Next.js package". + */ +const workspaceRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); + +const acceptsMarkdown = { + type: "header", + key: "accept", + value: ".*text/markdown.*", +} as const; + +/* + * Extensionless pages, as the docs site has them. Skips the markdown route + * itself, the API, MCP and social-card endpoints, Next's own assets, and anything with a + * file extension (`.md` urls, llms.txt, openapi.json), which has no markdown + * twin. + */ +const pages = "/:path((?!llms\\.mdx/|(?:api|_next|mcp|og)(?:/|$))(?!.*\\.[a-z0-9]+$).+)"; + +const nextConfig: NextConfig = { + // Pages prerender a shell around cached data (lib/data.ts), and links prefetch that shell. + cacheComponents: true, + partialPrefetching: true, + // The About and Methodology pages are MDX (app/*/content.mdx). + pageExtensions: ["ts", "tsx", "mdx"], + // Both workspace packages ship TypeScript source with no build step, so Next + // has to transpile them itself. Without this the build fails resolving + // `@benchmarks/data`'s `.ts` relative imports. + transpilePackages: ["@benchmarks/types", "@benchmarks/data"], + // takumi's native renderer (app/og) is loaded by Node, not bundled -- as vlt.io has it. + serverExternalPackages: ["@takumi-rs/image-response"], + turbopack: { root: workspaceRoot }, + // The docs' rewrites (vltpkg/www/docs/next.config.ts). The query survives a + // rewrite, so `/x.md?fixture=astro` is the markdown of that filtered view. + rewrites: async () => ({ + // Agents that ask for markdown get it at the page's own url; runs before + // the filesystem, which would otherwise match the html page first. + beforeFiles: [ + { source: "/", has: [acceptsMarkdown], destination: "/llms.mdx" }, + { source: pages, has: [acceptsMarkdown], destination: "/llms.mdx/:path" }, + ], + // `/package-managers/clean.md` serves that page as plain markdown. + afterFiles: [{ source: "/:path+.md", destination: "/llms.mdx/:path+" }], + }), + // Those urls answer with html or markdown depending on `Accept`, so caches must key on it. + headers: async () => + ["/", pages].map((source) => ({ + source, + headers: [{ key: "Vary", value: "Accept" }], + })), + // Compare was removed; its links land on the same section's report. + redirects: async () => [ + /* The front door is the default benchmark; what used to be here is /about. + Temporary, so the default can change without browsers caching it. */ + { source: "/", destination: "/registries/average", permanent: false }, + /* A benchmark opens on its average, query kept; temporary for the same reason. */ + { + source: "/:section(registries|package-managers|task-runners)", + destination: "/:section/average", + permanent: false, + }, + { source: "/compare", destination: "/package-managers", permanent: true }, + { source: "/compare/:section/:rest*", destination: "/:section", permanent: true }, + ], +}; + +export default createMDX()(nextConfig); diff --git a/apps/www/package.json b/apps/www/package.json new file mode 100644 index 0000000000..1f3bc686fe --- /dev/null +++ b/apps/www/package.json @@ -0,0 +1,55 @@ +{ + "name": "@benchmarks/www", + "version": "0.0.0", + "private": true, + "description": "The benchmarks site. A Next.js app that renders the published gh-pages data, fetched over HTTP through @benchmarks/data, as HTML, markdown and MCP.", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint:check": "oxlint src test", + "lint": "oxlint --fix src test", + "format:check": "oxfmt --check src test", + "format": "oxfmt src test", + "typecheck": "tsc --noEmit", + "test": "vitest run" + }, + "dependencies": { + "@benchmarks/data": "workspace:*", + "@benchmarks/types": "workspace:*", + "@mdx-js/loader": "^3.1.1", + "@mdx-js/react": "^3.1.1", + "@modelcontextprotocol/server": "^2.3.1", + "@next/mdx": "^16.4.1", + "@takumi-rs/image-response": "^0.69.2", + "class-variance-authority": "^0.7.1", + "clsx": "^2.1.1", + "cmdk": "^1.1.1", + "echarts": "^6.0.0", + "lucide-react": "^1.50.0", + "mcp-handler": "^2.3.0", + "motion": "^12.23.26", + "next": "^16.3.8", + "next-themes": "^0.4.6", + "nuqs": "^2.10.2", + "radix-ui": "^1.6.7", + "react": "^19.2.0", + "react-day-picker": "^10.0.2", + "react-dom": "^19.2.0", + "tailwind-merge": "^3.4.0", + "tailwindcss": "^4.3.3", + "zod": "^4.6.7" + }, + "devDependencies": { + "@tailwindcss/postcss": "^4.3.3", + "@types/mdx": "^2.0.14", + "@types/node": "catalog:hygiene", + "@types/react": "^19.2.2", + "@types/react-dom": "^19.2.1", + "oxfmt": "catalog:hygiene", + "oxlint": "catalog:hygiene", + "tw-animate-css": "^1.4.0", + "typescript": "catalog:hygiene", + "vitest": "catalog:testing" + } +} diff --git a/apps/www/postcss.config.mjs b/apps/www/postcss.config.mjs new file mode 100644 index 0000000000..61e36849cf --- /dev/null +++ b/apps/www/postcss.config.mjs @@ -0,0 +1,7 @@ +const config = { + plugins: { + "@tailwindcss/postcss": {}, + }, +}; + +export default config; diff --git a/app/src/assets/fonts/GeistMono-VariableFont_wght.ttf b/apps/www/public/assets/fonts/geist-mono.ttf similarity index 100% rename from app/src/assets/fonts/GeistMono-VariableFont_wght.ttf rename to apps/www/public/assets/fonts/geist-mono.ttf diff --git a/apps/www/public/assets/fonts/inter.ttf b/apps/www/public/assets/fonts/inter.ttf new file mode 100644 index 0000000000..5c88739bdf Binary files /dev/null and b/apps/www/public/assets/fonts/inter.ttf differ diff --git a/apps/www/public/images/marketing/open-source-initiative.png b/apps/www/public/images/marketing/open-source-initiative.png new file mode 100644 index 0000000000..07f8796044 Binary files /dev/null and b/apps/www/public/images/marketing/open-source-initiative.png differ diff --git a/apps/www/src/app/[section]/[[...params]]/page.tsx b/apps/www/src/app/[section]/[[...params]]/page.tsx new file mode 100644 index 0000000000..ef2258a190 --- /dev/null +++ b/apps/www/src/app/[section]/[[...params]]/page.tsx @@ -0,0 +1,722 @@ +import type { Metadata } from "next"; +import { Fragment, Suspense } from "react"; +import { notFound, redirect } from "next/navigation"; +import { ArrowRight } from "lucide-react"; +import { InlineCode, Marked } from "@/components/inline-code"; +import { Button } from "@/components/ui/button"; +import { TimeSeriesChart } from "@/components/charts-echarts"; +import { BarFigure, GroupedBarFigure } from "@/components/charts-figures"; +import { ToolLegend } from "@/components/chart-legend"; +import { FixtureTimeSeries } from "@/components/fixture-trend"; +import { TabbedFigure } from "@/components/tabbed-figure"; +import { NoData } from "@/components/no-data"; +import { Callout, Figure, ReportSection, Stat } from "@/components/report"; +import { DataAndSource } from "@/components/raw-results"; +import { FixtureMenu } from "@/components/fixture-menu"; +import { PathSkeleton } from "@/components/path-skeleton"; +import { PendingLink, PendingProvider, PendingRegion } from "@/components/report-pending"; +import { + DESCRIPTION, + FINDING, + FIXTURE_MENU, + MENUS, + ReportHead, + ReportIntro, + TITLE, + TOOLS_MENU, + TOP, + Trail, +} from "@/components/report-shell"; +import { SelectionChips, type ServerChip } from "@/components/selection-chips"; +import { ShowAllTools, ToolsMenu } from "@/components/tools-menu"; +import { CountTable, Leaderboard, PerFixtureTable, TrendByTool } from "@/components/sections"; +import { TOCMinimap } from "@/components/toc-minimap"; +import { FixtureName, ToolName } from "@/components/tool-name"; +import { formatDuration, longDate } from "@/lib/findings"; +import { fixtureBars } from "@/lib/figures"; +import { pageMetadata } from "@/lib/metadata"; +import { ogFigure } from "@/lib/og-figure"; +import { loadReportParams, reportHref, reportParams } from "@/lib/params"; +import { loadCatalog } from "@/lib/catalog"; +import { dataClient } from "@/lib/data"; +import { loadReport } from "@/lib/report"; +import { withAverage } from "@/lib/scenarios"; +import { isSectionId, SECTION_DEFS, SECTIONS } from "@/lib/sections"; +import { fixtureLabel, toolLabel } from "@/lib/tools"; + +/* Every scenario of the latest run, so each prerenders its shell; any other renders it on demand. */ +export const generateStaticParams = async () => { + const { scenarios } = await loadCatalog(dataClient()); + return SECTIONS.flatMap((section) => + withAverage(scenarios[section]).map((s) => ({ section, params: [s.variation] })), + ); +}; + +/** "Package managers — vlt benchmarks": the section names the page, whatever the scenario. */ +export const generateMetadata = async ({ + params, + searchParams, +}: { + params: Promise<{ section: string; params?: string[] }>; + searchParams: Promise; +}): Promise => { + const { section, params: rest = [] } = await params; + if (!isSectionId(section)) return {}; + const def = SECTION_DEFS[section]; + const path = ["", section, ...rest].map(encodeURIComponent).join("/"); + /* The card shows this run and fixture; every other parameter changes nothing on it -- + but a link to one figure (`?figure=`, see lib/og-figure.ts) gets that figure's card, + over the window and without the tools hidden on the page it links to. */ + const query = await loadReportParams(searchParams); + const { fixture, date } = query; + const raw = (await searchParams)["figure"]; + const figure = ogFigure(typeof raw === "string" ? raw : null); + const card = new URL( + reportHref(`/og${path}`, { + fixture, + date, + ...(figure === null + ? {} + : { hide: query.hide, range: query.range, from: query.from, to: query.to }), + }), + "http://card", + ); + if (figure !== null) card.searchParams.set("figure", figure); + const image = `${card.pathname}${card.search}`; + return pageMetadata({ + title: def.label, + description: def.about, + path, + image, + }); +}; +/** + * A benchmark's report, for one scenario. + * + * The BENCHMARK and the SCENARIO are chosen in the sidebar (or its search, + * ⌘K). The page opens on one row, level with the sidebar's search: the + * breadcrumb trail on the left, the tools and fixture menus on the right. + * Beneath it any other non-default selection as a chip that undoes it, then + * the scenario's badge and what was found, as the page's h1. Under it what was + * on disk before each run, and what the scenario is, running into the lede. + * + * `/{section}` redirects to `/{section}/average`, query kept (next.config.ts): + * the derived view, the mean of the measured scenarios. Where there are fewer + * than two to average, the report redirects on to the section's first scenario. + * + * Everything here but the report is the shell, shared by every link to a + * report and prefetched; the report reads the address and streams in. + */ +const Page = ({ + params, + searchParams, +}: { + params: Promise<{ section: string; params?: string[] }>; + searchParams: Promise; +}) => ( + + + + } + > + + +); + +type SearchParams = { [key: string]: string | string[] | undefined }; + +/** The report itself: everything that waits on the data. */ +const Report = async ({ + params, + searchParams, +}: { + params: Promise<{ section: string; params?: string[] }>; + searchParams: Promise; +}) => { + const { section, params: rest } = await params; + if (!isSectionId(section)) notFound(); + const requested = rest?.[0] === undefined ? undefined : decodeURIComponent(rest[0]); + const def = SECTION_DEFS[section]; + const query = await loadReportParams(searchParams); + const result = await loadReport({ section, scenario: requested, query }); + + if (result.status === "unknown-section") notFound(); + if (result.status === "no-dates") { + return ( + <> +
+

{def.label}

+
+

+ The served tree reports no dates. There is nothing to show. +

+ + ); + } + /* + * `/{section}` opens on the section's average (or its first scenario), and + * so does a scenario this date did not publish (a stale link, a pinned old + * run): by redirect, query kept, so the address -- and the sidebar's current + * item -- name what is on screen. Temporary: whether a date has an average + * depends on what it published. + */ + if (result.status === "unknown-scenario") redirect(result.href); + + const pinned = result.status === "ok" ? result.report.filters.date : null; + const sectionHref = reportHref(`/${section}`, { ...query, date: pinned }); + + if (result.status === "no-results") { + return ( + <> +
+ +
+
+

{def.label}

+
+

No {def.label.toLowerCase()} results in this run.

+ + + ); + } + + const report = result.report; + const { scenario, run, filters, finding, baseline, caveats, trend } = report; + const { date } = run; + const { fixture, hide: hidden } = filters; + const { measure, isScript, outline, ranked, publishedFixtures, path } = report; + + const visible = (tool: string) => !hidden.includes(tool); + const name = (tool: string) => toolLabel(tool, section); + const named = (tool: string) => ({ tool, label: name(tool) }); + const baselineName = baseline === null ? null : name(baseline); + + const timeBars = report.timeBars; + const perPackageBars = report.perPackageBars; + /* What the chart needs of each run besides its plotted means. */ + const trendDetail = trend.points.map( + ({ + byTool: _byTool, + perPackage: _perPackage, + perPackageByFixture: _perPackageByFixture, + ...rest + }) => rest, + ); + const fixtureCount = report.fixtures.length; + const across = + fixture !== null + ? `on the ${fixtureLabel(fixture)} fixture` + : `across ${fixtureCount} ${fixtureCount === 1 ? "fixture" : "fixtures"}`; + /* Axis titles: the quantity and what it is taken over, in words. */ + const over = fixture !== null ? `on ${fixtureLabel(fixture)}` : `across ${fixtureCount} fixtures`; + const timeAxis = + fixtureCount === 1 + ? `${isScript ? "Execution" : "Install"} time ${over}` + : `Mean ${measure} time ${over}`; + const toolAxis = + section === "registries" + ? "Registry" + : section === "task-runners" + ? "Task runner" + : "Package manager"; + + /** A section holding one table is headed by that table: no title said twice. */ + const solo = new Set( + outline + .filter((part) => part.items.length === 1 && part.items[0]?.kind === "table") + .flatMap((part) => part.items.map((item) => item.id)), + ); + const level = (id: string): 2 | 3 => (solo.has(id) ? 2 : 3); + + /** + * What a bar figure's guide may say about the baseline: only what is on + * screen. The dashed outline and rule are described only if the baseline's + * bar is drawn. If the baseline is hidden, neither is drawn, and the guide + * says only what the multiples are still against. If this figure has no + * baseline at all, it says nothing. + */ + const baselineGuide = (bars: readonly { tool: string }[]): string => { + if (baselineName === null || !bars.some((b) => b.tool === baseline)) return ""; + return visible(baseline ?? "") + ? ` Dashed outline and dashed rule: the baseline, ${baselineName}.` + : ` Multiples are against the baseline, ${baselineName}, which is hidden.`; + }; + + /* Each fixture's bars, for the reader's tools: hidden tools are neither drawn nor named. */ + const onEachFixture = (rows: typeof report.timeRows) => + fixtureBars(rows).map((f) => ({ + fixture: f.fixture, + bars: f.bars + .filter((b) => visible(b.tool)) + .map((b) => ({ ...named(b.tool), value: b.value })), + failed: f.failed.filter(visible).map(name), + reference: f.bars.find((b) => b.tool === baseline)?.value, + })); + /* Every tool these files hold, hidden ones too: the legends list them all. */ + const byFixtureTools = [...ranked, ...report.unranked].map(named); + + const bodies: { [itemId: string]: React.ReactNode } = { + "leaderboard-table": ( +
+ Lower is faster. ± is the pooled relative standard deviation. + {trend.points.length > 1 + ? ` Since last run compares with ${longDate(trend.points.at(-2)?.date ?? date)}.` + : ""} + + } + notes={caveats.leaderboard} + > +
+ ), + "trend-by-tool": ( +
+
+ ), + "time-bars": ( +
+ Lower is faster. Bars start at zero, each a bright cap on its tool's colour, + labelled with its time and its multiple of the baseline. + {baselineGuide(timeBars.bars)} + + } + notes={caveats.timeBars} + > + visible(b.tool)) + .map((b) => ({ ...named(b.tool), value: b.value }))} + unit="s" + baseline={baseline} + reference={timeBars.bars.find((b) => b.tool === baseline)?.value} + xLabel={timeAxis} + yLabel={toolAxis} + variant="stripped" + /> +
+ ), + "time-trend": ( + + ), + "time-by-fixture": ( + + Lower is faster; bars start at zero, each a bright cap on its tool's colour. All + fixtures: a group per fixture, the tools in leaderboard order, as in the legend; hover a + fixture to read each by name. Each fixture: its tools fastest first, each labelled with + its time. A tool is drawn on every fixture it finished; a failed run is named, not + drawn. The legend shows and hides tools everywhere on the page. + {baselineGuide(timeBars.bars)} + + } + tabs={[ + { + value: "all", + label: "All fixtures", + content: ( + ({ + fixture: r.fixture, + label: fixtureLabel(r.fixture), + values: Object.fromEntries( + r.values.filter((v) => visible(v.tool)).map((v) => [v.tool, v.value]), + ), + failed: r.failed.filter(visible), + }))} + tools={byFixtureTools.filter((t) => visible(t.tool))} + unit="s" + yLabel={`${isScript ? "Execution" : "Install"} time`} + legend={ + t.tool)} + values={Object.fromEntries( + timeBars.bars.map((b) => [b.tool, `${formatDuration(b.value)} mean`]), + )} + section={section} + baseline={baseline} + /> + } + /> + ), + }, + ...onEachFixture(report.timeRows).map((f) => ({ + value: f.fixture, + label: , + /* Keyed: elements made in a map are validated as a list's children. */ + content: ( + + t.tool)} + values={Object.fromEntries( + report.timeRows + .find((r) => r.fixture === f.fixture) + ?.values.map((v) => [v.tool, formatDuration(v.value)]) ?? [], + )} + section={section} + baseline={baseline} + /> + } + /> + {f.failed.length === 0 ? null : ( +

+ Did not finish, not drawn: {f.failed.join(", ")} +

+ )} +
+ ), + })), + ]} + /> + ), + "time-table": ( +
+ Each cell is the mean time + {baselineName === null + ? "" + : `, with its multiple of ${baselineName} on that fixture beneath`} + . Bold: fastest on the fixture. DNF: the run exited non-zero and is excluded from every + figure. A dash: not run on that fixture. + + } + > +
+ ), + "per-package-chart": ( +
+ Milliseconds per installed package; lower is faster. Tools disagree about what a + dependency graph contains, so a tool that installs fewer packages finishes sooner + without being faster — this is the figure that corrects for it. + {baselineGuide(perPackageBars.bars)} + + } + notes={caveats.perPackageBars} + > + visible(b.tool)) + .map((b) => ({ ...named(b.tool), value: b.value }))} + unit="ms" + baseline={baseline} + reference={perPackageBars.bars.find((b) => b.tool === baseline)?.value} + xLabel={`${fixtureCount === 1 ? "Time" : "Mean time"} per installed package ${over}`} + yLabel={toolAxis} + /> +
+ ), + "per-package-by-fixture": ( + + ), + "per-package-table": ( +
+
+ ), + "package-count": ( +
+
+ ), + "process-count": ( +
+
+ ), + }; + + /* The non-default selections the menus beside them do not already show, + as resolved: a value the page ignored gets no chip. */ + const chips: ServerChip[] = [ + ...(filters.window.custom + ? [ + { + label: "window", + value: `${longDate(filters.window.from)} – ${longDate(filters.window.to)}`, + clear: ["from", "to", "range"] as const, + }, + ] + : query.range === reportParams.range.defaultValue + ? [] + : [ + { + label: "window", + value: `${query.range} days`, + clear: ["range", "from", "to"] as const, + }, + ]), + ...(trend.focus === null + ? [] + : [ + { + label: "focus", + value: `${longDate(trend.focus.from)} – ${longDate(trend.focus.to)}`, + clear: ["focus"] as const, + }, + ]), + ...(filters.exclude.length === 0 + ? [] + : [ + { + label: "exclusions", + value: String(filters.exclude.length), + clear: ["exclude"] as const, + }, + ]), + ]; + const metric = + query.metric === "total" || (query.metric === "per-package" && !trend.perPackage) + ? null + : query.metric; + + return ( + + + + {publishedFixtures.length < 2 ? null : ( + + )} + + } + chips={ + p.date)} /> + } + /> + + + + Mean {measure} time {across} + {scenario.averageOf === null ? ( + <> + {" "} + in the {scenario.variation} scenario + + ) : ( + `, averaged over ${scenario.averageOf.scenarios.length} scenarios` + )} + , from the run of {longDate(date)} on GitHub-hosted{" "} + ubuntu-24.04-arm runners. + {section === "registries" + ? " Every registry is installed from with the same npm client." + : ""} + {run.isLatest ? null : " This is not the latest run."} + + } + action={ + run.isLatest ? undefined : ( + + ) + } + /> + {caveats.qualifier === null ? null : ( +
+ + + +
+ )} + + {report.stats === null ? null : ( +
+ {report.stats.map((stat) => ( + + ) + } + /> + ))} +
+ )} + + {caveats.hidden === null ? null : ( +
+ + {caveats.hidden.body}{" "} + + +
+ )} + + {caveats.disclosure === null ? null : ( +

+ Disclosure. {caveats.disclosure} +

+ )} + + {outline.map((part) => ( + solo.has(item.id)) ? undefined : part.title} + > + {part.items.map((item) => ( + {bodies[item.id]} + ))} + + ))} + +
+ {/* Fixed to the viewport's right edge, in the page margin: no column. */} + [ + { url: `#${part.id}`, title: part.title, depth: 2 }, + ...(part.items.every((item) => solo.has(item.id)) + ? [] + : part.items.map((item) => ({ + url: `#${item.id}`, + title: item.label, + depth: 3, + }))), + ])} + /> +
+ ); +}; + +export default Page; diff --git a/apps/www/src/app/about/components.tsx b/apps/www/src/app/about/components.tsx new file mode 100644 index 0000000000..74f0bec959 --- /dev/null +++ b/apps/www/src/app/about/components.tsx @@ -0,0 +1,481 @@ +import { + ArrowRightIcon, + BookOpenIcon, + BracesIcon, + FileTextIcon, + PackageIcon, + PlugIcon, + ServerIcon, + WorkflowIcon, + type LucideIcon, +} from "lucide-react"; +import Link from "next/link"; +import { AgentPrompt } from "@/components/agent-prompt"; +import { Blueprint } from "@/components/blueprint"; +import { Marked } from "@/components/inline-code"; +import { Figure } from "@/components/report"; +import { Button } from "@/components/ui/button"; +import { ToolName } from "@/components/tool-name"; +import { dataClient } from "@/lib/data"; +import { toolColor } from "@/lib/chart-model"; +import { meanBars, shownBars } from "@/lib/figures"; +import { formatDuration, longDate } from "@/lib/findings"; +import { scenarioFacts, type ScenarioFacts } from "@/lib/headline"; +import { loadScenario } from "@/lib/load"; +import { SITE_DESCRIPTION, SITE_NAME, siteOrigin } from "@/lib/metadata"; +import { fixturesWith } from "@/lib/outline"; +import { scenarioPath, scenariosFor, type Scenario } from "@/lib/scenarios"; +import { SECTION_DEFS, SECTIONS, type SectionId } from "@/lib/sections"; +import { toolLabel } from "@/lib/tools"; + +/* + * The About page's blocks, as content.mdx places them: everything on it that + * is computed, interactive or drawn. Each is set apart from the article's + * typography (`data-not-typeset`), as the docs set their embedded components. + */ + +interface Headline { + readonly section: SectionId; + readonly scenario: Scenario; + readonly scenarioCount: number; + readonly fixtureCount: number; + readonly facts: ScenarioFacts; +} + +const SECTION_ICONS: { readonly [S in SectionId]: LucideIcon } = { + registries: ServerIcon, + "package-managers": PackageIcon, + "task-runners": WorkflowIcon, +}; + +/* A count as a quiet chip: the number in the foreground, its noun muted. */ +const Chip = ({ n, one, many }: { n: number; one: string; many: string }) => ( +
  • + {n} + {n === 1 ? one : many} +
  • +); + +/** The agent prompt, in llms.txt's words (app/llms.txt/route.ts): what to read, the API, the MCP server, how to read a result. */ +const agentPrompt = ( + origin: string, +): string => `Help me use the vlt benchmarks. Start by reading ${origin}/llms.txt, which lists every page of the site and explains how to read the results. Every page is available as Markdown: append .md to its URL, or request the page with an Accept: text/markdown header. Query parameters apply to the Markdown as they do to the page: ${origin}/package-managers/clean.md?fixture=babylon is that scenario on one fixture. ${origin}/llms-full.txt holds every scenario of the latest run, at the default view, in one file. + +1. For structured data, use the JSON API, described at ${origin}/openapi.json: /api/v1/runs, /api/v1/catalog, /api/v1/{section}/{scenario} (the full report, same parameters as the page) and /api/v1/{section}/{scenario}/history (the trend, failures and coverage). +2. To query it as tools, add the read-only MCP server with \`claude mcp add --transport http vlt-benchmarks ${origin}/mcp\`. Its tools are list_benchmarks, get_results, get_history and compare_scenarios. +3. When you report a result: total time is the measurement and the ranking. A DNF (the run exited non-zero) is never counted as a time, a tool that did not complete every fixture is not ranked against tools that did, and derived metrics are labelled "derived". + +Only quote numbers that appear on the site, and name a run by its date.`; + +type Card = { + icon: LucideIcon; + title: string; + description: string; + /** Absent for a surface a browser cannot open (the MCP endpoint). */ + href?: string; +}; + +const cards: Card[] = [ + { + icon: BookOpenIcon, + title: "llms.txt", + description: "An index of every page on this site, for agents to read before anything else.", + href: "/llms.txt", + }, + { + icon: FileTextIcon, + title: "Markdown pages", + description: + "Add .md to any page's URL for a plain Markdown copy, or read every scenario of the latest run in one file.", + href: "/llms-full.txt", + }, + { + icon: BracesIcon, + title: "JSON API", + description: + "Runs, the catalog, and each scenario's full report and history, described by an OpenAPI document.", + href: "/openapi.json", + }, + { + icon: PlugIcon, + title: "MCP server", + description: + "Read-only tools at /mcp to list benchmarks, get results and history, and compare scenarios.", + }, +]; + +/* The docs' card grid (vltpkg/www/docs/src/app/page.tsx), verbatim but for the card without a link. */ +const CardGrid = ({ cards }: { cards: Card[] }) => ( +
      + {cards.map(({ icon: Icon, title, description, href }) => { + const body = ( + <> + +

      {title}

      +

      {description}

      + + ); + return ( +
    • + {href === undefined ? ( +
      {body}
      + ) : ( + + {body} + + )} +
    • + ); + })} +
    +); + +/* Who publishes the site and what it holds, for agents and search engines (schema.org JSON-LD), in + the docs' shape. The organisation is the docs' own; the dataset claims nothing a run could falsify. */ +const jsonLd = (origin: string, latest: string | null) => { + const org = { + "@type": "Organization", + "@id": "https://www.vlt.io/#organization", + name: "vlt", + legalName: "vlt technology inc", + url: "https://www.vlt.io", + logo: new URL("/icon1.png", origin).href, + sameAs: [ + "https://github.com/vltpkg", + "https://www.linkedin.com/company/vltpkg/", + "https://x.com/vltpkg", + "https://bsky.app/profile/vlt.sh", + ], + }; + return { + "@context": "https://schema.org", + "@graph": [ + org, + { + "@type": "WebSite", + name: SITE_NAME, + url: `${origin}/`, + description: SITE_DESCRIPTION, + publisher: { "@id": org["@id"] }, + }, + { + "@type": "Dataset", + name: SITE_NAME, + description: SITE_DESCRIPTION, + url: `${origin}/`, + ...(latest === null ? {} : { dateModified: latest }), + publisher: { "@id": org["@id"] }, + creator: { "@id": org["@id"] }, + distribution: [ + { + "@type": "DataDownload", + encodingFormat: "application/json", + contentUrl: `${origin}/api/v1/catalog`, + }, + { + "@type": "DataDownload", + encodingFormat: "text/markdown", + contentUrl: `${origin}/llms-full.txt`, + }, + ], + }, + ], + }; +}; + +export const JsonLd = async () => { + const origin = siteOrigin(); + const index = await dataClient().getIndex(); + return ( +