ci(perf): GOMEMLIMIT with GOGC=off, pinned toolchain, cachegrind I1, matrix mode [W0-15]

.github/workflows/bench-compare.yml

@@ -8,9 +8,15 @@ name: Competitor Benchmarks
 #   one-pass benchmark smoke run.
 # - workflow_dispatch and release tags: a full -count run that publishes the
 #   results JSON, the raw benchmark output and a benchstat summary as an
-#   artifact. Shared runners are noisy; the JSON records the CPU so a run is
-#   never presented without its hardware. Commit the artifact to
+#   artifact. Shared runners are noisy and differ in CPU model from job to
+#   job; the JSON and env.txt record the CPU so a run is never presented
+#   without its hardware, and the job summary compares GoSQLX with each peer
+#   by in-job time ratios rather than absolute ns/op. Commit the artifact to
 #   benchmarks/compare/results/ (and website/src/data/) in a PR to publish it.
+#
+# Both jobs build with PERF_GO_VERSION, the release perf-gate.yml also pins
+# (scripts/perf/gate/config_test.go keeps them equal); GOGC=off runs set
+# GOMEMLIMIT (cmd/publish -gomemlimit, default as in scripts/perf/lib.sh).
 
 on:
   pull_request:
@@ -58,6 +64,8 @@ defaults:
 env:
   CGO_ENABLED: '1'
   GOTOOLCHAIN: local
+  # One Go release for every perf measurement; perf-gate.yml pins the same.
+  PERF_GO_VERSION: '1.26.9'
 
 jobs:
   test:
@@ -70,8 +78,8 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
         with:
-          go-version-file: benchmarks/compare/go.mod
-          check-latest: true
+          go-version: ${{ env.PERF_GO_VERSION }}
+          check-latest: false
           cache-dependency-path: benchmarks/compare/go.sum
 
       - name: Verify go.mod is tidy
@@ -102,8 +110,8 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
         with:
-          go-version-file: benchmarks/compare/go.mod
-          check-latest: true
+          go-version: ${{ env.PERF_GO_VERSION }}
+          check-latest: false
           cache-dependency-path: benchmarks/compare/go.sum
 
       - name: Run harness
@@ -118,27 +126,43 @@ jobs:
             *[!0-9a-z.]*|'') echo "benchtime must look like 200ms or 100x, got '$BENCHTIME'" >&2; exit 1 ;;
           esac
           mkdir -p "$RUNNER_TEMP/bench"
+          {
+            echo "cpu model:   $(. ../../scripts/perf/lib.sh && perf_cpu_model)"
+            echo "go:          $(go version)"
+            echo "kernel:      $(uname -srm)"
+            echo "commit:      $(git rev-parse HEAD)"
+            echo
+            lscpu
+          } > "$RUNNER_TEMP/bench/env.txt"
           # The matrix: -cpu 4 and -cpu 1, each with GOGC at its default and
-          # off, so mutator CPU can be told apart from GC. The first entry
-          # (cpu4, GOGC default) feeds the schema v1 fields of the JSON.
+          # off (with GOMEMLIMIT), so mutator CPU can be told apart from GC.
+          # The first entry (cpu4, GOGC default) feeds the schema v1 fields.
           go run ./cmd/publish -count="$COUNT" -benchtime="$BENCHTIME" \
             -cpus=4,1 -gogc=default,off \
             -out "$RUNNER_TEMP/bench/latest.json" \
-            -raw "$RUNNER_TEMP/bench/latest.txt"
+            -raw "$RUNNER_TEMP/bench/latest.txt" \
+            -ratios "$RUNNER_TEMP/bench/ratios.md"
 
       - name: benchstat summary
         run: |
           {
             echo '## Competitor benchmarks'
+            echo
+            cat "$RUNNER_TEMP/bench/ratios.md"
+            echo
+            echo '### Absolute numbers (this job and this CPU only)'
             for f in "$RUNNER_TEMP"/bench/latest*.txt; do
               name=$(basename "$f" .txt)
               go run golang.org/x/perf/cmd/benchstat@v0.0.0-20260908200009-22c9c6c9d4da \
                 "$f" > "$RUNNER_TEMP/bench/benchstat-$name.txt"
               echo
-              echo "### $name"
+              echo "<details><summary>$name</summary>"
+              echo
               echo '```'
               cat "$RUNNER_TEMP/bench/benchstat-$name.txt"
               echo '```'
+              echo
+              echo '</details>'
             done
           } >> "$GITHUB_STEP_SUMMARY"
 

.github/workflows/perf-gate.yml

@@ -10,9 +10,20 @@ name: Perf Gate
 #   scripts/perf/gate judges the result; the report goes to the job summary,
 #   a sticky comment (same-repo PRs only) and an artifact.
 # - Nightly on main: the same gate between main now and main 24 hours ago.
-# - workflow_dispatch: mode "gate" with explicit base/head revisions, or mode
-#   "profile": pprof (-cpu 1, GOGC=off, -cpu 4) of the gap queries and
-#   perf stat counters for one revision (input to W0-14).
+# - workflow_dispatch: mode "gate" with explicit base/head revisions; mode
+#   "matrix": the same interleaved A/B over the full benchmark matrix
+#   (scripts/perf/matrix-suite.sh, about 900 benchmarks; default 5 rounds of
+#   100ms, roughly an hour; base = head gives an A/A of the whole matrix); or
+#   mode "profile": pprof (-cpu 1, GOGC=off, -cpu 4) of the gap queries, perf
+#   stat counters where the PMU allows, and cachegrind cache simulation (L1i
+#   included) for one revision (input to W0-14).
+#
+# Methodology (W0-15): one pinned Go release (PERF_GO_VERSION, shared with
+# bench-compare.yml; GOTOOLCHAIN=local stops go.mod from switching it); every
+# GOGC=off run also sets GOMEMLIMIT (scripts/perf/lib.sh, input "gomemlimit");
+# every artifact and report names the runner CPU model, and reports compare
+# base and head of the same job by ratio only, because hosted runners differ
+# in CPU model from job to job.
 #
 # SHADOW MODE: the gate reports and never fails a PR. After the shadow period,
 # set PERF_GATE_ENFORCE to 'true' below; a regression then fails the job
@@ -34,9 +45,9 @@ on:
   workflow_dispatch:
     inputs:
       mode:
-        description: 'gate: A/B of base vs head; profile: CPU profiles and perf counters of head'
+        description: 'gate: A/B of base vs head over the gated suite; matrix: the same A/B over the full benchmark matrix; profile: CPU profiles, perf counters and cachegrind of head'
         type: choice
-        options: [gate, profile]
+        options: [gate, matrix, profile]
         default: gate
       base:
         description: 'gate mode: base revision (default origin/main)'
@@ -47,32 +58,62 @@ on:
         required: false
         default: ''
       rounds:
-        description: 'gate mode: interleaved rounds'
+        description: 'gate/matrix mode: interleaved rounds (default 10; matrix 5)'
         required: false
-        default: '10'
+        default: ''
       benchtime:
-        description: 'gate mode: -test.benchtime per run; profile mode: per profile'
+        description: 'gate/matrix mode: -test.benchtime per run (default 200ms; matrix 100ms); profile mode: per profile (default 5s)'
+        required: false
+        default: ''
+      gomemlimit:
+        description: 'GOMEMLIMIT of the GOGC=off runs (default: scripts/perf/lib.sh; "off" leaves it unset)'
         required: false
         default: ''
 
 permissions:
   contents: read
 
 concurrency:
-  group: perf-gate-${{ github.event.pull_request.number || github.ref }}-${{ inputs.mode || 'gate' }}
+  group: perf-gate-${{ github.event.pull_request.number || github.ref }}-${{ inputs.mode || 'gate' }}-${{ inputs.gomemlimit }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
   GOTOOLCHAIN: local
+  # The one Go release every perf measurement uses; bench-compare.yml pins
+  # the same (scripts/perf/gate/config_test.go keeps them equal). 1.26.9 is
+  # the toolchain of the recorded C0 baselines.
+  PERF_GO_VERSION: '1.26.9'
+  PERF_GOMEMLIMIT: ${{ inputs.gomemlimit }}
   PERF_GATE_ENFORCE: 'false'
 
 jobs:
   gate:
-    name: A/B perf gate (shadow)
-    if: github.event_name != 'workflow_dispatch' || inputs.mode == 'gate'
+    # A matrix dispatch is a report, not a gate verdict: name it so.
+    name: ${{ inputs.mode == 'matrix' && 'Matrix A/B (report only)' || 'A/B perf gate (shadow)' }}
+    if: github.event_name != 'workflow_dispatch' || inputs.mode != 'profile'
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: ${{ inputs.mode == 'matrix' && 240 || 30 }}
     steps:
+      - name: Validate inputs
+        # A GOMEMLIMIT the runtime rejects would make every benchmark binary
+        # panic at startup; fail here with a clear input error instead. Same
+        # rule as perf_check_gomemlimit in scripts/perf/lib.sh (a test keeps
+        # them equal), inlined so it runs before any checked-out code.
+        env:
+          LIMIT: ${{ inputs.gomemlimit }}
+        run: |
+          n=$LIMIT
+          case $n in *B) n=${n%B} n=${n%[KMGT]i} ;; esac
+          case $LIMIT in
+            '' | off) ;;
+            *[!0-9A-Za-z]*) bad=1 ;;
+            *) case $n in '' | *[!0-9]*) bad=1 ;; esac ;;
+          esac
+          if [ -n "${bad:-}" ]; then
+            echo "::error::gomemlimit must be off or digits with an optional B/KiB/MiB/GiB/TiB suffix (e.g. 64MiB), got '$LIMIT'"
+            exit 1
+          fi
+
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
         with:
           ref: ${{ inputs.head || '' }}
@@ -82,8 +123,8 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
         with:
-          go-version-file: go.mod
-          check-latest: true
+          go-version: ${{ env.PERF_GO_VERSION }}
+          check-latest: false
           cache-dependency-path: go.sum
 
       - name: Install instruction counters (best effort)
@@ -121,32 +162,49 @@ jobs:
 
       - name: Run perf gate
         id: gate
-        continue-on-error: ${{ env.PERF_GATE_ENFORCE != 'true' }}
-        timeout-minutes: 24
+        continue-on-error: ${{ env.PERF_GATE_ENFORCE != 'true' || inputs.mode == 'matrix' }}
+        timeout-minutes: ${{ inputs.mode == 'matrix' && 230 || 24 }}
         env:
           BASE: ${{ steps.refs.outputs.base }}
-          ROUNDS: ${{ inputs.rounds || '10' }}
-          BENCHTIME: ${{ inputs.benchtime || '200ms' }}
+          MODE: ${{ inputs.mode || 'gate' }}
+          ROUNDS: ${{ inputs.rounds }}
+          BENCHTIME: ${{ inputs.benchtime }}
           ACCEPTED: ${{ contains(github.event.pull_request.labels.*.name, 'perf-regression-accepted') }}
         run: |
           set -o pipefail
-          flags=(-G -r "$ROUNDS" -t "$BENCHTIME" -o "$RUNNER_TEMP/perf-gate")
-          [ "$PERF_GATE_ENFORCE" = true ] && flags+=(-e)
-          [ "$ACCEPTED" = true ] && flags+=(-A)
           mkdir -p "$RUNNER_TEMP/perf-gate"
-          scripts/perf/ab.sh "${flags[@]}" "$BASE" 2>&1 | tee "$RUNNER_TEMP/perf-gate.log"
+          if [ "$MODE" = matrix ]; then
+            # The full matrix: base vs head interleaved like the gate, without
+            # the A/A arm and instruction counts (too slow at this size). It
+            # reports and never fails.
+            scripts/perf/matrix-suite.sh > "$RUNNER_TEMP/matrix-suite.txt"
+            cp "$RUNNER_TEMP/matrix-suite.txt" "$RUNNER_TEMP/perf-gate/"
+            flags=(-r "${ROUNDS:-5}" -t "${BENCHTIME:-100ms}" -s "$RUNNER_TEMP/matrix-suite.txt" -T 'Matrix A/B')
+          else
+            flags=(-G -r "${ROUNDS:-10}" -t "${BENCHTIME:-200ms}")
+            [ "$PERF_GATE_ENFORCE" = true ] && flags+=(-e)
+            [ "$ACCEPTED" = true ] && flags+=(-A)
+          fi
+          scripts/perf/ab.sh "${flags[@]}" -o "$RUNNER_TEMP/perf-gate" "$BASE" 2>&1 | tee "$RUNNER_TEMP/perf-gate.log"
 
       - name: Job summary
         if: always()
         env:
           PR_NUMBER: ${{ github.event.pull_request.number }}
+          MODE: ${{ inputs.mode || 'gate' }}
         run: |
           out="$RUNNER_TEMP/perf-gate"
           mkdir -p "$out"
           if [ ! -f "$out/report.md" ]; then
             {
               echo '<!-- perf-gate-report -->'
-              echo '## Perf gate: ERROR (shadow mode: report only)'
+              if [ "$MODE" = matrix ]; then
+                echo '## Matrix A/B: ERROR (report only, not a gate verdict)'
+              else
+                echo '## Perf gate: ERROR (shadow mode: report only)'
+              fi
+              echo
+              echo "Runner CPU: \`$(. scripts/perf/lib.sh && perf_cpu_model)\`."
               echo
               echo "The gate did not produce a report; see the \"Run perf gate\" step log of ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}. Last lines:"
               echo
@@ -226,14 +284,17 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
         with:
-          go-version-file: go.mod
-          check-latest: true
+          go-version: ${{ env.PERF_GO_VERSION }}
+          check-latest: false
           cache-dependency-path: go.sum
 
-      - name: Install perf (best effort)
+      - name: Install perf and valgrind (best effort)
         continue-on-error: true
         run: |
           sudo apt-get update -qq
+          # valgrind first: cachegrind's cache simulation is the only L1i data
+          # on hosted runners, whose virtualised PMU usually lacks perf events.
+          sudo apt-get install -y -qq --no-install-recommends valgrind >/dev/null
           sudo apt-get install -y -qq --no-install-recommends linux-tools-common "linux-tools-$(uname -r)" >/dev/null
           sudo sysctl -w kernel.perf_event_paranoid=-1
 

CHANGELOG.md

@@ -43,6 +43,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
     its never-enforced `TokenizationThroughput` and `EndToEndSustained` entries
     are removed.
 
+### Changed
+- **Perf measurement methodology** (W0-15; benchmark tooling only, no library
+  behaviour change):
+  - Every `GOGC=off` measurement now also sets `GOMEMLIMIT` (default `64MiB`;
+    override with `PERF_GOMEMLIMIT` or the `gomemlimit` dispatch input, `off`
+    unsets it): `scripts/perf/ab.sh` (so `task bench:ab` and `task bench:gate`),
+    `scripts/perf/profile.sh`, `task bench:matrix` and the competitor
+    harness's `cmd/publish` (`-gomemlimit`, recorded as
+    `configs[].gomemlimit` in the results JSON). An invalid value is refused
+    up front instead of making every benchmark binary panic.
+    Without a limit a GC-off heap only grows, so allocations land on fresh pages
+    whose first-touch faults were timed and profiled; the first benchmark of
+    each test binary absorbed most of them. GOGC=off timings are therefore not
+    comparable with runs made before this change; base and head of one run
+    always use the same setting. The limit lets the collector run, so timings
+    and instructions/op (`perfdriver`, cachegrind) are no longer pure mutator
+    work but include a small GC share (about 0.25% of Ir/op for `join`), and
+    the pool drains it causes make allocs/op vary for large-input and
+    low-iteration benchmarks (not in the gated suite, where it stays exact).
+  - `scripts/perf/baseline` deliberately keeps `GOGC=off` with no
+    `GOMEMLIMIT` (and drops an inherited one): it gates absolute allocs/op and
+    B/op only, which are exact only when no collection drains the pools.
+    `docs/PERFORMANCE_METHODOLOGY.md` documents the whole method.
+  - `perf-gate.yml` and `bench-compare.yml` build with one pinned Go release,
+    `PERF_GO_VERSION` 1.26.9 (the toolchain of the recorded baselines); the
+    gate used to take 1.26.1 from `go.mod`.
+  - `perf-gate.yml` profile mode adds cachegrind cache simulation (32 KiB
+    8-way L1i, the Zen 3/4/5 geometry) of the gap queries: instructions, L1i
+    and L1d misses per operation and the functions with the most L1i misses.
+  - New `matrix` dispatch mode in `perf-gate.yml`: the interleaved A/B over the
+    full benchmark matrix (`scripts/perf/matrix-suite.sh`), report only: the
+    job is named "Matrix A/B (report only)" and the report heading is
+    "Matrix A/B: VERDICT (report only, not a gate verdict)" (`ab.sh -T`,
+    `gate -report-only -title`).
+  - Every perf artifact and report names the runner CPU model, and the reports
+    compare by in-job ratio only: the gate report says so, and the competitor
+    harness summary leads with GoSQLX/peer time ratios (`cmd/publish -ratios`)
+    instead of absolute ns/op.
+  - The LSP benchmark server (`setupTestServer`) discards its output, so
+    benchmarks that publish diagnostics no longer time a growing buffer.
+
 ### Performance
 - **Linter**: `linter.Context` gains an additive `Nodes()` method returning every AST node in `ast.Inspect` preorder, built once per `Context` and cached (reset by `WithAST`). The 13 AST rules that each walked the tree with `ast.Inspect` now iterate that shared list; violations are unchanged (none of those walks pruned a subtree). [W6-05a]
 - **Security**: `Scanner.ScanSQL` now skips the dollar-quote stripping lexer, the identifier-tautology regex and the ClickHouse table-function regex on inputs that provably cannot match them, using linear-time necessary-condition guards (`hasDollarOpener`, `hasEqualIdentPair`, `hasTableFnCandidate`). The regexes still confirm every finding, so results are byte-identical (guarded by a guards-on/off equivalence test over the real-world corpus and 200k random inputs); no API or behaviour change. [W6-01]

Taskfile.yml

@@ -185,7 +185,7 @@ tasks:
       - go run golang.org/x/perf/cmd/benchstat@v0.0.0-20260908200009-22c9c6c9d4da results/latest.txt
 
   bench:ab:
-    desc: "Interleaved A/B of the working tree against BASE (GOGC=off, -cpu 1, ABBA, benchstat alpha 0.01). Vars: BASE (default origin/main), ROUNDS (10), BENCHTIME (200ms), BENCH='./pkg/gosqlx=^BenchmarkParseE2E$' (default: the gated suite)"
+    desc: "Interleaved A/B of the working tree against BASE (GOGC=off with GOMEMLIMIT, -cpu 1, ABBA, benchstat alpha 0.01). Vars: BASE (default origin/main), ROUNDS (10), BENCHTIME (200ms), BENCH='./pkg/gosqlx=^BenchmarkParseE2E$' (default: the gated suite)"
     cmds:
       - scripts/perf/ab.sh -r {{.ROUNDS}} -t {{.BENCHTIME}} {{if .BENCH}}-b '{{.BENCH}}'{{end}} {{.BASE}}
     vars:
@@ -203,9 +203,10 @@ tasks:
 
   bench:matrix:
     set: [pipefail]
-    desc: "Run the whole benchmark matrix once (every pkg benchmark except cbinding, which needs CGO: see bench:interop; -benchmem, -cpu 1, GOGC=off) and write it to OUT (default: .bench/matrix.txt, gitignored). Vars: BENCHTIME (default 100ms), OUT, FILTER (default .)"
+    desc: "Run the whole benchmark matrix once (every pkg benchmark except cbinding, which needs CGO: see bench:interop; -benchmem, -cpu 1, GOGC=off with GOMEMLIMIT=64MiB so the heap stays bounded and GC-off timings are not dominated by first-touch page faults) and write it to OUT (default: .bench/matrix.txt, gitignored). Vars: BENCHTIME (default 100ms), OUT, FILTER (default .)"
     env:
       GOGC: 'off'
+      GOMEMLIMIT: '64MiB'
     cmds:
       - mkdir -p "$(dirname '{{.OUT}}')"
       - go test -run '^$' -bench '{{.FILTER}}' -benchmem -benchtime {{.BENCHTIME}} -cpu 1 $(go list ./pkg/... ./wasm/internal/... | grep -v /cbinding) 2>&1 | tee {{.OUT}}
@@ -235,7 +236,7 @@ tasks:
       TMP_BIN: '{{.TMPDIR | default "/tmp"}}/gosqlx-pgo.test'
 
   bench:profile:
-    desc: "CPU profiles (-cpu 1, GOGC=off, -cpu 4) of the gap queries plus perf stat counters when available (Vars: BENCHTIME, default 3s)"
+    desc: "CPU profiles (-cpu 1, GOGC=off with GOMEMLIMIT, -cpu 4) of the gap queries plus perf stat counters when available (Vars: BENCHTIME, default 3s)"
     cmds:
       - scripts/perf/profile.sh -t {{.BENCHTIME}}
     vars: