From 688d0098297e20db015b721f3d3b79b393b0f134 Mon Sep 17 00:00:00 2001 From: error-path <95665869+error-path@users.noreply.github.com> Date: Sat, 24 Oct 2026 14:29:39 +0530 Subject: [PATCH] fix(docs): mermaid reserved-id parse error and broken .html/.md links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README's "How it works" flowchart failed to parse on GitHub: `graph` is a reserved Mermaid keyword and can't be a node id. Renamed it to `imports`. docs/**/*.md and CHANGELOG.md linked to sibling pages as `name.html`, which only exists in the Jekyll build (docs/Gemfile already enables jekyll-relative-links, on by default for GitHub Pages native builds). Those links 404 on github.com. Rewrote them as relative `.md` links so Jekyll converts them to `.html` at build time and GitHub renders them directly; also replaced three `{{ '...' | relative_url }}` Liquid links in plain doc prose with the same relative `.md` form (left docs/research/index.md's templated collection loop alone — Liquid is right there). Fixed one spec link (docs/specs/2026-10-21-seo-long-term-design.md) whose relative path was wrong regardless of extension. Could not build the Jekyll site locally to verify end to end: Ruby 4.0.7's native gem extensions (bigdecimal, eventmachine, json) fail to compile on this machine's clang/SDK headers (`stdckdint.h` not found), unrelated to this repo. Verified instead from jekyll-relative-links' documented behavior and docs/_config.yml/docs/Gemfile's existing plugin list. Added two offline checks so this class of bug fails CI instead of surfacing on github.com: - scripts/check-md-links.mjs: every relative link and #anchor in README/CHANGELOG/CONTRIBUTING/SECURITY/docs/**/skills/** must resolve to a real file and a real heading, computed with no Jekyll build. - scripts/check-mermaid.mjs: a ```mermaid flowchart node id can't be a Mermaid reserved word (graph, end, subgraph, ...) — the exact class of the README bug. Chose a dependency-free .mjs script + vitest wrapper (tests/scripts/*.test.ts) over lychee or another external tool: the repo already uses this exact pattern for offline checks (scripts/check-site.mjs, scripts/metrics-path.mjs), and it plugs into the existing `npm test`/`npm run check`/ci.yml gate — which already runs on every PR regardless of path — with no new workflow step, binary install or network access. A full Mermaid grammar check needs mermaid's renderer (a DOM via jsdom or a headless browser), too heavy for this repo's two diagrams; the reserved-id lint covers the reported failure class directly. Also caught up README.md with v0.5.0: added the Verification pass and Pre-PR self-check rows to the Features table and a Verification pass row to the Documentation table, both shipped in 0.5.0 but missing from README. docs/captain.md already correctly documents coupling.missing as info-only and the outcome-labels study; no stale claim found there. Timing (rule 14): the new checks' vitest run was 160ms (tests/scripts/check-md-links.test.ts + check-mermaid.test.ts, 16 tests). Full `npm run check` under the CI-like env (rule 15) passed: typecheck, test 25.87s (1190 passed), test:slow 210.52s (551 passed, 1 skipped). Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 18 +-- README.md | 5 +- docs/alternatives.md | 4 +- docs/auto-approve.md | 6 +- docs/blast-radius.md | 6 +- docs/calibration.md | 2 +- docs/captain.md | 8 +- docs/cli-and-mcp.md | 2 +- docs/configuration.md | 4 +- docs/experiments.md | 24 +-- docs/github-action.md | 6 +- docs/limitations.md | 6 +- docs/llm-judge.md | 6 +- docs/llm-reviewers.md | 6 +- docs/maintaining.md | 2 +- docs/policy.md | 12 +- .../2026-10-22-dev-loop-time-sinks.md | 2 +- docs/research/index.md | 2 +- docs/results.md | 4 +- docs/signals.md | 6 +- docs/specs/2026-10-21-seo-long-term-design.md | 2 +- docs/verification.md | 4 +- scripts/check-md-links.d.mts | 7 + scripts/check-md-links.mjs | 150 ++++++++++++++++++ scripts/check-mermaid.d.mts | 6 + scripts/check-mermaid.mjs | 76 +++++++++ tests/scripts/check-md-links.test.ts | 101 ++++++++++++ tests/scripts/check-mermaid.test.ts | 52 ++++++ 28 files changed, 462 insertions(+), 67 deletions(-) create mode 100644 scripts/check-md-links.d.mts create mode 100644 scripts/check-md-links.mjs create mode 100644 scripts/check-mermaid.d.mts create mode 100644 scripts/check-mermaid.mjs create mode 100644 tests/scripts/check-md-links.test.ts create mode 100644 tests/scripts/check-mermaid.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d933ccc..65ef95d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to opendrs. Versions follow [Semantic Versioning](https://se ## 0.5.0 - 2026-10-23 The verification pass (`llm.verify`) ships, measured against the field test's real corpus (49 PRs, -six repositories — [docs/experiments.md](docs/experiments.html#verifier-precision-study-on-the-field-test-corpus)). +six repositories — [docs/experiments.md](docs/experiments.md#verifier-precision-study-on-the-field-test-corpus)). The first run (v1) recalled the true-useful facts and correctly handled the noisy `coupling.missing` fires, but missed both `contract.dependency` WRONG fires and ran over the cost bar: $0.0210/PR against a $0.02 bar, with a dead `confidence` field required on every verified @@ -15,12 +15,12 @@ them before the verifier ever saw a candidate). Fixed at the root — the dead f elimination case now scored correctly — the re-run (v2) passes on all three legs: recall 1/14 wrongly dropped, `coupling.missing` 14/18 handled, both WRONG fires accounted for, and **$0.0192/PR**, under the cap. Full numbers, including the honest v1 FAIL, in -[docs/experiments.md](docs/experiments.html#verifier-precision-study-on-the-field-test-corpus). +[docs/experiments.md](docs/experiments.md#verifier-precision-study-on-the-field-test-corpus). ### Added - Verification pass (`llm.verify`, Phase 2 of the verification-pass spec, - [docs/verification.md](docs/verification.html)): with `llm.verify.enabled: true`, one batched + [docs/verification.md](docs/verification.md)): with `llm.verify.enabled: true`, one batched call to the judge's provider reads every captain fact, review-first hunk and check of the brief with fresh context and keeps, drops or downgrades each one. A drop needs a named class (docs, tests, fixtures, generated, formatting) and a cited line, else it is a downgrade; an uncited @@ -33,7 +33,7 @@ wrongly dropped, `coupling.missing` 14/18 handled, both WRONG fires accounted fo commit, `wrong`/`intentional`/`ignored` downgrade that fact on every later run without a model call, and `useful` pins it against any verifier drop or downgrade. - Pre-PR self-check (Phase 1 of the verification-pass spec, - [docs/specs/2026-10-20-verification-pass-design.md](docs/specs/2026-10-20-verification-pass-design.html)): + [docs/specs/2026-10-20-verification-pass-design.md](docs/specs/2026-10-20-verification-pass-design.md)): `opendrs brief --self-check-template` and the MCP tool `self_check_items` list the ids a pre-PR self-check should fix, explain, or (a real, unresolved problem the agent can't fix itself) flag as `needs-fix`; the Action reads a `` block from the head commit's @@ -41,13 +41,13 @@ wrongly dropped, `coupling.missing` 14/18 handled, both WRONG fires accounted fo (`author agent self-checked: N facts addressed, M explained, K flagged as still needing a fix · not part of the review decision`). Advisory only — never read by the gate or the captain's priority. See - [docs/llm-reviewers.md](docs/llm-reviewers.html#self-check-before-you-open-the-pr) and - [docs/github-action.md](docs/github-action.html#author-self-check-advisory). + [docs/llm-reviewers.md](docs/llm-reviewers.md#self-check-before-you-open-the-pr) and + [docs/github-action.md](docs/github-action.md#author-self-check-advisory). - Two new captain facts from the ApacheJIT captain-fact-lift experiment: `change.concurrency` (strong; a changed, non-comment line of non-test code uses a concurrency primitive — 2.04× bug lift) and `change.safety_removed` (info; net deleted error-handling or guard lines per hunk — 1.79× lift, but fires often enough to stay informational). See - [docs/captain.md](docs/captain.html#4-the-facts). + [docs/captain.md](docs/captain.md#4-the-facts). - The captain brief's layers detector (`layers.edge`, `layers.cycle`) and the drift detector's "outside the original modules" check now use auto-detected module boundaries instead of a flat path-segment split: package roots are found from `package.json` (JS/TS), `go.mod` directories @@ -80,8 +80,8 @@ for the module-boundaries design. either every hunk of the firing file removes lines with no identifier surviving in the partner's base text, or the firing file is a test whose source partner is untouched (spec §3.8 of - [docs/specs/2026-10-20-verification-pass-design.md](docs/specs/2026-10-20-verification-pass-design.html)). - See [docs/captain.md](docs/captain.html#4-the-facts). + [docs/specs/2026-10-20-verification-pass-design.md](docs/specs/2026-10-20-verification-pass-design.md)). + See [docs/captain.md](docs/captain.md#4-the-facts). - A repo that already sets `captain.moduleDepth` explicitly in `.opendrs.yml` keeps its exact current behaviour: setting it disables auto-detection repo-wide and module boundaries stay a flat split on that many path segments from the repo root, unchanged. diff --git a/README.md b/README.md index 7316ccd..c4d8fb5 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ default branch; see [how to auto-approve low-risk PRs](docs/auto-approve.md) and flowchart LR diff["git diff
merge-base...head"] --> signals hist["git history"] --> signals - graph["import graph"] --> signals + imports["import graph"] --> signals llm["LLM judge
(optional)"] --> signals signals["Signals
each with evidence
and provenance"] --> model["Logistic model
score = sigmoid(b + Σ w·x)"] model --> tier["Tier
green / yellow / red"] @@ -127,6 +127,8 @@ editor nor the AI review comments showed. See [the captain brief](docs/captain.m | Reviewer assignment | One of the top three experts is asked to review risky PRs | on in the Action | [Policy](docs/policy.md) | | Hotspots | Rank files by risk at rest | run on demand | [CLI and MCP](docs/cli-and-mcp.md) | | Review brief | Ranks hunks to read first, fixed check questions, safe-to-skim list, hard-rule reasons — recipes for claude-code-action, Bugbot, CodeRabbit (limited), Copilot and Codex (paste) | on | [LLM reviewers](docs/llm-reviewers.md), [CLI and MCP](docs/cli-and-mcp.md) | +| Pre-PR self-check | Lists the ids an author agent should fix or explain before opening the PR; opendrs reads the verdicts back from the head commit and adds one advisory line, never part of the gate | off | [Self-check recipe](docs/llm-reviewers.md#self-check-before-you-open-the-pr) | +| Verification pass | One batched LLM call re-checks every captain fact, review-first hunk and check with fresh context and keeps, drops or downgrades each one; never changes the score, tier or policy | off | [Verification pass](docs/verification.md) | | Incident feedback | `opendrs feedback` records incidents for calibration | run on demand | [Calibration](docs/calibration.md) | Every setting lives in `.opendrs.yml`, read from the base branch so a PR cannot loosen its own gate: @@ -167,6 +169,7 @@ Flags, exit codes, MCP tools and the agent skill: [CLI, MCP server, Claude Code | [LLM judge](docs/llm-judge.md) | turn on the optional LLM judge or the `requireSafe` gate | | [Feeding the review brief to an LLM reviewer](docs/llm-reviewers.md) | get opendrs's findings in front of Claude Code, claude-code-action, Bugbot, CodeRabbit, Copilot or Codex | | [The captain brief](docs/captain.md) | know what the final human reviewer sees: priority, reviewer, and the architecture and history facts | +| [Verification pass](docs/verification.md) | turn on an LLM re-check of the brief's facts before a human reads them | | [Calibration and backtesting](docs/calibration.md) | fit the model to your repository and choose thresholds | | [Benchmark results](docs/results.md) | see how opendrs ranks risk on public repositories | | [Experiments](docs/experiments.md) | see what we tried and what did not work | diff --git a/docs/alternatives.md b/docs/alternatives.md index 3b320d3..331c30b 100644 --- a/docs/alternatives.md +++ b/docs/alternatives.md @@ -50,11 +50,11 @@ what it measures, read [what PR blast radius is](blast-radius.md); to set it up, - **You've seen CodeRabbit's "Merge Risk" label and "Estimated code review effort":** they are useful, but CodeRabbit does not publish how either is computed. opendrs's score is calibrated on your own repository's history (`opendrs backtest`), every signal's contribution to it is shown in - the comment, and the [captain brief](captain.html) names the actual facts behind a verdict — a + the comment, and the [captain brief](captain.md) names the actual facts behind a verdict — a new import edge, a broken contract, a file that usually changes with one this PR didn't — instead of a label alone. It is also open source, with no service to host or account to create. - **You want the single highest accuracy number, and can host a model:** DRS-OSS reports AUC 0.895 on ApacheJIT's official split; opendrs's calibrated model reports AUC 0.823 on the same benchmark - (a different split; see [Calibration and backtesting](calibration.html)). That gap is the + (a different split; see [Calibration and backtesting](calibration.md)). That gap is the trade-off: DRS-OSS needs a fine-tuned Llama 3.1 8B classifier running behind a REST API you host and keep up; opendrs runs as a CLI, GitHub Action or MCP server with no model service to deploy. diff --git a/docs/auto-approve.md b/docs/auto-approve.md index 062f731..68b865b 100644 --- a/docs/auto-approve.md +++ b/docs/auto-approve.md @@ -12,7 +12,7 @@ up a gate that approves the low-risk ones on its own and sends everything else t > A PR whose risk score is below 0.3, that changes nothing on a blocked path, and whose AI review > passed is approved and goes to the merge queue. Everything else waits for a reviewer. -The score comes from [opendrs](./), which measures each PR's [blast radius](blast-radius.html), +The score comes from [opendrs](./), which measures each PR's [blast radius](blast-radius.md), file history, author experience, size and tests. ## 1. Score every PR without approving anything @@ -57,7 +57,7 @@ npx @error-path/opendrs backtest # compare default and calibrated The backtest reports what share of bug-introducing commits the score would have flagged at each threshold. Commit the calibrated model through a normal reviewed PR. See -[Calibration and backtesting](calibration.html) for how to read the numbers. +[Calibration and backtesting](calibration.md) for how to read the numbers. ## 3. Turn on the branch rules approval depends on @@ -129,4 +129,4 @@ Whatever the score: - PRs from forks, drafts, and PRs where a reviewer requested changes; - PRs where a signal could not be computed and that unknown could have changed the tier. -The full rule order and every option are in the [GitHub Action guide](github-action.html). +The full rule order and every option are in the [GitHub Action guide](github-action.md). diff --git a/docs/blast-radius.md b/docs/blast-radius.md index 9a1250c..5b2fdaf 100644 --- a/docs/blast-radius.md +++ b/docs/blast-radius.md @@ -35,7 +35,7 @@ dependents count from the graph. The dependents count drives `blast.dependents` and `blast.public_surface` in the score above, and it also shows up in the AI-facing review brief's hunk reasons. It reaches the -[captain brief](captain.html) only once: as the "imported by N files" evidence on a +[captain brief](captain.md) only once: as the "imported by N files" evidence on a `contract.api` fact, when a deleted public declaration has dependents. The captain brief does not otherwise repeat a raw dependents count — that is editor trivia the captain can see in one click. @@ -56,7 +56,7 @@ A logistic model turns the signals into a probability between 0 and 1. Each sign is shown in the PR comment, so you can see why a PR scored what it did. The default weights are priors; `opendrs calibrate` fits them to your repository's history. The full feature table and its link to Meta's Diff Risk Score are in the -[signal reference](signals.html#metas-drs-features-and-their-opendrs-signals). +[signal reference](signals.md#metas-drs-features-and-their-opendrs-signals). ## Check a change's blast radius yourself @@ -68,4 +68,4 @@ npx @error-path/opendrs explain src/auth.ts # one file's dependents, history an Coding agents can call the same checks through the MCP server (`score_diff`, `explain_file_risk`, `suggest_reviewers`) before they open a PR, and split a change whose blast radius is large. To use the score as an approval gate, see -[Auto-approve low-risk pull requests](auto-approve.html). +[Auto-approve low-risk pull requests](auto-approve.md). diff --git a/docs/calibration.md b/docs/calibration.md index e8f13cc..ba20973 100644 --- a/docs/calibration.md +++ b/docs/calibration.md @@ -139,7 +139,7 @@ labelled positive. close, and this benchmark is how to measure it. Every experiment behind these numbers, including the ones that did not help, is in the -[experiment log](experiments.html). +[experiment log](experiments.md). The full report, with per-project numbers, coverage and caveats, is in [`bench/apachejit/results.md`](https://github.com/error-path/opendrs/blob/main/bench/apachejit/results.md); [`bench/apachejit/README.md`](https://github.com/error-path/opendrs/blob/main/bench/apachejit/README.md) diff --git a/docs/captain.md b/docs/captain.md index d2c6344..60bd3f4 100644 --- a/docs/captain.md +++ b/docs/captain.md @@ -202,7 +202,7 @@ A quick confirm with no facts is one line and no block: `**Captain: quick confir facts (not capped at five) is also in the "For the captain" section of the brief for AI reviewers, and in the `captain` Action output described below. -With the [verification pass](verification.html) on, each fact in the comment and in the brief ends +With the [verification pass](verification.md) on, each fact in the comment and in the brief ends with its verdict (`[verified · new]`, `[verified · pre-existing]`, `[downgraded by verifier: …]`, `[unverified]`), a dropped fact becomes a `dropped by verifier` note in the brief, and the "why" and the five facts come from the verified list. The priority does not: it is computed before the @@ -232,7 +232,7 @@ captain: ``` As with the rest of `.opendrs.yml`, this is read from the base commit, so a PR cannot switch its -own captain brief off. See the [configuration reference](configuration.html) for the full file. +own captain brief off. See the [configuration reference](configuration.md) for the full file. ## 8. What is measured, and what is not @@ -245,7 +245,7 @@ own captain brief off. See the [configuration reference](configuration.html) for split only. That ApacheJIT lift measures whether the file *was buggy*, not whether a reviewer would have cared; measured against real review comments and fixes instead (the outcome-labels study, §4 above), it did not clear its own bar and was demoted to `info`-only with a narrower - trigger, as recorded there. Details: [Experiments](experiments.html). + trigger, as recorded there. Details: [Experiments](experiments.md). - **Contracts and change risk** (`contract.dependency`, `contract.api`, `contract.schema`, `change.concurrency`, `change.safety_removed`) cleared the same pre-registered bug-lift bar (≥ 1.2×, ≥ 50 firing test commits) on ApacheJIT's test split (24,690 commits, pooled): @@ -255,7 +255,7 @@ own captain brief off. See the [configuration reference](configuration.html) for (firing on 21 % of test commits — too common to weight `strong` without pushing most PRs to "careful review", hence `info`). The docs may say these predict bugs; the facts still ship regardless of the number, as decisions a captain signs off on either way. Details: - [Experiments](experiments.html#captain-fact-lift-on-apachejit). + [Experiments](experiments.md#captain-fact-lift-on-apachejit). - **Layers** (`layers.edge`, `layers.cycle`) is not shown to predict bugs: `layers.edge` lifted only 1.12× on a 2,000-commit sample (19 firings), short of the bar. It still ships — a new import edge is a decision a captain signs off on, not a claim that it causes incidents. diff --git a/docs/cli-and-mcp.md b/docs/cli-and-mcp.md index fcf6f6f..a67ee3f 100644 --- a/docs/cli-and-mcp.md +++ b/docs/cli-and-mcp.md @@ -47,7 +47,7 @@ Every tool also takes `repoDir`, the absolute path to the git repository (any di |---|---|---| | `score_diff` | `base` (base ref to diff against; defaults to `origin/HEAD`, or `main` when that is not set), `head` (head ref to score; defaults to `HEAD`) | The Diff Risk Score (0-1, calibrated probability that the change causes an incident) for the changes between the two refs: the score, tier (green/yellow/red), the gate decision and its reasons, the top contributing signals with evidence, signals that could not be computed (unknowns), suggested reviewers, and the LLM judge's rationale when enabled. Use it before opening a pull request, or to decide whether a change should be split. | | `review_brief` | `base`, `head` (as `score_diff`), `maxTokens` (override the token budget) | The review brief as Markdown text content and structured `ReviewBrief` JSON. Call it before reviewing a diff. | -| `self_check_items` | `base`, `head`, `maxTokens` (as `review_brief`) | The ids (captain facts, reviewFirst hunks, check questions) a pre-PR self-check should fix or explain, as `{ items: [{ id, file, line?, text }] }`. Write verdicts into the head commit's message as described in [docs/llm-reviewers.md](llm-reviewers.html#self-check-before-you-open-the-pr); advisory only, never part of the gate. | +| `self_check_items` | `base`, `head`, `maxTokens` (as `review_brief`) | The ids (captain facts, reviewFirst hunks, check questions) a pre-PR self-check should fix or explain, as `{ items: [{ id, file, line?, text }] }`. Write verdicts into the head commit's message as described in [docs/llm-reviewers.md](llm-reviewers.md#self-check-before-you-open-the-pr); advisory only, never part of the gate. | | `explain_file_risk` | `path` (repo-relative file path with forward slashes, e.g. `src/auth/session.ts`) | The standing risk profile of one file at HEAD: commit and bug-fix history, number of authors, direct and transitive dependents (blast radius; null when the language is not supported), whether it is on a critical or blocked path, and its top authors. Use it before editing a file you do not know, or to explain a high `blast.dependents` contribution. | | `list_hotspots` | `top` (how many files to return; defaults to 20) | The files tracked at HEAD ranked by code-at-rest risk: bug-fix commits, churn, dependents and critical paths in the history lookback window (authors shown, not scored), as `{ hotspots, warnings }`, with transitive dependents in the import graph (null when the language is not supported) and critical or blocked path flags. Use it to find where a codebase is most likely to break, or to pick files for reengineering or extra test coverage. | | `suggest_reviewers` | `paths` (repo-relative file paths the change touches; at least one) | Up to three reviewers (author emails) for the files, ranked by how often they recently changed them in git history. Use it to pick human reviewers for a risky change. | diff --git a/docs/configuration.md b/docs/configuration.md index 337b8c2..17f6680 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -17,7 +17,7 @@ description: "Every key in .opendrs.yml with its default: thresholds, auto-appro | `graph` | The import graph: on or off, depth and file cap | | `model` | Where `opendrs calibrate` writes the fitted model; priors are used until it exists | | `brief` | Token budget for the review brief's Markdown | -| `captain` | The [captain brief](captain.html): the module map for the layers detector (auto-detected by default, or a flat depth / literal globs), the change-coupling detector's thresholds, and extra schema/contract globs | +| `captain` | The [captain brief](captain.md): the module map for the layers detector (auto-detected by default, or a flat depth / literal globs), the change-coupling detector's thresholds, and extra schema/contract globs | | `content` | JS regexes matched against each added line (see [What opendrs refuses](#what-opendrs-refuses)) | | `freezes` | Windows in which auto-approval is suspended or held to a stricter green (see [What opendrs refuses](#what-opendrs-refuses)) | | `sources` | Per-source policy for agent, bot and human PRs; every setting only tightens the gate | @@ -201,7 +201,7 @@ The default `history.fixPattern` counts any commit subject matching `history.prior_fixes`. On a repository that uses Conventional Commits heavily, this also counts routine housekeeping — `fix(deps): update dependencies`, `chore(deps)`, `fix(ci)`, `fix(docs)`, `fix: typo` — as if it were a real bug fix. The [score-saturation -experiment](experiments.html#score-saturation-fixpattern-and-incident-granularity-on-apachejit) +experiment](experiments.md#score-saturation-fixpattern-and-incident-granularity-on-apachejit) measured this on `vitejs/vite`: excluding those commits dropped the share matched by `fixPattern` by 21.6 %. It was not significant pooled across ApacheJIT's mostly pre-Conventional- Commits Java projects, so it is not a new default — but it is worth setting per repo if you use the diff --git a/docs/experiments.md b/docs/experiments.md index 611e218..3b0e03c 100644 --- a/docs/experiments.md +++ b/docs/experiments.md @@ -13,7 +13,7 @@ Unless stated otherwise, ApacheJIT numbers are on the dataset's official split: test 2017–2019. They are pooled over the held-out test commits, excluding `apache/hadoop`, which has no positive labels: 24,686 commits, 5,818 positive. "Recall@X %" is the share of buggy commits in the riskiest X % of commits. How to read the metrics is in -[Calibration and backtesting](calibration.html#reading-the-metrics). +[Calibration and backtesting](calibration.md#reading-the-metrics). | Date | Question | Result | Verdict | Details | |---|---|---|---|---| @@ -912,7 +912,7 @@ was adopted separately as a correctness fix, not a saturation fix: folder-level unrelated files (confirmed on vite, where one old revert flagged hundreds of files under `packages/vite/src/node/`), and the change carries no accuracy cost (AUC +0.0009, CI 0.0005 to 0.0014). V1 (the Conventional-Commits `fixPattern` exclusion) is documented as a per-repo override -in [`docs/configuration.md`](configuration.html#repositories-that-use-conventional-commits) rather +in [`docs/configuration.md`](configuration.md#repositories-that-use-conventional-commits) rather than shipped as a new default. ## Captain-fact lift on ApacheJIT @@ -1106,7 +1106,7 @@ before any spend (see Amendments, below). No API spend yet: paid LLM tests are p user approves Phase 1. The bench wiring for arm 2 (spec §13 phase 4) is **deferred by decision**: it is not built until the user approves the paid experiment. The full plan, with the prompt drafts, the context block, the literature and the power analysis, is in -[the research note, Part 5](research/2026-10-18-review-risk-heuristics.html#part-5-pre-registered-plan-for-the-next-llm-judge-experiment). +[the research note, Part 5](research/2026-10-18-review-risk-heuristics.md#part-5-pre-registered-plan-for-the-next-llm-judge-experiment). **Question.** Does an LLM reviewer plus opendrs beat the same LLM alone, by enough to justify any extra tokens? It is not opendrs against the LLM. The three headline arms run on the same commits: @@ -1198,9 +1198,9 @@ review of every commit. 1's mean input tokens, and the pilot stop rule from +20 % to +35 %. The budget is recomputed to ≈ $38.6 of ≤ $43. Reason: the product decided its default brief size; the experiment must measure the product as shipped. Full detail in - [the research note, Amendments](research/2026-10-18-review-risk-heuristics.html#amendments). + [the research note, Amendments](research/2026-10-18-review-risk-heuristics.md#amendments). -**Where.** The plan is in [the research note, Part 5](research/2026-10-18-review-risk-heuristics.html#part-5-pre-registered-plan-for-the-next-llm-judge-experiment). +**Where.** The plan is in [the research note, Part 5](research/2026-10-18-review-risk-heuristics.md#part-5-pre-registered-plan-for-the-next-llm-judge-experiment). The free simulation is `bench/apachejit/judge-sim.ts`. The cached judgements are those of the [LLM judge run](#structured-llm-judge-on-sampled-apachejit). @@ -1260,14 +1260,14 @@ Pooled, opendrs now matches the Kamei LR within noise. It does not beat it. **Where.** Commits `c14da1c` (fit experiment), `ad0d368` (fix), `35862bc` (regenerated benchmark). - [`bench/apachejit/fit-experiment.md`](https://github.com/error-path/opendrs/blob/main/bench/apachejit/fit-experiment.md) and `bench/apachejit/fit-experiment.ts`. - [`bench/apachejit/results.md`](https://github.com/error-path/opendrs/blob/main/bench/apachejit/results.md). -- [Calibration and backtesting](calibration.html#what-calibrate-fits-on). +- [Calibration and backtesting](calibration.md#what-calibrate-fits-on). ## Review-heuristic signals **Question.** AI code reviewers check for concurrency changes, removed error handling or guards, and weakened tests. Do these, computed deterministically from the diff, add ranking signal? The background is in the -[research note on how AI code reviewers judge PR risk](research/2026-10-18-review-risk-heuristics.html), +[research note on how AI code reviewers judge PR risk](research/2026-10-18-review-risk-heuristics.md), recommendations R4–R6. **Setup.** @@ -1424,7 +1424,7 @@ failed commits were judged again with the same model and prompt. **Where.** Commit `a9f4882` (structured outputs), `59e07e3` (results); [`bench/apachejit/results-llm.md`](https://github.com/error-path/opendrs/blob/main/bench/apachejit/results-llm.md); -[Calibration and backtesting](calibration.html#with-the-structured-llm-judge). +[Calibration and backtesting](calibration.md#with-the-structured-llm-judge). ## Structured LLM judge on sampled ApacheJIT @@ -1469,14 +1469,14 @@ The test sample's base rate is 25.5 % buggy. - The gate is the win: the commits it passes are buggy at a fifth of the base rate. - The ranking gain (about +0.02 AUC) is within noise on 800 commits. - `logic_bug` and `structural` each flag about half the commits, which is why the categories rank - poorly. The [research note](research/2026-10-18-review-risk-heuristics.html) proposes prompt + poorly. The [research note](research/2026-10-18-review-risk-heuristics.md) proposes prompt changes to fix that. - This sample was fitted before the refit in [fit experiment](#fit-experiment-and-the-calibrate-refit) and has not been re-run. **Where.** Commits `2c9e452` and `59e07e3`; [`bench/apachejit/results-llm.md`](https://github.com/error-path/opendrs/blob/main/bench/apachejit/results-llm.md); -[LLM judge](llm-judge.html). +[LLM judge](llm-judge.md). ## ApacheJIT benchmark baseline @@ -1510,7 +1510,7 @@ would have to close. **Where.** Commit `50060c5`; `bench/apachejit/run.ts`; [`bench/apachejit/results.md`](https://github.com/error-path/opendrs/blob/main/bench/apachejit/results.md); -[Calibration and backtesting](calibration.html#results-on-apachejit). +[Calibration and backtesting](calibration.md#results-on-apachejit). ## Public-repository backtests @@ -1538,4 +1538,4 @@ default priors and after `calibrate`, with nothing configured? terraform and eslint) because their bugs had not been found yet. **Where.** Commits `1e557cd` (nine repositories) and `5442193` (18 repositories); -[Benchmark results](results.html). +[Benchmark results](results.md). diff --git a/docs/github-action.md b/docs/github-action.md index b034cb4..305e2a2 100644 --- a/docs/github-action.md +++ b/docs/github-action.md @@ -78,7 +78,7 @@ opendrs checks out and reads the PR's code. opendrs scores on events that carry | `score` | 0 to 1, four decimals | | `tier` | `green`, `yellow`, `red` | | `decision` | `auto-approve`, `deferred-review`, `waiting-ai-review`, `human-review` (the policy decision, even when approval is disabled or a landing delay is still running) | -| `captain` | `careful`, `standard` or `quick`: how much attention the captain brief says this PR needs. Empty when `captain.enabled` is false or `brief: false`. See [the captain brief](captain.html). | +| `captain` | `careful`, `standard` or `quick`: how much attention the captain brief says this PR needs. Empty when `captain.enabled` is false or `brief: false`. See [the captain brief](captain.md). | | `brief` | The review brief as Markdown (empty when `brief: false`) | | `brief-file` | Path to the review brief as JSON, under `$RUNNER_TEMP` (empty when `brief: false`) | @@ -314,7 +314,7 @@ such as `claude-review`, not `review` or `test`. ## Author self-check (advisory) -When the PR author (human or agent) followed [the self-check recipe](llm-reviewers.html#self-check-before-you-open-the-pr) +When the PR author (human or agent) followed [the self-check recipe](llm-reviewers.md#self-check-before-you-open-the-pr) and left a `` block in the head commit's own message, opendrs reads it from that exact commit, rebuilds the brief fresh as if the record did not exist, and adds one line next to the captain line: @@ -395,7 +395,7 @@ On a sweep the `score`, `tier` and `decision` outputs are those of the last PR s Every run that updates the comment also writes a hidden machine marker into it, for example ``, recording this run's result. `first=` is the head of opendrs's first run on this PR and `firstRun=` -that run's number, which the [captain brief](captain.html)'s drift detector compares against: both +that run's number, which the [captain brief](captain.md)'s drift detector compares against: both are written from a run's own head and number on the first run, and kept unchanged on every later run, so the drift text keeps saying "since run `firstRun`" instead of resetting to "run 1" once a later run writes the marker. They are optional only when reading, so a marker written by 0.3.x (no diff --git a/docs/limitations.md b/docs/limitations.md index b419bc5..ea1f9e0 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -6,8 +6,8 @@ description: "What opendrs cannot see or measure: unfit default weights, noisy i # Known limitations opendrs estimates incident risk from git history and a diff, and these are the places where that -estimate is weaker than it looks, grouped by the model, the [signals](signals.html), the -[LLM judge](llm-judge.html) and the GitHub Action. +estimate is weaker than it looks, grouped by the model, the [signals](signals.md), the +[LLM judge](llm-judge.md) and the GitHub Action. ## The model and its labels @@ -42,7 +42,7 @@ estimate is weaker than it looks, grouped by the model, the [signals](signals.ht ## The GitHub Action - **Required checks are matched by name.** opendrs does not verify which app produced a check run, - so protect the names you list (see the [GitHub Action guide](github-action.html)). + so protect the names you list (see the [GitHub Action guide](github-action.md)). - **GitHub only.** The core is host-agnostic, but only a GitHub Action exists today. The Action needs full history (`fetch-depth: 0`). In a shallow clone, `diffusion.authors` and `history.*` are reported as `unknown` ("shallow clone: history incomplete"), keeping what they saw as a lower diff --git a/docs/llm-judge.md b/docs/llm-judge.md index 85947be..be116dc 100644 --- a/docs/llm-judge.md +++ b/docs/llm-judge.md @@ -53,12 +53,12 @@ For OpenAI or any OpenAI-compatible server (Azure OpenAI, Ollama, vLLM), set `pr provider you configure. If the call fails, the signal is `unknown`, and the PR cannot be auto-approved on that run. -The same provider can also run the optional [verification pass](verification.html) +The same provider can also run the optional [verification pass](verification.md) (`llm.verify.enabled: true`): a second, batched call that reads the review brief's facts, hunks and checks with fresh context and keeps, drops or downgrades each one. Unlike the judge, it never enters the score: it changes what the brief shows, never the gate or the captain priority. The `llm.semantic` and `llm.risk.*` signals sit alongside the git-derived ones listed on -[Risk signals: what opendrs measures](signals.html). The other rules that apply before a PR is -auto-approved are on [Auto-approve low-risk pull requests](auto-approve.html), and the full +[Risk signals: what opendrs measures](signals.md). The other rules that apply before a PR is +auto-approved are on [Auto-approve low-risk pull requests](auto-approve.md), and the full project is at the [README](https://github.com/error-path/opendrs). diff --git a/docs/llm-reviewers.md b/docs/llm-reviewers.md index d2e093e..422adc4 100644 --- a/docs/llm-reviewers.md +++ b/docs/llm-reviewers.md @@ -48,7 +48,7 @@ message** — not a separate commit, not the PR description: This is advisory only, never part of opendrs's auto-approve gate or the captain's priority. opendrs recomputes the brief from scratch on the exact head commit and only shows where your record and the fresh facts agree, disagree, or say nothing; a claimed fix that is still present in the diff is -called out, not hidden. See [docs/github-action.md](github-action.html#author-self-check-advisory) +called out, not hidden. See [docs/github-action.md](github-action.md#author-self-check-advisory) for what the Action renders. ## anthropics/claude-code-action @@ -172,12 +172,12 @@ The brief's `## For the captain` section carries the same priority (careful, sta facts as the PR comment's captain line, but with every fact, not just the top five, plus any detector notes. It is a mandatory section: `fitBudget` never trims it, whatever the token budget. Whichever recipe above you use to get the brief in front of an LLM reviewer, this section comes -along with it. See [the captain brief](captain.html) for what each fact means and an example of +along with it. See [the captain brief](captain.md) for what each fact means and an example of each. ### Reading verified facts -With the [verification pass](verification.html) on (`llm.verify.enabled: true`), every fact, +With the [verification pass](verification.md) on (`llm.verify.enabled: true`), every fact, review-first hunk and check carries one suffix, and an agent reading the brief should treat them differently: diff --git a/docs/maintaining.md b/docs/maintaining.md index 1f86431..e9ddac6 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -70,7 +70,7 @@ it, replace both. To preview locally: `cd docs && bundle install && bundle exec A shared, gitignored, append-only `runs.jsonl` is a record of how long `check`, `test`, `test:slow`, `build:action`, `claude plugin eval` and any other repeatable command take, so "is the dev loop getting slower" has a measured answer instead of a guess. Background: see -[Where the dev loop's time goes]({{ '/research/2026-10-22-dev-loop-time-sinks.html' | relative_url }}). +[Where the dev loop's time goes](research/2026-10-22-dev-loop-time-sinks.md). **Where it lives:** `scripts/metrics-path.mjs` resolves the file the same way for every script, in this order: the `OPENDRS_METRICS_FILE` env var if set; otherwise diff --git a/docs/policy.md b/docs/policy.md index dcd7c00..8736b72 100644 --- a/docs/policy.md +++ b/docs/policy.md @@ -19,13 +19,13 @@ The decision rules run in this order; the first that matches wins: `human-review`; 3. more than `maxChangedLines` *counted* lines changed (docs, comment-only hunks and `autoApprove.sizeCapExempt` paths do not count by default; see - [Configuration](configuration.html#inert-lines-and-the-size-cap)) → `human-review`; + [Configuration](configuration.md#inert-lines-and-the-size-cap)) → `human-review`; 4. tier is not green (unless `autoApprove.deferredReview` is on and the score is in its band), or an active freeze window suspends auto-approval or holds it to a stricter green, or the PR's source (human, bot or AI agent) has its own stricter green, has hit its daily cap, or lacks the required track record → `human-review`; -5. the [LLM judge](llm-judge.html) failed, or `llm.requireSafe` is on and the judge did not call - every hunk safe with enough confidence, or the `unknown` [signals](signals.html), all at their +5. the [LLM judge](llm-judge.md) failed, or `llm.requireSafe` is on and the judge did not call + every hunk safe with enough confidence, or the `unknown` [signals](signals.md), all at their worst at once, could have changed the tier → `human-review`; 6. the PR is a draft, or a reviewer's latest review requests changes → `human-review`; 7. a required AI-review check has not passed on the head commit → `waiting-ai-review`; @@ -48,7 +48,7 @@ The rails that matter when the Action can approve PRs: | Rail | What it does | |---|---| | **Config comes from the base branch.** | The Action reads `.opendrs.yml` and the calibrated model from the PR's base commit, so a PR cannot loosen its own gate. | -| **Hard blocks.** | `paths.block` (by default `.github/**`, `.opendrs.yml`, `.opendrs/**`, `CODEOWNERS`, migrations, Terraform, `.gitmodules`, `.gitattributes`, and files that execute code on install, build or CI; see [Configuration](configuration.html)) always sends the PR to a human. | +| **Hard blocks.** | `paths.block` (by default `.github/**`, `.opendrs.yml`, `.opendrs/**`, `CODEOWNERS`, migrations, Terraform, `.gitmodules`, `.gitattributes`, and files that execute code on install, build or CI; see [Configuration](configuration.md)) always sends the PR to a human. | | **Binary files cannot pass as text.** | Git's own test only looks for a NUL in the first 8000 bytes and obeys `.gitattributes`, so opendrs also reads every changed file at the PR head and treats it as binary when it holds a NUL anywhere or is not valid UTF-8. With git 2.40 or later the diff reads attributes from the base commit (`--attr-source`), so the PR's own `.gitattributes` has no effect. Older git ignores only the global and system attribute files and still reads the ones in the checkout; the content check above still catches binaries there. | | **Submodules.** | Moving a submodule pointer pulls in code the diff does not show, so any gitlink change goes to a human, whatever `diff.submodule`, `diff.ignoreSubmodules` or a submodule's `ignore` setting says. Repointing a submodule's URL changes `.gitmodules`, which is blocked. | | **Self-modification guard.** | Independently of any config, the Action never approves a PR that adds, changes, deletes or renames `.opendrs.yml`, `.opendrs/**`, `.github/**`, `action.yml`, `action/**`, `.gitmodules` or any `.gitattributes`, or the configured model file. It also never approves PRs from forks. | @@ -58,8 +58,8 @@ The rails that matter when the Action can approve PRs: | **Freeze windows.** | Meta gates more diffs on weekends and around peak events, when fewer people can respond to an incident. A `freezes` window (weekdays and/or dates in a timezone) suspends auto-approval, or with `green` holds it to a stricter threshold. Tiers and labels do not change. | | **Per-source policy.** | Like RADAR, opendrs classifies each PR as written by a person, a bot account (Dependabot, Renovate) or an AI coding agent (Claude Code, GitHub Copilot's coding agent, Cursor, Codex, Devin, Jules, Aider, or your own `sources.agent.patterns`). It reads commit trailers, author emails and, in CI, the GitHub account type. The comment shows the source and why. `sources.` can set a stricter `green`, `requireSafe` (the LLM gate for that source only), a `dailyCap` on that source's commits landed in the last 24 hours, and a `trackRecord` (by default: at least 10 landed commits in 60 days, at most 5 % of them reverted). These settings can only tighten the gate. Trailers are written by the PR author's tools, so a source must never earn a looser threshold than a person gets. | | **Deferred review.** | RADAR lets some low-risk human diffs land first and be reviewed after. With `autoApprove.deferredReview` (`enabled`, `below`, `sources`, default `[human]`), a PR whose score is between `green` and `below` and that passes every other rule gets `deferred-review`: the Action approves it and lets it merge, requests a post-merge review from the suggested reviewers, and labels it `opendrs:review-after-merge`. Only the tier rule is waived, and only in that band. | -| **Landing delay.** | With `autoApprove.landingDelayMinutes` (or a longer `sources..landingDelayMinutes`), an eligible PR waits that long, on the same head commit, before opendrs approves it. Meanwhile it carries the label `opendrs:landing`, and anyone can veto by requesting changes, pushing a commit (which restarts the clock) or adding `opendrs:hold`. A scheduled run of the Action re-checks waiting PRs; see the [GitHub Action guide](github-action.html#deferred-review-and-landing-delay). | +| **Landing delay.** | With `autoApprove.landingDelayMinutes` (or a longer `sources..landingDelayMinutes`), an eligible PR waits that long, on the same head commit, before opendrs approves it. Meanwhile it carries the label `opendrs:landing`, and anyone can veto by requesting changes, pushing a commit (which restarts the clock) or adding `opendrs:hold`. A scheduled run of the Action re-checks waiting PRs; see the [GitHub Action guide](github-action.md#deferred-review-and-landing-delay). | | **Content blocklist.** | `content.block` regexes are checked against every added line, like Meta's RADAR blocklisted code phrases. A match goes to a human, and the reason names `file:line`. | | **Agent-instruction files always count.** | `paths.instructions` (by default `CLAUDE.md`, `AGENTS.md`, `GEMINI.md`, `.claude/**`, `.cursor/**`, `SKILL.md`, `*.prompt.md`, `prompts/**`, plus the already-blocked `.github/**`/`.opendrs/**`) is never excluded from the size cap, whatever its extension — additive only, with no `unblock`-style removal. | -| **Stale approvals are blocked by branch rules.** | The Action approves only if the base branch requires at least one approval and dismisses stale approvals on push. Without that rule, an approval for one commit could carry over to a later, riskier push. See the [GitHub Action guide](github-action.html#repository-settings-for-auto-approval) for the ruleset. | +| **Stale approvals are blocked by branch rules.** | The Action approves only if the base branch requires at least one approval and dismisses stale approvals on push. Without that rule, an approval for one commit could carry over to a later, riskier push. See the [GitHub Action guide](github-action.md#repository-settings-for-auto-approval) for the ruleset. | | **Prompt injection.** | The diff, PR title and PR body are untrusted. The LLM prompt wraps them in delimiters and tells the model to ignore instructions inside them; the PR body is not sent at all unless `llm.includeDescription` is true. The answer must be exactly one JSON object and is clamped to [0, 1]. The judge can only raise the score, never lower it, so talking it into "risk 0" gains nothing. Risk claims written in the PR text never change the score directly. | diff --git a/docs/research/2026-10-22-dev-loop-time-sinks.md b/docs/research/2026-10-22-dev-loop-time-sinks.md index 7ebe8a7..105882a 100644 --- a/docs/research/2026-10-22-dev-loop-time-sinks.md +++ b/docs/research/2026-10-22-dev-loop-time-sinks.md @@ -155,5 +155,5 @@ To make this kind of audit a five-minute `node scripts/metrics-report.mjs` inste transcript-mining exercise next time, every `check`/`test`/`test:slow`/`build:action` run now appends one line to the gitignored `.dev-metrics/runs.jsonl`, and `claude plugin eval` runs can be appended the same way from their `aggregate-result.json`. See -[Maintaining opendrs]({{ '/maintaining.html' | relative_url }}#dev-loop-metrics). The numbers in +[Maintaining opendrs](../maintaining.md#dev-loop-metrics). The numbers in this note are seeded into it with `"source": "backfill"`. diff --git a/docs/research/index.md b/docs/research/index.md index fabc8c7..d86549f 100644 --- a/docs/research/index.md +++ b/docs/research/index.md @@ -6,7 +6,7 @@ description: "Dated research notes behind opendrs: field tests on real repositor # Research notes Dated notes from building opendrs: what was tested, on which repositories, and what changed as a -result. Each note lists its sources. The [experiment log]({{ '/experiments.html' | relative_url }}) +result. Each note lists its sources. The [experiment log](../experiments.md) has the benchmark runs behind them. {% assign notes = site.pages | where_exp: "p", "p.dir == '/research/'" | where_exp: "p", "p.date" | where_exp: "p", "p.sitemap != false" | sort: "date" | reverse %} diff --git a/docs/results.md b/docs/results.md index fe09722..b872248 100644 --- a/docs/results.md +++ b/docs/results.md @@ -9,7 +9,7 @@ These are the numbers behind opendrs's claims: `opendrs backtest` replayed on op ## The nine-repository run -On the public [ApacheJIT](calibration.html#results-on-apachejit) benchmark (Apache Java +On the public [ApacheJIT](calibration.md#results-on-apachejit) benchmark (Apache Java projects, official 2017–2019 test years, `apache/hadoop` left out because it has no positive labels), calibrated opendrs reaches AUC 0.823 and catches 67 % of buggy commits in its riskiest 30 %, on par with a regression on the classic JIT metrics (0.821, 66 %). @@ -98,4 +98,4 @@ Runs took 1 to 16 minutes per repository on a 4-core Linux machine. ## ApacheJIT -On the public ApacheJIT benchmark, calibrated opendrs is compared with a logistic regression on the classic JIT metrics and with the published DRS-OSS numbers, pooled and per project, with and without the structured LLM judge. Those tables, and the caveats on what the split and labels let you read from them, are in [Calibration and backtesting: Results on ApacheJIT](calibration.html#results-on-apachejit). +On the public ApacheJIT benchmark, calibrated opendrs is compared with a logistic regression on the classic JIT metrics and with the published DRS-OSS numbers, pooled and per project, with and without the structured LLM judge. Those tables, and the caveats on what the split and labels let you read from them, are in [Calibration and backtesting: Results on ApacheJIT](calibration.md#results-on-apachejit). diff --git a/docs/signals.md b/docs/signals.md index 06eb4ef..adf4c1d 100644 --- a/docs/signals.md +++ b/docs/signals.md @@ -37,14 +37,14 @@ Meta's baseline is a logistic regression over about a dozen features, chosen fro The import graph uses regex-based extractors for TypeScript/JavaScript, Python, Go, Java/Kotlin, Rust and Ruby. Files in other languages get `unknown` blast-radius signals, which the policy treats as risk, not as zero. How the blast-radius signals are measured is explained in -[What is the blast radius of a pull request?](blast-radius.html). +[What is the blast radius of a pull request?](blast-radius.md). ## LLM signals The `llm.semantic` and `llm.risk.*` signals come from the optional LLM judge, which can only raise a score, never lower it. How it is configured, how it classifies hunks and how it can gate -auto-approval are on [LLM judge](llm-judge.html). +auto-approval are on [LLM judge](llm-judge.md). The signals are combined by a logistic model whose weights `opendrs calibrate` fits to your -repository's history; see [Calibration and backtesting](calibration.html) and the +repository's history; see [Calibration and backtesting](calibration.md) and the [README](https://github.com/error-path/opendrs). diff --git a/docs/specs/2026-10-21-seo-long-term-design.md b/docs/specs/2026-10-21-seo-long-term-design.md index af7ffda..b11388d 100644 --- a/docs/specs/2026-10-21-seo-long-term-design.md +++ b/docs/specs/2026-10-21-seo-long-term-design.md @@ -153,7 +153,7 @@ Anything else is fixed before the switch. > opendrs is maintained by error-path and released under the MIT license. The code, these docs > and the research notes are written with AI assistance (Claude) and reviewed by the maintainer > before they are merged. Every benchmark number on this site comes from a script in the - > repository, and [Results](results.html) shows how to reproduce each one. Corrections are + > repository, and [Results](../results.md) shows how to reproduce each one. Corrections are > welcome as [GitHub issues](https://github.com/error-path/opendrs/issues). `/about.html` does not match jekyll-seo-tag's homepage-or-about pattern (`/about/`), so it gets diff --git a/docs/verification.md b/docs/verification.md index 88ad072..14a96df 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -15,7 +15,7 @@ captain priority. This is Phase 2 of the [verification-pass spec](https://github.com/error-path/opendrs/blob/main/docs/specs/2026-10-20-verification-pass-design.md#3-phase-2--the-reviewer-side-verification-pass-llmverify); -Phase 1 is the author's [self-check](llm-reviewers.html#self-check-before-you-open-the-pr). +Phase 1 is the author's [self-check](llm-reviewers.md#self-check-before-you-open-the-pr). ## What it verifies @@ -130,7 +130,7 @@ llm: maxTokens: 40000 # estimated prompt tokens (chars / 3.5) above which the call is skipped ``` -The pass needs `llm.enabled: true`; it runs on the same provider as the [judge](llm-judge.html), +The pass needs `llm.enabled: true`; it runs on the same provider as the [judge](llm-judge.md), and `--no-llm` disables both. In the Action, the key comes from the `llm-api-key` input as for the judge. diff --git a/scripts/check-md-links.d.mts b/scripts/check-md-links.d.mts new file mode 100644 index 0000000..8c3fd84 --- /dev/null +++ b/scripts/check-md-links.d.mts @@ -0,0 +1,7 @@ +export const TARGETS: string[]; +export function listDocFiles(root: string): string[]; +export function slugify(heading: string): string; +export function headingSlugs(markdown: string): Set; +export function findLinks(markdown: string): { line: number; url: string }[]; +export function checkFile(file: string, slugCache: Map | null>): string[]; +export function checkAll(root: string): string[]; diff --git a/scripts/check-md-links.mjs b/scripts/check-md-links.mjs new file mode 100644 index 0000000..1039035 --- /dev/null +++ b/scripts/check-md-links.mjs @@ -0,0 +1,150 @@ +#!/usr/bin/env node +// Checks every internal link and anchor in the repo's Markdown docs offline, without a Jekyll +// build: a relative link's target file must exist, and a `#fragment` must match a heading in that +// file (or in the current file, for a same-page link). This is the GitHub-side half of +// scripts/check-site.mjs, which only checks the already-built Pages site; this one catches a link +// to a path that never exists on disk at all — e.g. a doc linking to a sibling page as `.html`, +// which 404s on github.com (`.html` only exists in the Pages build, via jekyll-relative-links). +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { dirname, join, relative, resolve, sep } from "node:path"; +import { pathToFileURL } from "node:url"; + +// Files and directories this check covers; kept in sync with the repo's user-facing docs. +export const TARGETS = [ + "README.md", + "CHANGELOG.md", + "CONTRIBUTING.md", + "SECURITY.md", + "docs", + "skills", +]; + +function walk(dir) { + return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + if (entry.name === "node_modules" || entry.name === "_site" || entry.name === "vendor") return []; + const path = join(dir, entry.name); + return entry.isDirectory() ? walk(path) : [path]; + }); +} + +export function listDocFiles(root) { + return TARGETS.flatMap((target) => { + const path = join(root, target); + const stat = statSync(path, { throwIfNoEntry: false }); + if (!stat) return []; + const files = stat.isDirectory() ? walk(path) : [path]; + return files.filter((f) => f.endsWith(".md")); + }); +} + +// GitHub's heading-to-anchor algorithm: lowercase, drop inline formatting and links, strip +// punctuation, spaces to hyphens, de-duplicate with a `-2`, `-3`, ... suffix. +export function slugify(heading) { + return heading + .trim() + .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") + .replace(/[`*_~]/g, "") + .toLowerCase() + .replace(/[^\w\s-]/g, "") + .trim() + .replace(/\s+/g, "-"); +} + +export function headingSlugs(markdown) { + const slugs = new Set(); + const counts = new Map(); + let inFence = false; + for (const line of markdown.split("\n")) { + if (/^\s*```/.test(line)) { + inFence = !inFence; + continue; + } + if (inFence) continue; + const heading = line.match(/^#{1,6}\s+(.+?)\s*$/); + if (!heading) continue; + let slug = slugify(heading[1]); + const seen = counts.get(slug) ?? 0; + counts.set(slug, seen + 1); + if (seen > 0) slug = `${slug}-${seen}`; + slugs.add(slug); + } + return slugs; +} + +// Markdown links/images outside fenced code blocks and inline code spans, as `{ line, url }` +// (1-indexed line numbers). Inline code is blanked out first so a backtick-quoted regex or path +// like `["']([A-Za-z_]\w*)["']` can't be misread as `[link text](url)`. +export function findLinks(markdown) { + const links = []; + let inFence = false; + const lines = markdown.split("\n"); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (/^\s*```/.test(line)) { + inFence = !inFence; + continue; + } + if (inFence) continue; + const withoutCodeSpans = line.replace(/`[^`]*`/g, (span) => " ".repeat(span.length)); + for (const m of withoutCodeSpans.matchAll(/!?\[[^\]]*\]\(([^)]+)\)/g)) { + links.push({ line: i + 1, url: m[1].trim().split(/\s+["']/)[0] }); + } + } + return links; +} + +function isExternal(url) { + return /^[a-z][a-z0-9+.-]*:/i.test(url) || url.startsWith("//"); +} + +// `{{ ... }}` is a Liquid expression (e.g. a collection loop's computed URL): not a static path +// this check can resolve, and valid Jekyll outside this check's scope. +function isTemplated(url) { + return url.includes("{{"); +} + +export function checkFile(file, slugCache) { + const problems = []; + const markdown = readFileSync(file, "utf8"); + const ownSlugs = () => { + if (!slugCache.has(file)) slugCache.set(file, headingSlugs(markdown)); + return slugCache.get(file); + }; + for (const { line, url } of findLinks(markdown)) { + if (!url || isExternal(url) || isTemplated(url)) continue; + const [pathPart, fragment] = url.split("#"); + let targetFile = file; + if (pathPart) { + targetFile = resolve(dirname(file), decodeURIComponent(pathPart)); + const stat = statSync(targetFile, { throwIfNoEntry: false }); + if (!stat) { + problems.push(`${file}:${line}: ${url} -> missing ${relative(process.cwd(), targetFile)}`); + continue; + } + if (stat.isDirectory()) targetFile = join(targetFile, "index.md"); + } + if (fragment === undefined) continue; + if (!slugCache.has(targetFile)) { + const stat = statSync(targetFile, { throwIfNoEntry: false }); + slugCache.set(targetFile, stat ? headingSlugs(readFileSync(targetFile, "utf8")) : null); + } + const slugs = targetFile === file ? ownSlugs() : slugCache.get(targetFile); + if (slugs && !slugs.has(fragment)) { + problems.push(`${file}:${line}: ${url} -> no heading "${fragment}" in ${relative(process.cwd(), targetFile)}`); + } + } + return problems; +} + +export function checkAll(root) { + const slugCache = new Map(); + return listDocFiles(root).flatMap((file) => checkFile(file, slugCache)); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const root = process.argv[2] ?? resolve(new URL(".", import.meta.url).pathname, ".."); + const problems = checkAll(root); + for (const problem of problems) console.error(problem); + console.log(problems.length ? `md link check: ${problems.length} problem(s)` : "md link check passed"); + process.exit(problems.length ? 1 : 0); +} diff --git a/scripts/check-mermaid.d.mts b/scripts/check-mermaid.d.mts new file mode 100644 index 0000000..d65fd20 --- /dev/null +++ b/scripts/check-mermaid.d.mts @@ -0,0 +1,6 @@ +export const TARGETS: string[]; +export const RESERVED_IDS: Set; +export function listDocFiles(root: string): string[]; +export function findReservedNodeIds(markdown: string): { line: number; id: string }[]; +export function checkFile(file: string): string[]; +export function checkAll(root: string): string[]; diff --git a/scripts/check-mermaid.mjs b/scripts/check-mermaid.mjs new file mode 100644 index 0000000..0c3c6d7 --- /dev/null +++ b/scripts/check-mermaid.mjs @@ -0,0 +1,76 @@ +#!/usr/bin/env node +// Lints ```mermaid fenced blocks in Markdown for flowchart node ids that collide with a Mermaid +// reserved word (e.g. `graph["import graph"] --> x` fails to parse: "got 'GRAPH'"). A real parse +// check would need mermaid's renderer, which pulls in a DOM (jsdom or a headless browser) — too +// heavy for this repo's one or two diagrams, so this checks the specific failure class the +// flowchart/graph grammar is known to hit: a bare node id that is also a keyword. +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +export const TARGETS = ["README.md", "CHANGELOG.md", "docs", "skills"]; + +// Keywords the flowchart/graph grammar reserves; using one as a bare node id is a parse error. +export const RESERVED_IDS = new Set(["graph", "flowchart", "subgraph", "end", "class", "click", "style", "classDef", "linkStyle", "direction", "default"]); + +function walk(dir) { + return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + if (entry.name === "node_modules" || entry.name === "_site" || entry.name === "vendor") return []; + const path = join(dir, entry.name); + return entry.isDirectory() ? walk(path) : [path]; + }); +} + +export function listDocFiles(root) { + return TARGETS.flatMap((target) => { + const path = join(root, target); + const stat = statSync(path, { throwIfNoEntry: false }); + if (!stat) return []; + const files = stat.isDirectory() ? walk(path) : [path]; + return files.filter((f) => f.endsWith(".md")); + }); +} + +// Mermaid blocks only accept a flowchart/graph node id written as `id[...]`, `id(...)`, `id{...}` +// or a bare `id`, at the start of a line or right after `-->`/`---`. This is deliberately narrow: +// it flags a reserved word used that way, not every occurrence of the word in the diagram. +export function findReservedNodeIds(markdown) { + const hits = []; + let inMermaid = false; + const lines = markdown.split("\n"); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (/^\s*```mermaid\s*$/.test(line)) { + inMermaid = true; + continue; + } + if (inMermaid && /^\s*```\s*$/.test(line)) { + inMermaid = false; + continue; + } + if (!inMermaid) continue; + for (const m of line.matchAll(/(?:^|-->|---|-\.-|==>)\s*([A-Za-z_][\w-]*)\s*[[({]/g)) { + const id = m[1].toLowerCase(); + if (RESERVED_IDS.has(id)) hits.push({ line: i + 1, id: m[1] }); + } + } + return hits; +} + +export function checkFile(file) { + return findReservedNodeIds(readFileSync(file, "utf8")).map( + ({ line, id }) => `${file}:${line}: node id "${id}" is a Mermaid reserved word`, + ); +} + +export function checkAll(root) { + return listDocFiles(root).flatMap(checkFile); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const root = process.argv[2] ?? resolve(new URL(".", import.meta.url).pathname, ".."); + const problems = checkAll(root); + for (const problem of problems) console.error(problem); + console.log(problems.length ? `mermaid lint: ${problems.length} problem(s)` : "mermaid lint passed"); + process.exit(problems.length ? 1 : 0); +} diff --git a/tests/scripts/check-md-links.test.ts b/tests/scripts/check-md-links.test.ts new file mode 100644 index 0000000..c84ef92 --- /dev/null +++ b/tests/scripts/check-md-links.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { checkAll, findLinks, headingSlugs, slugify } from "../../scripts/check-md-links.mjs"; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); + +function withFiles(files: Record, run: (dir: string) => void) { + const dir = mkdtempSync(join(tmpdir(), "check-md-links-")); + try { + for (const [path, content] of Object.entries(files)) { + mkdirSync(dirname(join(dir, path)), { recursive: true }); + writeFileSync(join(dir, path), content); + } + run(dir); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +describe("slugify", () => { + it("matches GitHub's heading-to-anchor rules", () => { + expect(slugify("Calibration and backtesting")).toBe("calibration-and-backtesting"); + expect(slugify("4. The facts")).toBe("4-the-facts"); + expect(slugify("`coupling.missing`")).toBe("couplingmissing"); + }); +}); + +describe("findLinks", () => { + it("ignores links inside fenced code blocks", () => { + const md = "[real](a.md)\n```md\n[fake](b.md)\n```\n[also real](c.md)"; + expect(findLinks(md).map((link) => link.url)).toEqual(["a.md", "c.md"]); + }); + + it("ignores a bracket/paren pair inside an inline code span", () => { + const md = "Add `/\\bos\\.LookupEnv\\((['\"])([A-Za-z_]\\w*)\\1\\)/` to `ENV_PATTERNS`."; + expect(findLinks(md)).toEqual([]); + }); +}); + +describe("headingSlugs", () => { + it("de-duplicates repeated headings with a numeric suffix", () => { + const slugs = headingSlugs("# Notes\n## Notes\n## Notes"); + expect(slugs).toEqual(new Set(["notes", "notes-1", "notes-2"])); + }); +}); + +describe("checkAll", () => { + it("reports a relative link to a file that does not exist", () => { + withFiles({ "docs/a.md": "[gone](missing.md)" }, (dir) => { + const problems = checkAll(dir); + expect(problems).toHaveLength(1); + expect(problems[0]).toMatch(/missing\.md$/); + }); + }); + + it("reports the exact bug class this check exists for: a .html link where only the .md exists", () => { + withFiles( + { "docs/a.md": "[Calibration](calibration.html)", "docs/calibration.md": "# Calibration" }, + (dir) => { + const problems = checkAll(dir); + expect(problems).toHaveLength(1); + expect(problems[0]).toContain("calibration.html"); + }, + ); + }); + + it("reports a fragment that matches no heading in the target file", () => { + withFiles( + { "docs/a.md": "[x](b.md#no-such-heading)", "docs/b.md": "# Real heading" }, + (dir) => { + const problems = checkAll(dir); + expect(problems).toHaveLength(1); + expect(problems[0]).toContain("no-such-heading"); + }, + ); + }); + + it("accepts a same-file fragment that matches an existing heading", () => { + withFiles({ "docs/a.md": "# Top\n\n[x](#top)" }, (dir) => { + expect(checkAll(dir)).toEqual([]); + }); + }); + + it("skips external URLs and templated Liquid expressions", () => { + withFiles( + { + "docs/a.md": "[ext](https://example.com/x.html)\n[loop]({{ note.url | relative_url }})", + }, + (dir) => { + expect(checkAll(dir)).toEqual([]); + }, + ); + }); + + it("passes on the repo's own README, CHANGELOG, docs and skills", () => { + expect(checkAll(REPO_ROOT)).toEqual([]); + }); +}); diff --git a/tests/scripts/check-mermaid.test.ts b/tests/scripts/check-mermaid.test.ts new file mode 100644 index 0000000..d6dc941 --- /dev/null +++ b/tests/scripts/check-mermaid.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { checkAll, findReservedNodeIds } from "../../scripts/check-mermaid.mjs"; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); + +describe("findReservedNodeIds", () => { + it("flags the exact failure this check exists for: `graph` used as a flowchart node id", () => { + const md = ["```mermaid", "flowchart LR", ' graph["import graph"] --> signals', "```"].join("\n"); + expect(findReservedNodeIds(md)).toEqual([{ line: 3, id: "graph" }]); + }); + + it("does not flag a reserved word used only as text inside another node's label", () => { + const md = ["```mermaid", "flowchart LR", ' imports["import graph"] --> signals', "```"].join("\n"); + expect(findReservedNodeIds(md)).toEqual([]); + }); + + it("ignores a mermaid-looking block outside a ```mermaid fence", () => { + const md = [' graph["x"] --> signals'].join("\n"); + expect(findReservedNodeIds(md)).toEqual([]); + }); + + it("flags `end` used as a node id but not the word inside a label", () => { + const md = ["```mermaid", "flowchart LR", " a --> end[stop]", "```"].join("\n"); + expect(findReservedNodeIds(md)).toEqual([{ line: 3, id: "end" }]); + }); +}); + +describe("checkAll", () => { + it("passes on the repo's README, CHANGELOG, docs and skills", () => { + expect(checkAll(REPO_ROOT)).toEqual([]); + }); + + it("reports a reserved node id introduced in a fresh mermaid block", () => { + const dir = mkdtempSync(join(tmpdir(), "check-mermaid-")); + try { + mkdirSync(join(dir, "docs"), { recursive: true }); + writeFileSync( + join(dir, "docs/a.md"), + ["```mermaid", "flowchart LR", " subgraph[nope] --> b", "```"].join("\n"), + ); + const problems = checkAll(dir); + expect(problems).toHaveLength(1); + expect(problems[0]).toContain("subgraph"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +});