diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 1b817fb5b6..339bb0d499 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -35,6 +35,26 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + # Shallow on EVERY trigger, because this job never publishes what the + # full clone would improve. + # + # The knowledge-extractor sets each page's `last_verified` from that + # file's last commit date, and under a depth-1 clone every file + # resolves to the same commit, so the plugin emits "" rather than a + # constant that looks like data. It warns and carries on; nothing here + # asserts on last_verified, so this cannot turn the job red. + # + # An earlier version of this kept full history on push/schedule, on + # the theory that those runs produce the artifact a deploy reads. They + # do not: this job has no deploy step and no upload-artifact step, so + # the index it builds is discarded with the runner on every trigger, + # push included. No workflow in this repo publishes it. + # + # docs-retrieval-evals.yml is the one that does use the dates - it + # reports freshness coverage and fails on --require-freshness - and it + # keeps fetch-depth: 0 for that reason. + fetch-depth: 1 - name: Setup Node.js uses: actions/setup-node@v4 diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index f891280f8a..7dcaf2edaf 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -21,6 +21,22 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + # Shallow is enough HERE, and only here. last_verified is the one + # thing that needs full history - the extractor reads each file's last + # commit date, and under a depth-1 clone every file resolves to the + # same commit - but the extractor now returns before any git call on a + # preview build, because it must not emit its ~22 MB of JSON into the + # gh-pages branch that pr-preview-action commits. A full clone would + # buy a freshness signal nobody computes. build-docs.yml makes the + # same call for the same reason and is shallow on EVERY trigger: it + # builds and verifies but never publishes the index, so the dates + # would be computed and discarded there too. + # + # docs-retrieval-evals.yml is the one workflow that keeps + # fetch-depth: 0, because it is the one that reads the dates - it + # reports freshness coverage and fails on --require-freshness. + fetch-depth: 1 - name: Setup Node.js uses: actions/setup-node@v4 diff --git a/.github/workflows/docs-retrieval-evals.yml b/.github/workflows/docs-retrieval-evals.yml new file mode 100644 index 0000000000..9a3277cffd --- /dev/null +++ b/.github/workflows/docs-retrieval-evals.yml @@ -0,0 +1,126 @@ +name: Docs Retrieval Evals + +# Measures retrieval quality of the build-generated knowledge index against a +# gold set of real queries (success@k / precision@k / MRR). Runs on PRs that +# touch docs, the extractor, or the gold set — a regression in descriptions or +# partials that hurts retrieval fails here. Also runs on push to main, so the +# deploy branch is checked too (an empty index from a cause outside the PR +# paths — e.g. a Docusaurus upgrade renaming article selectors — is caught here +# as well as by the plugin's own fail-loud floor check during the build). + +on: + pull_request: + branches: + - main + paths: + - 'docs/**' + # And the frozen snapshots. While docusaurus.config.ts pins lastVersion to + # one of them - a beta cycle - that snapshot IS the corpus this indexes, + # so a PR editing it changes exactly what is measured here and would + # otherwise run neither eval until after it merged. + - 'versioned_docs/**' + - 'src/plugins/knowledge-extractor/**' + - 'scripts/retrieval-evals/**' + - '.github/workflows/docs-retrieval-evals.yml' + # The extractor reads RENDERED html, so a component decides what the + # indexed text IS - it already carries a special case for FeatureList's + # "Loading features..." SSR placeholder, and productKeys() reads + # src/data. Without these, a PR that changes what a component renders + # (this one changes FeatureList) never runs the evals, and the first + # signal is the push-to-main trigger after it has merged. + - 'src/components/**' + - 'src/data/**' + # And src/theme, which decides the MARKUP: MDXComponents.js maps Tabs / + # TabItem and every custom component rendered inside article .markdown - + # tabsToMd is built entirely around that markup - and DocItem is swizzled. + # Swapping the Tabs implementation, or wrapping a component in one more + # element, changes the indexed text on hundreds of pages. + - 'src/theme/**' + # These decide WHAT gets indexed at all: assistantIgnoreFiles and + # lastVersion live in the config, and versions.json decides which + # trees are frozen. Without them a PR that widens or narrows the + # curation list changes the corpus and skips both evals. + - 'docusaurus.config.ts' + - 'versions.json' + push: + branches: + - main + +# Same pattern as docs-preview.yml, for the same reason: every run here is a +# full yarn build plus two eval passes, and this workflow watches paths a docs +# PR touches on nearly every push. Without a group they queue and all run; with +# it, a new push supersedes the one still running. Keyed on the PR number so +# runs on main are never cancelled by a PR. +concurrency: + group: docs-retrieval-evals-${{ github.event.number || github.sha }} + cancel-in-progress: true + +jobs: + retrieval-evals: + name: Docs Retrieval Evals + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + # Full history, and unlike build-docs.yml this job USES it. + # + # The knowledge-extractor takes each page's `last_verified` from its + # last commit date. Under the default depth-1 clone every file + # resolves to the same single commit, so the plugin deliberately + # emits "" rather than a constant that looks like data. + # + # run.cjs reports freshness coverage from that field - how many + # modules carry a date, the oldest and newest, and how many are over + # 180 days old - and fails in CI if NOT ONE module has one, which is + # what losing this line would look like. So the cost buys a signal + # that is actually read, and the check guards the setting that + # produces it. Do not drop to depth 1 here without deleting that + # report first. + # + # Deliberately NOT a ranking input: last_verified is a date, and the + # scorer tokenises on [a-z0-9]{2,}, so it would enter the term stream + # as "2026"/"09"/"18" and match any query mentioning those numbers. + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + cache: 'yarn' + + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: Build docs (generates the knowledge retrieval index) + id: build + run: yarn build + + - name: Run retrieval evals + # --require-freshness: this job is the one that checks out full history + # on purpose, so a dateless index here means the checkout lost its + # fetch-depth. Other callers (build-docs.yml is shallow on PRs) get the + # report without the failure. + run: node scripts/retrieval-evals/run.cjs --report reports/retrieval-evals.json --require-freshness + + # The gold-set run above measures 20 curated queries. This one asks every + # module to retrieve itself, which is the only check that covers pages + # nobody wrote a query for - and until now MIN_AUTO_SUCCESS gated nothing + # because no workflow passed --auto. Reported separately so a corpus-wide + # regression is attributable. + - name: Run corpus-wide self-retrieval + # Not always(): a gold-set breach above must not hide the corpus-wide + # number, but a FAILED BUILD must not add a second red step blaming a + # missing index instead - and always() fires on cancellation too. + if: ${{ !cancelled() && steps.build.outcome == 'success' }} + run: node scripts/retrieval-evals/run.cjs --auto --report reports/retrieval-evals-auto.json + + - name: Upload eval report + if: always() + uses: actions/upload-artifact@v4 + with: + name: retrieval-evals-report + path: | + reports/retrieval-evals.json + reports/retrieval-evals-auto.json + if-no-files-found: ignore diff --git a/docusaurus.config.ts b/docusaurus.config.ts index ff080e18b6..4d0d2ddb28 100644 --- a/docusaurus.config.ts +++ b/docusaurus.config.ts @@ -9,6 +9,7 @@ import remarkHideComments from "./src/plugins/remark-hide-comments"; import remarkOffloadPreviewMedia from "./src/plugins/remark-offload-preview-media"; import stripPreviewMediaPlugin from "./src/plugins/plugin-strip-preview-media"; import { UNRELEASED_FRAMEWORK_SLUGS } from "./src/constants/unreleasedFrameworks"; +import knowledgeExtractor from "./src/plugins/knowledge-extractor"; dotenv.config(); // Load environment variables from .env file const productionUrl = "https://docs.scandit.com"; @@ -95,19 +96,67 @@ const llmsRootRedirectOnlyDocs: string[] = [ "docs/system-requirements.mdx", ]; -// Paths are matched by docusaurus-plugin-llms relative to siteDir (e.g. docs/...). -const llmsIgnoreFiles: string[] = [ +/** + * CURATION: pages no AI consumer should be given at all - deprecated platforms, + * features a platform does not have, redirect-only stubs with no prose, and + * partials that are not routes. Shared by both AI exports, so a decision about + * what an assistant may see is made once. + */ +const assistantIgnoreFiles: string[] = [ "docs/connector-guides/**", // Partials are imported into real pages, not standalone doc routes; omit so llms.txt URLs work. "docs/partials/**", ...llmsRootRedirectOnlyDocs, ...llmsIgnoredSdkTrees, ...llmsLinuxPartialIgnore, - ...llmsNonWebSdkRoots.flatMap((root) => - llmsSharedPartialPageNames.map((name) => `docs/${root}/${name}`), - ), ]; +/** + * CORPUS SHAPE, for the llms export only: the same prose repeated under every + * non-Web SDK root, deduped to Web so a flat text corpus does not carry nine + * copies of it. + * + * Deliberately NOT shared with the knowledge-extractor, and this is the one + * place the two exports differ on purpose. That index exists to route a reader + * to the page they are on: someone reading Flutter docs needs the Flutter URL, + * not the Web one that happens to hold the canonical copy of the text. Applying + * this list there removed about 1,400 modules and with them the ability to + * answer "where in the Flutter docs does this live". + */ +const llmsDedupedToWeb: string[] = llmsNonWebSdkRoots.flatMap((root) => + llmsSharedPartialPageNames.map((name) => `docs/${root}/${name}`), +); + +/** + * CORPUS SHAPE, for the knowledge index only: the feature-availability matrix, + * every copy of it. + * + * The counterpart of llmsDedupedToWeb - each export states its own corpus shape + * while both share the curation decision in assistantIgnoreFiles. The llms + * export keeps ONE copy of this page, because a flat text corpus can carry a + * table; the index keeps none. + * + * A token-overlap consumer cannot use it and is actively harmed by it. The + * matrices are 120 modules over 10 URLs at 23-33 distinct tokens against an + * index median of 50, and they name every product - so they score well on ANY + * single-product query while answering none of them. Measured: the whole top 10 + * for the query "sparkscan" was the ten framework copies of this page, with the + * SparkScan pages below all of them. + * + * Nothing is lost. What the table says - which product exists on which + * framework - is exactly what the graph's AvailableOn / NotAvailableOn edges + * carry, in a form a consumer can query rather than one it has to parse. + */ +const knowledgeIndexDeduped: string[] = [ + "docs/features-by-framework.mdx", + ...llmsNonWebSdkRoots.map((root) => `docs/${root}/features-by-framework.mdx`), + "docs/sdks/web/features-by-framework.mdx", +]; + +// Paths are matched by docusaurus-plugin-llms relative to siteDir (e.g. docs/...). +const llmsIgnoreFiles: string[] = [...assistantIgnoreFiles, ...llmsDedupedToWeb]; +const knowledgeIndexIgnoreFiles: string[] = [...assistantIgnoreFiles, ...knowledgeIndexDeduped]; + // --------------------------------------------------------------------------- // SINGLE SOURCE OF TRUTH: docs versions and every `docusaurus_tag` derived // from them. @@ -465,6 +514,30 @@ const config: Config = { // reader on 6.28.11 finds the 6.28 API and never the 8.x one. apiReferenceTagsByVersionTag: buildApiReferenceTags(docsVersions), versionNumberByTag: buildVersionNumberByTag(docsVersions), + // The CURATION half of the llms ignore list - what no AI consumer should be + // given - exported so the knowledge-extractor applies the same decision + // rather than a paraphrase of it. Exporting only `llmsIgnoredSdkTrees` left + // the two disagreeing about 39 modules; exporting the WHOLE llms list + // over-corrected, because it also carries a corpus-shape decision (see + // llmsDedupedToWeb) that a routing index must not inherit. + // Neither assistantIgnoreFiles nor knowledgeIndexIgnoreFiles is exported + // here. Each consumer takes its own list as a plugin option instead, so + // there is one name per consumer rather than a customFields key that has to + // be kept in step with it. + // + // This does NOT keep them out of the client bundle, and it is worth being + // exact about that: Docusaurus serialises the resolved config, plugin + // options included, so the globs ship either way - the built main.js + // carries two `ignoreFiles:` arrays. Moving them was a naming fix, not a + // size one. + // Which version THIS build serves at the root. Read by the + // knowledge-extractor plugin, which cannot otherwise tell a frozen version + // served at the root from `current` - and the two need opposite handling. + lastVersion: effectiveLastVersion, + // Read by the knowledge-extractor, which must NOT emit its ~22 MB of JSON + // into a preview: pr-preview-action commits the whole build/ into gh-pages, + // so every preview push would add those blobs there permanently. + isPreviewBuild, }, // Set the production url of your site here @@ -754,6 +827,48 @@ const config: Config = { // gate can check a real build artifact instead of re-deriving the same // assumption from this file and agreeing with itself. searchTagsManifestPlugin, + // Build-generate AI layer: emits /assets/knowledge-retrieval-index.json and + // /assets/knowledge-graph.jsonld from the rendered HTML (see + // src/plugins/knowledge-extractor). + // + // Division of labour with docusaurus-plugin-llms above, which also exports for + // AI consumers: that plugin ships the PROSE (llms.txt as a link index, + // llms-full.txt as the text - 0.08 and 2.28 MiB on this build). This one ships + // TYPED METADATA AND EDGES for deciding what to read - intents, audiences, + // channels, frameworks, products, cites-API, see-also, availability - with + // only an excerpt of the prose, because the text is already published next to + // it (10.41 MiB index, 11.62 MiB graph). Neither replaces the other. + // + // They share assistantIgnoreFiles, so what an assistant may SEE is decided + // once. They differ, on purpose, on llmsDedupedToWeb: a flat corpus wants one + // copy of prose repeated across frameworks, a routing index needs the module + // that points at the framework the reader is actually on. + // + // Last in the array by convention, NOT by guarantee. Docusaurus runs every + // plugin's postBuild through Promise.all (core/lib/commands/build.js), so + // array position implies nothing about execution order and this hook races + // any other postBuild. + // + // There are THREE other postBuild hooks, not one: stripPreviewMediaPlugin, + // @docusaurus/plugin-client-redirects and docusaurus-plugin-llms. + // + // stripPreviewMediaPlugin deletes under build/img, which walkHtml skips, and + // docusaurus-plugin-llms only writes its own .txt files at the root. But + // plugin-client-redirects WRITES /index.html stubs into outDir - the + // very files walkHtml enumerates - so this plugin sees a different file count + // depending on who wins. The extractor now identifies those stubs by their + // meta refresh and excludes them from both terms of its drift ratio, so the + // outcome no longer depends on the race. + // + // A future postBuild that REWRITES page html would still race + // non-deterministically and must be sequenced explicitly, not moved above + // this line. + // The curation list plus this index's own corpus-shape exclusions (see + // knowledgeIndexDeduped), passed as a plugin option so this consumer reads it + // under one name - the same shape docusaurus-plugin-llms uses above. The + // plugin throws if it is missing rather than defaulting to "index + // everything", which is the property that matters here. + [knowledgeExtractor, { ignoreFiles: knowledgeIndexIgnoreFiles }], ], presets: [ diff --git a/package.json b/package.json index 73eb2b6763..d2d8cb4805 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "docs:gate:setup": "node scripts/setup-vale.cjs", "verify:search-tags": "node scripts/verify-search-tags.cjs", "test:search-facets": "node scripts/test-search-facets.cjs", + "docs:retrieval-evals": "node scripts/retrieval-evals/run.cjs", "verify:frameworks": "node scripts/verify-frameworks.cjs", "test:frameworks": "node scripts/test-frameworks.cjs", "test:docs-gate": "node scripts/test-docs-gate.cjs", @@ -32,9 +33,11 @@ "@docusaurus/plugin-client-redirects": "3.4.0", "@docusaurus/preset-classic": "^3.4.0", "@mdx-js/react": "^3.0.0", + "cheerio": "1.0.0-rc.12", "classnames": "^2.5.1", "clsx": "^1.2.1", "docusaurus-plugin-sass": "^0.2.5", + "gray-matter": "4.0.3", "prism-react-renderer": "^2.1.0", "react": "^18.0.0", "react-dom": "^18.0.0", diff --git a/scripts/retrieval-evals/baseline.json b/scripts/retrieval-evals/baseline.json new file mode 100644 index 0000000000..2da59cb1b0 --- /dev/null +++ b/scripts/retrieval-evals/baseline.json @@ -0,0 +1,8 @@ +{ + "page_success_at_k": 0.8596, + "page_mrr": 0.775, + "pages": 520, + "modules": 4228, + "k": 3, + "recorded_at": "2026-09-21" +} diff --git a/scripts/retrieval-evals/gold-set.json b/scripts/retrieval-evals/gold-set.json new file mode 100644 index 0000000000..123e952e46 --- /dev/null +++ b/scripts/retrieval-evals/gold-set.json @@ -0,0 +1,25 @@ +{ + "_comment": "Retrieval-eval gold set. Each entry is a real user query (seeded from Algolia top-queries + the product/intent taxonomy) mapped to the URL/path substrings a correct answer should live under. A retrieved knowledge module counts as relevant if its `url` contains ANY of `expect`. Add entries as new products/pages land; keep queries in the user's words.", + "queries": [ + { "query": "barcode capture get started", "expect": ["/barcode-capture/get-started"] }, + { "query": "sparkscan", "expect": ["/sparkscan/"] }, + { "query": "spark scan get started", "expect": ["/sparkscan/get-started", "/sparkscan/intro"] }, + { "query": "matrixscan count", "expect": ["/matrixscan-count/"] }, + { "query": "matrixscan find", "expect": ["/matrixscan-find/", "/matrixscan-pick/"] }, + { "query": "label capture", "expect": ["/label-capture/"] }, + { "query": "smart label capture", "expect": ["/label-capture/"] }, + { "query": "id capture supported documents", "expect": ["/id-capture/supported-documents", "/id-capture/", "id-documents"] }, + { "query": "id bolt", "expect": ["/id-bolt/"] }, + { "query": "parser", "expect": ["/parser/"] }, + { "query": "barcode selection", "expect": ["/barcode-selection/"] }, + { "query": "add the sdk", "expect": ["/add-sdk"] }, + { "query": "how to initialize data capture context", "expect": ["/add-sdk", "/get-started"] }, + { "query": "supported symbologies", "expect": ["barcode-symbologies", "symbology-properties"] }, + { "query": "qr code scanning", "expect": ["barcode-symbologies", "/barcode-capture/"] }, + { "query": "data matrix", "expect": ["barcode-symbologies", "symbology-properties"] }, + { "query": "ai powered barcode scanning", "expect": ["ai-powered-barcode-scanning"] }, + { "query": "scandit express inventory count", "expect": ["/express/", "inventory-count"] }, + { "query": "release notes", "expect": ["/release-notes"] }, + { "query": "agent skills", "expect": ["/agent-skills"] } + ] +} diff --git a/scripts/retrieval-evals/run.cjs b/scripts/retrieval-evals/run.cjs new file mode 100644 index 0000000000..a6919c690f --- /dev/null +++ b/scripts/retrieval-evals/run.cjs @@ -0,0 +1,580 @@ +#!/usr/bin/env node +"use strict"; +/** + * Retrieval-quality evals for the build-generated knowledge retrieval index + * (docs/assets/knowledge-retrieval-index.json, emitted by the knowledge-extractor + * plugin at build time). + * + * Scores each gold-set query with the same token-overlap ranking the index is + * designed for, then reports: + * - success@k : fraction of queries with a relevant module in the top k + * - precision@k : mean fraction of the top k that are relevant + * - MRR : mean reciprocal rank of the first relevant module + * + * A retrieved module is "relevant" when its `url` contains ANY of the query's + * `expect` substrings (a path-class, so the gold set stays stable even though + * module ids are auto-generated). Fails (exit 1) if success@k or MRR drops below + * the thresholds — so a regression in descriptions/partials is caught in CI. + * + * Adapted from the bundle's run_retrieval_evals.py (token mode), pure Node stdlib. + * + * Usage: node scripts/retrieval-evals/run.cjs [--index ] [--k 3] + * [--min-success 0.8] [--min-mrr 0.6] [--report ] + */ +const fs = require("fs"); +const path = require("path"); + +function arg(name, def) { + const i = process.argv.indexOf(`--${name}`); + return i !== -1 && process.argv[i + 1] ? process.argv[i + 1] : def; +} + +const INDEX = arg("index", "build/assets/knowledge-retrieval-index.json"); +const GOLD = arg("gold", path.join(__dirname, "gold-set.json")); +// Parsed through a checked reader, NOT bare parseFloat. +// +// `--min-success x` or `--auto-tolerance abc` yields NaN, every `metric < NaN` +// comparison is false, and the gate exits 0 reporting "ok" - a threshold +// typo silently disables the check it was meant to tighten. readBaseline() +// already refuses to run ungated for exactly this hazard on the baseline file; +// the flags deserve the same treatment. +function num(name, fallback, parse) { + const raw = arg(name, fallback); + const v = parse(raw, 10); + if (!Number.isFinite(v)) { + console.error( + `retrieval-evals: --${name} is "${raw}", which is not a number.\n` + + "Refusing to run: every comparison against NaN is false, so the gate " + + "would pass whatever the metrics were.", + ); + process.exit(1); + } + return v; +} + +const K = num("k", "3", parseInt); +const MIN_SUCCESS = num("min-success", "0.8", parseFloat); +const MIN_PRECISION = num("min-precision", "0.6", parseFloat); +const MIN_MRR = num("min-mrr", "0.6", parseFloat); +// --auto: corpus-wide self-retrieval over EVERY module (not just the 20-query +// gold set) — each module becomes a query built from its own title+summary and +// must retrieve itself in the top k. Measures coverage across all docs we +// create/edit. --auto-limit caps it; --min-auto-success gates it. +const AUTO = process.argv.includes("--auto"); +const AUTO_LIMIT = num("auto-limit", "0", parseInt); +// Gated as a REGRESSION against a recorded baseline, not against an absolute +// floor. +// +// An absolute floor is the wrong instrument for this metric. Self-retrieval asks +// whether a page surfaces itself, and this corpus is ~11 near-duplicate copies +// of the same prose, one per framework - so every page competes with its own +// siblings. Adding a framework makes the score fall for a reason that has +// nothing to do with retrieval quality, and the last measurement (0.8493 over +// 531 pages / 4,386 modules) sat under 5 points above a 0.80 floor. The next +// SDK would have turned a green pipeline red and taught everyone to raise the +// number, which is how a gate stops meaning anything. +// +// So: fail when the score drops more than AUTO_TOLERANCE below the baseline in +// baseline.json, and keep a low absolute floor as a backstop for the case where +// the baseline itself is wrong. Re-record with --update-baseline after a change +// that legitimately alters what is indexed, and say so in the commit. +const BASELINE_PATH = path.join(__dirname, "baseline.json"); +const AUTO_TOLERANCE = num("auto-tolerance", "0.03", parseFloat); +const AUTO_FLOOR = num("auto-floor", "0.60", parseFloat); +const AUTO_MRR_FLOOR = num("auto-mrr-floor", "0.40", parseFloat); +const MIN_AUTO_MRR = num("min-auto-mrr", "0", parseFloat); +const UPDATE_BASELINE = process.argv.includes("--update-baseline"); +function readBaseline() { + let raw; + try { + raw = fs.readFileSync(BASELINE_PATH, "utf8"); + } catch { + // Genuinely absent is a legitimate state - the first run records it. + return null; + } + // Present but unusable is NOT. Swallowing it dropped the gate to the bare + // backstop, so a 24-point regression would have gone green; and a parsed + // object missing the key made the floor NaN, which every comparison is false + // against - the gate then passed unconditionally and still reported "ok". + // --update-baseline is the ONE caller allowed past a bad baseline, because it + // is the thing that repairs one. Without this, both checks below exited 1 + // before the write at the end of main() could run - so the message "Re-record + // it with --update-baseline" named a command that could never succeed, and + // the only way out was to delete the file by hand. A guard whose own + // remediation it blocks is worse than no guard. + const refuse = (msg) => { + console.error(`retrieval-evals: ${msg}`); + if (UPDATE_BASELINE) { + console.error("--update-baseline was passed, so this run will overwrite it."); + return null; + } + console.error("Refusing to run an ungated check."); + process.exit(1); + }; + + let parsed; + try { + parsed = JSON.parse(raw); + } catch (err) { + return refuse(`${BASELINE_PATH} is not valid JSON (${err.message}).`); + } + for (const key of ["page_success_at_k", "page_mrr"]) { + // Both, not just the first. page_mrr went unchecked, so a baseline that + // dropped the key - a hand edit, or one written by a pre-MRR version of + // this script - set mrrFloor to 0, and nothing is ever below 0: the MRR + // gate disabled itself while the report still said "ok". + if (!parsed || !Number.isFinite(parsed[key])) { + return refuse(`${BASELINE_PATH} has no finite ${key}.`); + } + } + return parsed; +} +const MIN_AUTO_SUCCESS = num("min-auto-success", "0", parseFloat); +const REPORT = arg("report", ""); +// Set by the workflow that checks out full history on purpose. See +// reportFreshness: without it, an index with no dates is reported and allowed. +const REQUIRE_FRESHNESS = process.argv.includes("--require-freshness"); + +const TOKEN = /[a-z0-9]{2,}/gi; +const tokenize = (s) => new Set((String(s || "").toLowerCase().match(TOKEN) || [])); + +function docText(d) { + return [ + d.title, + d.summary, + d.docs_excerpt, + d.assistant_excerpt, + (d.keywords || []).join(" "), + (d.intents || []).join(" "), + ].join(" "); +} + +// BM25 over a binary term representation, replacing `overlap / sqrt(|q|*|d|)`. +// +// |q| is constant across the documents of one query, so that formula ranked on +// overlap / sqrt(|d|) - which rewards a document for being SHORT. A 10-token +// stub matching one query term scored 1/sqrt(10) = 0.316, beating a 100-token +// page matching two at 2/sqrt(100) = 0.200. The proxy retriever this gate reads +// its numbers from was therefore measuring page brevity as much as relevance, +// and precision looked stronger than it was. +// +// Two changes fix that. IDF weights a rare term above a common one, so matching +// "sparkscan" is no longer worth the same as matching "the". And the length term +// normalises against the corpus AVERAGE with the standard b=0.75, which damps +// long documents without handing short ones the ranking. +// +// Term frequency is binary because docText() is read as a Set. That is +// deliberate rather than a shortcut: the fields concatenated there already +// repeat the title and slug, so a raw count would measure field layout. With +// tf=1 the length factor is constant per document and BM25 reduces to +// IDF-weighted overlap under pivoted length normalisation - which is exactly +// the property that was missing. +const K1 = 1.2; +const B = 0.75; + +function makeScorer(tokenSets) { + const df = new Map(); + let total = 0; + for (const dt of tokenSets) { + total += dt.size; + for (const t of dt) df.set(t, (df.get(t) || 0) + 1); + } + const N = tokenSets.length || 1; + const avgdl = total / N || 1; + // Precomputed, not called per (query token, document) pair: --auto scores + // every page against every module, so Math.log in the inner loop is tens of + // millions of calls. Floored at 0 so a term in nearly every document + // contributes nothing rather than scoring negative. + const idf = new Map(); + for (const [t, n] of df) idf.set(t, Math.max(0, Math.log(1 + (N - n + 0.5) / (n + 0.5)))); + return function score(qt, dt) { + if (!qt.size || !dt.size) return 0; + let sum = 0; + for (const t of qt) if (dt.has(t)) sum += idf.get(t) || 0; + if (sum === 0) return 0; + return (sum * (K1 + 1)) / (1 + K1 * (1 - B + (B * dt.size) / avgdl)); + }; +} + +// Memoised per index array. Both halves of this script build their scorer from +// the same corpus through the same function, so "rank" cannot come to mean two +// different things in the two paths - the tie-break notes below depend on that. +const SCORERS = new WeakMap(); +function scorerFor(index) { + let entry = SCORERS.get(index); + if (!entry) { + const tokens = index.map((d) => tokenize(docText(d))); + entry = { tokens, score: makeScorer(tokens) }; + SCORERS.set(index, entry); + } + return entry; +} + +function search(index, query, k) { + const qt = tokenize(query); + const { tokens, score } = scorerFor(index); + const ranked = index + .map((d, i) => ({ id: d.id, url: String(d.url || d.source_site || ""), s: score(qt, tokens[i]) })) + .sort((a, b) => b.s - a.s || a.id.localeCompare(b.id)); + // One slot per PAGE, not per chunk. The index holds many chunks per page + // (4,364 modules over 532 pages) and this used to slice raw modules, so a + // single page could take every slot: "id capture supported documents" returned + // three chunks of /id-documents/ as its entire top 3. + // + // It does not move the gate - success@3 and MRR are the same either way, and + // precision@3 actually falls, because duplicate chunks of a relevant page each + // counted as relevant. That is the point. Un-deduped, precision@3 partly + // measured how finely a page happened to be chunked rather than how often the + // right page was found, and a consumer with three slots to spend got one page. + // + // The --auto half already ranks by "how many OTHER-page chunks outrank it", so + // it never spends a slot on a sibling. Deduping here makes both halves of this + // script agree about what a rank means - the same argument as the tie-break + // note above. + const seen = new Set(); + const out = []; + for (const hit of ranked) { + const page = hit.url.split("#")[0]; + if (seen.has(page)) continue; + seen.add(page); + out.push(hit); + if (out.length >= k) break; + } + return out; +} + +// Freshness coverage of the index. +// +// docs-retrieval-evals.yml checks out full history for exactly one reason: the +// extractor dates each page from that file's last commit, and under a shallow +// clone it emits "" rather than a constant that looks like data. Nothing read +// the result, which is what made the cost look unjustified - docs-preview.yml +// calls it "a freshness signal nobody computes". This computes it. +// +// A REPORT, not a ranking input, and that distinction is the whole point. +// last_verified is a date and this scorer tokenises on [a-z0-9]{2,}, so +// "2026-09-18" would enter the term stream as "2026", "09" and "18" - a query +// mentioning any of those numbers would then match every page verified that +// day. Recency is not relevance, and feeding one into the other would quietly +// degrade the metric the rest of this file exists to measure. +function reportFreshness(index) { + const dated = []; + let blank = 0; + for (const m of index) { + const d = String(m.last_verified || ""); + if (/^\d{4}-\d{2}-\d{2}$/.test(d)) dated.push(d); + else blank++; + } + console.log(`\nFreshness: ${dated.length}/${index.length} modules carry a last_verified date.`); + if (dated.length) { + dated.sort(); + const cutoff = new Date(Date.now() - 180 * 864e5).toISOString().slice(0, 10); + const stale = dated.filter((d) => d < cutoff).length; + console.log( + ` oldest ${dated[0]}, newest ${dated[dated.length - 1]}, ` + + `${stale} older than 180 days (before ${cutoff}).`, + ); + } + if (blank === index.length) { + console.error( + "\nretrieval-evals: NOT ONE module carries a date, so this index was built\n" + + "from a shallow clone - the extractor emits \"\" rather than a constant\n" + + "that looks like data.\n", + ); + // Opt-in, NOT `process.env.CI`. + // + // A shallow clone is a legitimate state: it is what build-docs.yml now uses + // on pull_request, because that job never publishes the index. Keying this + // off CI alone would fail any other CI job that ran the evals, and blame a + // workflow it had never touched. Only a caller that asked for full history + // can say that dateless output is a regression rather than a choice, so + // only that caller passes the flag. + if (REQUIRE_FRESHNESS) { + console.error( + "--require-freshness was passed, so the caller expects dated modules:\n" + + "the checkout step has lost its `fetch-depth: 0`.\n", + ); + process.exit(1); + } + } +} + +function relevant(hit, expect) { + const u = hit.url.toLowerCase(); + return expect.some((e) => u.includes(e.toLowerCase())); +} + +function main() { + if (!fs.existsSync(INDEX)) { + console.error(`retrieval-evals: index not found at ${INDEX} — run \`yarn build\` first.`); + process.exit(1); + } + const index = JSON.parse(fs.readFileSync(INDEX, "utf8")); + const gold = JSON.parse(fs.readFileSync(GOLD, "utf8")).queries || []; + if (!Array.isArray(index) || !index.length) { + console.error("retrieval-evals: empty or invalid index."); + process.exit(1); + } + reportFreshness(index); + if (AUTO) { + // Corpus-wide, PAGE-LEVEL self-retrieval over every doc: for each page, + // query with a chunk's own title+summary and check that a chunk from the + // SAME page (same url) lands in the top k. Page-level (not exact-chunk) is + // the meaningful coverage metric here, because the index holds many + // near-duplicate chunks per page (shared prose across frameworks, "Part N" + // splits) — so "did we surface the right page for this doc's content?" is + // what matters, not "did this exact chunk outrank its own siblings". + const docs = index.map((m) => ({ + id: String(m.id || ""), + url: String(m.url || ""), + tokens: tokenize(docText(m)), + query: [m.title, m.summary].filter(Boolean).join(" ").trim() || String(m.id || ""), + })); + // One representative query per unique page (first chunk seen). + const seen = new Set(); + let rows = docs.filter((d) => d.url && !seen.has(d.url) && seen.add(d.url)); + if (AUTO_LIMIT > 0) rows = rows.slice(0, AUTO_LIMIT); + // The SAME scorer the gold-set path uses, built over this corpus. It was a + // second copy of the old formula here, so a change to one ranking function + // silently left the other behind - and this is the half the baseline and + // the gate are computed from. + const fastScore = makeScorer(docs.map((d) => d.tokens)); + let ok = 0, + rrSum = 0; + const misses = []; + for (const r of rows) { + const qt = tokenize(r.query); + // Best-ranking chunk of the SAME page. Ties are broken by id, exactly as + // the gold-set path sorts (b.s - a.s || a.id.localeCompare(b.id)): the two + // halves of this script have to agree about what rank means. + let bestSame = 0; + let bestSameId = ""; + for (const d of docs) { + if (d.url !== r.url) continue; + const s = fastScore(qt, d.tokens); + if (s > bestSame || (s === bestSame && (!bestSameId || d.id.localeCompare(bestSameId) < 0))) { + bestSame = s; + bestSameId = d.id; + } + } + // Rank of that chunk = how many OTHER-page chunks outrank it. + // + // A tie counts against it when the competitor's id sorts first. The + // comparison used to be a bare `>`, which handed the page every tie - and + // this corpus is ~11 near-duplicate copies of the same prose, so ties are + // the common case, not the edge case. Measured on the same artifact: + // page-success@3 0.8459 optimistic vs 0.7350 with ties counted, page-MRR + // 0.7669 vs 0.6526. A baseline resting on that difference moves further + // than the tolerance whenever an edit perturbs tokenisation enough to flip + // a batch of ties - a red gate with no regression behind it - and hides a + // real regression that only converts strictly-better competitors into ties. + // Distinct competing PAGES, not competing chunks. + // + // search() spends one slot per page, so a rank there counts pages. This + // counted every outranking CHUNK, so three chunks of one rival page + // consumed all three slots and the page was scored a miss that the + // gold-set path scores a hit. The two halves of this script have to agree + // about what a rank means, and this is the half the baseline and the CI + // gate are computed from - so the gated number was the wrong one. + // + // It also made the metric move with CHUNK COUNTS: re-chunking a rival + // page, with no change in retrieval quality at all, changed this score. + const betterPages = new Set(); + for (const d of docs) { + if (d.url === r.url) continue; + const s = fastScore(qt, d.tokens); + if (s > bestSame || (s === bestSame && d.id.localeCompare(bestSameId) < 0)) { + betterPages.add(d.url); + if (betterPages.size >= K) break; + } + } + const better = betterPages.size; + if (better < K) { + ok++; + rrSum += 1 / (better + 1); + } else { + misses.push(r.url.replace(/^https?:\/\/[^/]+/, "")); + } + } + // No rows means no metric, and a metric that is NaN passes every gate: + // `NaN < floor` is false, so `breached` stays false and this prints "ok" + // and exits 0 on an index it never measured. That is the same hazard num() + // and readBaseline() were written to close, one level up - an index whose + // records carry no `url` produces rows.length === 0 and would sail through. + if (rows.length === 0) { + console.error( + "\nretrieval-evals: no page rows - every record is missing a `url`, " + + "so there is nothing to measure.\nRefusing to report a gate result " + + "on an index this run could not evaluate.\n", + ); + process.exit(1); + } + const metrics = { + mode: "auto-page-self-retrieval", + pages: rows.length, + modules: docs.length, + k: K, + page_success_at_k: +(ok / rows.length).toFixed(4), + page_mrr: +(rrSum / rows.length).toFixed(4), + }; + console.log(`\nRetrieval self-eval (AUTO, page-level over all docs): ${rows.length} pages / ${docs.length} modules, k=${K}`); + const baseline = readBaseline(); + // An explicit --min-auto-success still wins, so a caller can pin a number. + // --min-auto-mrr does the same for MRR: with only the success override the + // documented "a caller can pin a number" could not actually relax the AUTO + // gate, because MRR kept failing it. + const floor = MIN_AUTO_SUCCESS > 0 + ? MIN_AUTO_SUCCESS + : baseline + ? Math.max(AUTO_FLOOR, +(baseline.page_success_at_k - AUTO_TOLERANCE).toFixed(4)) + : AUTO_FLOOR; + // MRR is recorded, so gate it. Success@k alone cannot see a change that + // keeps every page inside the top 3 while pushing it from rank 1 to rank 3 - + // the corpus looks unchanged and ranking quality, which is what a retrieval + // consumer feels, has halved. + // readBaseline guarantees a finite page_mrr when a baseline exists, so the + // only way here without one is having no baseline at all - the first run. + const mrrFloor = MIN_AUTO_MRR > 0 + ? MIN_AUTO_MRR + : baseline + ? Math.max(AUTO_MRR_FLOOR, +(baseline.page_mrr - AUTO_TOLERANCE).toFixed(4)) + : AUTO_MRR_FLOOR; + const breached = + metrics.page_success_at_k < floor || metrics.page_mrr < mrrFloor; + console.log(` page-success@${K} = ${metrics.page_success_at_k} (min ${floor})`); + if (baseline) { + const delta = +(metrics.page_success_at_k - baseline.page_success_at_k).toFixed(4); + console.log( + ` baseline = ${baseline.page_success_at_k} ` + + `(${baseline.pages} pages / ${baseline.modules} modules, ${baseline.recorded_at})` + + ` delta ${delta >= 0 ? "+" : ""}${delta}`, + ); + // Corpus growth is the expected reason for a drop here, so say it out loud + // rather than leaving someone to infer it from a red pipeline. + if (metrics.pages !== baseline.pages || metrics.modules !== baseline.modules) { + console.log( + ` NOTE: corpus changed since the baseline ` + + `(${baseline.pages}->${metrics.pages} pages, ${baseline.modules}->${metrics.modules} modules). ` + + `If that was intended, re-record with --update-baseline.`, + ); + } + } else { + console.log(` baseline = none recorded; gating on the ${AUTO_FLOOR} backstop only`); + } + if (UPDATE_BASELINE) { + fs.writeFileSync( + BASELINE_PATH, + JSON.stringify( + { + page_success_at_k: metrics.page_success_at_k, + page_mrr: metrics.page_mrr, + pages: metrics.pages, + modules: metrics.modules, + k: metrics.k, + recorded_at: new Date().toISOString().slice(0, 10), + }, + null, + 2, + ) + "\n", + ); + console.log(` baseline recorded -> ${path.relative(process.cwd(), BASELINE_PATH)}`); + } + console.log(` page-MRR = ${metrics.page_mrr} (min ${mrrFloor})`); + console.log(` ${misses.length} page(s) not surfaced in top ${K} by their own content.`); + misses.slice(0, 10).forEach((m) => console.log(` ✗ ${m}`)); + if (REPORT) { + fs.mkdirSync(path.dirname(REPORT), { recursive: true }); + fs.writeFileSync( + REPORT, + JSON.stringify({ status: breached ? "breach" : "ok", floor, mrrFloor, baseline, metrics, misses: misses.slice(0, 200) }, null, 2) + "\n", + ); + } + process.exit(breached && !UPDATE_BASELINE ? 1 : 0); + } + + if (!gold.length) { + console.error("retrieval-evals: empty gold set."); + process.exit(1); + } + + // NOTE on recall: this gold set uses path-classes (a query maps to "any page + // under /sparkscan/", not one specific page), and every such class has more + // than K relevant modules in the corpus. A capped recall@k + // (relCount / min(totalRelevant, k)) therefore reduces algebraically to + // precision@k (relCount / k) — the same number query by query, not just on + // average. Reporting both would be one signal printed twice, and a + // "min-recall" gate could only ever fail when precision already had. So we + // report precision@k with an honestly-named MIN_PRECISION floor and DO NOT + // report a redundant recall. A real recall metric needs single-page gold + // entries (small, exact totalRelevant) or an uncapped denominator at a large + // k — a future gold-set change, not a rename. + let successSum = 0, + precisionSum = 0, + rrSum = 0; + const rows = []; + for (const g of gold) { + const hits = search(index, g.query, K); + const rel = hits.map((h) => relevant(h, g.expect)); + const firstRel = rel.indexOf(true); + const relCount = rel.filter(Boolean).length; + const success = firstRel !== -1 ? 1 : 0; + const precision = relCount / Math.max(hits.length, 1); + const rr = firstRel !== -1 ? 1 / (firstRel + 1) : 0; + successSum += success; + precisionSum += precision; + rrSum += rr; + rows.push({ + query: g.query, + success, + precision: +precision.toFixed(3), + rr: +rr.toFixed(3), + top: hits.map((h) => h.url.replace(/^https?:\/\/[^/]+/, "")), + }); + } + + const n = gold.length; + const metrics = { + success_at_k: +(successSum / n).toFixed(4), + precision_at_k: +(precisionSum / n).toFixed(4), + mrr: +(rrSum / n).toFixed(4), + k: K, + query_count: n, + index_size: index.length, + }; + + console.log(`\nRetrieval evals (token mode, k=${K}, ${n} queries over ${index.length} modules)`); + console.log(` success@${K} = ${metrics.success_at_k} (min ${MIN_SUCCESS})`); + console.log(` precision@${K} = ${metrics.precision_at_k} (min ${MIN_PRECISION})`); + console.log(` MRR = ${metrics.mrr} (min ${MIN_MRR})\n`); + for (const r of rows) { + if (!r.success) console.log(` ✗ MISS "${r.query}" → top: ${r.top.join(" , ") || "(none)"}`); + } + + const breaches = []; + + // NOTE: the artifact invariants (no excerpt ending on a heading/label with no + // content, no unbalanced code fence) are asserted by the knowledge-extractor + // plugin itself, in postBuild - it refuses to publish a violation. They were + // briefly duplicated here, but this workflow is paths-filtered and limited to + // base `main`, so it does not run on the builds that publish the index, and a + // second copy of the predicate could only drift from the generator's. + + if (metrics.success_at_k < MIN_SUCCESS) breaches.push(`success@${K}=${metrics.success_at_k} < ${MIN_SUCCESS}`); + if (metrics.precision_at_k < MIN_PRECISION) breaches.push(`precision@${K}=${metrics.precision_at_k} < ${MIN_PRECISION}`); + if (metrics.mrr < MIN_MRR) breaches.push(`MRR=${metrics.mrr} < ${MIN_MRR}`); + + if (REPORT) { + fs.mkdirSync(path.dirname(REPORT), { recursive: true }); + fs.writeFileSync(REPORT, JSON.stringify({ status: breaches.length ? "breach" : "ok", metrics, breaches, rows }, null, 2) + "\n"); + } + + if (breaches.length) { + console.log("Retrieval quality below threshold:"); + for (const b of breaches) console.log(` breach: ${b}`); + process.exit(1); + } + console.log("Retrieval quality OK."); + process.exit(0); +} + +main(); diff --git a/scripts/test-search-facets.cjs b/scripts/test-search-facets.cjs index d351ff270e..8bbce43137 100644 --- a/scripts/test-search-facets.cjs +++ b/scripts/test-search-facets.cjs @@ -177,7 +177,11 @@ check("an unknown version tag adds nothing rather than guessing", () => { check("typing a version moves the guides AND the API reference together", () => { const filters = ["language:en", [DEFAULT, SERVED, APILATEST]]; - const out = rewriteVersionTag(filters, "docs-default-7.6.14", MAP); + // The frozen tag from the manifest, not a literal. This file's premise is + // that no version may be hardcoded here - LEGACY and API76 either side of + // this line are derived - so retiring 7.6 would have failed the suite for a + // reason unrelated to search correctness. + const out = rewriteVersionTag(filters, LEGACY.replace("docusaurus_tag:", ""), MAP); const tags = tagsOf(out); assert.ok(tags.includes(LEGACY), "page tag must be swapped"); assert.ok(tags.includes(API76), "API reference must follow to 7.6"); diff --git a/src/components/FeatureList/index.tsx b/src/components/FeatureList/index.tsx index 570cf628df..5185879a27 100644 --- a/src/components/FeatureList/index.tsx +++ b/src/components/FeatureList/index.tsx @@ -200,15 +200,44 @@ const FeatureList: React.FC = ({ {Object.entries(feature.frameworks) .filter(([, info]) => info.version !== 'n/a') .map(([frameworkName, frameworkInfo]) => ( - - {frameworkName} v{frameworkInfo.version} - + // The ELEMENT changes, not just the href. Guarding the + // href alone rendered the same DOM - withCurrentDocsPath + // returns undefined for undefined and React omits the + // attribute - so the chip stayed an with no href: + // not keyboard-focusable, navigating nowhere, still + // showing the link hover. 17 cells in features.json + // carry no apiUrl, of which 16 reach this branch - the + // 17th is Web / 7-Segment Display, filtered out earlier + // by version === "n/a". + // Latent only while every usage is compact mode, which + // skips this column. Same shape as the feature-name cell + // above, which swaps for

. + frameworkInfo.apiUrl ? ( + + {frameworkName} v{frameworkInfo.version} + + ) : ( + + {frameworkName} v{frameworkInfo.version} + {/* The dashed border and the dimmed token say "unavailable" + to someone who can see them. Without this the chip + announces exactly what an available one announces - + "iOS v1.2.3" - so a screen-reader user gets no signal + that this one is different and does not link anywhere. */} + (not available) + + ) ))} diff --git a/src/components/FeatureList/styles.module.css b/src/components/FeatureList/styles.module.css index 7e41c35f1b..a5cf73ff35 100644 --- a/src/components/FeatureList/styles.module.css +++ b/src/components/FeatureList/styles.module.css @@ -87,8 +87,18 @@ } +/* The chips carry padding and a border but no margin, and React renders the + .map() output with no whitespace between siblings, so without a container + layout they sit flush against one another. This rule was `display: none` + from the file's first commit - the column has never rendered, because all 50 + FeatureList usages pass displayMode="compact", which takes a different path. + Hiding the content while the component still emits its + Available Frameworks header was incoherent; this is the smallest + layout that makes the column legible if that view is ever used. */ .frameworkList { - display: none; + display: flex; + flex-wrap: wrap; + gap: 0.25rem; } .frameworkItem { @@ -112,14 +122,41 @@ text-decoration: none; } +/* No API reference for this framework yet. Dimmed through the border and a + colour token rather than `opacity`, which composited the label to 3.19:1 in + light mode - below AA - while looking merely faint. Measured against the + BUILT css rather than assumed token values: 5.29:1 light, 12.26:1 dark. + `border-style: dashed` carries the state without relying on colour. */ +/* Reachable by a screen reader, absent from the visual layout. Not + `display: none` or `visibility: hidden`, which remove it from the + accessibility tree as well and would defeat the point. */ +.visuallyHidden { + position: absolute; + width: 1px; + height: 1px; + margin: -1px; + padding: 0; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + .frameworkItemUnavailable { - opacity: 0.5; - cursor: not-allowed; + color: var(--ifm-color-emphasis-700); + border-style: dashed; + cursor: default; } +/* Hold the dimmed colour on hover. This rule exists to cancel + .frameworkItem:hover, but it was left over from the opacity version and + restored --ifm-color-content - the full-contrast colour - so hovering an + unavailable chip made it look available, with only the dashed border + still carrying the state. The span is not interactive; cursor: default + already says so. */ .frameworkItemUnavailable:hover { background: var(--ifm-color-emphasis-100); - color: var(--ifm-color-content); + color: var(--ifm-color-emphasis-700); border-color: var(--ifm-color-emphasis-200); } diff --git a/src/plugins/knowledge-extractor/index.ts b/src/plugins/knowledge-extractor/index.ts new file mode 100644 index 0000000000..12c9e10d77 --- /dev/null +++ b/src/plugins/knowledge-extractor/index.ts @@ -0,0 +1,2456 @@ +/** + * knowledge-extractor — build-generate AI layer. + * + * Runs in Docusaurus `postBuild`, so it reads the FINAL rendered HTML (which + * already has all partials/MDX components inlined) rather than raw Markdown. + * That means pages whose prose lives in imported partials are captured in full, + * and every chunk gets the real, user-facing URL and the frontmatter-derived + * `` as its summary. + * + * It splits each page of the version SERVED AT THE ROOT into small + * self-contained knowledge + * modules (chunks, ~1400 chars) with rule-based metadata, preserving link URLs + * inside the prose (so citations — including external API-reference links — + * survive), then emits the two artifacts an assistant / in-docs search consume: + * - /assets/knowledge-retrieval-index.json (fast lookup) + * - /assets/knowledge-graph.jsonld (enriched concept graph) + * + * A POINTER INDEX, NOT A TEXT STORE - and deliberately so. `toIndexRecord` + * publishes `docs_excerpt` and `assistant_excerpt` - each a clipped preview, + * budgeted at 400 and 300 chars respectively, though `clipMarkdown` appends the + * fence closer AFTER slicing, so those are slice budgets and not enforced + * bounds - and does NOT publish `content.docs_markdown`. The consumer follows + * `url` for the full text, which is the page itself, so it is always reachable. + * + * The cost of the alternative is a MAGNITUDE argument, and it is stated in + * magnitudes on purpose. The index is around 10 MiB and the graph around 11; + * carrying the full chunk would add roughly 6 MiB more, against `llms-full.txt` + * holding the site's entire prose corpus in about 2.3. Per-module text is an + * expensive way to store text, and that holds at any of these sizes. + * + * Exact figures are deliberately NOT pinned here. They have been wrong three + * times: the module count, both artifact sizes and both excerpt averages all + * drifted as chunking and scoring changed, while this comment went on asserting + * them, and nothing checks a number in a comment. + * + * So the current values are REPORTED rather than written down. The success line + * at the end of postBuild prints modules, pages, both artifact sizes, node and + * edge counts, every one of them derived from the artifact it just wrote - + * `yarn build` and read the last [knowledge-extractor] line. Pages and modules + * are also in scripts/retrieval-evals/baseline.json, recorded by the evals. + * + * The argument rests on those magnitudes and on the `url`, NOT on a claim that + * each module's prose also ships in llms-full.txt: the two artifacts cut the + * corpus differently - llms.txt indexes whole PAGES while this indexes CHUNKS, + * several per page - so there is no per-module correspondence to appeal to. + * Units are MiB throughout, to match `mb()` and MAX_INDEX_MB below. + * + * So CHUNK_TARGET_CHARS is a GRANULARITY knob, not a payload size: it decides + * how finely a page is split into retrievable units, and the excerpt is a + * ranking and preview budget. Shrinking the chunk to match the excerpt would + * change what a module IS - more, smaller modules - which is a retrieval-quality + * decision to make against the evals, not a size cleanup. + * + * DIVISION OF LABOUR with docusaurus-plugin-llms, registered beside this one: + * that plugin emits the PROSE corpus for context-stuffing (`llms.txt` as a link + * index, `llms-full.txt` as the text). This one emits TYPED METADATA AND EDGES + * for deciding which page or chunk to read - intents, audiences, channels, + * frameworks, products, cites-API, see-also, availability - which a flat text + * dump cannot express. They share the curation decision about what an assistant + * may see - `assistantIgnoreFiles` in docusaurus.config.ts - but each takes its + * own list as a PLUGIN OPTION (`options.ignoreFiles` here, `ignoreFiles` on the + * llms plugin) rather than through `customFields`. That is deliberate: a + * customFields key is a second name that has to be kept in step with the list + * it mirrors, and it had already drifted once. Each consumer's list is a + * superset of the shared one, so the decision is still made in a single place. + * + * Known limit: 10 MiB is large for a browser consumer to fetch whole. Nothing + * in the repo consumes it that way yet; if something does, it needs a served + * index or a split by framework rather than a smaller excerpt. + * + * The graph is not just faceting: alongside intent/audience/channel/framework + * it mines real edges from the content — product membership, cites-API, + * see-also (internal links), and per-product availability (from "not available" + * stubs). Per-module intermediates are held in memory only, never committed. + * + * Faithful port of the bundle's Python pipeline, adapted to the Scandit repo. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import * as cheerio from "cheerio"; +import matter from "gray-matter"; + +const CHUNK_TARGET_CHARS = 1400; +// Budgets for the published artifacts. Deliberately close to today's real sizes +// so growth has to be an explicit decision rather than a silent one. +const MAX_INDEX_MB = 14; +const MAX_GRAPH_MB = 16; +const OWNER = "docsops-auto"; + +type Chunk = { heading: string; content: string }; + +// --------------------------------------------------------------------------- +// small helpers (ported) +// --------------------------------------------------------------------------- +/** + * Truncate markdown without leaving an unterminated code fence. + * + * Prefers cutting just before the fence that would be left open; if the text + * opens a fence too early for that to leave anything useful, closes the fence + * instead. Consumers read this field directly, so a dangling ``` is not cosmetic. + */ +/** + * A line that only INTRODUCES content and carries none itself: a heading, or a + * bold-only line (a tab-panel label, in practice). + * + * The bold cap is 80, not 40: this corpus already has "Install from Package + * Registry (Recommended)" at 43 characters, which a tighter cap silently + * exempted. There is no prose risk in that width - a prose bold line is always + * blank-separated, because orderedSegments joins prose segments with a blank + * line, whereas a label is glued to its body by a single newline. + */ +const INTRODUCER_LINE = /^\s*(?:>\s*)*(?:\*\*[^*]{1,80}\*\*|#{1,6}\s+\S.*)\s*$/; + +/** Trailing lines that introduce content the text does not (yet) contain. */ +function trailingIntroducers(lines: string[]): number { + let i = lines.length; + let sawIntroducer = false; + while (i > 0 && (!lines[i - 1].trim() || INTRODUCER_LINE.test(lines[i - 1]))) { + if (lines[i - 1].trim()) sawIntroducer = true; + i--; + } + // Trailing blank lines alone are not a broken promise - they are whitespace. + // Counting them made "ends on an introducer" true for "hello\n\n", which would + // have reported the wrong defect. + return sawIntroducer ? lines.length - i : 0; +} + +/** Drop a trailing introducer run, so a clip never ends on a broken promise. */ +function dropTrailingIntroducers(lines: string[]): string[] { + const drop = trailingIntroducers(lines); + return drop ? lines.slice(0, lines.length - drop) : lines.slice(); +} + +/** + * Strip the markers that make a line an INTRODUCER, keeping its words. + * + * The last-resort branch of clipMarkdown used to hand back the raw window when + * dropping introducers emptied it - which is exactly the output the invariant + * forbids, so with the assertion in postBuild it would be an unfixable build + * failure. The corpus is close: the longest existing leading introducer run is + * 247 characters against a 300-character assistant budget, and the longest single + * heading is 246. Demoting keeps the text and drops only the promise. + */ +function demoteIntroducers(text: string): string { + return text.split("\n").map(demoteLine).join("\n"); +} + +/** + * Strip markers from ONE line until it is no longer an introducer. + * + * A single pass was not enough, and the shortfall was not cosmetic: `### # Foo` + * demotes to `# Foo`, `**# Note**` to `# Note`, `**## # x**` to `## # x` - all + * still introducers, so the demotion handed the assertion precisely the shape it + * throws on. With that assertion in postBuild, a page containing one such line + * fails the main deploy and every PR preview, on content no docs author could + * clear without a code change. It also broke enforceChunkInvariant's fixpoint: + * a second application demoted `# Note` again, to `Note`. + * + * Terminates: while the line matches, one of the two patterns matches too - the + * heading branch needs a leading `#{1,6}` run, the bold branch a whole-line + * `**...**` - and either replacement strictly shortens the line. + */ +function demoteLine(line: string): string { + let out = line; + while (INTRODUCER_LINE.test(out)) { + const next = out + .replace(/^(\s*(?:>\s*)*)#{1,6}\s+/, "$1") + .replace(/\*\*([^*]*)\*\*/g, "$1"); + // Believed unreachable by the argument above, but the cost of being wrong is + // a build nobody can unblock, so make no-progress terminal rather than a + // spin - and hand back something that CANNOT match either branch. + if (next === out) return out.replace(/[#*>]/g, "").trim(); + out = next; + } + return out; +} + +/** + * Close a code fence the clip left open. + * + * Both of clipMarkdown's early returns need this, and for different reasons: + * the short path can be handed text that was already unbalanced, and the + * demote path can MANUFACTURE an opener by stripping the markers in front of + * one. Shared so the two cannot drift - they did, and the demote path shipped + * the shape the published-artifact assertion throws on. + * + * The closer copies the opener's quote prefix. A bare closer after "> ```" is + * not balanced: CommonMark ends the blockquote at the unquoted line, implicitly + * closing the quoted fence, then opens a new top-level one that never closes. + */ +function balanceFences(text: string): string { + const fences = text.match(/^\s*(?:>\s*)*```/gm) || []; + if (fences.length % 2 === 0) return text; + const openerPrefix = (fences[fences.length - 1] || "").match(/^\s*(?:>\s*)*/); + return `${text}\n${openerPrefix ? openerPrefix[0] : ""}\u0060\u0060\u0060`; +} + +function clipMarkdown(text: string, limit: number): string { + // The early return used to hand back the raw text, so for any chunk shorter + // than the limit the drop never ran at all - the "no excerpt ends on an + // introducer" property then held only by luck of the content. + if (text.length <= limit) { + const whole = dropTrailingIntroducers(text.split("\n")).join("\n").trimEnd(); + const short = whole || demoteIntroducers(text).trimEnd(); + // Balance here as well. This path returned its input verbatim, so an + // unbalanced fence arriving from the content itself reached + // assertArtifactInvariants, which throws and fails `docusaurus build` - + // for main and for every PR preview. That contradicts the rule the rest + // of this plugin follows, that extraction must never break a deploy, and + // the fix would have to be a code change on a docs-only PR with main red. + return balanceFences(short); + } + const cut = text.slice(0, limit); + const lines = cut.split("\n"); + let open = false; + let lastFenceStart = -1; + for (let i = 0; i < lines.length; i++) { + if (/^\s*(?:>\s*)*```/.test(lines[i])) { + if (!open) lastFenceStart = i; + open = !open; + } + } + // The label drop runs on BOTH paths. It used to sit only below this early + // return, so when the cut landed exactly at the end of a label line - the + // fence opener falling outside the window - the label shipped with nothing + // after it. Two records did precisely that. + if (!open) { + const plain = dropTrailingIntroducers(lines).join("\n").trimEnd(); + // Never nothing, and never the forbidden shape: demote instead of echoing. + // + // balanceFences on BOTH, because demotion can CREATE a fence where the + // counter saw none. `open` is computed over `cut` before demotion, and the + // fence test is line-anchored, so "### # ``` weird" holds no fence at all - + // then demoteLine strips the markers, the backticks land at column 0, and + // the result carries an opener with no closer. Measured: limit 12 on that + // input returned "``` we", which assertArtifactInvariants throws on, which + // fails `docusaurus build` on content no docs author could clear. + return balanceFences(plain || demoteIntroducers(cut).trimEnd()); + } + // Dropping the half-included block is better than shipping it broken, unless + // that would throw away almost everything. + const kept = dropTrailingIntroducers(lines.slice(0, lastFenceStart)); + const trimmed = kept.join("\n").trimEnd(); + if (trimmed.length >= Math.floor(limit / 3)) return trimmed; + // Match the opener's quote prefix. A bare closer after "> ```" is not + // balanced: CommonMark ends the blockquote at the unquoted line, implicitly + // closing the quoted fence, and then opens a new top-level one that never + // closes - the exact dangling fence this function exists to prevent. + const openerPrefix = (lines[lastFenceStart] || "").match(/^\s*(?:>\s*)*/); + const closer = `${openerPrefix ? openerPrefix[0] : ""}\u0060\u0060\u0060`; + return `${cut.trimEnd()}\n${closer}`; +} + +/** + * `assistant_context` with its chunk clipped fence-safely, and its boilerplate + * prefix left alone. Splitting on the chunk rather than on a character count is + * what stops the clip from eating all the content. + */ +const ASSISTANT_CHUNK_CHARS = 300; +function assistantExcerpt(assistantContext: string, chunk: string): string { + const at = chunk ? assistantContext.lastIndexOf(chunk) : -1; + if (at < 0) return clipMarkdown(assistantContext, ASSISTANT_CHUNK_CHARS); + const prefix = assistantContext.slice(0, at); + // The tail is empty today because assistant_context ENDS with the chunk, but + // nothing asserts that. Re-appending it costs nothing and means a future + // suffix - the curated user_intents/not_for that a comment below contemplates + // inlining - cannot silently vanish from the published field. + const tail = assistantContext.slice(at + chunk.length); + return `${prefix}${clipMarkdown(chunk, ASSISTANT_CHUNK_CHARS)}${tail}`; +} + +/** + * Escape EVERY `|` in a table cell, link targets included. + * + * I previously exempted link targets on the theory that a backslash would break + * the URL. That was wrong in both directions. CommonMark does recognise + * backslash escapes inside a link destination (spec 6.1), so `\|` round-trips to + * a literal pipe; and the GFM tables extension requires the escape for a pipe + * anywhere in a cell, "including inside other inline spans". Leaving it raw makes + * the parser split the row at the pipe: the URL is truncated mid-string, a + * phantom cell appears, and since the delimiter is sized to the widest row that + * phantom widens the delimiter and pads every other row with an empty column. + */ +function escapeCell(text: string): string { + return text.replace(/\|/g, "\\|"); +} + +function slug(value: string): string { + const clean = String(value) + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); + return clean.replace(/-{2,}/g, "-") || "module"; +} + +function firstHeading(text: string): string { + for (const line of text.split(/\r?\n/)) { + const m = line.match(/^\s*#{2,6}\s+(.+?)\s*$/); + if (m) return m[1].trim(); + } + return ""; +} + +/** Split body into ~chunkTarget-sized chunks at H2/H3 boundaries (ported). */ +/** + * Split `body` at every line the caller calls a boundary, but NEVER inside a + * fenced code block. + * + * Both chunk boundaries have to go through this. Splitting on headings alone was + * not enough: the paragraph-level fallback in chunkBody used a plain + * split("\n\n"), which was harmless only for as long as code samples were + * (wrongly) collapsed onto a single line and so contained no blank lines. + * Restoring real newlines in code samples turned that latent bug into a live one + * - measured at 386 of 4452 chunks carrying an unbalanced number of fence lines, + * i.e. a dangling ``` and raw code presented to a consumer as prose. + */ +function splitFenceAware( + body: string, + isBoundary: (line: string, prevLine: string) => boolean, + dropBoundary = false, +): string[] { + const out: string[] = []; + let buf: string[] = []; + let inFence = false; + const flush = () => { + const joined = buf.join("\n"); + if (joined.trim()) out.push(joined); + buf = []; + }; + for (const line of body.split("\n")) { + // The (?:> )* prefix matters: a fenced sample inside a blockquote is emitted + // as "> ```", which a bare /^\s*```/ never sees - so the splitter would cut + // straight through it. + if (/^\s*(?:>\s*)*```/.test(line)) inFence = !inFence; + if (!inFence && isBoundary(line, buf.length ? buf[buf.length - 1] : "")) { + if (dropBoundary) { + flush(); + continue; // the boundary itself is a separator, not content + } + if (buf.length) flush(); + } + buf.push(line); + } + flush(); + return out; +} + +/** + * Heading boundaries: "## " / "### " at the start of a line, outside a fence. + * + * Deliberately plain again. Special-casing "a heading right after a label" here + * only moved the orphaned-label symptom to "orphaned label plus headings", and + * cost two chunks their correct section title. The invariant that a chunk may not + * end on an introducer is enforced once, after chunking, in enforceChunkInvariant. + */ +function splitOnHeadings(body: string): string[] { + return splitFenceAware(body, (l) => /^(##\s|###\s)/.test(l)); +} + +/** Paragraph boundaries: a blank line outside a fence. */ +function splitParagraphs(body: string): string[] { + return splitFenceAware(body, (l) => l.trim() === "", true); +} + +/** + * THE invariant, enforced in one place instead of by boundary heuristics. + * + * A chunk must not END on a run of lines that merely introduces content - a tab + * label, a heading, or several of them - because such a chunk advertises + * something a consumer will not find in it. The run is MOVED to the head of the + * next chunk, never dropped: it belongs with the content it introduces, and + * dropping it would lose text outright. A chunk left with nothing but its + * introducers disappears into its successor. + * + * Headings are then re-derived from each chunk's OWN leading heading. Taking the + * first heading anywhere in the source part is what indexed two chunks under the + * wrong section once the boundaries moved. + */ +function enforceChunkInvariant(chunks: Chunk[]): Chunk[] { + const out = chunks.map((c) => ({ ...c })); + + // FORWARD, not backwards. Iterating from the end writes into a slot that has + // already been finalised: if out[i+1] was introducer-only it is emptied into + // out[i+2] first, and out[i]'s trailing run then lands in that now-empty slot, + // re-creating an introducer-only chunk no later step inspects. Going forward + // means every slot is re-examined after it is written, which is what makes this + // a fixpoint - re-running it changes nothing. + for (let i = 0; i <= out.length - 2; i++) { + const lines = out[i].content.split("\n"); + const drop = trailingIntroducers(lines); + if (!drop) continue; + const moved = lines.slice(lines.length - drop).join("\n").trim(); + const keep = lines.slice(0, lines.length - drop).join("\n").trim(); + if (moved) out[i + 1].content = `${moved}\n${out[i + 1].content}`.trim(); + out[i].content = keep; + } + + // The LAST chunk was exempt, because there is nowhere forward to move to. But a + // trailing introducer run at the end of the body introduces nothing anywhere - + // the section it names is empty in the source too - so keeping it would publish + // a promise the whole document cannot honour. Dropped, which costs only the + // heading of an empty trailing section. + if (out.length) { + const last = out[out.length - 1]; + const dropped = dropTrailingIntroducers(last.content.split("\n")).join("\n").trim(); + // Guarded on the RESULT, not on out.length. A body that is entirely + // introducer lines but long enough to split chunks twice has out.length > 1, + // so testing the array length let every chunk empty and the page leave the + // index - and postBuild skips a page with no chunks WITHOUT incrementing + // pagesProcessed, so it vanishes silently AND drags the drift-guard ratio + // down. Counting what still has content covers that case and the + // single-chunk one under one rule. + const survivors = out.slice(0, -1).filter((c) => c.content.trim()).length; + last.content = dropped || (survivors === 0 ? demoteIntroducers(last.content).trim() : ""); + } + + return out + .filter((c) => c.content.trim()) + .map((c) => ({ content: c.content, heading: leadingHeading(c.content) || c.heading })); +} + +/** + * The chunk's OWN leading heading, or "" when it does not open with one. + * + * Distinct from firstHeading(), which returns the first heading ANYWHERE in the + * text - so a chunk that begins mid-section and happens to contain a later + * heading was titled by that later section. 241 chunks were titled that way, 139 + * of them by a heading more than 400 characters in. Falling back to the + * chunkBody-derived heading is more accurate: that is the section the chunk + * STARTS in, which is what a reader of the chunk is actually inside. + */ +function leadingHeading(text: string): string { + for (const line of text.split(/\r?\n/)) { + if (!line.trim()) continue; + const m = line.match(/^\s*#{2,6}\s+(.+?)\s*$/); + return m ? m[1].trim() : ""; + } + return ""; +} + +function chunkBody(body: string, target: number): Chunk[] { + const parts = splitOnHeadings(body); + const chunks: Chunk[] = []; + let current = ""; + let currentHeading = ""; + for (let part of parts) { + part = part.trim(); + if (!part) continue; + const partHeading = firstHeading(part); + const candidate = current ? `${current}\n\n${part}`.trim() : part; + if (candidate.length <= target) { + current = candidate; + if (!currentHeading) currentHeading = partHeading; + continue; + } + if (current) chunks.push({ heading: currentHeading, content: current }); + if (part.length <= target) { + current = part; + currentHeading = partHeading; + continue; + } + let para = ""; + let paraHeading = partHeading; + // A single fenced block longer than the target stays intact and simply + // overshoots: an oversized chunk is recoverable, a bisected code sample is not. + for (let p of splitParagraphs(part)) { + p = p.trim(); + if (!p) continue; + const cand = para ? `${para}\n\n${p}`.trim() : p; + if (cand.length <= target) { + para = cand; + if (!paraHeading) paraHeading = firstHeading(p); + } else { + if (para) chunks.push({ heading: paraHeading, content: para }); + para = p; + // `|| partHeading`: this used to overwrite unconditionally, so a + // continuation paragraph carrying no heading of its own reset the + // heading to "". enforceChunkInvariant then had nothing to fall back on + // and buildModule substituted the page title, leaving 1,179 of 3,855 + // "(Part N)" modules titled " (Part N)" instead of the section + // they are in - and `topic`, which a reranker keys on, with them. + paraHeading = firstHeading(p) || partHeading; + } + } + current = para; + currentHeading = paraHeading; + } + if (current) chunks.push({ heading: currentHeading, content: current }); + return enforceChunkInvariant(chunks); +} + +function pickIntents(contentType: string, title: string, body: string): string[] { + const text = `${title} ${body}`.toLowerCase(); + const intents: string[] = []; + if (contentType === "tutorial" || contentType === "how-to" || text.includes("configure")) intents.push("configure"); + // `fix` is word-bounded for the same reason `auth` is, two lines down: as a + // bare substring it matched "prefix", "suffix" and "fixtures", and those + // words appear in ordinary reference prose, so pages with nothing wrong were + // filed under the troubleshooting intent - for many of them the ONLY intent + // they carried. The inflected forms are kept because they are the real + // signal; the trailing boundary is what excludes "fixtures". + if ( + contentType === "troubleshooting" || + text.includes("error") || + /\bfix(?:e[sd]|ing)?\b/.test(text) + ) + intents.push("troubleshoot"); + if (contentType === "reference" || contentType === "concept" || text.includes("integrat")) intents.push("integrate"); + // Word-boundary, not a substring. Counted across the built HTML, `" auth"` + // matched "authenticity" 12 times, "authentic" 10 and "authority" 5 - every + // one of them the ID-DOCUMENT sense, about whether a passport is genuine - + // against "authorization" twice. So the old test filed about 27 + // document-verification mentions under the security intent, and 2 real ones. + // + // Verb forms are deliberately left out: "authenticate" does not occur in + // this corpus at all, while "authorized" does - in "authorized dealer" - + // so adding them would re-admit the false positives to catch nothing. + if (text.includes("secure") || /\bauth(?:n|entication|orization)?\b/.test(text)) { + intents.push("secure"); + } + if (intents.length === 0) intents.push("configure"); + return Array.from(new Set(intents)).sort(); +} + +function pickAudiences(contentType: string): string[] { + if (contentType === "tutorial") return ["beginner", "practitioner"]; + if (contentType === "reference" || contentType === "concept") return ["developer", "operator"]; + if (contentType === "troubleshooting") return ["support", "operator"]; + return ["practitioner", "developer"]; +} + +function extractSummary(description: string, bodyChunk: string): string { + const desc = (description || "").trim(); + if (desc) return desc.slice(0, 240); + let text = bodyChunk.replace(/\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/\s+/g, " ").trim(); + if (text.length < 30) text = `${text} This module is auto-generated from docs content for retrieval and assistant context.`; + return text.slice(0, 240); +} + +// --------------------------------------------------------------------------- +// HTML -> markdown-ish text (link URLs preserved so citations survive) +// --------------------------------------------------------------------------- +const HEADING_LEVEL: Record = { h1: 1, h2: 2, h3: 3, h4: 4, h5: 5, h6: 6 }; + +/** Serialize inline content, keeping `[label](href)` for links and `code` spans. */ +function serializeInline($: cheerio.CheerioAPI, node: any): string { + let out = ""; + $(node) + .contents() + .each((_i, n: any) => { + if (n.type === "text") { + out += n.data || ""; + } else if (n.type === "tag") { + const tag = String(n.name || "").toLowerCase(); + if (tag === "a") { + // Docusaurus heading anchors are whose label is a + // single U+200B and whose href is the FULL page path plus a fragment - + // not a bare "#anchor". So neither the empty-label check nor the + // startsWith("#") guard caught them: they leaked into the heading, the + // topic and the excerpt, and classifyLinks then read the stripped href + // as a real reference, which is what made most SeeAlso edges point a + // module at its own page. + const cls = String($(n).attr("class") || ""); + if (/\bhash-link\b/.test(cls)) return; + const href = String($(n).attr("href") || ""); + // U+200B is not matched by \s, so trim() leaves it and !label is false. + const label = serializeInline($, n) + // Also the private-use range: /sdks/web/matrixscan/get-started/ renders a + // second permalink anchor with no hash-link class whose label is a + // Font-Awesome glyph (U+F0C1), so the zero-width strip alone left it + // non-empty and it was published as a real link inside the heading. + .replace(/[\u200b\u200c\u200d\ufeff\ue000-\uf8ff]/g, "") + .replace(/\s+/g, " ") + .trim(); + if (!label) return; + out += href && !href.startsWith("#") ? `[${label}](${href})` : label; + } else if (tag === "code") { + out += "`" + $(n).text() + "`"; + } else if (tag === "br") { + out += " "; + } else { + out += serializeInline($, n); + } + } + }); + return out; +} + +function inlineText($: cheerio.CheerioAPI, el: any): string { + return serializeInline($, el).replace(/​/g, "").replace(/\s+/g, " ").trim(); +} + +function tableToMd($: cheerio.CheerioAPI, el: any): string { + // Scoped to THIS table's own row groups: find("tr") also collects a nested + // table's rows and appends them as extra rows of the outer one. + const trs = $(el).children("thead,tbody,tfoot").children("tr"); + const rows: string[][] = []; + (trs.length ? trs : $(el).children("tr")).each((_i, tr) => { + const cells: string[] = []; + $(tr) + .children("th,td") + .each((_j, c) => { + // Braced for the same TS2322 reason as tabsToMd above. + cells.push(escapeCell(inlineText($, c))); + }); + if (cells.length) rows.push(cells); + }); + if (!rows.length) return ""; + // Widest row, not the header's width: a parser honours the delimiter, so a + // header narrower than a body row would silently DISCARD the extra cells. + const width = rows.reduce((w, r) => Math.max(w, r.length), 0); + const line = (cells: string[]) => + `| ${cells.concat(Array(width - cells.length).fill("")).join(" | ")} |`; + const out = [line(rows[0])]; + // Without a delimiter row after the header none of this is a table to a + // markdown parser - it is a run of literal pipe lines, with the header + // indistinguishable from the data. 595 chunks shipped tables; none had one. + out.push(`|${" --- |".repeat(width)}`); + for (const r of rows.slice(1)) out.push(line(r)); + return out.join("\n"); +} + +/** + * Split an element's children into ordered segments, gathering CONSECUTIVE + * INLINE children into runs and serializing each run as one unit. + * + * Both properties are load-bearing, and every round of review found the same two + * bugs in whichever walk site had not yet been fixed: + * + * - serializeInline() iterates the CONTENTS of the node it is handed, so passing + * it an skips the branch that emits [label](href) and keeps only the + * label. Same for (backticks) and
. Inline children therefore have + * to be serialized as a group, inside a wrapper, never one at a time. + * - joining separately-serialized pieces with " " inserts a space at every + * boundary, which produced "List-based workflows : Validate…". + * + * Runs also preserve document order, so a sentence that follows a code block is + * not hoisted above it, and a text node that CONTINUES a sentence across an + * inline element stays in the same segment instead of becoming its own paragraph. + */ +/** + * SSR loading placeholders are not documentation. FeatureList renders + * "Loading features..." before it hydrates, and a walk that returns bare text + * children publishes it. Matched narrowly - the word plus a trailing ellipsis - + * so prose that merely begins with "Loading" survives ("Loading Dialog", + * `setProgressBarMessage("Loading ...")` and the like all stay). + */ +function isLoadingPlaceholder(text: string): boolean { + // Deliberately narrow. The previous shape allowed 40 arbitrary characters, and + // once the test was also applied to whole blocks that meant a paragraph like + // "Loading the SDK asynchronously..." would be deleted silently, with no + // warning and no trace in the output. A placeholder is one or two bare words + // and an ellipsis; anything with more shape than that is prose. + return /^loading(\s+[a-z]+){1,2}\s*(\.{3}|\u2026)$/i.test(text.trim()); +} + +function orderedSegments($: cheerio.CheerioAPI, el: any): string[] { + const segs: string[] = []; + let pending: any[] = []; + const flush = () => { + if (!pending.length) return; + const wrap = $(""); + for (const nd of pending) wrap.append($(nd).clone()); + const t = inlineText($, wrap[0]); + if (t && !isLoadingPlaceholder(t)) segs.push(t); + pending = []; + }; + const isBlockTag = (tag: string, node: any) => + tag === "ul" || + tag === "ol" || + tag === "pre" || + tag === "p" || + tag === "div" || + tag === "section" || + tag === "details" || + tag === "article" || + tag === "aside" || + tag === "table" || + tag === "blockquote" || + Boolean(HEADING_LEVEL[tag]) || + $(node).find("pre").length > 0; + + $(el) + .contents() + .each((_i, c: any) => { + if (c.type === "text") { + pending.push(c); + return; + } + if (c.type !== "tag") return; + const tag = String(c.name || "").toLowerCase(); + if (!isBlockTag(tag, c)) { + pending.push(c); + return; + } + flush(); + const sub = blockToMd($, c); + // Guarded here as well: it worked only because FeatureList happens to use a + //
, which recurses back through the inline path. A

, or anything + // beside the text, and the placeholder was published again. + if (sub.trim() && !isLoadingPlaceholder(sub)) segs.push(sub.trim()); + }); + flush(); + return segs; +} + +/** + * A Docusaurus block, rendered as its labelled variants. + * + * Docusaurus emits every panel into the HTML - the inactive ones only carry + * `hidden` - as a