feat: build-generate AI retrieval layer (knowledge graph + index)

.github/workflows/build-docs.yml

@@ -35,6 +35,26 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          # Shallow on EVERY trigger, because this job never publishes what the
+          # full clone would improve.
+          #
+          # The knowledge-extractor sets each page's `last_verified` from that
+          # file's last commit date, and under a depth-1 clone every file
+          # resolves to the same commit, so the plugin emits "" rather than a
+          # constant that looks like data. It warns and carries on; nothing here
+          # asserts on last_verified, so this cannot turn the job red.
+          #
+          # An earlier version of this kept full history on push/schedule, on
+          # the theory that those runs produce the artifact a deploy reads. They
+          # do not: this job has no deploy step and no upload-artifact step, so
+          # the index it builds is discarded with the runner on every trigger,
+          # push included. No workflow in this repo publishes it.
+          #
+          # docs-retrieval-evals.yml is the one that does use the dates - it
+          # reports freshness coverage and fails on --require-freshness - and it
+          # keeps fetch-depth: 0 for that reason.
+          fetch-depth: 1
 
       - name: Setup Node.js
         uses: actions/setup-node@v4

.github/workflows/docs-preview.yml

@@ -21,6 +21,22 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          # Shallow is enough HERE, and only here. last_verified is the one
+          # thing that needs full history - the extractor reads each file's last
+          # commit date, and under a depth-1 clone every file resolves to the
+          # same commit - but the extractor now returns before any git call on a
+          # preview build, because it must not emit its ~22 MB of JSON into the
+          # gh-pages branch that pr-preview-action commits. A full clone would
+          # buy a freshness signal nobody computes. build-docs.yml makes the
+          # same call for the same reason and is shallow on EVERY trigger: it
+          # builds and verifies but never publishes the index, so the dates
+          # would be computed and discarded there too.
+          #
+          # docs-retrieval-evals.yml is the one workflow that keeps
+          # fetch-depth: 0, because it is the one that reads the dates - it
+          # reports freshness coverage and fails on --require-freshness.
+          fetch-depth: 1
 
       - name: Setup Node.js
         uses: actions/setup-node@v4

.github/workflows/docs-retrieval-evals.yml

@@ -0,0 +1,126 @@
+name: Docs Retrieval Evals
+
+# Measures retrieval quality of the build-generated knowledge index against a
+# gold set of real queries (success@k / precision@k / MRR). Runs on PRs that
+# touch docs, the extractor, or the gold set — a regression in descriptions or
+# partials that hurts retrieval fails here. Also runs on push to main, so the
+# deploy branch is checked too (an empty index from a cause outside the PR
+# paths — e.g. a Docusaurus upgrade renaming article selectors — is caught here
+# as well as by the plugin's own fail-loud floor check during the build).
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+      # And the frozen snapshots. While docusaurus.config.ts pins lastVersion to
+      # one of them - a beta cycle - that snapshot IS the corpus this indexes,
+      # so a PR editing it changes exactly what is measured here and would
+      # otherwise run neither eval until after it merged.
+      - 'versioned_docs/**'
+      - 'src/plugins/knowledge-extractor/**'
+      - 'scripts/retrieval-evals/**'
+      - '.github/workflows/docs-retrieval-evals.yml'
+      # The extractor reads RENDERED html, so a component decides what the
+      # indexed text IS - it already carries a special case for FeatureList's
+      # "Loading features..." SSR placeholder, and productKeys() reads
+      # src/data. Without these, a PR that changes what a component renders
+      # (this one changes FeatureList) never runs the evals, and the first
+      # signal is the push-to-main trigger after it has merged.
+      - 'src/components/**'
+      - 'src/data/**'
+      # And src/theme, which decides the MARKUP: MDXComponents.js maps Tabs /
+      # TabItem and every custom component rendered inside article .markdown -
+      # tabsToMd is built entirely around that markup - and DocItem is swizzled.
+      # Swapping the Tabs implementation, or wrapping a component in one more
+      # element, changes the indexed text on hundreds of pages.
+      - 'src/theme/**'
+      # These decide WHAT gets indexed at all: assistantIgnoreFiles and
+      # lastVersion live in the config, and versions.json decides which
+      # trees are frozen. Without them a PR that widens or narrows the
+      # curation list changes the corpus and skips both evals.
+      - 'docusaurus.config.ts'
+      - 'versions.json'
+  push:
+    branches:
+      - main
+
+# Same pattern as docs-preview.yml, for the same reason: every run here is a
+# full yarn build plus two eval passes, and this workflow watches paths a docs
+# PR touches on nearly every push. Without a group they queue and all run; with
+# it, a new push supersedes the one still running. Keyed on the PR number so
+# runs on main are never cancelled by a PR.
+concurrency:
+  group: docs-retrieval-evals-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  retrieval-evals:
+    name: Docs Retrieval Evals
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          # Full history, and unlike build-docs.yml this job USES it.
+          #
+          # The knowledge-extractor takes each page's `last_verified` from its
+          # last commit date. Under the default depth-1 clone every file
+          # resolves to the same single commit, so the plugin deliberately
+          # emits "" rather than a constant that looks like data.
+          #
+          # run.cjs reports freshness coverage from that field - how many
+          # modules carry a date, the oldest and newest, and how many are over
+          # 180 days old - and fails in CI if NOT ONE module has one, which is
+          # what losing this line would look like. So the cost buys a signal
+          # that is actually read, and the check guards the setting that
+          # produces it. Do not drop to depth 1 here without deleting that
+          # report first.
+          #
+          # Deliberately NOT a ranking input: last_verified is a date, and the
+          # scorer tokenises on [a-z0-9]{2,}, so it would enter the term stream
+          # as "2026"/"09"/"18" and match any query mentioning those numbers.
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+          cache: 'yarn'
+
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+
+      - name: Build docs (generates the knowledge retrieval index)
+        id: build
+        run: yarn build
+
+      - name: Run retrieval evals
+        # --require-freshness: this job is the one that checks out full history
+        # on purpose, so a dateless index here means the checkout lost its
+        # fetch-depth. Other callers (build-docs.yml is shallow on PRs) get the
+        # report without the failure.
+        run: node scripts/retrieval-evals/run.cjs --report reports/retrieval-evals.json --require-freshness
+
+      # The gold-set run above measures 20 curated queries. This one asks every
+      # module to retrieve itself, which is the only check that covers pages
+      # nobody wrote a query for - and until now MIN_AUTO_SUCCESS gated nothing
+      # because no workflow passed --auto. Reported separately so a corpus-wide
+      # regression is attributable.
+      - name: Run corpus-wide self-retrieval
+        # Not always(): a gold-set breach above must not hide the corpus-wide
+        # number, but a FAILED BUILD must not add a second red step blaming a
+        # missing index instead - and always() fires on cancellation too.
+        if: ${{ !cancelled() && steps.build.outcome == 'success' }}
+        run: node scripts/retrieval-evals/run.cjs --auto --report reports/retrieval-evals-auto.json
+
+      - name: Upload eval report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: retrieval-evals-report
+          path: |
+            reports/retrieval-evals.json
+            reports/retrieval-evals-auto.json
+          if-no-files-found: ignore

docusaurus.config.ts

@@ -9,6 +9,7 @@ import remarkHideComments from "./src/plugins/remark-hide-comments";
 import remarkOffloadPreviewMedia from "./src/plugins/remark-offload-preview-media";
 import stripPreviewMediaPlugin from "./src/plugins/plugin-strip-preview-media";
 import { UNRELEASED_FRAMEWORK_SLUGS } from "./src/constants/unreleasedFrameworks";
+import knowledgeExtractor from "./src/plugins/knowledge-extractor";
 dotenv.config();  // Load environment variables from .env file
 
 const productionUrl = "https://docs.scandit.com";
@@ -95,19 +96,67 @@ const llmsRootRedirectOnlyDocs: string[] = [
   "docs/system-requirements.mdx",
 ];
 
-// Paths are matched by docusaurus-plugin-llms relative to siteDir (e.g. docs/...).
-const llmsIgnoreFiles: string[] = [
+/**
+ * CURATION: pages no AI consumer should be given at all - deprecated platforms,
+ * features a platform does not have, redirect-only stubs with no prose, and
+ * partials that are not routes. Shared by both AI exports, so a decision about
+ * what an assistant may see is made once.
+ */
+const assistantIgnoreFiles: string[] = [
   "docs/connector-guides/**",
   // Partials are imported into real pages, not standalone doc routes; omit so llms.txt URLs work.
   "docs/partials/**",
   ...llmsRootRedirectOnlyDocs,
   ...llmsIgnoredSdkTrees,
   ...llmsLinuxPartialIgnore,
-  ...llmsNonWebSdkRoots.flatMap((root) =>
-    llmsSharedPartialPageNames.map((name) => `docs/${root}/${name}`),
-  ),
 ];
 
+/**
+ * CORPUS SHAPE, for the llms export only: the same prose repeated under every
+ * non-Web SDK root, deduped to Web so a flat text corpus does not carry nine
+ * copies of it.
+ *
+ * Deliberately NOT shared with the knowledge-extractor, and this is the one
+ * place the two exports differ on purpose. That index exists to route a reader
+ * to the page they are on: someone reading Flutter docs needs the Flutter URL,
+ * not the Web one that happens to hold the canonical copy of the text. Applying
+ * this list there removed about 1,400 modules and with them the ability to
+ * answer "where in the Flutter docs does this live".
+ */
+const llmsDedupedToWeb: string[] = llmsNonWebSdkRoots.flatMap((root) =>
+  llmsSharedPartialPageNames.map((name) => `docs/${root}/${name}`),
+);
+
+/**
+ * CORPUS SHAPE, for the knowledge index only: the feature-availability matrix,
+ * every copy of it.
+ *
+ * The counterpart of llmsDedupedToWeb - each export states its own corpus shape
+ * while both share the curation decision in assistantIgnoreFiles. The llms
+ * export keeps ONE copy of this page, because a flat text corpus can carry a
+ * table; the index keeps none.
+ *
+ * A token-overlap consumer cannot use it and is actively harmed by it. The
+ * matrices are 120 modules over 10 URLs at 23-33 distinct tokens against an
+ * index median of 50, and they name every product - so they score well on ANY
+ * single-product query while answering none of them. Measured: the whole top 10
+ * for the query "sparkscan" was the ten framework copies of this page, with the
+ * SparkScan pages below all of them.
+ *
+ * Nothing is lost. What the table says - which product exists on which
+ * framework - is exactly what the graph's AvailableOn / NotAvailableOn edges
+ * carry, in a form a consumer can query rather than one it has to parse.
+ */
+const knowledgeIndexDeduped: string[] = [
+  "docs/features-by-framework.mdx",
+  ...llmsNonWebSdkRoots.map((root) => `docs/${root}/features-by-framework.mdx`),
+  "docs/sdks/web/features-by-framework.mdx",
+];
+
+// Paths are matched by docusaurus-plugin-llms relative to siteDir (e.g. docs/...).
+const llmsIgnoreFiles: string[] = [...assistantIgnoreFiles, ...llmsDedupedToWeb];
+const knowledgeIndexIgnoreFiles: string[] = [...assistantIgnoreFiles, ...knowledgeIndexDeduped];
+
 // ---------------------------------------------------------------------------
 // SINGLE SOURCE OF TRUTH: docs versions and every `docusaurus_tag` derived
 // from them.
@@ -465,6 +514,30 @@ const config: Config = {
     // reader on 6.28.11 finds the 6.28 API and never the 8.x one.
     apiReferenceTagsByVersionTag: buildApiReferenceTags(docsVersions),
     versionNumberByTag: buildVersionNumberByTag(docsVersions),
+    // The CURATION half of the llms ignore list - what no AI consumer should be
+    // given - exported so the knowledge-extractor applies the same decision
+    // rather than a paraphrase of it. Exporting only `llmsIgnoredSdkTrees` left
+    // the two disagreeing about 39 modules; exporting the WHOLE llms list
+    // over-corrected, because it also carries a corpus-shape decision (see
+    // llmsDedupedToWeb) that a routing index must not inherit.
+    // Neither assistantIgnoreFiles nor knowledgeIndexIgnoreFiles is exported
+    // here. Each consumer takes its own list as a plugin option instead, so
+    // there is one name per consumer rather than a customFields key that has to
+    // be kept in step with it.
+    //
+    // This does NOT keep them out of the client bundle, and it is worth being
+    // exact about that: Docusaurus serialises the resolved config, plugin
+    // options included, so the globs ship either way - the built main.js
+    // carries two `ignoreFiles:` arrays. Moving them was a naming fix, not a
+    // size one.
+    // Which version THIS build serves at the root. Read by the
+    // knowledge-extractor plugin, which cannot otherwise tell a frozen version
+    // served at the root from `current` - and the two need opposite handling.
+    lastVersion: effectiveLastVersion,
+    // Read by the knowledge-extractor, which must NOT emit its ~22 MB of JSON
+    // into a preview: pr-preview-action commits the whole build/ into gh-pages,
+    // so every preview push would add those blobs there permanently.
+    isPreviewBuild,
   },
 
   // Set the production url of your site here
@@ -754,6 +827,48 @@ const config: Config = {
   // gate can check a real build artifact instead of re-deriving the same
   // assumption from this file and agreeing with itself.
   searchTagsManifestPlugin,
+  // Build-generate AI layer: emits /assets/knowledge-retrieval-index.json and
+  // /assets/knowledge-graph.jsonld from the rendered HTML (see
+  // src/plugins/knowledge-extractor).
+  //
+  // Division of labour with docusaurus-plugin-llms above, which also exports for
+  // AI consumers: that plugin ships the PROSE (llms.txt as a link index,
+  // llms-full.txt as the text - 0.08 and 2.28 MiB on this build). This one ships
+  // TYPED METADATA AND EDGES for deciding what to read - intents, audiences,
+  // channels, frameworks, products, cites-API, see-also, availability - with
+  // only an excerpt of the prose, because the text is already published next to
+  // it (10.41 MiB index, 11.62 MiB graph). Neither replaces the other.
+  //
+  // They share assistantIgnoreFiles, so what an assistant may SEE is decided
+  // once. They differ, on purpose, on llmsDedupedToWeb: a flat corpus wants one
+  // copy of prose repeated across frameworks, a routing index needs the module
+  // that points at the framework the reader is actually on.
+  //
+  // Last in the array by convention, NOT by guarantee. Docusaurus runs every
+  // plugin's postBuild through Promise.all (core/lib/commands/build.js), so
+  // array position implies nothing about execution order and this hook races
+  // any other postBuild.
+  //
+  // There are THREE other postBuild hooks, not one: stripPreviewMediaPlugin,
+  // @docusaurus/plugin-client-redirects and docusaurus-plugin-llms.
+  //
+  // stripPreviewMediaPlugin deletes under build/img, which walkHtml skips, and
+  // docusaurus-plugin-llms only writes its own .txt files at the root. But
+  // plugin-client-redirects WRITES <path>/index.html stubs into outDir - the
+  // very files walkHtml enumerates - so this plugin sees a different file count
+  // depending on who wins. The extractor now identifies those stubs by their
+  // meta refresh and excludes them from both terms of its drift ratio, so the
+  // outcome no longer depends on the race.
+  //
+  // A future postBuild that REWRITES page html would still race
+  // non-deterministically and must be sequenced explicitly, not moved above
+  // this line.
+  // The curation list plus this index's own corpus-shape exclusions (see
+  // knowledgeIndexDeduped), passed as a plugin option so this consumer reads it
+  // under one name - the same shape docusaurus-plugin-llms uses above. The
+  // plugin throws if it is missing rather than defaulting to "index
+  // everything", which is the property that matters here.
+  [knowledgeExtractor, { ignoreFiles: knowledgeIndexIgnoreFiles }],
 ],
 
   presets: [

package.json

@@ -19,6 +19,7 @@
     "docs:gate:setup": "node scripts/setup-vale.cjs",
     "verify:search-tags": "node scripts/verify-search-tags.cjs",
     "test:search-facets": "node scripts/test-search-facets.cjs",
+    "docs:retrieval-evals": "node scripts/retrieval-evals/run.cjs",
     "verify:frameworks": "node scripts/verify-frameworks.cjs",
     "test:frameworks": "node scripts/test-frameworks.cjs",
     "test:docs-gate": "node scripts/test-docs-gate.cjs",
@@ -32,9 +33,11 @@
     "@docusaurus/plugin-client-redirects": "3.4.0",
     "@docusaurus/preset-classic": "^3.4.0",
     "@mdx-js/react": "^3.0.0",
+    "cheerio": "1.0.0-rc.12",
     "classnames": "^2.5.1",
     "clsx": "^1.2.1",
     "docusaurus-plugin-sass": "^0.2.5",
+    "gray-matter": "4.0.3",
     "prism-react-renderer": "^2.1.0",
     "react": "^18.0.0",
     "react-dom": "^18.0.0",

scripts/retrieval-evals/baseline.json

@@ -0,0 +1,8 @@
+{
+  "page_success_at_k": 0.8596,
+  "page_mrr": 0.775,
+  "pages": 520,
+  "modules": 4228,
+  "k": 3,
+  "recorded_at": "2026-09-21"
+}

scripts/retrieval-evals/gold-set.json

@@ -0,0 +1,25 @@
+{
+  "_comment": "Retrieval-eval gold set. Each entry is a real user query (seeded from Algolia top-queries + the product/intent taxonomy) mapped to the URL/path substrings a correct answer should live under. A retrieved knowledge module counts as relevant if its `url` contains ANY of `expect`. Add entries as new products/pages land; keep queries in the user's words.",
+  "queries": [
+    { "query": "barcode capture get started", "expect": ["/barcode-capture/get-started"] },
+    { "query": "sparkscan", "expect": ["/sparkscan/"] },
+    { "query": "spark scan get started", "expect": ["/sparkscan/get-started", "/sparkscan/intro"] },
+    { "query": "matrixscan count", "expect": ["/matrixscan-count/"] },
+    { "query": "matrixscan find", "expect": ["/matrixscan-find/", "/matrixscan-pick/"] },
+    { "query": "label capture", "expect": ["/label-capture/"] },
+    { "query": "smart label capture", "expect": ["/label-capture/"] },
+    { "query": "id capture supported documents", "expect": ["/id-capture/supported-documents", "/id-capture/", "id-documents"] },
+    { "query": "id bolt", "expect": ["/id-bolt/"] },
+    { "query": "parser", "expect": ["/parser/"] },
+    { "query": "barcode selection", "expect": ["/barcode-selection/"] },
+    { "query": "add the sdk", "expect": ["/add-sdk"] },
+    { "query": "how to initialize data capture context", "expect": ["/add-sdk", "/get-started"] },
+    { "query": "supported symbologies", "expect": ["barcode-symbologies", "symbology-properties"] },
+    { "query": "qr code scanning", "expect": ["barcode-symbologies", "/barcode-capture/"] },
+    { "query": "data matrix", "expect": ["barcode-symbologies", "symbology-properties"] },
+    { "query": "ai powered barcode scanning", "expect": ["ai-powered-barcode-scanning"] },
+    { "query": "scandit express inventory count", "expect": ["/express/", "inventory-count"] },
+    { "query": "release notes", "expect": ["/release-notes"] },
+    { "query": "agent skills", "expect": ["/agent-skills"] }
+  ]
+}