feat: feat: the index, phase 1: facet schema, 4-digit id ranges, backfill of every demo, index.json and .md twins

AGENTS.md

@@ -104,6 +104,19 @@ Reading failures: locators fail with an ARIA snapshot; traces are in `__traces__
 
 ---
 
+## The gallery index (demo facets)
+
+Every numbered demo (`site/src/demos/<group>/<page>/<Variant>.svelte`, id in `site/src/demos/catalog.ts`) has a `Variant.meta.json` sidecar, and the sidecar carries an `index` block: the facets the gallery filters and searches on. `bun run --cwd packages/ui manifest -- --check` (and every build) fails when the block is missing or wrong, so a new demo cannot ship without it.
+
+- **Facets:** `kind` + `subtype`, `industry[]`, `style[]` (a `designStyle` id or one of the planned recipes), `motion` (`none` state-free, `subtle` state change, `expressive` entrance/scroll/loop, `immersive` physics/3D/shader/cursor-driven scene), `interaction[]`, `layout`, `colorMode`, `density`, `tech[]`, `cost`, `components[]`, `aliases[]` (lowercase search synonyms), `related[]` (catalog ids), `a11y { reducedMotion, keyboard }`, `added` (ISO date). The closed vocabularies live in `packages/ui/scripts/index-schema.ts`; an unknown value fails the check.
+- **Inferred, never hand-typed:** `kind`, `subtype`, `components` (must equal the package imports of the demo), `tech`, `cost`, `added`. For a new demo run `bun packages/ui/scripts/index-backfill.ts` (it keeps every value already in a sidecar and fills these), then write `industry`, `style`, `motion`, `interaction`, `layout`, `colorMode`, `density`, `aliases`, `related` and `a11y.keyboard` by reading what the demo renders. Do not copy another demo's block.
+- **Id ranges:** ids stay append-only; 3 digits to 999, 4 digits from 1000. `ID_RANGES` in `index-schema.ts` reserves a block per kind (components 001-299 and 400-999, motion 300-399 by subtype, sections 1000-2999 in blocks of 100 per subtype, backgrounds 3000-3199, techniques 3200-3399, 3D 3400-3499, shaders 3500-3599, pages 4000-4499, templates 4500-4999). The check fails when an id sits outside the block of its `kind`/`subtype`. Four legacy ids (`256`, `387-389`) are exempt (`LEGACY_OUT_OF_RANGE`); a new id never is.
+- **Publishing gate:** a hand-read facet reaches `index.json`, the markdown twins and every filter only when the one-sided 95% upper bound of its error rate on a seeded, folder-stratified audit is at most 5% (`AUDIT`, `errorUpperBound`, `MAX_ERROR_RATE` in `index-schema.ts`). 0 wrong of 73 reads is "at most ~4% wrong", never "0%". A field above the limit stays in the sidecar, marked `"review": { "<field>": "pending" }`, validated but hidden until it is corrected and re-audited with a new seed; flip it in `AUDIT` only with the new numbers. Only 203 of 382 entries were ever re-read; the other 179 hold the first pass's value as corrected by rules, and are unverified beyond them. Today `industry` is pending (5 wrong of 73, upper bound ~14%). Never fix a field by editing the sample you measured it on: re-audit on entries you have not read.
+- **Output:** the manifest build writes `site/public/api/v1/index.json` (every entry with its facets and per-facet counts) and one markdown twin per id at `site/public/gallery/<id>.md`. Both are generated: never edit them; commit the regenerated files.
+- **Originality:** write every title, summary, alias and prompt in our own words. Never name, link to or credit an external site or product in a demo, sidecar or generated file (`tests/demos-index.test.ts` scans the sidecars).
+
+---
+
 ## Critical rules
 
 0. **Test first, always.** Every bug fix and issue starts with a regression test that reproduces the problem. Run it and confirm it fails on the current code, then fix, then confirm it passes. A PR reports that fail-before/pass-after evidence. Tests exist to prove a behavior or a fix, never to pad coverage: no tests of wiring, copies, mock echoes or incidental defaults. If a test cannot fail for a plausible bug, delete it.

packages/ui/scripts/build-manifest.ts

@@ -27,6 +27,8 @@ import { loadMigrations, migrationMarkdown } from "../src/cli/migrations.ts";
 import { STYLES, type SurfaceStyle } from "../src/lib/theme/define.ts";
 import { DESIGN_STYLES } from "../src/lib/theme/styles.ts";
 import { ENTRY_PAGES, parseRecipes, type Recipe } from "./ai-entry.ts";
+import { buildIndexFiles } from "./index-build.ts";
+import { checkIdRange, checkIndexFacets, ID_PATTERN, type IndexFacets } from "./index-schema.ts";
 import { demoSources, mdxToMarkdown } from "./mdx-to-markdown.ts";
 
 // CJS require keeps svelte2tsx out of Vite's transform pipeline (its sourceMappingURL points at a file it does not ship).
@@ -1148,6 +1150,8 @@ export interface ManifestDemo {
 	/** Exact timing of the motion, printed in e/ and p/ so an AI never guesses it. */
 	motion?: ManifestMotion[];
 	shape: "square" | "wide" | "tall";
+	/** Index facets of the sidecar (`index.json`, `/gallery/<id>.md`). */
+	index: IndexFacets;
 	components: string[];
 	imports: { from: string; names: string[] }[];
 	interactive: boolean;
@@ -1197,7 +1201,7 @@ function demoUsages() {
 
 type DemoMeta = Pick<
 	ManifestDemo,
-	"summary" | "use" | "spec" | "tags" | "shape" | "names" | "motion"
+	"summary" | "use" | "spec" | "tags" | "shape" | "names" | "motion" | "index"
 >;
 
 const META_KEYS: Record<string, true> = {
@@ -1208,6 +1212,7 @@ const META_KEYS: Record<string, true> = {
 	shape: true,
 	names: true,
 	motion: true,
+	index: true,
 };
 const MOTION_REQUIRED = ["property", "duration", "easing"] as const;
 const MOTION_OPTIONAL = ["delay", "stagger", "spring"] as const;
@@ -1217,7 +1222,11 @@ const isLines = (v: unknown): v is string[] =>
 	Array.isArray(v) && v.every((s) => typeof s === "string" && s.trim() !== "");
 
 /** Problems of one sidecar against its demo source: shape of every field, and spec tokens quoted from the source. */
-export function checkMeta(meta: unknown, source: string): string[] {
+export function checkMeta(
+	meta: unknown,
+	source: string,
+	ctx: { known?: ReadonlySet<string>; ids?: ReadonlySet<string>; self?: string } = {}
+): string[] {
 	if (!meta || typeof meta !== "object" || Array.isArray(meta)) return ["not an object"];
 	const m = meta as Record<string, unknown>;
 	const errors = Object.keys(m)
@@ -1235,6 +1244,7 @@ export function checkMeta(meta: unknown, source: string): string[] {
 	if (typeof m.shape !== "string" || !SHAPES[m.shape]) errors.push("shape: square | wide | tall");
 	if ("names" in m && (!isLines(m.names) || m.names.length < 1))
 		errors.push("names: non-empty strings, at least one");
+	if ("index" in m) errors.push(...checkIndexFacets(m.index, ctx));
 	if ("motion" in m) {
 		if (!Array.isArray(m.motion) || m.motion.length < 1)
 			errors.push("motion: at least one entry when present");
@@ -1263,9 +1273,36 @@ export function checkMeta(meta: unknown, source: string): string[] {
 	return errors;
 }
 
-/** Problems that make the registry invalid: unregistered demos, missing files, duplicate ids/paths, missing or inaccurate sidecars. */
-export function checkCatalog(catalog: readonly { id: string; path: string }[]): string[] {
+/** Library exports a demo imports from the package (the `components` facet is derived from these, never hand-typed). */
+export function importedNames(source: string): string[] {
+	return [
+		...new Set(
+			[
+				...source.matchAll(/import\s*\{([^}]+)\}\s*from\s*["']@codeatlasdev\/ui[^"']*["']/g),
+			].flatMap(([, names]) =>
+				names
+					.split(",")
+					.map(
+						(n) =>
+							n
+								.trim()
+								.replace(/^type\s+/, "")
+								.split(/\s+as\s+/)[0]
+					)
+					.filter(Boolean)
+			)
+		),
+	];
+}
+
+/** Problems that make the registry invalid: unregistered demos, missing files, duplicate ids/paths, missing or inaccurate sidecars, missing or inconsistent index facets. */
+export function checkCatalog(
+	catalog: readonly { id: string; path: string }[],
+	components: readonly { name: string }[]
+): string[] {
 	const errors: string[] = [];
+	const ids = new Set(catalog.map((d) => d.id));
+	const known = new Set(components.map((c) => c.name));
 	const walk = (dir: string, ext: string): string[] =>
 		readdirSync(dir, { withFileTypes: true }).flatMap((e) =>
 			e.isDirectory()
@@ -1279,10 +1316,11 @@ export function checkCatalog(catalog: readonly { id: string; path: string }[]):
 		if (!HELPERS.includes(p) && !paths.has(p)) errors.push(`demo without id: ${p}`);
 	for (const p of walk(DEMOS_DIR, ".meta.json"))
 		if (!paths.has(p)) errors.push(`sidecar without demo: ${p}.meta.json`);
-	const ids = new Set<string>();
+	const seen = new Set<string>();
 	for (const d of catalog) {
-		if (ids.has(d.id)) errors.push(`duplicate id: ${d.id}`);
-		ids.add(d.id);
+		if (seen.has(d.id)) errors.push(`duplicate id: ${d.id}`);
+		seen.add(d.id);
+		if (!ID_PATTERN.test(d.id)) errors.push(`#${d.id} id: 3 digits up to 999, 4 digits from 1000`);
 		if (!existsSync(join(DEMOS_DIR, `${d.path}.svelte`))) {
 			errors.push(`#${d.id} missing file: ${d.path}`);
 			continue;
@@ -1300,7 +1338,21 @@ export function checkCatalog(catalog: readonly { id: string; path: string }[]):
 			continue;
 		}
 		const source = readFileSync(join(DEMOS_DIR, `${d.path}.svelte`), "utf8");
-		for (const e of checkMeta(meta, source)) errors.push(`#${d.id} ${d.path}.meta.json: ${e}`);
+		for (const e of checkMeta(meta, source, { known, ids, self: d.id }))
+			errors.push(`#${d.id} ${d.path}.meta.json: ${e}`);
+		const facets = (meta as { index?: IndexFacets }).index;
+		if (!facets) {
+			errors.push(`#${d.id} ${d.path}.meta.json: index: facets required (bun run index:backfill)`);
+			continue;
+		}
+		const range = checkIdRange(d.id, facets.kind, facets.subtype);
+		if (range) errors.push(`#${d.id} ${d.path}.meta.json: index: ${range}`);
+		const used = importedNames(source).filter((n) => known.has(n));
+		const listed = [...facets.components].sort().join();
+		if (listed !== [...used].sort().join())
+			errors.push(
+				`#${d.id} ${d.path}.meta.json: index.components must be the imports of the demo: ${used.join(", ") || "none"}`
+			);
 	}
 	if (paths.size !== catalog.length) errors.push("duplicate path in catalog");
 	return errors;
@@ -1359,6 +1411,7 @@ export function buildDemos(
 			...(meta.names && { names: meta.names }),
 			...(meta.motion && { motion: meta.motion }),
 			shape: meta.shape,
+			index: meta.index,
 			components,
 			imports,
 			interactive: use?.interactive ?? false,
@@ -1594,7 +1647,7 @@ if (import.meta.main) {
 		? (await import("../../../site/src/demos/catalog.ts")).CATALOG
 		: [];
 	// Every run (build included) refuses an invalid registry: a demo without an id must never ship.
-	const errors = catalog.length ? checkCatalog(catalog) : [];
+	const errors = catalog.length ? checkCatalog(catalog, manifest.components) : [];
 	if (errors.length) {
 		console.error(`demo catalog (site/src/demos/catalog.ts):\n  ${errors.join("\n  ")}`);
 		process.exit(1);
@@ -1634,6 +1687,12 @@ if (import.meta.main) {
 			: { demos: [], files: {} };
 		if (demos.length) {
 			write("demos.json", demos);
+			const indexFiles = buildIndexFiles(demos, manifest.version);
+			const siteRoot = resolve(ROOT, "../../site/public");
+			for (const [name, body] of Object.entries(indexFiles)) {
+				mkdirSync(dirname(join(siteRoot, name)), { recursive: true });
+				writeFileSync(join(siteRoot, name), body);
+			}
 			for (const base of [join(dist, "api/v1"), resolve(ROOT, "../../site/public/api/v1")])
 				for (const [name, body] of Object.entries(files)) {
 					mkdirSync(dirname(join(base, name)), { recursive: true });

packages/ui/scripts/index-backfill.ts

@@ -0,0 +1,140 @@
+/**
+ * Writes the `index` facet block into every demo sidecar.
+ *
+ *   bun packages/ui/scripts/index-backfill.ts [--infer] [--apply <curated.json>]
+ *
+ * `--infer` derives what the code can prove: kind, subtype, components (the package imports), tech, cost,
+ * the reduced-motion flag and the added date (first commit that registered the id). It keeps any value a
+ * human already curated in the sidecar. `--apply` merges a hand-read file `{ "<id>": { style, industry,
+ * motion, layout, colorMode, density, interaction, aliases, keyboard, related } }` over it. The result is
+ * validated by `checkCatalog` (`bun run --cwd packages/ui manifest -- --check`).
+ */
+import { execFileSync } from "node:child_process";
+import { readFileSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { CATALOG } from "../../../site/src/demos/catalog.ts";
+import { buildManifest, importedNames } from "./build-manifest.ts";
+import { type IndexFacets, PENDING_FACETS, SUBTYPES } from "./index-schema.ts";
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
+const DEMOS = join(ROOT, "site/src/demos");
+
+const MOTION_SUBTYPE: Record<string, string> = {
+	basics: "basics",
+	text: "text",
+	scroll: "scroll",
+	micro: "micro",
+	transitions: "transition",
+	"hero-ambient": "ambient",
+};
+const COMPONENT_SUBTYPE: Record<string, string> = {
+	inputs: "input",
+	display: "display",
+	feedback: "feedback",
+	navigation: "navigation",
+	surfaces: "surface",
+	layout: "layout",
+	charts: "chart",
+	foundation: "foundation",
+	map: "map",
+	"docs-kit": "display",
+};
+
+/** `kind` and `subtype` from the demo's folder. */
+export function inferKind(path: string): Pick<IndexFacets, "kind" | "subtype"> {
+	const [group, page] = path.split("/");
+	if (group === "motion") return { kind: "motion", subtype: MOTION_SUBTYPE[page] ?? "micro" };
+	const subtype = COMPONENT_SUBTYPE[group];
+	if (!subtype || !SUBTYPES.component.includes(subtype)) throw new Error(`no subtype for ${path}`);
+	return { kind: "component", subtype };
+}
+
+/** What the source proves about how the demo is built. */
+export function inferTech(source: string): Pick<IndexFacets, "tech" | "cost"> {
+	const tech = new Set<string>(["css"]);
+	if (
+		/\bspring\b|Spring\b|Tween\b|\banimate\(|requestAnimationFrame|IntersectionObserver|MotionStage/.test(
+			source
+		)
+	)
+		tech.add("svelte-motion");
+	if (/<canvas|getContext\(/.test(source)) tech.add("canvas2d");
+	if (/feTurbulence|feDisplacementMap|<filter/.test(source)) tech.add("svg-filter");
+	if (/webgl/i.test(source)) tech.add("webgl");
+	if (/startViewTransition|view-transition/.test(source)) tech.add("view-transitions");
+	const cost = tech.has("webgl") ? "gpu" : tech.size === 1 ? "css" : "js-light";
+	return { tech: [...tech], cost };
+}
+
+/** Date (YYYY-MM-DD) of the commit that first listed the id in catalog.ts. */
+function addedDates(): Map<string, string> {
+	const log = execFileSync(
+		"git",
+		["log", "--reverse", "--format=COMMIT %as", "-p", "-U0", "--", "site/src/demos/catalog.ts"],
+		{ cwd: ROOT, encoding: "utf8", maxBuffer: 256 * 1024 * 1024 }
+	);
+	const out = new Map<string, string>();
+	let date = "";
+	for (const line of log.split("\n")) {
+		if (line.startsWith("COMMIT ")) date = line.slice(7);
+		else {
+			const m = line.match(/^\+\s*\{ id: "(\d+)"/);
+			if (m && !out.has(m[1])) out.set(m[1], date);
+		}
+	}
+	return out;
+}
+
+if (import.meta.main) {
+	const applyAt = process.argv.indexOf("--apply");
+	const curated: Record<string, Partial<IndexFacets> & { keyboard?: boolean }> =
+		applyAt > 0 ? JSON.parse(readFileSync(process.argv[applyAt + 1], "utf8")) : {};
+	const known = new Set(buildManifest().components.map((c) => c.name));
+	const dates = addedDates();
+	let written = 0;
+	for (const { id, path } of CATALOG) {
+		const file = join(DEMOS, `${path}.meta.json`);
+		const meta = JSON.parse(readFileSync(file, "utf8"));
+		const source = readFileSync(join(DEMOS, `${path}.svelte`), "utf8");
+		const prior = (meta.index ?? {}) as Partial<IndexFacets>;
+		const hand = curated[id] ?? {};
+		const { keyboard, ...handFacets } = hand;
+		const inferred = {
+			...inferKind(path),
+			...inferTech(source),
+			components: importedNames(source).filter((n) => known.has(n)),
+			added: dates.get(id) ?? new Date().toISOString().slice(0, 10),
+		};
+		const reducedMotion =
+			/prefers-reduced-motion|\.reduced\b|MotionStage/.test(source) ||
+			!inferred.tech.includes("svelte-motion");
+		const index = {
+			kind: inferred.kind,
+			subtype: inferred.subtype,
+			industry: handFacets.industry ?? prior.industry,
+			style: handFacets.style ?? prior.style,
+			motion: handFacets.motion ?? prior.motion,
+			interaction: handFacets.interaction ?? prior.interaction,
+			layout: handFacets.layout ?? prior.layout,
+			colorMode: handFacets.colorMode ?? prior.colorMode,
+			density: handFacets.density ?? prior.density,
+			tech: inferred.tech,
+			cost: inferred.cost,
+			components: inferred.components,
+			aliases: handFacets.aliases ?? prior.aliases ?? [],
+			related: handFacets.related ?? prior.related ?? [],
+			a11y: { reducedMotion, keyboard: keyboard ?? prior.a11y?.keyboard ?? false },
+			added: prior.added ?? inferred.added,
+			...(PENDING_FACETS.length && {
+				review: Object.fromEntries(PENDING_FACETS.map((f) => [f, "pending"])),
+			}),
+		};
+		const { index: _drop, ...rest } = meta;
+		writeFileSync(file, `${JSON.stringify({ ...rest, index }, null, "\t")}\n`);
+		written++;
+	}
+	// Same formatter as the rest of the repo, so the diff of a sidecar is only its `index` key.
+	execFileSync("bunx", ["biome", "format", "--write", DEMOS], { cwd: ROOT, stdio: "ignore" });
+	console.log(`index facets written to ${written} sidecars`);
+}