diff --git a/README.md b/README.md index 7eebea9..fdc2039 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ the trigger, policy decisions, tool calls, approvals, the reply, and what it cos pnpm typecheck # strict TypeScript, no build step (tsx runs source) pnpm eval # 500+ deterministic checks, no API key needed RUN_LLM_EVALS=1 pnpm eval # plus live checks: grounded Q&A, red team, answer golden set per model -pnpm mutate # sabotage each of 408 guardrails; its eval must catch it +pnpm mutate # sabotage each of 412 guardrails; its eval must catch it pnpm auditlog verify # the audit log's hash chain holds ``` diff --git a/evals/retrieval-contract.test.ts b/evals/retrieval-contract.test.ts new file mode 100644 index 0000000..e284cfc --- /dev/null +++ b/evals/retrieval-contract.test.ts @@ -0,0 +1,87 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { Vault } from "@scriptorium/core"; +import { buildIndex, withEmbeddings, type Embedder, type SearchHit, type VaultIndex } from "@scriptorium/curator"; + +/** + * What every retriever behind `search_vault` promises, whichever it is: BM25 alone, or BM25 + * fused with embeddings. A retriever swapped in later runs this same contract. + */ + +const CONCEPTS: Record = { spreadsheet: 0, csv: 0, import: 1, thousands: 1, people: 2, subscribers: 2, downtime: 3, maintenance: 3 }; +const conceptEmbedder: Embedder = { + name: "concepts", + embedDocuments: async (texts) => texts.map(embed), + embedQuery: async (text) => embed(text), +}; +function embed(text: string): number[] { + const vector = [0, 0, 0, 0]; + for (const word of text.toLowerCase().split(/[^a-z]+/)) { + const dim = CONCEPTS[word]; + if (dim !== undefined) vector[dim] = (vector[dim] ?? 0) + 1; + } + return vector; +} + +const RETRIEVERS: Array<[string, (vault: Vault) => Promise]> = [ + ["BM25", (vault) => buildIndex(vault)], + ["hybrid", async (vault) => withEmbeddings(vault, await buildIndex(vault), conceptEmbedder)], +]; + +/** As the Q&A tool asks: the async search where there is one. */ +async function find(index: VaultIndex, query: string, limit?: number): Promise { + return index.searchAsync ? index.searchAsync(query, limit) : index.search(query, limit); +} + +let tmpRoot: string; +let vault: Vault; + +beforeEach(async () => { + tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "scriptorium-retrieval-contract-")); + vault = new Vault(tmpRoot); + await vault.ensure(); + for (let i = 0; i < 8; i += 1) await vault.writeNote(`docs/maintenance-${i}.md`, `# Maintenance window ${i}\n\nAnnounce planned downtime for maintenance ${i}.`, {}); + await vault.writeNote("_lessons/L-003-csv.md", "Always import subscribers from a CSV file.", { id: "L-003", status: "rejected", rejected_by: "Alex Kim" }); + await vault.writeNote("_memory/M-1.md", "Priya keeps a spreadsheet of people to import.", { id: "M-1", status: "approved", scope: "person:slack:U1" }); + await vault.writeNote("_gaps/G-1.md", "Someone asked how to import a spreadsheet of thousands.", {}); + await vault.writeNote("_inbox/unreleased.md", "# Unreleased\n\nImport thousands of people from a spreadsheet, launching next quarter.", {}); +}); + +afterEach(async () => { + await fs.rm(tmpRoot, { recursive: true, force: true, maxRetries: 5 }); +}); + +for (const [name, build] of RETRIEVERS) { + describe(`the retrieval contract: ${name}`, () => { + it("returns hits a citation can name, at most as many as asked for", async () => { + const hits = await find(await build(vault), "maintenance downtime", 3); + expect(hits.length).toBeGreaterThan(0); + expect(hits.length).toBeLessThanOrEqual(3); + for (const hit of hits) { + expect(hit.relPath).toMatch(/^docs\/maintenance-\d\.md$/); + expect(typeof hit.title).toBe("string"); + expect(typeof hit.score).toBe("number"); + expect(hit.snippet.length).toBeLessThanOrEqual(240); + } + }); + + it("never returns a private note, even asked in its own words", async () => { + const index = await build(vault); + for (const query of ["Priya keeps a spreadsheet of people to import", "Someone asked how to import a spreadsheet of thousands", "Unreleased import thousands of people launching next quarter"]) { + for (const hit of await find(index, query, 20)) expect(hit.relPath, query).not.toMatch(/^_(memory|gaps|inbox)\//); + } + }); + + it("a rule a human refused carries its notice, however it was found", async () => { + const index = await build(vault); + // In its own words, and (where there are embeddings) in none of them. + for (const query of ["import subscribers from a CSV", ...(name === "hybrid" ? ["spreadsheet of thousands"] : [])]) { + const refused = (await find(index, query, 10)).find((hit) => hit.relPath === "_lessons/L-003-csv.md"); + expect(refused, query).toBeDefined(); + expect(refused?.notice, query).toMatch(/^REJECTED — Alex Kim/); + } + }); + }); +} diff --git a/scripts/mutate.ts b/scripts/mutate.ts index 342feee..31f823d 100644 --- a/scripts/mutate.ts +++ b/scripts/mutate.ts @@ -2710,6 +2710,34 @@ export const MUTANTS: Mutant[] = [ replace: " if (true) await moveTo(ctx, key, pending.to, status);", evals: ["evals/jira-board.test.ts"], }, + { + control: "the embedding half never indexes a private note", + file: "packages/curator/src/hybrid.ts", + find: " if (isPrivateNote(relPath)) continue;\n", + replace: "", + evals: ["evals/retrieval-contract.test.ts", "evals/hybrid-retrieval.test.ts"], + }, + { + control: "a refused rule found only by meaning keeps its notice", + file: "packages/curator/src/hybrid.ts", + find: ", ...(doc.notice ? { notice: doc.notice } : {}) };", + replace: " };", + evals: ["evals/retrieval-contract.test.ts"], + }, + { + control: "retrieval falls back to BM25 when embeddings are down", + file: "packages/curator/src/hybrid.ts", + find: " console.warn(`[curator] embeddings unavailable, BM25 only: ${error instanceof Error ? error.message : error}`);\n return base;", + replace: " throw error;", + evals: ["evals/hybrid-retrieval.test.ts"], + }, + { + control: "a hybrid search returns at most what was asked for", + file: "packages/curator/src/hybrid.ts", + find: " const fused = reciprocalRankFusion([lexical.map((hit) => hit.relPath), semantic]).slice(0, limit);", + replace: " const fused = reciprocalRankFusion([lexical.map((hit) => hit.relPath), semantic]);", + evals: ["evals/retrieval-contract.test.ts"], + }, { control: "a delegation ends by itself", file: "apps/agents/src/teammate-bot/control.ts", diff --git a/site/data.js b/site/data.js index 72ebeea..6cd2adb 100644 --- a/site/data.js +++ b/site/data.js @@ -7,7 +7,7 @@ window.SITE = { stats: [ { value: "730+", label: "evals, deterministic, on every commit" }, - { value: "408", label: "guardrails proven by mutation testing" }, + { value: "412", label: "guardrails proven by mutation testing" }, { value: "30+", label: "tools, every write approve-tier" }, { value: "78", label: "reviewed pull requests merged" }, ],