diff --git a/tests/benchmark-acquisition.test.ts b/tests/benchmark-acquisition.test.ts index 29558e6..e3d2a5a 100644 --- a/tests/benchmark-acquisition.test.ts +++ b/tests/benchmark-acquisition.test.ts @@ -3,10 +3,9 @@ import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, it } from "node:test"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { - parseAgenticBenchmarkAcquisitionRequest, parseAgenticBenchmarkAcquisitionReplayResult, + parseAgenticBenchmarkAcquisitionRequest, parseAgenticBenchmarkAcquisitionResult, } from "../src/contracts/index.js"; import { @@ -18,9 +17,8 @@ import { sha256Canonical, } from "../src/core/index.js"; import { acquireAgenticBenchmark, benchmarkCacheDirectoryKey } from "../src/engine/index.js"; -import { runCli } from "../src/cli.js"; -import { createTestForgeServer } from "../src/mcp/index.js"; import { TestForge } from "../src/sdk/index.js"; +import { connectPublicMcp, runPublicCli } from "./support/public-surface.js"; const temporaryDirectories: string[] = []; @@ -436,12 +434,12 @@ describe("benchmark acquisition", () => { stderr += text; }, }; - assert.equal(await runCli(["benchmark-acquire", "-", "--json"], io), 4); + assert.equal(await runPublicCli(["benchmark-acquire", "-", "--json"], io), 4); assert.match(stderr, /--allow-unsafe-execution/); stdout = ""; stderr = ""; assert.equal( - await runCli(["benchmark-acquire", "-", "--allow-unsafe-execution", "--json"], io), + await runPublicCli(["benchmark-acquire", "-", "--allow-unsafe-execution", "--json"], io), 0, stderr, ); @@ -449,14 +447,10 @@ describe("benchmark acquisition", () => { assert.equal(cliResult.status, "COMPLETE"); stdout = ""; stdin = JSON.stringify(cliResult); - assert.equal(await runCli(["benchmark-acquire-replay", "-", "--json"], io), 0); + assert.equal(await runPublicCli(["benchmark-acquire-replay", "-", "--json"], io), 0); assert.equal(JSON.parse(stdout).valid, true); - const disabledServer = createTestForgeServer({ allowedRepositoryRoots: [root] }); - const disabledPair = InMemoryTransport.createLinkedPair(); - await disabledServer.connect(disabledPair[1]); - const disabledClient = new Client({ name: "test", version: "1.0.0" }); - await disabledClient.connect(disabledPair[0]); + const disabledClient = await connectPublicMcp({ root }); assert.ok( !(await disabledClient.listTools()).tools.some( (tool) => tool.name === "testforge_benchmark_acquire", @@ -468,16 +462,8 @@ describe("benchmark acquisition", () => { }); assert.equal((replayed.structuredContent as Record).valid, true); await disabledClient.close(); - await disabledServer.close(); - const server = createTestForgeServer({ - allowUnsafeExecution: true, - allowedRepositoryRoots: [root], - }); - const pair = InMemoryTransport.createLinkedPair(); - await server.connect(pair[1]); - const client = new Client({ name: "test", version: "1.0.0" }); - await client.connect(pair[0]); + const client = await connectPublicMcp({ root, allowUnsafeExecution: true }); const acquired = await client.callTool({ name: "testforge_benchmark_acquire", arguments: { request: request(root) }, @@ -485,6 +471,5 @@ describe("benchmark acquisition", () => { assert.equal(acquired.isError, undefined); assert.equal((acquired.structuredContent as Record).status, "COMPLETE"); await client.close(); - await server.close(); }); }); diff --git a/tests/bun-designated.test.ts b/tests/bun-designated.test.ts index 03ae773..635f9a4 100644 --- a/tests/bun-designated.test.ts +++ b/tests/bun-designated.test.ts @@ -11,7 +11,6 @@ import { classifyBunDesignatedRun, designatedBunConfig, } from "../integrations/bun/driver.mjs"; -import { runCli } from "../src/cli.js"; import { parseEvidenceManifestV4 } from "../src/contracts/index.js"; import { replayEvidenceManifest } from "../src/core/index.js"; import { verifyCampaign } from "../src/engine/index.js"; @@ -25,6 +24,7 @@ import { TEST_SOURCE, writeTree, } from "./support/designated-campaign.js"; +import { runPublicCli } from "./support/public-surface.js"; const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const driver = path.join(repositoryRoot, "integrations", "bun", "driver.mjs"); @@ -667,7 +667,7 @@ describe("v4 designated campaigns", () => { }, }; assert.equal( - await runCli( + await runPublicCli( ["verify", requestFile, "--allow-unsafe-execution", "--allow-windows-native-execution"], io, ), diff --git a/tests/bun-init.test.ts b/tests/bun-init.test.ts index 3b71b7f..3fee0fa 100644 --- a/tests/bun-init.test.ts +++ b/tests/bun-init.test.ts @@ -3,8 +3,8 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { it } from "node:test"; -import { runCli, type CliIo } from "../src/cli.js"; import { initializeRepository } from "../src/engine/index.js"; +import { type PublicCliIo as CliIo, runPublicCli } from "./support/public-surface.js"; it("initializes a Bun test repository with the official v2 adapter configuration", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "assertledger-bun-init-")); @@ -101,7 +101,7 @@ it("accepts an explicit Bun framework in the static CLI doctor", async () => { writeStderr: () => {}, }; assert.equal( - await runCli(["doctor", root, "--framework", "bun:test", "--json"], io), + await runPublicCli(["doctor", root, "--framework", "bun:test", "--json"], io), 0, stdout, ); diff --git a/tests/container-facades.test.ts b/tests/container-facades.test.ts index 704a2cc..077b492 100644 --- a/tests/container-facades.test.ts +++ b/tests/container-facades.test.ts @@ -5,7 +5,6 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, it } from "node:test"; import { fileURLToPath } from "node:url"; -import { runCli } from "../src/cli.js"; import type { EvidenceManifestContract, EvidenceManifestV2Contract, @@ -17,6 +16,7 @@ import { type GitRegressionOptions, type GitRegressionV2Options, } from "../src/sdk/index.js"; +import { runPublicCli } from "./support/public-surface.js"; const FAKE_RUNTIME = fileURLToPath( new URL("./support/fake-container-runtime.mjs", import.meta.url), @@ -43,7 +43,7 @@ async function temporaryDirectory(prefix: string): Promise { async function cli(argv: string[], cwd = process.cwd()) { let stdout = ""; let stderr = ""; - const code = await runCli(argv, { + const code = await runPublicCli(argv, { cwd, readStdin: async () => "", writeStdout: (text) => { diff --git a/tests/container-isolation-docker.test.ts b/tests/container-isolation-docker.test.ts index c557854..48619a5 100644 --- a/tests/container-isolation-docker.test.ts +++ b/tests/container-isolation-docker.test.ts @@ -4,7 +4,6 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, it } from "node:test"; import { fileURLToPath } from "node:url"; -import { runCli } from "../src/cli.js"; import { parseEvidenceManifestV2, parseEvidenceManifestV4 } from "../src/contracts/index.js"; import { replayEvidenceManifest } from "../src/core/index.js"; import { runProcess, verifyCampaign } from "../src/engine/index.js"; @@ -15,6 +14,7 @@ import { KEEPS_DOCS, OVERRUNS_UNDER_FAULT, } from "./support/designated-campaign.js"; +import { runPublicCli } from "./support/public-surface.js"; // Hostile scenarios against a real Docker Engine. They need an operator-provided runtime argv and a // locally present digest-pinned Node.js image, never pull, and fail instead of skipping when @@ -355,7 +355,7 @@ describe("real container isolation", { skip, concurrency: false }, () => { stderr += text; }, }; - const code = await runCli( + const code = await runPublicCli( [ "check", root, @@ -386,7 +386,7 @@ describe("real container isolation", { skip, concurrency: false }, () => { const manifest = parseEvidenceManifestV2(JSON.parse(await readFile(manifestPath, "utf8"))); assert.equal(manifest.decision.status, "VERIFIED"); assert.equal(manifest.evidenceContext.execution.backend.kind, "container"); - assert.equal(await runCli(["replay", manifestPath], io), 0, stderr); + assert.equal(await runPublicCli(["replay", manifestPath], io), 0, stderr); await assertCleanedUp(volumesBefore); }); }); diff --git a/tests/container-transport-options.test.ts b/tests/container-transport-options.test.ts index e7044a2..8316fe0 100644 --- a/tests/container-transport-options.test.ts +++ b/tests/container-transport-options.test.ts @@ -6,7 +6,7 @@ import os from "node:os"; import path from "node:path"; import { it } from "node:test"; import { fileURLToPath } from "node:url"; -import { runCli } from "../src/cli.js"; +import { runPublicCli } from "./support/public-surface.js"; async function refusedWitness(transport: string, capable = false) { const parent = await mkdtemp(path.join(os.tmpdir(), "assertledger-transport-")); @@ -51,7 +51,7 @@ async function refusedWitness(transport: string, capable = false) { import.meta.url, ), ); - const code = await runCli( + const code = await runPublicCli( [ "import-witness", "-", diff --git a/tests/corpus-allocation.test.ts b/tests/corpus-allocation.test.ts index 6a26d8c..6a31ea4 100644 --- a/tests/corpus-allocation.test.ts +++ b/tests/corpus-allocation.test.ts @@ -1,7 +1,6 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { describe, it } from "node:test"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { agenticCorpusAllocationJsonSchema, agenticCorpusAllocationReplayResultJsonSchema, @@ -16,10 +15,9 @@ import { createAgenticCorpusAllocation, replayAgenticCorpusAllocation, } from "../src/core/index.js"; -import { runCli } from "../src/cli.js"; import { parseAgenticCorpusCase } from "../src/evaluation/agentic-corpus.js"; -import { createTestForgeServer } from "../src/mcp/index.js"; import { TestForge } from "../src/sdk/index.js"; +import { connectPublicMcp, runPublicCli } from "./support/public-surface.js"; const SEED_DIGEST = "sha256:0000000000000000000000000000000000000000000000000000000000000001"; const CASE_IDS = [ @@ -233,14 +231,10 @@ describe("Agentic Corpus Allocation v1", () => { }, writeStderr() {}, }; - assert.equal(await runCli(["corpus-allocate", "-", "--json"], cliIo), 0); + assert.equal(await runPublicCli(["corpus-allocate", "-", "--json"], cliIo), 0); assert.deepEqual(JSON.parse(cliOutput.join("")), allocation); - const server = createTestForgeServer(); - const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); - const client = new Client({ name: "corpus-allocation-test", version: "1.0.0" }); - await server.connect(serverTransport); - await client.connect(clientTransport); + const client = await connectPublicMcp({ name: "corpus-allocation-test" }); const allocated = await client.callTool({ name: "testforge_corpus_allocate", arguments: { request: request() }, @@ -252,6 +246,5 @@ describe("Agentic Corpus Allocation v1", () => { }); assert.equal((replayed.structuredContent as { valid: boolean }).valid, true); await client.close(); - await server.close(); }); }); diff --git a/tests/corpus-experiment.test.ts b/tests/corpus-experiment.test.ts index 9705e11..f503685 100644 --- a/tests/corpus-experiment.test.ts +++ b/tests/corpus-experiment.test.ts @@ -36,9 +36,8 @@ import { agenticCorpusProvenanceSigningBytes, agenticCorpusTrustPolicyDigest, } from "../src/evaluation/agentic-corpus.js"; -import { createTestForgeServer } from "../src/mcp/index.js"; -import { runCli } from "../src/cli.js"; import { TestForge } from "../src/sdk/index.js"; +import { connectPublicMcp, listedInputSchemas, runPublicCli } from "./support/public-surface.js"; const SOURCE_IDENTITY_DIGEST = `sha256:${"c".repeat(64)}`; const SOURCE_B_IDENTITY_DIGEST = `sha256:${"2".repeat(64)}`; @@ -1041,7 +1040,7 @@ describe("externally anchored H3 experiment v1", () => { await writeFile(path.join(root, `${name}.json`), JSON.stringify(document), "utf8"); } const diagnostics: string[] = []; - const code = await runCli( + const code = await runPublicCli( [ "corpus-experiment-replay", "artifact.json", @@ -1089,7 +1088,7 @@ describe("externally anchored H3 experiment v1", () => { assert.equal(await run(fixture({ firstBaselineOutcome: "INFRA_ERROR" })), 3); const value = fixture(); assert.equal( - await runCli(["corpus-experiment-replay", "-", "--json"], { + await runPublicCli(["corpus-experiment-replay", "-", "--json"], { cwd: process.cwd(), async readStdin() { return JSON.stringify(value.artifact); @@ -1102,11 +1101,14 @@ describe("externally anchored H3 experiment v1", () => { }); it("does not expose H3 creation or replay as default MCP tools", async () => { - const server = createTestForgeServer(); - const tools = (server as unknown as { _registeredTools: Record }) - ._registeredTools; - assert.equal(Object.hasOwn(tools, "testforge_corpus_experiment"), false); - assert.equal(Object.hasOwn(tools, "testforge_corpus_experiment_replay"), false); - await server.close(); + const client = await connectPublicMcp(); + try { + const tools = await listedInputSchemas(client); + assert.ok(tools.has("testforge_corpus_allocate")); + assert.equal(tools.has("testforge_corpus_experiment"), false); + assert.equal(tools.has("testforge_corpus_experiment_replay"), false); + } finally { + await client.close(); + } }); }); diff --git a/tests/developer-experience.test.ts b/tests/developer-experience.test.ts index d37972a..8d2f8b9 100644 --- a/tests/developer-experience.test.ts +++ b/tests/developer-experience.test.ts @@ -31,6 +31,7 @@ import { runFixtureDemo } from "../src/engine/demo.js"; import { verifyCampaign as executeCampaign, initializeRepository } from "../src/engine/index.js"; import { type RepositorySetupResult, setupRepository } from "../src/engine/setup.js"; import { AssertLedger } from "../src/sdk/index.js"; +import { runPublicCli } from "./support/public-surface.js"; const packageMetadata = JSON.parse( await readFile(new URL("../package.json", import.meta.url), "utf8"), @@ -904,9 +905,10 @@ describe("developer entry points", () => { ); const capture = captureIo(root); - const exitCode = await runCli(["setup", root, "--client", "codex", "--dry-run"], capture.io, { - setupEntry: path.resolve("dist", "cli.js"), - }); + const exitCode = await runPublicCli( + ["setup", root, "--client", "codex", "--dry-run"], + capture.io, + ); assert.equal(exitCode, 4); assert.equal(capture.stderr(), ""); @@ -971,7 +973,7 @@ describe("developer entry points", () => { it("prints typed diagnostics for invalid plain setup arguments", async () => { const capture = captureIo(process.cwd()); - const exitCode = await runCli( + const exitCode = await runPublicCli( ["setup", ".", "--client", "codex", "--dry-run", "--write"], capture.io, ); @@ -1072,7 +1074,7 @@ describe("developer entry points", () => { it("keeps invalid and unexpected demo JSON failures typed and opaque", async () => { const invalidCapture = captureIo(process.cwd()); - assert.equal(await runCli(["demo", "--json", "--unknown"], invalidCapture.io), 64); + assert.equal(await runPublicCli(["demo", "--json", "--unknown"], invalidCapture.io), 64); assert.equal(invalidCapture.stderr(), ""); const invalid = JSON.parse(invalidCapture.stdout()) as { status: string; @@ -1273,7 +1275,7 @@ describe("developer entry points", () => { const before = await readFile(path.join(root, "package.json"), "utf8"); const capture = captureIo(root); - assert.equal(await runCli(["doctor", ".", "--json"], capture.io), 0); + assert.equal(await runPublicCli(["doctor", ".", "--json"], capture.io), 0); const result = parseRepositoryInitResult(JSON.parse(capture.stdout())); assert.equal(result.status, "WOULD_CREATE"); assert.deepEqual(result.requiredOperatorInputs, ["worlds", "candidates"]); @@ -1292,7 +1294,7 @@ describe("developer entry points", () => { const root = await fixtureRepository(); const capture = captureIo(root); - assert.equal(await runCli(["doctor", "."], capture.io), 0); + assert.equal(await runPublicCli(["doctor", "."], capture.io), 0); assert.match(capture.stdout(), /Status: WOULD_CREATE/u); assert.match(capture.stdout(), /Required operator inputs: worlds, candidates/u); assert.match(capture.stdout(), /UNSANDBOXED trusted-local/u); @@ -1303,19 +1305,19 @@ describe("developer entry points", () => { it("supports help and version aliases while preserving empty and unknown usage errors", async () => { for (const argv of [["help"], ["--help"], ["-h"]]) { const capture = captureIo(process.cwd()); - assert.equal(await runCli(argv, capture.io), 0); + assert.equal(await runPublicCli(argv, capture.io), 0); assert.match(capture.stdout(), /Usage: assertledger/u); assert.equal(capture.stderr(), ""); } for (const argv of [["version"], ["--version"], ["-v"]]) { const capture = captureIo(process.cwd()); - assert.equal(await runCli(argv, capture.io), 0); + assert.equal(await runPublicCli(argv, capture.io), 0); assert.equal(capture.stdout(), `assertledger ${packageMetadata.version}\n`); assert.equal(capture.stderr(), ""); } for (const argv of [[], ["unknown"]]) { const capture = captureIo(process.cwd()); - assert.equal(await runCli(argv, capture.io), 64); + assert.equal(await runPublicCli(argv, capture.io), 64); assert.equal(capture.stdout(), ""); assert.match(capture.stderr(), /Usage: assertledger/u); } @@ -1610,7 +1612,7 @@ describe("developer entry points", () => { ["disconnect", ".", "--client", "claude"], ]) { const capture = captureIo(root); - assert.equal(await runCli(argv, capture.io), 64); + assert.equal(await runPublicCli(argv, capture.io), 64); assert.match(capture.stderr(), /Usage: assertledger/u); assert.equal(capture.stdout(), ""); } @@ -1623,7 +1625,7 @@ describe("developer entry points", () => { ["mcp", "--root", "-x"], ]) { const capture = captureIo(process.cwd()); - assert.equal(await runCli(argv, capture.io), 64); + assert.equal(await runPublicCli(argv, capture.io), 64); assert.equal(capture.stdout(), ""); assert.match(capture.stderr(), /Usage: assertledger/u); } diff --git a/tests/diagnostics.test.ts b/tests/diagnostics.test.ts index bf5e932..629d62d 100644 --- a/tests/diagnostics.test.ts +++ b/tests/diagnostics.test.ts @@ -1,15 +1,13 @@ import assert from "node:assert/strict"; import { describe, it } from "node:test"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { runCli } from "../src/cli.js"; import { DiagnosticReportSchema, explainReasonCodes, quoteRepositoryEntry, renderRepositoryLinkRefusal, } from "../src/diagnostics.js"; -import { createAssertLedgerServer } from "../src/mcp/index.js"; import { AssertLedger } from "../src/sdk/index.js"; +import { connectPublicMcp, runPublicCli } from "./support/public-surface.js"; describe("versioned diagnostic guidance", () => { it("explains refusal without turning operational failures into evidence", () => { @@ -63,7 +61,7 @@ describe("versioned diagnostic guidance", () => { const codes = ["NO_ELIGIBLE_CANDIDATE", "REFERENCE_NOT_GREEN"]; const expected = new AssertLedger().explain(codes); let stdout = ""; - const code = await runCli(["explain", ...codes, "--json"], { + const code = await runPublicCli(["explain", ...codes, "--json"], { cwd: process.cwd(), readStdin: async () => "", writeStdout: (text) => { @@ -75,18 +73,13 @@ describe("versioned diagnostic guidance", () => { }); assert.equal(code, 0); assert.deepEqual(JSON.parse(stdout), expected); - const server = createAssertLedgerServer(); - const [ct, st] = InMemoryTransport.createLinkedPair(); - const client = new Client({ name: "diagnostics-client", version: "1.0.0" }); - await server.connect(st); - await client.connect(ct); + const client = await connectPublicMcp({ name: "diagnostics-client" }); try { const result = await client.callTool({ name: "assertledger_explain", arguments: { codes } }); assert.notEqual(result.isError, true); assert.deepEqual(result.structuredContent, expected); } finally { await client.close(); - await server.close(); } }); @@ -131,7 +124,7 @@ describe("versioned diagnostic guidance", () => { [["explain", "not-a-code"], 4], ] as const) { let stdout = ""; - const exitCode = await runCli([...argv], { + const exitCode = await runPublicCli([...argv], { cwd: process.cwd(), readStdin: async () => "", writeStdout: (value) => { diff --git a/tests/git-regression.test.ts b/tests/git-regression.test.ts index 447cd77..c5e782e 100644 --- a/tests/git-regression.test.ts +++ b/tests/git-regression.test.ts @@ -3,8 +3,6 @@ import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, it } from "node:test"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { runCli } from "../src/cli.js"; import { replayEvidenceManifest } from "../src/core/index.js"; import { batchOutputLimitForTesting, @@ -15,8 +13,8 @@ import { runBoundedProcessForTesting, } from "../src/engine/git-regression.js"; import { runProcess } from "../src/engine/index.js"; -import { createAssertLedgerServer } from "../src/mcp/index.js"; import { AssertLedger } from "../src/sdk/index.js"; +import { connectPublicMcp, listedInputSchemas, runPublicCli } from "./support/public-surface.js"; const temporaryDirectories: string[] = []; @@ -111,22 +109,26 @@ describe("Git regression qualification", () => { const value = await fixture(); const options = optionsFor(value, "mcp-evidence"); const { allowUnsafeExecution: _capability, ...request } = options; - const readOnly = createAssertLedgerServer({ allowedRepositoryRoots: [value.root] }); - assert.equal(readOnly.toolInputSchemaJson("assertledger_check"), undefined); - const server = createAssertLedgerServer({ + const readOnly = await connectPublicMcp({ root: value.root }); + try { + const listed = await listedInputSchemas(readOnly); + assert.ok(listed.has("assertledger_analyze")); + assert.equal(listed.has("assertledger_check"), false); + } finally { + await readOnly.close(); + } + const client = await connectPublicMcp({ + root: value.root, allowUnsafeExecution: true, - allowedRepositoryRoots: [value.root], + name: "git-check-client", }); - const schema = server.toolInputSchemaJson("assertledger_check"); - assert.ok(schema); - assert.equal(schema.additionalProperties, false); - assert.equal((schema.properties as Record).allowUnsafeExecution, undefined); - assert.deepEqual(server.toolInputSchemaJson("testforge_check"), schema); - const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); - const client = new Client({ name: "git-check-client", version: "1.0.0" }); - await server.connect(serverTransport); - await client.connect(clientTransport); try { + const listed = await listedInputSchemas(client); + const schema = listed.get("assertledger_check"); + assert.ok(schema); + assert.equal(schema.additionalProperties, false); + assert.equal((schema.properties as Record).allowUnsafeExecution, undefined); + assert.deepEqual(listed.get("testforge_check"), schema); const outside = await fixture(); const rejectedRoot = await client.callTool({ name: "assertledger_check", @@ -150,7 +152,6 @@ describe("Git regression qualification", () => { ); } finally { await client.close(); - await server.close(); } }); @@ -627,7 +628,7 @@ describe("Git regression qualification", () => { it("requires the CLI unsafe flag before parsing or executing Git input", async () => { let stderr = ""; - const code = await runCli(["check", ".", "--json"], { + const code = await runPublicCli(["check", ".", "--json"], { cwd: process.cwd(), readStdin: async () => "", writeStdout: () => undefined, diff --git a/tests/init-doctor-contracts.test.ts b/tests/init-doctor-contracts.test.ts index 6d731a4..b4ab0c8 100644 --- a/tests/init-doctor-contracts.test.ts +++ b/tests/init-doctor-contracts.test.ts @@ -3,10 +3,10 @@ import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promis import os from "node:os"; import path from "node:path"; import { it } from "node:test"; -import { type CliIo, runCli } from "../src/cli.js"; import { parseVersionedRepositoryInitResult } from "../src/contracts/index.js"; import { explainReasonCodes } from "../src/diagnostics.js"; import { initializeRepository } from "../src/engine/index.js"; +import { type PublicCliIo as CliIo, runPublicCli } from "./support/public-surface.js"; function capture(root: string) { let stdout = ""; @@ -60,7 +60,7 @@ for (const framework of ["node:test", "bun:test"] as const) { `${argv[0]}${argv.includes("--dry-run") ? " --dry-run" : ""}`, async () => { const output = capture(root); - assert.equal(await runCli(argv, output.io), 4); + assert.equal(await runPublicCli(argv, output.io), 4); assert.notEqual(output.stdout(), "", output.stderr()); assert.equal(output.stderr(), ""); const result = parseVersionedRepositoryInitResult(JSON.parse(output.stdout())); @@ -115,7 +115,7 @@ it("returns a structured conflict when an inventory evidence path exceeds its sc for (const command of ["doctor", "init"] as const) { const output = capture(root); assert.equal( - await runCli([command, root, "--framework", "node:test", "--json"], output.io), + await runPublicCli([command, root, "--framework", "node:test", "--json"], output.io), 4, ); assert.notEqual(output.stdout(), "", output.stderr()); @@ -145,7 +145,7 @@ it("returns a structured conflict when the canonical inventory lock exceeds the ); } const output = capture(root); - assert.equal(await runCli(["init", root, "--dry-run", "--json"], output.io), 4); + assert.equal(await runPublicCli(["init", root, "--dry-run", "--json"], output.io), 4); assert.notEqual(output.stdout(), "", output.stderr()); const result = parseVersionedRepositoryInitResult(JSON.parse(output.stdout())); assert.equal(result.status, "CONFLICT"); @@ -219,7 +219,7 @@ it("explains every static doctor/init reason and contract error with a next acti assert.match(ambiguous?.nextAction ?? "", /--framework/u); const output = capture(process.cwd()); - assert.equal(await runCli(["explain", ...INIT_CODES, "--json"], output.io), 0); + assert.equal(await runPublicCli(["explain", ...INIT_CODES, "--json"], output.io), 0); assert.deepEqual(JSON.parse(output.stdout()), report); }); diff --git a/tests/integration.test.ts b/tests/integration.test.ts index 911c11c..8c60d17 100644 --- a/tests/integration.test.ts +++ b/tests/integration.test.ts @@ -4,9 +4,9 @@ import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promis import os from "node:os"; import path from "node:path"; import { afterEach, describe, it } from "node:test"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { parseRepositoryInitResult } from "../src/contracts/index.js"; import { sha256Canonical } from "../src/core/index.js"; +import { connectPublicMcp, listedInputSchemas, runPublicCli } from "./support/public-surface.js"; type IntegrationApi = { TestForge: new () => { @@ -34,31 +34,19 @@ type IntegrationApi = { writeStderr(text: string): void; }, ): Promise; - createTestForgeServer(options?: { - allowUnsafeExecution?: boolean; - allowedRepositoryRoots?: string[]; - }): any; decideEvidence(input: unknown): any; sealManifestArtifact(input: unknown): any; }; async function loadIntegration(): Promise { try { - const [ - { TestForge }, - { runCli }, - { createTestForgeServer }, - { decideEvidence, sealManifestArtifact }, - ] = await Promise.all([ + const [{ TestForge }, { decideEvidence, sealManifestArtifact }] = await Promise.all([ import("../src/sdk/index.js"), - import("../src/cli.js"), - import("../src/mcp/index.js"), import("../src/core/index.js"), ]); return { TestForge, - runCli, - createTestForgeServer, + runCli: runPublicCli, decideEvidence, sealManifestArtifact, } as IntegrationApi; @@ -97,7 +85,6 @@ async function loadIntegration(): Promise { } }, runCli: async () => 99, - createTestForgeServer: () => undefined, decideEvidence: () => undefined, sealManifestArtifact: () => undefined, }; @@ -926,16 +913,29 @@ describe("MCP facade", () => { allowUnsafeExecution?: boolean; allowedRepositoryRoots?: string[]; }) { - const server = integration.createTestForgeServer(options); - const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); - const client = new Client({ name: "testforge-integration-test", version: "1.0.0" }); - await server.connect(serverTransport); - await client.connect(clientTransport); + assert.ok((options.allowedRepositoryRoots ?? []).length <= 1); + const client = await connectPublicMcp({ + root: options.allowedRepositoryRoots?.[0], + allowUnsafeExecution: options.allowUnsafeExecution, + name: "testforge-integration-test", + }); + const listed = await listedInputSchemas(client); + // The published input schemas, as `tools/list` returns them to any client. + const server = { + toolInputSchemaJson: (name: string) => listed.get(name), + close: async () => {}, + }; return { client, server }; } - it("is read-only by default and does not register the execution tool", () => { - const server = integration.createTestForgeServer(); + async function listedServer(options: { allowUnsafeExecution?: boolean } = {}) { + const { client, server } = await connectServer(options); + await client.close(); + return server; + } + + it("is read-only by default and does not register the execution tool", async () => { + const server = await listedServer(); assert.ok(server); for (const tool of [ @@ -1044,10 +1044,10 @@ describe("MCP facade", () => { } }); - it("registers verify only when the operator grants unsafe execution", () => { - const schema = integration - .createTestForgeServer({ allowUnsafeExecution: true }) - .toolInputSchemaJson("testforge_verify"); + it("registers verify only when the operator grants unsafe execution", async () => { + const schema = (await listedServer({ allowUnsafeExecution: true })).toolInputSchemaJson( + "testforge_verify", + ); assert.equal(schema.additionalProperties, false); assert.deepEqual(schema.required, ["request"]); @@ -1107,7 +1107,7 @@ describe("MCP facade", () => { it("publishes strict output schemas and the exhaustive facade schema names", async () => { const { client } = await connectServer({ allowUnsafeExecution: true }); - const server = integration.createTestForgeServer({ allowUnsafeExecution: true }); + const server = await listedServer({ allowUnsafeExecution: true }); const listed = await client.listTools(); for (const tool of [ "assertledger_qualification_plan", diff --git a/tests/naming-migration.test.ts b/tests/naming-migration.test.ts index 7555da0..72b832d 100644 --- a/tests/naming-migration.test.ts +++ b/tests/naming-migration.test.ts @@ -3,9 +3,10 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, it } from "node:test"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { createAssertLedgerServer, createTestForgeServer } from "../src/mcp/index.js"; import { AssertLedger, TestForge } from "../src/sdk/index.js"; +import { connectPublicMcp, listedInputSchemas } from "./support/public-surface.js"; + const temporaryDirectories: string[] = []; afterEach(async () => { @@ -75,8 +76,10 @@ describe("MCP naming compatibility", () => { assert.equal(createTestForgeServer, createAssertLedgerServer); }); - it("registers every read-only tool under assertledger_* and testforge_* bound to the identical config", () => { - const server = createAssertLedgerServer(); + it("registers every read-only tool under assertledger_* and testforge_* bound to the identical config", async () => { + const client = await connectPublicMcp(); + const listed = await listedInputSchemas(client); + await client.close(); const pairs = [ "analyze", "benchmark", @@ -96,8 +99,8 @@ describe("MCP naming compatibility", () => { ]; for (const suffix of pairs) { - const preferred = server.toolInputSchemaJson(`assertledger_${suffix}`); - const legacy = server.toolInputSchemaJson(`testforge_${suffix}`); + const preferred = listed.get(`assertledger_${suffix}`); + const legacy = listed.get(`testforge_${suffix}`); assert.ok(preferred, `missing assertledger_${suffix}`); assert.deepEqual( preferred, @@ -107,23 +110,24 @@ describe("MCP naming compatibility", () => { } }); - it("registers assertledger_verify only when the operator grants unsafe execution", () => { - const readOnly = createAssertLedgerServer(); - assert.equal(readOnly.toolInputSchemaJson("assertledger_verify"), undefined); - - const unsafe = createAssertLedgerServer({ allowUnsafeExecution: true }); - const preferred = unsafe.toolInputSchemaJson("assertledger_verify"); - const legacy = unsafe.toolInputSchemaJson("testforge_verify"); + it("registers assertledger_verify only when the operator grants unsafe execution", async () => { + const readOnly = await connectPublicMcp(); + const readOnlyTools = await listedInputSchemas(readOnly); + await readOnly.close(); + assert.ok(readOnlyTools.has("assertledger_analyze")); + assert.equal(readOnlyTools.has("assertledger_verify"), false); + + const unsafe = await connectPublicMcp({ allowUnsafeExecution: true }); + const unsafeTools = await listedInputSchemas(unsafe); + await unsafe.close(); + const preferred = unsafeTools.get("assertledger_verify"); + const legacy = unsafeTools.get("testforge_verify"); assert.ok(preferred); assert.deepEqual(preferred, legacy); }); it("returns byte-identical output when the preferred and legacy tool names are called", async () => { - const server = createAssertLedgerServer(); - const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); - const client = new Client({ name: "naming-migration-test", version: "1.0.0" }); - await server.connect(serverTransport); - await client.connect(clientTransport); + const client = await connectPublicMcp({ name: "naming-migration-test" }); try { const preferred = await client.callTool({ @@ -146,11 +150,7 @@ describe("MCP naming compatibility", () => { it("confines both the preferred and legacy analyze tool to the identical allowed repository root", async () => { const allowed = await fixtureRepository(); const forbidden = await fixtureRepository(); - const server = createAssertLedgerServer({ allowedRepositoryRoots: [allowed] }); - const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); - const client = new Client({ name: "naming-migration-test", version: "1.0.0" }); - await server.connect(serverTransport); - await client.connect(clientTransport); + const client = await connectPublicMcp({ root: allowed, name: "naming-migration-test" }); try { const preferredAccepted = await client.callTool({ @@ -173,7 +173,6 @@ describe("MCP naming compatibility", () => { assert.match(JSON.stringify(preferredRejected.content), /MCP_REPOSITORY_ROOT_FORBIDDEN/); } finally { await client.close(); - await server.close(); } }); }); diff --git a/tests/public-surface-anchoring.test.ts b/tests/public-surface-anchoring.test.ts new file mode 100644 index 0000000..71709ca --- /dev/null +++ b/tests/public-surface-anchoring.test.ts @@ -0,0 +1,74 @@ +/** Ratchet for ADR 0003: in 3.0.0 the CLI and the MCP server are the public contract, so tests reach + * them through the built package (`tests/support/public-surface.ts`), never through their source + * modules. Only the files below may still import those modules, each for a reason the process + * boundary cannot serve. */ +import assert from "node:assert/strict"; +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { test } from "node:test"; +import { PACKAGE_ROOT } from "./support/goldens.js"; + +const TESTS_ROOT = path.join(PACKAGE_ROOT, "tests"); +const SOURCE_SURFACES = [/^src\/cli\.[cm]?[jt]s$/u, /^src\/mcp\//u]; + +const IN_PROCESS_ALLOWLIST = new Map([ + [ + "bun-preflight-diagnostics.test.ts", + "mocks node:child_process inside the CLI process to script the Bun probe", + ], + [ + "developer-experience.test.ts", + "injects setup and demo dependencies, and checks the refusals of a source (unbuilt) run", + ], + [ + "naming-migration.test.ts", + "pins the createTestForgeServer alias of the JS API removed in 3.0.0", + ], +]); + +async function testModules(directory: string): Promise { + const files: string[] = []; + for (const entry of await readdir(directory, { withFileTypes: true })) { + const full = path.join(directory, entry.name); + if (entry.isDirectory()) { + if (entry.name !== "goldens" && entry.name !== "replay-corpus") { + files.push(...(await testModules(full))); + } + } else if (/\.[cm]?[jt]s$/u.test(entry.name)) { + files.push(full); + } + } + return files; +} + +/** Relative specifiers of static imports, re-exports and dynamic imports that land on a source surface. */ +function sourceSurfaceImports(file: string, source: string): string[] { + const specifiers = source.matchAll( + /(?:\bfrom\s*|\bimport\s*\(\s*|^\s*import\s+)["']([^"']+)["']/gmu, + ); + return [...specifiers] + .map((match) => match[1] as string) + .filter((specifier) => specifier.startsWith(".")) + .filter((specifier) => { + const target = path + .relative(PACKAGE_ROOT, path.resolve(path.dirname(file), specifier)) + .split(path.sep) + .join("/"); + return SOURCE_SURFACES.some((surface) => surface.test(target)); + }); +} + +test("tests reach the CLI and MCP server only through the built package, outside the allowlist", async () => { + const importing = new Map(); + for (const file of await testModules(TESTS_ROOT)) { + const found = sourceSurfaceImports(file, await readFile(file, "utf8")); + if (found.length > 0) { + importing.set(path.relative(TESTS_ROOT, file).split(path.sep).join("/"), found); + } + } + + const unlisted = [...importing].filter(([file]) => !IN_PROCESS_ALLOWLIST.has(file)); + assert.deepEqual(unlisted, [], "use tests/support/public-surface.ts instead of a source module"); + const stale = [...IN_PROCESS_ALLOWLIST.keys()].filter((file) => !importing.has(file)); + assert.deepEqual(stale, [], "remove allowlist entries that no longer import a source surface"); +}); diff --git a/tests/qualification-engine.test.ts b/tests/qualification-engine.test.ts index f41e92a..c2d1844 100644 --- a/tests/qualification-engine.test.ts +++ b/tests/qualification-engine.test.ts @@ -4,10 +4,10 @@ import os from "node:os"; import path from "node:path"; import { test } from "node:test"; import { smokeQualificationPlan, smokeTestSource } from "../scripts/qualification-smoke.js"; -import { runCli } from "../src/cli.js"; import type { QualificationPlan } from "../src/contracts/qualification.js"; import { sha256Canonical } from "../src/core/index.js"; import { AssertLedger } from "../src/sdk/index.js"; +import { runPublicCli } from "./support/public-surface.js"; async function capability() { const modulePath = "../src/engine/qualification.js"; @@ -236,7 +236,7 @@ writeFileSync(process.env.ASSERTLEDGER_QUALIFICATION_RESULT_FILE, JSON.stringify assert.equal(ledger.replayQualification(reportedResult).valid, true); } let cliOutput = ""; - const cliCode = await runCli(["qualify", "-", "--allow-unsafe-execution"], { + const cliCode = await runPublicCli(["qualify", "-", "--allow-unsafe-execution"], { cwd: root, readStdin: async () => JSON.stringify({ root, plan, planDigest: sha256Canonical(plan), candidate: { files: [] } }), diff --git a/tests/qualification-facades.test.ts b/tests/qualification-facades.test.ts index 5837f1b..36b81b1 100644 --- a/tests/qualification-facades.test.ts +++ b/tests/qualification-facades.test.ts @@ -6,7 +6,6 @@ import path from "node:path"; import { test } from "node:test"; import { fileURLToPath } from "node:url"; import { smokeQualificationPlan, smokeTestSource } from "../scripts/qualification-smoke.js"; -import { runCli } from "../src/cli.js"; import { sha256Canonical } from "../src/core/index.js"; import { sealQualificationPlan } from "../src/core/qualification.js"; import { @@ -14,6 +13,7 @@ import { qualificationRepositoryDigest, } from "../src/engine/qualification.js"; import { AssertLedger } from "../src/sdk/index.js"; +import { runPublicCli } from "./support/public-surface.js"; test("real CLI qualification and CI import preserve OPEN and REJECTED exits and reject forged replay", async () => { const temporary = await mkdtemp(path.join(tmpdir(), "al-cli-")); @@ -109,8 +109,8 @@ test("CLI names plan, execution and replay surfaces and refuses unauthorized exe output += text; }, }; - assert.equal(await runCli(["--help"], io), 0); + assert.equal(await runPublicCli(["--help"], io), 0); assert.match(output, /qualification-plan/u, "qualification plan command missing"); assert.match(output, /qualification-replay/u); - assert.equal(await runCli(["qualify", "-"], io), 4); + assert.equal(await runPublicCli(["qualify", "-"], io), 4); }); diff --git a/tests/qualification-mcp.test.ts b/tests/qualification-mcp.test.ts index a95202a..d2a827f 100644 --- a/tests/qualification-mcp.test.ts +++ b/tests/qualification-mcp.test.ts @@ -3,7 +3,7 @@ import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { test } from "node:test"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { pathToFileURL } from "node:url"; import { exerciseQualificationHandlers, smokeQualificationPlan, @@ -12,26 +12,25 @@ import { import { sha256Canonical } from "../src/core/index.js"; import { qualificationFileDigest, - qualificationMechanismDigest, qualificationRepositoryDigest, } from "../src/engine/qualification.js"; -import { createAssertLedgerServer } from "../src/mcp/index.js"; import { AssertLedger } from "../src/sdk/index.js"; +import { PACKAGE_ROOT } from "./support/goldens.js"; +import { connectPublicMcp } from "./support/public-surface.js"; + +/** The server runs the built package, so its mechanism is the digest of the built modules. */ +async function builtMechanismDigest(): Promise { + const built = pathToFileURL(path.join(PACKAGE_ROOT, "dist", "engine", "qualification.js")).href; + const engine = (await import(built)) as typeof import("../src/engine/qualification.js"); + return engine.qualificationMechanismDigest(); +} test("MCP seals, executes and replays node:test qualification and refuses safe execution", { timeout: 30_000, }, async () => { const root = await mkdtemp(path.join(os.tmpdir(), "qualification-mcp-")); async function connect(allowUnsafeExecution: boolean) { - const server = createAssertLedgerServer({ - allowedRepositoryRoots: [root], - allowUnsafeExecution, - }); - const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); - const client = new Client({ name: "qualification-handler-test", version: "1.0.0" }); - await server.connect(serverTransport); - await client.connect(clientTransport); - return client; + return connectPublicMcp({ root, allowUnsafeExecution, name: "qualification-handler-test" }); } try { await writeFile(path.join(root, "public.test.mjs"), smokeTestSource); @@ -57,7 +56,7 @@ test("MCP seals, executes and replays node:test qualification and refuses safe e const receipt = await exerciseQualificationHandlers(executing, request); assert.equal( (receipt.provenance.runtime as Record).mechanismDigest, - await qualificationMechanismDigest(), + await builtMechanismDigest(), ); assert.equal(await readFile(path.join(root, "public.test.mjs"), "utf8"), smokeTestSource); const outside = await executing.callTool({ diff --git a/tests/repository-audit.test.ts b/tests/repository-audit.test.ts index 722672f..90bed1a 100644 --- a/tests/repository-audit.test.ts +++ b/tests/repository-audit.test.ts @@ -4,8 +4,8 @@ import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; import { syncBuiltinESMExports } from "node:module"; import os from "node:os"; import path from "node:path"; -import { promisify } from "node:util"; import { afterEach, describe, it } from "node:test"; +import { promisify } from "node:util"; import { parseRepositoryAudit, parseRepositoryAuditV2, @@ -13,7 +13,7 @@ import { } from "../src/contracts/index.js"; import { auditRepository, initializeRepository } from "../src/engine/index.js"; import { AssertLedger } from "../src/sdk/index.js"; -import { runCli } from "../src/cli.js"; +import { runPublicCli } from "./support/public-surface.js"; const temporaryDirectories: string[] = []; const execFileAsync = promisify(execFile); @@ -23,7 +23,7 @@ describe("audit exclusions", () => { const ledger = new AssertLedger(); for (const name of ["repository-audit", "repository-audit-v2"] as const) { let stdout = ""; - const exit = await runCli(["schema", name], { + const exit = await runPublicCli(["schema", name], { cwd: process.cwd(), readStdin: async () => "", writeStdout: (text) => { @@ -115,7 +115,7 @@ describe("audit exclusions", () => { await writeFile(path.join(root, "assertledger.request.json"), JSON.stringify(request(root))); let stdout = ""; let stderr = ""; - const exit = await runCli(["audit", ".", "--no-git", "--json"], { + const exit = await runPublicCli(["audit", ".", "--no-git", "--json"], { cwd: root, readStdin: async () => "", writeStdout: (text) => { @@ -186,7 +186,7 @@ describe("audit exclusions", () => { ); let stdout = ""; let stderr = ""; - const exit = await runCli( + const exit = await runPublicCli( [ "audit", ".", @@ -542,32 +542,32 @@ describe("repository audit v1", () => { }, }; assert.equal( - await runCli(["audit", ".", "--emit-verification-request", "--no-git", "--json"], io), + await runPublicCli(["audit", ".", "--emit-verification-request", "--no-git", "--json"], io), 3, ); await writeFile(path.join(root, "assertledger.request.json"), JSON.stringify(request(root))); messages.stdout = ""; assert.equal( - await runCli(["audit", ".", "--emit-verification-request", "--no-git", "--json"], io), + await runPublicCli(["audit", ".", "--emit-verification-request", "--no-git", "--json"], io), 0, ); const emitted = JSON.parse(messages.stdout) as ReturnType; assert.equal(emitted.isolation.acknowledgedUnsafeExecution, false); messages.stdout = ""; - assert.equal(await runCli(["audit", ".", "--no-git", "--json"], io), 0); + assert.equal(await runPublicCli(["audit", ".", "--no-git", "--json"], io), 0); assert.equal( (JSON.parse(messages.stdout) as { cost: { executions: string } }).cost.executions, "444", ); await writeFile(path.join(root, "invalid.json"), "{}\n"); assert.equal( - await runCli(["audit", ".", "--verification-request", "invalid.json", "--json"], io), + await runPublicCli(["audit", ".", "--verification-request", "invalid.json", "--json"], io), 4, ); const otherRoot = await fixture(); await writeFile(path.join(root, "wrong-root.json"), JSON.stringify(request(otherRoot))); assert.equal( - await runCli(["audit", ".", "--verification-request", "wrong-root.json", "--json"], io), + await runPublicCli(["audit", ".", "--verification-request", "wrong-root.json", "--json"], io), 4, ); }); diff --git a/tests/repository-init.test.ts b/tests/repository-init.test.ts index 82b52a9..1dc99bf 100644 --- a/tests/repository-init.test.ts +++ b/tests/repository-init.test.ts @@ -16,7 +16,6 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, it, type TestContext } from "node:test"; import { promisify } from "node:util"; -import { type CliIo, runCli } from "../src/cli.js"; import { parseRepositoryAnalysis, parseRepositoryInitConfig, @@ -31,6 +30,7 @@ import { import { canonicalize } from "../src/core/index.js"; import { analyzeRepository, initializeRepository } from "../src/engine/index.js"; import { AssertLedger } from "../src/sdk/index.js"; +import { type PublicCliIo as CliIo, runPublicCli } from "./support/public-surface.js"; const temporaryDirectories: string[] = []; const execFileAsync = promisify(execFile); @@ -382,7 +382,7 @@ describe("repository init v1", () => { if (!linked) return; const refused = capture(root); - assert.equal(await runCli(["analyze", "."], refused.io), 4); + assert.equal(await runPublicCli(["analyze", "."], refused.io), 4); assert.equal(refused.stdout(), ""); const lines = refused.stderr().split("\n"); assert.equal(lines[0], "UNSUPPORTED_REPOSITORY_SYMLINK"); @@ -393,11 +393,11 @@ describe("repository init v1", () => { const declared = capture(root); assert.equal( - await runCli(["init", ".", "--exclude", ".local-tools", "--json"], declared.io), + await runPublicCli(["init", ".", "--exclude", ".local-tools", "--json"], declared.io), 0, ); const analyzed = capture(root); - assert.equal(await runCli(["analyze", "."], analyzed.io), 0); + assert.equal(await runPublicCli(["analyze", "."], analyzed.io), 0); assert.equal(analyzed.stderr(), ""); }); @@ -408,7 +408,7 @@ describe("repository init v1", () => { }); const baseline = capture(root); - assert.equal(await runCli(["analyze"], baseline.io), 0); + assert.equal(await runPublicCli(["analyze"], baseline.io), 0); const expected = baseline.stdout(); assert.notEqual(expected, ""); assert.equal(baseline.stderr(), ""); @@ -423,7 +423,7 @@ describe("repository init v1", () => { ["analyze", ".", "--json", "--json"], ]) { const accepted = capture(root); - assert.equal(await runCli(argv, accepted.io), 0, argv.join(" ")); + assert.equal(await runPublicCli(argv, accepted.io), 0, argv.join(" ")); assert.equal(accepted.stdout(), expected, argv.join(" ")); assert.equal(accepted.stderr(), "", argv.join(" ")); } @@ -439,7 +439,7 @@ describe("repository init v1", () => { ["analyze", "missing-directory", "--unknown"], ]) { const refused = capture(root); - assert.equal(await runCli(argv, refused.io), 64, argv.join(" ")); + assert.equal(await runPublicCli(argv, refused.io), 64, argv.join(" ")); assert.equal(refused.stdout(), "", argv.join(" ")); assert.match(refused.stderr(), /^Usage: assertledger /u, argv.join(" ")); } @@ -473,13 +473,13 @@ describe("repository init v1", () => { if (!linked) return; const refused = capture(root); - assert.equal(await runCli(["doctor", "."], refused.io), 4); + assert.equal(await runPublicCli(["doctor", "."], refused.io), 4); assert.match(refused.stdout(), /UNSUPPORTED_REPOSITORY_SYMLINK: /u); assert.doesNotMatch(refused.stdout(), /no explanation in the installed catalogue/u); const planned = capture(root); assert.equal( - await runCli( + await runPublicCli( ["doctor", ".", "--exclude", "vendor", "--exclude", ".local-tools", "--json"], planned.io, ), @@ -498,20 +498,23 @@ describe("repository init v1", () => { ); const written = capture(root); - assert.equal(await runCli(["init", ".", "--exclude", ".local-tools", "--json"], written.io), 0); + assert.equal( + await runPublicCli(["init", ".", "--exclude", ".local-tools", "--json"], written.io), + 0, + ); assert.equal(parseRepositoryInitResult(JSON.parse(written.stdout())).status, "CREATED"); const inherited = capture(root); - assert.equal(await runCli(["doctor", ".", "--json"], inherited.io), 0); + assert.equal(await runPublicCli(["doctor", ".", "--json"], inherited.io), 0); assert.equal(parseRepositoryInitResult(JSON.parse(inherited.stdout())).status, "UNCHANGED"); - assert.equal(await runCli(["analyze", "."], capture(root).io), 0); + assert.equal(await runPublicCli(["analyze", "."], capture(root).io), 0); for (const argv of [ ["doctor", ".", "--exclude"], ["doctor", ".", "--runtime", "--exclude", ".local-tools"], ]) { const usage = capture(root); - assert.equal(await runCli(argv, usage.io), 64, argv.join(" ")); + assert.equal(await runPublicCli(argv, usage.io), 64, argv.join(" ")); } }); @@ -1023,7 +1026,7 @@ describe("repository init v1", () => { "test/base.test.js": "import test from 'node:test';\n", }); const output = capture(root); - assert.equal(await runCli(["init", ".", "--dry-run", "--json"], output.io), 0); + assert.equal(await runPublicCli(["init", ".", "--dry-run", "--json"], output.io), 0); const result = JSON.parse(output.stdout()); assert.equal(result.status, "WOULD_CREATE"); assert.deepEqual(result.nextCommands[0], { @@ -1037,7 +1040,7 @@ describe("repository init v1", () => { { "bun.lock": "", "a.test.ts": "import { test } from 'bun:test';\n" }, ); const bunOutput = capture(bun); - assert.equal(await runCli(["init", ".", "--json"], bunOutput.io), 0); + assert.equal(await runPublicCli(["init", ".", "--json"], bunOutput.io), 0); assert.equal(JSON.parse(bunOutput.stdout()).schemaVersion, "2.0.0"); }); @@ -1059,7 +1062,10 @@ describe("repository init v1", () => { ); const output = capture(cwd); assert.equal( - await runCli(["init", repository, "--adapter-config", "adapter.json", "--json"], output.io), + await runPublicCli( + ["init", repository, "--adapter-config", "adapter.json", "--json"], + output.io, + ), 0, ); assert.equal(JSON.parse(output.stdout()).status, "CREATED"); diff --git a/tests/repository-link-closure.test.ts b/tests/repository-link-closure.test.ts index a16df4f..d66bb49 100644 --- a/tests/repository-link-closure.test.ts +++ b/tests/repository-link-closure.test.ts @@ -5,10 +5,10 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, it } from "node:test"; import { promisify } from "node:util"; -import { type CliIo, runCli } from "../src/cli.js"; import type { RepositoryLinkAssessment } from "../src/engine/index.js"; import { computeTestClosure } from "../src/engine/test-closure.js"; import { AssertLedger } from "../src/sdk/index.js"; +import { type PublicCliIo as CliIo, runPublicCli } from "./support/public-surface.js"; const execFileAsync = promisify(execFile); const temporaryDirectories: string[] = []; @@ -91,7 +91,7 @@ describe("repository links judged against the executed test closure", () => { assert.equal(assessment?.refused, undefined); const cli = capture(root); - assert.equal(await runCli(["doctor", "."], cli.io), 0); + assert.equal(await runPublicCli(["doctor", "."], cli.io), 0); assert.match(cli.stderr(), /Links outside every test closure: 1/u); // The repository analysis still refuses any link: only the closure-aware paths changed. @@ -119,7 +119,7 @@ describe("repository links judged against the executed test closure", () => { }); const cli = capture(root); - assert.equal(await runCli(["doctor", "."], cli.io), 4); + assert.equal(await runPublicCli(["doctor", "."], cli.io), 4); assert.match(cli.stderr(), /Link detail: "src\/linked" \(REPOSITORY_LINK_IN_TEST_CLOSURE\)/u); assert.doesNotMatch(cli.stdout(), /no explanation in the installed catalogue/u); }); diff --git a/tests/runtime-doctor.test.ts b/tests/runtime-doctor.test.ts index d666eac..e13928f 100644 --- a/tests/runtime-doctor.test.ts +++ b/tests/runtime-doctor.test.ts @@ -3,8 +3,6 @@ import { mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, it } from "node:test"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { type CliIo, runCli } from "../src/cli.js"; import type { RepositoryInitResult } from "../src/contracts/index.js"; import { parseRuntimeDoctorResult } from "../src/contracts/runtime-doctor.js"; import { runNodeTestRuntimePreflight } from "../src/engine/adapters/node-test-runtime.js"; @@ -13,8 +11,13 @@ import { type RuntimeDoctorDependencies, runRuntimeDoctorChecks, } from "../src/engine/runtime-doctor.js"; -import { createAssertLedgerServer } from "../src/mcp/index.js"; import { AssertLedger } from "../src/sdk/index.js"; +import { + type PublicCliIo as CliIo, + connectPublicMcp, + listedInputSchemas, + runPublicCli, +} from "./support/public-surface.js"; const temporaryDirectories: string[] = []; @@ -113,23 +116,25 @@ describe("runtime doctor", () => { it("keeps runtime MCP opt-in and matches SDK diagnostics within the allowed root", async () => { const root = await realpath(await fixtureRepository()); const outside = await fixtureRepository(); - assert.equal( - createAssertLedgerServer().toolInputSchemaJson("assertledger_doctor_runtime"), - undefined, - ); - const server = createAssertLedgerServer({ + const readOnly = await connectPublicMcp({ root }); + try { + const listed = await listedInputSchemas(readOnly); + assert.ok(listed.has("assertledger_doctor")); + assert.equal(listed.has("assertledger_doctor_runtime"), false); + } finally { + await readOnly.close(); + } + const client = await connectPublicMcp({ + root, allowUnsafeExecution: true, - allowedRepositoryRoots: [root], + name: "runtime-doctor-client", }); - const schema = server.toolInputSchemaJson("assertledger_doctor_runtime"); - assert.ok(schema); - assert.equal(schema.additionalProperties, false); - assert.deepEqual(server.toolInputSchemaJson("testforge_doctor_runtime"), schema); - const [ct, st] = InMemoryTransport.createLinkedPair(); - const client = new Client({ name: "runtime-doctor-client", version: "1.0.0" }); - await server.connect(st); - await client.connect(ct); try { + const listed = await listedInputSchemas(client); + const schema = listed.get("assertledger_doctor_runtime"); + assert.ok(schema); + assert.equal(schema.additionalProperties, false); + assert.deepEqual(listed.get("testforge_doctor_runtime"), schema); const refused = await client.callTool({ name: "assertledger_doctor_runtime", arguments: { root: outside }, @@ -145,7 +150,6 @@ describe("runtime doctor", () => { assert.deepEqual(observed.structuredContent, expected); } finally { await client.close(); - await server.close(); } }); @@ -354,13 +358,13 @@ describe("runtime doctor", () => { it("exposes strict CLI flags and a stable denied result", async () => { const root = await fixtureRepository(); const denied = captureIo(root); - assert.equal(await runCli(["doctor", ".", "--runtime", "--json"], denied.io), 3); + assert.equal(await runPublicCli(["doctor", ".", "--runtime", "--json"], denied.io), 3); assert.equal(parseRuntimeDoctorResult(JSON.parse(denied.stdout())).status, "BLOCKED"); assert.equal(denied.stderr(), ""); const authorized = captureIo(root); assert.equal( - await runCli( + await runPublicCli( ["doctor", ".", "--runtime", "--allow-unsafe-execution", "--json"], authorized.io, ), @@ -387,7 +391,7 @@ describe("runtime doctor", () => { ["doctor", ".", "--runtime", "--unknown"], ]) { const capture = captureIo(root); - assert.equal(await runCli(argv, capture.io), 64); + assert.equal(await runPublicCli(argv, capture.io), 64); assert.equal(capture.stdout(), ""); assert.match(capture.stderr(), /Usage: assertledger/u); } diff --git a/tests/support/public-surface.ts b/tests/support/public-surface.ts new file mode 100644 index 0000000..6c9a0ab --- /dev/null +++ b/tests/support/public-surface.ts @@ -0,0 +1,100 @@ +/** Drives the built package through its public process boundary only: the CLI as a separate + * process, with the same call shape as the in-process `runCli`, and the MCP server over stdio. + * Tests anchored here never reach a source module, so a later implementation of the same commands + * and tools can be swapped in. */ +import { spawn } from "node:child_process"; +import path from "node:path"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { PACKAGE_ROOT } from "./goldens.js"; + +const CLI_ENTRY = path.join(PACKAGE_ROOT, "dist", "cli.js"); +/** A hung command fails its test instead of keeping the runner alive. */ +const DEFAULT_DEADLINE_MS = 300_000; + +export interface PublicCliIo { + cwd: string; + readStdin?(): Promise; + writeStdout(text: string): void; + writeStderr(text: string): void; +} + +export interface PublicCliOptions { + env?: NodeJS.ProcessEnv; + deadlineMs?: number; +} + +export async function runPublicCli( + argv: readonly string[], + io: PublicCliIo, + options: PublicCliOptions = {}, +): Promise { + const stdin = io.readStdin === undefined ? "" : await io.readStdin(); + const child = spawn(process.execPath, [CLI_ENTRY, ...argv], { + cwd: io.cwd, + env: options.env ?? process.env, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk)); + child.stderr.on("data", (chunk: Buffer) => stderr.push(chunk)); + child.stdin.on("error", () => { + // A command that never reads stdin may exit before the write completes. + }); + child.stdin.end(stdin); + const deadlineMs = options.deadlineMs ?? DEFAULT_DEADLINE_MS; + const exitCode = await new Promise((resolve, reject) => { + const deadline = setTimeout(() => { + child.kill(); + reject(new Error(`CLI ${argv.join(" ")} exceeded ${deadlineMs} ms and was killed`)); + }, deadlineMs); + child.on("error", (error) => { + clearTimeout(deadline); + reject(error); + }); + child.on("close", (code) => { + clearTimeout(deadline); + resolve(code ?? -1); + }); + }); + // Like the in-process writer, an empty stream is never written. + const out = Buffer.concat(stdout).toString("utf8"); + const err = Buffer.concat(stderr).toString("utf8"); + if (out.length > 0) io.writeStdout(out); + if (err.length > 0) io.writeStderr(err); + return exitCode; +} + +export interface PublicMcpOptions { + /** Passed as `--root`; the server otherwise confines itself to its working directory. */ + root?: string | undefined; + allowUnsafeExecution?: boolean | undefined; + cwd?: string; + name?: string; +} + +/** Starts `assertledger mcp` over stdio and returns a connected client; close it to stop the server. */ +export async function connectPublicMcp(options: PublicMcpOptions = {}): Promise { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [ + CLI_ENTRY, + "mcp", + ...(options.root === undefined ? [] : ["--root", options.root]), + ...(options.allowUnsafeExecution === true ? ["--allow-unsafe-execution"] : []), + ], + cwd: options.cwd ?? process.cwd(), + stderr: "pipe", + }); + const client = new Client({ name: options.name ?? "assertledger-public-test", version: "1.0.0" }); + await client.connect(transport); + return client; +} + +/** The input schema each registered tool publishes through `tools/list`, by tool name. */ +export async function listedInputSchemas(client: Client): Promise> { + const { tools } = await client.listTools(); + return new Map(tools.map((tool) => [tool.name, tool.inputSchema])); +} diff --git a/tests/witness-import.test.ts b/tests/witness-import.test.ts index 670a022..5224dbf 100644 --- a/tests/witness-import.test.ts +++ b/tests/witness-import.test.ts @@ -4,10 +4,10 @@ import { mkdir, mkdtemp, readdir, readFile, realpath, rm, stat, writeFile } from import os from "node:os"; import path from "node:path"; import { afterEach, describe, it } from "node:test"; -import { type CliIo, runCli } from "../src/cli.js"; import { parseEvidenceManifestV4, parseWitnessImportRequest } from "../src/contracts/index.js"; import { replayEvidenceManifest } from "../src/core/index.js"; import { AssertLedger } from "../src/sdk/index.js"; +import { type PublicCliIo as CliIo, runPublicCli } from "./support/public-surface.js"; const bun = process.env.ASSERTLEDGER_BUN_EXECUTABLE ?? "bun"; const temporaryDirectories: string[] = []; @@ -175,7 +175,7 @@ describe("witness import", () => { stderr += text; }, }; - assert.equal(await runCli(["import-witness", requestFile, "--out", "evidence"], io), 4); + assert.equal(await runPublicCli(["import-witness", requestFile, "--out", "evidence"], io), 4); assert.match(stderr, /without an authorized backend/u); await assert.rejects(stat(path.join(parent, "evidence")), { code: "ENOENT" }); });