From 7036eafe8b623fbe7f568209a9ac6a653f507d64 Mon Sep 17 00:00:00 2001 From: Sean Perkins <1733750+seanperkins@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:02:06 -0400 Subject: [PATCH 1/2] fix: publish immutable compiled host generations (#272) --- README.md | 23 ++- scripts/run-compiled.mjs | 244 ++++++++++++++--------------- tests/scripts/run-compiled.test.ts | 217 +++++++++++++++++++++++++ tsconfig.build.json | 4 +- 4 files changed, 362 insertions(+), 126 deletions(-) create mode 100644 tests/scripts/run-compiled.test.ts diff --git a/README.md b/README.md index 70b4971..d534d23 100644 --- a/README.md +++ b/README.md @@ -212,12 +212,31 @@ npm run init-worker -- --install-launchd # + install a KeepAlive LaunchAgent (m `--install-launchd` writes `~/Library/LaunchAgents/com.switchyard.worker.plist` and loads it: the worker starts immediately, restarts if it crashes (a clean stop stays down — `launchctl unload` to stop it), and comes back after reboot. -No secrets and no shell are involved in the plist — launchd execs `tsx` -directly and the worker itself reads the repo `.env` (0600) at start. Two +No secrets and no shell are involved in the plist — launchd execs `node +scripts/run-compiled.mjs ` and the worker itself reads the repo `.env` +(0600) at start. Two worker loops can't run at once: the loop takes a pidfile lock (`.superpowers/worker.pid`), and the installer additionally refuses to load the LaunchAgent while any worker process is running. +The launcher compiles host services on demand, or ahead of time with +`npm run build:server`. It fingerprints source, configuration, dependency +lockfiles, the installed TypeScript compiler, and the Node version. Completed +builds live in `dist/server/<64-character hash>/`; the extra path component +is intentional. Each service imports its own immutable generation, so another +startup cannot replace files underneath it, including files imported later. +Concurrent cold starts may briefly run multiple compilers, but publication is +atomic and only one complete generation is retained per fingerprint. A failed +compile fails startup and leaves earlier builds intact. + +`dist/server/build-info.json` records the last published generation for +inspection; services do not follow this pointer after startup. Old generations +are retained because running services may still need them, so disk usage grows +with changed builds. To reclaim the cache, stop **all** host services, remove +`dist/server`, run `npm run build:server`, then restart the services. This also +cleans staging directories left by a forcibly killed compiler and legacy +`dist/server/scripts` output. Do not delete generations while services run. + To run it by hand instead: ```bash diff --git a/scripts/run-compiled.mjs b/scripts/run-compiled.mjs index b2abe39..d5288d0 100644 --- a/scripts/run-compiled.mjs +++ b/scripts/run-compiled.mjs @@ -1,95 +1,92 @@ #!/usr/bin/env node -// SYD-268: launcher for the compiled host scripts (dist/server/scripts/*.js). +// SYD-268: one resident node process per host service, without a tsx loader. +// #272: never compile into a directory a running service can import from. +// Each content-addressed generation is built privately, then published with +// one directory rename. Concurrent builders may do redundant work, but only +// a complete generation wins. There is no lock to strand after a crash. // -// Plain JS on purpose — this file is never itself compiled, so it needs no -// build step to run. It exists so launchd runs one plain `node` process per -// service (node -> run-compiled.mjs -> dist/server/scripts/.js) -// instead of the old node -> tsx shim -> node+esbuild loader chain, without -// ever risking a worker that silently runs month-old compiled code: -// -// 1. compute a content hash over the compile inputs (every .ts under -// scripts/ and src/, plus tsconfig.build.json, tsconfig.json, -// package.json), -// 2. compare it to dist/server/build-info.json's `sourceHash`, -// 3. if missing or different, run `tsc -p tsconfig.build.json` -// synchronously (a transient child that exits before the service -// starts) and write a fresh build-info.json, -// 4. `await import()` dist/server/scripts/.js in this same -// process — one resident node process, never stale. -// -// `npm run build:server` (and CI) share this exact logic via `--build-only`, -// so a manual/CI build and a launchd-triggered build produce byte-identical -// build-info.json semantics. +// A launcher imports its exact generation, not a mutable "current" symlink. +// Retain old generations for running services' lazy imports; remove dist/server +// only when all host services are stopped. Legacy dist/server/scripts output +// is left untouched, but new launchers never use it. // // Usage: // node scripts/run-compiled.mjs [...args passed to the entry] // node scripts/run-compiled.mjs --build-only -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { spawnSync } from "node:child_process"; import { createRequire } from "node:module"; -import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -// This file lives directly in scripts/ and is never compiled/moved, so one -// level up is always the real repo root — unlike the repoRoot() helpers in -// the compiled entries themselves, which have to handle two possible layouts. const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const distDir = path.join(repoRoot, "dist", "server"); -const buildInfoPath = path.join(distDir, "build-info.json"); - const require = createRequire(import.meta.url); +const entries = ["agent-worker", "deliver", "github-poll"]; -/** Recursively collects every `.ts` file under `dir`, skipping `node_modules` and dotdirs. */ -function collectTsFiles(dir, out) { - let entries; - try { - entries = readdirSync(dir, { withFileTypes: true }); - } catch { - return; // dir doesn't exist (e.g. a fresh checkout with no src/ yet) — nothing to hash - } - for (const entry of entries) { +function collectFiles(dir, extension, out) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { if (entry.name === "node_modules" || entry.name.startsWith(".")) continue; const full = path.join(dir, entry.name); - if (entry.isDirectory()) collectTsFiles(full, out); - else if (entry.isFile() && entry.name.endsWith(".ts")) out.push(full); + if (entry.isDirectory()) collectFiles(full, extension, out); + else if (entry.isFile() && entry.name.endsWith(extension)) out.push(full); } } -/** - * Content hash over the compile inputs (SYD-268): every `.ts` under - * `scripts/` and `src/`, plus the three config files whose contents change - * what tsc emits. `worker-sdk/` is deliberately excluded — it's never - * compiled (see scripts/agent-worker.ts's dispatchSdk), so a change there - * doesn't need to trigger a dist/server rebuild. - */ +/** Include both declared dependencies and the compiler actually installed. */ function computeSourceHash() { const files = []; - collectTsFiles(path.join(repoRoot, "scripts"), files); - collectTsFiles(path.join(repoRoot, "src"), files); - for (const extra of ["tsconfig.build.json", "tsconfig.json", "package.json"]) { + for (const dir of ["scripts", "src"]) { + collectFiles(path.join(repoRoot, dir), ".ts", files); + } + for (const extra of [ + "tsconfig.build.json", + "tsconfig.json", + "package.json", + "package-lock.json", + "scripts/run-compiled.mjs", + ]) { files.push(path.join(repoRoot, extra)); } - - const relPaths = files.map((f) => path.relative(repoRoot, f).split(path.sep).join("/")).sort(); - - const hash = createHash("sha256"); - for (const rel of relPaths) { - hash.update(rel); + // npm's installed-tree lock catches an install that differs from the root + // lock. The compiler package and JS bytes also catch a compiler replaced in + // place, even when neither lockfile was changed. + const installedLock = path.join(repoRoot, "node_modules", ".package-lock.json"); + if (existsSync(installedLock)) files.push(installedLock); + const compilerPackage = require.resolve("typescript/package.json"); + files.push(compilerPackage, require.resolve("typescript/bin/tsc")); + collectFiles(path.join(path.dirname(compilerPackage), "lib"), ".js", files); + + const hash = createHash("sha256").update(`generation-v1\0${process.version}\0`); + for (const file of files.sort()) { + hash.update(path.relative(repoRoot, file).split(path.sep).join("/")); hash.update("\0"); - hash.update(readFileSync(path.join(repoRoot, rel))); + hash.update(readFileSync(file)); hash.update("\0"); } return hash.digest("hex"); } -function readBuildInfo() { - if (!existsSync(buildInfoPath)) return null; +function readBuildInfo(dir, sourceHash) { try { - return JSON.parse(readFileSync(buildInfoPath, "utf8")); + const info = JSON.parse(readFileSync(path.join(dir, "build-info.json"), "utf8")); + return info.sourceHash === sourceHash && + entries.every((entry) => existsSync(path.join(dir, "scripts", `${entry}.js`))) + ? info + : null; } catch { - return null; // corrupt/partial build-info.json — treat like "no build yet" + return null; } } @@ -98,91 +95,92 @@ function gitHeadOf() { return res.status === 0 ? res.stdout.trim() : undefined; } -/** Runs `tsc -p tsconfig.build.json` synchronously. Returns its exit code. */ -function runTsc() { - const tscBin = require.resolve("typescript/bin/tsc"); - const result = spawnSync( - process.execPath, - [tscBin, "-p", path.join(repoRoot, "tsconfig.build.json")], - { - cwd: repoRoot, - stdio: "inherit", - }, - ); - return result.status ?? 1; +/** Atomic informational pointer; imports always use the immutable generation. */ +function publishBuildInfo(info) { + const temporary = path.join(distDir, `.build-info-${randomUUID()}.json`); + try { + writeFileSync(temporary, `${JSON.stringify(info, null, 2)}\n`, { flag: "wx" }); + renameSync(temporary, path.join(distDir, "build-info.json")); + } finally { + rmSync(temporary, { force: true }); + } } -/** - * Builds dist/server if its build-info.json is missing or stale (SYD-268's - * staleness guard). Exits the process with a clear message if the compile - * fails — a worker must never silently start against a half-built or - * month-old dist/server. - */ function ensureBuilt() { const sourceHash = computeSourceHash(); - const existing = readBuildInfo(); - if (existing && existing.sourceHash === sourceHash) { - return { rebuilt: false, sourceHash }; + const generation = path.join(distDir, sourceHash); + const existing = readBuildInfo(generation, sourceHash); + if (existing) return generation; + // An existing but incomplete generation is never overwritten: a running + // process could already hold imports from it. Fail closed for operator repair. + if (existsSync(generation)) { + // A competing builder may have published between our first read and the + // existence check. Re-read before treating this as a broken generation. + if (readBuildInfo(generation, sourceHash)) return generation; + throw new Error(`[run-compiled] invalid build generation: ${generation}`); } - console.log( - existing - ? "[run-compiled] source changed since the last compiled build — rebuilding dist/server..." - : "[run-compiled] no compiled build found — building dist/server...", - ); - const code = runTsc(); - if (code !== 0) { - console.error( - `\n[run-compiled] FATAL: \`tsc -p tsconfig.build.json\` failed (exit ${code}). ` + - "Fix the compile error above — a stale or half-built dist/server must never run.\n", + console.log("[run-compiled] compile inputs changed or missing — building a new generation..."); + mkdirSync(distDir, { recursive: true }); + const staging = mkdtempSync(path.join(distDir, ".building-")); + try { + const result = spawnSync( + process.execPath, + [ + require.resolve("typescript/bin/tsc"), + "-p", + path.join(repoRoot, "tsconfig.build.json"), + "--outDir", + staging, + ], + { cwd: repoRoot, stdio: "inherit" }, ); - process.exit(code || 1); + if (result.status !== 0) { + throw new Error( + `[run-compiled] tsc failed (exit ${result.status ?? 1}); previous builds are untouched.`, + { cause: result.error }, + ); + } + if (computeSourceHash() !== sourceHash) { + throw new Error("[run-compiled] compile inputs changed during the build; retry startup."); + } + const info = { sourceHash, builtAt: new Date().toISOString(), gitHead: gitHeadOf() }; + writeFileSync(path.join(staging, "build-info.json"), `${JSON.stringify(info, null, 2)}\n`); + if (!readBuildInfo(staging, sourceHash)) { + throw new Error("[run-compiled] compiler did not produce every required host entry."); + } + try { + renameSync(staging, generation); + } catch (error) { + // Another launcher can win the rename while we compile. A nonempty, + // complete generation is immutable, so safely reuse that winner. + if (!["EEXIST", "ENOTEMPTY"].includes(error.code)) throw error; + if (!readBuildInfo(generation, sourceHash)) throw error; + } + publishBuildInfo(readBuildInfo(generation, sourceHash)); + console.log(`[run-compiled] built generation ${sourceHash.slice(0, 12)}...`); + return generation; + } finally { + rmSync(staging, { recursive: true, force: true }); } - - mkdirSync(distDir, { recursive: true }); - const buildInfo = { sourceHash, builtAt: new Date().toISOString(), gitHead: gitHeadOf() }; - writeFileSync(buildInfoPath, `${JSON.stringify(buildInfo, null, 2)}\n`); - console.log(`[run-compiled] built dist/server (sourceHash ${sourceHash.slice(0, 12)}...)`); - return { rebuilt: true, sourceHash }; } async function main() { const [, , first, ...rest] = process.argv; - if (first === "--build-only") { ensureBuilt(); return; } - - if (!first || first.startsWith("-")) { - console.error( + if (!entries.includes(first)) { + throw new Error( "usage: node scripts/run-compiled.mjs [...args]\n" + " node scripts/run-compiled.mjs --build-only\n" + - " is one of: agent-worker, deliver, github-poll", + ` is one of: ${entries.join(", ")}`, ); - process.exit(1); } - - ensureBuilt(); - - const entryPath = path.join(distDir, "scripts", `${first}.js`); - if (!existsSync(entryPath)) { - console.error( - `[run-compiled] compiled entry not found: ${entryPath}\n` + - `(expected "${first}" to be one of agent-worker, deliver, github-poll)`, - ); - process.exit(1); - } - - // The entry module's own `if (import.meta.url === \`file://${process.argv[1]}\`)` - // guard (its equivalent of "am I the thing node was invoked on") — and its - // own `process.argv.slice(2)` argv parsing — both assume node was invoked - // directly on it. Since we're dynamically importing it instead, rewrite - // argv so both keep working unmodified: argv[1] becomes the compiled - // entry's own path, and everything after the entry name on our own argv - // becomes what the entry sees after its own path. + const entryPath = path.join(ensureBuilt(), "scripts", `${first}.js`); + // Preserve each entry's direct-invocation guard and argument parsing. process.argv = [process.argv[0], entryPath, ...rest]; - await import(pathToFileURL(entryPath).href); } diff --git a/tests/scripts/run-compiled.test.ts b/tests/scripts/run-compiled.test.ts new file mode 100644 index 0000000..5006574 --- /dev/null +++ b/tests/scripts/run-compiled.test.ts @@ -0,0 +1,217 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { spawn } from "node:child_process"; +import { + appendFileSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; + +let root: string; +const children: ReturnType[] = []; +const entries = ["agent-worker", "deliver", "github-poll"]; + +function write(relative: string, contents: string) { + const file = path.join(root, relative); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, contents); +} + +beforeEach(() => { + root = mkdtempSync(path.join(tmpdir(), "switchyard-compiled-")); + write("package.json", '{"type":"module"}'); + write("package-lock.json", '{"lockfileVersion":3}'); + write("tsconfig.json", "{}"); + write("tsconfig.build.json", "{}"); + write("node_modules/typescript/package.json", '{"version":"1.0.0","type":"module"}'); + write("node_modules/typescript/lib/compiler.js", "// compiler implementation"); + write("src/value.ts", 'export const version = "v1";'); + for (const entry of entries) { + write( + `scripts/${entry}.ts`, + `import { existsSync, writeFileSync } from "node:fs"; +import { setTimeout as delay } from "node:timers/promises"; +if (import.meta.url !== new URL("file://" + process.argv[1]).href) throw new Error("argv mismatch"); +if (process.env.ENTRY_GATE) { + writeFileSync(process.env.ENTRY_GATE + ".started", "ready"); + while (!existsSync(process.env.ENTRY_GATE)) await delay(10); +} +const { version } = await import("../src/value.js"); +console.log("ENTRY:" + version + ":" + process.argv.slice(2).join(","));`, + ); + } + copyFileSync( + new URL("../../scripts/run-compiled.mjs", import.meta.url), + path.join(root, "scripts/run-compiled.mjs"), + ); + // A controllable compiler subprocess exercises publication at the OS/process + // boundary. It deliberately emits a partial tree before waiting or failing. + write( + "node_modules/typescript/bin/tsc", + `import { appendFileSync, copyFileSync, existsSync, mkdirSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +const output = process.argv[process.argv.indexOf("--outDir") + 1]; +appendFileSync("compilations.log", process.pid + "\\n"); +mkdirSync(path.join(output, "scripts"), { recursive: true }); +copyFileSync("scripts/agent-worker.ts", path.join(output, "scripts/agent-worker.js")); +if (existsSync("fail-compile")) process.exit(2); +if (process.env.COMPILE_GATE) { + writeFileSync(process.env.COMPILE_GATE + "." + process.pid, "waiting"); + while (!existsSync(process.env.COMPILE_GATE)) await delay(10); +} +for (const entry of ["deliver", "github-poll"]) { + copyFileSync("scripts/" + entry + ".ts", path.join(output, "scripts", entry + ".js")); +} +mkdirSync(path.join(output, "src"), { recursive: true }); +copyFileSync("src/value.ts", path.join(output, "src/value.js"));`, + ); +}); + +afterEach(async () => { + for (const child of children.splice(0)) { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + await new Promise((resolve) => child.once("close", resolve)); + } + } + rmSync(root, { recursive: true, force: true }); +}); + +function launch(args = ["--build-only"], env: Record = {}) { + const child = spawn(process.execPath, [path.join(root, "scripts/run-compiled.mjs"), ...args], { + cwd: root, + env: { ...process.env, ...env }, + stdio: ["ignore", "pipe", "pipe"], + }); + children.push(child); + let output = ""; + child.stdout.on("data", (chunk) => (output += chunk)); + child.stderr.on("data", (chunk) => (output += chunk)); + const result = new Promise<{ code: number | null; output: string }>((resolve, reject) => { + child.once("error", reject); + child.once("close", (code) => resolve({ code, output })); + }); + return { child, result }; +} + +async function until(predicate: () => boolean) { + const deadline = Date.now() + 5_000; + while (!predicate()) { + if (Date.now() > deadline) throw new Error("timed out waiting for subprocess barrier"); + await delay(10); + } +} + +function buildInfo() { + return JSON.parse(readFileSync(path.join(root, "dist/server/build-info.json"), "utf8")) as { + sourceHash: string; + }; +} + +function compilationCount() { + return readFileSync(path.join(root, "compilations.log"), "utf8").trim().split("\n").length; +} + +describe("compiled host launcher", () => { + it("competing startups only import a complete generation and forward entry arguments", async () => { + const gate = path.join(root, "compile-gate"); + const launches = entries.map((entry) => launch([entry, "--once"], { COMPILE_GATE: gate })); + await until( + () => readdirSync(root).filter((name) => name.startsWith("compile-gate.")).length === 3, + ); + expect(existsSync(path.join(root, "dist/server/build-info.json"))).toBe(false); + expect(launches.every(({ child }) => child.exitCode === null)).toBe(true); + writeFileSync(gate, "release"); + for (const result of await Promise.all(launches.map(({ result }) => result))) { + expect(result.code, result.output).toBe(0); + expect(result.output).toContain("ENTRY:v1:--once"); + } + const hash = buildInfo().sourceHash; + expect(readdirSync(path.join(root, "dist/server")).sort()).toEqual( + ["build-info.json", hash].sort(), + ); + expect((await launch().result).code).toBe(0); + expect(compilationCount()).toBe(3); // unchanged startup is a cache hit + }); + + it("failed compilation preserves the prior complete build and never starts stale code", async () => { + expect((await launch().result).code).toBe(0); + const previous = readFileSync(path.join(root, "dist/server/build-info.json"), "utf8"); + const hash = buildInfo().sourceHash; + const previousOutput = readFileSync( + path.join(root, "dist/server", hash, "src/value.js"), + "utf8", + ); + write("src/value.ts", 'export const version = "v2";'); + write("fail-compile", "yes"); + const failed = await launch(["agent-worker"]).result; + expect(failed.code).not.toBe(0); + expect(failed.output).toContain("tsc failed"); + expect(failed.output).not.toContain("ENTRY:"); + expect(readFileSync(path.join(root, "dist/server/build-info.json"), "utf8")).toBe(previous); + expect(readFileSync(path.join(root, "dist/server", hash, "src/value.js"), "utf8")).toBe( + previousOutput, + ); + expect( + readdirSync(path.join(root, "dist/server")).some((name) => name.startsWith(".building-")), + ).toBe(false); + }); + + it("rebuilds when the dependency lock or installed compiler changes", async () => { + expect((await launch().result).code).toBe(0); + const hashes = new Set([buildInfo().sourceHash]); + for (const [file, contents] of [ + ["package-lock.json", '{"lockfileVersion":3,"dependencies":{"new":"1"}}'], + ["node_modules/.package-lock.json", '{"packages":{"typescript":{"version":"1.0.0"}}}'], + ["node_modules/typescript/package.json", '{"version":"2.0.0","type":"module"}'], + ["node_modules/typescript/lib/compiler.js", "// replaced compiler implementation"], + ]) { + write(file, contents); + const result = await launch().result; + expect(result.code, result.output).toBe(0); + hashes.add(buildInfo().sourceHash); + } + expect(compilationCount()).toBe(5); + expect(hashes.size).toBe(5); + }); + + it("rejects inputs changed while compilation is in progress", async () => { + expect((await launch().result).code).toBe(0); + const previous = buildInfo(); + appendFileSync(path.join(root, "src/value.ts"), "\n// change before startup"); + const gate = path.join(root, "compile-gate"); + const compiling = launch(["--build-only"], { COMPILE_GATE: gate }); + await until(() => readdirSync(root).some((name) => name.startsWith("compile-gate."))); + appendFileSync(path.join(root, "src/value.ts"), "\n// change during compile"); + writeFileSync(gate, "release"); + const result = await compiling.result; + expect(result.code).not.toBe(0); + expect(result.output).toContain("compile inputs changed during the build"); + expect(buildInfo()).toEqual(previous); + }); + + it("keeps lazy imports on the generation selected by a running service", async () => { + const gate = path.join(root, "entry-gate"); + const oldService = launch(["agent-worker"], { ENTRY_GATE: gate }); + await until(() => existsSync(`${gate}.started`)); + const oldHash = buildInfo().sourceHash; + write("src/value.ts", 'export const version = "v2";'); + const newService = await launch(["agent-worker"]).result; + expect(newService.code, newService.output).toBe(0); + expect(newService.output).toContain("ENTRY:v2:"); + expect(buildInfo().sourceHash).not.toBe(oldHash); + writeFileSync(gate, "release"); + const oldResult = await oldService.result; + expect(oldResult.code, oldResult.output).toBe(0); + expect(oldResult.output).toContain("ENTRY:v1:"); + }); +}); diff --git a/tsconfig.build.json b/tsconfig.build.json index bdca39d..4d1156d 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,7 +1,7 @@ { // SYD-268: compiles the host-side launchd entry points (and whatever they // pull in from scripts/ and src/) to plain JS so launchd can run a single - // `node dist/server/scripts/.js` process instead of + // `node scripts/run-compiled.mjs ` process instead of // `node -> tsx shim -> node+esbuild loader` per service. Deliberately does // NOT touch the root tsconfig.json's `noEmit: true` — `npm run typecheck` // depends on that staying a check-only config. @@ -13,6 +13,8 @@ "extends": "./tsconfig.json", "compilerOptions": { "noEmit": false, + // The launcher overrides this with a private staging directory, then + // publishes the complete output at dist/server/. "outDir": "dist/server", "rootDir": "." }, From d48dfba4036757692dcbce810bddd96ce334808f Mon Sep 17 00:00:00 2001 From: Sean Perkins <1733750+seanperkins@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:02:37 -0400 Subject: [PATCH 2/2] fix: preserve runtime sidecars in compiled generations (#272) --- README.md | 10 +++++-- scripts/run-compiled.mjs | 30 +++++++++++++++++--- tests/scripts/run-compiled.test.ts | 45 +++++++++++++++++++++++++++++- 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d534d23..0c34d45 100644 --- a/README.md +++ b/README.md @@ -223,14 +223,18 @@ The launcher compiles host services on demand, or ahead of time with `npm run build:server`. It fingerprints source, configuration, dependency lockfiles, the installed TypeScript compiler, and the Node version. Completed builds live in `dist/server/<64-character hash>/`; the extra path component -is intentional. Each service imports its own immutable generation, so another +is intentional. Hand-written `scripts/**/*.mjs` runtime files are copied into +each generation, and their contents and `.d.mts`/`.d.cts` declarations are +fingerprinted alongside TypeScript sources. Each service imports its own +immutable generation, so another startup cannot replace files underneath it, including files imported later. Concurrent cold starts may briefly run multiple compilers, but publication is atomic and only one complete generation is retained per fingerprint. A failed compile fails startup and leaves earlier builds intact. -`dist/server/build-info.json` records the last published generation for -inspection; services do not follow this pointer after startup. Old generations +`dist/server/build-info.json` records the last selected generation for +inspection, including cache hits. It does not describe every running service; +services do not follow this pointer after startup. Old generations are retained because running services may still need them, so disk usage grows with changed builds. To reclaim the cache, stop **all** host services, remove `dist/server`, run `npm run build:server`, then restart the services. This also diff --git a/scripts/run-compiled.mjs b/scripts/run-compiled.mjs index d5288d0..ded74b9 100644 --- a/scripts/run-compiled.mjs +++ b/scripts/run-compiled.mjs @@ -18,6 +18,7 @@ import { createHash, randomUUID } from "node:crypto"; import { spawnSync } from "node:child_process"; import { createRequire } from "node:module"; import { + copyFileSync, existsSync, mkdirSync, mkdtempSync, @@ -48,14 +49,16 @@ function collectFiles(dir, extension, out) { function computeSourceHash() { const files = []; for (const dir of ["scripts", "src"]) { - collectFiles(path.join(repoRoot, dir), ".ts", files); + for (const extension of [".ts", ".tsx", ".mts", ".cts"]) { + collectFiles(path.join(repoRoot, dir), extension, files); + } } + collectFiles(path.join(repoRoot, "scripts"), ".mjs", files); for (const extra of [ "tsconfig.build.json", "tsconfig.json", "package.json", "package-lock.json", - "scripts/run-compiled.mjs", ]) { files.push(path.join(repoRoot, extra)); } @@ -78,6 +81,17 @@ function computeSourceHash() { return hash.digest("hex"); } +/** TypeScript does not emit the hand-written runtime sidecars it imports. */ +function copyScriptAssets(staging) { + const assets = []; + collectFiles(path.join(repoRoot, "scripts"), ".mjs", assets); + for (const source of assets) { + const destination = path.join(staging, path.relative(repoRoot, source)); + mkdirSync(path.dirname(destination), { recursive: true }); + copyFileSync(source, destination); + } +} + function readBuildInfo(dir, sourceHash) { try { const info = JSON.parse(readFileSync(path.join(dir, "build-info.json"), "utf8")); @@ -110,13 +124,20 @@ function ensureBuilt() { const sourceHash = computeSourceHash(); const generation = path.join(distDir, sourceHash); const existing = readBuildInfo(generation, sourceHash); - if (existing) return generation; + if (existing) { + publishBuildInfo(existing); + return generation; + } // An existing but incomplete generation is never overwritten: a running // process could already hold imports from it. Fail closed for operator repair. if (existsSync(generation)) { // A competing builder may have published between our first read and the // existence check. Re-read before treating this as a broken generation. - if (readBuildInfo(generation, sourceHash)) return generation; + const winner = readBuildInfo(generation, sourceHash); + if (winner) { + publishBuildInfo(winner); + return generation; + } throw new Error(`[run-compiled] invalid build generation: ${generation}`); } @@ -141,6 +162,7 @@ function ensureBuilt() { { cause: result.error }, ); } + copyScriptAssets(staging); if (computeSourceHash() !== sourceHash) { throw new Error("[run-compiled] compile inputs changed during the build; retry startup."); } diff --git a/tests/scripts/run-compiled.test.ts b/tests/scripts/run-compiled.test.ts index 5006574..bb62d9c 100644 --- a/tests/scripts/run-compiled.test.ts +++ b/tests/scripts/run-compiled.test.ts @@ -184,6 +184,41 @@ describe("compiled host launcher", () => { expect(hashes.size).toBe(5); }); + it("copies imported MJS sidecars and rebuilds when their code or declarations change", async () => { + write("scripts/telemetry/metadata.mjs", 'export const model = "first";'); + write("scripts/telemetry/metadata.d.mts", "export const model: string;"); + appendFileSync( + path.join(root, "scripts/agent-worker.ts"), + '\nconst { model } = await import("./telemetry/metadata.mjs");\nconsole.log("MODEL:" + model);', + ); + const first = await launch(["agent-worker"]).result; + expect(first.code, first.output).toBe(0); + expect(first.output).toContain("MODEL:first"); + const firstHash = buildInfo().sourceHash; + + write("scripts/telemetry/metadata.mjs", 'export const model = "second";'); + const second = await launch(["agent-worker"]).result; + expect(second.code, second.output).toBe(0); + expect(second.output).toContain("MODEL:second"); + const secondHash = buildInfo().sourceHash; + expect(secondHash).not.toBe(firstHash); + expect( + readFileSync( + path.join(root, "dist/server", firstHash, "scripts/telemetry/metadata.mjs"), + "utf8", + ), + ).toContain('"first"'); + + write("scripts/telemetry/metadata.d.mts", 'export const model: "second";'); + expect((await launch().result).code).toBe(0); + expect(buildInfo().sourceHash).not.toBe(secondHash); + const declarationHash = buildInfo().sourceHash; + write("scripts/telemetry/commonjs.d.cts", "export const version: string;"); + expect((await launch().result).code).toBe(0); + expect(buildInfo().sourceHash).not.toBe(declarationHash); + expect(compilationCount()).toBe(4); + }); + it("rejects inputs changed while compilation is in progress", async () => { expect((await launch().result).code).toBe(0); const previous = buildInfo(); @@ -203,7 +238,8 @@ describe("compiled host launcher", () => { const gate = path.join(root, "entry-gate"); const oldService = launch(["agent-worker"], { ENTRY_GATE: gate }); await until(() => existsSync(`${gate}.started`)); - const oldHash = buildInfo().sourceHash; + const oldInfo = buildInfo(); + const oldHash = oldInfo.sourceHash; write("src/value.ts", 'export const version = "v2";'); const newService = await launch(["agent-worker"]).result; expect(newService.code, newService.output).toBe(0); @@ -213,5 +249,12 @@ describe("compiled host launcher", () => { const oldResult = await oldService.result; expect(oldResult.code, oldResult.output).toBe(0); expect(oldResult.output).toContain("ENTRY:v1:"); + + // Returning to old inputs selects the cached generation without compiling, + // and refreshes the informational pointer while retaining original builtAt. + write("src/value.ts", 'export const version = "v1";'); + expect((await launch().result).code).toBe(0); + expect(buildInfo()).toEqual(oldInfo); + expect(compilationCount()).toBe(2); }); });