diff --git a/packages/provider-opencode/src/server/OpenCodeRuntime.ts b/packages/provider-opencode/src/server/OpenCodeRuntime.ts index ea2d7a7d76cac..4d5ce2d078e8c 100644 --- a/packages/provider-opencode/src/server/OpenCodeRuntime.ts +++ b/packages/provider-opencode/src/server/OpenCodeRuntime.ts @@ -26,6 +26,7 @@ import * as Result from "effect/Result"; import * as Scope from "effect/Scope"; import * as Schema from "effect/Schema"; import * as Schedule from "effect/Schedule"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Stream from "effect/Stream"; import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; @@ -623,6 +624,7 @@ const makeOpenCodeRuntime = Effect.gen(function* () { // Scopes close in reverse order. Forking this before the group kill is // registered forgets the ledger entry only once the group is stopped. const ledgerScope = yield* Scope.fork(runtimeScope); + const marker = yield* Effect.sync(() => SecureRandom.uuidV4Unsafe()); const child = yield* spawner .spawn( ChildProcess.make(spawnCommand.command, spawnCommand.args, { @@ -634,6 +636,7 @@ const makeOpenCodeRuntime = Effect.gen(function* () { // The value is set explicitly (rather than relying on inheritance) because // `extendEnv` is false whenever `input.environment` is provided. OPENCODE_CONFIG_CONTENT: resolveOpenCodeConfigContent(input.environment), + [OpenCodeServerLedger.SERVER_MARKER_ENV]: marker, }, extendEnv: input.environment === undefined, }), @@ -658,7 +661,12 @@ const makeOpenCodeRuntime = Effect.gen(function* () { // Registered before recording, so an interrupt while the ledger writes // still stops the group. yield* Scope.addFinalizer(runtimeScope, terminateChild); - const forgetServer = yield* serverLedger.track({ pid: Number(child.pid), port, args }); + const forgetServer = yield* serverLedger.track({ + pid: Number(child.pid), + port, + args, + marker, + }); yield* Scope.addFinalizer(ledgerScope, forgetServer); const stdoutRef = yield* Ref.make(""); diff --git a/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts b/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts index 3fcb52c898d68..d071bbd24e841 100644 --- a/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts +++ b/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts @@ -13,6 +13,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Scope from "effect/Scope"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as OpenCodeRuntime from "./OpenCodeRuntime.ts"; @@ -46,10 +47,15 @@ const killGroup = (pgid: number) => const groupLeaderGuard = `pgid="$(ps -o pgid= -p $$ 2>&1 | tr -d ' ')"; [ "$pgid" = "$$" ] || { echo "ungrouped:$pgid"; exit 97; }; echo grouped; `; // Yields the group, or a description of why this shell cannot be used as one. -const spawnGroupLeader = (args: ReadonlyArray, script: string) => +const spawnGroupLeader = ( + args: ReadonlyArray, + script: string, + env: Record, +) => Effect.gen(function* () { const child = NodeChildProcess.spawn("/bin/sh", ["-c", groupLeaderGuard + script, ...args], { detached: true, + env: { ...process.env, ...env }, stdio: ["ignore", "pipe", "ignore"], }); const pid = child.pid; @@ -85,12 +91,16 @@ const spawnGroupLeader = (args: ReadonlyArray, script: string) => // mean the guard itself is broken (for example a missing or BusyBox `ps`). const GROUP_LEADER_ATTEMPTS = 5; -const spawnGroup = (args: ReadonlyArray, script = "sleep 600 & wait") => +const spawnGroup = ( + args: ReadonlyArray, + script = "sleep 600 & wait", + env: Record = {}, +) => Effect.acquireRelease( Effect.gen(function* () { let failure = ""; for (let attempt = 0; attempt < GROUP_LEADER_ATTEMPTS; attempt++) { - const group = yield* spawnGroupLeader(args, script); + const group = yield* spawnGroupLeader(args, script, env); if (typeof group !== "string") return group; failure = group; } @@ -101,14 +111,20 @@ const spawnGroup = (args: ReadonlyArray, script = "sleep 600 & wait") => ({ pid }) => killGroup(pid), ); -const recordFromDeadServer = (stateDir: string, server: { readonly pid: number }) => +const recordFromDeadServer = ( + stateDir: string, + server: { readonly pid: number }, + marker: string = SecureRandom.uuidV4Unsafe(), +) => Effect.gen(function* () { const previousServer = yield* spawnGroup([]); const previousLedger = yield* OpenCodeServerLedger.make({ stateDir, ownerPid: previousServer.pid, }); - yield* Effect.asVoid(previousLedger.track({ pid: server.pid, port: 4096, args: SERVE_ARGS })); + yield* Effect.asVoid( + previousLedger.track({ pid: server.pid, port: 4096, args: SERVE_ARGS, marker }), + ); yield* killGroup(previousServer.pid); yield* previousServer.exited; }); @@ -143,6 +159,38 @@ describe.each(observedPlatforms)("OpenCodeServerLedger observing as %s", (platfo }).pipe(provideHost), ); + // OpenCode's bash tool starts each shell command in its own session. + it.live("stops a shell command whose OpenCode server exited before restart", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-opencode-ledger-" }); + const gate = path.join(stateDir, "gate"); + const shellPidFile = path.join(stateDir, "shell-pid"); + NodeChildProcess.execFileSync("mkfifo", [gate, shellPidFile]); + const marker = SecureRandom.uuidV4Unsafe(); + const server = yield* spawnGroup( + SERVE_ARGS, + `perl -MPOSIX -e 'POSIX::setsid(); open(F, ">", shift); print F "$$\\n"; close(F); exec @ARGV' '${shellPidFile}' sleep 600 & read _ < '${gate}'; exit 0`, + { [OpenCodeServerLedger.SERVER_MARKER_ENV]: marker }, + ); + const shell = yield* Effect.acquireRelease( + fs.readFileString(shellPidFile).pipe(Effect.map((pid) => Number(pid.trim()))), + (pid) => killGroup(pid), + ); + expect(shell).not.toBe(server.pid); + expect(groupExists(shell)).toBe(true); + yield* recordFromDeadServer(stateDir, server, marker); + yield* fs.writeFileString(gate, "\n"); + yield* server.exited; + + const restarted = yield* OpenCodeServerLedger.make({ stateDir }); + yield* restarted.reapOrphans; + + expect(groupExists(shell)).toBe(false); + }).pipe(provideHost), + ); + it.live("records a group whose spawned wrapper already exited", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -249,7 +297,12 @@ describe.each(observedPlatforms)("OpenCodeServerLedger observing as %s", (platfo const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-opencode-ledger-" }); const running = yield* OpenCodeServerLedger.make({ stateDir }); const server = yield* spawnGroup(SERVE_ARGS); - const forget = yield* running.track({ pid: server.pid, port: 4096, args: SERVE_ARGS }); + const forget = yield* running.track({ + pid: server.pid, + port: 4096, + args: SERVE_ARGS, + marker: SecureRandom.uuidV4Unsafe(), + }); const other = yield* OpenCodeServerLedger.make({ stateDir }); yield* other.reapOrphans; diff --git a/packages/provider-opencode/src/server/OpenCodeServerLedger.ts b/packages/provider-opencode/src/server/OpenCodeServerLedger.ts index fb3ea7645e9bf..08cb53462bc9b 100644 --- a/packages/provider-opencode/src/server/OpenCodeServerLedger.ts +++ b/packages/provider-opencode/src/server/OpenCodeServerLedger.ts @@ -1,4 +1,5 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Config from "effect/Config"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -22,6 +23,7 @@ const OpenCodeServerEntry = Schema.Struct({ startTime: Schema.String, command: Schema.String, args: Schema.optionalKey(Schema.Array(Schema.String)), + marker: Schema.optionalKey(Schema.String), port: Schema.Int, stateDir: Schema.String, owner: ProcessIdentity, @@ -38,10 +40,15 @@ export class OpenCodeServerLedger extends Context.Service< readonly pid: number; readonly port: number; readonly args: ReadonlyArray; + readonly marker: string; }) => Effect.Effect>; } >()("@t3tools/provider-opencode/server/OpenCodeServerLedger") {} +// OpenCode starts each shell command in its own session, so once the server exits +// only this inherited variable, set to the entry's marker, ties a command to it. +export const SERVER_MARKER_ENV = "T3CODE_OPENCODE_SERVER_MARKER"; + const ENTRY_DIRECTORY = "opencode-servers"; const ENTRY_FILE = /^\d+\.json$/; const STOP_POLL_INTERVAL = "50 millis"; @@ -104,6 +111,8 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { const platform = yield* HostProcess.Platform; const directory = path.join(input.stateDir, ENTRY_DIRECTORY); const recordable = platform === "linux" || platform === "darwin"; + // A T3 Code server started from one of those commands carries a marker too. + const inheritedMarker = yield* Config.String(SERVER_MARKER_ENV).pipe(Config.withDefault("")); // A process-group scan calls this for every /proc entry; avoid a span per process. const observeLinux = Effect.fnUntraced(function* (pid: number, group?: number) { @@ -176,6 +185,46 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { return Effect.succeed([]); }; + // procps has no `ps -E`, so /proc serves wherever it exists; macOS has no /proc. + const markedGroups = (marker: string): Effect.Effect> => { + const variable = `${SERVER_MARKER_ENV}=${marker}`; + const fromProc = fs.readDirectory("/proc").pipe( + Effect.flatMap((names) => + Effect.forEach( + names.filter((name) => /^\d+$/.test(name)), + (name) => + fs.readFileString(`/proc/${name}/environ`).pipe( + Effect.flatMap((environ) => + environ.split("\0").includes(variable) + ? observeLinux(Number(name)) + : Effect.undefined, + ), + Effect.orElseSucceed(() => undefined), + ), + { concurrency: 16 }, + ), + ), + ); + const fromPs = psDarwin(["-A", "-E"]).pipe( + Effect.map((found) => + found.filter((candidate) => candidate.command.split(" ").includes(variable)), + ), + ); + return fs.exists("/proc/self/environ").pipe( + Effect.flatMap((hasProc) => + hasProc ? fromProc : platform === "darwin" ? fromPs : Effect.succeed([]), + ), + Effect.map((found) => [ + ...new Set( + found.flatMap((candidate) => + candidate === undefined || candidate.zombie ? [] : [candidate.pgid], + ), + ), + ]), + Effect.orElseSucceed(() => []), + ); + }; + const isRunning = (identity: ProcessIdentity) => observe(identity.pid).pipe( Effect.map( @@ -217,6 +266,7 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { startTime: member.startTime, command: member.command, args: server.args, + marker: server.marker, port: server.port, stateDir: input.stateDir, owner, @@ -243,6 +293,15 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { ), ); + const stopGroup = Effect.fn("OpenCodeServerLedger.stopGroup")(function* (pgid: number) { + signalGroup(pgid, "SIGTERM"); + for (let attempt = 0; attempt < STOP_POLL_ATTEMPTS && groupExists(pgid); attempt++) { + yield* Effect.sleep(STOP_POLL_INTERVAL); + } + // The group never emptied, so its pgid cannot have been reused. + if (groupExists(pgid)) signalGroup(pgid, "SIGKILL"); + }); + // Signals only a group whose recorded member still has the same pid, start // time, and group and still has the same command line or serve arguments, so // a recycled pid is never touched. @@ -264,12 +323,21 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { pid: entry.pgid, port: entry.port, }); - signalGroup(entry.pgid, "SIGTERM"); - for (let attempt = 0; attempt < STOP_POLL_ATTEMPTS && groupExists(entry.pgid); attempt++) { - yield* Effect.sleep(STOP_POLL_INTERVAL); - } - // The group never emptied, so its pgid cannot have been reused. - if (groupExists(entry.pgid)) signalGroup(entry.pgid, "SIGKILL"); + yield* stopGroup(entry.pgid); + }); + + // Runs after the server is stopped so it cannot start more commands meanwhile. + const stopMarked = Effect.fn("OpenCodeServerLedger.stopMarked")(function* ( + entry: OpenCodeServerEntry, + ) { + if (entry.marker === undefined || entry.marker === inheritedMarker) return; + const groups = yield* markedGroups(entry.marker); + if (groups.length === 0) return; + yield* Effect.logInfo("Stopping commands left by a previous OpenCode server", { + pid: entry.pgid, + groups, + }); + yield* Effect.forEach(groups, stopGroup, { concurrency: "unbounded", discard: true }); }); const reapEntry = Effect.fn("OpenCodeServerLedger.reapEntry")( @@ -278,6 +346,7 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { if (Option.isSome(entry) && entry.value.stateDir === input.stateDir) { if (yield* isRunning(entry.value.owner)) return; yield* stopOrphan(entry.value); + yield* stopMarked(entry.value); } yield* fs.remove(entryPath, { force: true }); },