From 496f21809dc407f1ea96dbae2bd3717b3297a41d Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 17 Nov 2026 02:09:54 +0000 Subject: [PATCH 1/2] test(opencode): restart stops a shell command left by an exited OpenCode server Reproduces #37848. OpenCode's bash tool runs each command in its own process group, so once the recorded server exits, restart cleanup finds nothing to stop and deletes the record while the shell keeps running. Co-Authored-By: Claude Opus 5.5 --- .../src/server/OpenCodeServerLedger.test.ts | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts b/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts index 3fcb52c898d68..3a48b50b6c917 100644 --- a/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts +++ b/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts @@ -143,6 +143,36 @@ describe.each(observedPlatforms)("OpenCodeServerLedger observing as %s", (platfo }).pipe(provideHost), ); + // OpenCode's bash tool starts each shell command in its own process group. + it.live("stops a shell command whose OpenCode server exited before restart", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-opencode-ledger-" }); + const gate = path.join(stateDir, "gate"); + const shellPidFile = path.join(stateDir, "shell-pid"); + NodeChildProcess.execFileSync("mkfifo", [gate, shellPidFile]); + const server = yield* spawnGroup( + SERVE_ARGS, + `perl -e 'setpgrp(0, 0); open(F, ">", shift); print F "$$\\n"; close(F); exec @ARGV' '${shellPidFile}' sleep 600 & read _ < '${gate}'; exit 0`, + ); + const shell = yield* Effect.acquireRelease( + fs.readFileString(shellPidFile).pipe(Effect.map((pid) => Number(pid.trim()))), + (pid) => killGroup(pid), + ); + expect(shell).not.toBe(server.pid); + expect(groupExists(shell)).toBe(true); + yield* recordFromDeadServer(stateDir, server); + yield* fs.writeFileString(gate, "\n"); + yield* server.exited; + + const restarted = yield* OpenCodeServerLedger.make({ stateDir }); + yield* restarted.reapOrphans; + + expect(groupExists(shell)).toBe(false); + }).pipe(provideHost), + ); + it.live("records a group whose spawned wrapper already exited", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; From 90dc106ffe4a5d00727a71f5d522c2760167417c Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 17 Nov 2026 02:27:05 +0000 Subject: [PATCH 2/2] fix(opencode): restart stops shell commands left by an exited OpenCode server OpenCode starts each bash tool command in its own session, so restart cleanup, which only signals the recorded server's process group, found nothing once that server had exited and dropped the record while the command kept running. Where no systemd scope cleans up (macOS, Linux without a user session) the command ran on indefinitely. Each OpenCode server now gets a random marker in its environment, which its commands inherit, and the ledger records it. Restart cleanup stops every process group carrying the marker after stopping the server. Fixes pingdotgg/t3code#37848 Co-Authored-By: Claude Opus 5.5 --- .../src/server/OpenCodeRuntime.ts | 10 ++- .../src/server/OpenCodeServerLedger.test.ts | 41 +++++++--- .../src/server/OpenCodeServerLedger.ts | 81 +++++++++++++++++-- 3 files changed, 116 insertions(+), 16 deletions(-) diff --git a/packages/provider-opencode/src/server/OpenCodeRuntime.ts b/packages/provider-opencode/src/server/OpenCodeRuntime.ts index ea2d7a7d76cac..4d5ce2d078e8c 100644 --- a/packages/provider-opencode/src/server/OpenCodeRuntime.ts +++ b/packages/provider-opencode/src/server/OpenCodeRuntime.ts @@ -26,6 +26,7 @@ import * as Result from "effect/Result"; import * as Scope from "effect/Scope"; import * as Schema from "effect/Schema"; import * as Schedule from "effect/Schedule"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Stream from "effect/Stream"; import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; @@ -623,6 +624,7 @@ const makeOpenCodeRuntime = Effect.gen(function* () { // Scopes close in reverse order. Forking this before the group kill is // registered forgets the ledger entry only once the group is stopped. const ledgerScope = yield* Scope.fork(runtimeScope); + const marker = yield* Effect.sync(() => SecureRandom.uuidV4Unsafe()); const child = yield* spawner .spawn( ChildProcess.make(spawnCommand.command, spawnCommand.args, { @@ -634,6 +636,7 @@ const makeOpenCodeRuntime = Effect.gen(function* () { // The value is set explicitly (rather than relying on inheritance) because // `extendEnv` is false whenever `input.environment` is provided. OPENCODE_CONFIG_CONTENT: resolveOpenCodeConfigContent(input.environment), + [OpenCodeServerLedger.SERVER_MARKER_ENV]: marker, }, extendEnv: input.environment === undefined, }), @@ -658,7 +661,12 @@ const makeOpenCodeRuntime = Effect.gen(function* () { // Registered before recording, so an interrupt while the ledger writes // still stops the group. yield* Scope.addFinalizer(runtimeScope, terminateChild); - const forgetServer = yield* serverLedger.track({ pid: Number(child.pid), port, args }); + const forgetServer = yield* serverLedger.track({ + pid: Number(child.pid), + port, + args, + marker, + }); yield* Scope.addFinalizer(ledgerScope, forgetServer); const stdoutRef = yield* Ref.make(""); diff --git a/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts b/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts index 3a48b50b6c917..d071bbd24e841 100644 --- a/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts +++ b/packages/provider-opencode/src/server/OpenCodeServerLedger.test.ts @@ -13,6 +13,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Scope from "effect/Scope"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import * as OpenCodeRuntime from "./OpenCodeRuntime.ts"; @@ -46,10 +47,15 @@ const killGroup = (pgid: number) => const groupLeaderGuard = `pgid="$(ps -o pgid= -p $$ 2>&1 | tr -d ' ')"; [ "$pgid" = "$$" ] || { echo "ungrouped:$pgid"; exit 97; }; echo grouped; `; // Yields the group, or a description of why this shell cannot be used as one. -const spawnGroupLeader = (args: ReadonlyArray, script: string) => +const spawnGroupLeader = ( + args: ReadonlyArray, + script: string, + env: Record, +) => Effect.gen(function* () { const child = NodeChildProcess.spawn("/bin/sh", ["-c", groupLeaderGuard + script, ...args], { detached: true, + env: { ...process.env, ...env }, stdio: ["ignore", "pipe", "ignore"], }); const pid = child.pid; @@ -85,12 +91,16 @@ const spawnGroupLeader = (args: ReadonlyArray, script: string) => // mean the guard itself is broken (for example a missing or BusyBox `ps`). const GROUP_LEADER_ATTEMPTS = 5; -const spawnGroup = (args: ReadonlyArray, script = "sleep 600 & wait") => +const spawnGroup = ( + args: ReadonlyArray, + script = "sleep 600 & wait", + env: Record = {}, +) => Effect.acquireRelease( Effect.gen(function* () { let failure = ""; for (let attempt = 0; attempt < GROUP_LEADER_ATTEMPTS; attempt++) { - const group = yield* spawnGroupLeader(args, script); + const group = yield* spawnGroupLeader(args, script, env); if (typeof group !== "string") return group; failure = group; } @@ -101,14 +111,20 @@ const spawnGroup = (args: ReadonlyArray, script = "sleep 600 & wait") => ({ pid }) => killGroup(pid), ); -const recordFromDeadServer = (stateDir: string, server: { readonly pid: number }) => +const recordFromDeadServer = ( + stateDir: string, + server: { readonly pid: number }, + marker: string = SecureRandom.uuidV4Unsafe(), +) => Effect.gen(function* () { const previousServer = yield* spawnGroup([]); const previousLedger = yield* OpenCodeServerLedger.make({ stateDir, ownerPid: previousServer.pid, }); - yield* Effect.asVoid(previousLedger.track({ pid: server.pid, port: 4096, args: SERVE_ARGS })); + yield* Effect.asVoid( + previousLedger.track({ pid: server.pid, port: 4096, args: SERVE_ARGS, marker }), + ); yield* killGroup(previousServer.pid); yield* previousServer.exited; }); @@ -143,7 +159,7 @@ describe.each(observedPlatforms)("OpenCodeServerLedger observing as %s", (platfo }).pipe(provideHost), ); - // OpenCode's bash tool starts each shell command in its own process group. + // OpenCode's bash tool starts each shell command in its own session. it.live("stops a shell command whose OpenCode server exited before restart", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -152,9 +168,11 @@ describe.each(observedPlatforms)("OpenCodeServerLedger observing as %s", (platfo const gate = path.join(stateDir, "gate"); const shellPidFile = path.join(stateDir, "shell-pid"); NodeChildProcess.execFileSync("mkfifo", [gate, shellPidFile]); + const marker = SecureRandom.uuidV4Unsafe(); const server = yield* spawnGroup( SERVE_ARGS, - `perl -e 'setpgrp(0, 0); open(F, ">", shift); print F "$$\\n"; close(F); exec @ARGV' '${shellPidFile}' sleep 600 & read _ < '${gate}'; exit 0`, + `perl -MPOSIX -e 'POSIX::setsid(); open(F, ">", shift); print F "$$\\n"; close(F); exec @ARGV' '${shellPidFile}' sleep 600 & read _ < '${gate}'; exit 0`, + { [OpenCodeServerLedger.SERVER_MARKER_ENV]: marker }, ); const shell = yield* Effect.acquireRelease( fs.readFileString(shellPidFile).pipe(Effect.map((pid) => Number(pid.trim()))), @@ -162,7 +180,7 @@ describe.each(observedPlatforms)("OpenCodeServerLedger observing as %s", (platfo ); expect(shell).not.toBe(server.pid); expect(groupExists(shell)).toBe(true); - yield* recordFromDeadServer(stateDir, server); + yield* recordFromDeadServer(stateDir, server, marker); yield* fs.writeFileString(gate, "\n"); yield* server.exited; @@ -279,7 +297,12 @@ describe.each(observedPlatforms)("OpenCodeServerLedger observing as %s", (platfo const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-opencode-ledger-" }); const running = yield* OpenCodeServerLedger.make({ stateDir }); const server = yield* spawnGroup(SERVE_ARGS); - const forget = yield* running.track({ pid: server.pid, port: 4096, args: SERVE_ARGS }); + const forget = yield* running.track({ + pid: server.pid, + port: 4096, + args: SERVE_ARGS, + marker: SecureRandom.uuidV4Unsafe(), + }); const other = yield* OpenCodeServerLedger.make({ stateDir }); yield* other.reapOrphans; diff --git a/packages/provider-opencode/src/server/OpenCodeServerLedger.ts b/packages/provider-opencode/src/server/OpenCodeServerLedger.ts index fb3ea7645e9bf..08cb53462bc9b 100644 --- a/packages/provider-opencode/src/server/OpenCodeServerLedger.ts +++ b/packages/provider-opencode/src/server/OpenCodeServerLedger.ts @@ -1,4 +1,5 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Config from "effect/Config"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -22,6 +23,7 @@ const OpenCodeServerEntry = Schema.Struct({ startTime: Schema.String, command: Schema.String, args: Schema.optionalKey(Schema.Array(Schema.String)), + marker: Schema.optionalKey(Schema.String), port: Schema.Int, stateDir: Schema.String, owner: ProcessIdentity, @@ -38,10 +40,15 @@ export class OpenCodeServerLedger extends Context.Service< readonly pid: number; readonly port: number; readonly args: ReadonlyArray; + readonly marker: string; }) => Effect.Effect>; } >()("@t3tools/provider-opencode/server/OpenCodeServerLedger") {} +// OpenCode starts each shell command in its own session, so once the server exits +// only this inherited variable, set to the entry's marker, ties a command to it. +export const SERVER_MARKER_ENV = "T3CODE_OPENCODE_SERVER_MARKER"; + const ENTRY_DIRECTORY = "opencode-servers"; const ENTRY_FILE = /^\d+\.json$/; const STOP_POLL_INTERVAL = "50 millis"; @@ -104,6 +111,8 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { const platform = yield* HostProcess.Platform; const directory = path.join(input.stateDir, ENTRY_DIRECTORY); const recordable = platform === "linux" || platform === "darwin"; + // A T3 Code server started from one of those commands carries a marker too. + const inheritedMarker = yield* Config.String(SERVER_MARKER_ENV).pipe(Config.withDefault("")); // A process-group scan calls this for every /proc entry; avoid a span per process. const observeLinux = Effect.fnUntraced(function* (pid: number, group?: number) { @@ -176,6 +185,46 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { return Effect.succeed([]); }; + // procps has no `ps -E`, so /proc serves wherever it exists; macOS has no /proc. + const markedGroups = (marker: string): Effect.Effect> => { + const variable = `${SERVER_MARKER_ENV}=${marker}`; + const fromProc = fs.readDirectory("/proc").pipe( + Effect.flatMap((names) => + Effect.forEach( + names.filter((name) => /^\d+$/.test(name)), + (name) => + fs.readFileString(`/proc/${name}/environ`).pipe( + Effect.flatMap((environ) => + environ.split("\0").includes(variable) + ? observeLinux(Number(name)) + : Effect.undefined, + ), + Effect.orElseSucceed(() => undefined), + ), + { concurrency: 16 }, + ), + ), + ); + const fromPs = psDarwin(["-A", "-E"]).pipe( + Effect.map((found) => + found.filter((candidate) => candidate.command.split(" ").includes(variable)), + ), + ); + return fs.exists("/proc/self/environ").pipe( + Effect.flatMap((hasProc) => + hasProc ? fromProc : platform === "darwin" ? fromPs : Effect.succeed([]), + ), + Effect.map((found) => [ + ...new Set( + found.flatMap((candidate) => + candidate === undefined || candidate.zombie ? [] : [candidate.pgid], + ), + ), + ]), + Effect.orElseSucceed(() => []), + ); + }; + const isRunning = (identity: ProcessIdentity) => observe(identity.pid).pipe( Effect.map( @@ -217,6 +266,7 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { startTime: member.startTime, command: member.command, args: server.args, + marker: server.marker, port: server.port, stateDir: input.stateDir, owner, @@ -243,6 +293,15 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { ), ); + const stopGroup = Effect.fn("OpenCodeServerLedger.stopGroup")(function* (pgid: number) { + signalGroup(pgid, "SIGTERM"); + for (let attempt = 0; attempt < STOP_POLL_ATTEMPTS && groupExists(pgid); attempt++) { + yield* Effect.sleep(STOP_POLL_INTERVAL); + } + // The group never emptied, so its pgid cannot have been reused. + if (groupExists(pgid)) signalGroup(pgid, "SIGKILL"); + }); + // Signals only a group whose recorded member still has the same pid, start // time, and group and still has the same command line or serve arguments, so // a recycled pid is never touched. @@ -264,12 +323,21 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { pid: entry.pgid, port: entry.port, }); - signalGroup(entry.pgid, "SIGTERM"); - for (let attempt = 0; attempt < STOP_POLL_ATTEMPTS && groupExists(entry.pgid); attempt++) { - yield* Effect.sleep(STOP_POLL_INTERVAL); - } - // The group never emptied, so its pgid cannot have been reused. - if (groupExists(entry.pgid)) signalGroup(entry.pgid, "SIGKILL"); + yield* stopGroup(entry.pgid); + }); + + // Runs after the server is stopped so it cannot start more commands meanwhile. + const stopMarked = Effect.fn("OpenCodeServerLedger.stopMarked")(function* ( + entry: OpenCodeServerEntry, + ) { + if (entry.marker === undefined || entry.marker === inheritedMarker) return; + const groups = yield* markedGroups(entry.marker); + if (groups.length === 0) return; + yield* Effect.logInfo("Stopping commands left by a previous OpenCode server", { + pid: entry.pgid, + groups, + }); + yield* Effect.forEach(groups, stopGroup, { concurrency: "unbounded", discard: true }); }); const reapEntry = Effect.fn("OpenCodeServerLedger.reapEntry")( @@ -278,6 +346,7 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { if (Option.isSome(entry) && entry.value.stateDir === input.stateDir) { if (yield* isRunning(entry.value.owner)) return; yield* stopOrphan(entry.value); + yield* stopMarked(entry.value); } yield* fs.remove(entryPath, { force: true }); },