From 59b13a2d81a8a13c3e57e9ebd3e296cd83eb8f9d Mon Sep 17 00:00:00 2001 From: Birzool Date: Mon, 19 Oct 2026 14:43:18 +0300 Subject: [PATCH 1/4] fix(server): stop a relay client left running by a server that died Tests: - RelayClientLedger: records a started relay client until it is forgotten - RelayClientLedger: stops a relay client left by a server that died - RelayClientLedger: leaves the relay client of a running server alone - RelayClientLedger: leaves a recycled pid alone when its start time / command line does not match - RelayClientLedger: leaves a relay client recorded for another state directory alone - CloudManagedEndpointRuntime: starts and stops the connector the same way without / with a relay client ledger (the record is forgotten only after the client is killed) Co-Authored-By: Claude Opus 5.5 --- .../src/cloud/ManagedEndpointRuntime.test.ts | 60 +++++ .../src/cloud/ManagedEndpointRuntime.ts | 16 ++ .../src/cloud/RelayClientLedger.test.ts | 178 ++++++++++++ apps/server/src/cloud/RelayClientLedger.ts | 255 ++++++++++++++++++ apps/server/src/server.ts | 9 + 5 files changed, 518 insertions(+) create mode 100644 apps/server/src/cloud/RelayClientLedger.test.ts create mode 100644 apps/server/src/cloud/RelayClientLedger.ts diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 10cf761e20a8b..2feac7ce8a01e 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -19,6 +19,7 @@ import * as RelayClient from "@t3tools/shared/relayClient"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; +import * as RelayClientLedger from "./RelayClientLedger.ts"; const layerRelayClientAvailable = Layer.succeed( RelayClient.RelayClient, @@ -581,6 +582,65 @@ describe("CloudManagedEndpointRuntime", () => { }), ); + it.effect.each([{ ledger: "without" }, { ledger: "with" }])( + "starts and stops the connector the same way $ledger a relay client ledger", + ({ ledger }) => + Effect.gen(function* () { + const events: Array = []; + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const handle = makeHandle({ + pid: 500, + onKill: () => { + events.push("kill:500"); + }, + }); + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore)); + return handle; + }), + ); + const layerRuntime = ManagedEndpointRuntime.layer.pipe( + Layer.provide(runtimeDependencies(spawner)), + ); + const context = yield* Layer.build( + ledger === "without" + ? layerRuntime + : layerRuntime.pipe( + Layer.provide( + Layer.succeed( + RelayClientLedger.RelayClientLedger, + RelayClientLedger.RelayClientLedger.of({ + track: ({ pid }) => + Effect.sync(() => { + events.push(`track:${pid}`); + return Effect.sync(() => { + events.push(`forget:${pid}`); + }); + }), + }), + ), + ), + ), + ); + const runtime = yield* Effect.service( + ManagedEndpointRuntime.CloudManagedEndpointRuntime, + ).pipe(Effect.provide(context)); + + const started = yield* runtime.applyConfig({ + providerKind: "cloudflare_tunnel", + connectorToken: "token", + }); + const stopped = yield* runtime.applyConfig(null); + + expect(started).toMatchObject({ status: "running", pid: 500 }); + expect(stopped).toEqual({ status: "disabled" }); + // The record outlives the client: it is forgotten only after the kill. + expect(events).toEqual( + ledger === "without" ? ["kill:500"] : ["track:500", "kill:500", "forget:500"], + ); + }), + ); + it.effect("stops an active connector when a non-Cloudflare runtime config is applied", () => Effect.gen(function* () { const killed: Array = []; diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 1302b2167d0ac..264f088d15a63 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -18,6 +18,8 @@ import { HttpClient } from "effect/http"; import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as RelayClientLedger from "./RelayClientLedger.ts"; + export type CloudManagedEndpointRuntimeStatus = | { readonly status: "disabled"; @@ -164,6 +166,7 @@ const stopConnector = (connector: ActiveConnector | null) => export const make = Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const relayClient = yield* RelayClient.RelayClient; + const relayClientLedger = yield* Effect.serviceOption(RelayClientLedger.RelayClientLedger); const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient); const activeRef = yield* Ref.make(null); const desiredConfigRef = yield* Ref.make(null); @@ -541,6 +544,14 @@ export const make = Effect.gen(function* () { } const connectorScope = yield* Scope.make("sequential"); + // Added before the spawn so it runs after the spawner's finalizer stops the + // client (finalizers run in reverse order): the ledger keeps the client + // recorded until it is actually stopped. + let forgetRelayClient: Effect.Effect = Effect.void; + yield* Scope.addFinalizer( + connectorScope, + Effect.suspend(() => forgetRelayClient), + ); const child = yield* spawner .spawn( ChildProcess.make( @@ -603,6 +614,11 @@ export const make = Effect.gen(function* () { } if (!("status" in child)) { + if (Option.isSome(relayClientLedger)) { + // Recorded so the next server stops it if this one dies without + // running finalizers; forgotten when the connector scope closes. + forgetRelayClient = yield* relayClientLedger.value.track({ pid: Number(child.pid) }); + } const connector = { child, executable, diff --git a/apps/server/src/cloud/RelayClientLedger.test.ts b/apps/server/src/cloud/RelayClientLedger.test.ts new file mode 100644 index 0000000000000..bb3da809ca843 --- /dev/null +++ b/apps/server/src/cloud/RelayClientLedger.test.ts @@ -0,0 +1,178 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeChildProcess from "node:child_process"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import * as RelayClientLedger from "./RelayClientLedger.ts"; + +const processExists = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +}; + +const spawnProcess = Effect.acquireRelease( + Effect.gen(function* () { + const child = NodeChildProcess.spawn("sleep", ["600"], { stdio: "ignore" }); + const exited = yield* Deferred.make(); + child.once("exit", (_code, signal) => Deferred.doneUnsafe(exited, Effect.succeed(signal))); + const pid = child.pid; + if (pid === undefined) return yield* Effect.die("spawn failed"); + return { pid, exited: Deferred.await(exited) }; + }), + ({ pid }) => + Effect.sync(() => { + if (processExists(pid)) process.kill(pid, "SIGKILL"); + }), +); + +const recordFromDeadServer = (stateDir: string, client: { readonly pid: number }) => + Effect.gen(function* () { + const previousServer = yield* spawnProcess; + const previousLedger = yield* RelayClientLedger.make({ + stateDir, + ownerPid: previousServer.pid, + }); + yield* Effect.asVoid(previousLedger.track({ pid: client.pid })); + process.kill(previousServer.pid, "SIGKILL"); + yield* previousServer.exited; + }); + +const hostPlatform = HostProcess.Platform.defaultValue(); +// procps accepts the same `ps` flags as macOS, so Linux also covers the macOS path. +const observedPlatforms: ReadonlyArray = + hostPlatform === "linux" ? ["linux", "darwin"] : hostPlatform === "darwin" ? ["darwin"] : []; + +describe.each(observedPlatforms)("RelayClientLedger observing as %s", (platform) => { + const provideHost = (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(HostProcess.Platform, platform), + Effect.provide(NodeServices.layer), + ); + + it.live("records a started relay client until it is forgotten", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const client = yield* spawnProcess; + const ledger = yield* RelayClientLedger.make({ stateDir }); + + const forget = yield* ledger.track({ pid: client.pid }); + + const entryPath = path.join(stateDir, "relay-clients", `${client.pid}.json`); + expect(yield* fs.readDirectory(path.join(stateDir, "relay-clients"))).toEqual([ + `${client.pid}.json`, + ]); + expect(JSON.parse(yield* fs.readFileString(entryPath))).toMatchObject({ + version: 1, + pid: client.pid, + command: "sleep 600", + stateDir, + owner: { pid: process.pid }, + }); + + yield* forget; + expect(yield* fs.readDirectory(path.join(stateDir, "relay-clients"))).toEqual([]); + expect(processExists(client.pid)).toBe(true); + }).pipe(provideHost), + ); + + it.live("stops a relay client left by a server that died", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const orphan = yield* spawnProcess; + yield* recordFromDeadServer(stateDir, orphan); + expect(yield* fs.readDirectory(path.join(stateDir, "relay-clients"))).toHaveLength(1); + + const restarted = yield* RelayClientLedger.make({ stateDir }); + yield* restarted.reapOrphans; + + expect(yield* orphan.exited).toBe("SIGTERM"); + expect(processExists(orphan.pid)).toBe(false); + expect(yield* fs.readDirectory(path.join(stateDir, "relay-clients"))).toEqual([]); + }).pipe(provideHost), + ); + + it.live("leaves the relay client of a running server alone", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const running = yield* RelayClientLedger.make({ stateDir }); + const client = yield* spawnProcess; + const forget = yield* running.track({ pid: client.pid }); + + const other = yield* RelayClientLedger.make({ stateDir }); + yield* other.reapOrphans; + + expect(processExists(client.pid)).toBe(true); + expect(yield* fs.readDirectory(path.join(stateDir, "relay-clients"))).toEqual([ + `${client.pid}.json`, + ]); + yield* forget; + }).pipe(provideHost), + ); + + it.live.each([ + { field: "start time", recorded: /"startTime":"[^"]+"/, replacement: '"startTime":"1"' }, + { + field: "command line", + recorded: /"command":"[^"]+"/, + replacement: '"command":"cloudflared tunnel run"', + }, + ])("leaves a recycled pid alone when its $field does not match", ({ recorded, replacement }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const unrelated = yield* spawnProcess; + yield* recordFromDeadServer(stateDir, unrelated); + const entryPath = path.join(stateDir, "relay-clients", `${unrelated.pid}.json`); + const entry = yield* fs.readFileString(entryPath); + expect(entry).toMatch(recorded); + yield* fs.writeFileString(entryPath, entry.replace(recorded, replacement)); + + const restarted = yield* RelayClientLedger.make({ stateDir }); + yield* restarted.reapOrphans; + + expect(processExists(unrelated.pid)).toBe(true); + expect(yield* fs.exists(entryPath)).toBe(false); + }).pipe(provideHost), + ); + + it.live("leaves a relay client recorded for another state directory alone", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const otherStateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const client = yield* spawnProcess; + yield* recordFromDeadServer(otherStateDir, client); + const entryName = `${client.pid}.json`; + const otherEntryPath = path.join(otherStateDir, "relay-clients", entryName); + const entryPath = path.join(stateDir, "relay-clients", entryName); + // A state directory copied from another install still names its origin. + yield* fs.makeDirectory(path.join(stateDir, "relay-clients"), { recursive: true }); + yield* fs.writeFileString(entryPath, yield* fs.readFileString(otherEntryPath)); + + const restarted = yield* RelayClientLedger.make({ stateDir }); + yield* restarted.reapOrphans; + + expect(processExists(client.pid)).toBe(true); + expect(yield* fs.exists(entryPath)).toBe(false); + expect(yield* fs.exists(otherEntryPath)).toBe(true); + }).pipe(provideHost), + ); +}); diff --git a/apps/server/src/cloud/RelayClientLedger.ts b/apps/server/src/cloud/RelayClientLedger.ts new file mode 100644 index 0000000000000..7d50db67601e4 --- /dev/null +++ b/apps/server/src/cloud/RelayClientLedger.ts @@ -0,0 +1,255 @@ +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +/** + * The relay client is a plain child of the server, stopped only by a scope + * finalizer. A server that dies without running finalizers (SIGKILL, a crash, + * an OOM kill) leaves it running under init/launchd, still serving the tunnel + * with the token it started with, and the next server starts a second one. + * Each started client is recorded here so the next server can stop it. + * + * Same identity rules as OpenCodeServerLedger: only a process whose pid, + * start time and full command line still match the record is signalled, and + * only after the server that started it is gone. + */ +const ProcessIdentity = Schema.Struct({ pid: Schema.Int, startTime: Schema.String }); +type ProcessIdentity = typeof ProcessIdentity.Type; + +const RelayClientEntry = Schema.Struct({ + version: Schema.Literal(1), + pid: Schema.Int, + startTime: Schema.String, + command: Schema.String, + stateDir: Schema.String, + owner: ProcessIdentity, +}); +type RelayClientEntry = typeof RelayClientEntry.Type; +const RelayClientEntryJson = Schema.fromJsonString(RelayClientEntry); +const decodeEntry = Schema.decodeUnknownOption(RelayClientEntryJson); +const encodeEntry = Schema.encodeEffect(RelayClientEntryJson); + +export class RelayClientLedger extends Context.Service< + RelayClientLedger, + { + /** Records a started relay client; the returned effect forgets it. */ + readonly track: (client: { readonly pid: number }) => Effect.Effect>; + } +>()("t3/cloud/RelayClientLedger") {} + +const ENTRY_DIRECTORY = "relay-clients"; +const ENTRY_FILE = /^\d+\.json$/; +const STOP_POLL_INTERVAL = "50 millis"; +const STOP_POLL_ATTEMPTS = 40; + +interface ObservedProcess { + readonly pid: number; + readonly startTime: string; + readonly command: string; + readonly zombie: boolean; +} + +// `ps -o lstart` prints e.g. `Sat Sep 6 20:51:57 2026`; C locale and UTC keep it stable. +const DARWIN_PS_LINE = /^\s*(\d+)\s+(\S+)\s+(\w{3} \w{3} +\d{1,2} \d{2}:\d{2}:\d{2} \d{4})\s+(.*)$/; + +const processExists = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch (cause) { + return (cause as NodeJS.ErrnoException | undefined)?.code !== "ESRCH"; + } +}; + +const signal = (pid: number, name: NodeJS.Signals) => { + try { + process.kill(pid, name); + } catch {} +}; + +export const make = Effect.fn("RelayClientLedger.make")(function* (input: { + readonly stateDir: string; + readonly ownerPid?: number; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const platform = yield* HostProcess.Platform; + const directory = path.join(input.stateDir, ENTRY_DIRECTORY); + const recordable = platform === "linux" || platform === "darwin"; + + const observeLinux = (pid: number) => + Effect.gen(function* () { + const stat = yield* fs.readFileString(`/proc/${pid}/stat`); + const fields = stat + .slice(stat.lastIndexOf(")") + 2) + .trim() + .split(/\s+/); + const startTime = fields[19]; + if (startTime === undefined) return undefined; + const cmdline = yield* fs.readFileString(`/proc/${pid}/cmdline`); + return { + pid, + startTime, + command: cmdline.replace(/\0$/, "").replaceAll("\0", " "), + zombie: fields[0] === "Z", + } satisfies ObservedProcess; + }); + + const observeDarwin = (pid: number) => + spawner + .string( + ChildProcess.make("/bin/ps", ["-ww", "-o", "pid=,stat=,lstart=,args=", "-p", String(pid)], { + env: { LC_ALL: "C", TZ: "UTC" }, + extendEnv: true, + stdin: "ignore", + stderr: "ignore", + }), + ) + .pipe( + Effect.timeout("5 seconds"), + Effect.map((output) => + output.split("\n").flatMap((line): ReadonlyArray => { + const match = DARWIN_PS_LINE.exec(line); + if (match === null || Number(match[1]) !== pid) return []; + return [ + { + pid, + zombie: match[2]!.startsWith("Z"), + startTime: match[3]!, + command: match[4]!.trimEnd(), + }, + ]; + }), + ), + Effect.map((found) => found[0]), + ); + + const observe = (pid: number): Effect.Effect => { + if (platform === "linux") return observeLinux(pid).pipe(Effect.orElseSucceed(() => undefined)); + if (platform === "darwin") { + return observeDarwin(pid).pipe(Effect.orElseSucceed(() => undefined)); + } + return Effect.succeed(undefined); + }; + + const sameProcess = ( + observed: ObservedProcess | undefined, + identity: { readonly startTime: string; readonly command?: string }, + ) => + observed !== undefined && + !observed.zombie && + observed.startTime === identity.startTime && + (identity.command === undefined || observed.command === identity.command); + + const ownerPid = input.ownerPid ?? process.pid; + const ownerProcess = yield* observe(ownerPid); + const owner = + ownerProcess === undefined ? undefined : { pid: ownerPid, startTime: ownerProcess.startTime }; + + const track: RelayClientLedger["Service"]["track"] = (client) => + Effect.gen(function* () { + if (!recordable) return Effect.void; + const observed = yield* observe(client.pid); + if (owner === undefined || observed === undefined || observed.zombie) { + yield* Effect.logWarning( + "Could not record the relay client; it will keep running if this server crashes", + { pid: client.pid }, + ); + return Effect.void; + } + const entryPath = path.join(directory, `${client.pid}.json`); + const entry: RelayClientEntry = { + version: 1, + pid: observed.pid, + startTime: observed.startTime, + command: observed.command, + stateDir: input.stateDir, + owner, + }; + yield* writeFileStringAtomically({ + filePath: entryPath, + contents: yield* encodeEntry(entry), + }).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + return fs + .remove(entryPath, { force: true }) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not forget a stopped relay client", { cause }), + ), + ); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not record the relay client process", { cause }).pipe( + Effect.as(Effect.void), + ), + ), + ); + + // SIGTERM lets cloudflared close its edge connections; SIGKILL only if the + // same process (not a recycled pid) is still there afterwards. + const stopOrphan = (entry: RelayClientEntry) => + Effect.gen(function* () { + if (!sameProcess(yield* observe(entry.pid), entry)) return; + yield* Effect.logInfo("Stopping a relay client left by a previous T3 Code server", { + pid: entry.pid, + }); + signal(entry.pid, "SIGTERM"); + for (let attempt = 0; attempt < STOP_POLL_ATTEMPTS && processExists(entry.pid); attempt++) { + yield* Effect.sleep(STOP_POLL_INTERVAL); + } + if (sameProcess(yield* observe(entry.pid), entry)) signal(entry.pid, "SIGKILL"); + }); + + const reapEntry = (entryPath: string) => + Effect.gen(function* () { + const entry = decodeEntry(yield* fs.readFileString(entryPath)); + if (Option.isSome(entry) && entry.value.stateDir === input.stateDir) { + if (sameProcess(yield* observe(entry.value.owner.pid), entry.value.owner)) return; + yield* stopOrphan(entry.value); + } + yield* fs.remove(entryPath, { force: true }); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not clean up a recorded relay client", { entryPath, cause }), + ), + ); + + const reapOrphans = Effect.gen(function* () { + if (!recordable) return; + const names = yield* fs.readDirectory(directory).pipe(Effect.orElseSucceed(() => [])); + yield* Effect.forEach( + names.filter((name) => ENTRY_FILE.test(name)), + (name) => reapEntry(path.join(directory, name)), + { concurrency: "unbounded", discard: true }, + ); + }); + + return { track, reapOrphans }; +}); + +export const layer = (input: { readonly stateDir: string }) => + Layer.effect( + RelayClientLedger, + Effect.gen(function* () { + const ledger = yield* make(input); + yield* ledger.reapOrphans.pipe(Effect.forkScoped); + return RelayClientLedger.of({ track: ledger.track }); + }), + ); + +export const layerTest = Layer.succeed( + RelayClientLedger, + RelayClientLedger.of({ track: () => Effect.succeed(Effect.void) }), +); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 59daedc151071..97c70aaf58cc9 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -142,6 +142,7 @@ import * as CloudLink from "./cloud/CloudLink.ts"; import { pendingServiceUpdateExists } from "./cloud/updateHandoff.ts"; import * as RelayTracing from "./cloud/relayTracing.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; +import * as RelayClientLedger from "./cloud/RelayClientLedger.ts"; import { MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER, MANAGED_TUNNEL_RECOVERY_COOLDOWN, @@ -457,6 +458,14 @@ const layerCloudManagedEndpointRuntime = Layer.mergeAll( CloudManagedEndpointRuntime.layer.pipe( Layer.provide(ServerSecretStore.layer), Layer.provide(layerRelayClient), + Layer.provide( + Layer.unwrap( + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + return RelayClientLedger.layer({ stateDir: config.stateDir }); + }), + ), + ), ), ); From 368688b901899e5361b44c9418bc21f0d1f8f16f Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Mon, 26 Oct 2026 17:05:53 +0000 Subject: [PATCH 2/4] fix(server): retain relay records when orphan cleanup fails --- .../src/cloud/ManagedEndpointRuntime.ts | 6 +- .../src/cloud/RelayClientLedger.test.ts | 25 ++++ apps/server/src/cloud/RelayClientLedger.ts | 119 ++++++++---------- 3 files changed, 81 insertions(+), 69 deletions(-) diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index aa8daf42402c4..e1636666bcf43 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -605,9 +605,7 @@ export const make = Effect.gen(function* () { const http2 = yield* Ref.get(http2Ref); const connectorScope = yield* Scope.make("sequential"); - // Added before the spawn so it runs after the spawner's finalizer stops the - // client (finalizers run in reverse order): the ledger keeps the client - // recorded until it is actually stopped. + // Finalizers run in reverse order; forget only after the spawner stops the client. let forgetRelayClient: Effect.Effect = Effect.void; yield* Scope.addFinalizer( connectorScope, @@ -678,8 +676,6 @@ export const make = Effect.gen(function* () { if (!("status" in child)) { if (Option.isSome(relayClientLedger)) { - // Recorded so the next server stops it if this one dies without - // running finalizers; forgotten when the connector scope closes. forgetRelayClient = yield* relayClientLedger.value.track({ pid: Number(child.pid) }); } const connector = { diff --git a/apps/server/src/cloud/RelayClientLedger.test.ts b/apps/server/src/cloud/RelayClientLedger.test.ts index bb3da809ca843..07f4b6360bdd3 100644 --- a/apps/server/src/cloud/RelayClientLedger.test.ts +++ b/apps/server/src/cloud/RelayClientLedger.test.ts @@ -8,6 +8,7 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import { vi } from "vite-plus/test"; import * as RelayClientLedger from "./RelayClientLedger.ts"; @@ -105,6 +106,30 @@ describe.each(observedPlatforms)("RelayClientLedger observing as %s", (platform) }).pipe(provideHost), ); + it.live("keeps the record when the orphan cannot be signalled", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const orphan = yield* spawnProcess; + yield* recordFromDeadServer(stateDir, orphan); + const kill = process.kill.bind(process); + using _kill = vi.spyOn(process, "kill").mockImplementation((pid, signal) => { + if (pid === orphan.pid && signal !== 0) + throw Object.assign(new Error("denied"), { code: "EPERM" }); + return kill(pid, signal); + }); + + const restarted = yield* RelayClientLedger.make({ stateDir }); + yield* restarted.reapOrphans; + + expect(processExists(orphan.pid)).toBe(true); + expect(yield* fs.exists(path.join(stateDir, "relay-clients", `${orphan.pid}.json`))).toBe( + true, + ); + }).pipe(provideHost), + ); + it.live("leaves the relay client of a running server alone", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/cloud/RelayClientLedger.ts b/apps/server/src/cloud/RelayClientLedger.ts index 7d50db67601e4..d91eeb4d0cd78 100644 --- a/apps/server/src/cloud/RelayClientLedger.ts +++ b/apps/server/src/cloud/RelayClientLedger.ts @@ -10,19 +10,7 @@ import * as Schema from "effect/Schema"; import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; -/** - * The relay client is a plain child of the server, stopped only by a scope - * finalizer. A server that dies without running finalizers (SIGKILL, a crash, - * an OOM kill) leaves it running under init/launchd, still serving the tunnel - * with the token it started with, and the next server starts a second one. - * Each started client is recorded here so the next server can stop it. - * - * Same identity rules as OpenCodeServerLedger: only a process whose pid, - * start time and full command line still match the record is signalled, and - * only after the server that started it is gone. - */ const ProcessIdentity = Schema.Struct({ pid: Schema.Int, startTime: Schema.String }); -type ProcessIdentity = typeof ProcessIdentity.Type; const RelayClientEntry = Schema.Struct({ version: Schema.Literal(1), @@ -72,7 +60,9 @@ const processExists = (pid: number) => { const signal = (pid: number, name: NodeJS.Signals) => { try { process.kill(pid, name); - } catch {} + } catch (cause) { + if ((cause as NodeJS.ErrnoException | undefined)?.code !== "ESRCH") throw cause; + } }; export const make = Effect.fn("RelayClientLedger.make")(function* (input: { @@ -86,23 +76,22 @@ export const make = Effect.fn("RelayClientLedger.make")(function* (input: { const directory = path.join(input.stateDir, ENTRY_DIRECTORY); const recordable = platform === "linux" || platform === "darwin"; - const observeLinux = (pid: number) => - Effect.gen(function* () { - const stat = yield* fs.readFileString(`/proc/${pid}/stat`); - const fields = stat - .slice(stat.lastIndexOf(")") + 2) - .trim() - .split(/\s+/); - const startTime = fields[19]; - if (startTime === undefined) return undefined; - const cmdline = yield* fs.readFileString(`/proc/${pid}/cmdline`); - return { - pid, - startTime, - command: cmdline.replace(/\0$/, "").replaceAll("\0", " "), - zombie: fields[0] === "Z", - } satisfies ObservedProcess; - }); + const observeLinux = Effect.fn("RelayClientLedger.observeLinux")(function* (pid: number) { + const stat = yield* fs.readFileString(`/proc/${pid}/stat`); + const fields = stat + .slice(stat.lastIndexOf(")") + 2) + .trim() + .split(/\s+/); + const startTime = fields[19]; + if (startTime === undefined) return undefined; + const cmdline = yield* fs.readFileString(`/proc/${pid}/cmdline`); + return { + pid, + startTime, + command: cmdline.replace(/\0$/, "").replaceAll("\0", " "), + zombie: fields[0] === "Z", + } satisfies ObservedProcess; + }); const observeDarwin = (pid: number) => spawner @@ -138,6 +127,7 @@ export const make = Effect.fn("RelayClientLedger.make")(function* (input: { if (platform === "darwin") { return observeDarwin(pid).pipe(Effect.orElseSucceed(() => undefined)); } + // @effect-diagnostics-next-line effectSucceedWithVoid:off - Missing process observations are undefined, not void. return Effect.succeed(undefined); }; @@ -155,8 +145,8 @@ export const make = Effect.fn("RelayClientLedger.make")(function* (input: { const owner = ownerProcess === undefined ? undefined : { pid: ownerPid, startTime: ownerProcess.startTime }; - const track: RelayClientLedger["Service"]["track"] = (client) => - Effect.gen(function* () { + const track: RelayClientLedger["Service"]["track"] = Effect.fn("RelayClientLedger.track")( + function* (client) { if (!recordable) return Effect.void; const observed = yield* observe(client.pid); if (owner === undefined || observed === undefined || observed.zombie) { @@ -189,42 +179,48 @@ export const make = Effect.fn("RelayClientLedger.make")(function* (input: { Effect.logWarning("Could not forget a stopped relay client", { cause }), ), ); - }).pipe( - Effect.catchCause((cause) => - Effect.logWarning("Could not record the relay client process", { cause }).pipe( - Effect.as(Effect.void), - ), + }, + Effect.catchCause((cause) => + Effect.logWarning("Could not record the relay client process", { cause }).pipe( + Effect.as(Effect.void), ), - ); + ), + ); - // SIGTERM lets cloudflared close its edge connections; SIGKILL only if the - // same process (not a recycled pid) is still there afterwards. - const stopOrphan = (entry: RelayClientEntry) => - Effect.gen(function* () { - if (!sameProcess(yield* observe(entry.pid), entry)) return; - yield* Effect.logInfo("Stopping a relay client left by a previous T3 Code server", { - pid: entry.pid, - }); - signal(entry.pid, "SIGTERM"); - for (let attempt = 0; attempt < STOP_POLL_ATTEMPTS && processExists(entry.pid); attempt++) { + const stopOrphan = Effect.fn("RelayClientLedger.stopOrphan")(function* (entry: RelayClientEntry) { + if (!sameProcess(yield* observe(entry.pid), entry)) return true; + yield* Effect.logInfo("Stopping a relay client left by a previous T3 Code server", { + pid: entry.pid, + }); + for (const name of ["SIGTERM", "SIGKILL"] as const) { + if (!sameProcess(yield* observe(entry.pid), entry)) return true; + signal(entry.pid, name); + for (let attempt = 0; attempt < STOP_POLL_ATTEMPTS; attempt++) { + if (!sameProcess(yield* observe(entry.pid), entry)) return true; yield* Effect.sleep(STOP_POLL_INTERVAL); } - if (sameProcess(yield* observe(entry.pid), entry)) signal(entry.pid, "SIGKILL"); - }); + } + return !sameProcess(yield* observe(entry.pid), entry); + }); - const reapEntry = (entryPath: string) => - Effect.gen(function* () { + const reapEntry = Effect.fn("RelayClientLedger.reapEntry")( + function* (entryPath: string) { const entry = decodeEntry(yield* fs.readFileString(entryPath)); if (Option.isSome(entry) && entry.value.stateDir === input.stateDir) { - if (sameProcess(yield* observe(entry.value.owner.pid), entry.value.owner)) return; - yield* stopOrphan(entry.value); + const owner = yield* observe(entry.value.owner.pid); + if ( + sameProcess(owner, entry.value.owner) || + (owner === undefined && processExists(entry.value.owner.pid)) + ) + return; + if (!(yield* stopOrphan(entry.value))) return; } yield* fs.remove(entryPath, { force: true }); - }).pipe( - Effect.catchCause((cause) => - Effect.logWarning("Could not clean up a recorded relay client", { entryPath, cause }), - ), - ); + }, + Effect.catchCause((cause) => + Effect.logWarning("Could not clean up a recorded relay client", { cause }), + ), + ); const reapOrphans = Effect.gen(function* () { if (!recordable) return; @@ -248,8 +244,3 @@ export const layer = (input: { readonly stateDir: string }) => return RelayClientLedger.of({ track: ledger.track }); }), ); - -export const layerTest = Layer.succeed( - RelayClientLedger, - RelayClientLedger.of({ track: () => Effect.succeed(Effect.void) }), -); From 8c4963d3bcc299a87191db3b2b0f600bc741ab28 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Mon, 26 Oct 2026 17:14:40 +0000 Subject: [PATCH 3/4] fix(server): forget relay clients only after confirmed exit --- .../src/cloud/RelayClientLedger.test.ts | 63 ++++++++++++++++++- apps/server/src/cloud/RelayClientLedger.ts | 34 ++++++---- 2 files changed, 84 insertions(+), 13 deletions(-) diff --git a/apps/server/src/cloud/RelayClientLedger.test.ts b/apps/server/src/cloud/RelayClientLedger.test.ts index 07f4b6360bdd3..bd051be4d6483 100644 --- a/apps/server/src/cloud/RelayClientLedger.test.ts +++ b/apps/server/src/cloud/RelayClientLedger.test.ts @@ -8,6 +8,9 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import { vi } from "vite-plus/test"; import * as RelayClientLedger from "./RelayClientLedger.ts"; @@ -60,7 +63,7 @@ describe.each(observedPlatforms)("RelayClientLedger observing as %s", (platform) Effect.provide(NodeServices.layer), ); - it.live("records a started relay client until it is forgotten", () => + it.live("keeps a running relay client recorded until it exits", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -82,9 +85,12 @@ describe.each(observedPlatforms)("RelayClientLedger observing as %s", (platform) owner: { pid: process.pid }, }); + yield* forget; + expect(yield* fs.exists(entryPath)).toBe(true); + process.kill(client.pid, "SIGTERM"); + yield* client.exited; yield* forget; expect(yield* fs.readDirectory(path.join(stateDir, "relay-clients"))).toEqual([]); - expect(processExists(client.pid)).toBe(true); }).pipe(provideHost), ); @@ -130,6 +136,59 @@ describe.each(observedPlatforms)("RelayClientLedger observing as %s", (platform) }).pipe(provideHost), ); + it.live.each([0, 2])( + "keeps the record when identity reads fail after %s observations", + (beforeFailure) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const orphan = yield* spawnProcess; + yield* recordFromDeadServer(stateDir, orphan); + const restarted = yield* RelayClientLedger.make({ stateDir }); + const denied = Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "process", + method: "observe", + }), + ); + let observations = 0; + const read = fs.readFileString.bind(fs); + using _read = vi + .spyOn(fs, "readFileString") + .mockImplementation((file, ...args) => + file === `/proc/${orphan.pid}/stat` && observations++ >= beforeFailure + ? denied + : read(file, ...args), + ); + const ps = spawner.string.bind(spawner); + using _ps = vi + .spyOn(spawner, "string") + .mockImplementation((command, ...args) => + ChildProcess.isStandardCommand(command) && + command.args.at(-1) === String(orphan.pid) && + observations++ >= beforeFailure + ? denied + : ps(command, ...args), + ); + const kill = process.kill.bind(process); + using _kill = vi + .spyOn(process, "kill") + .mockImplementation((pid, signal) => + pid === orphan.pid && signal !== 0 ? true : kill(pid, signal), + ); + + yield* restarted.reapOrphans; + + expect(processExists(orphan.pid)).toBe(true); + expect(yield* fs.exists(path.join(stateDir, "relay-clients", `${orphan.pid}.json`))).toBe( + true, + ); + }).pipe(provideHost), + ); + it.live("leaves the relay client of a running server alone", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/cloud/RelayClientLedger.ts b/apps/server/src/cloud/RelayClientLedger.ts index d91eeb4d0cd78..a37ad5f547abb 100644 --- a/apps/server/src/cloud/RelayClientLedger.ts +++ b/apps/server/src/cloud/RelayClientLedger.ts @@ -140,6 +140,14 @@ export const make = Effect.fn("RelayClientLedger.make")(function* (input: { observed.startTime === identity.startTime && (identity.command === undefined || observed.command === identity.command); + const matchesRecordedProcess = Effect.fn("RelayClientLedger.matchesRecordedProcess")(function* ( + entry: RelayClientEntry, + ) { + const observed = yield* observe(entry.pid); + if (observed === undefined && processExists(entry.pid)) return undefined; + return sameProcess(observed, entry); + }); + const ownerPid = input.ownerPid ?? process.pid; const ownerProcess = yield* observe(ownerPid); const owner = @@ -172,13 +180,14 @@ export const make = Effect.fn("RelayClientLedger.make")(function* (input: { Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), ); - return fs - .remove(entryPath, { force: true }) - .pipe( - Effect.catchCause((cause) => - Effect.logWarning("Could not forget a stopped relay client", { cause }), - ), - ); + return matchesRecordedProcess(entry).pipe( + Effect.flatMap((matching) => + matching === false ? fs.remove(entryPath, { force: true }) : Effect.void, + ), + Effect.catchCause((cause) => + Effect.logWarning("Could not forget a stopped relay client", { cause }), + ), + ); }, Effect.catchCause((cause) => Effect.logWarning("Could not record the relay client process", { cause }).pipe( @@ -188,19 +197,22 @@ export const make = Effect.fn("RelayClientLedger.make")(function* (input: { ); const stopOrphan = Effect.fn("RelayClientLedger.stopOrphan")(function* (entry: RelayClientEntry) { - if (!sameProcess(yield* observe(entry.pid), entry)) return true; + const matching = yield* matchesRecordedProcess(entry); + if (matching !== true) return matching === false; yield* Effect.logInfo("Stopping a relay client left by a previous T3 Code server", { pid: entry.pid, }); for (const name of ["SIGTERM", "SIGKILL"] as const) { - if (!sameProcess(yield* observe(entry.pid), entry)) return true; + const matching = yield* matchesRecordedProcess(entry); + if (matching !== true) return matching === false; signal(entry.pid, name); for (let attempt = 0; attempt < STOP_POLL_ATTEMPTS; attempt++) { - if (!sameProcess(yield* observe(entry.pid), entry)) return true; + const matching = yield* matchesRecordedProcess(entry); + if (matching !== true) return matching === false; yield* Effect.sleep(STOP_POLL_INTERVAL); } } - return !sameProcess(yield* observe(entry.pid), entry); + return (yield* matchesRecordedProcess(entry)) === false; }); const reapEntry = Effect.fn("RelayClientLedger.reapEntry")( From 049e5a714b45156640a8841efaf6aff18dbb4010 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Mon, 26 Oct 2026 17:25:05 +0000 Subject: [PATCH 4/4] fix(server): reap relay clients before accepting replacements --- .../src/cloud/RelayClientLedger.test.ts | 29 +++++++++++++++++-- apps/server/src/cloud/RelayClientLedger.ts | 2 +- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/apps/server/src/cloud/RelayClientLedger.test.ts b/apps/server/src/cloud/RelayClientLedger.test.ts index bd051be4d6483..80f8a2c497bd4 100644 --- a/apps/server/src/cloud/RelayClientLedger.test.ts +++ b/apps/server/src/cloud/RelayClientLedger.test.ts @@ -7,6 +7,8 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as ChildProcess from "effect/process/ChildProcess"; @@ -103,8 +105,31 @@ describe.each(observedPlatforms)("RelayClientLedger observing as %s", (platform) yield* recordFromDeadServer(stateDir, orphan); expect(yield* fs.readDirectory(path.join(stateDir, "relay-clients"))).toHaveLength(1); - const restarted = yield* RelayClientLedger.make({ stateDir }); - yield* restarted.reapOrphans; + const cleaning = yield* Deferred.make(); + const releaseCleanup = yield* Deferred.make(); + const exposed = yield* Deferred.make(); + const entryPath = path.join(stateDir, "relay-clients", `${orphan.pid}.json`); + const remove = fs.remove.bind(fs); + using _remove = vi + .spyOn(fs, "remove") + .mockImplementation((file, options) => + file === entryPath + ? Deferred.succeed(cleaning, undefined).pipe( + Effect.andThen(Deferred.await(releaseCleanup)), + Effect.andThen(remove(file, options)), + ) + : remove(file, options), + ); + const build = yield* Layer.build(RelayClientLedger.layer({ stateDir })).pipe( + Effect.tap(() => Deferred.succeed(exposed, undefined)), + Effect.forkChild, + ); + yield* Deferred.await(cleaning); + const exposedBeforeCleanup = yield* Deferred.isDone(exposed); + yield* Deferred.succeed(releaseCleanup, undefined); + yield* Fiber.join(build); + + expect(exposedBeforeCleanup).toBe(false); expect(yield* orphan.exited).toBe("SIGTERM"); expect(processExists(orphan.pid)).toBe(false); diff --git a/apps/server/src/cloud/RelayClientLedger.ts b/apps/server/src/cloud/RelayClientLedger.ts index a37ad5f547abb..4b721e9021dff 100644 --- a/apps/server/src/cloud/RelayClientLedger.ts +++ b/apps/server/src/cloud/RelayClientLedger.ts @@ -252,7 +252,7 @@ export const layer = (input: { readonly stateDir: string }) => RelayClientLedger, Effect.gen(function* () { const ledger = yield* make(input); - yield* ledger.reapOrphans.pipe(Effect.forkScoped); + yield* ledger.reapOrphans; return RelayClientLedger.of({ track: ledger.track }); }), );