diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 3b9d55ce3f6cd..00c30210702e3 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -18,6 +18,7 @@ import * as RelayClient from "@t3tools/shared/relayClient"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; +import * as RelayClientLedger from "./RelayClientLedger.ts"; const layerRelayClientAvailable = Layer.succeed( RelayClient.RelayClient, @@ -528,6 +529,65 @@ describe("CloudManagedEndpointRuntime", () => { }), ); + it.effect.each([{ ledger: "without" }, { ledger: "with" }])( + "starts and stops the connector the same way $ledger a relay client ledger", + ({ ledger }) => + Effect.gen(function* () { + const events: Array = []; + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const handle = makeHandle({ + pid: 500, + onKill: () => { + events.push("kill:500"); + }, + }); + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore)); + return handle; + }), + ); + const layerRuntime = ManagedEndpointRuntime.layer.pipe( + Layer.provide(runtimeDependencies(spawner)), + ); + const context = yield* Layer.build( + ledger === "without" + ? layerRuntime + : layerRuntime.pipe( + Layer.provide( + Layer.succeed( + RelayClientLedger.RelayClientLedger, + RelayClientLedger.RelayClientLedger.of({ + track: ({ pid }) => + Effect.sync(() => { + events.push(`track:${pid}`); + return Effect.sync(() => { + events.push(`forget:${pid}`); + }); + }), + }), + ), + ), + ), + ); + const runtime = yield* Effect.service( + ManagedEndpointRuntime.CloudManagedEndpointRuntime, + ).pipe(Effect.provide(context)); + + const started = yield* runtime.applyConfig({ + providerKind: "cloudflare_tunnel", + connectorToken: "token", + }); + const stopped = yield* runtime.applyConfig(null); + + expect(started).toMatchObject({ status: "running", pid: 500 }); + expect(stopped).toEqual({ status: "disabled" }); + // The record outlives the client: it is forgotten only after the kill. + expect(events).toEqual( + ledger === "without" ? ["kill:500"] : ["track:500", "kill:500", "forget:500"], + ); + }), + ); + it.effect("supervises the active connector and restarts it after process exit", () => Effect.gen(function* () { const spawned: Array = []; diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 5ffd31de9acab..e1636666bcf43 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -19,6 +19,8 @@ import { HttpClient, HttpClientResponse } from "effect/http"; import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as RelayClientLedger from "./RelayClientLedger.ts"; + export type CloudManagedEndpointRuntimeStatus = | { readonly status: "disabled"; @@ -177,6 +179,7 @@ const stopConnector = (connector: ActiveConnector | null) => export const make = Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const relayClient = yield* RelayClient.RelayClient; + const relayClientLedger = yield* Effect.serviceOption(RelayClientLedger.RelayClientLedger); const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient); const activeRef = yield* Ref.make(null); const desiredConfigRef = yield* Ref.make(null); @@ -602,6 +605,12 @@ export const make = Effect.gen(function* () { const http2 = yield* Ref.get(http2Ref); const connectorScope = yield* Scope.make("sequential"); + // Finalizers run in reverse order; forget only after the spawner stops the client. + let forgetRelayClient: Effect.Effect = Effect.void; + yield* Scope.addFinalizer( + connectorScope, + Effect.suspend(() => forgetRelayClient), + ); const child = yield* spawner .spawn( ChildProcess.make( @@ -666,6 +675,9 @@ export const make = Effect.gen(function* () { } if (!("status" in child)) { + if (Option.isSome(relayClientLedger)) { + forgetRelayClient = yield* relayClientLedger.value.track({ pid: Number(child.pid) }); + } const connector = { child, executable, diff --git a/apps/server/src/cloud/RelayClientLedger.test.ts b/apps/server/src/cloud/RelayClientLedger.test.ts new file mode 100644 index 0000000000000..80f8a2c497bd4 --- /dev/null +++ b/apps/server/src/cloud/RelayClientLedger.test.ts @@ -0,0 +1,287 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeChildProcess from "node:child_process"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import { vi } from "vite-plus/test"; + +import * as RelayClientLedger from "./RelayClientLedger.ts"; + +const processExists = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +}; + +const spawnProcess = Effect.acquireRelease( + Effect.gen(function* () { + const child = NodeChildProcess.spawn("sleep", ["600"], { stdio: "ignore" }); + const exited = yield* Deferred.make(); + child.once("exit", (_code, signal) => Deferred.doneUnsafe(exited, Effect.succeed(signal))); + const pid = child.pid; + if (pid === undefined) return yield* Effect.die("spawn failed"); + return { pid, exited: Deferred.await(exited) }; + }), + ({ pid }) => + Effect.sync(() => { + if (processExists(pid)) process.kill(pid, "SIGKILL"); + }), +); + +const recordFromDeadServer = (stateDir: string, client: { readonly pid: number }) => + Effect.gen(function* () { + const previousServer = yield* spawnProcess; + const previousLedger = yield* RelayClientLedger.make({ + stateDir, + ownerPid: previousServer.pid, + }); + yield* Effect.asVoid(previousLedger.track({ pid: client.pid })); + process.kill(previousServer.pid, "SIGKILL"); + yield* previousServer.exited; + }); + +const hostPlatform = HostProcess.Platform.defaultValue(); +// procps accepts the same `ps` flags as macOS, so Linux also covers the macOS path. +const observedPlatforms: ReadonlyArray = + hostPlatform === "linux" ? ["linux", "darwin"] : hostPlatform === "darwin" ? ["darwin"] : []; + +describe.each(observedPlatforms)("RelayClientLedger observing as %s", (platform) => { + const provideHost = (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(HostProcess.Platform, platform), + Effect.provide(NodeServices.layer), + ); + + it.live("keeps a running relay client recorded until it exits", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const client = yield* spawnProcess; + const ledger = yield* RelayClientLedger.make({ stateDir }); + + const forget = yield* ledger.track({ pid: client.pid }); + + const entryPath = path.join(stateDir, "relay-clients", `${client.pid}.json`); + expect(yield* fs.readDirectory(path.join(stateDir, "relay-clients"))).toEqual([ + `${client.pid}.json`, + ]); + expect(JSON.parse(yield* fs.readFileString(entryPath))).toMatchObject({ + version: 1, + pid: client.pid, + command: "sleep 600", + stateDir, + owner: { pid: process.pid }, + }); + + yield* forget; + expect(yield* fs.exists(entryPath)).toBe(true); + process.kill(client.pid, "SIGTERM"); + yield* client.exited; + yield* forget; + expect(yield* fs.readDirectory(path.join(stateDir, "relay-clients"))).toEqual([]); + }).pipe(provideHost), + ); + + it.live("stops a relay client left by a server that died", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const orphan = yield* spawnProcess; + yield* recordFromDeadServer(stateDir, orphan); + expect(yield* fs.readDirectory(path.join(stateDir, "relay-clients"))).toHaveLength(1); + + const cleaning = yield* Deferred.make(); + const releaseCleanup = yield* Deferred.make(); + const exposed = yield* Deferred.make(); + const entryPath = path.join(stateDir, "relay-clients", `${orphan.pid}.json`); + const remove = fs.remove.bind(fs); + using _remove = vi + .spyOn(fs, "remove") + .mockImplementation((file, options) => + file === entryPath + ? Deferred.succeed(cleaning, undefined).pipe( + Effect.andThen(Deferred.await(releaseCleanup)), + Effect.andThen(remove(file, options)), + ) + : remove(file, options), + ); + const build = yield* Layer.build(RelayClientLedger.layer({ stateDir })).pipe( + Effect.tap(() => Deferred.succeed(exposed, undefined)), + Effect.forkChild, + ); + yield* Deferred.await(cleaning); + const exposedBeforeCleanup = yield* Deferred.isDone(exposed); + yield* Deferred.succeed(releaseCleanup, undefined); + yield* Fiber.join(build); + + expect(exposedBeforeCleanup).toBe(false); + + expect(yield* orphan.exited).toBe("SIGTERM"); + expect(processExists(orphan.pid)).toBe(false); + expect(yield* fs.readDirectory(path.join(stateDir, "relay-clients"))).toEqual([]); + }).pipe(provideHost), + ); + + it.live("keeps the record when the orphan cannot be signalled", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const orphan = yield* spawnProcess; + yield* recordFromDeadServer(stateDir, orphan); + const kill = process.kill.bind(process); + using _kill = vi.spyOn(process, "kill").mockImplementation((pid, signal) => { + if (pid === orphan.pid && signal !== 0) + throw Object.assign(new Error("denied"), { code: "EPERM" }); + return kill(pid, signal); + }); + + const restarted = yield* RelayClientLedger.make({ stateDir }); + yield* restarted.reapOrphans; + + expect(processExists(orphan.pid)).toBe(true); + expect(yield* fs.exists(path.join(stateDir, "relay-clients", `${orphan.pid}.json`))).toBe( + true, + ); + }).pipe(provideHost), + ); + + it.live.each([0, 2])( + "keeps the record when identity reads fail after %s observations", + (beforeFailure) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const orphan = yield* spawnProcess; + yield* recordFromDeadServer(stateDir, orphan); + const restarted = yield* RelayClientLedger.make({ stateDir }); + const denied = Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "process", + method: "observe", + }), + ); + let observations = 0; + const read = fs.readFileString.bind(fs); + using _read = vi + .spyOn(fs, "readFileString") + .mockImplementation((file, ...args) => + file === `/proc/${orphan.pid}/stat` && observations++ >= beforeFailure + ? denied + : read(file, ...args), + ); + const ps = spawner.string.bind(spawner); + using _ps = vi + .spyOn(spawner, "string") + .mockImplementation((command, ...args) => + ChildProcess.isStandardCommand(command) && + command.args.at(-1) === String(orphan.pid) && + observations++ >= beforeFailure + ? denied + : ps(command, ...args), + ); + const kill = process.kill.bind(process); + using _kill = vi + .spyOn(process, "kill") + .mockImplementation((pid, signal) => + pid === orphan.pid && signal !== 0 ? true : kill(pid, signal), + ); + + yield* restarted.reapOrphans; + + expect(processExists(orphan.pid)).toBe(true); + expect(yield* fs.exists(path.join(stateDir, "relay-clients", `${orphan.pid}.json`))).toBe( + true, + ); + }).pipe(provideHost), + ); + + it.live("leaves the relay client of a running server alone", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const running = yield* RelayClientLedger.make({ stateDir }); + const client = yield* spawnProcess; + const forget = yield* running.track({ pid: client.pid }); + + const other = yield* RelayClientLedger.make({ stateDir }); + yield* other.reapOrphans; + + expect(processExists(client.pid)).toBe(true); + expect(yield* fs.readDirectory(path.join(stateDir, "relay-clients"))).toEqual([ + `${client.pid}.json`, + ]); + yield* forget; + }).pipe(provideHost), + ); + + it.live.each([ + { field: "start time", recorded: /"startTime":"[^"]+"/, replacement: '"startTime":"1"' }, + { + field: "command line", + recorded: /"command":"[^"]+"/, + replacement: '"command":"cloudflared tunnel run"', + }, + ])("leaves a recycled pid alone when its $field does not match", ({ recorded, replacement }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const unrelated = yield* spawnProcess; + yield* recordFromDeadServer(stateDir, unrelated); + const entryPath = path.join(stateDir, "relay-clients", `${unrelated.pid}.json`); + const entry = yield* fs.readFileString(entryPath); + expect(entry).toMatch(recorded); + yield* fs.writeFileString(entryPath, entry.replace(recorded, replacement)); + + const restarted = yield* RelayClientLedger.make({ stateDir }); + yield* restarted.reapOrphans; + + expect(processExists(unrelated.pid)).toBe(true); + expect(yield* fs.exists(entryPath)).toBe(false); + }).pipe(provideHost), + ); + + it.live("leaves a relay client recorded for another state directory alone", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const otherStateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-ledger-" }); + const client = yield* spawnProcess; + yield* recordFromDeadServer(otherStateDir, client); + const entryName = `${client.pid}.json`; + const otherEntryPath = path.join(otherStateDir, "relay-clients", entryName); + const entryPath = path.join(stateDir, "relay-clients", entryName); + // A state directory copied from another install still names its origin. + yield* fs.makeDirectory(path.join(stateDir, "relay-clients"), { recursive: true }); + yield* fs.writeFileString(entryPath, yield* fs.readFileString(otherEntryPath)); + + const restarted = yield* RelayClientLedger.make({ stateDir }); + yield* restarted.reapOrphans; + + expect(processExists(client.pid)).toBe(true); + expect(yield* fs.exists(entryPath)).toBe(false); + expect(yield* fs.exists(otherEntryPath)).toBe(true); + }).pipe(provideHost), + ); +}); diff --git a/apps/server/src/cloud/RelayClientLedger.ts b/apps/server/src/cloud/RelayClientLedger.ts new file mode 100644 index 0000000000000..4b721e9021dff --- /dev/null +++ b/apps/server/src/cloud/RelayClientLedger.ts @@ -0,0 +1,258 @@ +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +const ProcessIdentity = Schema.Struct({ pid: Schema.Int, startTime: Schema.String }); + +const RelayClientEntry = Schema.Struct({ + version: Schema.Literal(1), + pid: Schema.Int, + startTime: Schema.String, + command: Schema.String, + stateDir: Schema.String, + owner: ProcessIdentity, +}); +type RelayClientEntry = typeof RelayClientEntry.Type; +const RelayClientEntryJson = Schema.fromJsonString(RelayClientEntry); +const decodeEntry = Schema.decodeUnknownOption(RelayClientEntryJson); +const encodeEntry = Schema.encodeEffect(RelayClientEntryJson); + +export class RelayClientLedger extends Context.Service< + RelayClientLedger, + { + /** Records a started relay client; the returned effect forgets it. */ + readonly track: (client: { readonly pid: number }) => Effect.Effect>; + } +>()("t3/cloud/RelayClientLedger") {} + +const ENTRY_DIRECTORY = "relay-clients"; +const ENTRY_FILE = /^\d+\.json$/; +const STOP_POLL_INTERVAL = "50 millis"; +const STOP_POLL_ATTEMPTS = 40; + +interface ObservedProcess { + readonly pid: number; + readonly startTime: string; + readonly command: string; + readonly zombie: boolean; +} + +// `ps -o lstart` prints e.g. `Sat Sep 6 20:51:57 2026`; C locale and UTC keep it stable. +const DARWIN_PS_LINE = /^\s*(\d+)\s+(\S+)\s+(\w{3} \w{3} +\d{1,2} \d{2}:\d{2}:\d{2} \d{4})\s+(.*)$/; + +const processExists = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch (cause) { + return (cause as NodeJS.ErrnoException | undefined)?.code !== "ESRCH"; + } +}; + +const signal = (pid: number, name: NodeJS.Signals) => { + try { + process.kill(pid, name); + } catch (cause) { + if ((cause as NodeJS.ErrnoException | undefined)?.code !== "ESRCH") throw cause; + } +}; + +export const make = Effect.fn("RelayClientLedger.make")(function* (input: { + readonly stateDir: string; + readonly ownerPid?: number; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const platform = yield* HostProcess.Platform; + const directory = path.join(input.stateDir, ENTRY_DIRECTORY); + const recordable = platform === "linux" || platform === "darwin"; + + const observeLinux = Effect.fn("RelayClientLedger.observeLinux")(function* (pid: number) { + const stat = yield* fs.readFileString(`/proc/${pid}/stat`); + const fields = stat + .slice(stat.lastIndexOf(")") + 2) + .trim() + .split(/\s+/); + const startTime = fields[19]; + if (startTime === undefined) return undefined; + const cmdline = yield* fs.readFileString(`/proc/${pid}/cmdline`); + return { + pid, + startTime, + command: cmdline.replace(/\0$/, "").replaceAll("\0", " "), + zombie: fields[0] === "Z", + } satisfies ObservedProcess; + }); + + const observeDarwin = (pid: number) => + spawner + .string( + ChildProcess.make("/bin/ps", ["-ww", "-o", "pid=,stat=,lstart=,args=", "-p", String(pid)], { + env: { LC_ALL: "C", TZ: "UTC" }, + extendEnv: true, + stdin: "ignore", + stderr: "ignore", + }), + ) + .pipe( + Effect.timeout("5 seconds"), + Effect.map((output) => + output.split("\n").flatMap((line): ReadonlyArray => { + const match = DARWIN_PS_LINE.exec(line); + if (match === null || Number(match[1]) !== pid) return []; + return [ + { + pid, + zombie: match[2]!.startsWith("Z"), + startTime: match[3]!, + command: match[4]!.trimEnd(), + }, + ]; + }), + ), + Effect.map((found) => found[0]), + ); + + const observe = (pid: number): Effect.Effect => { + if (platform === "linux") return observeLinux(pid).pipe(Effect.orElseSucceed(() => undefined)); + if (platform === "darwin") { + return observeDarwin(pid).pipe(Effect.orElseSucceed(() => undefined)); + } + // @effect-diagnostics-next-line effectSucceedWithVoid:off - Missing process observations are undefined, not void. + return Effect.succeed(undefined); + }; + + const sameProcess = ( + observed: ObservedProcess | undefined, + identity: { readonly startTime: string; readonly command?: string }, + ) => + observed !== undefined && + !observed.zombie && + observed.startTime === identity.startTime && + (identity.command === undefined || observed.command === identity.command); + + const matchesRecordedProcess = Effect.fn("RelayClientLedger.matchesRecordedProcess")(function* ( + entry: RelayClientEntry, + ) { + const observed = yield* observe(entry.pid); + if (observed === undefined && processExists(entry.pid)) return undefined; + return sameProcess(observed, entry); + }); + + const ownerPid = input.ownerPid ?? process.pid; + const ownerProcess = yield* observe(ownerPid); + const owner = + ownerProcess === undefined ? undefined : { pid: ownerPid, startTime: ownerProcess.startTime }; + + const track: RelayClientLedger["Service"]["track"] = Effect.fn("RelayClientLedger.track")( + function* (client) { + if (!recordable) return Effect.void; + const observed = yield* observe(client.pid); + if (owner === undefined || observed === undefined || observed.zombie) { + yield* Effect.logWarning( + "Could not record the relay client; it will keep running if this server crashes", + { pid: client.pid }, + ); + return Effect.void; + } + const entryPath = path.join(directory, `${client.pid}.json`); + const entry: RelayClientEntry = { + version: 1, + pid: observed.pid, + startTime: observed.startTime, + command: observed.command, + stateDir: input.stateDir, + owner, + }; + yield* writeFileStringAtomically({ + filePath: entryPath, + contents: yield* encodeEntry(entry), + }).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + return matchesRecordedProcess(entry).pipe( + Effect.flatMap((matching) => + matching === false ? fs.remove(entryPath, { force: true }) : Effect.void, + ), + Effect.catchCause((cause) => + Effect.logWarning("Could not forget a stopped relay client", { cause }), + ), + ); + }, + Effect.catchCause((cause) => + Effect.logWarning("Could not record the relay client process", { cause }).pipe( + Effect.as(Effect.void), + ), + ), + ); + + const stopOrphan = Effect.fn("RelayClientLedger.stopOrphan")(function* (entry: RelayClientEntry) { + const matching = yield* matchesRecordedProcess(entry); + if (matching !== true) return matching === false; + yield* Effect.logInfo("Stopping a relay client left by a previous T3 Code server", { + pid: entry.pid, + }); + for (const name of ["SIGTERM", "SIGKILL"] as const) { + const matching = yield* matchesRecordedProcess(entry); + if (matching !== true) return matching === false; + signal(entry.pid, name); + for (let attempt = 0; attempt < STOP_POLL_ATTEMPTS; attempt++) { + const matching = yield* matchesRecordedProcess(entry); + if (matching !== true) return matching === false; + yield* Effect.sleep(STOP_POLL_INTERVAL); + } + } + return (yield* matchesRecordedProcess(entry)) === false; + }); + + const reapEntry = Effect.fn("RelayClientLedger.reapEntry")( + function* (entryPath: string) { + const entry = decodeEntry(yield* fs.readFileString(entryPath)); + if (Option.isSome(entry) && entry.value.stateDir === input.stateDir) { + const owner = yield* observe(entry.value.owner.pid); + if ( + sameProcess(owner, entry.value.owner) || + (owner === undefined && processExists(entry.value.owner.pid)) + ) + return; + if (!(yield* stopOrphan(entry.value))) return; + } + yield* fs.remove(entryPath, { force: true }); + }, + Effect.catchCause((cause) => + Effect.logWarning("Could not clean up a recorded relay client", { cause }), + ), + ); + + const reapOrphans = Effect.gen(function* () { + if (!recordable) return; + const names = yield* fs.readDirectory(directory).pipe(Effect.orElseSucceed(() => [])); + yield* Effect.forEach( + names.filter((name) => ENTRY_FILE.test(name)), + (name) => reapEntry(path.join(directory, name)), + { concurrency: "unbounded", discard: true }, + ); + }); + + return { track, reapOrphans }; +}); + +export const layer = (input: { readonly stateDir: string }) => + Layer.effect( + RelayClientLedger, + Effect.gen(function* () { + const ledger = yield* make(input); + yield* ledger.reapOrphans; + return RelayClientLedger.of({ track: ledger.track }); + }), + ); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index f461a99186b31..ca9cf4c650e9c 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -149,6 +149,7 @@ import * as ConnectReachability from "./cloud/ConnectReachability.ts"; import { pendingServiceUpdateExists } from "./cloud/updateHandoff.ts"; import * as RelayTracing from "./cloud/relayTracing.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; +import * as RelayClientLedger from "./cloud/RelayClientLedger.ts"; import { MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER, makeManagedTunnelRecovery, @@ -481,6 +482,14 @@ const layerCloudManagedEndpointRuntime = Layer.mergeAll( CloudManagedEndpointRuntime.layer.pipe( Layer.provide(ServerSecretStore.layer), Layer.provide(layerRelayClient), + Layer.provide( + Layer.unwrap( + Effect.gen(function* () { + const config = yield* ServerConfig.ServerConfig; + return RelayClientLedger.layer({ stateDir: config.stateDir }); + }), + ), + ), ), );