diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 523f495a5d6e9..ad926f2b5154d 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -1,3 +1,4 @@ +import { DESKTOP_UPDATE_RESTART_MARKER_FILE } from "@t3tools/contracts"; // @effect-diagnostics-next-line nodeBuiltinImport:off -- Cross-process launcher contention uses real filesystem leases and IPC barriers. import * as NodeChildProcess from "node:child_process"; import * as NodeSqlite from "node:sqlite"; @@ -204,11 +205,14 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( yield* fs.writeFileString(runtime.entryPath, "#!/bin/sh\n"); yield* fs.writeFileString(runtime.sentinelPath, "1.2.3\n"); + const markersAtStop: boolean[] = []; + const markersAtPreparation: boolean[] = []; const commands: string[] = []; const invocations: ProcessRunner.ProcessRunInput[] = []; const timeouts = new Map(); const control: { failCommand: string | undefined; + interruptCommand?: string; stateAfterStop?: string; /** `undefined` makes `loginctl show-user` fail, as when logind does not know the user. */ linger: string | undefined; @@ -240,6 +244,23 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( ) { const command = `${input.command} ${input.args.join(" ")}`; commands.push(command); + const marker = path.join(baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE); + if ( + command === "systemctl --user stop t3code.service" || + command === "systemctl --user disable --now t3code.service" || + command === "systemctl --user restart --no-block t3code.service" || + command.startsWith("launchctl bootout --wait ") || + (input.command === "schtasks" && input.args[0] === "/End") + ) { + markersAtStop.push(yield* fs.exists(marker)); + } + if ( + command === "systemctl --user daemon-reload" || + (input.command === "schtasks" && input.args[0] === "/Create") + ) { + markersAtPreparation.push(yield* fs.exists(marker)); + } + if (command === control.interruptCommand) return yield* Effect.interrupt; invocations.push(input); timeouts.set(command, input.timeout); if (command === "systemctl --user reset-failed t3code.service") control.startLimitHit = false; @@ -403,12 +424,51 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( commands, invocations, timeouts, + markersAtStop, + markersAtPreparation, control, runtime, }; }); it.layer(NodeServices.layer)("boot service install", (it) => { + it.effect( + "marks immediate and deferred service restarts without blocking on a marker error", + () => + Effect.gen(function* () { + const { service, fs, baseDir, control, markersAtStop, makeService } = yield* makeHarness(); + const path = yield* Path.Path; + const marker = path.join(baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE); + yield* service.install(); + expect(markersAtStop).toEqual([]); + expect(yield* fs.exists(marker)).toBe(false); + + const newer = yield* makeService(undefined, "1.2.4"); + yield* newer.install(); + expect(markersAtStop).toEqual([true]); + expect(yield* fs.exists(marker)).toBe(false); + + const next = yield* makeService(undefined, "1.2.5"); + yield* next.install({ start: false }); + expect(markersAtStop).toEqual([true]); + expect(yield* fs.exists(marker)).toBe(false); + yield* next.restart; + expect(markersAtStop).toEqual([true, true]); + expect(yield* fs.exists(marker)).toBe(false); + + control.failCommand = "systemctl --user stop t3code.service"; + const error = yield* next.restart.pipe(Effect.flip); + expect(error._tag).toBe("BootServiceCommandError"); + expect(markersAtStop).toEqual([true, true, true]); + expect(yield* fs.exists(marker)).toBe(false); + + control.failCommand = undefined; + yield* fs.makeDirectory(marker); + expect(yield* next.restart).toBe("restarted"); + expect((yield* fs.stat(marker)).type).toBe("Directory"); + }), + ); + it.effect( "fails before installing files or validating a runtime when lingering needs an administrator", () => @@ -1158,12 +1218,13 @@ process.disconnect(); it.effect("pauses only this home's running service for maintenance, even when it fails", () => Effect.gen(function* () { - const { service, commands, control, makeService } = yield* makeHarness(); + const { service, commands, control, makeService, markersAtStop } = yield* makeHarness(); yield* service.install(); const failure = yield* service .withStopped(Effect.failSync(() => (control.active ? "running" : "repair failed"))) .pipe(Effect.flip); expect(failure).toBe("repair failed"); + expect(markersAtStop).toEqual([false]); expect(control.active).toBe(true); control.failCommand = "systemctl --user stop t3code.service"; @@ -1299,9 +1360,19 @@ process.disconnect(); "queues an in-service restart without stopping the caller, even on handoff failure", () => Effect.gen(function* () { - const { service, fs, statePath, commands, control, makeService } = yield* makeHarness(); + const { + service, + fs, + statePath, + commands, + control, + makeService, + markersAtStop, + markersAtPreparation, + } = yield* makeHarness(); const plan = yield* service.install(); const state = yield* fs.readFileString(statePath); + const relayMarker = `${plan.baseDir}/runtime/${DESKTOP_UPDATE_RESTART_MARKER_FILE}`; const marker = `${plan.baseDir}/runtime/${SERVICE_RESTART_PENDING_FILE}`; yield* fs.writeFileString(marker, "1.2.3\n"); const inside = yield* makeService(undefined, "1.2.3", undefined, true); @@ -1312,6 +1383,23 @@ process.disconnect(); expect(error).toMatchObject({ _tag: "BootServiceCommandError", exitCode: 1 }); expect(commands).toEqual(["systemctl --user daemon-reload"]); expect(control.active).toBe(true); + expect(yield* fs.exists(relayMarker)).toBe(false); + expect(markersAtPreparation.at(-1)).toBe(false); + expect(markersAtStop).toEqual([]); + + const queueCommand = "systemctl --user restart --no-block t3code.service"; + control.failCommand = queueCommand; + expect((yield* inside.restart.pipe(Effect.flip))._tag).toBe("BootServiceCommandError"); + expect(markersAtStop).toEqual([true]); + expect(yield* fs.exists(relayMarker)).toBe(false); + + control.failCommand = undefined; + control.interruptCommand = queueCommand; + const interrupted = yield* inside.restart.pipe(Effect.forkScoped); + expect((yield* Fiber.await(interrupted))._tag).toBe("Failure"); + expect(markersAtStop).toEqual([true, true]); + expect(yield* fs.exists(relayMarker)).toBe(false); + delete control.interruptCommand; control.failCommand = undefined; commands.length = 0; @@ -1323,6 +1411,9 @@ process.disconnect(); "systemctl --user restart --no-block t3code.service", ]); expect(outcome).toBe("queued"); + expect(markersAtStop).toEqual([true, true, true]); + expect(markersAtPreparation).not.toContain(true); + expect(yield* fs.exists(relayMarker)).toBe(true); expect(yield* fs.readFileString(statePath)).toBe(state); expect(yield* fs.readFileString(marker)).toBe("1.2.3\n"); }), @@ -1406,7 +1497,8 @@ process.disconnect(); it.effect("installs, reports current state, and uninstalls on macOS", () => Effect.gen(function* () { - const { service, fs, statePath, commands, timeouts, runtime } = yield* makeHarness("darwin"); + const { service, fs, statePath, commands, timeouts, runtime, markersAtStop } = + yield* makeHarness("darwin"); const path = yield* Path.Path; const plan = yield* service.install(); @@ -1429,7 +1521,13 @@ process.disconnect(); current: true, installedVersion: "1.2.3", }); + const relayMarker = path.join(plan.baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE); + expect(yield* service.restart).toBe("restarted"); + expect(markersAtStop).toEqual([true]); + expect(yield* fs.exists(relayMarker)).toBe(false); expect(yield* service.uninstall).toBe(true); + expect(markersAtStop).toEqual([true, false]); + expect(yield* fs.exists(relayMarker)).toBe(false); expect((yield* service.status).installed).toBe(false); expect(commands.some((command) => command.startsWith("systemctl "))).toBe(false); // launchd can take 90s to bootout; the runner's 60s default could let bootstrap race it. @@ -1532,9 +1630,19 @@ process.disconnect(); it.effect("hands an in-service install and restart to Task Scheduler on Windows", () => Effect.gen(function* () { - const { service, fs, statePath, commands, makeService } = yield* makeHarness("win32"); + const { + service, + fs, + statePath, + commands, + control, + makeService, + markersAtStop, + markersAtPreparation, + } = yield* makeHarness("win32"); const plan = yield* service.install(); const inside = yield* makeService(undefined, "1.2.4", undefined, true); + const relayMarker = `${plan.baseDir}/runtime/${DESKTOP_UPDATE_RESTART_MARKER_FILE}`; const handoff = [ `schtasks /Create /XML ${plan.unitPath} /TN t3code-PC-theo /F`, "schtasks /End /TN t3code-PC-theo", @@ -1549,9 +1657,23 @@ process.disconnect(); yield* fs.readFileString(`${plan.baseDir}/runtime/${SERVICE_RESTART_PENDING_FILE}`), ).toBe("1.2.4\n"); + expect(markersAtStop).toEqual([true]); + expect(yield* fs.exists(relayMarker)).toBe(true); + yield* fs.remove(relayMarker); + commands.length = 0; expect(yield* inside.restart).toBe("queued"); expect(commands).toEqual(handoff); + expect(markersAtStop).toEqual([true, true]); + expect(yield* fs.exists(relayMarker)).toBe(true); + yield* fs.remove(relayMarker); + + expect(markersAtPreparation).not.toContain(true); + control.failCommand = handoff[0]; + expect((yield* inside.restart.pipe(Effect.flip))._tag).toBe("BootServiceCommandError"); + expect(yield* fs.exists(relayMarker)).toBe(false); + expect(markersAtPreparation.at(-1)).toBe(false); + expect(markersAtStop).toEqual([true, true]); }), ); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index d624e02c7ecfb..4f81ec8641931 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -1,3 +1,4 @@ +import { DESKTOP_UPDATE_RESTART_MARKER_FILE } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Config from "effect/Config"; import * as Context from "effect/Context"; @@ -285,6 +286,11 @@ export type BootServiceStep = { /** Must exceed systemd TimeoutStopSec and launchd ExitTimeOut (both 90s) to avoid shutdown races. */ const STOP_STEP_TIMEOUT = Duration.seconds(120); +interface BootServiceHandoff { + readonly prepare: ReadonlyArray; + readonly restart: ReadonlyArray; +} + export interface BootServiceManager { readonly kind: "systemd" | "launchd" | "schtasks"; readonly unitPath: string; @@ -293,7 +299,7 @@ export interface BootServiceManager { readonly activate: ReadonlyArray; readonly restart: ReadonlyArray; /** Queue a restart owned by the manager, since stopping would kill the caller. */ - readonly handoff?: ReadonlyArray; + readonly handoff?: BootServiceHandoff; readonly deactivate: ReadonlyArray; readonly finalize: ReadonlyArray; } @@ -364,14 +370,16 @@ function systemdManager(input: { args: ["--user", "restart", BOOT_SERVICE_UNIT_FILE], }, ], - handoff: [ - ...prepareActivation, - { - step: "restarting the service", - command: "systemctl", - args: ["--user", "restart", "--no-block", BOOT_SERVICE_UNIT_FILE], - }, - ], + handoff: { + prepare: prepareActivation, + restart: [ + { + step: "restarting the service", + command: "systemctl", + args: ["--user", "restart", "--no-block", BOOT_SERVICE_UNIT_FILE], + }, + ], + }, deactivate: [ { step: "stopping the service", @@ -514,16 +522,18 @@ function schtasksManager(input: { ], // Ending the task kills only its console host; the launcher sees the console close and // stops gracefully while the replacement waits for its lock. - handoff: [ - register, - { - step: "ending the scheduled task", - command: "schtasks", - args: ["/End", ...task], - optional: true, - }, - { ...run, step: "restarting the service" }, - ], + handoff: { + prepare: [register], + restart: [ + { + step: "ending the scheduled task", + command: "schtasks", + args: ["/End", ...task], + optional: true, + }, + { ...run, step: "restarting the service" }, + ], + }, deactivate: [...stop, { step: "removing the scheduled task", run: input.steps.remove }], finalize: [], }; @@ -1211,6 +1221,29 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { return problems; }); + // Reuse the desktop marker so already released servers recognize the restart. + const restartMarker = path.join(input.baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE); + const markRestart = fs.writeFileString(restartMarker, "", { mode: 0o600 }).pipe( + Effect.as(true), + Effect.catch((error) => + Effect.logWarning("Could not mark the service restart to preserve its relay tunnel", { + errorTag: error._tag, + }).pipe(Effect.as(false)), + ), + ); + const stopForRestart = (steps: ReadonlyArray, queued = false) => + Effect.acquireUseRelease( + markRestart, + () => runSteps(steps), + (marked, exit) => + // When a restart is queued, the outgoing server still needs to read the marker. + marked && (!queued || Exit.isFailure(exit)) + ? fs.remove(restartMarker, { force: true }).pipe(Effect.ignore) + : Effect.void, + ); + const handOffRestart = (handoff: BootServiceHandoff) => + runSteps(handoff.prepare).pipe(Effect.andThen(stopForRestart(handoff.restart, true))); + const install = Effect.fn("cloud.boot_service.install")(function* (options?: { readonly allowDowngrade?: boolean; readonly start?: boolean; @@ -1334,9 +1367,12 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { } // Task Scheduler restarts the task itself, so the files are written as for a deferred // restart while this launcher keeps running, then the restart is handed to the manager. - const handOff = installed && start && runningInsideService && manager.kind === "schtasks"; - if (installed && start && !handOff) { - yield* runSteps(manager.stop); + const handoff = + installed && start && runningInsideService && manager.kind === "schtasks" + ? manager.handoff + : undefined; + if (installed && start && handoff === undefined) { + yield* stopForRestart(manager.stop); } // State first, then unit. The launcher acquires only one at a time, and a @@ -1396,7 +1432,10 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { 2, )}\n`, ); - if ((!start || handOff) && (currentlyInstalled || Option.isSome(previousStateText))) { + if ( + (!start || handoff !== undefined) && + (currentlyInstalled || Option.isSome(previousStateText)) + ) { const written = yield* fs.readFileString(statePath); if (serviceStateActiveVersion(written) !== input.cliVersion) { return yield* new BootServiceUpdatePendingError(); @@ -1414,8 +1453,8 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { ).pipe( Effect.tap( Effect.gen(function* () { - if (handOff) { - yield* runSteps(manager.handoff ?? []); + if (handoff !== undefined) { + yield* handOffRestart(handoff); } else if (start) { yield* runSteps(manager.activate); yield* withSelectionLock( @@ -1435,12 +1474,12 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { cause._tag === "PlatformError" ? new BootServiceInstallError({ cause }) : cause, ), Effect.tapError(() => - installed && start && !handOff + installed && start && handoff === undefined ? runSteps(manager.restart).pipe(Effect.ignore) : Effect.void, ), ); - return handOff ? { ...installedPlan, handedOff: true } : installedPlan; + return handoff !== undefined ? { ...installedPlan, handedOff: true } : installedPlan; }); const restart: BootService["Service"]["restart"] = Effect.gen(function* () { @@ -1457,10 +1496,10 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { // Writes no state, and the launcher lets an update transition in flight // finish before the queued stop tears it down. if (runningInsideService && manager.handoff !== undefined) { - yield* runSteps(manager.handoff); + yield* handOffRestart(manager.handoff); return "queued" as const; } - yield* runSteps(manager.stop); + yield* stopForRestart(manager.stop); yield* runSteps(manager.activate).pipe( Effect.tapError(() => runSteps(manager.restart).pipe(Effect.ignore)), );