diff --git a/apps/server/src/cli/config.test.ts b/apps/server/src/cli/config.test.ts index ddbc49e1ced74..75d6454fef44f 100644 --- a/apps/server/src/cli/config.test.ts +++ b/apps/server/src/cli/config.test.ts @@ -128,11 +128,13 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-cli-preflight-" }); const child = yield* liveProcess; assert(child.pid !== undefined); - for (const [name, pid, mode, rejectRunningServer] of [ - ["stale", 2_147_483_647, "web", true], - ["own-pid", process.pid, "web", true], - ["desktop", child.pid, "desktop", true], - ["serve", child.pid, "web", false], + for (const [name, pid, mode, rejectRunningServer, hostname] of [ + ["stale", 2_147_483_647, "web", true, undefined], + ["own-pid", process.pid, "web", true, undefined], + // A recreated container or a machine sharing the home: the ownership lock decides. + ["foreign-host", child.pid, "web", true, `${NodeOS.hostname()}-peer`], + ["desktop", child.pid, "desktop", true, undefined], + ["serve", child.pid, "web", false, undefined], ] as const) { const baseDir = path.join(root, name); const stateDir = path.join(baseDir, "userdata"); @@ -142,6 +144,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => { yield* encodeUnknownJson({ version: 1, pid, + hostname, port: 3773, origin: "http://127.0.0.1:3773", startedAt: "2026-10-01T00:00:00.000Z", diff --git a/apps/server/src/cli/config.ts b/apps/server/src/cli/config.ts index 25d6c6eb62dd6..f2516488f6bc5 100644 --- a/apps/server/src/cli/config.ts +++ b/apps/server/src/cli/config.ts @@ -26,7 +26,11 @@ import { readBootstrapEnvelope } from "../bootstrap.ts"; import { resolveServiceLauncherMode } from "../cloud/serviceLauncherClient.ts"; import * as ServerConfig from "../config.ts"; import { expandHomePath, resolveBaseDir } from "../os-jank.ts"; -import { isProcessAlive, readPersistedServerRuntimeState } from "../serverRuntimeState.ts"; +import { + isProcessAlive, + isRuntimeStateFromAnotherMachine, + readPersistedServerRuntimeState, +} from "../serverRuntimeState.ts"; import { resolveHostCliCommand } from "./invocation.ts"; const modeFlag = Flag.Literals("mode", ServerConfig.RuntimeMode.literals).pipe( @@ -349,9 +353,11 @@ export const resolveServerConfig = Effect.fn("config.resolveServerConfig")(funct // and supervisor handoff are separate; this preflight cannot arbitrate two starts. if (options?.rejectRunningServer && mode === "web") { const runtime = yield* readPersistedServerRuntimeState(derivedPaths.serverRuntimeStatePath); - // A killed container can leave a record with the new server's own PID. + // A killed container can leave a record with the new server's own PID. Another + // host's PID means nothing here, so its record is left to the ownership lock. if ( Option.isSome(runtime) && + !isRuntimeStateFromAnotherMachine(runtime.value) && runtime.value.pid > 0 && runtime.value.pid !== process.pid && isProcessAlive(runtime.value.pid) diff --git a/apps/server/src/cli/pair.test.ts b/apps/server/src/cli/pair.test.ts index baafbca9817ac..4685f2623343b 100644 --- a/apps/server/src/cli/pair.test.ts +++ b/apps/server/src/cli/pair.test.ts @@ -432,6 +432,39 @@ describe("t3 pair", () => { ).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); + // A shared $HOME makes another machine's record visible here. The server that + // owns the home still pairs, since a hostname can change while it runs. + it.effect("names the other machine unless the server that owns the home answers", () => + withDescriptorServer((origin) => + Effect.gen(function* () { + const baseDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-pair-foreign-test-")); + const statePath = NodePath.join(baseDir, "userdata", "server-runtime.json"); + const state = yield* makePersistedServerRuntimeState({ + config: { host: "127.0.0.1", devUrl: undefined }, + port: Number(new URL(origin).port), + }); + const hostname = `${NodeOS.hostname()}-peer`; + + yield* persistServerRuntimeState({ path: statePath, state: { ...state, hostname } }); + const environmentIdPath = NodePath.join(baseDir, "userdata", "environment-id"); + NodeFS.writeFileSync(environmentIdPath, `${testDescriptor.environmentId}\n`); + const output = yield* captureStdout(runCli(["pair", "--base-dir", baseDir])); + assert.include(output, `Pairing URL: ${origin}/pair#token=`); + + // The recorded PID and port answer here, but for a server of another home. + NodeFS.writeFileSync(environmentIdPath, "another-home-environment\n"); + const error = yield* provideCliTestLayers( + runCli(["pair", "--base-dir", baseDir]).pipe(Effect.flip), + ); + const rendered = String( + typeof error === "object" && error !== null && "cause" in error ? error.cause : error, + ); + assert.include(rendered, "running on another machine"); + assert.include(rendered, hostname); + }), + ).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); + // regression: #10516 minting opens the target database with this checkout's // migrations, so a worktree must never fall back to the live install. it.effect("does not fall back to the shared home from a linked worktree", () => diff --git a/apps/server/src/cli/pair.ts b/apps/server/src/cli/pair.ts index 2d45d45befeec..9ddc90ac1142c 100644 --- a/apps/server/src/cli/pair.ts +++ b/apps/server/src/cli/pair.ts @@ -23,6 +23,7 @@ import * as Console from "effect/Console"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as References from "effect/References"; @@ -38,6 +39,7 @@ import { resolveBaseDir } from "../os-jank.ts"; import { type PersistedServerRuntimeState, isProcessAlive, + isRuntimeStateFromAnotherMachine, readPersistedServerRuntimeState, } from "../serverRuntimeState.ts"; import { @@ -82,6 +84,22 @@ export class NoRunningServerError extends Schema.TaggedError()( + "ServerRunningOnAnotherMachineError", + { + hostname: Schema.String, + statePath: Schema.String, + }, +) { + override get message(): string { + return [ + `The T3 Code server for this T3 home is running on another machine (\`${this.hostname}\`).`, + `Its record is ${this.statePath}, visible here because this machine shares that T3 home.`, + `Run \`t3 pair\` on \`${this.hostname}\` to pair with it. To run a separate server on this machine, give it its own --base-dir.`, + ].join("\n"); + } +} + export class IncompatibleServerProtocolError extends Schema.TaggedError()( "IncompatibleServerProtocolError", { @@ -292,7 +310,9 @@ const discoverPairTarget = Effect.fn("pair.discoverPairTarget")(function* ( } } + const fs = yield* FileSystem.FileSystem; const checkedStatePaths: Array = []; + let foreignServer: { readonly hostname: string; readonly statePath: string } | undefined; for (const baseDir of new Set(bases)) { for (const variant of ["userdata", "dev"] as const) { const derivedPaths = yield* ServerConfig.deriveServerPaths( @@ -307,21 +327,38 @@ const discoverPairTarget = Effect.fn("pair.discoverPairTarget")(function* ( continue; } // A dead server's port may be reused; do not mint tokens in its old database for another server. - if (!isProcessAlive(state.value.pid)) { - continue; - } - const probed = yield* probeEnvironmentDescriptor(state.value.origin); - if (probed._tag !== "descriptor") { + const probed = isProcessAlive(state.value.pid) + ? yield* probeEnvironmentDescriptor(state.value.origin) + : undefined; + const foreign = isRuntimeStateFromAnotherMachine(state.value); + // Hostnames change under running servers (DHCP), so a foreign record still pairs, but only + // with the server that owns this home: its recorded PID and port can be reused here. + const descriptor = + probed?._tag === "descriptor" && + (!foreign || + (yield* fs.readFileString(derivedPaths.environmentIdPath).pipe( + Effect.map((id) => id.trim()), + Effect.orElseSucceed(() => undefined), + )) === probed.descriptor.environmentId) + ? probed.descriptor + : undefined; + if (descriptor === undefined) { + if (foreign) { + foreignServer ??= { hostname: state.value.hostname ?? "unknown", statePath }; + } continue; } return { baseDir, variant, state: state.value, - descriptor: probed.descriptor, + descriptor, } satisfies DiscoveredPairTarget; } } + if (foreignServer !== undefined) { + return yield* new ServerRunningOnAnotherMachineError(foreignServer); + } const baseDirArgs = hasExplicitBaseDir ? ["--base-dir", explicitBaseDir] : []; return yield* new NoRunningServerError({ checkedStatePaths, diff --git a/apps/server/src/cli/service.test.ts b/apps/server/src/cli/service.test.ts index ada548ba2bf39..459fa408343fd 100644 --- a/apps/server/src/cli/service.test.ts +++ b/apps/server/src/cli/service.test.ts @@ -76,8 +76,12 @@ it.layer(Layer.mergeAll(NodeServices.layer, NetService.layer))("service commands { command, label: "another home's service", - status: { ...newerServiceStatus, otherBaseDir: "/srv/t3" }, - error: { _tag: "BootServiceOtherBaseDirError", installedBaseDir: "/srv/t3" }, + status: { ...newerServiceStatus, otherBaseDir: "/srv/t3", otherBaseDirHostname: "peer" }, + error: { + _tag: "BootServiceOtherBaseDirError", + installedBaseDir: "/srv/t3", + hostname: "peer", + }, }, ]), )( diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index fc2aa8d0889aa..3b6d5fb9eca01 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -120,6 +120,9 @@ export const reconcileService = Effect.fn("cli.service.reconcile")(function* (op return yield* new BootService.BootServiceOtherBaseDirError({ installedBaseDir: status.otherBaseDir, baseDir, + ...(status.otherBaseDirHostname === undefined + ? {} + : { hostname: status.otherBaseDirHostname }), }); } if (status.installed && status.current) { @@ -189,11 +192,15 @@ function formatServiceStatus( return `T3 Code service\n Status: not installed\n Next: Run \`${commands.install}\`.`; } if (status.otherBaseDir !== undefined) { + const manage = + status.otherBaseDirHostname === undefined + ? `Pass \`${BootService.formatBootServiceBaseDirOption(status.otherBaseDir)}\` to manage it.` + : `Manage it on \`${status.otherBaseDirHostname}\` with \`${BootService.formatBootServiceBaseDirOption(status.otherBaseDir)}\`.`; return [ "T3 Code service", " Status: installed for another T3 home", ` Base dir: ${status.otherBaseDir}`, - ` Next: Pass \`${BootService.formatBootServiceBaseDirOption(status.otherBaseDir)}\` to manage it.`, + ` Next: ${manage}`, ].join("\n"); } const installedVersion = status.installedVersion ?? cliVersion; diff --git a/apps/server/src/cli/up.test.ts b/apps/server/src/cli/up.test.ts index 9a38208af9829..5fc84f63886af 100644 --- a/apps/server/src/cli/up.test.ts +++ b/apps/server/src/cli/up.test.ts @@ -261,7 +261,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, NetService.layer)) { label: "a service for another home is refused without a restart", installed: true, - status: { current: false, otherBaseDir: "/srv/t3 home" }, + status: { current: false, otherBaseDir: "/srv/t3 home", otherBaseDirHostname: "peer" }, fails: true, answer: "y", env: "T3CODE_LAN=false\n", @@ -273,10 +273,10 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, NetService.layer)) { label: "another home's service supplies no local network default", installed: true, - status: { current: false, otherBaseDir: "/srv/t3 home" }, + status: { current: false, otherBaseDir: "/srv/t3 home", otherBaseDirHostname: "peer" }, reportStatus: true, json: { - service: { otherBaseDir: "/srv/t3 home" }, + service: { otherBaseDir: "/srv/t3 home", otherBaseDirHostname: "peer" }, lan: { enabled: false, inheritedDefault: false }, }, env: "", @@ -285,6 +285,18 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, NetService.layer)) restarts: 0, envAfter: "", }, + { + label: "status names the machine running another home's service", + installed: true, + status: { current: false, otherBaseDir: "/srv/t3 home", otherBaseDirHostname: "peer" }, + reportStatus: true, + prints: ["installed for another T3 home · /srv/t3 home on `peer`", "Manage it on `peer`"], + env: "", + args: [], + installs: 0, + restarts: 0, + envAfter: "", + }, { label: "status hints keep the caller's --base-dir", installed: true, @@ -362,6 +374,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, NetService.layer)) new BootService.BootServiceOtherBaseDirError({ installedBaseDir: status.otherBaseDir, baseDir, + ...(status.otherBaseDirHostname === undefined + ? {} + : { hostname: status.otherBaseDirHostname }), }), ) : input.installFails === true diff --git a/apps/server/src/cli/up.ts b/apps/server/src/cli/up.ts index a56e478c7f45c..f11e6fe8ddd40 100644 --- a/apps/server/src/cli/up.ts +++ b/apps/server/src/cli/up.ts @@ -147,10 +147,16 @@ const formatSetup = (setup: Setup, hints: Hints) => { const { service } = setup; // This home's access settings reach no service, so only the service is shown. if (service.otherBaseDir !== undefined) { + const machine = + service.otherBaseDirHostname === undefined ? "" : ` on \`${service.otherBaseDirHostname}\``; + const manage = + service.otherBaseDirHostname === undefined + ? `Pass \`${BootService.formatBootServiceBaseDirOption(service.otherBaseDir)}\` to manage it.` + : `Manage it on \`${service.otherBaseDirHostname}\` with \`${BootService.formatBootServiceBaseDirOption(service.otherBaseDir)}\`.`; return [ - ` ${"Service".padEnd(15)}installed for another T3 home · ${service.otherBaseDir}`, + ` ${"Service".padEnd(15)}installed for another T3 home · ${service.otherBaseDir}${machine}`, "", - `Pass \`${BootService.formatBootServiceBaseDirOption(service.otherBaseDir)}\` to manage it.`, + manage, ].join("\n"); } const rows: ReadonlyArray = [ @@ -236,6 +242,9 @@ const setUp = Effect.fn("cli.up")(function* (flags: { return yield* new BootService.BootServiceOtherBaseDirError({ installedBaseDir: before.service.otherBaseDir, baseDir, + ...(before.service.otherBaseDirHostname === undefined + ? {} + : { hostname: before.service.otherBaseDirHostname }), }); } if (before.connect === null && Option.getOrElse(flags.connect, () => false)) { @@ -516,6 +525,9 @@ const setupJson = (setup: Setup) => installed: setup.service.installed, installedVersion: setup.service.installedVersion ?? null, otherBaseDir: setup.service.otherBaseDir ?? null, + ...(setup.service.otherBaseDirHostname === undefined + ? {} + : { otherBaseDirHostname: setup.service.otherBaseDirHostname }), }, connect: setup.connect === null ? null : { enabled: setup.connect, revoked: setup.connectRevoked }, diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 523f495a5d6e9..0105c450c90b2 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -1226,15 +1226,32 @@ process.disconnect(); const otherHome = yield* fs.makeTempDirectoryScoped({ prefix: "t3-other-home-" }); const other = yield* makeService(undefined, "1.2.3", path.join(otherHome, ".t3")); + // A record from another machine sharing this home names that machine. + yield* fs.makeDirectory(path.join(baseDir, "userdata"), { recursive: true }); + yield* fs.writeFileString( + path.join(baseDir, "userdata", "server-runtime.json"), + JSON.stringify({ + version: 1, + pid: 1234, + hostname: "another-machine", + port: 3773, + origin: "http://127.0.0.1:3773", + startedAt: "2026-10-01T00:00:00.000Z", + }), + ); commands.length = 0; expect(yield* other.restart).toBe("skipped"); const refused = yield* other.install().pipe(Effect.flip); expect(refused._tag).toBe("BootServiceOtherBaseDirError"); + expect(refused).toMatchObject({ hostname: "another-machine" }); + expect(refused.message).toContain("`another-machine`"); + expect(refused.message).not.toContain("uninstall"); // The CLI prints a user message without a stack trace. expect(CliError.getUserMessage(refused)).toBe(refused.message); expect(yield* fs.readFileString(plan.unitPath)).toBe(unit); expect(commands).toEqual([]); expect((yield* other.status).otherBaseDir).toBe(baseDir); + expect((yield* other.status).otherBaseDirHostname).toBe("another-machine"); const quoted = yield* makeService(undefined, "1.2.3", path.join(otherHome, 'T3 "Data"')); commands.length = 0; diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index d624e02c7ecfb..a557fab99631e 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -17,6 +17,11 @@ import * as Schema from "effect/Schema"; import { CLI_RELEASE_BASE_URL_ENV } from "@t3tools/shared/cliRelease"; import * as ProcessRunner from "../processRunner.ts"; +import * as ServerConfig from "../config.ts"; +import { + isRuntimeStateFromAnotherMachine, + readPersistedServerRuntimeState, +} from "../serverRuntimeState.ts"; import { acquireServiceSelectionLock } from "./serviceSelectionLock.ts"; import { serviceLauncherIsRunning, @@ -718,10 +723,15 @@ export class BootServiceOtherBaseDirError extends Schema.TaggedError; readonly unitPath: string; readonly logPath: string; @@ -1017,6 +1029,21 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { ? installedBaseDir : undefined; }; + // The runtime record names the machine that wrote it; a foreign record means + // the installed home's service runs on another machine sharing this home. + // `status` and `install` expose no Effect context, so the services `make` + // already captured are provided here instead of taken from the environment. + const foreignHostnameFor = (dir: string) => + ServerConfig.deriveServerPaths(dir, undefined, {}).pipe( + Effect.flatMap((paths) => readPersistedServerRuntimeState(paths.serverRuntimeStatePath)), + Effect.map((state) => + Option.isSome(state) && isRuntimeStateFromAnotherMachine(state.value) + ? state.value.hostname + : undefined, + ), + Effect.provideService(Path.Path, path), + Effect.provideService(FileSystem.FileSystem, fs), + ); const writeDurably = (filePath: string, contents: string | Uint8Array) => Effect.scoped( Effect.gen(function* () { @@ -1227,9 +1254,11 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { ? otherBaseDirOf(existingUnit.value) : undefined; if (installedBaseDir !== undefined) { + const hostname = yield* foreignHostnameFor(installedBaseDir); return yield* new BootServiceOtherBaseDirError({ installedBaseDir, baseDir: input.baseDir, + ...(hostname === undefined ? {} : { hostname }), }); } }); @@ -1565,6 +1594,8 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { : undefined; const installedBaseDir = bootServiceBaseDirOf(unit); const otherBaseDir = otherBaseDirOf(unit); + const otherBaseDirHostname = + otherBaseDir === undefined ? undefined : yield* foreignHostnameFor(otherBaseDir); const normalizeUnit = (contents: string) => manager.kind === "launchd" ? contents.replace(/(PATH<\/key>\n\s*)[^<]*(<\/string>)/, "$1$2") @@ -1582,6 +1613,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { ...(installedVersion === undefined ? {} : { installedVersion }), ...(installedBaseDir === undefined ? {} : { installedBaseDir }), ...(otherBaseDir === undefined ? {} : { otherBaseDir }), + ...(otherBaseDirHostname === undefined ? {} : { otherBaseDirHostname }), problems, current: problems.length === 0 && diff --git a/apps/server/src/serverOwnership.ts b/apps/server/src/serverOwnership.ts index 193233fa8abeb..2a36a36650212 100644 --- a/apps/server/src/serverOwnership.ts +++ b/apps/server/src/serverOwnership.ts @@ -1,5 +1,6 @@ // @effect-diagnostics nodeBuiltinImport:off - The ownership lock is a native SQLite handle held for the server's lifetime. import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import * as NodeSqlite from "node:sqlite"; @@ -35,6 +36,7 @@ import * as ProcessRunner from "./processRunner.ts"; import { clearPersistedServerRuntimeState, isProcessAlive, + isRuntimeStateFromAnotherMachine, persistServerRuntimeState, readPersistedServerRuntimeState, type PersistedServerRuntimeState, @@ -42,24 +44,35 @@ import { export class ServerAlreadyRunningError extends Schema.TaggedError()( "ServerAlreadyRunningError", - { stateDir: Schema.String }, + { + stateDir: Schema.String, + /** The machine recorded as owning the home, when it is another machine. */ + hostname: Schema.optional(Schema.String), + }, ) { // Distinct exit codes let supervisors stop restarting an already-owned server. override readonly [Runtime.errorExitCode] = SERVER_EXIT_CODE_STATE_DIR_OWNED; override get message(): string { - return `A T3 Code server already owns ${this.stateDir}. Stop that server, or use a separate T3 home and pair with it, then retry. No server was stopped. server-owner.sqlite is the ownership lock; do not delete it while a server is running.`; + return this.hostname === undefined + ? `A T3 Code server already owns ${this.stateDir}. Stop that server, or use a separate T3 home and pair with it, then retry. No server was stopped. server-owner.sqlite is the ownership lock; do not delete it while a server is running.` + : `A T3 Code server on \`${this.hostname}\` already owns ${this.stateDir}. Stop that server on \`${this.hostname}\`, or run \`t3 pair\` on \`${this.hostname}\` to use it, or use a separate T3 home, then retry. No server was stopped. server-owner.sqlite is the ownership lock; do not delete it while a server is running.`; } } export class OlderBootServiceError extends Schema.TaggedError()( "OlderBootServiceError", - { baseDir: Schema.String }, + { + baseDir: Schema.String, + /** The machine recorded as owning the home, when it is another machine. */ + hostname: Schema.optional(Schema.String), + }, ) { override readonly [Runtime.errorExitCode] = SERVER_EXIT_CODE_STATE_DIR_OWNED; override get message(): string { - return `A T3 Code background service set up for ${this.baseDir} may still run a release without the ownership lock. On that machine, try \`t3 update\` and let it restart (or run \`t3 service restart\` if the update is already installed). If the older launcher cannot update, run \`t3 service uninstall\`, then reinstall with \`t3 service install\` from a current CLI. You can also leave the service uninstalled. Pass \`--base-dir ${this.baseDir}\` to each command, and use \`npx t3@latest\` if \`t3\` is not installed or is outdated. Then retry. No server was stopped.`; + const machine = this.hostname === undefined ? "that machine" : `\`${this.hostname}\``; + return `A T3 Code background service set up for ${this.baseDir} may still run a release without the ownership lock. On ${machine}, try \`t3 update\` and let it restart (or run \`t3 service restart\` if the update is already installed). If the older launcher cannot update, run \`t3 service uninstall\`, then reinstall with \`t3 service install\` from a current CLI. You can also leave the service uninstalled. Pass \`--base-dir ${this.baseDir}\` to each command, and use \`npx t3@latest\` if \`t3\` is not installed or is outdated. Then retry. No server was stopped.`; } } @@ -159,18 +172,42 @@ const olderBootServiceHome = Effect.fn("olderBootServiceHome")( } } // A deferred update names the staged runtime, not the loaded service's runtime. - if (loaded && (yield* fs.exists(NodePath.join(runtimeDir, SERVICE_RESTART_PENDING_FILE)))) { - return baseDir; - } + const restartPending = + loaded && (yield* fs.exists(NodePath.join(runtimeDir, SERVICE_RESTART_PENDING_FILE))); // Stable 0.1.0 shipped the ownership lock. Earlier lock-aware prereleases // are conservatively refused because older runtimes expose no capability marker. - return compareExactServiceVersions(version, "0.1.0") < 0 ? baseDir : undefined; + if (!restartPending && compareExactServiceVersions(version, "0.1.0") >= 0) return undefined; + // The unit file is shared with the home, so the older service may run on + // another machine; its runtime record names that machine. + const record = yield* readPersistedServerRuntimeState( + NodePath.join(stateDir, "server-runtime.json"), + ); + const hostname = + Option.isSome(record) && isRuntimeStateFromAnotherMachine(record.value) + ? record.value.hostname + : undefined; + return { baseDir, ...(hostname === undefined ? {} : { hostname }) }; }, Effect.orElseSucceed(() => undefined), ); const ownershipLockPath = (stateDir: string) => NodePath.join(stateDir, "server-owner.sqlite"); +// The runtime record names the machine that wrote it; a foreign record means +// the lock holder runs on another machine sharing this T3 home. +const foreignOwnerHostname = (stateDir: string): string | undefined => { + try { + const record = JSON.parse( + NodeFS.readFileSync(NodePath.join(stateDir, "server-runtime.json"), "utf8"), + ) as unknown; + if (typeof record !== "object" || record === null) return undefined; + const hostname = "hostname" in record ? record.hostname : undefined; + return typeof hostname === "string" && hostname !== NodeOS.hostname() ? hostname : undefined; + } catch { + return undefined; + } +}; + /** * The lock is an exclusive transaction on `server-owner.sqlite`, which never * holds data and must never be unlinked: the OS releases it when the process @@ -201,10 +238,14 @@ const openOwnershipLock = (stateDir: string) => throw cause; } }, - catch: (cause) => - isSqliteBusy(cause) - ? new ServerAlreadyRunningError({ stateDir }) - : new ServerOwnershipError({ stateDir, cause }), + catch: (cause) => { + if (!isSqliteBusy(cause)) return new ServerOwnershipError({ stateDir, cause }); + const hostname = foreignOwnerHostname(stateDir); + return new ServerAlreadyRunningError({ + stateDir, + ...(hostname === undefined ? {} : { hostname }), + }); + }, }); /** Hold only the ownership lock until the scope closes, without claiming the runtime state. */ @@ -246,11 +287,17 @@ export const acquireServerOwnership = Effect.fn("acquireServerOwnership")(functi previous.value.ownerId === undefined && (yield* legacyOwnerIsLive(previous.value)) ) { - return yield* new ServerAlreadyRunningError({ stateDir }); + const hostname = isRuntimeStateFromAnotherMachine(previous.value) + ? previous.value.hostname + : undefined; + return yield* new ServerAlreadyRunningError({ + stateDir, + ...(hostname === undefined ? {} : { hostname }), + }); } const olderServiceHome = yield* olderBootServiceHome(stateDir); if (olderServiceHome !== undefined) { - return yield* new OlderBootServiceError({ baseDir: olderServiceHome }); + return yield* new OlderBootServiceError(olderServiceHome); } yield* Effect.gen(function* () { diff --git a/apps/server/src/serverRuntimeState.test.ts b/apps/server/src/serverRuntimeState.test.ts index 5a14cf139577e..b985a504020ed 100644 --- a/apps/server/src/serverRuntimeState.test.ts +++ b/apps/server/src/serverRuntimeState.test.ts @@ -1,5 +1,6 @@ // @effect-diagnostics nodeBuiltinImport:off - Injects a real file replacement during the native SQLite transaction. import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; import * as NodeSqlite from "node:sqlite"; import { assert, describe, it } from "@effect/vitest"; @@ -34,6 +35,8 @@ const startBesideBootService = Effect.fn("test.startBesideBootService")(function readonly missingServiceState?: boolean; readonly environment?: NodeJS.ProcessEnv; readonly connected?: boolean; + /** The machine a runtime record in this home names, when it is another machine. */ + readonly runtimeHostname?: string; }) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -43,6 +46,17 @@ const startBesideBootService = Effect.fn("test.startBesideBootService")(function for (const home of [t3Home, serviceHome]) { yield* fs.makeDirectory(path.join(home, "userdata"), { recursive: true }); } + if (input.runtimeHostname !== undefined) { + const state = yield* ServerRuntimeState.makePersistedServerRuntimeState({ + config: { host: undefined, devUrl: undefined }, + port: 3773, + }); + yield* ServerRuntimeState.persistServerRuntimeState({ + path: path.join(t3Home, "userdata", "server-runtime.json"), + // The recorded PID means nothing on this machine; only the hostname matters here. + state: { ...state, hostname: input.runtimeHostname, pid: 0 }, + }); + } yield* fs.makeDirectory(path.join(serviceHome, "runtime")); const activeVersion = input.activeVersion ?? "0.0.42"; if (!input.missingServiceState) { @@ -135,6 +149,29 @@ const startBesideBootService = Effect.fn("test.startBesideBootService")(function return { result, t3Home }; }); +describe("server runtime state machine identity", () => { + it.effect("stamps the writing machine and recognises records from other machines", () => + Effect.gen(function* () { + const state = yield* ServerRuntimeState.makePersistedServerRuntimeState({ + config: { host: undefined, devUrl: undefined }, + port: 3773, + }); + assert.equal(state.hostname, NodeOS.hostname()); + assert.isFalse(ServerRuntimeState.isRuntimeStateFromAnotherMachine(state)); + assert.isTrue( + ServerRuntimeState.isRuntimeStateFromAnotherMachine({ + ...state, + hostname: `${NodeOS.hostname()}-peer`, + }), + ); + // Records written before the field existed carry no hostname and stay local. + assert.isFalse( + ServerRuntimeState.isRuntimeStateFromAnotherMachine({ ...state, hostname: undefined }), + ); + }), + ); +}); + describe("server ownership", () => { // An empty home, so no boot service unit installed on this machine is read. const layerOwnership = Layer.mergeAll( @@ -171,6 +208,31 @@ describe("server ownership", () => { }).pipe(Effect.provide(layerOwnership)), ); + it.effect("names the other machine when the lock holder's record is foreign", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-foreign-" }); + const statePath = path.join(root, "server-runtime.json"); + const state = yield* makeState; + const hostname = `${NodeOS.hostname()}-peer`; + yield* Effect.scoped( + Effect.gen(function* () { + const owner = yield* ServerOwnership.acquireServerOwnership(statePath); + yield* owner.publish({ ...state, hostname }); + const refused = yield* ServerOwnership.acquireServerOwnership(statePath).pipe( + Effect.flip, + ); + assert.equal(refused._tag, "ServerAlreadyRunningError"); + if (refused._tag !== "ServerAlreadyRunningError") return; + assert.equal(refused.hostname, hostname); + assert.include(refused.message, `\`${hostname}\``); + assert.include(refused.message, "do not delete it while a server is running"); + }), + ); + }).pipe(Effect.provide(layerOwnership)), + ); + it.effect("refuses a second owner without touching its record, then allows a restart", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -303,6 +365,29 @@ describe("server ownership", () => { }).pipe(Effect.provide(layerOwnership)), ); + it.effect("names the other machine for a live pre-lock server on another machine", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-legacy-foreign-" }); + const statePath = path.join(root, "server-runtime.json"); + const state = yield* makeState; + const hostname = `${NodeOS.hostname()}-peer`; + yield* ServerRuntimeState.persistServerRuntimeState({ + path: statePath, + // Any record time after this process started proves the PID was not reused. + state: { ...state, hostname, startedAt: "2100-01-01T00:00:00.000Z" }, + }); + const refused = yield* Effect.scoped(ServerOwnership.acquireServerOwnership(statePath)).pipe( + Effect.flip, + ); + assert.equal(refused._tag, "ServerAlreadyRunningError"); + if (refused._tag !== "ServerAlreadyRunningError") return; + assert.equal(refused.hostname, hostname); + assert.include(refused.message, `\`${hostname}\``); + }).pipe(Effect.provide(layerOwnership)), + ); + it.effect("never removes a runtime record another server published", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -377,6 +462,25 @@ describe("server ownership", () => { }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); + it.effect("names the other machine when the older service runs on another machine", () => + Effect.gen(function* () { + const hostname = `${NodeOS.hostname()}-peer`; + const { result } = yield* startBesideBootService({ + platform: "linux", + runtimeHostname: hostname, + outputs: { + "systemctl is-active": { code: 0, stdout: "active\n" }, + }, + }); + assert.equal(result._tag, "Failure"); + if (result._tag !== "Failure") return; + assert.equal(result.failure._tag, "OlderBootServiceError"); + if (result.failure._tag !== "OlderBootServiceError") return; + assert.equal(result.failure.hostname, hostname); + assert.include(result.failure.message, `On \`${hostname}\``); + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); + it.effect( "refuses an active npm-installed service with no standalone executable or runtime record", () => diff --git a/apps/server/src/serverRuntimeState.ts b/apps/server/src/serverRuntimeState.ts index fc5c85a2a9c1e..e27b14f16848e 100644 --- a/apps/server/src/serverRuntimeState.ts +++ b/apps/server/src/serverRuntimeState.ts @@ -1,3 +1,5 @@ +import * as NodeOS from "node:os"; + import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -13,6 +15,8 @@ export const PersistedServerRuntimeState = Schema.Struct({ version: Schema.Literal(1), pid: Schema.Int, ownerId: Schema.optional(Schema.String), + // The machine that wrote this record; every machine sharing the T3 home can read it. + hostname: Schema.optional(Schema.String), host: Schema.optional(Schema.String), privateNetworkOnly: Schema.optional(Schema.Boolean), port: Schema.Int, @@ -73,6 +77,7 @@ export const makePersistedServerRuntimeState = (input: { Effect.map(DateTime.now, (now) => ({ version: 1, pid: process.pid, + hostname: NodeOS.hostname(), ...(input.config.host ? { host: input.config.host } : {}), ...(input.config.privateNetworkOnly ? { privateNetworkOnly: true } : {}), port: input.port, @@ -128,6 +133,10 @@ export const clearPersistedServerRuntimeState = Effect.fn( ); }); +/** A record written by another machine sharing this T3 home; its pid means nothing here. */ +export const isRuntimeStateFromAnotherMachine = (state: PersistedServerRuntimeState): boolean => + state.hostname !== undefined && state.hostname !== NodeOS.hostname(); + /** Signal 0 only probes existence; EPERM still means alive. */ export const isProcessAlive = (pid: number): boolean => { if (!Number.isInteger(pid) || pid <= 0) return false; diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index c7259b4bd84fb..569553db1fd97 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -967,6 +967,83 @@ process.exit(0); }), ); + it.effect("names the other machine when the owner runs on another machine", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-foreign-owner-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const ownerPath = path.join(root, "userdata", "server-owner.sqlite"); + yield* fs.makeDirectory(path.dirname(ownerPath), { recursive: true }); + yield* fs.writeFileString( + path.join(root, "userdata", "server-runtime.json"), + JSON.stringify({ + version: 1, + pid: 1234, + hostname: "another-machine", + port: 3773, + origin: "http://127.0.0.1:3773", + startedAt: "2026-10-01T00:00:00.000Z", + }), + ); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.0.0"), + [ + `const db = new (require("node:sqlite").DatabaseSync)(${JSON.stringify(ownerPath)});`, + `try { db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); }`, + `catch { process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED}); }`, + 'process.send("owner");', + "setInterval(() => {}, 1_000);", + ].join("\n"), + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + using otherOwner = new NodeSqlite.DatabaseSync(ownerPath); + otherOwner.exec("BEGIN EXCLUSIVE;"); + const lines: Array = []; + const refused = Promise.withResolvers(); + yield* Effect.acquireRelease( + Effect.sync(() => + vi.spyOn(process.stderr, "write").mockImplementation((chunk) => { + lines.push(String(chunk)); + if (String(chunk).includes("owns this T3 home")) refused.resolve(); + return true; + }), + ), + (spy) => Effect.sync(() => spy.mockRestore()), + ); + const owned = Promise.withResolvers(); + yield* Effect.acquireRelease( + Effect.sync(() => { + spawnHook.current = (child) => child.once("message", () => owned.resolve()); + }), + () => + Effect.sync(() => { + spawnHook.current = undefined; + }), + ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { stateDirOwnerPollMs: 10 }, + ); + const running = launcher.run(); + yield* Effect.promise(() => refused.promise); + assert.include(lines.join(""), "`another-machine`"); + assert.include(lines.join(""), "Stop it on `another-machine`"); + otherOwner.close(); + yield* Effect.promise(() => owned.promise); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }), + ); + it.effect( "keeps a pending update while owned and rolls a failed resumed trial back to the other owner's writes", () => diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 51662f7be3309..717f5b5821f9d 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -5,6 +5,7 @@ import * as NodeChildProcess from "node:child_process"; import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import * as NodeUtil from "node:util"; import { acquireServiceSelectionLock } from "./cloud/serviceSelectionLock.ts"; @@ -846,6 +847,22 @@ export async function stateDirOwned(baseDir: string): Promise { return legacyOwnerStartedBeforeRecord(output, recordedAt, platform); } +// The runtime record names the machine that wrote it; a foreign record means +// the owner runs on another machine sharing this T3 home. +async function foreignRuntimeHostname(baseDir: string): Promise { + let record: unknown; + try { + record = JSON.parse( + await NodeFSP.readFile(NodePath.join(baseDir, "userdata", "server-runtime.json"), "utf8"), + ); + } catch { + return undefined; + } + if (typeof record !== "object" || record === null) return undefined; + const hostname = "hostname" in record ? record.hostname : undefined; + return typeof hostname === "string" && hostname !== NodeOS.hostname() ? hostname : undefined; +} + const stopMarkerPath = (baseDir: string) => NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE); const restartPendingPath = (baseDir: string) => @@ -1463,8 +1480,11 @@ export class Launcher { } } this.#releaseServing(); + const hostname = await foreignRuntimeHostname(this.#baseDir); this.#log( - "[service-launcher] Another T3 Code server owns this T3 home. This service starts again once it stops.\n", + hostname === undefined + ? "[service-launcher] Another T3 Code server owns this T3 home. This service starts again once it stops.\n" + : `[service-launcher] Another T3 Code server on \`${hostname}\` owns this T3 home. Stop it on \`${hostname}\`; this service starts again once it stops.\n`, ); this.#stateDirFreeProbes = 0; // The interval also keeps Node alive: signal listeners alone do not.