From 742011a637ca03606bc6b9c77ca070c8d98cba9a Mon Sep 17 00:00:00 2001 From: greyxp1 Date: Sat, 31 Oct 2026 16:42:34 -0400 Subject: [PATCH] fix(connect): preserve managed tunnels across server restarts Co-authored-by: maria-rcks --- .../src/updates/DesktopUpdates.test.ts | 16 +- apps/desktop/src/updates/DesktopUpdates.ts | 29 +- .../desktop/src/updates/updatesTestHarness.ts | 4 - .../src/cloud/CloudLink.lifecycle.test.ts | 263 ++---------------- apps/server/src/cloud/CloudLink.ts | 92 +----- apps/server/src/cloud/serviceProtocol.ts | 2 - apps/server/src/cloud/updateHandoff.ts | 60 ---- apps/server/src/server.lifecycle.test.ts | 218 +++++++++++++++ apps/server/src/server.ts | 23 -- apps/server/src/serviceLauncher.test.ts | 7 +- apps/server/src/serviceLauncher.ts | 12 +- docs/internals/t3-connect.md | 18 +- docs/operations/release.md | 5 +- infra/relay/.env.example | 6 +- infra/relay/src/Config.test.ts | 51 ++++ infra/relay/src/Config.ts | 12 +- .../ManagedEndpointReaper.test.ts | 2 +- .../src/environments/ManagedEndpointReaper.ts | 2 +- packages/contracts/src/desktopBootstrap.ts | 2 +- 19 files changed, 335 insertions(+), 489 deletions(-) delete mode 100644 apps/server/src/cloud/updateHandoff.ts create mode 100644 apps/server/src/server.lifecycle.test.ts create mode 100644 infra/relay/src/Config.test.ts diff --git a/apps/desktop/src/updates/DesktopUpdates.test.ts b/apps/desktop/src/updates/DesktopUpdates.test.ts index 535e1123d291..9bc9e236b27e 100644 --- a/apps/desktop/src/updates/DesktopUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.test.ts @@ -1,7 +1,7 @@ import * as NodeModule from "node:module"; import { assert, describe, it } from "@effect/vitest"; -import { DESKTOP_UPDATE_RESTART_MARKER_FILE, DesktopUpdateStateSchema } from "@t3tools/contracts"; +import { DesktopUpdateStateSchema } from "@t3tools/contracts"; import { computeReleaseNotes } from "electron-updater/out/providers/GitHubProvider.js"; import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; @@ -20,7 +20,6 @@ import * as TestClock from "effect/testing/TestClock"; import * as ElectronUpdater from "../electron/ElectronUpdater.ts"; import * as DesktopBackendPool from "../backend/DesktopBackendPool.ts"; import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts"; -import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; import * as DesktopState from "../app/DesktopState.ts"; import * as DesktopUpdatePolicy from "./DesktopUpdatePolicy.ts"; import * as DesktopUpdates from "./DesktopUpdates.ts"; @@ -716,12 +715,10 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); - it.effect("marks the backend stop for an install as an update restart", () => { - let markersAtStop: ReadonlyArray = []; + it.effect("marks the backend stop while an update is in progress", () => { let inProgressAtStop: string | undefined; const harness = makeHarness({ stopBackend: Effect.sync(() => { - markersAtStop = [...harness.updateRestartMarkers]; inProgressAtStop = harness.updateInProgressMarker(); }), }); @@ -729,16 +726,12 @@ describe("DesktopUpdates", () => { return Effect.scoped( Effect.gen(function* () { yield* TestClock.setTime(1_000); - const environment = yield* DesktopEnvironment.DesktopEnvironment; const updates = yield* DesktopUpdates.DesktopUpdates; yield* updates.configure; harness.emit("update-downloaded", { version: "1.2.4" }); yield* flushCallbacks; assert.isTrue((yield* updates.install).accepted); - assert.deepEqual(markersAtStop, [ - environment.path.join(environment.baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE), - ]); // SSH launchers wait on this until the replacement backend publishes its runtime. assert.deepEqual(JSON.parse(inProgressAtStop ?? "null"), { startedAt: 1_000, @@ -826,11 +819,11 @@ describe("DesktopUpdates", () => { const installFiber = yield* updates.install.pipe(Effect.forkScoped); yield* Deferred.await(stopping); - assert.equal(harness.updateRestartMarkers.size, 1); + assert.equal(harness.installRecords.size, 1); assert.isDefined(harness.updateInProgressMarker()); yield* Fiber.interrupt(installFiber); - assert.equal(harness.updateRestartMarkers.size, 0); + assert.equal(harness.installRecords.size, 0); assert.isUndefined(harness.updateInProgressMarker()); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); @@ -865,7 +858,6 @@ describe("DesktopUpdates", () => { "quitAndInstall", "startBackend", ]); - assert.equal(harness.updateRestartMarkers.size, 0); assert.isUndefined(harness.updateInProgressMarker()); assert.equal(harness.installRecords.size, 0); }), diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index 7976503d9e76..825152edd607 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -1,5 +1,4 @@ import { - DESKTOP_UPDATE_RESTART_MARKER_FILE, DESKTOP_UPDATE_IN_PROGRESS_MARKER_FILE, DESKTOP_UPDATE_IN_PROGRESS_TTL_MS, DesktopUpdateChannelSchema, @@ -754,19 +753,6 @@ export const make = Effect.gen(function* () { }); const updateRestartMarkerDir = environment.path.join(environment.baseDir, "runtime"); - const updateRestartMarkerPath = environment.path.join( - updateRestartMarkerDir, - DESKTOP_UPDATE_RESTART_MARKER_FILE, - ); - const writeUpdateRestartMarker = fileSystem - .makeDirectory(updateRestartMarkerDir, { recursive: true }) - .pipe( - Effect.andThen(fileSystem.writeFileString(updateRestartMarkerPath, "")), - Effect.catch((error) => - logUpdaterWarning("Could not write the update restart marker.", { errorTag: error._tag }), - ), - ); - const updateInProgressMarkerPath = environment.path.join( updateRestartMarkerDir, DESKTOP_UPDATE_IN_PROGRESS_MARKER_FILE, @@ -781,6 +767,7 @@ export const make = Effect.gen(function* () { startedAt, expiresAt: startedAt + DESKTOP_UPDATE_IN_PROGRESS_TTL_MS, }); + yield* fileSystem.makeDirectory(updateRestartMarkerDir, { recursive: true }); yield* fileSystem.writeFileString(updateInProgressTempPath, contents); yield* fileSystem.rename(updateInProgressTempPath, updateInProgressMarkerPath); }).pipe( @@ -795,18 +782,17 @@ export const make = Effect.gen(function* () { .remove(updateInProgressMarkerPath, { force: true }) .pipe(Effect.ignore); - // Failed or interrupted installs leave no replacement backend; later quits must release the tunnel. - const removeUpdateRestartMarker = fileSystem - .remove(updateRestartMarkerPath, { force: true }) - .pipe(Effect.ignore); - const installRecordPath = environment.path.join( updateRestartMarkerDir, DESKTOP_UPDATE_INSTALL_RECORD_FILE, ); const writeInstallRecord = (targetVersion: string) => encodeInstallRecord({ fromVersion: environment.appVersion, targetVersion }).pipe( - Effect.flatMap((text) => fileSystem.writeFileString(installRecordPath, text)), + Effect.flatMap((text) => + fileSystem + .makeDirectory(updateRestartMarkerDir, { recursive: true }) + .pipe(Effect.andThen(fileSystem.writeFileString(installRecordPath, text))), + ), Effect.catch((error) => logUpdaterWarning("Could not record the update install.", { errorTag: error._tag }), ), @@ -849,7 +835,6 @@ export const make = Effect.gen(function* () { finishUpdateAction("install"), Ref.set(desktopState.quitting, false), Ref.set(installRunningBackendsRef, []), - removeUpdateRestartMarker, removeUpdateInProgressMarker, removeInstallRecord, ], @@ -867,7 +852,6 @@ export const make = Effect.gen(function* () { if (!ownsRecovery) return; yield* Ref.set(desktopState.quitting, false); - yield* removeUpdateRestartMarker; yield* removeUpdateInProgressMarker; yield* removeInstallRecord; yield* Effect.gen(function* () { @@ -931,7 +915,6 @@ export const make = Effect.gen(function* () { yield* Ref.set(installRunningBackendsRef, []); return yield* Effect.gen(function* () { - yield* writeUpdateRestartMarker; yield* writeUpdateInProgressMarker; const { downloadedVersion } = yield* Ref.get(updateStateRef); if (downloadedVersion !== null) yield* writeInstallRecord(downloadedVersion); diff --git a/apps/desktop/src/updates/updatesTestHarness.ts b/apps/desktop/src/updates/updatesTestHarness.ts index b118d15ced67..5920f18c634f 100644 --- a/apps/desktop/src/updates/updatesTestHarness.ts +++ b/apps/desktop/src/updates/updatesTestHarness.ts @@ -243,7 +243,6 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { } satisfies DesktopAppSettings.DesktopAppSettings["Service"]) : DesktopAppSettings.layer; - const updateRestartMarkers = new Set(); const updateMarkerContents = new Map(); const installRecords = new Map(); let previousInstallRecord = options.installRecord; @@ -271,7 +270,6 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { Effect.sync(() => { if (isInstallRecord(path)) installRecords.set(path, data); else { - if (path.endsWith("/desktop-update-restart")) updateRestartMarkers.add(path); updateMarkerContents.set(path, data); } }), @@ -283,7 +281,6 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { }), remove: (path) => Effect.sync(() => { - updateRestartMarkers.delete(path); updateMarkerContents.delete(path); installRecords.delete(path); if (isInstallRecord(path)) previousInstallRecord = undefined; @@ -316,7 +313,6 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { checkCount: () => checkCount, quitAndInstalls: () => quitAndInstallCount, installSteps, - updateRestartMarkers, updateInProgressMarker: () => [...updateMarkerContents].find(([path]) => path.endsWith("/desktop-update-in-progress"))?.[1], installRecords, diff --git a/apps/server/src/cloud/CloudLink.lifecycle.test.ts b/apps/server/src/cloud/CloudLink.lifecycle.test.ts index a71a8f00b81c..688c09c6e708 100644 --- a/apps/server/src/cloud/CloudLink.lifecycle.test.ts +++ b/apps/server/src/cloud/CloudLink.lifecycle.test.ts @@ -3,10 +3,8 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import * as ConfigProvider from "effect/ConfigProvider"; import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; @@ -19,14 +17,6 @@ import { EnvironmentId } from "@t3tools/contracts"; import * as EnvironmentKeys from "./environmentKeys.ts"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; -import * as ServerConfigModule from "../config.ts"; -import { writeServiceState } from "../serviceLauncher.ts"; -import { - SERVICE_LAUNCHER_PROTOCOL, - SERVICE_STATE_FILE, - SERVICE_STOP_MARKER_FILE, - type ServiceUpdateRecord, -} from "./serviceProtocol.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; @@ -48,7 +38,6 @@ import { import * as CloudLink from "./CloudLink.ts"; import * as ConnectReachability from "./ConnectReachability.ts"; import { consumeCloudReplayGuards } from "./CloudLink.ts"; -import { pendingServiceUpdateExists } from "./updateHandoff.ts"; import { managedTunnelStartupAction } from "./managedTunnelStartup.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; @@ -109,7 +98,7 @@ describe("consumeCloudReplayGuards", () => { ); }); -describe("releaseManagedTunnelOnShutdown", () => { +describe("managed tunnel recovery", () => { const cliToken: CliTokenManager.PersistedToken = { accessToken: "cli-access-token", refreshToken: "cli-refresh-token", @@ -139,7 +128,7 @@ describe("releaseManagedTunnelOnShutdown", () => { return { store, values }; } - interface ReleaseHarness { + interface RecoveryHarness { readonly store: ServerSecretStore.ServerSecretStore["Service"]; readonly applyConfigCalls: Array; readonly requests: Array; @@ -152,22 +141,8 @@ describe("releaseManagedTunnelOnShutdown", () => { readonly connector?: ManagedEndpointRuntime.ConnectorFacts; } - const writeLauncherState = (update: ServiceUpdateRecord) => - Effect.gen(function* () { - const path = yield* Path.Path; - const config = yield* ServerConfigModule.ServerConfig; - const statePath = path.join(config.baseDir, "runtime", SERVICE_STATE_FILE); - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "0.0.30", - update, - }), - ); - }); - - const provideReleaseHarness = - (harness: ReleaseHarness) => + const provideRecoveryHarness = + (harness: RecoveryHarness) => (effect: Effect.Effect) => effect.pipe( Effect.provide( @@ -258,14 +233,7 @@ describe("releaseManagedTunnelOnShutdown", () => { ), ), - Effect.provide([ - ServerConfigModule.layerTest("/", { prefix: "t3-http-release-test-" }).pipe( - Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), - ), - ConfigProvider.layer( - ConfigProvider.fromEnv({ env: { T3CODE_RELAY_URL: "https://relay.example.test" } }), - ), - ]), + Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), Effect.scoped, ); @@ -329,7 +297,7 @@ describe("releaseManagedTunnelOnShutdown", () => { environmentId: "env_123", scopes: ["agent_activity_notifications"], }); - }).pipe(provideReleaseHarness({ store, applyConfigCalls: [], requests: [] })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls: [], requests: [] })); }, ); @@ -353,186 +321,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(yield* link.reconcileDesiredLinkIfStillDesired("http://127.0.0.1:3773")).toBeNull(); expect(requests).toEqual([]); expect(applyConfigCalls).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); - }); - - it.effect("stops the connector, releases the relay tunnel, and drops the dead token", () => { - const { store, values } = makeMemorySecretStore(managedLinkSecrets); - const applyConfigCalls: Array = []; - const requests: Array = []; - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(true); - expect(applyConfigCalls).toEqual([null]); - expect(requests).toHaveLength(1); - const request = requests[0]!; - expect(request.method).toBe("DELETE"); - expect(request.url).toBe( - "https://relay.example.test/v1/client/environment-links/env_123/tunnel", - ); - expect(request.headers.authorization).toBe("Bearer cli-access-token"); - expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); - expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); - }); - - it.effect("releases a tunnel only through the configured relay", () => { - const { store, values } = makeMemorySecretStore(managedLinkSecrets); - values.set(RELAY_URL_SECRET, new TextEncoder().encode("https://old-relay.example.test")); - const applyConfigCalls: Array = []; - const requests: Array = []; - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - expect(yield* link.releaseManagedTunnelOnShutdown()).toBe(false); - expect(applyConfigCalls).toEqual([]); - expect(requests).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); - }); - - it.effect("leaves the tunnel of a web/mobile-installed link untouched", () => { - const { store, values } = makeMemorySecretStore([ - [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"], - [RELAY_URL_SECRET, "https://relay.example.test"], - ]); - const applyConfigCalls: Array = []; - const requests: Array = []; - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(false); - expect(applyConfigCalls).toEqual([]); - expect(requests).toEqual([]); - expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); - }); - - it.effect("keeps the tunnel when shutdown hands off to a pending update", () => { - const { store, values } = makeMemorySecretStore(managedLinkSecrets); - const applyConfigCalls: Array = []; - const requests: Array = []; - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - yield* writeLauncherState({ - id: "update-1", - fromVersion: "0.0.30", - targetVersion: "0.0.31", - dbPath: "/tmp/state.sqlite", - status: "pending", - }); - - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(false); - expect(applyConfigCalls).toEqual([]); - expect(requests).toEqual([]); - expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); - }); - - it.effect("still releases a pending update when the launcher is stopping", () => { - const { store, values } = makeMemorySecretStore(managedLinkSecrets); - const applyConfigCalls: Array = []; - const requests: Array = []; - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - yield* writeLauncherState({ - id: "update-1", - fromVersion: "0.0.30", - targetVersion: "0.0.31", - dbPath: "/tmp/state.sqlite", - status: "pending", - }); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const config = yield* ServerConfigModule.ServerConfig; - yield* fs.writeFileString(path.join(config.baseDir, "runtime", SERVICE_STOP_MARKER_FILE), ""); - - expect(yield* pendingServiceUpdateExists).toBe(true); - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(true); - expect(requests).toHaveLength(1); - expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); - }); - - // regression: 808d6853555 keep the managed tunnel across remote update restarts - it.effect("still releases when the recorded update already settled", () => { - const { store, values } = makeMemorySecretStore(managedLinkSecrets); - const applyConfigCalls: Array = []; - const requests: Array = []; - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - yield* writeLauncherState({ - id: "update-1", - fromVersion: "0.0.30", - targetVersion: "0.0.31", - status: "committed", - }); - - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(true); - expect(requests).toHaveLength(1); - expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); - }); - - it.effect("keeps a runtime config that a fast restart replaced mid-release", () => { - const { store, values } = makeMemorySecretStore(managedLinkSecrets); - const applyConfigCalls: Array = []; - const requests: Array = []; - const freshConfig = new TextEncoder().encode("fresh-runtime-config"); - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(true); - - expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig); - }).pipe( - provideReleaseHarness({ - store, - applyConfigCalls, - requests, - respond: () => { - values.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, freshConfig); - return Response.json({ ok: true }); - }, - }), - ); - }); - - it.effect("keeps the stored connector token when the relay skipped the release", () => { - const { store, values } = makeMemorySecretStore(managedLinkSecrets); - const applyConfigCalls: Array = []; - const requests: Array = []; - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(false); - expect(requests).toHaveLength(1); - expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); - }).pipe( - provideReleaseHarness({ - store, - applyConfigCalls, - requests, - respond: () => Response.json({ ok: false }), - }), - ); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }); it.effect("registers an existing tunnel and starts the confirmed connector", () => { @@ -572,7 +361,7 @@ describe("releaseManagedTunnelOnShutdown", () => { } expect(applyConfigCalls).toHaveLength(1); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -613,7 +402,7 @@ describe("releaseManagedTunnelOnShutdown", () => { }); expect(applyConfigCalls).toEqual([null, config]); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -649,7 +438,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(values.has(RELAY_ENVIRONMENT_CREDENTIAL_SECRET)).toBe(true); expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -685,7 +474,7 @@ describe("releaseManagedTunnelOnShutdown", () => { }); expect(values.has(CLOUD_LINK_REVOKED)).toBe(false); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls: [], requests: [], @@ -796,7 +585,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(values.has(CLOUD_CLI_DESIRED_LINK_SECRET)).toBe(true); } }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls: [], requests: [], @@ -846,7 +635,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(yield* link.startManagedTunnelIfOriginConfirmed("http://127.0.0.1:3773")).toBe(true); expect(applyConfigCalls).toEqual([config]); expect(requests).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }, ); @@ -875,7 +664,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(yield* link.startManagedTunnelIfOriginConfirmed(origin)).toBe(false); expect(applyConfigCalls).toEqual([]); expect(requests).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }); // regression: #9386 remove tunnels after hosts go offline @@ -902,7 +691,7 @@ describe("releaseManagedTunnelOnShutdown", () => { ).toBe(true); expect(applyConfigCalls).toEqual([config]); expect(requests).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }, ); @@ -928,7 +717,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(applyConfigCalls).toEqual([]); expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -981,7 +770,7 @@ describe("releaseManagedTunnelOnShutdown", () => { }); expect(requests).toEqual([]); expect(applyConfigCalls).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }); it.effect("recovers a web-linked tunnel and clears its revoked mark", () => { @@ -1017,7 +806,7 @@ describe("releaseManagedTunnelOnShutdown", () => { ).toEqual(nextConfig); expect(values.has(CLOUD_LINK_REVOKED)).toBe(false); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -1047,7 +836,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(yield* link.recoverManagedTunnel("http://127.0.0.1:3773")).toBe(false); expect(applyConfigCalls).toEqual([]); expect(requests).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }); it.effect("ignores recovery requests for a tunnel that has already been replaced", () => { @@ -1074,7 +863,7 @@ describe("releaseManagedTunnelOnShutdown", () => { ).toBe(false); expect(requests).toEqual([]); expect(applyConfigCalls).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }); it.effect.each([ @@ -1109,7 +898,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(requests).toHaveLength(1); expect(applyConfigCalls).toEqual([]); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -1136,7 +925,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(error.message).toContain("Trace ID: trace-expired"); expect(CloudLink.shouldRetryCloudLink(error)).toBe(true); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -1170,7 +959,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig); expect(applyConfigCalls).toEqual([]); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -1241,7 +1030,7 @@ describe("releaseManagedTunnelOnShutdown", () => { managedTunnelActive: false, }); expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); - }).pipe(provideReleaseHarness({ store: failingStore, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store: failingStore, applyConfigCalls, requests })); }); it.effect("does not report a not-linked check once the managed tunnel config is stored", () => { @@ -1267,7 +1056,7 @@ describe("releaseManagedTunnelOnShutdown", () => { // The address shows before any check passes. expect(state.tunnelHostname).toBe("prod-abc.t3coderelay.com"); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls: [], requests: [], @@ -1297,7 +1086,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(state.reachability).toEqual(reachable); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls: [], requests: [], diff --git a/apps/server/src/cloud/CloudLink.ts b/apps/server/src/cloud/CloudLink.ts index 6c9c70246b67..d39256037913 100644 --- a/apps/server/src/cloud/CloudLink.ts +++ b/apps/server/src/cloud/CloudLink.ts @@ -23,7 +23,7 @@ import { RelayManagedEndpointRecoveryRegistrationResponse, RelayManagedEndpointRecoveryResponse, type RelayManagedEndpointRuntimeConfig, - RelayOkResponse, + type RelayOkResponse, } from "@t3tools/contracts/relay"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; import { @@ -37,7 +37,7 @@ import { RELAY_MINT_RESPONSE_TYP, verifyRelayJwt, } from "@t3tools/shared/relayJwt"; -import { isSecureRelayUrl, normalizeSecureRelayUrl } from "@t3tools/shared/relayUrl"; +import { isSecureRelayUrl } from "@t3tools/shared/relayUrl"; import * as Context from "effect/Context"; import * as Digest from "effect/crypto/Digest"; import * as SecureRandom from "effect/crypto/SecureRandom"; @@ -48,21 +48,17 @@ import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FetchHttpClient from "effect/http/FetchHttpClient"; -import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as Path from "effect/Path"; import type * as PlatformError from "effect/PlatformError"; import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"; -import type * as HttpClientError from "effect/http/HttpClientError"; import * as HttpServer from "effect/http/HttpServer"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; -import * as ServerConfig from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import { makeRelayEnvironmentClient } from "../relay/relayEnvironmentClient.ts"; @@ -114,7 +110,6 @@ import { managedEndpointRuntimeConfigsMatch, parseManagedEndpointLocalOrigin, } from "./linkChecks.ts"; -import { desktopUpdateRestartPending, pendingUpdateHandoffExists } from "./updateHandoff.ts"; const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-"; const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-"; @@ -501,12 +496,6 @@ export class CloudLink extends Context.Service< | CloudLinkOriginInvalidError | ServerSecretStore.SecretStoreError >; - readonly releaseManagedTunnelOnShutdown: () => Effect.Effect< - boolean, - | CloudLinkBackgroundError - | CliTokenManager.CloudCliTokenManagerError - | HttpClientError.HttpClientError - >; } >()("t3/cloud/CloudLink") {} @@ -525,9 +514,6 @@ const make = Effect.gen(function* () { const digest = yield* Digest.Digest; const signature = yield* Signature.Signature; const signingKey = yield* EnvironmentSigningKey; - const config = yield* ServerConfig.ServerConfig; - const fileSystem = yield* FileSystem.FileSystem; - const path = yield* Path.Path; const managedPolicy = yield* ManagedPolicy.ManagedPolicy; // Relay-signed text must not name a file directly; a digest is also a fixed, safe length. @@ -538,14 +524,6 @@ const make = Effect.gen(function* () { }); const withSecrets = Effect.provideService(ServerSecretStore.ServerSecretStore, secrets); - const withRuntimeFiles = ( - effect: Effect.Effect, - ) => - effect.pipe( - Effect.provideService(ServerConfig.ServerConfig, config), - Effect.provideService(FileSystem.FileSystem, fileSystem), - Effect.provideService(Path.Path, path), - ); const validateLinkedCloudUser = ( cloudUserId: string, @@ -1228,70 +1206,6 @@ const make = Effect.gen(function* () { }), ); - // Cloudflare bills provisioned tunnels even while environments are offline. - const releaseManagedTunnelOnShutdown = Effect.fn( - "environment.cloud.releaseManagedTunnelOnShutdown", - )(function* () { - const runtimeConfig = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); - if (Option.isNone(runtimeConfig)) { - return false; - } - // CLI authorization releases only CLI-desired managed links; web/mobile links recover by credential and expire through the relay reaper. - if ( - !(yield* readCliDesiredCloudLink.pipe(withSecrets)) || - (yield* readCliDesiredLinkMode.pipe(withSecrets)) !== "managed" - ) { - return false; - } - // Update handoffs reuse the tunnel: replacing it adds 1-2 minutes of hostname propagation. - if ( - (yield* withRuntimeFiles(desktopUpdateRestartPending)) || - (yield* withRuntimeFiles(pendingUpdateHandoffExists)) - ) { - yield* Effect.logInfo("Keeping the managed tunnel across the update restart"); - return false; - } - const relayUrl = yield* requireRelayUrl.pipe(Effect.option); - const installedRelayUrl = yield* secrets.get(RELAY_URL_SECRET); - if ( - Option.isNone(relayUrl) || - Option.isNone(installedRelayUrl) || - normalizeSecureRelayUrl(bytesToString(installedRelayUrl.value)) !== relayUrl.value - ) { - return false; - } - const token = yield* cliTokenManager.getExisting; - if (Option.isNone(token)) { - return false; - } - const environmentId = yield* environment.getEnvironmentId; - yield* endpointRuntime.applyConfig(null); - const response = yield* HttpClientRequest.delete( - `${relayUrl.value}/v1/client/environment-links/${encodeURIComponent(environmentId)}/tunnel`, - ).pipe( - HttpClientRequest.bearerToken(token.value.accessToken), - httpClient.execute, - Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), - Effect.flatMap(filterRelayResponse), - Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)), - withRelayClientTracing, - ); - // ok:false means concurrent provisioning owns the tunnel; preserve its config. - if (!response.ok) { - return false; - } - // Released tokens are dead; remove only this shutdown's config to preserve a fast restart's replacement. - const storedConfig = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); - if ( - Option.isSome(storedConfig) && - bytesToString(storedConfig.value) === bytesToString(runtimeConfig.value) - ) { - yield* secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); - yield* secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN); - } - return true; - }); - const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function* () { const [ cloudUserId, @@ -1716,7 +1630,6 @@ const make = Effect.gen(function* () { registerManagedTunnelRecovery: notLinked, recoverManagedTunnel: notStarted, startManagedTunnelIfOriginConfirmed: notStarted, - releaseManagedTunnelOnShutdown, }); } @@ -1733,7 +1646,6 @@ const make = Effect.gen(function* () { registerManagedTunnelRecovery, recoverManagedTunnel, startManagedTunnelIfOriginConfirmed, - releaseManagedTunnelOnShutdown, }); }); diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 18dbb5199e54..a019c7555bb2 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -17,8 +17,6 @@ export const SERVICE_STATE_FILE = "service-state.json"; /** Set by the unit or launch agent to its own file name, so the launcher can find the definition that started it. */ export const SERVICE_UNIT_ENV = "T3_BOOT_SERVICE_UNIT"; -/** Written before explicit stop to distinguish shutdown from update handoff. */ -export const SERVICE_STOP_MARKER_FILE = ".service-stopping"; /** Marks a repointed unit whose service still runs the old version; launcher startup clears it. */ export const SERVICE_RESTART_PENDING_FILE = ".restart-pending"; /** Held by a Windows launcher while it runs: Task Scheduler tracks only the task's console host. */ diff --git a/apps/server/src/cloud/updateHandoff.ts b/apps/server/src/cloud/updateHandoff.ts deleted file mode 100644 index 27b66c7f5b2e..000000000000 --- a/apps/server/src/cloud/updateHandoff.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { DESKTOP_UPDATE_RESTART_MARKER_FILE } from "@t3tools/contracts"; -import * as Clock from "effect/Clock"; -import * as Duration from "effect/Duration"; -import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; -import * as Option from "effect/Option"; -import * as Path from "effect/Path"; - -import * as ServerConfig from "../config.ts"; -import { - SERVICE_STATE_FILE, - SERVICE_STOP_MARKER_FILE, - serviceStateHasPendingUpdate, -} from "./serviceProtocol.ts"; - -const DESKTOP_UPDATE_RESTART_MARKER_TTL = Duration.minutes(1); - -export const pendingServiceUpdateExists = Effect.gen(function* () { - const config = yield* ServerConfig.ServerConfig; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const runtimeDir = path.join(config.baseDir, "runtime"); - const stateText = yield* fs - .readFileString(path.join(runtimeDir, SERVICE_STATE_FILE)) - .pipe(Effect.option); - return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value); -}); - -export const pendingUpdateHandoffExists = Effect.gen(function* () { - if (!(yield* pendingServiceUpdateExists)) { - return false; - } - const config = yield* ServerConfig.ServerConfig; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const runtimeDir = path.join(config.baseDir, "runtime"); - const stopping = yield* fs - .exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE)) - .pipe(Effect.orElseSucceed(() => false)); - return !stopping; -}); - -// Reading consumes it, so shutdown checks it first. -export const desktopUpdateRestartPending = Effect.gen(function* () { - const config = yield* ServerConfig.ServerConfig; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const markerPath = path.join(config.baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE); - const marker = yield* fs.stat(markerPath).pipe(Effect.option); - if (Option.isNone(marker)) { - return false; - } - yield* fs.remove(markerPath).pipe(Effect.ignore); - const now = yield* Clock.currentTimeMillis; - return Option.match(marker.value.mtime, { - onNone: () => false, - onSome: (writtenAt) => - now - writtenAt.getTime() < Duration.toMillis(DESKTOP_UPDATE_RESTART_MARKER_TTL), - }); -}); diff --git a/apps/server/src/server.lifecycle.test.ts b/apps/server/src/server.lifecycle.test.ts new file mode 100644 index 000000000000..9fb203323cd5 --- /dev/null +++ b/apps/server/src/server.lifecycle.test.ts @@ -0,0 +1,218 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Exercises CLI subprocess shutdown against a local relay fixture. +import * as NodeChildProcess from "node:child_process"; +import * as NodeEvents from "node:events"; +import * as NodeHttps from "node:https"; +import * as NodeURL from "node:url"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import * as NetService from "@t3tools/shared/Net"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as TestClock from "effect/testing/TestClock"; + +import { + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, + CLOUD_ENDPOINT_RUNTIME_CONFIG, + CLOUD_LINKED_USER_ID, + decodeConfirmedOrigin, + RELAY_ENVIRONMENT_CREDENTIAL_SECRET, + RELAY_ISSUER_SECRET, + RELAY_URL_SECRET, +} from "./cloud/config.ts"; +import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./cloud/CliState.ts"; +import { makeProcessProbe, processProbeSource } from "./testUtils/processProbe.ts"; + +it.live("retains a managed tunnel on server shutdown and reuses it after restart", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const net = yield* NetService.NetService; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-tunnel-lifecycle-" }); + const secretsDir = path.join(root, "userdata", "secrets"); + yield* fs.makeDirectory(secretsDir, { recursive: true }); + const relayCertPath = yield* path.fromFileUrl( + new URL("./peer/fixtures/localhost-cert.pem", import.meta.url), + ); + const relayKey = yield* fs.readFileString( + yield* path.fromFileUrl(new URL("./peer/fixtures/localhost-key.pem", import.meta.url)), + ); + const relayCert = yield* fs.readFileString(relayCertPath); + const requests: string[] = []; + const relay = yield* Effect.acquireRelease( + Effect.promise(async () => { + const server = NodeHttps.createServer( + { key: relayKey, cert: relayCert }, + (request, response) => { + requests.push(`${request.method} ${request.url}`); + request.resume(); + response.setHeader("content-type", "application/json"); + response.end( + JSON.stringify(request.method === "DELETE" ? { ok: true } : { status: "ready" }), + ); + }, + ); + server.listen(0, "127.0.0.1"); + await NodeEvents.EventEmitter.once(server, "listening"); + return server; + }), + (server) => + Effect.promise(() => new Promise((resolve) => server.close(() => resolve()))), + ); + const address = relay.address(); + if (address === null || typeof address === "string") return yield* Effect.die("No relay port"); + const relayUrl = `https://127.0.0.1:${address.port}`; + const port = yield* net.reserveLoopbackPort(); + const config = { + providerKind: "cloudflare_tunnel", + tunnelId: "existing-tunnel", + connectorToken: "existing-token", + }; + const configJson = JSON.stringify(config); + const confirmedOrigin = { + config, + origin: { localHttpHost: "127.0.0.1", localHttpPort: port }, + }; + yield* Effect.forEach( + Object.entries({ + [CLOUD_ENDPOINT_RUNTIME_CONFIG]: configJson, + [CLOUD_ENDPOINT_CONFIRMED_ORIGIN]: JSON.stringify(confirmedOrigin), + [CLOUD_CLI_DESIRED_LINK_SECRET]: "managed", + [CLOUD_LINKED_USER_ID]: "user-123", + [RELAY_URL_SECRET]: relayUrl, + [RELAY_ISSUER_SECRET]: relayUrl, + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET]: "environment-credential", + "cloud-cli-oauth-token": JSON.stringify({ + accessToken: "cli-access-token", + refreshToken: "cli-refresh-token", + expiresAtEpochMs: Number.MAX_SAFE_INTEGER, + }), + }), + ([name, value]) => fs.writeFileString(path.join(secretsDir, `${name}.bin`), value), + ); + yield* fs.writeFileString(path.join(root, "userdata", "environment-id"), "env_123"); + const bin = yield* path.fromFileUrl(new URL("./bin.ts", import.meta.url)); + const inheritedEnv = Object.fromEntries( + Object.entries(process.env).filter(([name]) => !/^(?:T3CODE_|VITE_)/u.test(name)), + ); + + const startAndStop = Effect.fn("test.managedTunnel.startAndStop")( + function* () { + const connector = yield* makeProcessProbe; + const preload = path.join(root, `connector-${connector.port}.mjs`); + // Node acts as cloudflared so the fixture also works without a shell launcher on Windows. + yield* fs.writeFileString( + preload, + ` + import { basename } from "node:path"; + const command = basename(process.argv[1] ?? ""); + if (command === "version") { + console.log("cloudflared version 2026.10.4"); + process.exit(0); + } + if (command === "tunnel") { + ${processProbeSource(connector.port)} + console.error("INF Registered tunnel connection connIndex=0"); + await new Promise((resolve) => probe.on("close", resolve)); + process.exit(0); + } + process.on("message", (message) => { + if (message === "shutdown") process.emit("SIGTERM"); + }); + `, + ); + const child = yield* Effect.acquireRelease( + Effect.sync(() => + NodeChildProcess.fork( + bin, + ["serve", "--base-dir", root, "--host", "127.0.0.1", "--port", String(port)], + { + cwd: root, + execArgv: [], + silent: true, + env: { + ...inheritedEnv, + NODE_OPTIONS: `--import=${NodeURL.pathToFileURL(preload).href}`, + NODE_EXTRA_CA_CERTS: relayCertPath, + T3CODE_CLOUDFLARED_PATH: process.execPath, + T3CODE_RELAY_URL: relayUrl, + T3CODE_CLERK_PUBLISHABLE_KEY: `pk_test_${Buffer.from("clerk.example.test$").toString("base64")}`, + T3CODE_CLERK_CLI_OAUTH_CLIENT_ID: "test-client", + T3CODE_TELEMETRY_ENABLED: "false", + }, + }, + ), + ), + (process) => + Effect.promise(async () => { + if ( + process.pid === undefined || + process.exitCode !== null || + process.signalCode !== null + ) + return; + const exited = NodeEvents.EventEmitter.once(process, "close"); + process.kill("SIGKILL"); + await exited; + }), + ); + const ready = Promise.withResolvers(); + const exited = NodeEvents.EventEmitter.once(child, "close"); + let output = ""; + for (const stream of [child.stdout, child.stderr]) { + stream?.on("data", (chunk: Buffer) => { + output += chunk.toString(); + if ( + output.includes("T3 Code server is ready.") && + output.includes("T3 Connect managed tunnel recovery registered") && + output.includes("Relay client tunnel connection registered") + ) { + ready.resolve(); + } + }); + } + yield* Effect.promise(() => + Promise.race([ + ready.promise, + exited.then(() => { + throw new Error(`Server exited before tunnel registration:\n${output}`); + }), + ]), + ); + yield* connector.started; + yield* Effect.promise(async () => { + child.send("shutdown"); + await exited; + }); + yield* connector.exited; + expect(output).toContain("V2 orchestration shutdown reconciliation completed"); + }, + Effect.scoped, + // The shared process probe uses TestClock.withLive for its event deadlines. + Effect.provide(TestClock.layer()), + ); + + yield* startAndStop(); + expect(requests).toEqual(["POST /v1/environments/env_123/tunnel/recovery"]); + expect( + yield* fs.readFileString(path.join(secretsDir, `${CLOUD_ENDPOINT_RUNTIME_CONFIG}.bin`)), + ).toBe(configJson); + expect( + decodeConfirmedOrigin( + yield* fs.readFileString(path.join(secretsDir, `${CLOUD_ENDPOINT_CONFIRMED_ORIGIN}.bin`)), + ), + ).toEqual(Option.some(confirmedOrigin)); + + yield* startAndStop(); + expect(requests).toEqual([ + "POST /v1/environments/env_123/tunnel/recovery", + "POST /v1/environments/env_123/tunnel/recovery", + ]); + expect( + yield* fs.readFileString(path.join(secretsDir, `${CLOUD_ENDPOINT_RUNTIME_CONFIG}.bin`)), + ).toBe(configJson); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NetService.layer))), +); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index d06186beb198..aa1d71cd5828 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -189,7 +189,6 @@ import * as GuestHttp from "./guest/http.ts"; import * as LinkThreadHttp from "./peer/linkThreadHttp.ts"; import * as CloudLink from "./cloud/CloudLink.ts"; import * as ConnectReachability from "./cloud/ConnectReachability.ts"; -import { pendingServiceUpdateExists } from "./cloud/updateHandoff.ts"; import * as RelayTracing from "./cloud/relayTracing.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; import * as RelayClientLedger from "./cloud/RelayClientLedger.ts"; @@ -1159,30 +1158,8 @@ const layerMakeServer = Layer.unwrap( yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); return; } - const releaseManagedTunnel = cloudLink.releaseManagedTunnelOnShutdown().pipe( - Effect.timeout("10 seconds"), - Effect.tap((released) => - released ? Effect.logInfo("Released the managed tunnel on shutdown") : Effect.void, - ), - Effect.catchCause((cause) => - Effect.logWarning( - "Failed to release the managed tunnel on shutdown; the next link reuses it", - { errors: Cause.prettyErrors(cause).map((error) => error.message) }, - ), - ), - Effect.asVoid, - ); - // Trials own cleanup immediately after the old server exits, retaining tunnels for commit/rollback unless explicitly stopped. - // Other runtimes wait for activation so failed standbys cannot tear down active tunnels. - const cleanupBeforeActivation = yield* pendingServiceUpdateExists; - if (cleanupBeforeActivation) { - yield* Effect.addFinalizer(() => releaseManagedTunnel); - } yield* ServerActivation.forkParked( Effect.gen(function* () { - if (!cleanupBeforeActivation) { - yield* Effect.addFinalizer(() => releaseManagedTunnel); - } const server = yield* HttpServer.HttpServer; const address = server.address; if (typeof address === "string" || !("port" in address)) return; diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index c7259b4bd84f..980078593d96 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -43,7 +43,6 @@ import { SERVICE_ENV_FILE, SERVICE_LAUNCHER_PROTOCOL, SERVICE_RESTART_PENDING_FILE, - SERVICE_STOP_MARKER_FILE, } from "./cloud/serviceProtocol.ts"; const spawnedChildren = vi.hoisted((): Array => []); @@ -1696,7 +1695,7 @@ if (context.update?.status === "pending") { }), ); - it.effect("holds its lock while it serves and stops the server over IPC, marker set", () => + it.effect("holds its lock while it serves and stops the server over IPC, no marker", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -1709,7 +1708,7 @@ if (context.update?.status === "pending") { const encoded = JSON.stringify({ databasePath, reportPath, - markerPath: path.join(root, "runtime", SERVICE_STOP_MARKER_FILE), + markerPath: path.join(root, "runtime", ".service-stopping"), }); const childSource = ` const fs = require("node:fs"); @@ -1760,7 +1759,7 @@ process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.dat // What losing the task's console does. yield* Effect.promise(() => launcher.stop("SIGTERM")); yield* Effect.promise(() => running); - assert.equal(yield* fs.readFileString(reportPath), "true"); + assert.equal(yield* fs.readFileString(reportPath), "false"); assert.isFalse(yield* Effect.promise(() => serviceLauncherIsRunning(lockPath))); }), ); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 8dcd7886fce5..dbec6289b004 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -37,7 +37,6 @@ import { SERVICE_LAUNCHER_PROTOCOL, SERVICE_STATE_FILE, SERVICE_RESTART_PENDING_FILE, - SERVICE_STOP_MARKER_FILE, SERVICE_UNIT_ENV, systemdUnitDirectoryOf, } from "./cloud/serviceProtocol.ts"; @@ -882,8 +881,6 @@ export async function stateDirOwned(baseDir: string): Promise { return legacyOwnerStartedBeforeRecord(output, recordedAt, platform); } -const stopMarkerPath = (baseDir: string) => - NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE); const restartPendingPath = (baseDir: string) => NodePath.join(baseDir, "runtime", SERVICE_RESTART_PENDING_FILE); @@ -1023,10 +1020,6 @@ export class Launcher { } async stop(signal: NodeJS.Signals): Promise { - // Write synchronously on signal receipt; a terminating child needs the marker in its finalizer. - try { - NodeFS.writeFileSync(stopMarkerPath(this.#baseDir), "", { mode: 0o600 }); - } catch {} if (this.#stopRequested || this.#stopping) { await this.#completion.promise.catch(() => undefined); return; @@ -1035,7 +1028,7 @@ export class Launcher { this.#stopSignal.abort(); this.#clearTimer(); this.#enqueue(async () => { - // Let in-progress updates start replacements first; replacements observe the marker and own pre-activation tunnel cleanup. + // Let in-progress updates start replacements first. this.#stopping = true; // An exit-78 transition queued before this one may have started the owner poll. this.#clearTimer(); @@ -1124,8 +1117,7 @@ export class Launcher { async #recover(): Promise { await this.#claimServing(); - // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races. - await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); + // Clear a deferred restart only after the requested version starts. const restartPending = restartPendingPath(this.#baseDir); try { const release = await acquireServiceSelectionLock(this.#statePath); diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index 7237df5cfe38..19e9624a6d96 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -104,17 +104,11 @@ reconciles that intent. CLI logout removes the stored cloud credential and disables exposure without uninstalling the environment's background service. Managed allocations belong to a user/environment pair. Provisioning checkpoints -external tunnel and DNS resources so retries can reconcile partial work. A -normal shutdown of a CLI-managed link releases its tunnel to avoid paying for -an idle resource, retaining the hostname reservation for the next startup. -It also retains the allocation record so the environment remains "offline" -rather than becoming "not authorized". - -Two cases must retain the tunnel across shutdown. A link installed through a -client has no startup provisioning path and depends on its stored connector -token. An update handoff immediately starts a replacement server, and replacing -the tunnel would add routing propagation delay to every update. These exceptions -belong to [shutdown handling](../../apps/server/src/cloud/CloudLink.ts). +external tunnel and DNS resources so retries can reconcile partial work. +Server shutdown stops the connector but keeps the tunnel and its stored config, +so restarts reconnect without provisioning or routing propagation. Explicit +unlink removes the link and tunnel. The relay reclaims tunnels that remain +offline after the grace period below. Release and unlink claim the allocation generation before deleting external resources. A delayed cleanup must not delete a tunnel reused by a concurrent @@ -128,7 +122,7 @@ teardown retains enough state to retry. See the Cloudflare bills a tunnel whether or not a connector is attached, so a laptop that sleeps with a linked environment leaves a paid tunnel behind. The relay's five-minute maintenance job can reclaim those tunnels. `RELAY_TUNNEL_CLEANUP_MODE` -selects `off`, `dry-run`, or `enabled`, with `off` as the default. The mode is +selects `off`, `dry-run`, or `enabled`, with `enabled` as the default. The mode is read at deploy time, so changing it means a relay deploy, not a variable flip. A candidate is a same-stage tunnel that Cloudflare reports down for at least five minutes, or one that never connected and is at least an hour old. The diff --git a/docs/operations/release.md b/docs/operations/release.md index 831c70cf2776..950960a5953a 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -144,7 +144,7 @@ Optional `production` environment variables: - `RELAY_DOMAIN` when overriding the derived `relay.` domain - `RELAY_TUNNEL_CLEANUP_MODE` with `off`, `dry-run`, or `enabled`. Missing and blank values use - `off`. + `enabled`. - `RELAY_LEGACY_TUNNEL_CLEANUP_MODE` with the same values, for tunnels whose host never registered recovery. Missing and blank values use `off`. @@ -183,7 +183,8 @@ whose teardown failed. Fork PRs get no preview. ### Managed tunnel cleanup rollout -Keep `RELAY_TUNNEL_CLEANUP_MODE=off` for the first production deploy. That deploy applies the +Set `RELAY_TUNNEL_CLEANUP_MODE=off` explicitly before the first production deploy. An unset or +blank value enables cleanup. That deploy applies the nullable allocation migration and adds the recovery endpoints. Web and mobile clients need no coordinated release. CLI and desktop server builds must reach users before cleanup is enabled, because those builds register recovery and replace a deleted tunnel after wake. diff --git a/infra/relay/.env.example b/infra/relay/.env.example index 87a9a1ca32cb..639515f6121f 100644 --- a/infra/relay/.env.example +++ b/infra/relay/.env.example @@ -5,9 +5,9 @@ RELAY_API_ZONE_NAME=example.com RELAY_TUNNEL_ZONE_NAME=tunnels.example.com -# Optional: inactive tunnel cleanup. Start with dry-run, verify the cleanup -# logs, then set enabled. Unset and off both disable cleanup. -# RELAY_TUNNEL_CLEANUP_MODE=off +# Optional: inactive tunnel cleanup. Defaults to enabled for recoverable tunnels. +# Use dry-run to inspect candidates or off to disable cleanup. +# RELAY_TUNNEL_CLEANUP_MODE=enabled # Optional: cleanup for tunnels whose host never registered recovery, once they # have been down for 30 days. Same values, independent of the mode above. # RELAY_LEGACY_TUNNEL_CLEANUP_MODE=off diff --git a/infra/relay/src/Config.test.ts b/infra/relay/src/Config.test.ts new file mode 100644 index 000000000000..c458838dd347 --- /dev/null +++ b/infra/relay/src/Config.test.ts @@ -0,0 +1,51 @@ +import { expect, it } from "@effect/vitest"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Effect from "effect/Effect"; + +import { + legacyManagedEndpointCleanupModeConfig, + managedEndpointCleanupModeConfig, +} from "./Config.ts"; + +it.effect.each([ + { name: "missing", env: {}, expected: "enabled" }, + { name: "empty", env: { RELAY_TUNNEL_CLEANUP_MODE: "" }, expected: "enabled" }, + { name: "whitespace", env: { RELAY_TUNNEL_CLEANUP_MODE: " \t" }, expected: "enabled" }, + { name: "off", env: { RELAY_TUNNEL_CLEANUP_MODE: "off" }, expected: "off" }, + { + name: "dry-run", + env: { RELAY_TUNNEL_CLEANUP_MODE: "dry-run" }, + expected: "dry-run", + }, + { name: "enabled", env: { RELAY_TUNNEL_CLEANUP_MODE: "enabled" }, expected: "enabled" }, +] as const)("loads $name cleanup mode as $expected", ({ env, expected }) => + Effect.gen(function* () { + const provider = ConfigProvider.fromEnv({ env }); + expect(yield* managedEndpointCleanupModeConfig.parse(provider)).toBe(expected); + }), +); + +it.effect("rejects an invalid cleanup mode", () => + Effect.gen(function* () { + const provider = ConfigProvider.fromEnv({ + env: { RELAY_TUNNEL_CLEANUP_MODE: "delete-everything" }, + }); + const error = yield* Effect.flip(managedEndpointCleanupModeConfig.parse(provider)); + + expect(error._tag).toBe("ConfigError"); + expect(error.message).toContain('Expected "off" | "dry-run" | "enabled"'); + }), +); + +it.effect("reads the legacy cleanup mode independently of the main one", () => + Effect.gen(function* () { + const provider = ConfigProvider.fromEnv({ + env: { RELAY_TUNNEL_CLEANUP_MODE: "enabled", RELAY_LEGACY_TUNNEL_CLEANUP_MODE: "dry-run" }, + }); + expect(yield* managedEndpointCleanupModeConfig.parse(provider)).toBe("enabled"); + expect(yield* legacyManagedEndpointCleanupModeConfig.parse(provider)).toBe("dry-run"); + expect( + yield* legacyManagedEndpointCleanupModeConfig.parse(ConfigProvider.fromEnv({ env: {} })), + ).toBe("off"); + }), +); diff --git a/infra/relay/src/Config.ts b/infra/relay/src/Config.ts index ecc71a5a96a4..00af2b561609 100644 --- a/infra/relay/src/Config.ts +++ b/infra/relay/src/Config.ts @@ -15,10 +15,10 @@ const decodeManagedEndpointCleanupMode = Schema.decodeUnknownEffect(ManagedEndpo const RELAY_TUNNEL_CLEANUP_MODE = "RELAY_TUNNEL_CLEANUP_MODE"; const RELAY_LEGACY_TUNNEL_CLEANUP_MODE = "RELAY_LEGACY_TUNNEL_CLEANUP_MODE"; -const cleanupModeConfig = (name: string) => +const cleanupModeConfig = (name: string, defaultMode: ManagedEndpointCleanupMode) => Config.String(name).pipe( - Config.withDefault("off"), - Config.map((value) => value.trim() || "off"), + Config.withDefault(defaultMode), + Config.map((value) => value.trim() || defaultMode), Config.mapEffect((value) => decodeManagedEndpointCleanupMode(value).pipe( Effect.mapError((error) => new Config.ConfigError(error)), @@ -26,9 +26,13 @@ const cleanupModeConfig = (name: string) => ), ); -export const managedEndpointCleanupModeConfig = cleanupModeConfig(RELAY_TUNNEL_CLEANUP_MODE); +export const managedEndpointCleanupModeConfig = cleanupModeConfig( + RELAY_TUNNEL_CLEANUP_MODE, + "enabled", +); export const legacyManagedEndpointCleanupModeConfig = cleanupModeConfig( RELAY_LEGACY_TUNNEL_CLEANUP_MODE, + "off", ); const RELAY_LEGACY_TUNNEL_GRACE_MINUTES = "RELAY_LEGACY_TUNNEL_GRACE_MINUTES"; diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index 5ce24e6b7e5c..6581e5c45ba9 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -237,7 +237,7 @@ function harness(input?: { cloudMintPublicKey: "cloud-public-key", managedEndpointBaseDomain: "example.test", managedEndpointNamespace: input?.namespace ?? "prod", - managedEndpointCleanupMode: input?.cleanupMode ?? "enabled", + ...(input?.cleanupMode === undefined ? {} : { managedEndpointCleanupMode: input.cleanupMode }), legacyManagedEndpointCleanupMode: input?.legacyCleanupMode ?? "off", ...(input?.legacyTunnelGraceMinutes === undefined ? {} diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index 71442b536d00..07e2f84a27b2 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -157,7 +157,7 @@ const make = Effect.gen(function* () { const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; const sweep = Effect.gen(function* () { - const mode = config.managedEndpointCleanupMode ?? "off"; + const mode = config.managedEndpointCleanupMode ?? "enabled"; const legacyMode = config.legacyManagedEndpointCleanupMode ?? "off"; const namespace = config.managedEndpointNamespace; if ((mode === "off" && legacyMode === "off") || !namespace) { diff --git a/packages/contracts/src/desktopBootstrap.ts b/packages/contracts/src/desktopBootstrap.ts index 5148aa2c1c82..e37852dca776 100644 --- a/packages/contracts/src/desktopBootstrap.ts +++ b/packages/contracts/src/desktopBootstrap.ts @@ -35,7 +35,7 @@ export const DesktopBackendBootstrap = Schema.Struct({ export type DesktopBackendBootstrap = typeof DesktopBackendBootstrap.Type; -/** Written to `/runtime` before update shutdown so the backend retains its managed tunnel. */ +// The service CLI writes this marker for installed servers that release their tunnel on shutdown. export const DESKTOP_UPDATE_RESTART_MARKER_FILE = "desktop-update-restart"; export const DESKTOP_UPDATE_IN_PROGRESS_MARKER_FILE = "desktop-update-in-progress";