From 068251d4a8b8ee137af16556308042d409b50153 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Thu, 22 Oct 2026 05:20:10 -0500 Subject: [PATCH 01/16] fix(relay): keep a tunnel whose host registered recovery after the cutoff A starting host registers tunnel recovery before cloudflared connects, so Cloudflare still reports the tunnel's old inactivity time. A cleanup sweep in that window deleted the tunnel the host was about to use. Release now leaves a tunnel alone when its recovery registration is newer than the sweep's inactivity cutoff. Co-Authored-By: Claude Opus 5.5 --- .../environments/EnvironmentConnector.test.ts | 3 ++ .../ManagedEndpointAllocations.ts | 12 ++--- .../ManagedEndpointProvider.test.ts | 44 +++++++++++++++++++ .../environments/ManagedEndpointProvider.ts | 14 ++++++ .../ManagedEndpointReaper.test.ts | 1 + infra/relay/src/hooks/HeldHooks.test.ts | 1 + infra/relay/src/hooks/HookForwarder.test.ts | 1 + 7 files changed, 68 insertions(+), 8 deletions(-) diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index 731cce9c22009..3ebf2dc3b4b9a 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -188,6 +188,7 @@ function makeAllocations( updatedAt: "2026-05-25T00:00:00.000Z", generation: 1, tunnelReleasedAt: null, + recoveryEnabledAt: null, }, ): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] { return { @@ -426,6 +427,7 @@ describe("EnvironmentConnector", () => { updatedAt: "2026-05-25T00:00:00.000Z", generation: 1, tunnelReleasedAt: null, + recoveryEnabledAt: null, }), }), ), @@ -485,6 +487,7 @@ describe("EnvironmentConnector", () => { updatedAt: "2026-05-25T00:00:00.000Z", generation: 2, tunnelReleasedAt: "2026-05-26T00:00:00.000Z", + recoveryEnabledAt: null, }); return Effect.gen(function* () { diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index d416508ad4397..285db12b695b7 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -25,6 +25,7 @@ export interface ManagedEndpointAllocation { readonly updatedAt: string; readonly generation: number; readonly tunnelReleasedAt: string | null; + readonly recoveryEnabledAt: string | null; } export interface ManagedEndpointTunnelAllocation extends ManagedEndpointAllocation { @@ -204,6 +205,7 @@ const allocationSelection = { updatedAt: relayManagedEndpointAllocations.updatedAt, generation: relayManagedEndpointAllocations.generation, tunnelReleasedAt: relayManagedEndpointAllocations.tunnelReleasedAt, + recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, }; const whereAllocation = (input: ManagedEndpointAllocationKey) => @@ -483,7 +485,6 @@ export const make = Effect.gen(function* () { db .select({ ...allocationSelection, - recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, recoveryEnvironmentPublicKey: relayManagedEndpointAllocations.recoveryEnvironmentPublicKey, linkedEnvironmentPublicKey: relayEnvironmentLinks.environmentPublicKey, @@ -504,15 +505,10 @@ export const make = Effect.gen(function* () { .pipe( Effect.map((rows) => rows.map( - ({ - recoveryEnabledAt, - recoveryEnvironmentPublicKey, - linkedEnvironmentPublicKey, - ...allocation - }) => ({ + ({ recoveryEnvironmentPublicKey, linkedEnvironmentPublicKey, ...allocation }) => ({ ...allocation, recoveryEnabled: - recoveryEnabledAt !== null && + allocation.recoveryEnabledAt !== null && recoveryEnvironmentPublicKey !== null && recoveryEnvironmentPublicKey === linkedEnvironmentPublicKey, }), diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index ead803b0d2e09..8a5a2294a2f9b 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -224,6 +224,7 @@ function makeAllocations(calls: AllocationCall[] = []) { updatedAt: `generation-${++generation}`, generation: 0, tunnelReleasedAt: null, + recoveryEnabledAt: null, }; allocations.set(allocationKey(input), allocation); return allocation; @@ -276,6 +277,10 @@ function makeAllocations(calls: AllocationCall[] = []) { return false; } recoveryEnabled.add(allocationKey(input)); + mutate(allocationKey(input), (current) => ({ + ...current, + recoveryEnabledAt: "2026-06-02T00:00:00.000Z", + })); return true; }), listByTunnelNames: (tunnelNames) => @@ -1279,6 +1284,45 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect.each([ + { when: "after", inactiveBefore: "2026-06-01T00:05:00.000Z", released: false }, + { when: "before", inactiveBefore: "2026-06-02T00:05:00.000Z", released: true }, + ] as const)( + "releases a down tunnel only if its host registered recovery before the cutoff ($when)", + ({ inactiveBefore, released }) => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + const layer = layerProvider( + makePersistentTunnelClient(tunnelCalls), + makeDnsClient(), + allocations, + ); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const; + yield* provider.provision({ ...key, origin }); + yield* allocations.enableRecovery({ + ...key, + tunnelId: "tunnel-id", + environmentPublicKey: "environment-public-key", + origin, + }); + + expect( + yield* provider.release({ + ...key, + expectedTunnelId: "tunnel-id", + expectedStatus: "down", + expectedInactiveBefore: inactiveBefore, + }), + ).toBe(released); + expect(tunnelCalls.some((call) => call.operation === "delete")).toBe(released); + }).pipe(Effect.provide(layer)); + }, + ); + it.effect.each([ { name: "the tunnel is gone", tunnelRemoved: true, released: true }, { name: "the tunnel still exists", tunnelRemoved: false, released: false }, diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 6a5d1f8bc3c9a..ddcc071fcff7a 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -993,6 +993,20 @@ export const make = Effect.gen(function* () { if (input.expectedTunnelId !== undefined && input.expectedTunnelId !== tunnelId) { return false; } + // A starting host registers recovery before its connector comes up, so + // Cloudflare still reports the tunnel's old inactivity. A registration + // after the cutoff means the host is about to reconnect this tunnel. + if (input.expectedInactiveBefore !== undefined && allocation.recoveryEnabledAt !== null) { + const registeredAt = DateTime.make(allocation.recoveryEnabledAt); + const cutoff = DateTime.make(input.expectedInactiveBefore); + if ( + Option.isSome(registeredAt) && + Option.isSome(cutoff) && + registeredAt.value.epochMilliseconds > cutoff.value.epochMilliseconds + ) { + return false; + } + } // Claim the release against the allocation's current generation before // touching Cloudflare. A provision racing this release (fast environment // restart) increments the generation when it records its tunnel, so a stale diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index a451d8f87e06c..d74355470a092 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -55,6 +55,7 @@ function allocation(input: { updatedAt: "2026-08-25T11:00:00.000Z", generation: 1, tunnelReleasedAt: null, + recoveryEnabledAt: null, recoveryEnabled: input.recoveryEnabled, }; } diff --git a/infra/relay/src/hooks/HeldHooks.test.ts b/infra/relay/src/hooks/HeldHooks.test.ts index 00a4c6bbf382d..a7f1b21f769b6 100644 --- a/infra/relay/src/hooks/HeldHooks.test.ts +++ b/infra/relay/src/hooks/HeldHooks.test.ts @@ -32,6 +32,7 @@ const allocationFor = ( dnsRecordId: "dns-record-id", readyAt: ready ? "2026-05-25T00:00:00.000Z" : null, tunnelReleasedAt: null, + recoveryEnabledAt: null, origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, updatedAt: "2026-05-25T00:00:00.000Z", generation: 1, diff --git a/infra/relay/src/hooks/HookForwarder.test.ts b/infra/relay/src/hooks/HookForwarder.test.ts index e3e9c111108ca..5c00759ed3aa4 100644 --- a/infra/relay/src/hooks/HookForwarder.test.ts +++ b/infra/relay/src/hooks/HookForwarder.test.ts @@ -63,6 +63,7 @@ const readyAllocation: ManagedEndpointAllocations.ManagedEndpointAllocation = { dnsRecordId: "dns-record-id", readyAt: "2026-05-25T00:00:00.000Z", tunnelReleasedAt: null, + recoveryEnabledAt: null, origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, updatedAt: "2026-05-25T00:00:00.000Z", generation: 1, From 17b659b1aa3b864a894b6e63bb336b0a88297b44 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Thu, 22 Oct 2026 06:09:44 -0500 Subject: [PATCH 02/16] chore(relay): format the recovery allocation listing Co-Authored-By: Claude Opus 5.5 --- infra/relay/src/environments/ManagedEndpointAllocations.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 285db12b695b7..94f770ac1b42a 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -505,7 +505,11 @@ export const make = Effect.gen(function* () { .pipe( Effect.map((rows) => rows.map( - ({ recoveryEnvironmentPublicKey, linkedEnvironmentPublicKey, ...allocation }) => ({ + ({ + recoveryEnvironmentPublicKey, + linkedEnvironmentPublicKey, + ...allocation + }) => ({ ...allocation, recoveryEnabled: allocation.recoveryEnabledAt !== null && From a192d2b15fd2a56407a2437898b52a84668779f6 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Thu, 22 Oct 2026 06:10:01 -0500 Subject: [PATCH 03/16] docs(relay): say when a host registers recovery relative to cloudflared Co-Authored-By: Claude Opus 5.5 --- infra/relay/src/environments/ManagedEndpointProvider.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index ddcc071fcff7a..0ac807f8376ff 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -993,9 +993,9 @@ export const make = Effect.gen(function* () { if (input.expectedTunnelId !== undefined && input.expectedTunnelId !== tunnelId) { return false; } - // A starting host registers recovery before its connector comes up, so - // Cloudflare still reports the tunnel's old inactivity. A registration - // after the cutoff means the host is about to reconnect this tunnel. + // A starting host registers recovery while cloudflared is still + // connecting, so Cloudflare keeps reporting the tunnel's old inactivity. + // A registration after the cutoff means the host is coming back to it. if (input.expectedInactiveBefore !== undefined && allocation.recoveryEnabledAt !== null) { const registeredAt = DateTime.make(allocation.recoveryEnabledAt); const cutoff = DateTime.make(input.expectedInactiveBefore); From e05942b541e2ed03914926df33440e696992dc01 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Thu, 22 Oct 2026 06:10:01 -0500 Subject: [PATCH 04/16] test(relay): give the API test allocations a recovery timestamp Co-Authored-By: Claude Opus 5.5 --- infra/relay/src/http/Api.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 5e9899edfa075..d365d154374ae 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -1019,6 +1019,7 @@ describe("relay managed tunnel recovery", () => { updatedAt: "replacement-generation", generation: 3, tunnelReleasedAt: null, + recoveryEnabledAt: null, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { @@ -1089,6 +1090,7 @@ describe("relay environment unlink", () => { dnsRecordId: "dns-1", readyAt: "2026-07-28T00:00:00.000Z", tunnelReleasedAt: null, + recoveryEnabledAt: null, origin: null, updatedAt: "2026-07-28T00:00:00.000Z", generation: 1, @@ -1127,6 +1129,7 @@ describe("relay environment unlink", () => { updatedAt: "generation-before-unlink", generation: 1, tunnelReleasedAt: null, + recoveryEnabledAt: null, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { @@ -1274,6 +1277,7 @@ describe("relay environment unlink", () => { updatedAt: "original-generation", generation: 1, tunnelReleasedAt: null, + recoveryEnabledAt: null, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { From 7b0de646e940449955a6331f1b4bfaf871ae01ea Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Fri, 23 Oct 2026 21:13:37 -0500 Subject: [PATCH 05/16] fix(relay): skip the recovery timestamp guard when recovery is unusable A relink that replaces the environment key keeps the tunnel's old recoveryEnabledAt. The reaper already treats that owner as legacy, but release still skipped the tunnel when the stale registration was newer than the legacy cutoff. The reaper now passes the owner's recoveryEnabled and release applies the guard only when it is not false. Co-Authored-By: Claude Opus 5.5 --- .../src/environments/ManagedEndpointProvider.test.ts | 9 ++++++++- infra/relay/src/environments/ManagedEndpointProvider.ts | 9 ++++++++- .../relay/src/environments/ManagedEndpointReaper.test.ts | 1 + infra/relay/src/environments/ManagedEndpointReaper.ts | 1 + 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 8a5a2294a2f9b..2a891db07dd42 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -1287,9 +1287,15 @@ describe("ManagedEndpointProvider", () => { it.effect.each([ { when: "after", inactiveBefore: "2026-06-01T00:05:00.000Z", released: false }, { when: "before", inactiveBefore: "2026-06-02T00:05:00.000Z", released: true }, + { + when: "after, with a replaced environment key", + inactiveBefore: "2026-06-01T00:05:00.000Z", + recoveryEnabled: false, + released: true, + }, ] as const)( "releases a down tunnel only if its host registered recovery before the cutoff ($when)", - ({ inactiveBefore, released }) => { + ({ inactiveBefore, released, ...rest }) => { const tunnelCalls: TunnelCall[] = []; const allocations = makeAllocations(); const layer = layerProvider( @@ -1316,6 +1322,7 @@ describe("ManagedEndpointProvider", () => { expectedTunnelId: "tunnel-id", expectedStatus: "down", expectedInactiveBefore: inactiveBefore, + ...("recoveryEnabled" in rest ? { expectedRecoveryEnabled: rest.recoveryEnabled } : {}), }), ).toBe(released); expect(tunnelCalls.some((call) => call.operation === "delete")).toBe(released); diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 0ac807f8376ff..69216633abcae 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -194,6 +194,7 @@ export class ManagedEndpointProvider extends Context.Service< readonly environmentId: string; readonly expectedTunnelId?: string; readonly expectedInactiveBefore?: string; + readonly expectedRecoveryEnabled?: boolean; readonly expectedStatus?: "inactive" | "down"; readonly markReleased?: boolean; }) => Effect.Effect; @@ -996,7 +997,13 @@ export const make = Effect.gen(function* () { // A starting host registers recovery while cloudflared is still // connecting, so Cloudflare keeps reporting the tunnel's old inactivity. // A registration after the cutoff means the host is coming back to it. - if (input.expectedInactiveBefore !== undefined && allocation.recoveryEnabledAt !== null) { + // A relink that replaced the environment key keeps the old timestamp, + // but that registration can no longer recover the tunnel. + if ( + input.expectedInactiveBefore !== undefined && + input.expectedRecoveryEnabled !== false && + allocation.recoveryEnabledAt !== null + ) { const registeredAt = DateTime.make(allocation.recoveryEnabledAt); const cutoff = DateTime.make(input.expectedInactiveBefore); if ( diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index d74355470a092..1d15d88f5bc33 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -488,6 +488,7 @@ describe("ManagedEndpointReaper", () => { expect(state.releases.every((release) => release.markReleased === true)).toBe(true); for (const release of state.releases) { expect(release.expectedInactiveBefore).toBe("2026-08-18T12:00:00.000Z"); + expect(release.expectedRecoveryEnabled).toBe(false); } }).pipe(Effect.provide(state.layer)); }); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index cbaf63577e739..3d3914d86134b 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -358,6 +358,7 @@ export const make = Effect.gen(function* () { environmentId: candidate.owner.environmentId, expectedTunnelId: candidate.tunnel.id, expectedInactiveBefore: candidate.inactiveBefore, + expectedRecoveryEnabled: candidate.owner.recoveryEnabled, expectedStatus: candidate.status, ...(candidate.legacy ? { markReleased: true } : {}), }) From 783ea31837bf68673eafb0f5604a6617d77ae561 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Fri, 23 Oct 2026 21:26:37 -0500 Subject: [PATCH 06/16] fix(relay): keep the recovery guard for a registration after the listing The reaper decides recoveryEnabled when it lists tunnels, but release reloads the allocation later. A host that registers recovery in between bumps the generation before release reads it, so the claim still succeeds, and the stale false skipped the timestamp guard and deleted the tunnel the host was coming back to. The reaper now also passes the listed recoveryEnabledAt, and release skips the guard only while the allocation still holds that timestamp. Co-Authored-By: Claude Opus 5.5 --- .../ManagedEndpointProvider.test.ts | 28 +++++++++++++++---- .../environments/ManagedEndpointProvider.ts | 11 ++++++-- .../ManagedEndpointReaper.test.ts | 1 + .../src/environments/ManagedEndpointReaper.ts | 1 + 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 2a891db07dd42..2435d1aa0b492 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -1284,18 +1284,29 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); - it.effect.each([ + it.effect.each<{ + readonly when: string; + readonly inactiveBefore: string; + readonly listedRecovery?: { readonly enabled: boolean; readonly enabledAt: string | null }; + readonly released: boolean; + }>([ { when: "after", inactiveBefore: "2026-06-01T00:05:00.000Z", released: false }, { when: "before", inactiveBefore: "2026-06-02T00:05:00.000Z", released: true }, { when: "after, with a replaced environment key", inactiveBefore: "2026-06-01T00:05:00.000Z", - recoveryEnabled: false, + listedRecovery: { enabled: false, enabledAt: "2026-06-02T00:00:00.000Z" }, released: true, }, - ] as const)( - "releases a down tunnel only if its host registered recovery before the cutoff ($when)", - ({ inactiveBefore, released, ...rest }) => { + { + when: "after, registered again since a replaced-key listing", + inactiveBefore: "2026-06-01T00:05:00.000Z", + listedRecovery: { enabled: false, enabledAt: "2026-05-01T00:00:00.000Z" }, + released: false, + }, + ])( + "releases a down tunnel unless a usable recovery registration came after the cutoff ($when)", + ({ inactiveBefore, listedRecovery, released }) => { const tunnelCalls: TunnelCall[] = []; const allocations = makeAllocations(); const layer = layerProvider( @@ -1322,7 +1333,12 @@ describe("ManagedEndpointProvider", () => { expectedTunnelId: "tunnel-id", expectedStatus: "down", expectedInactiveBefore: inactiveBefore, - ...("recoveryEnabled" in rest ? { expectedRecoveryEnabled: rest.recoveryEnabled } : {}), + ...(listedRecovery === undefined + ? {} + : { + expectedRecoveryEnabled: listedRecovery.enabled, + expectedRecoveryEnabledAt: listedRecovery.enabledAt, + }), }), ).toBe(released); expect(tunnelCalls.some((call) => call.operation === "delete")).toBe(released); diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 69216633abcae..00677943293a6 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -195,6 +195,7 @@ export class ManagedEndpointProvider extends Context.Service< readonly expectedTunnelId?: string; readonly expectedInactiveBefore?: string; readonly expectedRecoveryEnabled?: boolean; + readonly expectedRecoveryEnabledAt?: string | null; readonly expectedStatus?: "inactive" | "down"; readonly markReleased?: boolean; }) => Effect.Effect; @@ -998,10 +999,16 @@ export const make = Effect.gen(function* () { // connecting, so Cloudflare keeps reporting the tunnel's old inactivity. // A registration after the cutoff means the host is coming back to it. // A relink that replaced the environment key keeps the old timestamp, - // but that registration can no longer recover the tunnel. + // but that registration can no longer recover the tunnel. A newer + // registration since the caller's listing can, so the guard still + // applies to it. + const recoveryUnusable = + input.expectedRecoveryEnabled === false && + input.expectedRecoveryEnabledAt !== undefined && + input.expectedRecoveryEnabledAt === allocation.recoveryEnabledAt; if ( input.expectedInactiveBefore !== undefined && - input.expectedRecoveryEnabled !== false && + !recoveryUnusable && allocation.recoveryEnabledAt !== null ) { const registeredAt = DateTime.make(allocation.recoveryEnabledAt); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index 1d15d88f5bc33..fe1b30c46a5db 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -489,6 +489,7 @@ describe("ManagedEndpointReaper", () => { for (const release of state.releases) { expect(release.expectedInactiveBefore).toBe("2026-08-18T12:00:00.000Z"); expect(release.expectedRecoveryEnabled).toBe(false); + expect(release.expectedRecoveryEnabledAt).toBeNull(); } }).pipe(Effect.provide(state.layer)); }); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index 3d3914d86134b..d168feca82aa3 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -359,6 +359,7 @@ export const make = Effect.gen(function* () { expectedTunnelId: candidate.tunnel.id, expectedInactiveBefore: candidate.inactiveBefore, expectedRecoveryEnabled: candidate.owner.recoveryEnabled, + expectedRecoveryEnabledAt: candidate.owner.recoveryEnabledAt, expectedStatus: candidate.status, ...(candidate.legacy ? { markReleased: true } : {}), }) From 3768a6afb658ca7f6c4af5efdc483992c7abbea2 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Fri, 23 Oct 2026 21:30:23 -0500 Subject: [PATCH 07/16] fix(relay): release only the allocation generation the reaper listed Pinning the recovery timestamp still let other writes since the listing, such as a relink or a revoked link, reach release with a stale judgment. The reaper now passes the listed allocation generation instead, and release keeps the tunnel when the allocation changed since then. The next sweep decides with fresh data. Co-Authored-By: Claude Opus 5.5 --- .../ManagedEndpointProvider.test.ts | 21 ++++++++++++------- .../environments/ManagedEndpointProvider.ts | 21 +++++++++++-------- .../ManagedEndpointReaper.test.ts | 2 +- .../src/environments/ManagedEndpointReaper.ts | 2 +- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 2435d1aa0b492..16723c46c9e8d 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -1287,7 +1287,10 @@ describe("ManagedEndpointProvider", () => { it.effect.each<{ readonly when: string; readonly inactiveBefore: string; - readonly listedRecovery?: { readonly enabled: boolean; readonly enabledAt: string | null }; + readonly listed?: { + readonly recoveryEnabled: boolean; + readonly at: "before-registration" | "after-registration"; + }; readonly released: boolean; }>([ { when: "after", inactiveBefore: "2026-06-01T00:05:00.000Z", released: false }, @@ -1295,18 +1298,18 @@ describe("ManagedEndpointProvider", () => { { when: "after, with a replaced environment key", inactiveBefore: "2026-06-01T00:05:00.000Z", - listedRecovery: { enabled: false, enabledAt: "2026-06-02T00:00:00.000Z" }, + listed: { recoveryEnabled: false, at: "after-registration" }, released: true, }, { when: "after, registered again since a replaced-key listing", inactiveBefore: "2026-06-01T00:05:00.000Z", - listedRecovery: { enabled: false, enabledAt: "2026-05-01T00:00:00.000Z" }, + listed: { recoveryEnabled: false, at: "before-registration" }, released: false, }, ])( "releases a down tunnel unless a usable recovery registration came after the cutoff ($when)", - ({ inactiveBefore, listedRecovery, released }) => { + ({ inactiveBefore, listed, released }) => { const tunnelCalls: TunnelCall[] = []; const allocations = makeAllocations(); const layer = layerProvider( @@ -1320,12 +1323,16 @@ describe("ManagedEndpointProvider", () => { const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const; yield* provider.provision({ ...key, origin }); + const beforeRegistration = yield* allocations.get(key); yield* allocations.enableRecovery({ ...key, tunnelId: "tunnel-id", environmentPublicKey: "environment-public-key", origin, }); + const listing = + listed?.at === "before-registration" ? beforeRegistration : yield* allocations.get(key); + expect(listing).not.toBeNull(); expect( yield* provider.release({ @@ -1333,11 +1340,11 @@ describe("ManagedEndpointProvider", () => { expectedTunnelId: "tunnel-id", expectedStatus: "down", expectedInactiveBefore: inactiveBefore, - ...(listedRecovery === undefined + ...(listed === undefined || listing === null ? {} : { - expectedRecoveryEnabled: listedRecovery.enabled, - expectedRecoveryEnabledAt: listedRecovery.enabledAt, + expectedRecoveryEnabled: listed.recoveryEnabled, + expectedGeneration: listing.generation, }), }), ).toBe(released); diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 00677943293a6..09fcdcca42e65 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -195,7 +195,7 @@ export class ManagedEndpointProvider extends Context.Service< readonly expectedTunnelId?: string; readonly expectedInactiveBefore?: string; readonly expectedRecoveryEnabled?: boolean; - readonly expectedRecoveryEnabledAt?: string | null; + readonly expectedGeneration?: number; readonly expectedStatus?: "inactive" | "down"; readonly markReleased?: boolean; }) => Effect.Effect; @@ -995,20 +995,23 @@ export const make = Effect.gen(function* () { if (input.expectedTunnelId !== undefined && input.expectedTunnelId !== tunnelId) { return false; } + // The caller judged the allocation as it was when listed. Any write since + // then, such as a recovery registration or a relink, may have changed + // that judgment, so the next sweep decides with fresh data. + if ( + input.expectedGeneration !== undefined && + input.expectedGeneration !== allocation.generation + ) { + return false; + } // A starting host registers recovery while cloudflared is still // connecting, so Cloudflare keeps reporting the tunnel's old inactivity. // A registration after the cutoff means the host is coming back to it. // A relink that replaced the environment key keeps the old timestamp, - // but that registration can no longer recover the tunnel. A newer - // registration since the caller's listing can, so the guard still - // applies to it. - const recoveryUnusable = - input.expectedRecoveryEnabled === false && - input.expectedRecoveryEnabledAt !== undefined && - input.expectedRecoveryEnabledAt === allocation.recoveryEnabledAt; + // but that registration can no longer recover the tunnel. if ( input.expectedInactiveBefore !== undefined && - !recoveryUnusable && + input.expectedRecoveryEnabled !== false && allocation.recoveryEnabledAt !== null ) { const registeredAt = DateTime.make(allocation.recoveryEnabledAt); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index fe1b30c46a5db..560def48712ee 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -489,7 +489,7 @@ describe("ManagedEndpointReaper", () => { for (const release of state.releases) { expect(release.expectedInactiveBefore).toBe("2026-08-18T12:00:00.000Z"); expect(release.expectedRecoveryEnabled).toBe(false); - expect(release.expectedRecoveryEnabledAt).toBeNull(); + expect(release.expectedGeneration).toBe(1); } }).pipe(Effect.provide(state.layer)); }); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index d168feca82aa3..f0a78cff0fcae 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -359,7 +359,7 @@ export const make = Effect.gen(function* () { expectedTunnelId: candidate.tunnel.id, expectedInactiveBefore: candidate.inactiveBefore, expectedRecoveryEnabled: candidate.owner.recoveryEnabled, - expectedRecoveryEnabledAt: candidate.owner.recoveryEnabledAt, + expectedGeneration: candidate.owner.generation, expectedStatus: candidate.status, ...(candidate.legacy ? { markReleased: true } : {}), }) From 862be6398558f5fd0017d7b565a5928985b47bab Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Fri, 23 Oct 2026 21:32:50 -0500 Subject: [PATCH 08/16] docs(relay): name only allocation writes in the release generation pin Relinks and revocations update the links table and leave the allocation generation alone, so the comment no longer cites a relink. Co-Authored-By: Claude Opus 5.5 --- infra/relay/src/environments/ManagedEndpointProvider.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 09fcdcca42e65..f3ba98ceea4d7 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -995,9 +995,9 @@ export const make = Effect.gen(function* () { if (input.expectedTunnelId !== undefined && input.expectedTunnelId !== tunnelId) { return false; } - // The caller judged the allocation as it was when listed. Any write since - // then, such as a recovery registration or a relink, may have changed - // that judgment, so the next sweep decides with fresh data. + // The caller judged the allocation as it was when listed. A write since + // then, such as a recovery registration, may have made its tunnel + // recoverable again, so the next sweep decides with fresh data. if ( input.expectedGeneration !== undefined && input.expectedGeneration !== allocation.generation From fd1fe900d3b0eceb98be3748b8fa071874c79000 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Fri, 23 Oct 2026 23:00:40 -0500 Subject: [PATCH 09/16] fix(relay): a relink that restores the recovery key moves the allocation generation The reaper can list an allocation while its stored recovery key differs from the link, pass recoveryEnabled=false with that generation, and release it later without the recent-registration guard. A relink proven by the stored key restores recovery by writing only the link row, so the pinned generation still matched and release deleted the tunnel the relink had just provisioned. The link upsert now runs in a transaction that first bumps the allocation generation when its recovery key matches the new link key. The update takes the allocation row lock, which a claimed release holds while it deletes the tunnel, so the link cannot change in that window. Correction to 3768a6afb6: its message says the pin covers a relink or a revoked link. It covered only allocation writes. With this commit the reaper passes the listed allocation generation, and release keeps the tunnel when that allocation changed since the listing, which now includes a relink that restores the recovery key. A revocation alone cannot restore recovery and still bumps nothing. Co-Authored-By: Claude Opus 5.5 --- .../src/environments/EnvironmentLinks.test.ts | 76 +++++++++++++++- .../src/environments/EnvironmentLinks.ts | 90 +++++++++++-------- 2 files changed, 129 insertions(+), 37 deletions(-) diff --git a/infra/relay/src/environments/EnvironmentLinks.test.ts b/infra/relay/src/environments/EnvironmentLinks.test.ts index 9707a90034ff6..28a9ae740d21b 100644 --- a/infra/relay/src/environments/EnvironmentLinks.test.ts +++ b/infra/relay/src/environments/EnvironmentLinks.test.ts @@ -4,13 +4,15 @@ import * as Layer from "effect/Layer"; import { PgDialect } from "drizzle-orm/pg-core"; import * as RelayDb from "../db.ts"; -import { relayEnvironmentLinks } from "../persistence/schema.ts"; +import { relayEnvironmentLinks, relayManagedEndpointAllocations } from "../persistence/schema.ts"; import * as EnvironmentLinks from "./EnvironmentLinks.ts"; describe("EnvironmentLinks", () => { it.effect("carries the webhook-hold opt-in over only from links with the same key", () => { const inserted: Array> = []; const fakeDb = { + $client: { withTransaction: (effect: Effect.Effect) => effect }, + update: () => ({ set: () => ({ where: () => Effect.void }) }), insert: () => ({ values: (values: Record) => { inserted.push(values); @@ -45,6 +47,78 @@ describe("EnvironmentLinks", () => { ); }); + // The reaper can list an allocation while its recovery key differs from the + // link and release it later at that generation. A link that restores the key + // must move the generation before it commits, or the release deletes the + // tunnel the relink just provisioned. + it.effect("moves the allocation generation before a link restores its recovery key", () => { + const events: Array = []; + const allocationUpdates: Array<{ + readonly set: Record; + readonly where: unknown; + }> = []; + const fakeDb = { + $client: { + withTransaction: (effect: Effect.Effect) => + Effect.sync(() => events.push("begin")).pipe( + Effect.andThen(effect), + Effect.tap(() => Effect.sync(() => events.push("commit"))), + ), + }, + update: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + set: (set: Record) => ({ + where: (where: unknown) => + Effect.sync(() => { + events.push("allocation"); + allocationUpdates.push({ set, where }); + }), + }), + }; + }, + insert: (table: unknown) => { + expect(table).toBe(relayEnvironmentLinks); + return { + values: () => ({ + onConflictDoUpdate: () => Effect.sync(() => events.push("link")), + }), + }; + }, + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const links = yield* EnvironmentLinks.EnvironmentLinks; + yield* links.upsert({ + userId: "user-1", + request: { + notificationsEnabled: false, + liveActivitiesEnabled: false, + managedTunnelsEnabled: true, + } as never, + proof: { + environmentId: "env-1", + environmentPublicKey: "public-key-1", + descriptor: { label: "Laptop" }, + } as never, + endpoint: { httpBaseUrl: "https://a.example", wsBaseUrl: "wss://a.example" } as never, + }); + + expect(events).toEqual(["begin", "allocation", "link", "commit"]); + const dialect = new PgDialect(); + const where = dialect.sqlToQuery(allocationUpdates[0]?.where as never); + expect(where.sql).toContain('"recovery_environment_public_key" = $3'); + expect(where.params).toEqual(["user-1", "env-1", "public-key-1"]); + expect(dialect.sqlToQuery(allocationUpdates[0]?.set.generation as never).sql).toContain( + '"generation" + 1', + ); + }).pipe( + Effect.provide( + EnvironmentLinks.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, fakeDb))), + ), + ); + }); + it.effect("revokes only the active link owned by the requesting user", () => { const updateValues: Array> = []; const whereConditions: Array = []; diff --git a/infra/relay/src/environments/EnvironmentLinks.ts b/infra/relay/src/environments/EnvironmentLinks.ts index a7fdfc09d0353..7d921566d98a8 100644 --- a/infra/relay/src/environments/EnvironmentLinks.ts +++ b/infra/relay/src/environments/EnvironmentLinks.ts @@ -12,7 +12,7 @@ import * as Schema from "effect/Schema"; import { and, eq, isNull, or, sql } from "drizzle-orm"; import * as RelayDb from "../db.ts"; -import { relayEnvironmentLinks } from "../persistence/schema.ts"; +import { relayEnvironmentLinks, relayManagedEndpointAllocations } from "../persistence/schema.ts"; export interface RelayLinkedEnvironmentRecord extends RelayClientEnvironmentRecord { readonly environmentPublicKey: string; @@ -194,30 +194,41 @@ const make = Effect.gen(function* () { AND other.hold_webhooks_while_offline )`; const { endpoint } = input; - yield* db - .insert(relayEnvironmentLinks) - .values({ + const upsertError = (cause: unknown) => + new EnvironmentLinkUpsertPersistenceError({ userId: input.userId, environmentId, - holdWebhooksWhileOffline: inheritedHoldWebhooks, - environmentLabel: proof.descriptor.label, - environmentPublicKey: proof.environmentPublicKey, - endpointHttpBaseUrl: endpoint.httpBaseUrl, - endpointWsBaseUrl: endpoint.wsBaseUrl, - endpointProviderKind: endpoint.providerKind, - notificationsEnabled: request.notificationsEnabled, - liveActivitiesEnabled: request.liveActivitiesEnabled, - managedTunnelsEnabled: request.managedTunnelsEnabled, - createdByDeviceId: request.deviceId ?? null, - revokedAt: null, - createdAt: now, - updatedAt: now, - }) - .onConflictDoUpdate({ - target: [relayEnvironmentLinks.userId, relayEnvironmentLinks.environmentId], - set: { - environmentPublicKey: proof.environmentPublicKey, + ...(request.deviceId === undefined ? {} : { deviceId: request.deviceId }), + cause, + }); + // The reaper may have listed this allocation while its recovery key did + // not match the link, and then releases it without the recent + // registration guard. A link proven by that key makes recovery usable + // again, so it moves the generation the release was pinned to. The + // update runs first to take the allocation row lock, which a claimed + // release holds while it deletes the tunnel. + const writeLink = Effect.gen(function* () { + yield* db + .update(relayManagedEndpointAllocations) + .set({ generation: sql`${relayManagedEndpointAllocations.generation} + 1` }) + .where( + and( + eq(relayManagedEndpointAllocations.userId, input.userId), + eq(relayManagedEndpointAllocations.environmentId, environmentId), + eq( + relayManagedEndpointAllocations.recoveryEnvironmentPublicKey, + proof.environmentPublicKey, + ), + ), + ); + yield* db + .insert(relayEnvironmentLinks) + .values({ + userId: input.userId, + environmentId, + holdWebhooksWhileOffline: inheritedHoldWebhooks, environmentLabel: proof.descriptor.label, + environmentPublicKey: proof.environmentPublicKey, endpointHttpBaseUrl: endpoint.httpBaseUrl, endpointWsBaseUrl: endpoint.wsBaseUrl, endpointProviderKind: endpoint.providerKind, @@ -226,21 +237,28 @@ const make = Effect.gen(function* () { managedTunnelsEnabled: request.managedTunnelsEnabled, createdByDeviceId: request.deviceId ?? null, revokedAt: null, + createdAt: now, updatedAt: now, - holdWebhooksWhileOffline: inheritedHoldWebhooks, - }, - }) - .pipe( - Effect.mapError( - (cause) => - new EnvironmentLinkUpsertPersistenceError({ - userId: input.userId, - environmentId, - ...(request.deviceId === undefined ? {} : { deviceId: request.deviceId }), - cause, - }), - ), - ); + }) + .onConflictDoUpdate({ + target: [relayEnvironmentLinks.userId, relayEnvironmentLinks.environmentId], + set: { + environmentPublicKey: proof.environmentPublicKey, + environmentLabel: proof.descriptor.label, + endpointHttpBaseUrl: endpoint.httpBaseUrl, + endpointWsBaseUrl: endpoint.wsBaseUrl, + endpointProviderKind: endpoint.providerKind, + notificationsEnabled: request.notificationsEnabled, + liveActivitiesEnabled: request.liveActivitiesEnabled, + managedTunnelsEnabled: request.managedTunnelsEnabled, + createdByDeviceId: request.deviceId ?? null, + revokedAt: null, + updatedAt: now, + holdWebhooksWhileOffline: inheritedHoldWebhooks, + }, + }); + }); + yield* db.$client.withTransaction(writeLink).pipe(Effect.mapError(upsertError)); }), listDeliveryUsersForEnvironment: Effect.fn( From 794d5b8945448c3334a6a3344e384ff086f592ce Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Fri, 23 Oct 2026 23:11:15 -0500 Subject: [PATCH 10/16] fix(relay): write the link before the allocation bump The relink upsert updated the allocation row and then wrote the link row. enableRecovery takes the same two locks the other way round: its EXISTS subquery locks the link row FOR UPDATE before its UPDATE locks the allocation row. A same-key relink concurrent with tunnel registration or recovery could deadlock, and Postgres would abort one request. The upsert now writes the link first and bumps the allocation generation second, in the same transaction. That is the link-then-allocation order enableRecovery uses. Neither write is visible before commit, so the listing, get and claim windows stay closed, and the bump still waits on the allocation row lock a claimed release holds while it deletes the tunnel. Co-Authored-By: Claude Opus 5.5 --- .../src/environments/EnvironmentLinks.test.ts | 7 ++-- .../src/environments/EnvironmentLinks.ts | 34 ++++++++++--------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/infra/relay/src/environments/EnvironmentLinks.test.ts b/infra/relay/src/environments/EnvironmentLinks.test.ts index 28a9ae740d21b..3d39854b6a5d4 100644 --- a/infra/relay/src/environments/EnvironmentLinks.test.ts +++ b/infra/relay/src/environments/EnvironmentLinks.test.ts @@ -50,8 +50,9 @@ describe("EnvironmentLinks", () => { // The reaper can list an allocation while its recovery key differs from the // link and release it later at that generation. A link that restores the key // must move the generation before it commits, or the release deletes the - // tunnel the relink just provisioned. - it.effect("moves the allocation generation before a link restores its recovery key", () => { + // tunnel the relink just provisioned. The link write comes first, matching + // enableRecovery's link-then-allocation lock order so the two cannot deadlock. + it.effect("moves the allocation generation when a link restores its recovery key", () => { const events: Array = []; const allocationUpdates: Array<{ readonly set: Record; @@ -104,7 +105,7 @@ describe("EnvironmentLinks", () => { endpoint: { httpBaseUrl: "https://a.example", wsBaseUrl: "wss://a.example" } as never, }); - expect(events).toEqual(["begin", "allocation", "link", "commit"]); + expect(events).toEqual(["begin", "link", "allocation", "commit"]); const dialect = new PgDialect(); const where = dialect.sqlToQuery(allocationUpdates[0]?.where as never); expect(where.sql).toContain('"recovery_environment_public_key" = $3'); diff --git a/infra/relay/src/environments/EnvironmentLinks.ts b/infra/relay/src/environments/EnvironmentLinks.ts index 7d921566d98a8..742e3ff19cb25 100644 --- a/infra/relay/src/environments/EnvironmentLinks.ts +++ b/infra/relay/src/environments/EnvironmentLinks.ts @@ -204,23 +204,12 @@ const make = Effect.gen(function* () { // The reaper may have listed this allocation while its recovery key did // not match the link, and then releases it without the recent // registration guard. A link proven by that key makes recovery usable - // again, so it moves the generation the release was pinned to. The - // update runs first to take the allocation row lock, which a claimed - // release holds while it deletes the tunnel. + // again, so it moves the generation the release was pinned to. Both + // writes commit together, and the bump waits on the allocation row lock + // a claimed release holds while it deletes the tunnel. The link is + // written first to keep enableRecovery's order: its EXISTS locks the + // link row before its UPDATE locks the allocation row. const writeLink = Effect.gen(function* () { - yield* db - .update(relayManagedEndpointAllocations) - .set({ generation: sql`${relayManagedEndpointAllocations.generation} + 1` }) - .where( - and( - eq(relayManagedEndpointAllocations.userId, input.userId), - eq(relayManagedEndpointAllocations.environmentId, environmentId), - eq( - relayManagedEndpointAllocations.recoveryEnvironmentPublicKey, - proof.environmentPublicKey, - ), - ), - ); yield* db .insert(relayEnvironmentLinks) .values({ @@ -257,6 +246,19 @@ const make = Effect.gen(function* () { holdWebhooksWhileOffline: inheritedHoldWebhooks, }, }); + yield* db + .update(relayManagedEndpointAllocations) + .set({ generation: sql`${relayManagedEndpointAllocations.generation} + 1` }) + .where( + and( + eq(relayManagedEndpointAllocations.userId, input.userId), + eq(relayManagedEndpointAllocations.environmentId, environmentId), + eq( + relayManagedEndpointAllocations.recoveryEnvironmentPublicKey, + proof.environmentPublicKey, + ), + ), + ); }); yield* db.$client.withTransaction(writeLink).pipe(Effect.mapError(upsertError)); }), From 99183732519b91018156d1c94b15415e2d18b141 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Mon, 26 Oct 2026 12:05:51 +0000 Subject: [PATCH 11/16] refactor(relay): shorten reconnect cleanup rationale --- infra/relay/src/environments/EnvironmentLinks.test.ts | 5 ----- infra/relay/src/environments/EnvironmentLinks.ts | 10 ++-------- .../relay/src/environments/ManagedEndpointProvider.ts | 10 ++-------- 3 files changed, 4 insertions(+), 21 deletions(-) diff --git a/infra/relay/src/environments/EnvironmentLinks.test.ts b/infra/relay/src/environments/EnvironmentLinks.test.ts index 3d39854b6a5d4..792626c925237 100644 --- a/infra/relay/src/environments/EnvironmentLinks.test.ts +++ b/infra/relay/src/environments/EnvironmentLinks.test.ts @@ -47,11 +47,6 @@ describe("EnvironmentLinks", () => { ); }); - // The reaper can list an allocation while its recovery key differs from the - // link and release it later at that generation. A link that restores the key - // must move the generation before it commits, or the release deletes the - // tunnel the relink just provisioned. The link write comes first, matching - // enableRecovery's link-then-allocation lock order so the two cannot deadlock. it.effect("moves the allocation generation when a link restores its recovery key", () => { const events: Array = []; const allocationUpdates: Array<{ diff --git a/infra/relay/src/environments/EnvironmentLinks.ts b/infra/relay/src/environments/EnvironmentLinks.ts index 27655a1af3297..5b648522359bc 100644 --- a/infra/relay/src/environments/EnvironmentLinks.ts +++ b/infra/relay/src/environments/EnvironmentLinks.ts @@ -194,14 +194,8 @@ const make = Effect.gen(function* () { ...(request.deviceId === undefined ? {} : { deviceId: request.deviceId }), cause, }); - // The reaper may have listed this allocation while its recovery key did - // not match the link, and then releases it without the recent - // registration guard. A link proven by that key makes recovery usable - // again, so it moves the generation the release was pinned to. Both - // writes commit together, and the bump waits on the allocation row lock - // a claimed release holds while it deletes the tunnel. The link is - // written first to keep enableRecovery's order: its EXISTS locks the - // link row before its UPDATE locks the allocation row. + // Restoring the recovery key invalidates a stale reaper claim atomically. + // Lock the link before the allocation, matching enableRecovery. const writeLink = Effect.gen(function* () { yield* db .insert(relayEnvironmentLinks) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 5b3c755b52560..2b6c0a4bec9db 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -983,20 +983,14 @@ export const make = Effect.gen(function* () { if (input.expectedTunnelId !== undefined && input.expectedTunnelId !== tunnelId) { return false; } - // The caller judged the allocation as it was when listed. A write since - // then, such as a recovery registration, may have made its tunnel - // recoverable again, so the next sweep decides with fresh data. + // Recovery may become usable after the reaper lists the allocation. if ( input.expectedGeneration !== undefined && input.expectedGeneration !== allocation.generation ) { return false; } - // A starting host registers recovery while cloudflared is still - // connecting, so Cloudflare keeps reporting the tunnel's old inactivity. - // A registration after the cutoff means the host is coming back to it. - // A relink that replaced the environment key keeps the old timestamp, - // but that registration can no longer recover the tunnel. + // Cloudflare reports old inactivity while a registered host is reconnecting. if ( input.expectedInactiveBefore !== undefined && input.expectedRecoveryEnabled !== false && From a03f62251bbb35ed991d0be3f9b0d3b793213b06 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Wed, 18 Nov 2026 19:11:04 +0000 Subject: [PATCH 12/16] fix(relay): the sweep skips a reconnecting host's tunnel before counting it The reaper now skips a tunnel whose host registered usable recovery after the cutoff while it lists candidates. Dry-run no longer counts such a tunnel in wouldDelete, it no longer takes a slot in the deletion batch, and skippedReconnecting counts it. Recovery counts as usable only while the active link is a tunnel link, since the recovery endpoints reject any other link. The listing reports this as recoveryUsable without changing recoveryEnabled, so the legacy split stays as it was, and the reaper passes it to release() as expectedRecoveryUsable. The reaper and release() share one timestamp check, and release() keeps its own check for a registration that lands after the listing. Its comment now says when it returns false. --- .../ManagedEndpointAllocations.test.ts | 24 ++++++-- .../ManagedEndpointAllocations.ts | 33 +++++++++-- .../ManagedEndpointProvider.test.ts | 9 +-- .../environments/ManagedEndpointProvider.ts | 24 ++++---- .../ManagedEndpointReaper.test.ts | 55 ++++++++++++++++--- .../src/environments/ManagedEndpointReaper.ts | 14 ++++- 6 files changed, 123 insertions(+), 36 deletions(-) diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index 931e8a2b8893d..df7d771c51ca1 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -35,6 +35,7 @@ describe("ManagedEndpointAllocations", () => { recoveryEnabledAt: "2026-08-25T12:00:00.000Z", recoveryEnvironmentPublicKey: "current-key", linkedEnvironmentPublicKey: "current-key", + linkedEndpointProviderKind: "cloudflare_tunnel", }, { ...base, @@ -43,6 +44,7 @@ describe("ManagedEndpointAllocations", () => { recoveryEnabledAt: null, recoveryEnvironmentPublicKey: null, linkedEnvironmentPublicKey: "current-key", + linkedEndpointProviderKind: "cloudflare_tunnel", }, { ...base, @@ -51,6 +53,16 @@ describe("ManagedEndpointAllocations", () => { recoveryEnabledAt: "2026-08-25T12:00:00.000Z", recoveryEnvironmentPublicKey: "old-key", linkedEnvironmentPublicKey: "new-key", + linkedEndpointProviderKind: "cloudflare_tunnel", + }, + { + ...base, + environmentId: "environment-4", + tunnelId: "tunnel-4", + recoveryEnabledAt: "2026-08-25T12:00:00.000Z", + recoveryEnvironmentPublicKey: "current-key", + linkedEnvironmentPublicKey: "current-key", + linkedEndpointProviderKind: "manual", }, ]), }), @@ -65,12 +77,16 @@ describe("ManagedEndpointAllocations", () => { "first-tunnel", "second-tunnel", "third-tunnel", + "fourth-tunnel", ]); - expect(result.map((entry) => [entry.tunnelId, entry.recoveryEnabled])).toEqual([ - ["tunnel-1", true], - ["tunnel-2", false], - ["tunnel-3", false], + expect( + result.map((entry) => [entry.tunnelId, entry.recoveryEnabled, entry.recoveryUsable]), + ).toEqual([ + ["tunnel-1", true, true], + ["tunnel-2", false, false], + ["tunnel-3", false, false], + ["tunnel-4", true, false], ]); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 7c88457f42de6..ea72acf1492f5 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -30,10 +30,26 @@ export interface ManagedEndpointAllocation { export interface ManagedEndpointTunnelAllocation extends ManagedEndpointAllocation { readonly recoveryEnabled: boolean; + // The recovery endpoints accept only a tunnel link. + readonly recoveryUsable: boolean; } export const MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE = 500; +export function isRecoveryRegisteredAfter( + recoveryEnabledAt: string | null, + cutoff: DateTime.DateTime.Input, +): boolean { + if (recoveryEnabledAt === null) return false; + const registeredAt = DateTime.make(recoveryEnabledAt); + const cutoffAt = DateTime.make(cutoff); + return ( + Option.isSome(registeredAt) && + Option.isSome(cutoffAt) && + DateTime.isGreaterThan(registeredAt.value, cutoffAt.value) + ); +} + export function resolveReadyManagedEndpoint(input: { readonly allocation: ManagedEndpointAllocation; readonly baseDomain: string | undefined; @@ -475,6 +491,7 @@ export const make = Effect.gen(function* () { recoveryEnvironmentPublicKey: relayManagedEndpointAllocations.recoveryEnvironmentPublicKey, linkedEnvironmentPublicKey: relayEnvironmentLinks.environmentPublicKey, + linkedEndpointProviderKind: relayEnvironmentLinks.endpointProviderKind, }) .from(relayManagedEndpointAllocations) .leftJoin( @@ -495,14 +512,20 @@ export const make = Effect.gen(function* () { ({ recoveryEnvironmentPublicKey, linkedEnvironmentPublicKey, + linkedEndpointProviderKind, ...allocation - }) => ({ - ...allocation, - recoveryEnabled: + }) => { + const recoveryEnabled = allocation.recoveryEnabledAt !== null && recoveryEnvironmentPublicKey !== null && - recoveryEnvironmentPublicKey === linkedEnvironmentPublicKey, - }), + recoveryEnvironmentPublicKey === linkedEnvironmentPublicKey; + return { + ...allocation, + recoveryEnabled, + recoveryUsable: + recoveryEnabled && linkedEndpointProviderKind === "cloudflare_tunnel", + }; + }, ), ), Effect.mapError( diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 16723c46c9e8d..92e5138b619ca 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -290,6 +290,7 @@ function makeAllocations(calls: AllocationCall[] = []) { .map((allocation) => ({ ...allocation, recoveryEnabled: recoveryEnabled.has(allocationKey(allocation)), + recoveryUsable: recoveryEnabled.has(allocationKey(allocation)), })), ), claimRelease: (input) => @@ -1288,7 +1289,7 @@ describe("ManagedEndpointProvider", () => { readonly when: string; readonly inactiveBefore: string; readonly listed?: { - readonly recoveryEnabled: boolean; + readonly recoveryUsable: boolean; readonly at: "before-registration" | "after-registration"; }; readonly released: boolean; @@ -1298,13 +1299,13 @@ describe("ManagedEndpointProvider", () => { { when: "after, with a replaced environment key", inactiveBefore: "2026-06-01T00:05:00.000Z", - listed: { recoveryEnabled: false, at: "after-registration" }, + listed: { recoveryUsable: false, at: "after-registration" }, released: true, }, { when: "after, registered again since a replaced-key listing", inactiveBefore: "2026-06-01T00:05:00.000Z", - listed: { recoveryEnabled: false, at: "before-registration" }, + listed: { recoveryUsable: false, at: "before-registration" }, released: false, }, ])( @@ -1343,7 +1344,7 @@ describe("ManagedEndpointProvider", () => { ...(listed === undefined || listing === null ? {} : { - expectedRecoveryEnabled: listed.recoveryEnabled, + expectedRecoveryUsable: listed.recoveryUsable, expectedGeneration: listing.generation, }), }), diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 2b6c0a4bec9db..5d272abe19bff 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -179,13 +179,16 @@ export class ManagedEndpointProvider extends Context.Service< readonly environmentId: string; readonly target?: ManagedEndpointDeprovisionTarget | null; }) => Effect.Effect; - /** True means the connector token is dead; false means concurrent provisioning kept its tunnel live. */ + /** + * True means the connector token is dead. False means the tunnel or allocation changed, usable + * recovery came after the cutoff, the tunnel is no longer inactive, or it is still in use. + */ readonly release: (input: { readonly userId: string; readonly environmentId: string; readonly expectedTunnelId?: string; readonly expectedInactiveBefore?: string; - readonly expectedRecoveryEnabled?: boolean; + readonly expectedRecoveryUsable?: boolean; readonly expectedGeneration?: number; readonly expectedStatus?: "inactive" | "down"; readonly markReleased?: boolean; @@ -993,18 +996,13 @@ export const make = Effect.gen(function* () { // Cloudflare reports old inactivity while a registered host is reconnecting. if ( input.expectedInactiveBefore !== undefined && - input.expectedRecoveryEnabled !== false && - allocation.recoveryEnabledAt !== null + input.expectedRecoveryUsable !== false && + ManagedEndpointAllocations.isRecoveryRegisteredAfter( + allocation.recoveryEnabledAt, + input.expectedInactiveBefore, + ) ) { - const registeredAt = DateTime.make(allocation.recoveryEnabledAt); - const cutoff = DateTime.make(input.expectedInactiveBefore); - if ( - Option.isSome(registeredAt) && - Option.isSome(cutoff) && - registeredAt.value.epochMilliseconds > cutoff.value.epochMilliseconds - ) { - return false; - } + return false; } // Claim before deleting: concurrent provisioning can advance the generation and reuse the tunnel. const claimedGeneration = yield* allocations diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index 560def48712ee..263aade9447c8 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -42,6 +42,8 @@ function recoverableOwners( function allocation(input: { readonly tunnelId: string | null; readonly recoveryEnabled: boolean; + readonly recoveryUsable?: boolean; + readonly recoveryEnabledAt?: string; }): ManagedEndpointAllocations.ManagedEndpointTunnelAllocation { return { userId: "user-1", @@ -55,8 +57,9 @@ function allocation(input: { updatedAt: "2026-08-25T11:00:00.000Z", generation: 1, tunnelReleasedAt: null, - recoveryEnabledAt: null, + recoveryEnabledAt: input.recoveryEnabledAt ?? null, recoveryEnabled: input.recoveryEnabled, + recoveryUsable: input.recoveryUsable ?? input.recoveryEnabled, }; } @@ -278,10 +281,26 @@ describe("ManagedEndpointReaper", () => { status: "inactive", timestamp: "2026-08-25T10:59:00.000Z", }), + tunnel({ + id: "reconnecting", + suffix: "cccccccccccccccc", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), ], allocations: [ allocation({ tunnelId: "down-1", recoveryEnabled: true }), - allocation({ tunnelId: "inactive-1", recoveryEnabled: true }), + allocation({ + tunnelId: "inactive-1", + recoveryEnabled: true, + recoveryUsable: false, + recoveryEnabledAt: "2026-08-25T11:58:00.000Z", + }), + allocation({ + tunnelId: "reconnecting", + recoveryEnabled: true, + recoveryEnabledAt: "2026-08-25T11:58:00.000Z", + }), ], }); @@ -289,16 +308,20 @@ describe("ManagedEndpointReaper", () => { yield* TestClock.setTime(NOW_MILLIS); const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; expect(yield* reaper.sweep).toMatchObject({ - scanned: 2, + scanned: 3, + attempted: 2, + skippedReconnecting: 1, deleted: 2, skippedLegacy: 0, failed: 0, }); expect(state.deleted).toEqual(["down-1", "inactive-1"]); expect(state.releases.some((release) => release.markReleased === true)).toBe(false); - expect(state.releases.map((request) => request.expectedTunnelId)).toEqual([ - "down-1", - "inactive-1", + expect( + state.releases.map((request) => [request.expectedTunnelId, request.expectedRecoveryUsable]), + ).toEqual([ + ["down-1", true], + ["inactive-1", false], ]); expect(state.listRequests).toEqual([ { @@ -488,7 +511,7 @@ describe("ManagedEndpointReaper", () => { expect(state.releases.every((release) => release.markReleased === true)).toBe(true); for (const release of state.releases) { expect(release.expectedInactiveBefore).toBe("2026-08-18T12:00:00.000Z"); - expect(release.expectedRecoveryEnabled).toBe(false); + expect(release.expectedRecoveryUsable).toBe(false); expect(release.expectedGeneration).toBe(1); } }).pipe(Effect.provide(state.layer)); @@ -660,8 +683,21 @@ describe("ManagedEndpointReaper", () => { status: "down", timestamp: "2026-08-25T11:00:00.000Z", }), + tunnel({ + id: "reconnecting", + suffix: "bbbbbbbbbbbbbbbb", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [ + allocation({ tunnelId: "expired", recoveryEnabled: true }), + allocation({ + tunnelId: "reconnecting", + recoveryEnabled: true, + recoveryEnabledAt: "2026-08-25T11:58:00.000Z", + }), ], - allocations: [allocation({ tunnelId: "expired", recoveryEnabled: true })], }); return Effect.gen(function* () { @@ -669,10 +705,11 @@ describe("ManagedEndpointReaper", () => { const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; expect(yield* reaper.sweep).toMatchObject({ mode: "dry-run", - scanned: 1, + scanned: 2, attempted: 0, deleted: 0, wouldDelete: 1, + skippedReconnecting: 1, }); expect(state.deleted).toEqual([]); expect(state.releases).toEqual([]); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index 5f745f17f0256..de1e394ee2e9e 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -44,6 +44,7 @@ export interface ManagedEndpointSweepResult { readonly skippedOrphan: number; readonly skippedReplaced: number; readonly skippedUnrecorded: number; + readonly skippedReconnecting: number; readonly totalDown: number | null; readonly totalInactive: number | null; readonly failed: number; @@ -141,6 +142,7 @@ const emptyResult = ( skippedOrphan: 0, skippedReplaced: 0, skippedUnrecorded: 0, + skippedReconnecting: 0, totalDown: null, totalInactive: null, failed: 0, @@ -271,6 +273,7 @@ export const make = Effect.gen(function* () { let skippedOrphan = 0; let skippedReplaced = 0; let skippedUnrecorded = 0; + let skippedReconnecting = 0; let failed = 0; for (const { tunnel, status, cutoff } of uniqueExpired) { @@ -312,6 +315,14 @@ export const make = Effect.gen(function* () { if (legacyMode === "dry-run") continue; } else { if (mode === "off") continue; + // Cloudflare reports old inactivity while a registered host is reconnecting. + if ( + owner.recoveryUsable && + ManagedEndpointAllocations.isRecoveryRegisteredAfter(owner.recoveryEnabledAt, cutoff) + ) { + skippedReconnecting += 1; + continue; + } wouldDelete += 1; if (mode === "dry-run") continue; } @@ -346,7 +357,7 @@ export const make = Effect.gen(function* () { environmentId: candidate.owner.environmentId, expectedTunnelId: candidate.tunnel.id, expectedInactiveBefore: candidate.inactiveBefore, - expectedRecoveryEnabled: candidate.owner.recoveryEnabled, + expectedRecoveryUsable: candidate.owner.recoveryUsable, expectedGeneration: candidate.owner.generation, expectedStatus: candidate.status, ...(candidate.legacy ? { markReleased: true } : {}), @@ -395,6 +406,7 @@ export const make = Effect.gen(function* () { skippedOrphan, skippedReplaced, skippedUnrecorded, + skippedReconnecting, totalDown: totals.down, totalInactive: totals.inactive, failed, From 5b55597ce07638126634e49243591dcbbf4dd912 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Wed, 18 Nov 2026 19:11:04 +0000 Subject: [PATCH 13/16] fix(relay): an unchanged relink no longer cancels a tunnel release The link upsert first reads the existing link under a row lock in the same transaction, so the lock order stays link row, then allocation, as in enableRecovery. When no link exists yet, it inserts one, or locks the one a concurrent first link inserted. It moves the allocation generation only when the new link is a tunnel link and the old one was missing, revoked, held another key or was not a tunnel link, so a relink that keeps a usable link no longer makes a concurrent explicit release fail its claim. EnvironmentLinkUpsertPersistenceError now carries the stage that failed: the lock read, the link write, the generation update or the transaction itself. --- .../src/environments/EnvironmentLinks.test.ts | 216 ++++++++++++------ .../src/environments/EnvironmentLinks.ts | 101 +++++--- 2 files changed, 224 insertions(+), 93 deletions(-) diff --git a/infra/relay/src/environments/EnvironmentLinks.test.ts b/infra/relay/src/environments/EnvironmentLinks.test.ts index 792626c925237..da7e5129aef23 100644 --- a/infra/relay/src/environments/EnvironmentLinks.test.ts +++ b/infra/relay/src/environments/EnvironmentLinks.test.ts @@ -12,11 +12,14 @@ describe("EnvironmentLinks", () => { const inserted: Array> = []; const fakeDb = { $client: { withTransaction: (effect: Effect.Effect) => effect }, - update: () => ({ set: () => ({ where: () => Effect.void }) }), + select: () => ({ from: () => ({ where: () => ({ for: () => Effect.succeed([]) }) }) }), insert: () => ({ values: (values: Record) => { inserted.push(values); - return { onConflictDoUpdate: () => Effect.void }; + return { + onConflictDoNothing: () => ({ returning: () => Effect.succeed([{}]) }), + onConflictDoUpdate: () => Effect.void, + }; }, }), } as unknown as RelayDb.RelayDb["Service"]; @@ -47,73 +50,156 @@ describe("EnvironmentLinks", () => { ); }); - it.effect("moves the allocation generation when a link restores its recovery key", () => { - const events: Array = []; - const allocationUpdates: Array<{ - readonly set: Record; - readonly where: unknown; - }> = []; - const fakeDb = { - $client: { - withTransaction: (effect: Effect.Effect) => - Effect.sync(() => events.push("begin")).pipe( - Effect.andThen(effect), - Effect.tap(() => Effect.sync(() => events.push("commit"))), - ), - }, - update: (table: unknown) => { - expect(table).toBe(relayManagedEndpointAllocations); - return { - set: (set: Record) => ({ - where: (where: unknown) => - Effect.sync(() => { - events.push("allocation"); - allocationUpdates.push({ set, where }); + interface LinkRow { + readonly environmentPublicKey: string; + readonly endpointProviderKind: string; + readonly revokedAt: string | null; + } + const tunnelLink: LinkRow = { + environmentPublicKey: "public-key-1", + endpointProviderKind: "cloudflare_tunnel", + revokedAt: null, + }; + it.effect.each<{ + readonly when: string; + readonly previous?: LinkRow; + // The link a concurrent first link inserted after this lock read found none. + readonly raced?: LinkRow; + readonly providerKind?: string; + readonly moved: boolean; + }>([ + { when: "a first link", moved: true }, + { when: "a concurrent first link", raced: tunnelLink, moved: false }, + { + when: "a replaced key", + previous: { ...tunnelLink, environmentPublicKey: "public-key-0" }, + moved: true, + }, + { + when: "a revoked link", + previous: { ...tunnelLink, revokedAt: "2026-06-01T00:00:00.000Z" }, + moved: true, + }, + { + when: "a publish-only link", + previous: { ...tunnelLink, endpointProviderKind: "manual" }, + moved: true, + }, + { when: "an unchanged tunnel link", previous: tunnelLink, moved: false }, + { + when: "a relink to publish-only", + previous: { ...tunnelLink, environmentPublicKey: "public-key-0" }, + providerKind: "manual", + moved: false, + }, + ])( + "moves the allocation generation only when a link makes recovery usable again ($when)", + ({ previous, raced, providerKind = "cloudflare_tunnel", moved }) => { + const events: Array = []; + const locks: Array = []; + const allocationUpdates: Array<{ + readonly set: Record; + readonly where: unknown; + }> = []; + const fakeDb = { + $client: { + withTransaction: (effect: Effect.Effect) => + Effect.sync(() => events.push("begin")).pipe( + Effect.andThen(effect), + Effect.tap(() => Effect.sync(() => events.push("commit"))), + ), + }, + select: () => ({ + from: (table: unknown) => { + expect(table).toBe(relayEnvironmentLinks); + return { + where: (where: unknown) => ({ + for: (strength: string) => + Effect.sync(() => { + expect(strength).toBe("update"); + events.push("lock"); + locks.push(where); + const row = locks.length === 1 ? previous : raced; + return row === undefined ? [] : [row]; + }), }), - }), - }; - }, - insert: (table: unknown) => { - expect(table).toBe(relayEnvironmentLinks); - return { - values: () => ({ - onConflictDoUpdate: () => Effect.sync(() => events.push("link")), - }), - }; - }, - } as unknown as RelayDb.RelayDb["Service"]; + }; + }, + }), + update: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + set: (set: Record) => ({ + where: (where: unknown) => + Effect.sync(() => { + events.push("allocation"); + allocationUpdates.push({ set, where }); + }), + }), + }; + }, + insert: (table: unknown) => { + expect(table).toBe(relayEnvironmentLinks); + return { + values: () => ({ + onConflictDoNothing: () => ({ + returning: () => + Effect.sync(() => { + events.push("insert"); + return raced === undefined ? [{ environmentId: "env-1" }] : []; + }), + }), + onConflictDoUpdate: () => Effect.sync(() => events.push("link")), + }), + }; + }, + } as unknown as RelayDb.RelayDb["Service"]; - return Effect.gen(function* () { - const links = yield* EnvironmentLinks.EnvironmentLinks; - yield* links.upsert({ - userId: "user-1", - request: { - notificationsEnabled: false, - liveActivitiesEnabled: false, - managedTunnelsEnabled: true, - } as never, - proof: { - environmentId: "env-1", - environmentPublicKey: "public-key-1", - descriptor: { label: "Laptop" }, - } as never, - endpoint: { httpBaseUrl: "https://a.example", wsBaseUrl: "wss://a.example" } as never, - }); + return Effect.gen(function* () { + const links = yield* EnvironmentLinks.EnvironmentLinks; + yield* links.upsert({ + userId: "user-1", + request: { + notificationsEnabled: false, + liveActivitiesEnabled: false, + managedTunnelsEnabled: providerKind === "cloudflare_tunnel", + } as never, + proof: { + environmentId: "env-1", + environmentPublicKey: "public-key-1", + descriptor: { label: "Laptop" }, + } as never, + endpoint: { + httpBaseUrl: "https://a.example", + wsBaseUrl: "wss://a.example", + providerKind, + } as never, + }); - expect(events).toEqual(["begin", "link", "allocation", "commit"]); - const dialect = new PgDialect(); - const where = dialect.sqlToQuery(allocationUpdates[0]?.where as never); - expect(where.sql).toContain('"recovery_environment_public_key" = $3'); - expect(where.params).toEqual(["user-1", "env-1", "public-key-1"]); - expect(dialect.sqlToQuery(allocationUpdates[0]?.set.generation as never).sql).toContain( - '"generation" + 1', + expect(events).toEqual([ + "begin", + "lock", + ...(previous === undefined ? ["insert"] : []), + ...(raced === undefined ? [] : ["lock"]), + "link", + ...(moved ? ["allocation"] : []), + "commit", + ]); + const dialect = new PgDialect(); + expect(dialect.sqlToQuery(locks[0] as never).params).toEqual(["user-1", "env-1"]); + for (const { set, where } of allocationUpdates) { + const query = dialect.sqlToQuery(where as never); + expect(query.sql).toContain('"recovery_environment_public_key" = $3'); + expect(query.params).toEqual(["user-1", "env-1", "public-key-1"]); + expect(dialect.sqlToQuery(set.generation as never).sql).toContain('"generation" + 1'); + } + }).pipe( + Effect.provide( + EnvironmentLinks.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, fakeDb))), + ), ); - }).pipe( - Effect.provide( - EnvironmentLinks.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, fakeDb))), - ), - ); - }); + }, + ); it.effect("revokes only the active link owned by the requesting user", () => { const updateValues: Array> = []; diff --git a/infra/relay/src/environments/EnvironmentLinks.ts b/infra/relay/src/environments/EnvironmentLinks.ts index 5b648522359bc..7ef227765a209 100644 --- a/infra/relay/src/environments/EnvironmentLinks.ts +++ b/infra/relay/src/environments/EnvironmentLinks.ts @@ -30,6 +30,7 @@ export class EnvironmentLinkUpsertPersistenceError extends Schema.TaggedError - new EnvironmentLinkUpsertPersistenceError({ - userId: input.userId, - environmentId, - ...(request.deviceId === undefined ? {} : { deviceId: request.deviceId }), - cause, - }); - // Restoring the recovery key invalidates a stale reaper claim atomically. + const upsertError = + (stage: EnvironmentLinkUpsertPersistenceError["stage"]) => (cause: unknown) => + new EnvironmentLinkUpsertPersistenceError({ + userId: input.userId, + environmentId, + ...(request.deviceId === undefined ? {} : { deviceId: request.deviceId }), + stage, + cause, + }); + const linkValues = { + userId: input.userId, + environmentId, + holdWebhooksWhileOffline: inheritedHoldWebhooks, + environmentLabel: proof.descriptor.label, + environmentPublicKey: proof.environmentPublicKey, + endpointHttpBaseUrl: endpoint.httpBaseUrl, + endpointWsBaseUrl: endpoint.wsBaseUrl, + endpointProviderKind: endpoint.providerKind, + notificationsEnabled: request.notificationsEnabled, + liveActivitiesEnabled: request.liveActivitiesEnabled, + managedTunnelsEnabled: request.managedTunnelsEnabled, + createdByDeviceId: request.deviceId ?? null, + revokedAt: null, + createdAt: now, + updatedAt: now, + }; + const lockLink = db + .select({ + environmentPublicKey: relayEnvironmentLinks.environmentPublicKey, + endpointProviderKind: relayEnvironmentLinks.endpointProviderKind, + revokedAt: relayEnvironmentLinks.revokedAt, + }) + .from(relayEnvironmentLinks) + .where( + and( + eq(relayEnvironmentLinks.userId, input.userId), + eq(relayEnvironmentLinks.environmentId, environmentId), + ), + ) + .for("update") + .pipe(Effect.mapError(upsertError("link-lock"))); // Lock the link before the allocation, matching enableRecovery. const writeLink = Effect.gen(function* () { + let [previous] = yield* lockLink; + if (previous === undefined) { + // FOR UPDATE locks nothing for a missing link; a concurrent first link may insert it first. + const inserted = yield* db + .insert(relayEnvironmentLinks) + .values(linkValues) + .onConflictDoNothing() + .returning({ environmentId: relayEnvironmentLinks.environmentId }) + .pipe(Effect.mapError(upsertError("link-upsert"))); + if (inserted.length === 0) { + [previous] = yield* lockLink; + } + } yield* db .insert(relayEnvironmentLinks) - .values({ - userId: input.userId, - environmentId, - holdWebhooksWhileOffline: inheritedHoldWebhooks, - environmentLabel: proof.descriptor.label, - environmentPublicKey: proof.environmentPublicKey, - endpointHttpBaseUrl: endpoint.httpBaseUrl, - endpointWsBaseUrl: endpoint.wsBaseUrl, - endpointProviderKind: endpoint.providerKind, - notificationsEnabled: request.notificationsEnabled, - liveActivitiesEnabled: request.liveActivitiesEnabled, - managedTunnelsEnabled: request.managedTunnelsEnabled, - createdByDeviceId: request.deviceId ?? null, - revokedAt: null, - createdAt: now, - updatedAt: now, - }) + .values(linkValues) .onConflictDoUpdate({ target: [relayEnvironmentLinks.userId, relayEnvironmentLinks.environmentId], set: { @@ -232,7 +263,16 @@ const make = Effect.gen(function* () { updatedAt: now, holdWebhooksWhileOffline: inheritedHoldWebhooks, }, - }); + }) + .pipe(Effect.mapError(upsertError("link-upsert"))); + const recoveryWasUsable = + previous?.revokedAt === null && + previous.environmentPublicKey === proof.environmentPublicKey && + previous.endpointProviderKind === "cloudflare_tunnel"; + // Only a relink that makes recovery usable again invalidates a stale reaper claim. + if (recoveryWasUsable || endpoint.providerKind !== "cloudflare_tunnel") { + return; + } yield* db .update(relayManagedEndpointAllocations) .set({ generation: sql`${relayManagedEndpointAllocations.generation} + 1` }) @@ -245,9 +285,14 @@ const make = Effect.gen(function* () { proof.environmentPublicKey, ), ), - ); + ) + .pipe(Effect.mapError(upsertError("allocation-generation"))); }); - yield* db.$client.withTransaction(writeLink).pipe(Effect.mapError(upsertError)); + yield* db.$client.withTransaction(writeLink).pipe( + Effect.catchTags({ + SqlError: (cause) => Effect.fail(upsertError("transaction")(cause)), + }), + ); }), listDeliveryUsersForEnvironment: Effect.fn( From f8d0c85d7aee72edf408abae220284a8df50b23c Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Wed, 18 Nov 2026 19:11:04 +0000 Subject: [PATCH 14/16] test(relay): the release cutoff cases pass the reaper's listed recovery state Every row of the release cutoff table now passes the listed recovery state and generation, as the reaper does, so the sweep's normal input is covered. --- .../ManagedEndpointProvider.test.ts | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 92e5138b619ca..265dc92d7dfa8 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -1288,14 +1288,24 @@ describe("ManagedEndpointProvider", () => { it.effect.each<{ readonly when: string; readonly inactiveBefore: string; - readonly listed?: { + readonly listed: { readonly recoveryUsable: boolean; readonly at: "before-registration" | "after-registration"; }; readonly released: boolean; }>([ - { when: "after", inactiveBefore: "2026-06-01T00:05:00.000Z", released: false }, - { when: "before", inactiveBefore: "2026-06-02T00:05:00.000Z", released: true }, + { + when: "after", + inactiveBefore: "2026-06-01T00:05:00.000Z", + listed: { recoveryUsable: true, at: "after-registration" }, + released: false, + }, + { + when: "before", + inactiveBefore: "2026-06-02T00:05:00.000Z", + listed: { recoveryUsable: true, at: "after-registration" }, + released: true, + }, { when: "after, with a replaced environment key", inactiveBefore: "2026-06-01T00:05:00.000Z", @@ -1332,7 +1342,7 @@ describe("ManagedEndpointProvider", () => { origin, }); const listing = - listed?.at === "before-registration" ? beforeRegistration : yield* allocations.get(key); + listed.at === "before-registration" ? beforeRegistration : yield* allocations.get(key); expect(listing).not.toBeNull(); expect( @@ -1341,12 +1351,8 @@ describe("ManagedEndpointProvider", () => { expectedTunnelId: "tunnel-id", expectedStatus: "down", expectedInactiveBefore: inactiveBefore, - ...(listed === undefined || listing === null - ? {} - : { - expectedRecoveryUsable: listed.recoveryUsable, - expectedGeneration: listing.generation, - }), + expectedRecoveryUsable: listed.recoveryUsable, + ...(listing === null ? {} : { expectedGeneration: listing.generation }), }), ).toBe(released); expect(tunnelCalls.some((call) => call.operation === "delete")).toBe(released); From bb008bb9984b9c3238d9f61c5af9c4bd44d9ec2c Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Wed, 18 Nov 2026 19:47:19 +0000 Subject: [PATCH 15/16] fix(relay): a first link writes its row once When no link row exists, the insert already writes the whole row, so the upsert after it no longer runs. An insert that loses to a concurrent first link still locks that row and updates it, and a link that makes recovery usable again still moves the allocation generation. --- .../src/environments/EnvironmentLinks.test.ts | 2 +- .../src/environments/EnvironmentLinks.ts | 45 ++++++++++--------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/infra/relay/src/environments/EnvironmentLinks.test.ts b/infra/relay/src/environments/EnvironmentLinks.test.ts index da7e5129aef23..2017384a2acaa 100644 --- a/infra/relay/src/environments/EnvironmentLinks.test.ts +++ b/infra/relay/src/environments/EnvironmentLinks.test.ts @@ -181,7 +181,7 @@ describe("EnvironmentLinks", () => { "lock", ...(previous === undefined ? ["insert"] : []), ...(raced === undefined ? [] : ["lock"]), - "link", + ...(previous === undefined && raced === undefined ? [] : ["link"]), ...(moved ? ["allocation"] : []), "commit", ]); diff --git a/infra/relay/src/environments/EnvironmentLinks.ts b/infra/relay/src/environments/EnvironmentLinks.ts index 7ef227765a209..33f664dc8d33a 100644 --- a/infra/relay/src/environments/EnvironmentLinks.ts +++ b/infra/relay/src/environments/EnvironmentLinks.ts @@ -244,27 +244,30 @@ const make = Effect.gen(function* () { [previous] = yield* lockLink; } } - yield* db - .insert(relayEnvironmentLinks) - .values(linkValues) - .onConflictDoUpdate({ - target: [relayEnvironmentLinks.userId, relayEnvironmentLinks.environmentId], - set: { - environmentPublicKey: proof.environmentPublicKey, - environmentLabel: proof.descriptor.label, - endpointHttpBaseUrl: endpoint.httpBaseUrl, - endpointWsBaseUrl: endpoint.wsBaseUrl, - endpointProviderKind: endpoint.providerKind, - notificationsEnabled: request.notificationsEnabled, - liveActivitiesEnabled: request.liveActivitiesEnabled, - managedTunnelsEnabled: request.managedTunnelsEnabled, - createdByDeviceId: request.deviceId ?? null, - revokedAt: null, - updatedAt: now, - holdWebhooksWhileOffline: inheritedHoldWebhooks, - }, - }) - .pipe(Effect.mapError(upsertError("link-upsert"))); + // Still undefined only when the insert above wrote the row. + if (previous !== undefined) { + yield* db + .insert(relayEnvironmentLinks) + .values(linkValues) + .onConflictDoUpdate({ + target: [relayEnvironmentLinks.userId, relayEnvironmentLinks.environmentId], + set: { + environmentPublicKey: proof.environmentPublicKey, + environmentLabel: proof.descriptor.label, + endpointHttpBaseUrl: endpoint.httpBaseUrl, + endpointWsBaseUrl: endpoint.wsBaseUrl, + endpointProviderKind: endpoint.providerKind, + notificationsEnabled: request.notificationsEnabled, + liveActivitiesEnabled: request.liveActivitiesEnabled, + managedTunnelsEnabled: request.managedTunnelsEnabled, + createdByDeviceId: request.deviceId ?? null, + revokedAt: null, + updatedAt: now, + holdWebhooksWhileOffline: inheritedHoldWebhooks, + }, + }) + .pipe(Effect.mapError(upsertError("link-upsert"))); + } const recoveryWasUsable = previous?.revokedAt === null && previous.environmentPublicKey === proof.environmentPublicKey && From 5509c279ee70d891257292b83844ede5b717f1ad Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Fri, 20 Nov 2026 04:08:17 +0000 Subject: [PATCH 16/16] test(relay): add recoveryEnabledAt to the failed-unlink linker fixture --- infra/relay/src/environments/EnvironmentLinker.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index 23ae7c890fb94..b3950b39c4715 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -370,6 +370,7 @@ describe("EnvironmentLinker", () => { updatedAt: "2026-01-01T00:00:00.000Z", generation: 1, tunnelReleasedAt: null, + recoveryEnabledAt: null, }), deprovision: () => Effect.sync(() => touched.push("deprovision") > 0 && tornDown), upsert: () => Effect.sync(() => void touched.push("upsert")),