diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index c023703ef3a12..5e4e03a552db2 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -190,6 +190,7 @@ function makeAllocations( updatedAt: "2026-05-25T00:00:00.000Z", generation: 1, tunnelReleasedAt: null, + recoveryEnabledAt: null, }, ): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] { return { @@ -428,6 +429,7 @@ describe("EnvironmentConnector", () => { updatedAt: "2026-05-25T00:00:00.000Z", generation: 1, tunnelReleasedAt: null, + recoveryEnabledAt: null, }), }), ), @@ -487,6 +489,7 @@ describe("EnvironmentConnector", () => { updatedAt: "2026-05-25T00:00:00.000Z", generation: 2, tunnelReleasedAt: "2026-05-26T00:00:00.000Z", + recoveryEnabledAt: null, }); return Effect.gen(function* () { diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index 23ae7c890fb94..b3950b39c4715 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -370,6 +370,7 @@ describe("EnvironmentLinker", () => { updatedAt: "2026-01-01T00:00:00.000Z", generation: 1, tunnelReleasedAt: null, + recoveryEnabledAt: null, }), deprovision: () => Effect.sync(() => touched.push("deprovision") > 0 && tornDown), upsert: () => Effect.sync(() => void touched.push("upsert")), diff --git a/infra/relay/src/environments/EnvironmentLinker.ts b/infra/relay/src/environments/EnvironmentLinker.ts index c771f51841b49..cf56f3733c2c9 100644 --- a/infra/relay/src/environments/EnvironmentLinker.ts +++ b/infra/relay/src/environments/EnvironmentLinker.ts @@ -373,6 +373,7 @@ const make = Effect.gen(function* () { ? new EnvironmentLinks.EnvironmentLinkUpsertPersistenceError({ userId: input.userId, environmentId: verified.environmentId, + stage: "transaction", cause, }) : cause, diff --git a/infra/relay/src/environments/EnvironmentLinks.test.ts b/infra/relay/src/environments/EnvironmentLinks.test.ts index 46e26f4a76549..50ece3996ba51 100644 --- a/infra/relay/src/environments/EnvironmentLinks.test.ts +++ b/infra/relay/src/environments/EnvironmentLinks.test.ts @@ -10,7 +10,7 @@ import type { SQL } from "drizzle-orm"; import { PgDialect } from "drizzle-orm/pg-core"; import * as RelayDb from "../db.ts"; -import { relayEnvironmentLinks } from "../persistence/schema.ts"; +import { relayEnvironmentLinks, relayManagedEndpointAllocations } from "../persistence/schema.ts"; import * as EnvironmentLinks from "./EnvironmentLinks.ts"; describe("EnvironmentLinks", () => { @@ -18,9 +18,8 @@ describe("EnvironmentLinks", () => { const inserted: Array> = []; const revoked: Array<{ values: Record; condition: unknown }> = []; const fakeDb = { - $client: { - withTransaction: (effect: Effect.Effect) => effect, - }, + $client: { withTransaction: (effect: Effect.Effect) => effect }, + select: () => ({ from: () => ({ where: () => ({ for: () => Effect.succeed([]) }) }) }), execute: () => Effect.void, update: () => ({ set: (values: Record) => ({ @@ -33,7 +32,10 @@ describe("EnvironmentLinks", () => { insert: () => ({ values: (values: Record) => { inserted.push(values); - return { onConflictDoUpdate: () => Effect.void }; + return { + onConflictDoNothing: () => ({ returning: () => Effect.succeed([{}]) }), + onConflictDoUpdate: () => Effect.void, + }; }, }), } as unknown as RelayDb.RelayDb["Service"]; @@ -70,6 +72,167 @@ describe("EnvironmentLinks", () => { ); }); + interface LinkRow { + readonly environmentPublicKey: string; + readonly endpointProviderKind: string; + readonly revokedAt: string | null; + } + const tunnelLink: LinkRow = { + environmentPublicKey: "public-key-1", + endpointProviderKind: "cloudflare_tunnel", + revokedAt: null, + }; + it.effect.each<{ + readonly when: string; + readonly previous?: LinkRow; + // The link a concurrent first link inserted after this lock read found none. + readonly raced?: LinkRow; + readonly providerKind?: string; + readonly moved: boolean; + }>([ + { when: "a first link", moved: true }, + { + when: "a concurrent first link", + raced: { ...tunnelLink, environmentPublicKey: "public-key-0" }, + moved: true, + }, + { + when: "a replaced key", + previous: { ...tunnelLink, environmentPublicKey: "public-key-0" }, + moved: true, + }, + { + when: "a revoked link", + previous: { ...tunnelLink, revokedAt: "2026-06-01T00:00:00.000Z" }, + moved: true, + }, + { + when: "a publish-only link", + previous: { ...tunnelLink, endpointProviderKind: "manual" }, + moved: true, + }, + { when: "an unchanged tunnel link", previous: tunnelLink, moved: false }, + { + when: "a relink to publish-only", + previous: { ...tunnelLink, environmentPublicKey: "public-key-0" }, + providerKind: "manual", + moved: false, + }, + ])( + "moves the allocation generation only when a link makes recovery usable again ($when)", + ({ previous, raced, providerKind = "cloudflare_tunnel", moved }) => { + const events: Array = []; + const locks: Array = []; + const allocationUpdates: Array<{ + readonly set: Record; + readonly where: unknown; + }> = []; + const fakeDb = { + $client: { + withTransaction: (effect: Effect.Effect) => + Effect.sync(() => events.push("begin")).pipe( + Effect.andThen(effect), + Effect.tap(() => Effect.sync(() => events.push("commit"))), + ), + }, + select: () => ({ + from: (table: unknown) => { + expect(table).toBe(relayEnvironmentLinks); + return { + where: (where: unknown) => ({ + for: (strength: string) => + Effect.sync(() => { + expect(strength).toBe("update"); + events.push("lock"); + locks.push(where); + const row = locks.length === 1 ? previous : raced; + return row === undefined ? [] : [row]; + }), + }), + }; + }, + }), + execute: () => Effect.sync(() => events.push("advisory-lock")), + update: (table: unknown) => { + if (table === relayEnvironmentLinks) { + return { set: () => ({ where: () => Effect.sync(() => events.push("revoke")) }) }; + } + expect(table).toBe(relayManagedEndpointAllocations); + return { + set: (set: Record) => ({ + where: (where: unknown) => + Effect.sync(() => { + events.push("allocation"); + allocationUpdates.push({ set, where }); + }), + }), + }; + }, + insert: (table: unknown) => { + expect(table).toBe(relayEnvironmentLinks); + return { + values: () => ({ + onConflictDoNothing: () => ({ + returning: () => + Effect.sync(() => { + events.push("insert"); + return raced === undefined ? [{ environmentId: "env-1" }] : []; + }), + }), + onConflictDoUpdate: () => Effect.sync(() => events.push("link")), + }), + }; + }, + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const links = yield* EnvironmentLinks.EnvironmentLinks; + yield* links.upsert({ + userId: "user-1", + request: { + notificationsEnabled: false, + liveActivitiesEnabled: false, + managedTunnelsEnabled: providerKind === "cloudflare_tunnel", + } as never, + proof: { + environmentId: "env-1", + environmentPublicKey: "public-key-1", + descriptor: { label: "Laptop" }, + } as never, + endpoint: { + httpBaseUrl: "https://a.example", + wsBaseUrl: "wss://a.example", + providerKind, + } as never, + }); + + expect(events).toEqual([ + "begin", + "advisory-lock", + "lock", + ...(previous === undefined ? ["insert"] : []), + ...(raced === undefined ? [] : ["lock"]), + ...(previous === undefined && raced === undefined ? [] : ["link"]), + "revoke", + ...(moved ? ["allocation"] : []), + "commit", + ]); + const dialect = new PgDialect(); + expect(dialect.sqlToQuery(locks[0] as never).params).toEqual(["user-1", "env-1"]); + for (const { set, where } of allocationUpdates) { + const query = dialect.sqlToQuery(where as never); + expect(query.sql).toContain('"recovery_environment_public_key" = $3'); + expect(query.params).toEqual(["user-1", "env-1", "public-key-1"]); + expect(dialect.sqlToQuery(set.generation as never).sql).toContain('"generation" + 1'); + } + }).pipe( + Effect.provide( + EnvironmentLinks.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, fakeDb))), + ), + ); + }, + ); + it.effect("updates Live Activities only on the matching active link", () => Effect.gen(function* () { const pg = yield* Effect.acquireRelease( diff --git a/infra/relay/src/environments/EnvironmentLinks.ts b/infra/relay/src/environments/EnvironmentLinks.ts index 877e50f4f259a..4c560030048fb 100644 --- a/infra/relay/src/environments/EnvironmentLinks.ts +++ b/infra/relay/src/environments/EnvironmentLinks.ts @@ -12,7 +12,7 @@ import * as Schema from "effect/Schema"; import { and, eq, isNull, ne, or, sql } from "drizzle-orm"; import * as RelayDb from "../db.ts"; -import { relayEnvironmentLinks } from "../persistence/schema.ts"; +import { relayEnvironmentLinks, relayManagedEndpointAllocations } from "../persistence/schema.ts"; export interface RelayLinkedEnvironmentRecord extends RelayClientEnvironmentRecord { readonly environmentPublicKey: string; @@ -30,6 +30,13 @@ export class EnvironmentLinkUpsertPersistenceError extends Schema.TaggedError (cause: unknown) => + new EnvironmentLinkUpsertPersistenceError({ + userId: input.userId, + environmentId, + ...(request.deviceId === undefined ? {} : { deviceId: request.deviceId }), + stage, + cause, + }); + const linkValues = { + userId: input.userId, + environmentId, + holdWebhooksWhileOffline: inheritedHoldWebhooks, + environmentLabel: proof.descriptor.label, + environmentPublicKey: proof.environmentPublicKey, + endpointHttpBaseUrl: endpoint.httpBaseUrl, + endpointWsBaseUrl: endpoint.wsBaseUrl, + endpointProviderKind: endpoint.providerKind, + notificationsEnabled: request.notificationsEnabled, + liveActivitiesEnabled: request.liveActivitiesEnabled, + managedTunnelsEnabled: request.managedTunnelsEnabled, + createdByDeviceId: request.deviceId ?? null, + revokedAt: null, + createdAt: now, + updatedAt: now, + }; + const lockLink = db + .select({ + environmentPublicKey: relayEnvironmentLinks.environmentPublicKey, + endpointProviderKind: relayEnvironmentLinks.endpointProviderKind, + revokedAt: relayEnvironmentLinks.revokedAt, + }) + .from(relayEnvironmentLinks) + .where( + and( + eq(relayEnvironmentLinks.userId, input.userId), + eq(relayEnvironmentLinks.environmentId, environmentId), + ), + ) + .for("update") + .pipe(Effect.mapError(upsertError("link-lock"))); + // Lock the link before the allocation, matching enableRecovery. + const writeLink = Effect.gen(function* () { + // Serialize account switches for the same proven environment identity. + yield* db + .execute( + sql`select pg_advisory_xact_lock(hashtext(${environmentId}), hashtext(${proof.environmentPublicKey}))`, + ) + .pipe(Effect.mapError(upsertError("link-lock"))); + let [previous] = yield* lockLink; + if (previous === undefined) { + // FOR UPDATE locks nothing for a missing link; a concurrent first link may insert it first. + const inserted = yield* db + .insert(relayEnvironmentLinks) + .values(linkValues) + .onConflictDoNothing() + .returning({ environmentId: relayEnvironmentLinks.environmentId }) + .pipe(Effect.mapError(upsertError("link-upsert"))); + if (inserted.length === 0) { + [previous] = yield* lockLink; + } + } + // Still undefined only when the insert above wrote the row. + if (previous !== undefined) { + yield* db + .insert(relayEnvironmentLinks) + .values(linkValues) + .onConflictDoUpdate({ + target: [relayEnvironmentLinks.userId, relayEnvironmentLinks.environmentId], + set: { environmentPublicKey: proof.environmentPublicKey, + environmentLabel: proof.descriptor.label, endpointHttpBaseUrl: endpoint.httpBaseUrl, endpointWsBaseUrl: endpoint.wsBaseUrl, endpointProviderKind: endpoint.providerKind, @@ -216,50 +280,52 @@ const make = Effect.gen(function* () { managedTunnelsEnabled: request.managedTunnelsEnabled, createdByDeviceId: request.deviceId ?? null, revokedAt: null, - createdAt: now, updatedAt: now, - }) - .onConflictDoUpdate({ - target: [relayEnvironmentLinks.userId, relayEnvironmentLinks.environmentId], - set: { - environmentPublicKey: proof.environmentPublicKey, - environmentLabel: proof.descriptor.label, - endpointHttpBaseUrl: endpoint.httpBaseUrl, - endpointWsBaseUrl: endpoint.wsBaseUrl, - endpointProviderKind: endpoint.providerKind, - notificationsEnabled: request.notificationsEnabled, - liveActivitiesEnabled: request.liveActivitiesEnabled, - managedTunnelsEnabled: request.managedTunnelsEnabled, - createdByDeviceId: request.deviceId ?? null, - revokedAt: null, - updatedAt: now, - holdWebhooksWhileOffline: inheritedHoldWebhooks, - }, - }); - yield* db - .update(relayEnvironmentLinks) - .set({ revokedAt: now, updatedAt: now }) - .where( - and( - eq(relayEnvironmentLinks.environmentId, environmentId), - eq(relayEnvironmentLinks.environmentPublicKey, proof.environmentPublicKey), - ne(relayEnvironmentLinks.userId, input.userId), - isNull(relayEnvironmentLinks.revokedAt), - ), - ); - }), - ) - .pipe( - Effect.mapError( - (cause) => - new EnvironmentLinkUpsertPersistenceError({ - userId: input.userId, - environmentId, - ...(request.deviceId === undefined ? {} : { deviceId: request.deviceId }), - cause, - }), - ), - ); + holdWebhooksWhileOffline: inheritedHoldWebhooks, + }, + }) + .pipe(Effect.mapError(upsertError("link-upsert"))); + } + yield* db + .update(relayEnvironmentLinks) + .set({ revokedAt: now, updatedAt: now }) + .where( + and( + eq(relayEnvironmentLinks.environmentId, environmentId), + eq(relayEnvironmentLinks.environmentPublicKey, proof.environmentPublicKey), + ne(relayEnvironmentLinks.userId, input.userId), + isNull(relayEnvironmentLinks.revokedAt), + ), + ) + .pipe(Effect.mapError(upsertError("revoke-previous-links"))); + const recoveryWasUsable = + previous?.revokedAt === null && + previous.environmentPublicKey === proof.environmentPublicKey && + previous.endpointProviderKind === "cloudflare_tunnel"; + // Only a relink that makes recovery usable again invalidates a stale reaper claim. + if (recoveryWasUsable || endpoint.providerKind !== "cloudflare_tunnel") { + return; + } + yield* db + .update(relayManagedEndpointAllocations) + .set({ generation: sql`${relayManagedEndpointAllocations.generation} + 1` }) + .where( + and( + eq(relayManagedEndpointAllocations.userId, input.userId), + eq(relayManagedEndpointAllocations.environmentId, environmentId), + eq( + relayManagedEndpointAllocations.recoveryEnvironmentPublicKey, + proof.environmentPublicKey, + ), + ), + ) + .pipe(Effect.mapError(upsertError("allocation-generation"))); + }); + yield* db.$client.withTransaction(writeLink).pipe( + Effect.catchTags({ + SqlError: (cause) => Effect.fail(upsertError("transaction")(cause)), + }), + ); }), listDeliveryUsersForEnvironment: Effect.fn( diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index f0cd86bef9669..0dcea6528dbde 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -35,6 +35,7 @@ describe("ManagedEndpointAllocations", () => { recoveryEnabledAt: "2026-08-25T12:00:00.000Z", recoveryEnvironmentPublicKey: "current-key", linkedEnvironmentPublicKey: "current-key", + linkedEndpointProviderKind: "cloudflare_tunnel", }, { ...base, @@ -43,6 +44,7 @@ describe("ManagedEndpointAllocations", () => { recoveryEnabledAt: null, recoveryEnvironmentPublicKey: null, linkedEnvironmentPublicKey: "current-key", + linkedEndpointProviderKind: "cloudflare_tunnel", }, { ...base, @@ -51,6 +53,16 @@ describe("ManagedEndpointAllocations", () => { recoveryEnabledAt: "2026-08-25T12:00:00.000Z", recoveryEnvironmentPublicKey: "old-key", linkedEnvironmentPublicKey: "new-key", + linkedEndpointProviderKind: "cloudflare_tunnel", + }, + { + ...base, + environmentId: "environment-4", + tunnelId: "tunnel-4", + recoveryEnabledAt: "2026-08-25T12:00:00.000Z", + recoveryEnvironmentPublicKey: "current-key", + linkedEnvironmentPublicKey: "current-key", + linkedEndpointProviderKind: "manual", }, ]), }), @@ -65,12 +77,16 @@ describe("ManagedEndpointAllocations", () => { "first-tunnel", "second-tunnel", "third-tunnel", + "fourth-tunnel", ]); - expect(result.map((entry) => [entry.tunnelId, entry.recoveryEnabled])).toEqual([ - ["tunnel-1", true], - ["tunnel-2", false], - ["tunnel-3", false], + expect( + result.map((entry) => [entry.tunnelId, entry.recoveryEnabled, entry.recoveryUsable]), + ).toEqual([ + ["tunnel-1", true, true], + ["tunnel-2", false, false], + ["tunnel-3", false, false], + ["tunnel-4", true, false], ]); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 918c410e35176..22d346411a846 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -25,14 +25,31 @@ export interface ManagedEndpointAllocation { readonly updatedAt: string; readonly generation: number; readonly tunnelReleasedAt: string | null; + readonly recoveryEnabledAt: string | null; } export interface ManagedEndpointTunnelAllocation extends ManagedEndpointAllocation { readonly recoveryEnabled: boolean; + // The recovery endpoints accept only a tunnel link. + readonly recoveryUsable: boolean; } const MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE = 500; +export function isRecoveryRegisteredAfter( + recoveryEnabledAt: string | null, + cutoff: DateTime.DateTime.Input, +): boolean { + if (recoveryEnabledAt === null) return false; + const registeredAt = DateTime.make(recoveryEnabledAt); + const cutoffAt = DateTime.make(cutoff); + return ( + Option.isSome(registeredAt) && + Option.isSome(cutoffAt) && + DateTime.isGreaterThan(registeredAt.value, cutoffAt.value) + ); +} + export function resolveReadyManagedEndpoint(input: { readonly allocation: ManagedEndpointAllocation; readonly baseDomain: string | undefined; @@ -188,6 +205,7 @@ const allocationSelection = { updatedAt: relayManagedEndpointAllocations.updatedAt, generation: relayManagedEndpointAllocations.generation, tunnelReleasedAt: relayManagedEndpointAllocations.tunnelReleasedAt, + recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, }; const whereAllocation = (input: ManagedEndpointAllocationKey) => @@ -454,10 +472,10 @@ const make = Effect.gen(function* () { db .select({ ...allocationSelection, - recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, recoveryEnvironmentPublicKey: relayManagedEndpointAllocations.recoveryEnvironmentPublicKey, linkedEnvironmentPublicKey: relayEnvironmentLinks.environmentPublicKey, + linkedEndpointProviderKind: relayEnvironmentLinks.endpointProviderKind, }) .from(relayManagedEndpointAllocations) .leftJoin( @@ -476,17 +494,22 @@ const make = Effect.gen(function* () { Effect.map((rows) => rows.map( ({ - recoveryEnabledAt, recoveryEnvironmentPublicKey, linkedEnvironmentPublicKey, + linkedEndpointProviderKind, ...allocation - }) => ({ - ...allocation, - recoveryEnabled: - recoveryEnabledAt !== null && + }) => { + const recoveryEnabled = + allocation.recoveryEnabledAt !== null && recoveryEnvironmentPublicKey !== null && - recoveryEnvironmentPublicKey === linkedEnvironmentPublicKey, - }), + recoveryEnvironmentPublicKey === linkedEnvironmentPublicKey; + return { + ...allocation, + recoveryEnabled, + recoveryUsable: + recoveryEnabled && linkedEndpointProviderKind === "cloudflare_tunnel", + }; + }, ), ), Effect.mapError( diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 0b22b764e911f..eb6675ad835e5 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -254,6 +254,7 @@ function makeAllocations(calls: AllocationCall[] = []) { updatedAt: `generation-${++generation}`, generation: 0, tunnelReleasedAt: null, + recoveryEnabledAt: null, }; allocations.set(allocationKey(input), allocation); return allocation; @@ -307,6 +308,10 @@ function makeAllocations(calls: AllocationCall[] = []) { return false; } recoveryEnabled.add(allocationKey(input)); + mutate(allocationKey(input), (current) => ({ + ...current, + recoveryEnabledAt: "2026-06-02T00:00:00.000Z", + })); return true; }), listByTunnelNames: (tunnelNames) => @@ -316,6 +321,7 @@ function makeAllocations(calls: AllocationCall[] = []) { .map((allocation) => ({ ...allocation, recoveryEnabled: recoveryEnabled.has(allocationKey(allocation)), + recoveryUsable: recoveryEnabled.has(allocationKey(allocation)), })), ), claimRelease: (input) => @@ -1438,6 +1444,81 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect.each<{ + readonly when: string; + readonly inactiveBefore: string; + readonly listed: { + readonly recoveryUsable: boolean; + readonly at: "before-registration" | "after-registration"; + }; + readonly released: boolean; + }>([ + { + when: "after", + inactiveBefore: "2026-06-01T00:05:00.000Z", + listed: { recoveryUsable: true, at: "after-registration" }, + released: false, + }, + { + when: "before", + inactiveBefore: "2026-06-02T00:05:00.000Z", + listed: { recoveryUsable: true, at: "after-registration" }, + released: true, + }, + { + when: "after, with a replaced environment key", + inactiveBefore: "2026-06-01T00:05:00.000Z", + listed: { recoveryUsable: false, at: "after-registration" }, + released: true, + }, + { + when: "after, registered again since a replaced-key listing", + inactiveBefore: "2026-06-01T00:05:00.000Z", + listed: { recoveryUsable: false, at: "before-registration" }, + released: false, + }, + ])( + "releases a down tunnel unless a usable recovery registration came after the cutoff ($when)", + ({ inactiveBefore, listed, released }) => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + const layer = layerProvider( + makePersistentTunnelClient(tunnelCalls), + makeDnsClient(), + allocations, + ); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const; + yield* provider.provision({ ...key, origin }); + const beforeRegistration = yield* allocations.get(key); + yield* allocations.enableRecovery({ + ...key, + tunnelId: "tunnel-id", + environmentPublicKey: "environment-public-key", + origin, + }); + const listing = + listed.at === "before-registration" ? beforeRegistration : yield* allocations.get(key); + expect(listing).not.toBeNull(); + + expect( + yield* provider.release({ + ...key, + expectedTunnelId: "tunnel-id", + expectedStatus: "down", + expectedInactiveBefore: inactiveBefore, + expectedRecoveryUsable: listed.recoveryUsable, + ...(listing === null ? {} : { expectedGeneration: listing.generation }), + }), + ).toBe(released); + expect(tunnelCalls.some((call) => call.operation === "delete")).toBe(released); + }).pipe(Effect.provide(layer)); + }, + ); + it.effect.each([ { name: "the tunnel is gone", tunnelRemoved: true, released: true }, { name: "the tunnel still exists", tunnelRemoved: false, released: false }, diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index b2946c980358e..1b334dd9655ef 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -179,12 +179,17 @@ export class ManagedEndpointProvider extends Context.Service< readonly environmentId: string; readonly target?: ManagedEndpointDeprovisionTarget | null; }) => Effect.Effect; - /** True means the connector token is dead; false means concurrent provisioning kept its tunnel live. */ + /** + * True means the connector token is dead. False means the tunnel or allocation changed, usable + * recovery came after the cutoff, the tunnel is no longer inactive, or it is still in use. + */ readonly release: (input: { readonly userId: string; readonly environmentId: string; readonly expectedTunnelId?: string; readonly expectedInactiveBefore?: string; + readonly expectedRecoveryUsable?: boolean; + readonly expectedGeneration?: number; readonly expectedStatus?: "inactive" | "down"; readonly markReleased?: boolean; }) => Effect.Effect; @@ -1045,6 +1050,24 @@ const make = Effect.gen(function* () { if (input.expectedTunnelId !== undefined && input.expectedTunnelId !== tunnelId) { return false; } + // Recovery may become usable after the reaper lists the allocation. + if ( + input.expectedGeneration !== undefined && + input.expectedGeneration !== allocation.generation + ) { + return false; + } + // Cloudflare reports old inactivity while a registered host is reconnecting. + if ( + input.expectedInactiveBefore !== undefined && + input.expectedRecoveryUsable !== false && + ManagedEndpointAllocations.isRecoveryRegisteredAfter( + allocation.recoveryEnabledAt, + input.expectedInactiveBefore, + ) + ) { + return false; + } // Claim before deleting: concurrent provisioning can advance the generation and reuse the tunnel. const claimedGeneration = yield* allocations .claimRelease({ diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index 5ce24e6b7e5c1..c749fdf710bb3 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -42,6 +42,8 @@ function recoverableOwners( function allocation(input: { readonly tunnelId: string | null; readonly recoveryEnabled: boolean; + readonly recoveryUsable?: boolean; + readonly recoveryEnabledAt?: string; }): ManagedEndpointAllocations.ManagedEndpointTunnelAllocation { return { userId: "user-1", @@ -55,7 +57,9 @@ function allocation(input: { updatedAt: "2026-08-25T11:00:00.000Z", generation: 1, tunnelReleasedAt: null, + recoveryEnabledAt: input.recoveryEnabledAt ?? null, recoveryEnabled: input.recoveryEnabled, + recoveryUsable: input.recoveryUsable ?? input.recoveryEnabled, }; } @@ -277,10 +281,26 @@ describe("ManagedEndpointReaper", () => { status: "inactive", timestamp: "2026-08-25T10:59:00.000Z", }), + tunnel({ + id: "reconnecting", + suffix: "cccccccccccccccc", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), ], allocations: [ allocation({ tunnelId: "down-1", recoveryEnabled: true }), - allocation({ tunnelId: "inactive-1", recoveryEnabled: true }), + allocation({ + tunnelId: "inactive-1", + recoveryEnabled: true, + recoveryUsable: false, + recoveryEnabledAt: "2026-08-25T11:58:00.000Z", + }), + allocation({ + tunnelId: "reconnecting", + recoveryEnabled: true, + recoveryEnabledAt: "2026-08-25T11:58:00.000Z", + }), ], }); @@ -288,16 +308,20 @@ describe("ManagedEndpointReaper", () => { yield* TestClock.setTime(NOW_MILLIS); const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; expect(yield* reaper.sweep).toMatchObject({ - scanned: 2, + scanned: 3, + attempted: 2, + skippedReconnecting: 1, deleted: 2, skippedLegacy: 0, failed: 0, }); expect(state.deleted).toEqual(["down-1", "inactive-1"]); expect(state.releases.some((release) => release.markReleased === true)).toBe(false); - expect(state.releases.map((request) => request.expectedTunnelId)).toEqual([ - "down-1", - "inactive-1", + expect( + state.releases.map((request) => [request.expectedTunnelId, request.expectedRecoveryUsable]), + ).toEqual([ + ["down-1", true], + ["inactive-1", false], ]); expect(state.listRequests).toEqual([ { @@ -487,6 +511,8 @@ describe("ManagedEndpointReaper", () => { expect(state.releases.every((release) => release.markReleased === true)).toBe(true); for (const release of state.releases) { expect(release.expectedInactiveBefore).toBe("2026-08-18T12:00:00.000Z"); + expect(release.expectedRecoveryUsable).toBe(false); + expect(release.expectedGeneration).toBe(1); } }).pipe(Effect.provide(state.layer)); }); @@ -657,8 +683,21 @@ describe("ManagedEndpointReaper", () => { status: "down", timestamp: "2026-08-25T11:00:00.000Z", }), + tunnel({ + id: "reconnecting", + suffix: "bbbbbbbbbbbbbbbb", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [ + allocation({ tunnelId: "expired", recoveryEnabled: true }), + allocation({ + tunnelId: "reconnecting", + recoveryEnabled: true, + recoveryEnabledAt: "2026-08-25T11:58:00.000Z", + }), ], - allocations: [allocation({ tunnelId: "expired", recoveryEnabled: true })], }); return Effect.gen(function* () { @@ -666,10 +705,11 @@ describe("ManagedEndpointReaper", () => { const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; expect(yield* reaper.sweep).toMatchObject({ mode: "dry-run", - scanned: 1, + scanned: 2, attempted: 0, deleted: 0, wouldDelete: 1, + skippedReconnecting: 1, }); expect(state.deleted).toEqual([]); expect(state.releases).toEqual([]); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index 71442b536d007..4edd4d8b9817f 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -47,6 +47,7 @@ export interface ManagedEndpointSweepResult { readonly skippedOrphan: number; readonly skippedReplaced: number; readonly skippedUnrecorded: number; + readonly skippedReconnecting: number; readonly totalDown: number | null; readonly totalInactive: number | null; readonly failed: number; @@ -144,6 +145,7 @@ const emptyResult = ( skippedOrphan: 0, skippedReplaced: 0, skippedUnrecorded: 0, + skippedReconnecting: 0, totalDown: null, totalInactive: null, failed: 0, @@ -277,6 +279,7 @@ const make = Effect.gen(function* () { let skippedOrphan = 0; let skippedReplaced = 0; let skippedUnrecorded = 0; + let skippedReconnecting = 0; let failed = 0; for (const { tunnel, status, cutoff } of uniqueExpired) { @@ -318,6 +321,14 @@ const make = Effect.gen(function* () { if (legacyMode === "dry-run") continue; } else { if (mode === "off") continue; + // Cloudflare reports old inactivity while a registered host is reconnecting. + if ( + owner.recoveryUsable && + ManagedEndpointAllocations.isRecoveryRegisteredAfter(owner.recoveryEnabledAt, cutoff) + ) { + skippedReconnecting += 1; + continue; + } wouldDelete += 1; if (mode === "dry-run") continue; } @@ -352,6 +363,8 @@ const make = Effect.gen(function* () { environmentId: candidate.owner.environmentId, expectedTunnelId: candidate.tunnel.id, expectedInactiveBefore: candidate.inactiveBefore, + expectedRecoveryUsable: candidate.owner.recoveryUsable, + expectedGeneration: candidate.owner.generation, expectedStatus: candidate.status, ...(candidate.legacy ? { markReleased: true } : {}), }) @@ -399,6 +412,7 @@ const make = Effect.gen(function* () { skippedOrphan, skippedReplaced, skippedUnrecorded, + skippedReconnecting, totalDown: totals.down, totalInactive: totals.inactive, failed, diff --git a/infra/relay/src/hooks/HeldHooks.test.ts b/infra/relay/src/hooks/HeldHooks.test.ts index f07551b17207e..e9403c759ed74 100644 --- a/infra/relay/src/hooks/HeldHooks.test.ts +++ b/infra/relay/src/hooks/HeldHooks.test.ts @@ -32,6 +32,7 @@ const allocationFor = ( dnsRecordId: "dns-record-id", readyAt: ready ? "2026-05-25T00:00:00.000Z" : null, tunnelReleasedAt: null, + recoveryEnabledAt: null, origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, updatedAt: "2026-05-25T00:00:00.000Z", generation: 1, diff --git a/infra/relay/src/hooks/HookForwarder.test.ts b/infra/relay/src/hooks/HookForwarder.test.ts index 15090d3bd8cba..328a0af82077d 100644 --- a/infra/relay/src/hooks/HookForwarder.test.ts +++ b/infra/relay/src/hooks/HookForwarder.test.ts @@ -63,6 +63,7 @@ const readyAllocation: ManagedEndpointAllocations.ManagedEndpointAllocation = { dnsRecordId: "dns-record-id", readyAt: "2026-05-25T00:00:00.000Z", tunnelReleasedAt: null, + recoveryEnabledAt: null, origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, updatedAt: "2026-05-25T00:00:00.000Z", generation: 1, diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 36582f0d1fd13..2399ee4264aa4 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -1034,6 +1034,7 @@ describe("relay managed tunnel recovery", () => { updatedAt: "replacement-generation", generation: 3, tunnelReleasedAt: null, + recoveryEnabledAt: null, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { @@ -1106,6 +1107,7 @@ describe("relay environment unlink", () => { dnsRecordId: "dns-1", readyAt: "2026-07-28T00:00:00.000Z", tunnelReleasedAt: null, + recoveryEnabledAt: null, origin: null, updatedAt: "2026-07-28T00:00:00.000Z", generation: 1, @@ -1161,6 +1163,7 @@ describe("relay environment unlink", () => { updatedAt: "generation-before-unlink", generation: 1, tunnelReleasedAt: null, + recoveryEnabledAt: null, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { @@ -1308,6 +1311,7 @@ describe("relay environment unlink", () => { updatedAt: "original-generation", generation: 1, tunnelReleasedAt: null, + recoveryEnabledAt: null, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () {