From a35b7e148f07cbb51c0278b318746b27f2776e25 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 19 Nov 2026 18:26:21 -0800 Subject: [PATCH 1/4] fix(relay): refuse to relink an active environment under a different key Linking verified the proof against the key inside it and never compared that key with the active link's. A relink of the same environment id in the same account replaced the trusted key and reused the existing tunnel with an origin the new key signed. The relay now rejects it; unlinking first releases the tunnel, so nothing of the old link carries over. Co-Authored-By: Claude Fable 5.2 --- apps/server/src/cloud/relayResponse.ts | 6 ++- docs/internals/t3-connect.md | 4 ++ .../environments/EnvironmentLinker.test.ts | 42 ++++++++++++++++++- .../src/environments/EnvironmentLinker.ts | 16 +++++++ packages/contracts/src/relay.ts | 1 + 5 files changed, 67 insertions(+), 2 deletions(-) diff --git a/apps/server/src/cloud/relayResponse.ts b/apps/server/src/cloud/relayResponse.ts index 2074212d83172..ac16308deb780 100644 --- a/apps/server/src/cloud/relayResponse.ts +++ b/apps/server/src/cloud/relayResponse.ts @@ -65,8 +65,12 @@ function recoveryHint(error: RelayProtectedError): string { return "Unlink an unused environment in T3 Connect, then restart T3 Code on this machine."; case "RelayAuthInvalidError": return "Run `t3 connect login` to check this machine's authorization. If the stored credential was revoked, sign out with `t3 connect logout`, then run `t3 connect` again. Restart T3 Code after signing in."; - case "RelayEnvironmentLinkProofExpiredError": case "RelayEnvironmentLinkProofInvalidError": + if (error.reason === "environment_key_mismatch") { + return "T3 Connect already has this environment linked with a different key. Unlink it in T3 Connect, then restart T3 Code on this machine."; + } + return "Check this machine's date and time, update T3 Code, then restart it."; + case "RelayEnvironmentLinkProofExpiredError": return "Check this machine's date and time, update T3 Code, then restart it."; default: return "Retry when the relay is available. If this continues, include the trace ID when reporting it."; diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index ae3938fd0cd7b..7237df5cfe381 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -86,6 +86,10 @@ Managed tunnels expose only this server's own loopback HTTP origin. The link proof signs that origin from the server's listening address and ignores the caller's, so any client holding `relay:configure`, local or remote, can link without being able to point the tunnel at another service on the host. The +relay refuses to relink an active environment id under a different key: an +account bearer alone would otherwise take over that environment's tunnel and +trusted key with an origin it signed itself. Unlinking first releases the +tunnel. The relay resolves endpoints from its own managed allocations rather than a caller-supplied URL. Health and mint requests must not follow redirects. These restrictions keep endpoint discovery diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index 40ba2220dbfaa..3fca0bca8cc5e 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -112,6 +112,7 @@ const makeRequest = Effect.gen(function* () { function layerTest(input?: { readonly upsert?: EnvironmentLinks.EnvironmentLinks["Service"]["upsert"]; + readonly getForUser?: EnvironmentLinks.EnvironmentLinks["Service"]["getForUser"]; readonly consume?: DpopProofs.DpopProofReplay["Service"]["consume"]; readonly deprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["deprovision"]; }) { @@ -130,7 +131,7 @@ function layerTest(input?: { upsert: input?.upsert ?? (() => Effect.void), listDeliveryUsersForEnvironment: () => Effect.succeed([]), listForUser: () => Effect.succeed([]), - getForUser: () => Effect.succeed(null), + getForUser: input?.getForUser ?? (() => Effect.succeed(null)), findActiveManagedForEnvironment: () => Effect.succeed([]), updateLiveActivityPreference: () => Effect.die("unexpected Live Activity preference update"), @@ -293,6 +294,45 @@ describe("EnvironmentLinker", () => { ); }); + it.effect("refuses to move an active link to a different environment key", () => { + const touched: Array = []; + const activeLink = (environmentPublicKey: string) => ({ + environmentId: "env-link-test" as RelayEnvironmentLinkProofPayload["environmentId"], + label: "Link Test Environment", + endpoint: { + httpBaseUrl: "https://managed.example.test/", + wsBaseUrl: "wss://managed.example.test/ws", + providerKind: "cloudflare_tunnel" as const, + }, + environmentPublicKey, + linkedAt: "2026-01-01T00:00:00.000Z", + }); + let linkedKey = "another-environment-key"; + return Effect.gen(function* () { + const linker = yield* EnvironmentLinker.EnvironmentLinker; + const refused = yield* linker + .link({ userId: "user_123", request: (yield* makeRequest).request }) + .pipe(Effect.flip); + expect(refused).toMatchObject({ + _tag: "EnvironmentLinkProofInvalid", + reason: "environment_key_mismatch", + }); + expect(touched).toEqual([]); + + linkedKey = environmentKeyPair.publicKey.trim(); + yield* linker.link({ userId: "user_123", request: (yield* makeRequest).request }); + expect(touched).toEqual(["deprovision", "upsert"]); + }).pipe( + Effect.provide( + layerTest({ + getForUser: () => Effect.succeed(activeLink(linkedKey)), + upsert: () => Effect.sync(() => void touched.push("upsert")), + deprovision: () => Effect.sync(() => touched.push("deprovision") > 0), + }), + ), + ); + }); + it.effect("rejects replayed JWT ids", () => Effect.gen(function* () { const { request } = yield* makeRequest; diff --git a/infra/relay/src/environments/EnvironmentLinker.ts b/infra/relay/src/environments/EnvironmentLinker.ts index c0acd41c36c2e..930b7a1cab827 100644 --- a/infra/relay/src/environments/EnvironmentLinker.ts +++ b/infra/relay/src/environments/EnvironmentLinker.ts @@ -55,6 +55,7 @@ export class EnvironmentLinkProofInvalid extends Schema.TaggedError Date: Thu, 19 Nov 2026 19:02:23 -0800 Subject: [PATCH 2/4] fix(relay): finish a failed unlink teardown before a new link provisions An unlink that revoked the link but failed to delete its tunnel left the tunnel and its allocation behind. A later link under another key passed the active-link check and provisioning reused that tunnel by name. A link with no active predecessor now completes the leftover teardown first, and is refused while that teardown cannot finish. Co-Authored-By: Claude Fable 5.2 --- .../environments/EnvironmentLinker.test.ts | 45 ++++++++++++++++++- .../src/environments/EnvironmentLinker.ts | 19 +++++++- infra/relay/src/http/Api.ts | 6 +++ 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index 3fca0bca8cc5e..40fe61c574d62 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -113,6 +113,7 @@ const makeRequest = Effect.gen(function* () { function layerTest(input?: { readonly upsert?: EnvironmentLinks.EnvironmentLinks["Service"]["upsert"]; readonly getForUser?: EnvironmentLinks.EnvironmentLinks["Service"]["getForUser"]; + readonly prepareDeprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["prepareDeprovision"]; readonly consume?: DpopProofs.DpopProofReplay["Service"]["consume"]; readonly deprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["deprovision"]; }) { @@ -145,7 +146,7 @@ function layerTest(input?: { }), Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, { reconcileOrigin: () => Effect.succeed("ready"), - prepareDeprovision: () => Effect.succeed(null), + prepareDeprovision: input?.prepareDeprovision ?? (() => Effect.succeed(null)), deprovision: input?.deprovision ?? (() => Effect.succeed(true)), release: () => Effect.succeed(true), provision: () => @@ -333,6 +334,48 @@ describe("EnvironmentLinker", () => { ); }); + it.effect("finishes a failed unlink teardown before a new link takes the tunnel", () => { + const touched: Array = []; + let tornDown = false; + return Effect.gen(function* () { + const linker = yield* EnvironmentLinker.EnvironmentLinker; + const link = Effect.gen(function* () { + const { request } = yield* makeRequest; + return yield* linker.link({ + userId: "user_123", + request: { ...request, managedTunnelsEnabled: true }, + }); + }); + expect((yield* link.pipe(Effect.flip))._tag).toBe("ManagedEndpointDeprovisioningFailed"); + expect(touched).toEqual(["deprovision"]); + + tornDown = true; + yield* link; + expect(touched).toEqual(["deprovision", "deprovision", "upsert"]); + }).pipe( + Effect.provide( + layerTest({ + prepareDeprovision: () => + Effect.succeed({ + userId: "user_123", + environmentId: "env-link-test", + hostname: "managed.example.test", + tunnelId: "left-behind-tunnel", + tunnelName: "left-behind", + dnsRecordId: null, + readyAt: null, + origin: null, + updatedAt: "2026-01-01T00:00:00.000Z", + generation: 1, + tunnelReleasedAt: null, + }), + deprovision: () => Effect.sync(() => touched.push("deprovision") > 0 && tornDown), + upsert: () => Effect.sync(() => void touched.push("upsert")), + }), + ), + ); + }); + it.effect("rejects replayed JWT ids", () => Effect.gen(function* () { const { request } = yield* makeRequest; diff --git a/infra/relay/src/environments/EnvironmentLinker.ts b/infra/relay/src/environments/EnvironmentLinker.ts index 930b7a1cab827..6e66fa6f98935 100644 --- a/infra/relay/src/environments/EnvironmentLinker.ts +++ b/infra/relay/src/environments/EnvironmentLinker.ts @@ -72,6 +72,7 @@ export type EnvironmentLinkError = | DpopProofs.DpopProofReplayPersistenceError | EnvironmentLinks.EnvironmentLinkUpsertPersistenceError | EnvironmentLinks.EnvironmentLinkLookupPersistenceError + | ManagedEndpointProvider.ManagedEndpointDeprovisioningFailed | EnvironmentCredentials.EnvironmentCredentialCreatePersistenceError | ManagedEndpointProvider.ManagedEndpointProviderError; @@ -294,7 +295,7 @@ const make = Effect.gen(function* () { }); } // The account bearer alone must not move an active link, its tunnel and its trusted key to - // another key. Unlinking first releases the tunnel, so nothing of the old link is handed over. + // another key. Unlinking first releases the tunnel. const existing = yield* links.getForUser({ userId: input.userId, environmentId: verified.environmentId, @@ -307,6 +308,22 @@ const make = Effect.gen(function* () { stage: "validate_existing_link", }); } + // An unlink whose teardown failed leaves its tunnel behind, and provisioning would reuse it by + // name. Finish the teardown first so a new link never inherits the old link's tunnel. + if (existing === null && input.request.managedTunnelsEnabled) { + const key = { userId: input.userId, environmentId: verified.environmentId }; + const leftover = yield* managedEndpointProvider.prepareDeprovision(key); + if ( + leftover !== null && + !(yield* managedEndpointProvider.deprovision({ ...key, target: leftover })) + ) { + return yield* new ManagedEndpointProvider.ManagedEndpointDeprovisioningFailed({ + ...key, + stage: "claim-deprovision", + cause: "an earlier tunnel for this environment is still being torn down", + }); + } + } // Non-tunnel links must release provisioned tunnel/DNS resources; best-effort deprovision is idempotent. if (!input.request.managedTunnelsEnabled) { yield* managedEndpointProvider diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 74d2b5bbd7a6d..e43a86a13a0a8 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -897,6 +897,12 @@ export const layerClientApi = HttpApiBuilder.group( reason: "origin_not_allowed", traceId, }), + ManagedEndpointDeprovisioningFailed: (_error, traceId) => + new RelayEnvironmentLinkUnavailableError({ + code: "environment_link_unavailable", + reason: "managed_endpoint_provisioning_failed", + traceId, + }), ManagedTunnelLimitExceeded: (limitError, traceId) => new RelayEnvironmentLinkLimitExceededError({ code: "environment_link_limit_exceeded", From 9ac69d1aacaccd6f51a12b1327537f6e524affe2 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 19 Nov 2026 19:19:25 -0800 Subject: [PATCH 3/4] fix(relay): clear held webhooks before an unlink's tunnel teardown The teardown can fail, and the clear came after it, so a failed unlink kept the endpoint's held webhooks. The inbox is keyed by user and environment, not by environment key, so a later link under another key would have received them. Co-Authored-By: Claude Fable 5.2 --- infra/relay/src/http/Api.test.ts | 111 ++++++++++++++++++------------- infra/relay/src/http/Api.ts | 13 ++-- 2 files changed, 72 insertions(+), 52 deletions(-) diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index d59df1e172e28..36582f0d1fd13 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -1081,52 +1081,71 @@ describe("relay managed tunnel recovery", () => { }); describe("relay environment unlink", () => { - it.effect("drops the unlinked endpoint's held webhooks, even if clearing fails", () => { - const cleared: Array = []; - const endpointKey = "0123456789abcdef"; - const unlink = (clearFails: boolean) => - unlinkEnvironmentRecord({ - userId: "user-1", - environmentId: "environment-1", - managedEndpointNamespace: "dev", - }).pipe( - Effect.provide( - layerRelayUnlinkTest({ - getForUser: () => Effect.succeed(linkedEnvironmentRecord), - revokeForUser: () => Effect.succeed(true), - prepareDeprovision: () => - Effect.succeed({ - userId: "user-1", - environmentId: "environment-1", - hostname: "dev-0123456789abcdef.example.test", - tunnelId: "tunnel-1", - tunnelName: `t3coderelay-managedendpoint-dev-${endpointKey}`, - dnsRecordId: "dns-1", - readyAt: "2026-07-28T00:00:00.000Z", - tunnelReleasedAt: null, - origin: null, - updatedAt: "2026-07-28T00:00:00.000Z", - generation: 1, - }), - clearInbox: (input) => - clearFails - ? Effect.fail( - new HookInbox.HookInboxError({ - operation: "clear", - endpointKey: input.endpointKey, - cause: new Error("unavailable"), - }), - ) - : Effect.sync(() => void cleared.push(input.endpointKey)), - }), - ), - ); - return Effect.gen(function* () { - expect(yield* unlink(false)).toBe(true); - expect(cleared).toEqual([endpointKey]); - expect(yield* unlink(true)).toBe(true); - }); - }); + it.effect( + "drops the unlinked endpoint's held webhooks, even if clearing or teardown fails", + () => { + const cleared: Array = []; + const endpointKey = "0123456789abcdef"; + const unlink = (clearFails: boolean, teardownFails = false) => + unlinkEnvironmentRecord({ + userId: "user-1", + environmentId: "environment-1", + managedEndpointNamespace: "dev", + }).pipe( + Effect.provide( + layerRelayUnlinkTest({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + revokeForUser: () => Effect.succeed(true), + prepareDeprovision: () => + Effect.succeed({ + userId: "user-1", + environmentId: "environment-1", + hostname: "dev-0123456789abcdef.example.test", + tunnelId: "tunnel-1", + tunnelName: `t3coderelay-managedendpoint-dev-${endpointKey}`, + dnsRecordId: "dns-1", + readyAt: "2026-07-28T00:00:00.000Z", + tunnelReleasedAt: null, + origin: null, + updatedAt: "2026-07-28T00:00:00.000Z", + generation: 1, + }), + clearInbox: (input) => + clearFails + ? Effect.fail( + new HookInbox.HookInboxError({ + operation: "clear", + endpointKey: input.endpointKey, + cause: new Error("unavailable"), + }), + ) + : Effect.sync(() => void cleared.push(input.endpointKey)), + ...(teardownFails + ? { + deprovision: (input) => + Effect.fail( + new ManagedEndpointProvider.ManagedEndpointDeprovisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "delete-tunnel", + cause: new Error("unavailable"), + }), + ), + } + : {}), + }), + ), + ); + return Effect.gen(function* () { + expect(yield* unlink(false)).toBe(true); + expect(cleared).toEqual([endpointKey]); + expect(yield* unlink(true)).toBe(true); + // A later link under another key shares the inbox, so it is emptied before teardown can fail. + yield* unlink(false, true).pipe(Effect.flip); + expect(cleared).toEqual([endpointKey, endpointKey]); + }); + }, + ); it.effect("commits database revocation before deprovisioning the managed endpoint", () => { const calls: Array = []; diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index e43a86a13a0a8..49bfda68a7f95 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -527,12 +527,7 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron environmentId: link.environmentId, environmentPublicKey: link.environmentPublicKey, }); - // Deprovision after revocation commits; retry even if already revoked to finish failed Cloudflare cleanup. - const deprovisioned = yield* managedEndpointProvider.deprovision({ - userId: input.userId, - environmentId: input.environmentId, - target: deprovisionTarget, - }); + // Cleared before teardown, which can fail: a later link under another key shares this inbox. const endpointKey = deprovisionTarget && input.managedEndpointNamespace ? HeldHooks.endpointKeyForTunnelName( @@ -551,6 +546,12 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron ), ); } + // Deprovision after revocation commits; retry even if already revoked to finish failed Cloudflare cleanup. + const deprovisioned = yield* managedEndpointProvider.deprovision({ + userId: input.userId, + environmentId: input.environmentId, + target: deprovisionTarget, + }); if (!deprovisioned) { const key = { userId: input.userId, environmentId: input.environmentId }; const retryTarget = yield* managedEndpointProvider.prepareDeprovision(key); From 90b4726928935b98d6bcc179d38b5c328f274c27 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 19 Nov 2026 19:21:28 -0800 Subject: [PATCH 4/4] fix(relay): a publish-only link also waits for a leftover tunnel teardown A new key could link publish-only while an old tunnel's teardown was still failing, then turn tunnels on as the active link and inherit that tunnel. Any link with no active predecessor now requires the leftover teardown. Co-Authored-By: Claude Fable 5.2 --- .../environments/EnvironmentLinker.test.ts | 81 ++++++++++--------- .../src/environments/EnvironmentLinker.ts | 14 ++-- 2 files changed, 48 insertions(+), 47 deletions(-) diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index 40fe61c574d62..23ae7c890fb94 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -334,47 +334,50 @@ describe("EnvironmentLinker", () => { ); }); - it.effect("finishes a failed unlink teardown before a new link takes the tunnel", () => { - const touched: Array = []; - let tornDown = false; - return Effect.gen(function* () { - const linker = yield* EnvironmentLinker.EnvironmentLinker; - const link = Effect.gen(function* () { - const { request } = yield* makeRequest; - return yield* linker.link({ - userId: "user_123", - request: { ...request, managedTunnelsEnabled: true }, + it.effect.each([{ managedTunnelsEnabled: true }, { managedTunnelsEnabled: false }])( + "finishes a failed unlink teardown before a new link: %o", + ({ managedTunnelsEnabled }) => { + const touched: Array = []; + let tornDown = false; + return Effect.gen(function* () { + const linker = yield* EnvironmentLinker.EnvironmentLinker; + const link = Effect.gen(function* () { + const { request } = yield* makeRequest; + return yield* linker.link({ + userId: "user_123", + request: { ...request, managedTunnelsEnabled }, + }); }); - }); - expect((yield* link.pipe(Effect.flip))._tag).toBe("ManagedEndpointDeprovisioningFailed"); - expect(touched).toEqual(["deprovision"]); + expect((yield* link.pipe(Effect.flip))._tag).toBe("ManagedEndpointDeprovisioningFailed"); + expect(touched).toEqual(["deprovision"]); - tornDown = true; - yield* link; - expect(touched).toEqual(["deprovision", "deprovision", "upsert"]); - }).pipe( - Effect.provide( - layerTest({ - prepareDeprovision: () => - Effect.succeed({ - userId: "user_123", - environmentId: "env-link-test", - hostname: "managed.example.test", - tunnelId: "left-behind-tunnel", - tunnelName: "left-behind", - dnsRecordId: null, - readyAt: null, - origin: null, - updatedAt: "2026-01-01T00:00:00.000Z", - generation: 1, - tunnelReleasedAt: null, - }), - deprovision: () => Effect.sync(() => touched.push("deprovision") > 0 && tornDown), - upsert: () => Effect.sync(() => void touched.push("upsert")), - }), - ), - ); - }); + tornDown = true; + yield* link; + expect(touched).toEqual(["deprovision", "deprovision", "upsert"]); + }).pipe( + Effect.provide( + layerTest({ + prepareDeprovision: () => + Effect.succeed({ + userId: "user_123", + environmentId: "env-link-test", + hostname: "managed.example.test", + tunnelId: "left-behind-tunnel", + tunnelName: "left-behind", + dnsRecordId: null, + readyAt: null, + origin: null, + updatedAt: "2026-01-01T00:00:00.000Z", + generation: 1, + tunnelReleasedAt: null, + }), + deprovision: () => Effect.sync(() => touched.push("deprovision") > 0 && tornDown), + upsert: () => Effect.sync(() => void touched.push("upsert")), + }), + ), + ); + }, + ); it.effect("rejects replayed JWT ids", () => Effect.gen(function* () { diff --git a/infra/relay/src/environments/EnvironmentLinker.ts b/infra/relay/src/environments/EnvironmentLinker.ts index 6e66fa6f98935..c771f51841b49 100644 --- a/infra/relay/src/environments/EnvironmentLinker.ts +++ b/infra/relay/src/environments/EnvironmentLinker.ts @@ -309,14 +309,12 @@ const make = Effect.gen(function* () { }); } // An unlink whose teardown failed leaves its tunnel behind, and provisioning would reuse it by - // name. Finish the teardown first so a new link never inherits the old link's tunnel. - if (existing === null && input.request.managedTunnelsEnabled) { + // name. Finish the teardown before any new link, publish-only too: once that link is active + // it could turn tunnels on and inherit the old one. + if (existing === null) { const key = { userId: input.userId, environmentId: verified.environmentId }; const leftover = yield* managedEndpointProvider.prepareDeprovision(key); - if ( - leftover !== null && - !(yield* managedEndpointProvider.deprovision({ ...key, target: leftover })) - ) { + if (!(yield* managedEndpointProvider.deprovision({ ...key, target: leftover }))) { return yield* new ManagedEndpointProvider.ManagedEndpointDeprovisioningFailed({ ...key, stage: "claim-deprovision", @@ -324,8 +322,8 @@ const make = Effect.gen(function* () { }); } } - // Non-tunnel links must release provisioned tunnel/DNS resources; best-effort deprovision is idempotent. - if (!input.request.managedTunnelsEnabled) { + // An active link going publish-only releases its own tunnel/DNS; best-effort deprovision is idempotent. + if (existing !== null && !input.request.managedTunnelsEnabled) { yield* managedEndpointProvider .deprovision({ userId: input.userId,