diff --git a/apps/server/src/cloud/CloudLink.lifecycle.test.ts b/apps/server/src/cloud/CloudLink.lifecycle.test.ts index 6c33460d1afdb..cbbf4670d6f67 100644 --- a/apps/server/src/cloud/CloudLink.lifecycle.test.ts +++ b/apps/server/src/cloud/CloudLink.lifecycle.test.ts @@ -1194,6 +1194,7 @@ describe("releaseManagedTunnelOnShutdown", () => { pid: 123, readyConnections: 0, hostname: "prod-abc.t3coderelay.com", + origin: null, protocol: null, }, }), diff --git a/apps/server/src/cloud/ConnectReachability.test.ts b/apps/server/src/cloud/ConnectReachability.test.ts index 9420d6940ea78..91ef5bdc785d5 100644 --- a/apps/server/src/cloud/ConnectReachability.test.ts +++ b/apps/server/src/cloud/ConnectReachability.test.ts @@ -25,8 +25,14 @@ import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; vi.mock("node:net", { spy: true }); const checkedAt = "2026-10-19T00:00:00.000Z"; -const connected = { pid: 1, readyConnections: 4, hostname: "env.example.test", protocol: "quic" }; -const notConnected = { pid: 1, readyConnections: 0, hostname: null, protocol: null }; +const connected = { + pid: 1, + readyConnections: 4, + hostname: "env.example.test", + origin: null, + protocol: "quic", +}; +const notConnected = { pid: 1, readyConnections: 0, hostname: null, origin: null, protocol: null }; const facts = (overrides: Partial): ReachabilityFacts => ({ managedTunnelConfigured: true, connector: connected, @@ -64,6 +70,7 @@ describe("classifyReachability", () => { pid: 1, readyConnections: null, hostname: "env.example.test", + origin: null, protocol: null, }, endpoint: { kind: "status", status: 502 }, @@ -80,6 +87,7 @@ describe("classifyReachability", () => { pid: 1, readyConnections: null, hostname: "env.example.test", + origin: null, protocol: null, }, endpoint: { kind: "dns_failed" }, diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 7ce07f8e98677..dae9d870c8a75 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -22,7 +22,7 @@ import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; -import { HttpClient } from "effect/http"; +import { HttpClient, HttpClientResponse } from "effect/http"; import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; import * as RelayClient from "@t3tools/shared/relayClient"; @@ -259,6 +259,68 @@ describe("CloudManagedEndpointRuntime", () => { }), ); + it.effect("reads the public hostname and its origin from cloudflared's remote config", () => + Effect.gen(function* () { + let ingress: unknown = [ + { hostname: "env.example.t3.codes", service: "http://127.0.0.1:3773" }, + { service: "http_status:404" }, + ]; + const httpClient = HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + Response.json( + request.url.endsWith("/config") + ? { config: ingress === undefined ? {} : { ingress } } + : { readyConnections: 4 }, + ), + ), + ), + ); + const output = yield* Queue.unbounded(); + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const handle = makeHandle({ + pid: 702, + onKill: () => {}, + output: Stream.fromQueue(output), + }); + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore)); + return handle; + }), + ); + const runtime = yield* buildCloudManagedEndpointRuntime( + spawner, + layerRelayClientAvailable, + httpClient, + ); + yield* runtime.applyConfig({ + providerKind: "cloudflare_tunnel", + connectorToken: "connector-token", + tunnelId: "tunnel-1", + }); + yield* Queue.offer( + output, + new TextEncoder().encode( + "2026-10-06T00:00:01Z INF Starting metrics server on 127.0.0.1:20241/metrics\n" + + "2026-10-06T00:00:01Z INF Registered tunnel connection connIndex=0\n", + ), + ); + yield* Stream.runHead(runtime.tunnelConnected); + + expect(yield* runtime.inspectConnector).toMatchObject({ + readyConnections: 4, + hostname: "env.example.t3.codes", + origin: "http://127.0.0.1:3773", + }); + // Before the edge pushes a config, the origin is unknown rather than wrong. + for (const missing of [null, undefined]) { + ingress = missing; + expect(yield* runtime.inspectConnector).toMatchObject({ hostname: null, origin: null }); + } + }), + ); + it.effect("logs a connection unregistration at info with the connector token redacted", () => { const logs: Array<{ readonly level: string; readonly message: unknown }> = []; const unregistrationLogged = Deferred.makeUnsafe(); diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 399f6e0d644b2..6d14e23182344 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -60,6 +60,8 @@ export interface ConnectorFacts { readonly readyConnections: number | null; /** The public tunnel hostname from the remote config cloudflared receives once it connects. */ readonly hostname: string | null; + /** The local origin that remote config routes the public hostname to. */ + readonly origin: string | null; readonly protocol: string | null; } @@ -123,7 +125,14 @@ const ConnectorReadyBody = Schema.Struct({ readyConnections: Schema.Number }); const ConnectorConfigBody = Schema.Struct({ config: Schema.Struct({ ingress: Schema.optional( - Schema.NullOr(Schema.Array(Schema.Struct({ hostname: Schema.optional(Schema.String) }))), + Schema.NullOr( + Schema.Array( + Schema.Struct({ + hostname: Schema.optional(Schema.String), + service: Schema.optional(Schema.String), + }), + ), + ), ), }), }); @@ -924,7 +933,13 @@ export const make = Effect.gen(function* () { const pid = Number(active.child.pid); const protocol = yield* Ref.get(active.protocol); if (!(yield* Deferred.isDone(active.metricsAddress))) { - return { pid, readyConnections: null, hostname: null, protocol } satisfies ConnectorFacts; + return { + pid, + readyConnections: null, + hostname: null, + origin: null, + protocol, + } satisfies ConnectorFacts; } const metricsUrl = `http://${yield* Deferred.await(active.metricsAddress)}`; const [ready, config] = yield* Effect.all( @@ -934,10 +949,12 @@ export const make = Effect.gen(function* () { ], { concurrency: 2 }, ); + const hostnameRule = config?.config.ingress?.find((rule) => rule.hostname); return { pid, readyConnections: ready?.readyConnections ?? null, - hostname: config?.config.ingress?.find((rule) => rule.hostname)?.hostname ?? null, + hostname: hostnameRule?.hostname ?? null, + origin: hostnameRule?.service ?? null, protocol, } satisfies ConnectorFacts; }); diff --git a/apps/server/src/cloud/managedTunnelStartup.test.ts b/apps/server/src/cloud/managedTunnelStartup.test.ts index fbf4fc452c256..6752307b1d010 100644 --- a/apps/server/src/cloud/managedTunnelStartup.test.ts +++ b/apps/server/src/cloud/managedTunnelStartup.test.ts @@ -1,7 +1,12 @@ import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as TestClock from "effect/testing/TestClock"; -import { retryManagedTunnelRegistration } from "./managedTunnelStartup.ts"; +import { + retryManagedTunnelRegistration, + watchManagedTunnelOrigin, +} from "./managedTunnelStartup.ts"; describe("retryManagedTunnelRegistration", () => { it.effect("does not fall back or retry when registration is permanently rejected", () => @@ -26,3 +31,76 @@ describe("retryManagedTunnelRegistration", () => { }), ); }); + +describe("watchManagedTunnelOrigin", () => { + const localOrigin = "http://127.0.0.1:3773"; + + /** Runs the watcher for `minutes`, serving each check the next ingress in `ingress`. */ + const watch = (input: { + readonly ingress: ReadonlyArray; + readonly minutes: number; + readonly failRegistration?: boolean; + }) => + Effect.gen(function* () { + let checks = 0; + const registeredAtMinute: Array = []; + let minute = 0; + const fiber = yield* watchManagedTunnelOrigin({ + localOrigin, + readIngressOrigin: Effect.sync( + () => input.ingress[Math.min(checks++, input.ingress.length - 1)] ?? null, + ), + register: Effect.suspend(() => { + registeredAtMinute.push(minute); + return input.failRegistration ? Effect.fail("relay unavailable") : Effect.void; + }), + }).pipe(Effect.forkChild); + while (minute < input.minutes) { + minute += 1; + yield* TestClock.adjust("1 minute"); + } + yield* Fiber.interrupt(fiber); + return { checks, registeredAtMinute }; + }); + + it.effect("leaves a matching or unknown ingress alone", () => + Effect.gen(function* () { + const result = yield* watch({ + ingress: [null, `${localOrigin}/`, localOrigin], + minutes: 5, + }); + expect(result.checks).toBe(5); + expect(result.registeredAtMinute).toEqual([]); + }), + ); + + it.effect("keeps checking after a failed registration", () => + Effect.gen(function* () { + const result = yield* watch({ + ingress: ["http://127.0.0.1:46169", "http://127.0.0.1:46169", localOrigin], + minutes: 8, + failRegistration: true, + }); + expect(result.registeredAtMinute).toEqual([1, 3]); + // Minutes 1, 3, and 7; the match at 7 restores the one-minute interval. + expect(result.checks).toBe(4); + }), + ); + + it.effect("backs off while the relay keeps routing elsewhere", () => + Effect.gen(function* () { + const result = yield* watch({ ingress: ["http://127.0.0.1:46169"], minutes: 20 }); + expect(result.registeredAtMinute).toEqual([1, 3, 7, 15]); + }), + ); + + it.effect("keeps the backoff when one check cannot read the ingress", () => + Effect.gen(function* () { + const result = yield* watch({ + ingress: ["http://127.0.0.1:46169", null, "http://127.0.0.1:46169"], + minutes: 10, + }); + expect(result.registeredAtMinute).toEqual([1, 5, 9]); + }), + ); +}); diff --git a/apps/server/src/cloud/managedTunnelStartup.ts b/apps/server/src/cloud/managedTunnelStartup.ts index 806d6412cf359..ed7dd7aa4d8eb 100644 --- a/apps/server/src/cloud/managedTunnelStartup.ts +++ b/apps/server/src/cloud/managedTunnelStartup.ts @@ -1,4 +1,5 @@ import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; +import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -212,3 +213,56 @@ export const makeManagedTunnelRecovery = Effect.fn("ManagedTunnelRecovery.make") ), ); }); + +const MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL = Duration.minutes(1); + +const urlOrigin = (url: string) => { + try { + return new URL(url).origin; + } catch { + return null; + } +}; + +/** + * Only this server runs the connector, so an ingress naming another origin is a dead + * registration. Checks every minute and re-registers this origin on a mismatch, + * doubling the wait (up to 30 minutes) while the relay keeps routing elsewhere. + */ +export const watchManagedTunnelOrigin = Effect.fn("ManagedTunnelOrigin.watch")(function* < + E, + R, +>(options: { + readonly localOrigin: string; + /** The service URL cloudflared routes the public hostname to, or `null` while unknown. */ + readonly readIngressOrigin: Effect.Effect; + readonly register: Effect.Effect; +}) { + const localOrigin = urlOrigin(options.localOrigin); + let wait = MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL; + while (true) { + yield* Effect.sleep(wait); + const ingress = yield* options.readIngressOrigin; + // An unreadable ingress says nothing, so it keeps whatever backoff is running. + if (ingress === null) continue; + const ingressOrigin = urlOrigin(ingress); + if (ingressOrigin !== null && ingressOrigin === localOrigin) { + wait = MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL; + continue; + } + yield* Effect.logWarning( + "T3 Connect routes this server's tunnel to another origin; registering this server again", + { ingressOrigin: ingressOrigin ?? "unparseable", localOrigin }, + ); + yield* options.register.pipe( + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to register this server's T3 Connect origin again", { + cause, + }), + ), + ); + wait = Duration.min(Duration.times(wait, 2), MANAGED_TUNNEL_RECOVERY_MAX_COOLDOWN); + } +}); diff --git a/apps/server/src/guest/GuestShareService.test.ts b/apps/server/src/guest/GuestShareService.test.ts index cf2f9e3f70782..ae687c437b681 100644 --- a/apps/server/src/guest/GuestShareService.test.ts +++ b/apps/server/src/guest/GuestShareService.test.ts @@ -83,6 +83,7 @@ const layerConnect = Layer.mock(CloudManagedEndpointRuntime)({ pid: 1, readyConnections: 1, hostname: value, + origin: null, protocol: "http2", })), }); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 5c549551276f1..d5e42154898cd 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -183,6 +183,7 @@ import { managedTunnelStartupAction, retryDesiredLinkReconcile, retryManagedTunnelRegistration, + watchManagedTunnelOrigin, } from "./cloud/managedTunnelStartup.ts"; import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts"; import * as CloudCliState from "./cloud/CliState.ts"; @@ -1152,6 +1153,28 @@ const layerMakeServer = Layer.unwrap( Stream.runForEach(recoverManagedTunnel), Effect.forkScoped, ); + yield* watchManagedTunnelOrigin({ + localOrigin, + readIngressOrigin: endpointRuntime.inspectConnector.pipe( + Effect.map((connector) => connector?.origin ?? null), + ), + // Like startup, a publish-only link never routes the tunnel to this host. + register: CloudCliState.readCliDesiredLinkMode.pipe( + Effect.flatMap((mode) => + mode === "publish_only" + ? Effect.void + : cloudLink + .registerManagedTunnelRecovery(localOrigin) + .pipe( + Effect.flatMap((registration) => + registration.status === "recovery_required" + ? endpointRuntime.requestRecovery(registration.config) + : Effect.void, + ), + ), + ), + ), + }).pipe(Effect.forkScoped); const wantsCliLink = hasCloudPublicConfig ? yield* CloudCliState.readCliDesiredCloudLink.pipe( Effect.catch((cause) =>