From de39ed9e7007e273e2cc5e2b18eec31169aec8ab Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 19 Nov 2026 23:29:13 +0000 Subject: [PATCH 1/2] test(client-runtime): supervisor background tasks stay dead after a defect (#39419) The connection supervisor's network listener, wakeup listener and better-route checker are forked with nothing around them. One defect in each ends it for good, silently. Each test injects that defect once and fails on main; all three pass with the fault removed. Co-Authored-By: Claude Opus 5.5 --- .../src/connection/supervisor.test.ts | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index 46b9757d0cd83..078abb94f45d0 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -2629,3 +2629,123 @@ describe("EnvironmentSupervisor routes", () => { }), ); }); + +describe("EnvironmentSupervisor background fibers", () => { + it.effect("hears the network return after its connectivity stream died", () => + Effect.gen(function* () { + const harness = yield* makeHarness(); + const status = yield* SubscriptionRef.make("offline"); + const subscriptions = yield* Ref.make(0); + const dying = Connectivity.Connectivity.of({ + status: SubscriptionRef.get(status), + changes: Stream.unwrap( + Ref.updateAndGet(subscriptions, (count) => count + 1).pipe( + Effect.map((count) => + count === 1 + ? Stream.die(new Error("platform connectivity stream died")) + : SubscriptionRef.changes(status), + ), + ), + ), + }); + const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { + initiallyDesired: true, + }).pipe( + Effect.provideService(Connectivity.Connectivity, dying), + Effect.provide(harness.dependencies), + ); + yield* TestClock.adjust("10 minutes"); + expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("offline"); + + yield* SubscriptionRef.set(status, "online"); + yield* TestClock.adjust("10 minutes"); + expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connected"); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect("replaces a stalled session on return to the app after its wakeup stream died", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ probe: () => Effect.never }); + const returns = yield* SubscriptionRef.make(0); + const subscriptions = yield* Ref.make(0); + const dying = ConnectionWakeups.ConnectionWakeups.of({ + changes: Stream.unwrap( + Ref.updateAndGet(subscriptions, (count) => count + 1).pipe( + Effect.map((count) => + count === 1 + ? Stream.die(new Error("platform wakeup stream died")) + : SubscriptionRef.changes(returns).pipe( + Stream.drop(1), + Stream.map(() => "application-active" as const), + ), + ), + ), + ), + }); + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe( + Effect.provideService(ConnectionWakeups.ConnectionWakeups, dying), + Effect.provide(harness.dependencies), + ); + yield* TestClock.adjust("10 minutes"); + expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({ + phase: "connected", + generation: 1, + }); + + // The user returns to the app; the session's probe never answers. + yield* SubscriptionRef.update(returns, (count) => count + 1); + yield* TestClock.adjust("15 seconds"); + yield* TestClock.adjust("15 seconds"); + yield* TestClock.adjust("10 minutes"); + expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({ + phase: "connected", + generation: 2, + }); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect("still moves to a better route after one route check crashed", () => + Effect.gen(function* () { + const lanReachable = yield* Ref.make(false); + const preflights = yield* Ref.make(0); + const harness = yield* makeHarness({ + checkRoute: (route) => + route.target._tag === "BearerConnectionTarget" + ? Ref.get(lanReachable).pipe( + Effect.map((reachable) => (reachable ? "answered" : "silent")), + ) + : Effect.succeed("unchecked"), + preflight: () => + Ref.updateAndGet(preflights, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 1 + ? Effect.die(new Error("route check threw")) + : Ref.get(lanReachable).pipe( + Effect.map((reachable) => (reachable ? "ready" : "silent")), + ), + ), + ), + prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), + }); + const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 1, + ); + yield* TestClock.adjust("60 seconds"); + expect(yield* Ref.get(preflights)).toBeGreaterThanOrEqual(1); + + yield* Ref.set(lanReachable, true); + for (let minute = 0; minute < 60; minute += 1) { + yield* TestClock.adjust("60 seconds"); + } + expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( + "BearerConnectionTarget", + ); + }).pipe(Effect.provide(TestClock.layer())), + ); +}); From 2d144e56539233113525a8376765119ab2dc80e9 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 19 Nov 2026 23:37:05 +0000 Subject: [PATCH 2/2] fix(client-runtime): connection supervisor restarts its background listeners after a defect The network listener, wakeup listener and better-route checker were forked bare, so one defect ended each for good with nothing logged. Each now logs the failure and starts again after a second; the network listener reads the current status first, since it may have missed a change while it was down. Fixes pingdotgg/t3code#39419 Co-Authored-By: Claude Opus 5.5 --- .../src/connection/supervisor.ts | 86 ++++++++++++------- 1 file changed, 56 insertions(+), 30 deletions(-) diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts index 58411d795b7fc..bc2b2098123c4 100644 --- a/packages/client-runtime/src/connection/supervisor.ts +++ b/packages/client-runtime/src/connection/supervisor.ts @@ -1327,40 +1327,66 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( } }); - yield* connectivity.changes.pipe( - Stream.runForEach((network) => - Ref.modify(intent, (current) => { - const next = targetNetwork(network); - return current.network === next - ? [false, current] - : ([true, { ...current, network: next }] as const); - }).pipe( - Effect.flatMap((changed) => - changed - ? (network === "offline" ? Effect.void : forgetSilentRoutes).pipe( - Effect.andThen(signal({ _tag: "NetworkChanged", network })), - ) - : Effect.void, - ), + // A defect in a background listener would otherwise end it for good, silently. + // Untraced: a span here would stay open for the supervisor's whole lifetime. + const keepRunning = Effect.fnUntraced( + function* (task: string, effect: Effect.Effect, onRestart?: Effect.Effect) { + let current = effect; + while (true) { + const exit = yield* Effect.exit(current); + if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) return yield* exit; + yield* Effect.logError("Connection supervisor task failed; restarting it.").pipe( + Effect.annotateLogs({ + "environment.id": target.environmentId, + "environment.label": target.label, + task, + ...safeErrorLogAttributes(exit.cause.reasons.find(Cause.isDieReason)?.defect), + }), + ); + yield* Effect.sleep(RETRY_BASE_DELAY_MS); + current = onRestart ? onRestart.pipe(Effect.andThen(effect)) : effect; + } + }, + (self) => Effect.forkScoped(self), + ); + + const applyNetwork = (network: NetworkStatus) => + Ref.modify(intent, (current) => { + const next = targetNetwork(network); + return current.network === next + ? [false, current] + : ([true, { ...current, network: next }] as const); + }).pipe( + Effect.flatMap((changed) => + changed + ? (network === "offline" ? Effect.void : forgetSilentRoutes).pipe( + Effect.andThen(signal({ _tag: "NetworkChanged", network })), + ) + : Effect.void, ), - ), - Effect.forkScoped, + ); + // A restarted listener may have missed changes while it was down. + yield* keepRunning( + "network-listener", + connectivity.changes.pipe(Stream.runForEach(applyNetwork)), + connectivity.status.pipe(Effect.flatMap(applyNetwork)), ); - yield* wakeups.changes.pipe( - Stream.filter(ConnectionWakeups.reachesConnectionSupervisor), - Stream.runForEach((reason) => - // A long suspension can hide a network change the platform never reported. - (reason === "network-changed" || reason === "application-active-long" - ? forgetSilentRoutes - : Effect.void - ).pipe(Effect.andThen(signal({ _tag: "Wakeup", reason }))), + yield* keepRunning( + "wakeup-listener", + wakeups.changes.pipe( + Stream.filter(ConnectionWakeups.reachesConnectionSupervisor), + Stream.runForEach((reason) => + // A long suspension can hide a network change the platform never reported. + (reason === "network-changed" || reason === "application-active-long" + ? forgetSilentRoutes + : Effect.void + ).pipe(Effect.andThen(signal({ _tag: "Wakeup", reason }))), + ), ), - Effect.forkScoped, ); - yield* Queue.take(betterRouteChecks).pipe( - Effect.flatMap(checkBetterRoutes), - Effect.forever, - Effect.forkScoped, + yield* keepRunning( + "better-route-checker", + Queue.take(betterRouteChecks).pipe(Effect.flatMap(checkBetterRoutes), Effect.forever), ); yield* run().pipe(Effect.forkScoped);