diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index 81c8bf4064a45..c8e241f8f69c0 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -2936,3 +2936,123 @@ describe("EnvironmentSupervisor routes", () => { }), ); }); + +describe("EnvironmentSupervisor background fibers", () => { + it.effect("hears the network return after its connectivity stream died", () => + Effect.gen(function* () { + const harness = yield* makeHarness(); + const status = yield* SubscriptionRef.make("offline"); + const subscriptions = yield* Ref.make(0); + const dying = Connectivity.Connectivity.of({ + status: SubscriptionRef.get(status), + changes: Stream.unwrap( + Ref.updateAndGet(subscriptions, (count) => count + 1).pipe( + Effect.map((count) => + count === 1 + ? Stream.die(new Error("platform connectivity stream died")) + : SubscriptionRef.changes(status), + ), + ), + ), + }); + const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { + initiallyDesired: true, + }).pipe( + Effect.provideService(Connectivity.Connectivity, dying), + Effect.provide(harness.dependencies), + ); + yield* TestClock.adjust("10 minutes"); + expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("offline"); + + yield* SubscriptionRef.set(status, "online"); + yield* TestClock.adjust("10 minutes"); + expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connected"); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect("replaces a stalled session on return to the app after its wakeup stream died", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ probe: () => Effect.never }); + const returns = yield* SubscriptionRef.make(0); + const subscriptions = yield* Ref.make(0); + const dying = ConnectionWakeups.ConnectionWakeups.of({ + changes: Stream.unwrap( + Ref.updateAndGet(subscriptions, (count) => count + 1).pipe( + Effect.map((count) => + count === 1 + ? Stream.die(new Error("platform wakeup stream died")) + : SubscriptionRef.changes(returns).pipe( + Stream.drop(1), + Stream.map(() => "application-active" as const), + ), + ), + ), + ), + }); + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe( + Effect.provideService(ConnectionWakeups.ConnectionWakeups, dying), + Effect.provide(harness.dependencies), + ); + yield* TestClock.adjust("10 minutes"); + expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({ + phase: "connected", + generation: 1, + }); + + // The user returns to the app; the session's probe never answers. + yield* SubscriptionRef.update(returns, (count) => count + 1); + yield* TestClock.adjust("15 seconds"); + yield* TestClock.adjust("15 seconds"); + yield* TestClock.adjust("10 minutes"); + expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({ + phase: "connected", + generation: 2, + }); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect("still moves to a better route after one route check crashed", () => + Effect.gen(function* () { + const lanReachable = yield* Ref.make(false); + const preflights = yield* Ref.make(0); + const harness = yield* makeHarness({ + checkRoute: (route) => + route.target._tag === "BearerConnectionTarget" + ? Ref.get(lanReachable).pipe( + Effect.map((reachable) => (reachable ? "answered" : "silent")), + ) + : Effect.succeed("unchecked"), + preflight: () => + Ref.updateAndGet(preflights, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 1 + ? Effect.die(new Error("route check threw")) + : Ref.get(lanReachable).pipe( + Effect.map((reachable) => (reachable ? "ready" : "silent")), + ), + ), + ), + prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), + }); + const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 1, + ); + yield* TestClock.adjust("60 seconds"); + expect(yield* Ref.get(preflights)).toBeGreaterThanOrEqual(1); + + yield* Ref.set(lanReachable, true); + for (let minute = 0; minute < 60; minute += 1) { + yield* TestClock.adjust("60 seconds"); + } + expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( + "BearerConnectionTarget", + ); + }).pipe(Effect.provide(TestClock.layer())), + ); +}); diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts index 79e6802805205..0964db0359b18 100644 --- a/packages/client-runtime/src/connection/supervisor.ts +++ b/packages/client-runtime/src/connection/supervisor.ts @@ -1332,40 +1332,66 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( // Registered before the fibers below so closing the scope interrupts them first. // A loop still running when the queue shuts down would report that as a failure. yield* Effect.addFinalizer(() => Queue.shutdown(signals).pipe(Effect.andThen(clearLease))); - yield* connectivity.changes.pipe( - Stream.runForEach((network) => - Ref.modify(intent, (current) => { - const next = targetNetwork(network); - return current.network === next - ? [false, current] - : ([true, { ...current, network: next }] as const); - }).pipe( - Effect.flatMap((changed) => - changed - ? (network === "offline" ? Effect.void : forgetSilentRoutes).pipe( - Effect.andThen(signal({ _tag: "NetworkChanged", network })), - ) - : Effect.void, - ), + // A defect in a background listener would otherwise end it for good, silently. + // Untraced: a span here would stay open for the supervisor's whole lifetime. + const keepRunning = Effect.fnUntraced( + function* (task: string, effect: Effect.Effect, onRestart?: Effect.Effect) { + let current = effect; + while (true) { + const exit = yield* Effect.exit(current); + if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) return yield* exit; + yield* Effect.logError("Connection supervisor task failed; restarting it.").pipe( + Effect.annotateLogs({ + "environment.id": target.environmentId, + "environment.label": target.label, + task, + ...safeErrorLogAttributes(exit.cause.reasons.find(Cause.isDieReason)?.defect), + }), + ); + yield* Effect.sleep(RETRY_BASE_DELAY_MS); + current = onRestart ? onRestart.pipe(Effect.andThen(effect)) : effect; + } + }, + (self) => Effect.forkScoped(self), + ); + + const applyNetwork = (network: NetworkStatus) => + Ref.modify(intent, (current) => { + const next = targetNetwork(network); + return current.network === next + ? [false, current] + : ([true, { ...current, network: next }] as const); + }).pipe( + Effect.flatMap((changed) => + changed + ? (network === "offline" ? Effect.void : forgetSilentRoutes).pipe( + Effect.andThen(signal({ _tag: "NetworkChanged", network })), + ) + : Effect.void, ), - ), - Effect.forkScoped, + ); + // A restarted listener may have missed changes while it was down. + yield* keepRunning( + "network-listener", + connectivity.changes.pipe(Stream.runForEach(applyNetwork)), + connectivity.status.pipe(Effect.flatMap(applyNetwork)), ); - yield* wakeups.changes.pipe( - Stream.filter(ConnectionWakeups.reachesConnectionSupervisor), - Stream.runForEach((reason) => - // A long suspension can hide a network change the platform never reported. - (reason === "network-changed" || reason === "application-active-long" - ? forgetSilentRoutes - : Effect.void - ).pipe(Effect.andThen(signal({ _tag: "Wakeup", reason }))), + yield* keepRunning( + "wakeup-listener", + wakeups.changes.pipe( + Stream.filter(ConnectionWakeups.reachesConnectionSupervisor), + Stream.runForEach((reason) => + // A long suspension can hide a network change the platform never reported. + (reason === "network-changed" || reason === "application-active-long" + ? forgetSilentRoutes + : Effect.void + ).pipe(Effect.andThen(signal({ _tag: "Wakeup", reason }))), + ), ), - Effect.forkScoped, ); - yield* Queue.take(betterRouteChecks).pipe( - Effect.flatMap(checkBetterRoutes), - Effect.forever, - Effect.forkScoped, + yield* keepRunning( + "better-route-checker", + Queue.take(betterRouteChecks).pipe(Effect.flatMap(checkBetterRoutes), Effect.forever), ); // A restarted loop has forgotten its retry ladder, so it waits out the // ladder's ceiling unless a signal cuts the wait short.