From 89017c15a0af177645fee044226d9d794b028051 Mon Sep 17 00:00:00 2001 From: viktorce Date: Thu, 19 Nov 2026 22:47:56 +0000 Subject: [PATCH 1/2] fix(client-runtime): an unexpected failure no longer stops the reconnect loop for good The connection supervisor's reconnect loop ran as one forked fiber with nothing around it. A defect or a stray interruption in it ended the fiber silently. The state stayed where it was and Retry did nothing. One route real code takes: a session's `closed` can fail with a cause that mixes a failure with an interruption, and `exitUnlessInterrupted` turned any cause holding an interruption into an interruption of the loop. The loop now survives all of that: - `runAttempt` uses `Effect.exit` where it used `exitUnlessInterrupted`, so a stray interruption becomes a failed attempt that keeps whether the session was live and stable. - A defect elsewhere in an attempt is caught around the attempt, logged, and retried on the normal backoff ladder. - A defect from closing an attempt's scope is logged as a failed release, and the attempt's outcome stands. - The generation number advances where the session is published as connected, so no lost outcome or restart gives two sessions the same number. - A defect in the loop outside an attempt is logged, and the loop restarts after the longest retry delay or the next signal. - The shutdown finalizer is registered before the fibers, so closing a supervisor does not look like a failure to the loop. A failure inside the restart handler itself still ends the loop, after one log line. The supervisor's other background fibers are unchanged. --- .../src/connection/supervisor.test.ts | 397 +++++++++++++++++- .../src/connection/supervisor.ts | 213 ++++++---- ...ig-stream-failing-while-interrupted.e2e.ts | 85 ++++ 3 files changed, 589 insertions(+), 106 deletions(-) create mode 100644 scripts/scenarios/reconnect-survives-config-stream-failing-while-interrupted.e2e.ts diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index 46b9757d0cd83..b2d67a38865c6 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -6,14 +6,19 @@ import { } from "@t3tools/contracts"; import { describe, expect, it } from "@effect/vitest"; import { RelayClientTracer } from "@t3tools/shared/relayTracing"; +import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Random from "effect/Random"; import * as Ref from "effect/Ref"; +import * as References from "effect/References"; +import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; import * as Clock from "effect/Clock"; @@ -167,6 +172,18 @@ function awaitState( ); } +function recordLogs(logs: Array, annotations?: Array) { + return Logger.layer( + [ + Logger.make(({ message, fiber }) => { + logs.push(message); + annotations?.push(fiber.getRef(References.CurrentLogAnnotations)); + }), + ], + { mergeWithExisting: false }, + ); +} + const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?: { readonly networkStatus?: NetworkStatus; readonly prepare?: ( @@ -174,6 +191,7 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?: target: ConnectionTarget, ) => Effect.Effect; readonly ready?: (attempt: number) => Effect.Effect; + readonly release?: (attempt: number) => Effect.Effect; readonly probe?: (attempt: number) => Effect.Effect; readonly configEvents?: Stream.Stream | undefined; readonly initialConfig?: (attempt: number) => Effect.Effect; @@ -245,7 +263,10 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?: inboundFrames: options?.inboundFrames ?? Effect.succeed(0), closed: Deferred.await(closed), } satisfies RpcSession.RpcSession), - () => Ref.update(releaseCount, (count) => count + 1), + () => + Ref.update(releaseCount, (count) => count + 1).pipe( + Effect.andThen(options?.release?.(attempt) ?? Effect.void), + ), ); yield* reportProgress({ stage: "synchronizing", prepared }); @@ -308,12 +329,12 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?: reason, })), closeLatestSession: Effect.fn("TestConnectionHarness.closeLatestSession")(function* ( - error = transient("Session closed."), + exit: Exit.Exit = Exit.fail(transient("Session closed.")), ) { const sessions = yield* Ref.get(closedSessions); const latest = sessions.at(-1); if (latest) { - yield* Deferred.fail(latest, error); + yield* Deferred.done(latest, exit); } }), }; @@ -581,9 +602,11 @@ describe("EnvironmentSupervisor", () => { ? Effect.die(new Error("Native transport defect.")) : Effect.succeed(PREPARED_CONNECTION), }); + const logs: unknown[] = []; + const annotations: unknown[] = []; const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { initiallyDesired: true, - }).pipe(Effect.provide(harness.dependencies)); + }).pipe(Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs, annotations)))); const failed = yield* awaitState( supervisor.state, @@ -596,6 +619,10 @@ describe("EnvironmentSupervisor", () => { message: "Test environment connection failed unexpectedly.", }, }); + expect(logs).toEqual([["Connection attempt failed with an unexpected defect."]]); + expect(annotations).toMatchObject([ + { "environment.id": RELAY_TARGET.environmentId, errorName: "Error" }, + ]); yield* TestClock.adjust("3 seconds"); yield* awaitState(supervisor.state, (state) => state.phase === "connected"); @@ -603,42 +630,380 @@ describe("EnvironmentSupervisor", () => { }).pipe(Effect.provide(TestClock.layer())), ); - it.effect("explicit retry interrupts the current backoff", () => + it.effect.each([ + { + label: "the attempt's network-status read dies", + firstPrepare: Effect.never, + statusDefect: true, + log: "Connection attempt failed with an unexpected defect.", + }, + { + label: "an attempt interrupts itself", + firstPrepare: Effect.interrupt, + statusDefect: false, + log: "Connection attempt failed with an unexpected interruption.", + }, + ])("retries after $label", ({ firstPrepare, statusDefect, log }) => + Effect.gen(function* () { + const harness = yield* makeHarness({ + prepare: (attempt) => (attempt === 1 ? firstPrepare : Effect.succeed(PREPARED_CONNECTION)), + }); + const statusReads = yield* Ref.make(0); + const logs: unknown[] = []; + const annotations: Array> = []; + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe( + Effect.provideService( + Connectivity.Connectivity, + Connectivity.Connectivity.of({ + // The supervisor reads once at startup; the second read belongs to the first attempt. + status: Ref.updateAndGet(statusReads, (count) => count + 1).pipe( + Effect.flatMap((count) => + statusDefect && count === 2 + ? Effect.die(new Error("Platform connectivity defect.")) + : Effect.succeed("online" as const), + ), + ), + changes: Stream.never, + }), + ), + Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs, annotations))), + ); + + const failed = yield* awaitState( + supervisor.state, + (state) => state.phase === "backoff" && state.attempt === 1, + ); + expect(failed).toMatchObject({ + generation: 0, + lastFailure: { _tag: "ConnectionTransientError", reason: "transport" }, + }); + expect(logs).toEqual([[log]]); + expect(Object.hasOwn(annotations[0]!, "errorType")).toBe(statusDefect); + + yield* TestClock.adjust("999 millis"); + expect(yield* Ref.get(harness.prepareCount)).toBe(1); + yield* TestClock.adjust("2 seconds"); + yield* awaitState(supervisor.state, (state) => state.phase === "connected"); + expect(yield* Ref.get(harness.prepareCount)).toBe(2); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect.each([ + { + label: "is interrupted", + closed: Exit.interrupt(), + releaseDefect: false, + log: "Connection attempt failed with an unexpected interruption.", + }, + { + label: "fails and dies while interrupted", + closed: Exit.failCause( + Cause.combine( + Cause.combine( + Cause.fail(transient("Session closed.")), + Cause.die(new Error("Session closed defect.")), + ), + Cause.interrupt(), + ), + ), + releaseDefect: false, + log: "Connection attempt failed with an unexpected defect.", + }, + { + label: "fails while interrupted", + closed: Exit.failCause( + Cause.combine(Cause.fail(transient("Session closed.")), Cause.interrupt()), + ), + releaseDefect: false, + log: null, + }, + { + label: "closes while its release defects", + closed: Exit.fail(transient("Session closed.")), + releaseDefect: true, + log: "Connection release failed with an unexpected defect.", + }, + ])("reconnects soon after a stable session $label", ({ closed, releaseDefect, log }) => Effect.gen(function* () { const harness = yield* makeHarness({ prepare: (attempt) => attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION), + release: (session) => + releaseDefect && session === 1 + ? Effect.die(new Error("Session release defect.")) + : Effect.void, }); + const logs: unknown[] = []; const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { initiallyDesired: true, - }).pipe(Effect.provide(harness.dependencies)); + }).pipe(Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs)))); yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); - yield* supervisor.retryNow; + yield* TestClock.adjust("2 seconds"); + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 1, + ); + yield* TestClock.adjust("1 minute"); + yield* harness.closeLatestSession(closed); + const lost = yield* awaitState(supervisor.state, (state) => state.phase !== "connected"); + expect(logs).toEqual(log === null ? [] : [[log]]); + expect(lost).toMatchObject({ phase: "backoff", generation: 1 }); + expect(lost.retryAt).toBeLessThanOrEqual((yield* Clock.currentTimeMillis) + 250); + + yield* TestClock.adjust("250 millis"); + const replaced = yield* awaitState(supervisor.state, (state) => state.phase === "connected"); + expect(replaced.generation).toBe(2); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect.each([ + { + label: "its attempt defects after its session went live", + closed: Exit.interrupt(), + releaseDefect: false, + lost: { phase: "backoff", generation: 1, lastFailure: null }, + // The earlier failed attempt still counts, so this is the ladder's second step. + soonestMs: 2000, + latestMs: 4000, + }, + { + label: "logging its release defect dies", + closed: Exit.fail(transient("Session closed.")), + releaseDefect: true, + lost: { phase: "backoff", generation: 1 }, + soonestMs: 300_000, + latestMs: 300_000, + }, + ])( + "does not reuse a live session's generation when $label", + ({ closed, releaseDefect, lost, soonestMs, latestMs }) => + Effect.gen(function* () { + const harness = yield* makeHarness({ + prepare: (attempt) => + attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION), + release: (session) => + releaseDefect && session === 1 + ? Effect.die(new Error("Session release defect.")) + : Effect.void, + }); + let logCalls = 0; + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe( + Effect.provide( + Layer.merge( + harness.dependencies, + Logger.layer( + [ + // The first log line throws, which is how each row injects its defect. + Logger.make(() => { + logCalls += 1; + if (logCalls === 1) throw new Error("Logger defect."); + }), + ], + { mergeWithExisting: false }, + ), + ), + ), + ); + + yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); + yield* TestClock.adjust("2 seconds"); + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 1, + ); + yield* harness.closeLatestSession(closed); + const backoff = yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); + expect(logCalls).toBe(2); + expect(backoff).toMatchObject(lost); + const retryInMs = (backoff.retryAt ?? 0) - (yield* Clock.currentTimeMillis); + expect(retryInMs).toBeGreaterThanOrEqual(soonestMs); + expect(retryInMs).toBeLessThanOrEqual(latestMs); + + yield* TestClock.adjust(`${latestMs} millis`); + const replaced = yield* awaitState( + supervisor.state, + (state) => state.phase === "connected", + ); + expect(replaced.generation).toBe(2); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect("keeps a released session's outcome", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ + release: () => Effect.die(new Error("Session release defect.")), + }); + const logs: unknown[] = []; + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs)))); + yield* awaitState(supervisor.state, (state) => state.phase === "connected"); + yield* supervisor.disconnect; + const released = yield* awaitState(supervisor.state, (state) => state.phase !== "connected"); + expect(released.phase).toBe("available"); + expect(logs).toEqual([["Connection release failed with an unexpected defect."]]); + }).pipe(Effect.provide(TestClock.layer())), + ); - expect(yield* Ref.get(harness.prepareCount)).toBe(2); - }), + it.effect("restarts the loop after a defect outside an attempt", () => + Effect.gen(function* () { + const harness = yield* makeHarness(); + const logs: unknown[] = []; + let randomReads = 0; + const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { + initiallyDesired: true, + }).pipe( + Effect.provideService(Random.Random, { + // Both session losses below die here, so the second reaches the restart handler again. + nextDoubleUnsafe: () => { + randomReads += 1; + if (randomReads <= 2) throw new Error("Random defect."); + return 1 - Number.EPSILON; + }, + nextIntUnsafe: () => 0, + }), + Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs))), + ); + + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 1, + ); + yield* harness.closeLatestSession(); + const restarted = yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); + expect(logs).toEqual([["Connection loop failed with an unexpected defect."]]); + expect(restarted).toMatchObject({ + attempt: 1, + generation: 1, + lastFailure: { _tag: "ConnectionTransientError", reason: "transport" }, + retryAt: (yield* Clock.currentTimeMillis) + 300_000, + }); + + yield* TestClock.adjust("299999 millis"); + expect(yield* Ref.get(harness.sessionCount)).toBe(1); + yield* TestClock.adjust("1 millis"); + const reconnected = yield* awaitState( + supervisor.state, + (state) => state.phase === "connected", + ); + expect(reconnected.generation).toBe(2); + + yield* harness.setNetworkStatus("offline"); + yield* harness.closeLatestSession(); + const offline = yield* awaitState(supervisor.state, (state) => state.phase !== "connected"); + expect(offline).toMatchObject({ + phase: "offline", + attempt: 1, + lastFailure: { _tag: "ConnectionTransientError", reason: "transport" }, + }); + + yield* supervisor.retryNow; + const retried = yield* awaitState(supervisor.state, (state) => state.phase === "connected"); + expect(retried.generation).toBe(3); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect.each(["connecting", "connected"] as const)( + "stops attempting when its scope closes while %s", + (phase) => + Effect.gen(function* () { + const preparing = yield* Deferred.make(); + const harness = yield* makeHarness({ + prepare: () => + Deferred.succeed(preparing, undefined).pipe( + Effect.andThen( + phase === "connected" ? Effect.succeed(PREPARED_CONNECTION) : Effect.never, + ), + ), + }); + const logs: unknown[] = []; + const scope = yield* Scope.make(); + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe( + Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs))), + Scope.provide(scope), + ); + + yield* Deferred.await(preparing); + yield* awaitState(supervisor.state, (state) => state.phase === phase); + // Yielding between finalizers lets the loop run if the queue shuts down before its interrupt. + yield* Scope.close(scope, Exit.void).pipe( + Effect.provideService(References.MaxOpsBeforeYield, 1), + ); + yield* TestClock.adjust("10 minutes"); + expect(logs).toEqual([]); + expect(yield* Ref.get(harness.prepareCount)).toBe(1); + }).pipe(Effect.provide(TestClock.layer())), ); - it.effect("keeps blocked failures idle until an external signal requests another attempt", () => + it.effect("explicit retry interrupts the current backoff", () => Effect.gen(function* () { const harness = yield* makeHarness({ prepare: (attempt) => - attempt === 1 ? Effect.fail(blocked()) : Effect.succeed(PREPARED_CONNECTION), + attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION), }); const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { initiallyDesired: true, }).pipe(Effect.provide(harness.dependencies)); - yield* awaitState(supervisor.state, (state) => state.phase === "blocked"); - yield* TestClock.adjust("1 hour"); - expect(yield* Ref.get(harness.prepareCount)).toBe(1); - + yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); yield* supervisor.retryNow; yield* awaitState(supervisor.state, (state) => state.phase === "connected"); + expect(yield* Ref.get(harness.prepareCount)).toBe(2); - }).pipe(Effect.provide(TestClock.layer())), + }), + ); + + it.effect.each([ + { label: "fails", cause: Cause.fail(blocked()) }, + { + label: "fails while interrupted", + cause: Cause.combine(Cause.fail(blocked()), Cause.interrupt()), + }, + ])( + "keeps a blocked host that $label idle until an external signal requests another attempt", + ({ cause }) => + Effect.gen(function* () { + const harness = yield* makeHarness(); + const driver = yield* ConnectionDriver.ConnectionDriver.pipe( + Effect.provide(harness.dependencies), + ); + const connects = yield* Ref.make(0); + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe( + // The driver keeps only the typed error of a failed route, so the cause is mixed here. + Effect.provideService(ConnectionDriver.ConnectionDriver, { + ...driver, + connect: (entry, reportProgress) => + Ref.updateAndGet(connects, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 1 ? Effect.failCause(cause) : driver.connect(entry, reportProgress), + ), + ), + }), + Effect.provide(harness.dependencies), + ); + + const failed = yield* awaitState(supervisor.state, (state) => state.lastFailure !== null); + expect(failed).toMatchObject({ + phase: "blocked", + lastFailure: { _tag: "ConnectionBlockedError", detail: "Authentication required." }, + }); + yield* TestClock.adjust("1 hour"); + expect(yield* Ref.get(connects)).toBe(1); + + yield* supervisor.retryNow; + yield* awaitState(supervisor.state, (state) => state.phase === "connected"); + expect(yield* Ref.get(connects)).toBe(2); + }).pipe(Effect.provide(TestClock.layer())), ); it.effect("retries a blocked connection when platform credentials change", () => diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts index 58411d795b7fc..79e6802805205 100644 --- a/packages/client-runtime/src/connection/supervisor.ts +++ b/packages/client-runtime/src/connection/supervisor.ts @@ -149,16 +149,6 @@ type EstablishmentEvent = | { readonly _tag: "Interrupted"; readonly resetRetry: boolean } | { readonly _tag: "TimedOut" }; -function exitUnlessInterrupted( - effect: Effect.Effect, -): Effect.Effect, never, R> { - return Effect.matchCauseEffect(effect, { - onFailure: (cause) => - Cause.hasInterrupts(cause) ? Effect.interrupt : Effect.succeed(Exit.failCause(cause)), - onSuccess: (value) => Effect.succeed(Exit.succeed(value)), - }); -} - export interface EnvironmentSupervisorOptions { readonly initiallyDesired?: boolean; readonly learnRoutes?: (input: { @@ -247,37 +237,27 @@ function connectingState( }; } -function failureFromExit( +function failureFromCause( target: ConnectionTarget, - exit: Exit.Exit, + cause: Cause.Cause, established: boolean, stable: boolean, -): AttemptOutcome { - if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) { - return { _tag: "Interrupted", established, stable, resetRetry: false }; - } - const typedFailure = exit.cause.reasons.find(Cause.isFailReason); - if (typedFailure) { - return { - _tag: "Failure", - established, - stable, - failure: typedFailure.error, - }; - } +) { return { _tag: "Failure", established, stable, - failure: { - error: new ConnectionTransientError({ - reason: "transport", - ...translatableDetail( - msg("{host} connection failed unexpectedly.", { host: target.label }), - ), - }), - attemptSpan: Option.none(), - }, + failure: cause.reasons.find(Cause.isFailReason)?.error ?? unexpectedFailure(target), + } satisfies AttemptOutcome; +} + +function unexpectedFailure(target: ConnectionTarget): TracedAttemptFailure { + return { + error: new ConnectionTransientError({ + reason: "transport", + ...translatableDetail(msg("{host} connection failed unexpectedly.", { host: target.label })), + }), + attemptSpan: Option.none(), }; } @@ -496,9 +476,27 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( }), ); + const logUnexpectedFailure = ( + failed: "attempt" | "release" | "loop", + cause: Cause.Cause, + ) => { + const died = cause.reasons.find(Cause.isDieReason); + if (!died && cause.reasons.some(Cause.isFailReason)) return Effect.void; + return Effect.logError( + `Connection ${failed} failed with an unexpected ${died ? "defect" : "interruption"}.`, + ).pipe( + Effect.annotateLogs({ + "environment.id": target.environmentId, + "environment.label": target.label, + "cause.reason_count": cause.reasons.length, + ...(died ? safeErrorLogAttributes(died.defect) : {}), + }), + ); + }; + const reportProgress = Effect.fn("EnvironmentSupervisor.reportProgress")(function* ( attempt: number, - generation: number, + nextGeneration: number, lastFailure: ConnectionAttemptError | null, progress: ConnectionDriver.ConnectionDriverProgress, ) { @@ -506,13 +504,13 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( yield* SubscriptionRef.set(prepared, Option.some(progress.prepared)); } yield* setState( - connectingState(yield* Ref.get(intent), generation, attempt, lastFailure, progress.stage), + connectingState(yield* Ref.get(intent), nextGeneration, attempt, lastFailure, progress.stage), ); }); const establishConnection = Effect.fnUntraced(function* ( attempt: number, - generation: number, + nextGeneration: number, lastFailure: ConnectionAttemptError | null, synchronizedRoutes: Ref.Ref, ) { @@ -520,7 +518,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( (progress.stage === "synchronizing" ? Ref.update(synchronizedRoutes, (count) => count + 1) : Effect.void - ).pipe(Effect.andThen(reportProgress(attempt, generation, lastFailure, progress))), + ).pipe(Effect.andThen(reportProgress(attempt, nextGeneration, lastFailure, progress))), ); }); @@ -559,7 +557,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( Scope.Scope >, attempt: number, - generation: number, + nextGeneration: number, pendingRetry: Option.Option, ) => { // The span must end with the ConnectionAttemptError itself: the exporter @@ -572,7 +570,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( "environment.id": target.environmentId, "environment.target.kind": target._tag, "connection.attempt": attempt, - "connection.generation": generation, + "connection.generation": nextGeneration, "connection.retry.failure_count": Option.match(pendingRetry, { onNone: () => 0, onSome: (retry) => retry.failureCount, @@ -599,20 +597,25 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( const establishTracedConnection = Effect.fnUntraced(function* ( attempt: number, - generation: number, + nextGeneration: number, lastFailure: ConnectionAttemptError | null, pendingRetry: Option.Option, synchronizedRoutes: Ref.Ref, ) { if (usesRelay) { return yield* traceRelayEstablishment( - establishConnection(attempt, generation, lastFailure, synchronizedRoutes), + establishConnection(attempt, nextGeneration, lastFailure, synchronizedRoutes), attempt, - generation, + nextGeneration, pendingRetry, ); } - return yield* establishConnection(attempt, generation, lastFailure, synchronizedRoutes).pipe( + return yield* establishConnection( + attempt, + nextGeneration, + lastFailure, + synchronizedRoutes, + ).pipe( Effect.map((lease) => ({ attemptSpan: Option.none(), lease, @@ -946,9 +949,11 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( } }); + // Advanced as a session goes live and kept across loop restarts, so no two sessions share one. + let generation = 0; const runAttempt = Effect.fnUntraced(function* ( attempt: number, - generation: number, + nextGeneration: number, lastFailure: ConnectionAttemptError | null, pendingRetry: Option.Option, ignoreOffline: boolean, @@ -957,10 +962,10 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( yield* SubscriptionRef.set(prepared, Option.none()); const synchronizedRoutes = yield* Ref.make(0); const establishment = yield* Effect.raceAllFirst([ - exitUnlessInterrupted( + Effect.exit( establishTracedConnection( attempt, - generation, + nextGeneration, lastFailure, pendingRetry, synchronizedRoutes, @@ -1031,22 +1036,8 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( } satisfies AttemptOutcome; } if (Exit.isFailure(establishment.exit)) { - const isUnexpectedDefect = - !Cause.hasInterruptsOnly(establishment.exit.cause) && - !establishment.exit.cause.reasons.some(Cause.isFailReason); - const outcome = failureFromExit(target, establishment.exit, false, false); - if (isUnexpectedDefect) { - const defect = establishment.exit.cause.reasons.find(Cause.isDieReason)?.defect; - yield* Effect.logError("Connection attempt failed with an unexpected defect.").pipe( - Effect.annotateLogs({ - "environment.id": target.environmentId, - "environment.label": target.label, - "cause.reason_count": establishment.exit.cause.reasons.length, - ...safeErrorLogAttributes(defect), - }), - ); - } - return outcome; + yield* logUnexpectedFailure("attempt", establishment.exit.cause); + return failureFromCause(target, establishment.exit.cause, false, false); } const active = establishment.exit.value; @@ -1075,7 +1066,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( const connectedLease: ConnectedLease = { ...active.lease, activeRoute: routes.find((route) => connectionRouteId(route.target) === activeRouteId), - generation, + generation: nextGeneration, }; const refusedRouteIds = yield* Ref.updateAndGet(refusedRoutes, (known) => mergeRefusedRoutes( @@ -1086,6 +1077,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( ), ); const connectedAt = yield* Clock.currentTimeMillis; + generation = nextGeneration; yield* SubscriptionRef.set(prepared, Option.some(connectedLease.prepared)); yield* SubscriptionRef.set(session, Option.some(connectedLease.session)); if (options?.learnRoutes !== undefined) { @@ -1097,7 +1089,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( phase: "connected", stage: null, attempt, - generation, + generation: nextGeneration, lastFailure: null, retryAt: null, refusedRouteIds, @@ -1116,7 +1108,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( attemptSpan: active.attemptSpan, })), ), - ).pipe(exitUnlessInterrupted); + ).pipe(Effect.exit); const connectedForMs = (yield* Clock.currentTimeMillis) - connectedAt; if (Exit.isSuccess(connectedExit)) { return { @@ -1126,27 +1118,26 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( resetRetry: connectedExit.value, } satisfies AttemptOutcome; } - const outcome = failureFromExit( + yield* logUnexpectedFailure("attempt", connectedExit.cause); + const outcome = failureFromCause( target, - connectedExit, + connectedExit.cause, true, connectedForMs >= BACKOFF_RESET_AFTER_MS, ); - if (outcome._tag === "Failure") { - yield* Effect.void.pipe( - Effect.withSpan("EnvironmentSupervisor.connectionLost", { - root: true, - attributes: { - "environment.id": target.environmentId, - "environment.label": target.label, - "environment.target.kind": target._tag, - "connection.connected_ms": connectedForMs, - "connection.failure.reason": outcome.failure.error.reason, - "connection.failure.detail": outcome.failure.error.detail, - }, - }), - ); - } + yield* Effect.void.pipe( + Effect.withSpan("EnvironmentSupervisor.connectionLost", { + root: true, + attributes: { + "environment.id": target.environmentId, + "environment.label": target.label, + "environment.target.kind": target._tag, + "connection.connected_ms": connectedForMs, + "connection.failure.reason": outcome.failure.error.reason, + "connection.failure.detail": outcome.failure.error.detail, + }, + }), + ); return outcome; }, Effect.ensuring(clearLease)); @@ -1185,7 +1176,6 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( // Approval waits count as attempts but not toward backoff, so a later outage starts low. let backoffFailures = 0; let approvalWaits = 0; - let generation = 0; let latestFailure: ConnectionAttemptError | null = null; let pendingRetry = Option.none(); const resetRetryLadder = () => { @@ -1229,13 +1219,25 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( const attempt = failureCount + 1; const nextGeneration = generation + 1; - const outcome: AttemptOutcome = yield* Effect.scoped( - runAttempt(attempt, nextGeneration, latestFailure, pendingRetry, ignoreOffline), + const outcome: AttemptOutcome = yield* Effect.acquireUseRelease( + Scope.make(), + (scope) => + runAttempt(attempt, nextGeneration, latestFailure, pendingRetry, ignoreOffline).pipe( + Scope.provide(scope), + Effect.catchCause((cause) => + logUnexpectedFailure("attempt", cause).pipe( + Effect.as(failureFromCause(target, cause, generation === nextGeneration, false)), + ), + ), + ), + (scope, exit) => + Scope.close(scope, exit).pipe( + Effect.catchCause((cause) => logUnexpectedFailure("release", cause)), + ), ); replacing = false; const failedProbe = yield* Ref.getAndSet(probeUnanswered, false); if (outcome.established) { - generation = nextGeneration; // A session came up, so no earlier failure describes the connection. // Losing a live session is not a failed connection either, for example // a socket the OS killed while the app was suspended: its replacement @@ -1327,6 +1329,9 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( } }); + // Registered before the fibers below so closing the scope interrupts them first. + // A loop still running when the queue shuts down would report that as a failure. + yield* Effect.addFinalizer(() => Queue.shutdown(signals).pipe(Effect.andThen(clearLease))); yield* connectivity.changes.pipe( Stream.runForEach((network) => Ref.modify(intent, (current) => { @@ -1362,7 +1367,37 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( Effect.forever, Effect.forkScoped, ); - yield* run().pipe(Effect.forkScoped); + // A restarted loop has forgotten its retry ladder, so it waits out the + // ladder's ceiling unless a signal cuts the wait short. + yield* run().pipe( + Effect.catchCause((cause) => + Effect.gen(function* () { + yield* logUnexpectedFailure("loop", cause); + const currentIntent = yield* Ref.get(intent); + const lastFailure = unexpectedFailure(target).error; + yield* setState( + !currentIntent.desired + ? availableState(currentIntent, generation) + : currentIntent.network === "offline" + ? offlineState(currentIntent, generation, 1, lastFailure) + : { + desired: true, + network: currentIntent.network, + phase: "backoff", + stage: null, + attempt: 1, + generation, + lastFailure, + retryAt: (yield* Clock.currentTimeMillis) + RETRY_MAX_DELAY_MS, + refusedRouteIds: NO_REFUSED_ROUTES, + }, + ); + yield* waitForRetrySignal(RETRY_MAX_DELAY_MS); + }), + ), + Effect.forever, + Effect.forkScoped, + ); const connect = Ref.update(intent, (current) => ({ ...current, @@ -1385,8 +1420,6 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( Effect.withSpan("EnvironmentSupervisor.retryNow"), ); - yield* Effect.addFinalizer(() => Queue.shutdown(signals).pipe(Effect.andThen(clearLease))); - return EnvironmentSupervisor.of({ target, state, diff --git a/scripts/scenarios/reconnect-survives-config-stream-failing-while-interrupted.e2e.ts b/scripts/scenarios/reconnect-survives-config-stream-failing-while-interrupted.e2e.ts new file mode 100644 index 0000000000000..8e6a2606c1ba9 --- /dev/null +++ b/scripts/scenarios/reconnect-survives-config-stream-failing-while-interrupted.e2e.ts @@ -0,0 +1,85 @@ +import { expect } from "e2e"; +import * as Effect from "effect/Effect"; + +import { runPromise, test, WEB } from "./t3.ts"; + +interface Fault { + readonly failConfigStreamWhileInterrupted: () => boolean; +} + +test( + "The app reconnects after its config stream fails while interrupted", + WEB, + async ({ t3, app, screen, browser }) => { + const project = await t3.project("demo"); + const thread = await t3.thread({ + project, + title: "Earlier work", + messages: ["Print the JSON"], + model: "command-output", + }); + // Injected fault: the page hands the app one server frame that ends the config + // subscription with a typed failure and an interruption in the same cause. + await browser.addInitScript(() => { + let deliver: (() => void) | null = null; + const NativeWebSocket = window.WebSocket; + window.WebSocket = class extends NativeWebSocket { + override send(data: Parameters[0]) { + if (typeof data === "string" && data.includes('"tag":"subscribeServerConfig"')) { + const requestId = (JSON.parse(data) as { id: number | string }).id; + deliver = () => + this.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ + _tag: "Exit", + requestId, + exit: { + _tag: "Failure", + cause: [ + { + _tag: "Fail", + error: { + _tag: "KeybindingsConfigParseError", + configPath: "/demo/keybindings.json", + detail: "Injected failure.", + }, + }, + { _tag: "Interrupt", fiberId: 1 }, + ], + }, + }), + }), + ); + } + super.send(data); + } + }; + (window as unknown as { __fault: Fault }).__fault = { + failConfigStreamWhileInterrupted: () => { + deliver?.(); + return deliver !== null; + }, + }; + }); + await browser.emulateMedia({ colorScheme: "dark", reducedMotion: "reduce" }); + await t3.open(thread); + + const reply = screen.getByText("The command printed one JSON line."); + await expect(reply.first()).toBeVisible(); + await app.screenshot("1-connected"); + + const delivered = await browser.evaluate(() => + (window as unknown as { __fault: Fault }).__fault.failConfigStreamWhileInterrupted(), + ); + expect(delivered).toBe(true); + // A live loop reconnects well within each wait; a dead one never does. + await runPromise(Effect.sleep("5 seconds")); + await app.screenshot("2-five-seconds-after-the-failure"); + + await screen.getByRole("textbox").fill("Are you still there?"); + await screen.getByRole("textbox").press("Enter"); + await runPromise(Effect.sleep("3 seconds")); + await app.screenshot("3-after-pressing-enter"); + await expect(reply.nth(1)).toBeVisible(); + }, +); From fda679a036720f915395f0f69b1edd5b35e4d415 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 19 Nov 2026 23:21:29 +0000 Subject: [PATCH 2/2] test(client-runtime): trim overlapping reconnect loop cases Co-Authored-By: Claude Opus 5.5 --- .../src/connection/supervisor.test.ts | 72 ++----------------- 1 file changed, 7 insertions(+), 65 deletions(-) diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index b2d67a38865c6..81c8bf4064a45 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -697,20 +697,6 @@ describe("EnvironmentSupervisor", () => { releaseDefect: false, log: "Connection attempt failed with an unexpected interruption.", }, - { - label: "fails and dies while interrupted", - closed: Exit.failCause( - Cause.combine( - Cause.combine( - Cause.fail(transient("Session closed.")), - Cause.die(new Error("Session closed defect.")), - ), - Cause.interrupt(), - ), - ), - releaseDefect: false, - log: "Connection attempt failed with an unexpected defect.", - }, { label: "fails while interrupted", closed: Exit.failCause( @@ -759,35 +745,13 @@ describe("EnvironmentSupervisor", () => { }).pipe(Effect.provide(TestClock.layer())), ); - it.effect.each([ - { - label: "its attempt defects after its session went live", - closed: Exit.interrupt(), - releaseDefect: false, - lost: { phase: "backoff", generation: 1, lastFailure: null }, - // The earlier failed attempt still counts, so this is the ladder's second step. - soonestMs: 2000, - latestMs: 4000, - }, - { - label: "logging its release defect dies", - closed: Exit.fail(transient("Session closed.")), - releaseDefect: true, - lost: { phase: "backoff", generation: 1 }, - soonestMs: 300_000, - latestMs: 300_000, - }, - ])( - "does not reuse a live session's generation when $label", - ({ closed, releaseDefect, lost, soonestMs, latestMs }) => + it.effect( + "does not reuse a live session's generation when its attempt defects after going live", + () => Effect.gen(function* () { const harness = yield* makeHarness({ prepare: (attempt) => attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION), - release: (session) => - releaseDefect && session === 1 - ? Effect.die(new Error("Session release defect.")) - : Effect.void, }); let logCalls = 0; const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { @@ -798,7 +762,7 @@ describe("EnvironmentSupervisor", () => { harness.dependencies, Logger.layer( [ - // The first log line throws, which is how each row injects its defect. + // Logging the lost session's interruption throws, so the attempt itself defects. Logger.make(() => { logCalls += 1; if (logCalls === 1) throw new Error("Logger defect."); @@ -816,15 +780,11 @@ describe("EnvironmentSupervisor", () => { supervisor.state, (state) => state.phase === "connected" && state.generation === 1, ); - yield* harness.closeLatestSession(closed); + yield* harness.closeLatestSession(Exit.interrupt()); const backoff = yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); - expect(logCalls).toBe(2); - expect(backoff).toMatchObject(lost); - const retryInMs = (backoff.retryAt ?? 0) - (yield* Clock.currentTimeMillis); - expect(retryInMs).toBeGreaterThanOrEqual(soonestMs); - expect(retryInMs).toBeLessThanOrEqual(latestMs); + expect(backoff).toMatchObject({ generation: 1, lastFailure: null }); - yield* TestClock.adjust(`${latestMs} millis`); + yield* TestClock.adjust("4 seconds"); const replaced = yield* awaitState( supervisor.state, (state) => state.phase === "connected", @@ -833,24 +793,6 @@ describe("EnvironmentSupervisor", () => { }).pipe(Effect.provide(TestClock.layer())), ); - it.effect("keeps a released session's outcome", () => - Effect.gen(function* () { - const harness = yield* makeHarness({ - release: () => Effect.die(new Error("Session release defect.")), - }); - const logs: unknown[] = []; - const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { - initiallyDesired: true, - }).pipe(Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs)))); - - yield* awaitState(supervisor.state, (state) => state.phase === "connected"); - yield* supervisor.disconnect; - const released = yield* awaitState(supervisor.state, (state) => state.phase !== "connected"); - expect(released.phase).toBe("available"); - expect(logs).toEqual([["Connection release failed with an unexpected defect."]]); - }).pipe(Effect.provide(TestClock.layer())), - ); - it.effect("restarts the loop after a defect outside an attempt", () => Effect.gen(function* () { const harness = yield* makeHarness();