From d680ed1a112f001894847feb8771cf54e2abbde3 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 22 Oct 2026 01:02:58 +0000 Subject: [PATCH 1/5] test(client-runtime): reproduce #7231 slow foreground probe tearing down a live session Co-Authored-By: Claude Opus 5.5 --- .../src/connection/supervisor.test.ts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index a1acb6f245f51..76342a3a05b22 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -775,6 +775,44 @@ describe("EnvironmentSupervisor", () => { }), ); + it.effect("keeps a live session through one slow foreground probe that later answers", () => + Effect.gen(function* () { + const probeCalls = yield* Ref.make(0); + const firstProbeStarted = yield* Deferred.make(); + const secondProbeAnswered = yield* Deferred.make(); + const harness = yield* makeHarness({ + probe: () => + Ref.updateAndGet(probeCalls, (count) => count + 1).pipe( + Effect.flatMap((call) => + call === 1 + ? Deferred.succeed(firstProbeStarted, undefined).pipe(Effect.andThen(Effect.never)) + : Deferred.succeed(secondProbeAnswered, undefined), + ), + ), + }); + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 1, + ); + yield* harness.wake("application-active"); + yield* Deferred.await(firstProbeStarted); + // A busy server misses the foreground probe deadline once; the socket + // never closed. + yield* TestClock.adjust("18 seconds"); + yield* awaitState(supervisor.state, (state) => state.phase === "connected"); + yield* harness.wake("application-active"); + yield* Deferred.await(secondProbeAnswered); + + expect(yield* Ref.get(harness.releaseCount)).toBe(0); + expect(yield* Ref.get(harness.sessionCount)).toBe(1); + expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(1); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("does not churn a healthy session when credentials change", () => Effect.gen(function* () { const harness = yield* makeHarness(); From fb1ceafea8dbdda4010a5504ec4f9ad7e54d60a1 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 22 Oct 2026 01:34:08 +0000 Subject: [PATCH 2/5] fix(client-runtime): one slow foreground probe no longer tears down a live session A busy server that missed a single application-active health probe deadline had its healthy websocket replaced. The connected lease now tolerates one missed foreground probe; an answered probe resets that tolerance, and a second consecutive miss replaces the session as before. Probe failures, quick probes, and long resumes are unchanged. Fixes #7231 Co-Authored-By: Claude Opus 5.5 --- .../src/connection/supervisor.test.ts | 40 +++++++++++++++++++ .../src/connection/supervisor.ts | 16 ++++++++ 2 files changed, 56 insertions(+) diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index 76342a3a05b22..bf390f6859d93 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -4,6 +4,7 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; import * as Random from "effect/Random"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; @@ -813,6 +814,45 @@ describe("EnvironmentSupervisor", () => { }).pipe(Effect.provide(TestClock.layer())), ); + it.effect("replaces a session after consecutive unanswered foreground probes", () => + Effect.gen(function* () { + const probeCalls = yield* Ref.make(0); + const probeStarted = yield* Queue.unbounded(); + const harness = yield* makeHarness({ + // Misses, answers, then misses twice in a row. + probe: () => + Ref.updateAndGet(probeCalls, (count) => count + 1).pipe( + Effect.tap((call) => Queue.offer(probeStarted, call)), + Effect.flatMap((call) => (call === 2 ? Effect.void : Effect.never)), + ), + }); + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 1, + ); + for (const attempt of [1, 2, 3]) { + yield* harness.wake("application-active"); + expect(yield* Queue.take(probeStarted)).toBe(attempt); + yield* TestClock.adjust("18 seconds"); + } + // The answer in between reset the tolerance, so the session survives. + expect(yield* Ref.get(harness.releaseCount)).toBe(0); + + yield* harness.wake("application-active"); + expect(yield* Queue.take(probeStarted)).toBe(4); + yield* TestClock.adjust("18 seconds"); + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 2, + ); + expect(yield* Ref.get(harness.releaseCount)).toBe(1); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("does not churn a healthy session when credentials change", () => Effect.gen(function* () { const harness = yield* makeHarness(); diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts index d500fe6d8347c..be6e46546f755 100644 --- a/packages/client-runtime/src/connection/supervisor.ts +++ b/packages/client-runtime/src/connection/supervisor.ts @@ -584,6 +584,9 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( } }; + const isForegroundProbe = (next: SupervisorSignal) => + next._tag === "Wakeup" && next.reason === "application-active"; + // Returning to the app, an explicit retry, and the network reporting offline // all probe the live session instead of replacing it, so a healthy socket is // not torn down (the offline report is often wrong, for example for a @@ -605,6 +608,9 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( Stream.runForEach(() => requestBetterRouteCheck(lease)), Effect.forkScoped, ); + // A busy server can miss one foreground probe while the socket is still + // healthy, so only a second consecutive miss replaces the session. + let missedForegroundProbe = false; for (;;) { const next = yield* takeSignal; const end = yield* connectedLeaseEnd(next, lease); @@ -623,6 +629,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( } yield* Ref.set(probeUnanswered, true); const probe = yield* Effect.forkChild(lease.session.probe); + let foreground = isForegroundProbe(next); let deadline = (yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(probeTimeout); for (;;) { const remaining = deadline - (yield* Clock.monotonicTimeNanos); @@ -637,6 +644,11 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( ]); if (probeEvent._tag === "TimedOut") { yield* Fiber.interrupt(probe); + if (foreground && !missedForegroundProbe) { + missedForegroundProbe = true; + yield* Ref.set(probeUnanswered, false); + break; + } return yield* new ConnectionTransientError({ reason: "timeout", detail: `${target.label} did not respond to a connection health check.`, @@ -644,6 +656,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( } if (probeEvent._tag === "ProbeCompleted") { if (Exit.isSuccess(probeEvent.exit)) { + missedForegroundProbe = false; yield* Ref.set(probeUnanswered, false); } yield* probeEvent.exit; @@ -656,6 +669,9 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( } const signalTimeout = probeTimeoutFor(probeEvent.signal); if (signalTimeout !== undefined) { + // A retry or network signal wants a definite answer, even when its + // own deadline lands after the foreground one. + if (!isForegroundProbe(probeEvent.signal)) foreground = false; const signalDeadline = (yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(signalTimeout); if (signalDeadline < deadline) deadline = signalDeadline; From 9173f32272472d5755503524ead4ea72215570dc Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 22 Oct 2026 01:36:39 +0000 Subject: [PATCH 3/5] test(client-runtime): a late retry still ends a slow foreground probe definitely An explicit retry that arrives near the end of an application-active probe has a later deadline than the probe, but it asks for a definite answer, so the one-miss foreground tolerance must not swallow it. Co-Authored-By: Claude Opus 5.5 --- .../src/connection/supervisor.test.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index bf390f6859d93..0b0b6e6074b31 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -853,6 +853,35 @@ describe("EnvironmentSupervisor", () => { }).pipe(Effect.provide(TestClock.layer())), ); + it.effect("replaces a session when a retry lands late in a slow foreground probe", () => + Effect.gen(function* () { + const probeStarted = yield* Deferred.make(); + const harness = yield* makeHarness({ + probe: () => Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never)), + }); + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 1, + ); + yield* harness.wake("application-active"); + yield* Deferred.await(probeStarted); + yield* TestClock.adjust("13 seconds"); + // The retry's own deadline falls after the foreground one, but it still + // asks for a definite answer, so the first miss is not forgiven. + yield* supervisor.retryNow; + yield* TestClock.adjust("5 seconds"); + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 2, + ); + expect(yield* Ref.get(harness.releaseCount)).toBe(1); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("does not churn a healthy session when credentials change", () => Effect.gen(function* () { const harness = yield* makeHarness(); From 3eaf43fe9ce30c36e3ea04ef0b9b99a466ec88e2 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 22 Oct 2026 07:00:30 +0000 Subject: [PATCH 4/5] fix(client-runtime): a missed foreground probe asks again on its own The first tolerated foreground probe timeout ended the probe without a second attempt, so a session whose transport kept answering pings but whose app RPC stalled stayed connected until the user refocused. The first miss now starts one follow-up probe with the same deadline; its answer keeps the lease and its miss replaces the session. Co-Authored-By: Claude Opus 5.5 --- .../src/connection/supervisor.test.ts | 97 +++++++++---------- .../src/connection/supervisor.ts | 22 ++--- 2 files changed, 55 insertions(+), 64 deletions(-) diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index 0b0b6e6074b31..42f8631c693e2 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -776,55 +776,50 @@ describe("EnvironmentSupervisor", () => { }), ); - it.effect("keeps a live session through one slow foreground probe that later answers", () => - Effect.gen(function* () { - const probeCalls = yield* Ref.make(0); - const firstProbeStarted = yield* Deferred.make(); - const secondProbeAnswered = yield* Deferred.make(); - const harness = yield* makeHarness({ - probe: () => - Ref.updateAndGet(probeCalls, (count) => count + 1).pipe( - Effect.flatMap((call) => - call === 1 - ? Deferred.succeed(firstProbeStarted, undefined).pipe(Effect.andThen(Effect.never)) - : Deferred.succeed(secondProbeAnswered, undefined), + it.effect( + "keeps a live session when the automatic follow-up to a slow foreground probe answers", + () => + Effect.gen(function* () { + const probeCount = yield* Ref.make(0); + const probeStarted = yield* Queue.unbounded(); + const harness = yield* makeHarness({ + probe: () => + Ref.updateAndGet(probeCount, (count) => count + 1).pipe( + Effect.tap((call) => Queue.offer(probeStarted, call)), + Effect.flatMap((call) => (call === 1 ? Effect.never : Effect.void)), ), - ), - }); - const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { - initiallyDesired: true, - }).pipe(Effect.provide(harness.dependencies)); - - yield* awaitState( - supervisor.state, - (state) => state.phase === "connected" && state.generation === 1, - ); - yield* harness.wake("application-active"); - yield* Deferred.await(firstProbeStarted); - // A busy server misses the foreground probe deadline once; the socket - // never closed. - yield* TestClock.adjust("18 seconds"); - yield* awaitState(supervisor.state, (state) => state.phase === "connected"); - yield* harness.wake("application-active"); - yield* Deferred.await(secondProbeAnswered); - - expect(yield* Ref.get(harness.releaseCount)).toBe(0); - expect(yield* Ref.get(harness.sessionCount)).toBe(1); - expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(1); - }).pipe(Effect.provide(TestClock.layer())), + }); + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 1, + ); + yield* harness.wake("application-active"); + expect(yield* Queue.take(probeStarted)).toBe(1); + // A busy server misses the foreground probe deadline once while the + // socket stays open; the supervisor asks again on its own. + yield* TestClock.adjust("15 seconds"); + expect(yield* Queue.take(probeStarted)).toBe(2); + yield* TestClock.adjust("30 seconds"); + + expect(yield* Ref.get(harness.releaseCount)).toBe(0); + expect(yield* Ref.get(harness.sessionCount)).toBe(1); + expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({ + phase: "connected", + generation: 1, + }); + }).pipe(Effect.provide(TestClock.layer())), ); - it.effect("replaces a session after consecutive unanswered foreground probes", () => + it.effect("replaces a session when the automatic follow-up probe also goes unanswered", () => Effect.gen(function* () { - const probeCalls = yield* Ref.make(0); - const probeStarted = yield* Queue.unbounded(); + const probeCount = yield* Ref.make(0); const harness = yield* makeHarness({ - // Misses, answers, then misses twice in a row. probe: () => - Ref.updateAndGet(probeCalls, (count) => count + 1).pipe( - Effect.tap((call) => Queue.offer(probeStarted, call)), - Effect.flatMap((call) => (call === 2 ? Effect.void : Effect.never)), - ), + Ref.update(probeCount, (count) => count + 1).pipe(Effect.andThen(Effect.never)), }); const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { initiallyDesired: true, @@ -834,21 +829,17 @@ describe("EnvironmentSupervisor", () => { supervisor.state, (state) => state.phase === "connected" && state.generation === 1, ); - for (const attempt of [1, 2, 3]) { - yield* harness.wake("application-active"); - expect(yield* Queue.take(probeStarted)).toBe(attempt); - yield* TestClock.adjust("18 seconds"); - } - // The answer in between reset the tolerance, so the session survives. - expect(yield* Ref.get(harness.releaseCount)).toBe(0); - + // One foreground event and no other signal: the follow-up's miss alone + // replaces the session. yield* harness.wake("application-active"); - expect(yield* Queue.take(probeStarted)).toBe(4); - yield* TestClock.adjust("18 seconds"); + yield* TestClock.adjust("15 seconds"); + yield* TestClock.adjust("15 seconds"); yield* awaitState( supervisor.state, (state) => state.phase === "connected" && state.generation === 2, ); + + expect(yield* Ref.get(probeCount)).toBe(2); expect(yield* Ref.get(harness.releaseCount)).toBe(1); }).pipe(Effect.provide(TestClock.layer())), ); diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts index be6e46546f755..f203ef1c03b5c 100644 --- a/packages/client-runtime/src/connection/supervisor.ts +++ b/packages/client-runtime/src/connection/supervisor.ts @@ -608,9 +608,6 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( Stream.runForEach(() => requestBetterRouteCheck(lease)), Effect.forkScoped, ); - // A busy server can miss one foreground probe while the socket is still - // healthy, so only a second consecutive miss replaces the session. - let missedForegroundProbe = false; for (;;) { const next = yield* takeSignal; const end = yield* connectedLeaseEnd(next, lease); @@ -628,8 +625,11 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( continue; } yield* Ref.set(probeUnanswered, true); - const probe = yield* Effect.forkChild(lease.session.probe); - let foreground = isForegroundProbe(next); + let probe = yield* Effect.forkChild(lease.session.probe); + // A busy server can miss one foreground probe while the socket is still + // healthy, so the first miss asks again and only the follow-up's miss + // replaces the session. + let toleratesMiss = isForegroundProbe(next); let deadline = (yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(probeTimeout); for (;;) { const remaining = deadline - (yield* Clock.monotonicTimeNanos); @@ -644,10 +644,11 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( ]); if (probeEvent._tag === "TimedOut") { yield* Fiber.interrupt(probe); - if (foreground && !missedForegroundProbe) { - missedForegroundProbe = true; - yield* Ref.set(probeUnanswered, false); - break; + if (toleratesMiss) { + toleratesMiss = false; + probe = yield* Effect.forkChild(lease.session.probe); + deadline = (yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(probeTimeout); + continue; } return yield* new ConnectionTransientError({ reason: "timeout", @@ -656,7 +657,6 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( } if (probeEvent._tag === "ProbeCompleted") { if (Exit.isSuccess(probeEvent.exit)) { - missedForegroundProbe = false; yield* Ref.set(probeUnanswered, false); } yield* probeEvent.exit; @@ -671,7 +671,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( if (signalTimeout !== undefined) { // A retry or network signal wants a definite answer, even when its // own deadline lands after the foreground one. - if (!isForegroundProbe(probeEvent.signal)) foreground = false; + if (!isForegroundProbe(probeEvent.signal)) toleratesMiss = false; const signalDeadline = (yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(signalTimeout); if (signalDeadline < deadline) deadline = signalDeadline; From f382f90df9004484c9f587fe122b26b9e1519255 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 02:06:03 +0000 Subject: [PATCH 5/5] fix(client-runtime): the foreground follow-up probe leaves the silent socket check alone AI-assisted; verified with the tests and tools listed above. Co-Authored-By: Claude Opus 5.5 --- packages/client-runtime/src/connection/supervisor.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts index 36e1ead9a5d6f..83fe1ecf4d928 100644 --- a/packages/client-runtime/src/connection/supervisor.ts +++ b/packages/client-runtime/src/connection/supervisor.ts @@ -844,16 +844,17 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( return true; } if (probeEvent._tag === "TimedOut") { - yield* stopProbe; - // A long suspension leaves the socket suspect even when the network - // reports offline, so it is replaced like a silent one. - if (longResume) return true; if (toleratesMiss) { toleratesMiss = false; + yield* Fiber.interrupt(probe); probe = yield* Effect.forkChild(lease.session.probe); deadline = (yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(probeTimeout); continue; } + yield* stopProbe; + // A long suspension leaves the socket suspect even when the network + // reports offline, so it is replaced like a silent one. + if (longResume) return true; return yield* new ConnectionTransientError({ reason: "timeout", detail: `${target.label} did not respond to a connection health check.`,