From cf39986f7871b0e4bacb8a0918a549414c34168a Mon Sep 17 00:00:00 2001 From: Mike Olson Date: Fri, 6 Nov 2026 13:44:08 -0500 Subject: [PATCH] fix(client-runtime): Recover failed subscriptions from a fresh snapshot A failed subscription may leave its cursor ahead of unapplied data. Upstream already retries the stream, but resuming from that cursor can lose updates. Mark a cached thread synchronizing and request a fresh snapshot on recovery, while retaining upstream's retry timing. Remove a deleted thread returned by the snapshot, since it replaces the replay that would carry thread.deleted. --- .../src/state/threads-sync.test.ts | 67 +++++++++++++++++++ packages/client-runtime/src/state/threads.ts | 27 ++++++-- 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/packages/client-runtime/src/state/threads-sync.test.ts b/packages/client-runtime/src/state/threads-sync.test.ts index 5a771646f03f..7cfbcc939b8f 100644 --- a/packages/client-runtime/src/state/threads-sync.test.ts +++ b/packages/client-runtime/src/state/threads-sync.test.ts @@ -2452,6 +2452,73 @@ describe("EnvironmentThreads", () => { }), ); + it.effect.each(["stream", "apply"] as const)( + "resubscribes from a fresh snapshot after a %s failure", + (failure) => + Effect.gen(function* () { + const harness = yield* makeHarness({ cached: BASE_PROJECTION, completionMarker: true }); + yield* Queue.offer(harness.inputs, synchronized()); + yield* awaitThreadState(harness.observed, (value) => value.status === "live"); + expect(yield* Ref.get(harness.lastSubscribeAfterSequence)).toBe(CACHED_SNAPSHOT_SEQUENCE); + + const dieAndResubscribe = (expectedSubscriptions: number) => + Effect.gen(function* () { + const brokenSnapshot = { + ...snapshot(BASE_PROJECTION, CACHED_SNAPSHOT_SEQUENCE + 1), + get snapshotSequence(): number { + throw new Error("Snapshot application failed"); + }, + }; + yield* Queue.offer(harness.inputs, failure === "stream" ? TEST_DEFECT : brokenSnapshot); + const failed = yield* awaitThreadState(harness.observed, (value) => + Option.isSome(value.error), + ); + expect(failed.status).toBe("synchronizing"); + yield* TestClock.adjust("250 millis"); + yield* SubscriptionRef.changes(harness.subscriptionCount).pipe( + Stream.filter((count) => count >= expectedSubscriptions), + Stream.runHead, + ); + expect(yield* Ref.get(harness.lastSubscribeAfterSequence)).toBeUndefined(); + }); + + yield* dieAndResubscribe(2); + yield* Queue.offerAll(harness.inputs, [ + snapshot( + { + ...BASE_PROJECTION, + thread: { ...BASE_PROJECTION.thread, title: "Recovered thread" }, + }, + CACHED_SNAPSHOT_SEQUENCE + 5, + ), + synchronized(), + ]); + const recovered = yield* awaitThreadState( + harness.observed, + (value) => + value.status === "live" && + Option.isSome(value.data) && + value.data.value.thread.title === "Recovered thread", + ); + expect(Option.isNone(recovered.error)).toBe(true); + + // The resync snapshot stands in for the replay that would carry thread.deleted. + yield* dieAndResubscribe(3); + yield* Queue.offer( + harness.inputs, + snapshot( + { ...BASE_PROJECTION, thread: { ...BASE_PROJECTION.thread, deletedAt: v2Now } }, + CACHED_SNAPSHOT_SEQUENCE + 9, + ), + ); + const removed = yield* awaitThreadState( + harness.observed, + (value) => value.status === "deleted", + ); + expect(Option.isNone(removed.data)).toBe(true); + }), + ); + it.effect( "resubscribes after a snapshot decode defect ends the thread subscription (#23041)", () => diff --git a/packages/client-runtime/src/state/threads.ts b/packages/client-runtime/src/state/threads.ts index 462be846d84a..4a236aa4804d 100644 --- a/packages/client-runtime/src/state/threads.ts +++ b/packages/client-runtime/src/state/threads.ts @@ -317,6 +317,8 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make }; if (resumeCache?.owner === owner) resumeCache.snapshot = committed; const awaitingCompletion = yield* Ref.make(false); + // A failed application may leave the cursor ahead of data, so recovery needs a snapshot. + const resyncFromSnapshot = yield* Ref.make(false); const applyLock = yield* Semaphore.make(1); // Save only completed data/cursor updates. A canceled scope must not cache // a cursor whose event has not reached the data yet. @@ -674,13 +676,19 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make } if (item.kind === "snapshot") { + const resynced = yield* Ref.getAndSet(resyncFromSnapshot, false); yield* SubscriptionRef.set(lastSequence, item.snapshotSequence); + const projection = boundedSnapshotProjection(item); + if (resynced && projection.thread.deletedAt !== null) { + yield* setDeleted(); + return; + } const hasProgressiveHistory = item.historyCursor !== undefined || item.hasMoreHistory !== undefined || item.latestLocalTurnOrdinal !== undefined; yield* setThread( - boundedSnapshotProjection(item), + projection, hasProgressiveHistory ? { history: { @@ -1013,10 +1021,20 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make // A 404 can arrive before the thread exists, so a miss asks HTTP again while mounted. let httpMisses = 0; let defectRetries = 0; - const backOffAfterDefect = Effect.suspend(() => { + const backOffAfterDefect = Effect.gen(function* () { const retryDelay = subscriptionRetryDelay(THREAD_SUBSCRIPTION_RETRY_DELAY, defectRetries); defectRetries += 1; - return setStreamError(THREAD_SYNC_ERROR).pipe(Effect.andThen(Effect.sleep(retryDelay))); + yield* Ref.set(resyncFromSnapshot, true); + yield* Ref.set(resumingLive, false); + yield* Ref.set(awaitingCompletion, false); + yield* SubscriptionRef.update(state, (current) => { + let status = current.status; + if (status !== "deleted") { + status = Option.isSome(current.data) ? "synchronizing" : "empty"; + } + return { ...current, status, error: Option.some(THREAD_SYNC_ERROR) }; + }); + yield* Effect.sleep(retryDelay); }); const followThread = (): Effect.Effect< void, @@ -1087,6 +1105,7 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make } : EMPTY_THREAD_HISTORY_META; yield* setThread(httpResult.snapshot.projection, { history }); + yield* Ref.set(resyncFromSnapshot, false); yield* remember; }), ); @@ -1110,7 +1129,7 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make } const sequence = yield* SubscriptionRef.get(lastSequence); - const canResume = Option.isSome(current.data); + const canResume = Option.isSome(current.data) && !(yield* Ref.get(resyncFromSnapshot)); const acceptBoundedSnapshot = yield* Ref.get(acceptsBoundedSocketSnapshots); return {