Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions packages/client-runtime/src/connection/supervisor.test.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import { AuthStandardClientScopes, EnvironmentId, type ServerConfig } from "@t3tools/contracts";
import { describe, expect, it } from "@effect/vitest";
import * as Clock from "effect/Clock";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Queue from "effect/Queue";
import * as Random from "effect/Random";
import * as Ref from "effect/Ref";
import * as Stream from "effect/Stream";
Expand Down Expand Up @@ -615,6 +617,76 @@ describe("EnvironmentSupervisor", () => {
}).pipe(Effect.provide(TestClock.layer())),
);

it.effect.each([
"application-active",
"application-active-probe",
"application-active-reconnect",
] as const)("retries at once on %s but keeps climbing the ladder", (reason) =>
Effect.gen(function* () {
const attempts = yield* Queue.unbounded<number>();
const harness = yield* makeHarness({
prepare: (attempt) =>
Queue.offer(attempts, attempt).pipe(
Effect.andThen(Effect.fail(transient("Could not resolve hostname."))),
),
});
const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
initiallyDesired: true,
}).pipe(Effect.provide(harness.dependencies));
const nextBackoffDelay = Effect.gen(function* () {
yield* Queue.take(attempts);
const waiting = yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
return waiting.retryAt! - (yield* Clock.currentTimeMillis);
});

const delays = [yield* nextBackoffDelay];
for (let wakeup = 0; wakeup < 4; wakeup += 1) {
yield* harness.wake(reason);
// The clock never moves here, so each attempt was started by the wakeup.
delays.push(yield* nextBackoffDelay);
}
expect(delays).toEqual([2_000, 2_000, 4_000, 8_000, 16_000]);

yield* TestClock.adjust("5 minutes");
yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
yield* harness.wake(reason);
const restarted = yield* awaitState(
supervisor.state,
(state) => state.phase === "backoff" && state.attempt === 1,
);
expect(restarted.retryAt! - (yield* Clock.currentTimeMillis)).toBe(2_000);
}).pipe(Effect.provide(TestClock.layer())),
);

it.effect("restarts the retry ladder on every network change", () =>
Effect.gen(function* () {
const attempts = yield* Queue.unbounded<number>();
const harness = yield* makeHarness({
prepare: (attempt) =>
Queue.offer(attempts, attempt).pipe(Effect.andThen(Effect.fail(transient()))),
});
const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
initiallyDesired: true,
}).pipe(Effect.provide(harness.dependencies));
const nextBackoffAttempt = Effect.gen(function* () {
yield* Queue.take(attempts);
const waiting = yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
return waiting.attempt;
});

const ladder = [yield* nextBackoffAttempt];
for (const reason of [
"application-active",
"application-active",
"network-changed",
] as const) {
yield* harness.wake(reason);
ladder.push(yield* nextBackoffAttempt);
}
expect(ladder).toEqual([1, 1, 2, 1]);
}).pipe(Effect.provide(TestClock.layer())),
);

it.effect("probes the active session without reconnecting on application activation", () =>
Effect.gen(function* () {
const probeCount = yield* Ref.make(0);
Expand Down
43 changes: 24 additions & 19 deletions packages/client-runtime/src/connection/supervisor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -842,20 +842,20 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
}, Effect.ensuring(clearLease));

const waitForRetrySignal = Effect.fnUntraced(function* (delayMs: number) {
// @effect-diagnostics-next-line raceFirstWithSleepToTimeout:off - the sleep is the retry delay (false), not a timeout around the signal loop
// @effect-diagnostics-next-line raceFirstWithSleepToTimeout:off - the sleep is the retry delay, not a timeout around the signal loop
return yield* Effect.raceFirst(
Effect.sleep(delayMs).pipe(Effect.as(false)),
Effect.sleep(delayMs).pipe(Effect.as(undefined)),
Effect.gen(function* () {
for (;;) {
const next = yield* Queue.take(signals);
switch (next._tag) {
case "Wakeup":
return ConnectionWakeups.resetsRetryBackoff(next.reason);
return next.reason;
case "ConnectRequested":
case "DisconnectRequested":
case "RetryRequested":
case "NetworkChanged":
return false;
return undefined;
case "BetterRouteAvailable":
break;
}
Expand All @@ -866,9 +866,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (

const waitForSignal = Queue.take(signals).pipe(
Effect.repeat({ while: (next) => next._tag === "BetterRouteAvailable" }),
Effect.map(
(next) => next._tag === "Wakeup" && ConnectionWakeups.resetsRetryBackoff(next.reason),
),
Effect.map((next) => (next._tag === "Wakeup" ? next.reason : undefined)),
);

const run = Effect.fnUntraced(function* () {
Expand All @@ -880,6 +878,22 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
failureCount = 0;
pendingRetry = Option.none();
};
let lastForegroundResetAt = Number.NEGATIVE_INFINITY;
// Returning to the app still ends the wait, but restarts the ladder at most
// once per cap window. A window can become visible many times a minute,
// which would otherwise keep an unreachable host on the shortest delays.
// A network change can make the host reachable, so it always resets.
const resetRetryLadderAfterWakeup = Effect.fnUntraced(function* (
reason: ConnectionWakeups.ConnectionWakeup | undefined,
) {
if (reason === undefined || !ConnectionWakeups.resetsRetryBackoff(reason)) return;
if (reason !== "network-changed") {
const now = yield* Clock.currentTimeMillis;
if (now - lastForegroundResetAt < RETRY_MAX_DELAY_MS) return;
lastForegroundResetAt = now;
}
resetRetryLadder();
});
let replacing = false;

for (;;) {
Expand All @@ -900,10 +914,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
if (currentIntent.network === "offline" && !replacing) {
yield* clearLease;
yield* setState(offlineState(currentIntent, generation, failureCount + 1, latestFailure));
const applicationActivated = yield* waitForSignal;
if (applicationActivated) {
resetRetryLadder();
}
yield* resetRetryLadderAfterWakeup(yield* waitForSignal);
continue;
}

Expand Down Expand Up @@ -950,10 +961,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
lastFailure: error,
retryAt: null,
});
const applicationActivated = yield* waitForSignal;
if (applicationActivated) {
resetRetryLadder();
}
yield* resetRetryLadderAfterWakeup(yield* waitForSignal);
continue;
}

Expand Down Expand Up @@ -982,10 +990,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
lastFailure: latestFailure,
retryAt: (yield* Clock.currentTimeMillis) + delayMs,
});
const applicationActivated = yield* waitForRetrySignal(delayMs);
if (applicationActivated) {
resetRetryLadder();
}
yield* resetRetryLadderAfterWakeup(yield* waitForRetrySignal(delayMs));
}
});

Expand Down
Loading