From 2d7abe6fd886b3f33076167de3bf478de065c58e Mon Sep 17 00:00:00 2001 From: Mike Olson Date: Fri, 13 Nov 2026 17:12:50 -0500 Subject: [PATCH] fix(relay): Quiet running work stays current on the phone's agent activity card The server publishes a thread's activity only when it changes, and the relay sends a card only when it differs from the last one. A healthy turn that stays "Working" therefore got no push, so the iOS card showed "Out of date" after ten minutes, and the relay dropped the thread's row, ending the card, after two hours. The server now repeats a starting or running thread's state every three minutes with a heartbeat time. The relay carries the heartbeat on the thread's card row, which sends the card again, and measures the row's expiry from it, for the Android notification too. --- .../src/relay/AgentAwarenessRelay.test.ts | 99 ++++++++++++++++++- apps/server/src/relay/AgentAwarenessRelay.ts | 59 ++++++++++- .../AgentActivityPublisher.test.ts | 15 +++ .../agentActivity/agentActivityAggregate.ts | 2 + .../agentActivity/agentActivityPayloads.ts | 9 +- packages/contracts/src/relay.ts | 3 + 6 files changed, 179 insertions(+), 8 deletions(-) diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 68bd9e600469..428485d991c1 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -21,6 +21,7 @@ import * as BackgroundRelaySettings from "./BackgroundRelaySettings.ts"; import { BackgroundRelayConfig } from "./BackgroundRelayConfig.ts"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; @@ -710,6 +711,66 @@ describe("AgentAwarenessRelay", () => { }), ); + it.effect("reports quiet running work again, and stops once the run completes", () => + Effect.gen(function* () { + const { relay, currentShell, publications } = yield* makeTestRelay(); + yield* relay.publishThread(THREAD_ID); + for (let i = 0; i < 3; i++) { + yield* TestClock.adjust("3 minutes"); + yield* relay.drain; + } + assert.deepEqual( + publications.map((publication) => [ + publication.state?.phase, + publication.state?.updatedAt, + publication.state?.heartbeatAt, + ]), + [ + ["running", NOW, "1970-01-01T00:00:00.000Z"], + ["running", NOW, "1970-01-01T00:03:00.000Z"], + ["running", NOW, "1970-01-01T00:06:00.000Z"], + ["running", NOW, "1970-01-01T00:09:00.000Z"], + ], + ); + + yield* Ref.set(currentShell, shell({ status: "completed" })); + yield* relay.publishThread(THREAD_ID); + yield* TestClock.adjust("10 minutes"); + yield* relay.drain; + assert.deepEqual( + publications + .slice(4) + .map((publication) => [publication.state?.phase, publication.state?.heartbeatAt]), + [["completed", undefined]], + ); + }), + ); + + it.effect("sends the next heartbeat while waiting for a steady publish retry", () => + Effect.gen(function* () { + const { relay, publications } = yield* makeTestRelay({ + respond: (attempt) => + attempt >= 2 && attempt <= 9 + ? new Response("unavailable", { status: 503 }) + : Response.json({ ok: true, deliveries: [] }), + }); + yield* relay.publishThread(THREAD_ID); + yield* TestClock.adjust("3 minutes"); + yield* relay.drain; + for (const seconds of [1, 2, 4, 8, 16, 60, 60]) { + yield* TestClock.adjust(Duration.seconds(seconds)); + yield* relay.drain; + } + assert.equal(publications.length, 9); + + // The next retry is at 6:31, so only the heartbeat can publish at 6:00. + yield* TestClock.adjust("29 seconds"); + yield* relay.drain; + assert.equal(publications.length, 10); + assert.equal(publications[9]?.state?.heartbeatAt, "1970-01-01T00:06:00.000Z"); + }), + ); + it.effect("republishes unchanged state with fresh credentials after relinking", () => Effect.gen(function* () { const { relay, secrets, publications } = yield* makeTestRelay(); @@ -830,7 +891,7 @@ describe("AgentAwarenessRelay", () => { yield* TestClock.adjust("5 seconds"); yield* relay.drain; assert.deepEqual( - publications.map((publication) => publication.state?.phase ?? null), + withoutHeartbeats(publications).map((publication) => publication.state?.phase ?? null), ["running", null], ); }), @@ -1249,7 +1310,7 @@ describe("AgentAwarenessRelay", () => { ); }); -it.effect("resnapshots on explicit catch-up while idle configuration checks avoid snapshots", () => +it.effect("resnapshots on explicit catch-up without resnapshotting for heartbeats", () => Effect.gen(function* () { const { relay, secrets, catchUp, publications } = yield* makeTestRelay({ catchUpThreads: [shell()], @@ -1287,8 +1348,8 @@ it.effect("resnapshots on explicit catch-up while idle configuration checks avoi yield* TestClock.adjust("1 second"); yield* relay.drain; assert.equal(catchUp.activitySnapshotReads, 3); - assert.equal(publications.length, 2); - assert.equal(publications[1]?.state?.threadId, THREAD_ID); + assert.equal(withoutHeartbeats(publications).length, 2); + assert.equal(withoutHeartbeats(publications)[1]?.state?.threadId, THREAD_ID); }), ); @@ -1346,7 +1407,12 @@ describe("background relay resync", () => { assert.equal(publications.length, 2); assert.equal(publications[1]?.authorization, "Bearer second-credential"); yield* TestClock.adjust("10 minutes"); - assert.equal(publications.length, 2); + assert.equal(withoutHeartbeats(publications).length, 2); + assert.isTrue( + publications + .slice(2) + .every((publication) => publication.authorization === "Bearer second-credential"), + ); }).pipe(Effect.scoped), ); @@ -1474,3 +1540,26 @@ describe("background relay resync", () => { }).pipe(Effect.scoped), ); }); + +// A heartbeat repeats a thread's previous report at least three minutes later. Tests about +// other publishes leave heartbeats out. +function withoutHeartbeats< + P extends { + readonly authorization: string | null; + readonly state: { readonly threadId: string; readonly heartbeatAt?: string | undefined } | null; + }, +>(publications: ReadonlyArray

): Array

{ + const previousByThread = new Map(); + return publications.filter((publication) => { + const key = `${publication.authorization} ${publication.state?.threadId}`; + const previous = previousByThread.get(key); + previousByThread.set(key, publication); + if (!previous?.state?.heartbeatAt || !publication.state?.heartbeatAt) return true; + const { heartbeatAt: previousAt, ...previousState } = previous.state; + const { heartbeatAt, ...state } = publication.state; + return ( + JSON.stringify(state) !== JSON.stringify(previousState) || + Date.parse(heartbeatAt) - Date.parse(previousAt) < 3 * 60_000 + ); + }); +} diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index e1e1a80defc4..11b4f97b3425 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -227,6 +227,15 @@ const encodeRelayConfiguration = Schema.encodeEffect( ), ); +// The relay sends a card only on change and phones mark it out of date ten minutes after the +// last push, so quiet live work is reported again well inside that window. +const RELAY_AGENT_ACTIVITY_HEARTBEAT_MS = 3 * 60_000; + +// Waits and results alert, and a repeat could ring again; only silent live phases repeat. +function phaseHeartbeats(phase: RelayAgentActivityState["phase"]): boolean { + return phase === "starting" || phase === "running"; +} + function sanitizeRelayAgentActivityState( state: RelayAgentActivityState | null, ): RelayAgentActivityState | null { @@ -527,6 +536,17 @@ export const make = Effect.gen(function* () { // Threads whose confirmation holds a completion, so a thread that settles during the hold still alerts. const heldCompletions = new Set(); let schedulePublishConfirm: (threadId: ThreadId) => Effect.Effect = () => Effect.void; + const heartbeats = new Map< + ThreadId, + { readonly dueAtMs: number; readonly timer: Fiber.Fiber } + >(); + const cancelHeartbeat = Effect.fnUntraced(function* (threadId: ThreadId) { + const heartbeat = heartbeats.get(threadId); + heartbeats.delete(threadId); + if (heartbeat !== undefined) yield* Fiber.interrupt(heartbeat.timer); + }); + let scheduleHeartbeat: (threadId: ThreadId, dueAtMs: number) => Effect.Effect = () => + Effect.void; const publishRetries = new Map< ThreadId, { readonly attempts: number; timer: Fiber.Fiber | undefined } @@ -546,6 +566,7 @@ export const make = Effect.gen(function* () { cachedRelayClient = undefined; publishConfirmDeadlines.clear(); heldCompletions.clear(); + yield* Effect.forEach([...heartbeats.keys()], cancelHeartbeat, { discard: true }); yield* Ref.set(publishedStateByThreadRef, new Map()); }); @@ -661,10 +682,14 @@ export const make = Effect.gen(function* () { return; } } + const heartbeatDue = + (heartbeats.get(threadId)?.dueAtMs ?? Number.POSITIVE_INFINITY) <= + (yield* DateTime.now).epochMilliseconds; const published = publishedStateByThread.get(threadId); if ( published?.revision === relayConfig.revision && published.identity === publishIdentity && + !heartbeatDue && // A held completion settled after an earlier clear still has to alert. !(snapshot.state === null && heldCompletions.has(threadId)) ) { @@ -734,6 +759,7 @@ export const make = Effect.gen(function* () { reason: unsettled.reason, }).pipe(withRelayClientTracing); heldCompletions.delete(threadId); + yield* cancelHeartbeat(threadId); publishConfirmDeadlines.set(threadId, nowMs + 5_000); yield* Ref.update(publishedStateByThreadRef, (publishedStates) => new Map(publishedStates).set(threadId, { @@ -771,14 +797,24 @@ export const make = Effect.gen(function* () { childAlert?.requestId === childRequestId ? { ...snapshot.state, updatedAt: childAlert.updatedAt } : snapshot.state; + const heartbeatAt = state !== null && phaseHeartbeats(state.phase) ? yield* DateTime.now : null; yield* publishState({ projectId: snapshot.projectId, - state, + state: + state === null || heartbeatAt === null + ? state + : { ...state, heartbeatAt: DateTime.formatIso(heartbeatAt) }, reason: snapshot.reason, }).pipe(withRelayClientTracing); if (state !== null && childRequestId !== undefined) { childAlertByThread.set(threadId, { requestId: childRequestId, updatedAt: state.updatedAt }); } + yield* heartbeatAt === null + ? cancelHeartbeat(threadId) + : scheduleHeartbeat( + threadId, + heartbeatAt.epochMilliseconds + RELAY_AGENT_ACTIVITY_HEARTBEAT_MS, + ); outcome("acknowledged"); publishConfirmDeadlines.delete(threadId); heldCompletions.delete(threadId); @@ -1035,6 +1071,27 @@ export const make = Effect.gen(function* () { scope, ).pipe(Effect.asVoid); + scheduleHeartbeat = Effect.fn("AgentAwarenessRelay.scheduleHeartbeat")( + function* (threadId, dueAtMs) { + yield* cancelHeartbeat(threadId); + // Keep the heartbeat interval even when a failed publish is waiting for a retry. + const timer = yield* Effect.forkIn( + Effect.sleep(RELAY_AGENT_ACTIVITY_HEARTBEAT_MS).pipe( + Effect.andThen(worker.enqueue(threadId)), + Effect.forever, + Effect.catchCause((cause) => + Effect.logWarning("agent activity heartbeat failed", { + threadId, + cause: Cause.pretty(cause), + }), + ), + ), + scope, + ); + heartbeats.set(threadId, { dueAtMs, timer }); + }, + ); + const start: AgentAwarenessRelay["Service"]["start"] = Effect.fn("AgentAwarenessRelay.start")( function* () { const [relayConfig, publishEnabled] = yield* Effect.all([ diff --git a/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts b/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts index e37d4621d20e..c3ac6a2d6f68 100644 --- a/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts +++ b/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts @@ -1,4 +1,5 @@ import { makeAggregateState } from "./agentActivityAggregate.ts"; +import { androidActivityData } from "./fcmPayloads.ts"; import type { RelayAgentActivityState, RelayDeliveryResult } from "@t3tools/contracts/relay"; import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; @@ -494,6 +495,20 @@ describe("makeAggregateState", () => { expect(aggregate?.activities).toMatchObject([{ threadId: "thread-fresh" }]); }); + it("keeps a quiet row alive on its heartbeat, on the card and in Android's expiry", () => { + const heartbeatAt = "1970-01-01T02:59:00.000Z"; + const aggregate = makeAggregateState({ + activeStates: [{ ...state, heartbeatAt }], + terminalState: null, + nowMs: 3 * hourMs, + }); + + expect(aggregate?.activities).toMatchObject([{ updatedAt: state.updatedAt, heartbeatAt }]); + expect(androidActivityData(aggregate).activity_expires_at).toBe( + String(Date.parse(heartbeatAt) + 2 * hourMs), + ); + }); + it("returns null when every row has expired and nothing terminal remains", () => { expect( makeAggregateState({ diff --git a/infra/relay/src/agentActivity/agentActivityAggregate.ts b/infra/relay/src/agentActivity/agentActivityAggregate.ts index 99fdcf71ef90..8291eed4fc0c 100644 --- a/infra/relay/src/agentActivity/agentActivityAggregate.ts +++ b/infra/relay/src/agentActivity/agentActivityAggregate.ts @@ -41,6 +41,8 @@ function aggregateRowForState(state: RelayAgentActivityState) { phase: state.phase, status: statusForPhase(state.phase), updatedAt: state.updatedAt, + // A new heartbeat makes the card differ, so it is sent again and its stale date moves. + ...(state.heartbeatAt === undefined ? {} : { heartbeatAt: state.heartbeatAt }), deepLink: state.deepLink, }; } diff --git a/infra/relay/src/agentActivity/agentActivityPayloads.ts b/infra/relay/src/agentActivity/agentActivityPayloads.ts index 4195c8117180..a70d842c8e01 100644 --- a/infra/relay/src/agentActivity/agentActivityPayloads.ts +++ b/infra/relay/src/agentActivity/agentActivityPayloads.ts @@ -17,9 +17,14 @@ const RUNNING_AGENT_ACTIVITY_ROW_TTL_MS = 2 * 60 * 60 * 1_000; const WAITING_AGENT_ACTIVITY_ROW_TTL_MS = 24 * 60 * 60 * 1_000; export function agentActivityExpiresAt( - state: Pick, + state: Pick, ): number { - const updatedAtMs = Option.match(DateTime.make(state.updatedAt), { + // A heartbeat repeats an unchanged state, so it is the newer sign that the environment is alive. + const reportedAt = + state.heartbeatAt !== undefined && state.heartbeatAt.localeCompare(state.updatedAt) > 0 + ? state.heartbeatAt + : state.updatedAt; + const updatedAtMs = Option.match(DateTime.make(reportedAt), { onNone: () => Number.NaN, onSome: (dt) => dt.epochMilliseconds, }); diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index c32475de4be3..38c55570d736 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -126,6 +126,8 @@ export const RelayAgentActivityState = Schema.Struct({ detail: Schema.optional(TrimmedNonEmptyString), modelTitle: TrimmedNonEmptyString, updatedAt: TrimmedNonEmptyString, + /** When the environment last reported this unchanged state; absent from older servers. */ + heartbeatAt: Schema.optional(TrimmedNonEmptyString), deepLink: TrimmedNonEmptyString, }); export type RelayAgentActivityState = typeof RelayAgentActivityState.Type; @@ -139,6 +141,7 @@ export const RelayAgentActivityAggregateRow = Schema.Struct({ phase: RelayAgentAwarenessPhase, status: TrimmedNonEmptyString, updatedAt: TrimmedNonEmptyString, + heartbeatAt: Schema.optional(TrimmedNonEmptyString), deepLink: TrimmedNonEmptyString, }); export type RelayAgentActivityAggregateRow = typeof RelayAgentActivityAggregateRow.Type;