diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 68bd9e600469..428485d991c1 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -21,6 +21,7 @@ import * as BackgroundRelaySettings from "./BackgroundRelaySettings.ts"; import { BackgroundRelayConfig } from "./BackgroundRelayConfig.ts"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; @@ -710,6 +711,66 @@ describe("AgentAwarenessRelay", () => { }), ); + it.effect("reports quiet running work again, and stops once the run completes", () => + Effect.gen(function* () { + const { relay, currentShell, publications } = yield* makeTestRelay(); + yield* relay.publishThread(THREAD_ID); + for (let i = 0; i < 3; i++) { + yield* TestClock.adjust("3 minutes"); + yield* relay.drain; + } + assert.deepEqual( + publications.map((publication) => [ + publication.state?.phase, + publication.state?.updatedAt, + publication.state?.heartbeatAt, + ]), + [ + ["running", NOW, "1970-01-01T00:00:00.000Z"], + ["running", NOW, "1970-01-01T00:03:00.000Z"], + ["running", NOW, "1970-01-01T00:06:00.000Z"], + ["running", NOW, "1970-01-01T00:09:00.000Z"], + ], + ); + + yield* Ref.set(currentShell, shell({ status: "completed" })); + yield* relay.publishThread(THREAD_ID); + yield* TestClock.adjust("10 minutes"); + yield* relay.drain; + assert.deepEqual( + publications + .slice(4) + .map((publication) => [publication.state?.phase, publication.state?.heartbeatAt]), + [["completed", undefined]], + ); + }), + ); + + it.effect("sends the next heartbeat while waiting for a steady publish retry", () => + Effect.gen(function* () { + const { relay, publications } = yield* makeTestRelay({ + respond: (attempt) => + attempt >= 2 && attempt <= 9 + ? new Response("unavailable", { status: 503 }) + : Response.json({ ok: true, deliveries: [] }), + }); + yield* relay.publishThread(THREAD_ID); + yield* TestClock.adjust("3 minutes"); + yield* relay.drain; + for (const seconds of [1, 2, 4, 8, 16, 60, 60]) { + yield* TestClock.adjust(Duration.seconds(seconds)); + yield* relay.drain; + } + assert.equal(publications.length, 9); + + // The next retry is at 6:31, so only the heartbeat can publish at 6:00. + yield* TestClock.adjust("29 seconds"); + yield* relay.drain; + assert.equal(publications.length, 10); + assert.equal(publications[9]?.state?.heartbeatAt, "1970-01-01T00:06:00.000Z"); + }), + ); + it.effect("republishes unchanged state with fresh credentials after relinking", () => Effect.gen(function* () { const { relay, secrets, publications } = yield* makeTestRelay(); @@ -830,7 +891,7 @@ describe("AgentAwarenessRelay", () => { yield* TestClock.adjust("5 seconds"); yield* relay.drain; assert.deepEqual( - publications.map((publication) => publication.state?.phase ?? null), + withoutHeartbeats(publications).map((publication) => publication.state?.phase ?? null), ["running", null], ); }), @@ -1249,7 +1310,7 @@ describe("AgentAwarenessRelay", () => { ); }); -it.effect("resnapshots on explicit catch-up while idle configuration checks avoid snapshots", () => +it.effect("resnapshots on explicit catch-up without resnapshotting for heartbeats", () => Effect.gen(function* () { const { relay, secrets, catchUp, publications } = yield* makeTestRelay({ catchUpThreads: [shell()], @@ -1287,8 +1348,8 @@ it.effect("resnapshots on explicit catch-up while idle configuration checks avoi yield* TestClock.adjust("1 second"); yield* relay.drain; assert.equal(catchUp.activitySnapshotReads, 3); - assert.equal(publications.length, 2); - assert.equal(publications[1]?.state?.threadId, THREAD_ID); + assert.equal(withoutHeartbeats(publications).length, 2); + assert.equal(withoutHeartbeats(publications)[1]?.state?.threadId, THREAD_ID); }), ); @@ -1346,7 +1407,12 @@ describe("background relay resync", () => { assert.equal(publications.length, 2); assert.equal(publications[1]?.authorization, "Bearer second-credential"); yield* TestClock.adjust("10 minutes"); - assert.equal(publications.length, 2); + assert.equal(withoutHeartbeats(publications).length, 2); + assert.isTrue( + publications + .slice(2) + .every((publication) => publication.authorization === "Bearer second-credential"), + ); }).pipe(Effect.scoped), ); @@ -1474,3 +1540,26 @@ describe("background relay resync", () => { }).pipe(Effect.scoped), ); }); + +// A heartbeat repeats a thread's previous report at least three minutes later. Tests about +// other publishes leave heartbeats out. +function withoutHeartbeats< + P extends { + readonly authorization: string | null; + readonly state: { readonly threadId: string; readonly heartbeatAt?: string | undefined } | null; + }, +>(publications: ReadonlyArray

): Array

{ + const previousByThread = new Map(); + return publications.filter((publication) => { + const key = `${publication.authorization} ${publication.state?.threadId}`; + const previous = previousByThread.get(key); + previousByThread.set(key, publication); + if (!previous?.state?.heartbeatAt || !publication.state?.heartbeatAt) return true; + const { heartbeatAt: previousAt, ...previousState } = previous.state; + const { heartbeatAt, ...state } = publication.state; + return ( + JSON.stringify(state) !== JSON.stringify(previousState) || + Date.parse(heartbeatAt) - Date.parse(previousAt) < 3 * 60_000 + ); + }); +} diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index e1e1a80defc4..11b4f97b3425 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -227,6 +227,15 @@ const encodeRelayConfiguration = Schema.encodeEffect( ), ); +// The relay sends a card only on change and phones mark it out of date ten minutes after the +// last push, so quiet live work is reported again well inside that window. +const RELAY_AGENT_ACTIVITY_HEARTBEAT_MS = 3 * 60_000; + +// Waits and results alert, and a repeat could ring again; only silent live phases repeat. +function phaseHeartbeats(phase: RelayAgentActivityState["phase"]): boolean { + return phase === "starting" || phase === "running"; +} + function sanitizeRelayAgentActivityState( state: RelayAgentActivityState | null, ): RelayAgentActivityState | null { @@ -527,6 +536,17 @@ export const make = Effect.gen(function* () { // Threads whose confirmation holds a completion, so a thread that settles during the hold still alerts. const heldCompletions = new Set(); let schedulePublishConfirm: (threadId: ThreadId) => Effect.Effect = () => Effect.void; + const heartbeats = new Map< + ThreadId, + { readonly dueAtMs: number; readonly timer: Fiber.Fiber } + >(); + const cancelHeartbeat = Effect.fnUntraced(function* (threadId: ThreadId) { + const heartbeat = heartbeats.get(threadId); + heartbeats.delete(threadId); + if (heartbeat !== undefined) yield* Fiber.interrupt(heartbeat.timer); + }); + let scheduleHeartbeat: (threadId: ThreadId, dueAtMs: number) => Effect.Effect = () => + Effect.void; const publishRetries = new Map< ThreadId, { readonly attempts: number; timer: Fiber.Fiber | undefined } @@ -546,6 +566,7 @@ export const make = Effect.gen(function* () { cachedRelayClient = undefined; publishConfirmDeadlines.clear(); heldCompletions.clear(); + yield* Effect.forEach([...heartbeats.keys()], cancelHeartbeat, { discard: true }); yield* Ref.set(publishedStateByThreadRef, new Map()); }); @@ -661,10 +682,14 @@ export const make = Effect.gen(function* () { return; } } + const heartbeatDue = + (heartbeats.get(threadId)?.dueAtMs ?? Number.POSITIVE_INFINITY) <= + (yield* DateTime.now).epochMilliseconds; const published = publishedStateByThread.get(threadId); if ( published?.revision === relayConfig.revision && published.identity === publishIdentity && + !heartbeatDue && // A held completion settled after an earlier clear still has to alert. !(snapshot.state === null && heldCompletions.has(threadId)) ) { @@ -734,6 +759,7 @@ export const make = Effect.gen(function* () { reason: unsettled.reason, }).pipe(withRelayClientTracing); heldCompletions.delete(threadId); + yield* cancelHeartbeat(threadId); publishConfirmDeadlines.set(threadId, nowMs + 5_000); yield* Ref.update(publishedStateByThreadRef, (publishedStates) => new Map(publishedStates).set(threadId, { @@ -771,14 +797,24 @@ export const make = Effect.gen(function* () { childAlert?.requestId === childRequestId ? { ...snapshot.state, updatedAt: childAlert.updatedAt } : snapshot.state; + const heartbeatAt = state !== null && phaseHeartbeats(state.phase) ? yield* DateTime.now : null; yield* publishState({ projectId: snapshot.projectId, - state, + state: + state === null || heartbeatAt === null + ? state + : { ...state, heartbeatAt: DateTime.formatIso(heartbeatAt) }, reason: snapshot.reason, }).pipe(withRelayClientTracing); if (state !== null && childRequestId !== undefined) { childAlertByThread.set(threadId, { requestId: childRequestId, updatedAt: state.updatedAt }); } + yield* heartbeatAt === null + ? cancelHeartbeat(threadId) + : scheduleHeartbeat( + threadId, + heartbeatAt.epochMilliseconds + RELAY_AGENT_ACTIVITY_HEARTBEAT_MS, + ); outcome("acknowledged"); publishConfirmDeadlines.delete(threadId); heldCompletions.delete(threadId); @@ -1035,6 +1071,27 @@ export const make = Effect.gen(function* () { scope, ).pipe(Effect.asVoid); + scheduleHeartbeat = Effect.fn("AgentAwarenessRelay.scheduleHeartbeat")( + function* (threadId, dueAtMs) { + yield* cancelHeartbeat(threadId); + // Keep the heartbeat interval even when a failed publish is waiting for a retry. + const timer = yield* Effect.forkIn( + Effect.sleep(RELAY_AGENT_ACTIVITY_HEARTBEAT_MS).pipe( + Effect.andThen(worker.enqueue(threadId)), + Effect.forever, + Effect.catchCause((cause) => + Effect.logWarning("agent activity heartbeat failed", { + threadId, + cause: Cause.pretty(cause), + }), + ), + ), + scope, + ); + heartbeats.set(threadId, { dueAtMs, timer }); + }, + ); + const start: AgentAwarenessRelay["Service"]["start"] = Effect.fn("AgentAwarenessRelay.start")( function* () { const [relayConfig, publishEnabled] = yield* Effect.all([ diff --git a/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts b/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts index e37d4621d20e..c3ac6a2d6f68 100644 --- a/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts +++ b/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts @@ -1,4 +1,5 @@ import { makeAggregateState } from "./agentActivityAggregate.ts"; +import { androidActivityData } from "./fcmPayloads.ts"; import type { RelayAgentActivityState, RelayDeliveryResult } from "@t3tools/contracts/relay"; import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; @@ -494,6 +495,20 @@ describe("makeAggregateState", () => { expect(aggregate?.activities).toMatchObject([{ threadId: "thread-fresh" }]); }); + it("keeps a quiet row alive on its heartbeat, on the card and in Android's expiry", () => { + const heartbeatAt = "1970-01-01T02:59:00.000Z"; + const aggregate = makeAggregateState({ + activeStates: [{ ...state, heartbeatAt }], + terminalState: null, + nowMs: 3 * hourMs, + }); + + expect(aggregate?.activities).toMatchObject([{ updatedAt: state.updatedAt, heartbeatAt }]); + expect(androidActivityData(aggregate).activity_expires_at).toBe( + String(Date.parse(heartbeatAt) + 2 * hourMs), + ); + }); + it("returns null when every row has expired and nothing terminal remains", () => { expect( makeAggregateState({ diff --git a/infra/relay/src/agentActivity/agentActivityAggregate.ts b/infra/relay/src/agentActivity/agentActivityAggregate.ts index 99fdcf71ef90..8291eed4fc0c 100644 --- a/infra/relay/src/agentActivity/agentActivityAggregate.ts +++ b/infra/relay/src/agentActivity/agentActivityAggregate.ts @@ -41,6 +41,8 @@ function aggregateRowForState(state: RelayAgentActivityState) { phase: state.phase, status: statusForPhase(state.phase), updatedAt: state.updatedAt, + // A new heartbeat makes the card differ, so it is sent again and its stale date moves. + ...(state.heartbeatAt === undefined ? {} : { heartbeatAt: state.heartbeatAt }), deepLink: state.deepLink, }; } diff --git a/infra/relay/src/agentActivity/agentActivityPayloads.ts b/infra/relay/src/agentActivity/agentActivityPayloads.ts index 4195c8117180..a70d842c8e01 100644 --- a/infra/relay/src/agentActivity/agentActivityPayloads.ts +++ b/infra/relay/src/agentActivity/agentActivityPayloads.ts @@ -17,9 +17,14 @@ const RUNNING_AGENT_ACTIVITY_ROW_TTL_MS = 2 * 60 * 60 * 1_000; const WAITING_AGENT_ACTIVITY_ROW_TTL_MS = 24 * 60 * 60 * 1_000; export function agentActivityExpiresAt( - state: Pick, + state: Pick, ): number { - const updatedAtMs = Option.match(DateTime.make(state.updatedAt), { + // A heartbeat repeats an unchanged state, so it is the newer sign that the environment is alive. + const reportedAt = + state.heartbeatAt !== undefined && state.heartbeatAt.localeCompare(state.updatedAt) > 0 + ? state.heartbeatAt + : state.updatedAt; + const updatedAtMs = Option.match(DateTime.make(reportedAt), { onNone: () => Number.NaN, onSome: (dt) => dt.epochMilliseconds, }); diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index c32475de4be3..38c55570d736 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -126,6 +126,8 @@ export const RelayAgentActivityState = Schema.Struct({ detail: Schema.optional(TrimmedNonEmptyString), modelTitle: TrimmedNonEmptyString, updatedAt: TrimmedNonEmptyString, + /** When the environment last reported this unchanged state; absent from older servers. */ + heartbeatAt: Schema.optional(TrimmedNonEmptyString), deepLink: TrimmedNonEmptyString, }); export type RelayAgentActivityState = typeof RelayAgentActivityState.Type; @@ -139,6 +141,7 @@ export const RelayAgentActivityAggregateRow = Schema.Struct({ phase: RelayAgentAwarenessPhase, status: TrimmedNonEmptyString, updatedAt: TrimmedNonEmptyString, + heartbeatAt: Schema.optional(TrimmedNonEmptyString), deepLink: TrimmedNonEmptyString, }); export type RelayAgentActivityAggregateRow = typeof RelayAgentActivityAggregateRow.Type;