diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts
index 68bd9e600469..428485d991c1 100644
--- a/apps/server/src/relay/AgentAwarenessRelay.test.ts
+++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts
@@ -21,6 +21,7 @@ import * as BackgroundRelaySettings from "./BackgroundRelaySettings.ts";
import { BackgroundRelayConfig } from "./BackgroundRelayConfig.ts";
import * as DateTime from "effect/DateTime";
import * as Deferred from "effect/Deferred";
+import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
@@ -710,6 +711,66 @@ describe("AgentAwarenessRelay", () => {
}),
);
+ it.effect("reports quiet running work again, and stops once the run completes", () =>
+ Effect.gen(function* () {
+ const { relay, currentShell, publications } = yield* makeTestRelay();
+ yield* relay.publishThread(THREAD_ID);
+ for (let i = 0; i < 3; i++) {
+ yield* TestClock.adjust("3 minutes");
+ yield* relay.drain;
+ }
+ assert.deepEqual(
+ publications.map((publication) => [
+ publication.state?.phase,
+ publication.state?.updatedAt,
+ publication.state?.heartbeatAt,
+ ]),
+ [
+ ["running", NOW, "1970-01-01T00:00:00.000Z"],
+ ["running", NOW, "1970-01-01T00:03:00.000Z"],
+ ["running", NOW, "1970-01-01T00:06:00.000Z"],
+ ["running", NOW, "1970-01-01T00:09:00.000Z"],
+ ],
+ );
+
+ yield* Ref.set(currentShell, shell({ status: "completed" }));
+ yield* relay.publishThread(THREAD_ID);
+ yield* TestClock.adjust("10 minutes");
+ yield* relay.drain;
+ assert.deepEqual(
+ publications
+ .slice(4)
+ .map((publication) => [publication.state?.phase, publication.state?.heartbeatAt]),
+ [["completed", undefined]],
+ );
+ }),
+ );
+
+ it.effect("sends the next heartbeat while waiting for a steady publish retry", () =>
+ Effect.gen(function* () {
+ const { relay, publications } = yield* makeTestRelay({
+ respond: (attempt) =>
+ attempt >= 2 && attempt <= 9
+ ? new Response("unavailable", { status: 503 })
+ : Response.json({ ok: true, deliveries: [] }),
+ });
+ yield* relay.publishThread(THREAD_ID);
+ yield* TestClock.adjust("3 minutes");
+ yield* relay.drain;
+ for (const seconds of [1, 2, 4, 8, 16, 60, 60]) {
+ yield* TestClock.adjust(Duration.seconds(seconds));
+ yield* relay.drain;
+ }
+ assert.equal(publications.length, 9);
+
+ // The next retry is at 6:31, so only the heartbeat can publish at 6:00.
+ yield* TestClock.adjust("29 seconds");
+ yield* relay.drain;
+ assert.equal(publications.length, 10);
+ assert.equal(publications[9]?.state?.heartbeatAt, "1970-01-01T00:06:00.000Z");
+ }),
+ );
+
it.effect("republishes unchanged state with fresh credentials after relinking", () =>
Effect.gen(function* () {
const { relay, secrets, publications } = yield* makeTestRelay();
@@ -830,7 +891,7 @@ describe("AgentAwarenessRelay", () => {
yield* TestClock.adjust("5 seconds");
yield* relay.drain;
assert.deepEqual(
- publications.map((publication) => publication.state?.phase ?? null),
+ withoutHeartbeats(publications).map((publication) => publication.state?.phase ?? null),
["running", null],
);
}),
@@ -1249,7 +1310,7 @@ describe("AgentAwarenessRelay", () => {
);
});
-it.effect("resnapshots on explicit catch-up while idle configuration checks avoid snapshots", () =>
+it.effect("resnapshots on explicit catch-up without resnapshotting for heartbeats", () =>
Effect.gen(function* () {
const { relay, secrets, catchUp, publications } = yield* makeTestRelay({
catchUpThreads: [shell()],
@@ -1287,8 +1348,8 @@ it.effect("resnapshots on explicit catch-up while idle configuration checks avoi
yield* TestClock.adjust("1 second");
yield* relay.drain;
assert.equal(catchUp.activitySnapshotReads, 3);
- assert.equal(publications.length, 2);
- assert.equal(publications[1]?.state?.threadId, THREAD_ID);
+ assert.equal(withoutHeartbeats(publications).length, 2);
+ assert.equal(withoutHeartbeats(publications)[1]?.state?.threadId, THREAD_ID);
}),
);
@@ -1346,7 +1407,12 @@ describe("background relay resync", () => {
assert.equal(publications.length, 2);
assert.equal(publications[1]?.authorization, "Bearer second-credential");
yield* TestClock.adjust("10 minutes");
- assert.equal(publications.length, 2);
+ assert.equal(withoutHeartbeats(publications).length, 2);
+ assert.isTrue(
+ publications
+ .slice(2)
+ .every((publication) => publication.authorization === "Bearer second-credential"),
+ );
}).pipe(Effect.scoped),
);
@@ -1474,3 +1540,26 @@ describe("background relay resync", () => {
}).pipe(Effect.scoped),
);
});
+
+// A heartbeat repeats a thread's previous report at least three minutes later. Tests about
+// other publishes leave heartbeats out.
+function withoutHeartbeats<
+ P extends {
+ readonly authorization: string | null;
+ readonly state: { readonly threadId: string; readonly heartbeatAt?: string | undefined } | null;
+ },
+>(publications: ReadonlyArray
): Array
{
+ const previousByThread = new Map();
+ return publications.filter((publication) => {
+ const key = `${publication.authorization} ${publication.state?.threadId}`;
+ const previous = previousByThread.get(key);
+ previousByThread.set(key, publication);
+ if (!previous?.state?.heartbeatAt || !publication.state?.heartbeatAt) return true;
+ const { heartbeatAt: previousAt, ...previousState } = previous.state;
+ const { heartbeatAt, ...state } = publication.state;
+ return (
+ JSON.stringify(state) !== JSON.stringify(previousState) ||
+ Date.parse(heartbeatAt) - Date.parse(previousAt) < 3 * 60_000
+ );
+ });
+}
diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts
index e1e1a80defc4..11b4f97b3425 100644
--- a/apps/server/src/relay/AgentAwarenessRelay.ts
+++ b/apps/server/src/relay/AgentAwarenessRelay.ts
@@ -227,6 +227,15 @@ const encodeRelayConfiguration = Schema.encodeEffect(
),
);
+// The relay sends a card only on change and phones mark it out of date ten minutes after the
+// last push, so quiet live work is reported again well inside that window.
+const RELAY_AGENT_ACTIVITY_HEARTBEAT_MS = 3 * 60_000;
+
+// Waits and results alert, and a repeat could ring again; only silent live phases repeat.
+function phaseHeartbeats(phase: RelayAgentActivityState["phase"]): boolean {
+ return phase === "starting" || phase === "running";
+}
+
function sanitizeRelayAgentActivityState(
state: RelayAgentActivityState | null,
): RelayAgentActivityState | null {
@@ -527,6 +536,17 @@ export const make = Effect.gen(function* () {
// Threads whose confirmation holds a completion, so a thread that settles during the hold still alerts.
const heldCompletions = new Set();
let schedulePublishConfirm: (threadId: ThreadId) => Effect.Effect = () => Effect.void;
+ const heartbeats = new Map<
+ ThreadId,
+ { readonly dueAtMs: number; readonly timer: Fiber.Fiber }
+ >();
+ const cancelHeartbeat = Effect.fnUntraced(function* (threadId: ThreadId) {
+ const heartbeat = heartbeats.get(threadId);
+ heartbeats.delete(threadId);
+ if (heartbeat !== undefined) yield* Fiber.interrupt(heartbeat.timer);
+ });
+ let scheduleHeartbeat: (threadId: ThreadId, dueAtMs: number) => Effect.Effect = () =>
+ Effect.void;
const publishRetries = new Map<
ThreadId,
{ readonly attempts: number; timer: Fiber.Fiber | undefined }
@@ -546,6 +566,7 @@ export const make = Effect.gen(function* () {
cachedRelayClient = undefined;
publishConfirmDeadlines.clear();
heldCompletions.clear();
+ yield* Effect.forEach([...heartbeats.keys()], cancelHeartbeat, { discard: true });
yield* Ref.set(publishedStateByThreadRef, new Map());
});
@@ -661,10 +682,14 @@ export const make = Effect.gen(function* () {
return;
}
}
+ const heartbeatDue =
+ (heartbeats.get(threadId)?.dueAtMs ?? Number.POSITIVE_INFINITY) <=
+ (yield* DateTime.now).epochMilliseconds;
const published = publishedStateByThread.get(threadId);
if (
published?.revision === relayConfig.revision &&
published.identity === publishIdentity &&
+ !heartbeatDue &&
// A held completion settled after an earlier clear still has to alert.
!(snapshot.state === null && heldCompletions.has(threadId))
) {
@@ -734,6 +759,7 @@ export const make = Effect.gen(function* () {
reason: unsettled.reason,
}).pipe(withRelayClientTracing);
heldCompletions.delete(threadId);
+ yield* cancelHeartbeat(threadId);
publishConfirmDeadlines.set(threadId, nowMs + 5_000);
yield* Ref.update(publishedStateByThreadRef, (publishedStates) =>
new Map(publishedStates).set(threadId, {
@@ -771,14 +797,24 @@ export const make = Effect.gen(function* () {
childAlert?.requestId === childRequestId
? { ...snapshot.state, updatedAt: childAlert.updatedAt }
: snapshot.state;
+ const heartbeatAt = state !== null && phaseHeartbeats(state.phase) ? yield* DateTime.now : null;
yield* publishState({
projectId: snapshot.projectId,
- state,
+ state:
+ state === null || heartbeatAt === null
+ ? state
+ : { ...state, heartbeatAt: DateTime.formatIso(heartbeatAt) },
reason: snapshot.reason,
}).pipe(withRelayClientTracing);
if (state !== null && childRequestId !== undefined) {
childAlertByThread.set(threadId, { requestId: childRequestId, updatedAt: state.updatedAt });
}
+ yield* heartbeatAt === null
+ ? cancelHeartbeat(threadId)
+ : scheduleHeartbeat(
+ threadId,
+ heartbeatAt.epochMilliseconds + RELAY_AGENT_ACTIVITY_HEARTBEAT_MS,
+ );
outcome("acknowledged");
publishConfirmDeadlines.delete(threadId);
heldCompletions.delete(threadId);
@@ -1035,6 +1071,27 @@ export const make = Effect.gen(function* () {
scope,
).pipe(Effect.asVoid);
+ scheduleHeartbeat = Effect.fn("AgentAwarenessRelay.scheduleHeartbeat")(
+ function* (threadId, dueAtMs) {
+ yield* cancelHeartbeat(threadId);
+ // Keep the heartbeat interval even when a failed publish is waiting for a retry.
+ const timer = yield* Effect.forkIn(
+ Effect.sleep(RELAY_AGENT_ACTIVITY_HEARTBEAT_MS).pipe(
+ Effect.andThen(worker.enqueue(threadId)),
+ Effect.forever,
+ Effect.catchCause((cause) =>
+ Effect.logWarning("agent activity heartbeat failed", {
+ threadId,
+ cause: Cause.pretty(cause),
+ }),
+ ),
+ ),
+ scope,
+ );
+ heartbeats.set(threadId, { dueAtMs, timer });
+ },
+ );
+
const start: AgentAwarenessRelay["Service"]["start"] = Effect.fn("AgentAwarenessRelay.start")(
function* () {
const [relayConfig, publishEnabled] = yield* Effect.all([
diff --git a/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts b/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts
index e37d4621d20e..c3ac6a2d6f68 100644
--- a/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts
+++ b/infra/relay/src/agentActivity/AgentActivityPublisher.test.ts
@@ -1,4 +1,5 @@
import { makeAggregateState } from "./agentActivityAggregate.ts";
+import { androidActivityData } from "./fcmPayloads.ts";
import type { RelayAgentActivityState, RelayDeliveryResult } from "@t3tools/contracts/relay";
import { describe, expect, it } from "@effect/vitest";
import * as Effect from "effect/Effect";
@@ -494,6 +495,20 @@ describe("makeAggregateState", () => {
expect(aggregate?.activities).toMatchObject([{ threadId: "thread-fresh" }]);
});
+ it("keeps a quiet row alive on its heartbeat, on the card and in Android's expiry", () => {
+ const heartbeatAt = "1970-01-01T02:59:00.000Z";
+ const aggregate = makeAggregateState({
+ activeStates: [{ ...state, heartbeatAt }],
+ terminalState: null,
+ nowMs: 3 * hourMs,
+ });
+
+ expect(aggregate?.activities).toMatchObject([{ updatedAt: state.updatedAt, heartbeatAt }]);
+ expect(androidActivityData(aggregate).activity_expires_at).toBe(
+ String(Date.parse(heartbeatAt) + 2 * hourMs),
+ );
+ });
+
it("returns null when every row has expired and nothing terminal remains", () => {
expect(
makeAggregateState({
diff --git a/infra/relay/src/agentActivity/agentActivityAggregate.ts b/infra/relay/src/agentActivity/agentActivityAggregate.ts
index 99fdcf71ef90..8291eed4fc0c 100644
--- a/infra/relay/src/agentActivity/agentActivityAggregate.ts
+++ b/infra/relay/src/agentActivity/agentActivityAggregate.ts
@@ -41,6 +41,8 @@ function aggregateRowForState(state: RelayAgentActivityState) {
phase: state.phase,
status: statusForPhase(state.phase),
updatedAt: state.updatedAt,
+ // A new heartbeat makes the card differ, so it is sent again and its stale date moves.
+ ...(state.heartbeatAt === undefined ? {} : { heartbeatAt: state.heartbeatAt }),
deepLink: state.deepLink,
};
}
diff --git a/infra/relay/src/agentActivity/agentActivityPayloads.ts b/infra/relay/src/agentActivity/agentActivityPayloads.ts
index 4195c8117180..a70d842c8e01 100644
--- a/infra/relay/src/agentActivity/agentActivityPayloads.ts
+++ b/infra/relay/src/agentActivity/agentActivityPayloads.ts
@@ -17,9 +17,14 @@ const RUNNING_AGENT_ACTIVITY_ROW_TTL_MS = 2 * 60 * 60 * 1_000;
const WAITING_AGENT_ACTIVITY_ROW_TTL_MS = 24 * 60 * 60 * 1_000;
export function agentActivityExpiresAt(
- state: Pick,
+ state: Pick,
): number {
- const updatedAtMs = Option.match(DateTime.make(state.updatedAt), {
+ // A heartbeat repeats an unchanged state, so it is the newer sign that the environment is alive.
+ const reportedAt =
+ state.heartbeatAt !== undefined && state.heartbeatAt.localeCompare(state.updatedAt) > 0
+ ? state.heartbeatAt
+ : state.updatedAt;
+ const updatedAtMs = Option.match(DateTime.make(reportedAt), {
onNone: () => Number.NaN,
onSome: (dt) => dt.epochMilliseconds,
});
diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts
index c32475de4be3..38c55570d736 100644
--- a/packages/contracts/src/relay.ts
+++ b/packages/contracts/src/relay.ts
@@ -126,6 +126,8 @@ export const RelayAgentActivityState = Schema.Struct({
detail: Schema.optional(TrimmedNonEmptyString),
modelTitle: TrimmedNonEmptyString,
updatedAt: TrimmedNonEmptyString,
+ /** When the environment last reported this unchanged state; absent from older servers. */
+ heartbeatAt: Schema.optional(TrimmedNonEmptyString),
deepLink: TrimmedNonEmptyString,
});
export type RelayAgentActivityState = typeof RelayAgentActivityState.Type;
@@ -139,6 +141,7 @@ export const RelayAgentActivityAggregateRow = Schema.Struct({
phase: RelayAgentAwarenessPhase,
status: TrimmedNonEmptyString,
updatedAt: TrimmedNonEmptyString,
+ heartbeatAt: Schema.optional(TrimmedNonEmptyString),
deepLink: TrimmedNonEmptyString,
});
export type RelayAgentActivityAggregateRow = typeof RelayAgentActivityAggregateRow.Type;