Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 94 additions & 5 deletions apps/server/src/relay/AgentAwarenessRelay.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import * as BackgroundRelaySettings from "./BackgroundRelaySettings.ts";
import { BackgroundRelayConfig } from "./BackgroundRelayConfig.ts";
import * as DateTime from "effect/DateTime";
import * as Deferred from "effect/Deferred";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
Expand Down Expand Up @@ -710,6 +711,66 @@ describe("AgentAwarenessRelay", () => {
}),
);

it.effect("reports quiet running work again, and stops once the run completes", () =>
Effect.gen(function* () {
const { relay, currentShell, publications } = yield* makeTestRelay();
yield* relay.publishThread(THREAD_ID);
for (let i = 0; i < 3; i++) {
yield* TestClock.adjust("3 minutes");
yield* relay.drain;
}
assert.deepEqual(
publications.map((publication) => [
publication.state?.phase,
publication.state?.updatedAt,
publication.state?.heartbeatAt,
]),
[
["running", NOW, "1970-01-01T00:00:00.000Z"],
["running", NOW, "1970-01-01T00:03:00.000Z"],
["running", NOW, "1970-01-01T00:06:00.000Z"],
["running", NOW, "1970-01-01T00:09:00.000Z"],
],
);

yield* Ref.set(currentShell, shell({ status: "completed" }));
yield* relay.publishThread(THREAD_ID);
yield* TestClock.adjust("10 minutes");
yield* relay.drain;
assert.deepEqual(
publications
.slice(4)
.map((publication) => [publication.state?.phase, publication.state?.heartbeatAt]),
[["completed", undefined]],
);
}),
);

it.effect("sends the next heartbeat while waiting for a steady publish retry", () =>
Effect.gen(function* () {
const { relay, publications } = yield* makeTestRelay({
respond: (attempt) =>
attempt >= 2 && attempt <= 9
? new Response("unavailable", { status: 503 })
: Response.json({ ok: true, deliveries: [] }),
});
yield* relay.publishThread(THREAD_ID);
yield* TestClock.adjust("3 minutes");
yield* relay.drain;
for (const seconds of [1, 2, 4, 8, 16, 60, 60]) {
yield* TestClock.adjust(Duration.seconds(seconds));
yield* relay.drain;
}
assert.equal(publications.length, 9);

// The next retry is at 6:31, so only the heartbeat can publish at 6:00.
yield* TestClock.adjust("29 seconds");
yield* relay.drain;
assert.equal(publications.length, 10);
assert.equal(publications[9]?.state?.heartbeatAt, "1970-01-01T00:06:00.000Z");
}),
);

it.effect("republishes unchanged state with fresh credentials after relinking", () =>
Effect.gen(function* () {
const { relay, secrets, publications } = yield* makeTestRelay();
Expand Down Expand Up @@ -830,7 +891,7 @@ describe("AgentAwarenessRelay", () => {
yield* TestClock.adjust("5 seconds");
yield* relay.drain;
assert.deepEqual(
publications.map((publication) => publication.state?.phase ?? null),
withoutHeartbeats(publications).map((publication) => publication.state?.phase ?? null),
["running", null],
);
}),
Expand Down Expand Up @@ -1249,7 +1310,7 @@ describe("AgentAwarenessRelay", () => {
);
});

it.effect("resnapshots on explicit catch-up while idle configuration checks avoid snapshots", () =>
it.effect("resnapshots on explicit catch-up without resnapshotting for heartbeats", () =>
Effect.gen(function* () {
const { relay, secrets, catchUp, publications } = yield* makeTestRelay({
catchUpThreads: [shell()],
Expand Down Expand Up @@ -1287,8 +1348,8 @@ it.effect("resnapshots on explicit catch-up while idle configuration checks avoi
yield* TestClock.adjust("1 second");
yield* relay.drain;
assert.equal(catchUp.activitySnapshotReads, 3);
assert.equal(publications.length, 2);
assert.equal(publications[1]?.state?.threadId, THREAD_ID);
assert.equal(withoutHeartbeats(publications).length, 2);
assert.equal(withoutHeartbeats(publications)[1]?.state?.threadId, THREAD_ID);
}),
);

Expand Down Expand Up @@ -1346,7 +1407,12 @@ describe("background relay resync", () => {
assert.equal(publications.length, 2);
assert.equal(publications[1]?.authorization, "Bearer second-credential");
yield* TestClock.adjust("10 minutes");
assert.equal(publications.length, 2);
assert.equal(withoutHeartbeats(publications).length, 2);
assert.isTrue(
publications
.slice(2)
.every((publication) => publication.authorization === "Bearer second-credential"),
);
}).pipe(Effect.scoped),
);

Expand Down Expand Up @@ -1474,3 +1540,26 @@ describe("background relay resync", () => {
}).pipe(Effect.scoped),
);
});

// A heartbeat repeats a thread's previous report at least three minutes later. Tests about
// other publishes leave heartbeats out.
function withoutHeartbeats<
P extends {
readonly authorization: string | null;
readonly state: { readonly threadId: string; readonly heartbeatAt?: string | undefined } | null;
},
>(publications: ReadonlyArray<P>): Array<P> {
const previousByThread = new Map<string, P>();
return publications.filter((publication) => {
const key = `${publication.authorization} ${publication.state?.threadId}`;
const previous = previousByThread.get(key);
previousByThread.set(key, publication);
if (!previous?.state?.heartbeatAt || !publication.state?.heartbeatAt) return true;
const { heartbeatAt: previousAt, ...previousState } = previous.state;
const { heartbeatAt, ...state } = publication.state;
return (
JSON.stringify(state) !== JSON.stringify(previousState) ||
Date.parse(heartbeatAt) - Date.parse(previousAt) < 3 * 60_000
);
});
}
59 changes: 58 additions & 1 deletion apps/server/src/relay/AgentAwarenessRelay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,15 @@ const encodeRelayConfiguration = Schema.encodeEffect(
),
);

// The relay sends a card only on change and phones mark it out of date ten minutes after the
// last push, so quiet live work is reported again well inside that window.
const RELAY_AGENT_ACTIVITY_HEARTBEAT_MS = 3 * 60_000;

// Waits and results alert, and a repeat could ring again; only silent live phases repeat.
function phaseHeartbeats(phase: RelayAgentActivityState["phase"]): boolean {
return phase === "starting" || phase === "running";
}

function sanitizeRelayAgentActivityState(
state: RelayAgentActivityState | null,
): RelayAgentActivityState | null {
Expand Down Expand Up @@ -527,6 +536,17 @@ export const make = Effect.gen(function* () {
// Threads whose confirmation holds a completion, so a thread that settles during the hold still alerts.
const heldCompletions = new Set<ThreadId>();
let schedulePublishConfirm: (threadId: ThreadId) => Effect.Effect<void> = () => Effect.void;
const heartbeats = new Map<
ThreadId,
{ readonly dueAtMs: number; readonly timer: Fiber.Fiber<void> }
>();
const cancelHeartbeat = Effect.fnUntraced(function* (threadId: ThreadId) {
const heartbeat = heartbeats.get(threadId);
heartbeats.delete(threadId);
if (heartbeat !== undefined) yield* Fiber.interrupt(heartbeat.timer);
});
let scheduleHeartbeat: (threadId: ThreadId, dueAtMs: number) => Effect.Effect<void> = () =>
Effect.void;
const publishRetries = new Map<
ThreadId,
{ readonly attempts: number; timer: Fiber.Fiber<void> | undefined }
Expand All @@ -546,6 +566,7 @@ export const make = Effect.gen(function* () {
cachedRelayClient = undefined;
publishConfirmDeadlines.clear();
heldCompletions.clear();
yield* Effect.forEach([...heartbeats.keys()], cancelHeartbeat, { discard: true });
yield* Ref.set(publishedStateByThreadRef, new Map());
});

Expand Down Expand Up @@ -661,10 +682,14 @@ export const make = Effect.gen(function* () {
return;
}
}
const heartbeatDue =
(heartbeats.get(threadId)?.dueAtMs ?? Number.POSITIVE_INFINITY) <=
(yield* DateTime.now).epochMilliseconds;
const published = publishedStateByThread.get(threadId);
if (
published?.revision === relayConfig.revision &&
published.identity === publishIdentity &&
!heartbeatDue &&
// A held completion settled after an earlier clear still has to alert.
!(snapshot.state === null && heldCompletions.has(threadId))
) {
Expand Down Expand Up @@ -734,6 +759,7 @@ export const make = Effect.gen(function* () {
reason: unsettled.reason,
}).pipe(withRelayClientTracing);
heldCompletions.delete(threadId);
yield* cancelHeartbeat(threadId);
publishConfirmDeadlines.set(threadId, nowMs + 5_000);
yield* Ref.update(publishedStateByThreadRef, (publishedStates) =>
new Map(publishedStates).set(threadId, {
Expand Down Expand Up @@ -771,14 +797,24 @@ export const make = Effect.gen(function* () {
childAlert?.requestId === childRequestId
? { ...snapshot.state, updatedAt: childAlert.updatedAt }
: snapshot.state;
const heartbeatAt = state !== null && phaseHeartbeats(state.phase) ? yield* DateTime.now : null;
yield* publishState({
projectId: snapshot.projectId,
state,
state:
state === null || heartbeatAt === null
? state
: { ...state, heartbeatAt: DateTime.formatIso(heartbeatAt) },
reason: snapshot.reason,
}).pipe(withRelayClientTracing);
if (state !== null && childRequestId !== undefined) {
childAlertByThread.set(threadId, { requestId: childRequestId, updatedAt: state.updatedAt });
}
yield* heartbeatAt === null
? cancelHeartbeat(threadId)
: scheduleHeartbeat(
threadId,
heartbeatAt.epochMilliseconds + RELAY_AGENT_ACTIVITY_HEARTBEAT_MS,
);
outcome("acknowledged");
publishConfirmDeadlines.delete(threadId);
heldCompletions.delete(threadId);
Expand Down Expand Up @@ -1035,6 +1071,27 @@ export const make = Effect.gen(function* () {
scope,
).pipe(Effect.asVoid);

scheduleHeartbeat = Effect.fn("AgentAwarenessRelay.scheduleHeartbeat")(
function* (threadId, dueAtMs) {
yield* cancelHeartbeat(threadId);
// Keep the heartbeat interval even when a failed publish is waiting for a retry.
const timer = yield* Effect.forkIn(
Effect.sleep(RELAY_AGENT_ACTIVITY_HEARTBEAT_MS).pipe(
Effect.andThen(worker.enqueue(threadId)),
Effect.forever,
Effect.catchCause((cause) =>
Effect.logWarning("agent activity heartbeat failed", {
threadId,
cause: Cause.pretty(cause),
}),
),
),
scope,
);
heartbeats.set(threadId, { dueAtMs, timer });
},
);

const start: AgentAwarenessRelay["Service"]["start"] = Effect.fn("AgentAwarenessRelay.start")(
function* () {
const [relayConfig, publishEnabled] = yield* Effect.all([
Expand Down
15 changes: 15 additions & 0 deletions infra/relay/src/agentActivity/AgentActivityPublisher.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { makeAggregateState } from "./agentActivityAggregate.ts";
import { androidActivityData } from "./fcmPayloads.ts";
import type { RelayAgentActivityState, RelayDeliveryResult } from "@t3tools/contracts/relay";
import { describe, expect, it } from "@effect/vitest";
import * as Effect from "effect/Effect";
Expand Down Expand Up @@ -494,6 +495,20 @@ describe("makeAggregateState", () => {
expect(aggregate?.activities).toMatchObject([{ threadId: "thread-fresh" }]);
});

it("keeps a quiet row alive on its heartbeat, on the card and in Android's expiry", () => {
const heartbeatAt = "1970-01-01T02:59:00.000Z";
const aggregate = makeAggregateState({
activeStates: [{ ...state, heartbeatAt }],
terminalState: null,
nowMs: 3 * hourMs,
});

expect(aggregate?.activities).toMatchObject([{ updatedAt: state.updatedAt, heartbeatAt }]);
expect(androidActivityData(aggregate).activity_expires_at).toBe(
String(Date.parse(heartbeatAt) + 2 * hourMs),
);
});

it("returns null when every row has expired and nothing terminal remains", () => {
expect(
makeAggregateState({
Expand Down
2 changes: 2 additions & 0 deletions infra/relay/src/agentActivity/agentActivityAggregate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ function aggregateRowForState(state: RelayAgentActivityState) {
phase: state.phase,
status: statusForPhase(state.phase),
updatedAt: state.updatedAt,
// A new heartbeat makes the card differ, so it is sent again and its stale date moves.
...(state.heartbeatAt === undefined ? {} : { heartbeatAt: state.heartbeatAt }),
deepLink: state.deepLink,
};
}
Expand Down
9 changes: 7 additions & 2 deletions infra/relay/src/agentActivity/agentActivityPayloads.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,14 @@ const RUNNING_AGENT_ACTIVITY_ROW_TTL_MS = 2 * 60 * 60 * 1_000;
const WAITING_AGENT_ACTIVITY_ROW_TTL_MS = 24 * 60 * 60 * 1_000;

export function agentActivityExpiresAt(
state: Pick<RelayAgentActivityState, "phase" | "updatedAt">,
state: Pick<RelayAgentActivityState, "phase" | "updatedAt" | "heartbeatAt">,
): number {
const updatedAtMs = Option.match(DateTime.make(state.updatedAt), {
// A heartbeat repeats an unchanged state, so it is the newer sign that the environment is alive.
const reportedAt =
state.heartbeatAt !== undefined && state.heartbeatAt.localeCompare(state.updatedAt) > 0
? state.heartbeatAt
: state.updatedAt;
const updatedAtMs = Option.match(DateTime.make(reportedAt), {
onNone: () => Number.NaN,
onSome: (dt) => dt.epochMilliseconds,
});
Expand Down
3 changes: 3 additions & 0 deletions packages/contracts/src/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ export const RelayAgentActivityState = Schema.Struct({
detail: Schema.optional(TrimmedNonEmptyString),
modelTitle: TrimmedNonEmptyString,
updatedAt: TrimmedNonEmptyString,
/** When the environment last reported this unchanged state; absent from older servers. */
heartbeatAt: Schema.optional(TrimmedNonEmptyString),
deepLink: TrimmedNonEmptyString,
});
export type RelayAgentActivityState = typeof RelayAgentActivityState.Type;
Expand All @@ -139,6 +141,7 @@ export const RelayAgentActivityAggregateRow = Schema.Struct({
phase: RelayAgentAwarenessPhase,
status: TrimmedNonEmptyString,
updatedAt: TrimmedNonEmptyString,
heartbeatAt: Schema.optional(TrimmedNonEmptyString),
deepLink: TrimmedNonEmptyString,
});
export type RelayAgentActivityAggregateRow = typeof RelayAgentActivityAggregateRow.Type;
Expand Down