diff --git a/apps/server/src/lazurio/agentCliConnection.ts b/apps/server/src/lazurio/agentCliConnection.ts index b89e8a0d76df4..6b80d726c3ecb 100644 --- a/apps/server/src/lazurio/agentCliConnection.ts +++ b/apps/server/src/lazurio/agentCliConnection.ts @@ -192,6 +192,10 @@ export interface AgentCliConnection { route: string, body: unknown, ) => Effect.Effect<{ readonly status: number; readonly body: unknown }, AgentCliFailure>; + /** GET one of the overlay's routes as the person's client surface; every status comes back. */ + readonly getRoute: ( + route: string, + ) => Effect.Effect<{ readonly status: number; readonly body: unknown }, AgentCliFailure>; /** The live thread list, as the sidebar follows it. */ readonly subscribeShell: ( input: OrchestrationV2SubscribeShellInput, @@ -402,6 +406,27 @@ export const withAgentCliConnection = Effect.fn("LazurioAgentCli.withConnection" body: Option.getOrUndefined(decodeJson(text)), })), ), + getRoute: (route) => + ownerToken.pipe( + Effect.flatMap((token) => + httpClient + .execute( + HttpClientRequest.get(new URL(route, origin)).pipe( + HttpClientRequest.bearerToken(token), + ), + ) + .pipe( + Effect.flatMap((response) => + Effect.map(response.text, (text) => ({ status: response.status, text })), + ), + Effect.mapError(requestFailed), + ), + ), + Effect.map(({ status, text }) => ({ + status, + body: Option.getOrUndefined(decodeJson(text)), + })), + ), subscribeShell: (input) => Stream.unwrap( ws.pipe( diff --git a/apps/server/src/lazurio/buddySignal.test.ts b/apps/server/src/lazurio/buddySignal.test.ts new file mode 100644 index 0000000000000..8dfd87eb00221 --- /dev/null +++ b/apps/server/src/lazurio/buddySignal.test.ts @@ -0,0 +1,255 @@ +import { assert, describe, it } from "@effect/vitest"; +import { MessageId, RunId, ScheduledTaskId, ThreadId } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; + +import { + type BuddySignalInput, + type BuddySignalStarterMessage, + buddySignalMessageIds, + buddySignalOf, +} from "./buddySignal.ts"; + +const THREAD = ThreadId.make("buddy-thread"); +const at = (minute: number) => + DateTime.makeUnsafe(`2026-11-18T20:${String(minute).padStart(2, "0")}:00.000Z`); +const iso = (minute: number) => DateTime.formatIso(at(minute)); + +type Status = BuddySignalInput["runs"][number]["status"]; +/** A run started by `by`'s message `m`, settled at `minute` (null while it runs). */ +const run = (ordinal: number, status: Status, minute: number | null) => ({ + id: RunId.make(`run-${ordinal}`), + ordinal, + status, + userMessageId: MessageId.make(`m${ordinal}`), + completedAt: minute === null ? null : at(minute), +}); +const person = { createdBy: "user" as const }; +const agent = { createdBy: "agent" as const }; +const otherThread = { createdBy: "user" as const, senderThreadId: ThreadId.make("other") }; +const task = { createdBy: "user" as const, scheduledTaskId: ScheduledTaskId.make("task-1") }; + +const signal = ( + input: Partial> & { + readonly senders?: Record; + }, +) => { + const full = { + threadId: THREAD, + lastVisitedAt: null, + runs: [], + pendingRequests: [], + stopRequested: new Set(), + failedTaskRuns: [], + ...input, + }; + const messages = new Map( + Object.entries(input.senders ?? {}).map(([ordinal, sender]) => [ + MessageId.make(`m${ordinal}`), + sender, + ]), + ); + // The reader loads exactly the senders the signal names, so a test that leaves one out fails. + for (const id of buddySignalMessageIds(full)) + assert.isTrue(messages.has(id), `the test names the sender of ${id}`); + const { version, threadId, ...rest } = buddySignalOf({ ...full, messages }); + assert.strictEqual(version, 1); + assert.strictEqual(threadId, THREAD); + return rest; +}; + +describe("the sign by Buddy's avatar", () => { + it("is quiet for a new thread and once the person read every answer", () => { + assert.deepStrictEqual(signal({}), { + state: "quiet", + since: null, + request: null, + startedBy: null, + }); + assert.strictEqual( + signal({ + runs: [run(1, "completed", 5)], + lastVisitedAt: at(6), + senders: { 1: person }, + }).state, + "quiet", + ); + }); + + it("waits on a pending request, before a failure and an unread answer", () => { + const pendingRequests = [ + { kind: "user_input" as const, createdAt: at(9), runId: RunId.make("run-3") }, + { kind: "command" as const, createdAt: at(8), runId: RunId.make("run-3") }, + ]; + assert.deepStrictEqual( + signal({ + runs: [run(1, "completed", 2), run(2, "failed", 4), run(3, "running", null)], + pendingRequests, + senders: { 1: person, 2: person, 3: agent }, + }), + // The oldest request is the one it waits on; the run that asked is an agent's. + { state: "waiting", since: iso(8), request: { kind: "command" }, startedBy: "other" }, + ); + }); + + it("fails for a failed last turn or an interrupt nobody asked for, and a completed turn clears it", () => { + const failed = signal({ + runs: [run(1, "completed", 2), run(2, "failed", 4)], + lastVisitedAt: at(1), + senders: { 1: person, 2: person }, + }); + // The unread answer before it waits behind the failure. + assert.deepStrictEqual(failed, { + state: "failed", + since: iso(4), + request: null, + startedBy: "person", + }); + assert.strictEqual( + signal({ runs: [run(1, "interrupted", 3)], senders: { 1: task } }).state, + "failed", + ); + // Stopped on request: no fault, and nothing new either. + assert.strictEqual( + signal({ + runs: [run(1, "interrupted", 3)], + stopRequested: new Set([RunId.make("run-1")]), + lastVisitedAt: at(2), + senders: { 1: person }, + }).state, + "quiet", + ); + // A turn still running after the failure keeps it; a completed one clears it. + assert.strictEqual( + signal({ + runs: [run(1, "failed", 3), run(2, "running", null)], + senders: { 1: person }, + }).state, + "failed", + ); + assert.deepStrictEqual( + signal({ + runs: [run(1, "failed", 3), run(2, "completed", 5)], + lastVisitedAt: at(4), + senders: { 1: person, 2: person }, + }), + { state: "answered", since: iso(5), request: null, startedBy: "person" }, + ); + // A cancelled queued turn or a rollback settles nothing. + assert.strictEqual( + signal({ + runs: [run(1, "failed", 3), run(2, "cancelled", 4), run(3, "rolled_back", 5)], + senders: { 1: person }, + }).state, + "failed", + ); + }); + + it("fails for a bound task that failed after the last completed turn", () => { + assert.deepStrictEqual( + signal({ + runs: [run(1, "completed", 2)], + lastVisitedAt: at(3), + failedTaskRuns: [at(7), at(5)], + senders: { 1: person }, + }), + { state: "failed", since: iso(5), request: null, startedBy: "other" }, + ); + assert.strictEqual( + signal({ + runs: [run(1, "completed", 6)], + lastVisitedAt: at(7), + failedTaskRuns: [at(5)], + senders: { 1: person }, + }).state, + "quiet", + ); + }); + + it("keeps a task's failure cleared by a completed turn when a later turn was stopped on request", () => { + // The task fails at :05, a turn completes at :06, another is stopped on request at :07. + assert.strictEqual( + signal({ + runs: [run(1, "completed", 6), run(2, "interrupted", 7)], + stopRequested: new Set([RunId.make("run-2")]), + lastVisitedAt: at(8), + failedTaskRuns: [at(5)], + senders: { 1: person, 2: person }, + }).state, + "quiet", + ); + }); + + it("answers from the oldest unread answer, the person's if they started any of them", () => { + // The person writes, Buddy answers, then three turns of another thread follow. + const runs = [ + run(1, "completed", 2), + run(2, "completed", 3), + run(3, "completed", 4), + run(4, "completed", 5), + ]; + assert.deepStrictEqual( + signal({ + runs, + lastVisitedAt: at(1), + senders: { 1: person, 2: otherThread, 3: otherThread, 4: task }, + }), + { state: "answered", since: iso(2), request: null, startedBy: "person" }, + ); + // All unread turns are others'; then the person's own turn completes. + const others = signal({ + runs: runs.slice(1), + lastVisitedAt: at(1), + senders: { 2: otherThread, 3: otherThread, 4: task }, + }); + assert.deepStrictEqual(others, { + state: "answered", + since: iso(3), + request: null, + startedBy: "other", + }); + assert.deepStrictEqual( + signal({ + runs: [...runs.slice(1), run(5, "completed", 8)], + lastVisitedAt: at(1), + senders: { 2: otherThread, 3: otherThread, 4: task, 5: person }, + }), + { ...others, startedBy: "person" }, + ); + // An agent's message, sent from outside any thread, is not the person's. + assert.strictEqual( + signal({ runs: [run(1, "completed", 2)], senders: { 1: agent } }).startedBy, + "other", + ); + }); + + it("reading while a turn runs leaves that turn's answer unread", () => { + const runs = [run(1, "completed", 2), run(2, "running", null)]; + assert.strictEqual( + signal({ runs, lastVisitedAt: at(3), senders: { 1: person } }).state, + "quiet", + ); + assert.deepStrictEqual( + signal({ + runs: [run(1, "completed", 2), run(2, "completed", 6)], + lastVisitedAt: at(3), + senders: { 1: person, 2: person }, + }), + { state: "answered", since: iso(6), request: null, startedBy: "person" }, + ); + }); + + it("names no starter it could not read, rather than guessing one", () => { + assert.strictEqual( + buddySignalOf({ + threadId: THREAD, + lastVisitedAt: null, + runs: [run(1, "failed", 2)], + pendingRequests: [], + stopRequested: new Set(), + failedTaskRuns: [], + messages: new Map(), + }).startedBy, + null, + ); + }); +}); diff --git a/apps/server/src/lazurio/buddySignal.ts b/apps/server/src/lazurio/buddySignal.ts new file mode 100644 index 0000000000000..92038de97f1f6 --- /dev/null +++ b/apps/server/src/lazurio/buddySignal.ts @@ -0,0 +1,286 @@ +/** + * Lazurio overlay (plan LZD-0033): `GET /api/lazurio/buddy/signal`, the one source of the sign by + * Buddy's avatar, versioned for every reader. Absent Buddy or thread is a 404, never `quiet`. + */ +import { + AuthOrchestrationReadScope, + type MessageId, + type OrchestrationV2ConversationMessage, + type OrchestrationV2Run, + type OrchestrationV2RuntimeRequest, + type RunId, + type ThreadId, +} from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { HttpRouter, HttpServerResponse } from "effect/http"; + +import * as ServerConfig from "../config.ts"; +import { OrchestratorV2 } from "../orchestration-v2/Orchestrator.ts"; +import { ProjectStoreV2 } from "../orchestration-v2/ProjectStore.ts"; +import { isBuddySurfaceEnvironment } from "./buddySurface.ts"; +import { BuddyTaskFailures } from "./buddyTaskFailures.ts"; +import { BUDDY_THREAD_ID, findSharedBuddyThread, sharedBuddyFolder } from "./buddySurfaceThread.ts"; +import { authenticateOverlayRoute } from "./routeAuth.ts"; + +export const BUDDY_SIGNAL_ROUTE = "/api/lazurio/buddy/signal"; + +export const BuddySignalState = Schema.Literals(["waiting", "failed", "answered", "quiet"]); +export type BuddySignalState = typeof BuddySignalState.Type; +export const BuddySignalStarter = Schema.Literals(["person", "other"]); +export type BuddySignalStarter = typeof BuddySignalStarter.Type; + +export const BuddySignal = Schema.Struct({ + version: Schema.Literal(1), + state: BuddySignalState, + threadId: Schema.String, + since: Schema.NullOr(Schema.String), + request: Schema.NullOr(Schema.Struct({ kind: Schema.String })), + startedBy: Schema.NullOr(BuddySignalStarter), +}); +export type BuddySignal = typeof BuddySignal.Type; + +type SignalRun = Pick< + OrchestrationV2Run, + "id" | "ordinal" | "status" | "userMessageId" | "completedAt" +>; +type SignalRequest = Pick & { + /** The run whose turn asked, where its node names one. */ + readonly runId: RunId | null; +}; +export type BuddySignalStarterMessage = Pick< + OrchestrationV2ConversationMessage, + "createdBy" | "senderThreadId" | "scheduledTaskId" +>; + +export interface BuddySignalInput { + readonly threadId: ThreadId; + readonly lastVisitedAt: DateTime.Utc | null; + readonly runs: ReadonlyArray; + readonly pendingRequests: ReadonlyArray; + /** Runs someone asked to stop (a `run_interrupt_request` item); any other interrupt is a fault. */ + readonly stopRequested: ReadonlySet; + /** The last failed run of each scheduled task bound to the thread. */ + readonly failedTaskRuns: ReadonlyArray; + readonly messages: ReadonlyMap; +} + +const millis = (value: DateTime.Utc) => DateTime.toEpochMillis(value); +const SETTLED = new Set(["completed", "failed", "interrupted"]); + +/** The last run that settled the thread's line; a stopped or cancelled one is no fault. */ +const lastSettledRun = (runs: ReadonlyArray) => + runs + .filter((run) => SETTLED.has(run.status) && run.completedAt !== null) + .toSorted((left, right) => left.ordinal - right.ordinal) + .at(-1); + +const isFault = (run: SignalRun, stopRequested: ReadonlySet) => + run.status === "failed" || (run.status === "interrupted" && !stopRequested.has(run.id)); + +const unreadCompletedRuns = (input: Pick) => + input.runs.filter( + (run) => + run.status === "completed" && + run.completedAt !== null && + (input.lastVisitedAt === null || millis(run.completedAt) > millis(input.lastVisitedAt)), + ); + +/** The user messages whose senders the signal needs, so a reader loads only those. */ +export function buddySignalMessageIds( + input: Omit, +): Array { + const byId = new Map(input.runs.map((run) => [run.id, run])); + const runs = [ + ...input.pendingRequests.flatMap((request) => + request.runId === null ? [] : [byId.get(request.runId)], + ), + lastSettledRun(input.runs), + ...unreadCompletedRuns(input), + ]; + return [...new Set(runs.flatMap((run) => (run === undefined ? [] : [run.userMessageId])))]; +} + +function starterOf( + run: SignalRun | undefined, + messages: BuddySignalInput["messages"], +): BuddySignalStarter | null { + if (run === undefined) return null; + const message = messages.get(run.userMessageId); + if (message === undefined) return null; + return message.createdBy === "user" && + message.senderThreadId === undefined && + message.scheduledTaskId === undefined + ? "person" + : "other"; +} + +/** + * Waiting on a pending request, else failed (the last settled turn failed or was interrupted + * without a stop request, or a bound task failed after it), else answered (completed after + * `lastVisitedAt`), else quiet. `since` is the oldest cause; `startedBy` is the person's if any. + */ +export function buddySignalOf(input: BuddySignalInput): BuddySignal { + const base = { version: 1 as const, threadId: input.threadId }; + const byId = new Map(input.runs.map((run) => [run.id, run])); + const iso = (value: DateTime.Utc) => DateTime.formatIso(value); + + const pending = input.pendingRequests.toSorted( + (left, right) => millis(left.createdAt) - millis(right.createdAt), + )[0]; + if (pending !== undefined) + return { + ...base, + state: "waiting", + since: iso(pending.createdAt), + request: { kind: pending.kind }, + startedBy: starterOf( + pending.runId === null ? undefined : byId.get(pending.runId), + input.messages, + ), + }; + + const settled = lastSettledRun(input.runs); + // The latest completed turn clears a task's failure, also when a later turn was stopped. + const lastCompleted = Math.max( + Number.NEGATIVE_INFINITY, + ...input.runs.flatMap((run) => + run.status === "completed" && run.completedAt !== null ? [millis(run.completedAt)] : [], + ), + ); + const taskFailure = input.failedTaskRuns + .filter((at) => millis(at) > lastCompleted) + .toSorted((left, right) => millis(left) - millis(right))[0]; + if (settled !== undefined && isFault(settled, input.stopRequested)) + return { + ...base, + state: "failed", + since: iso(settled.completedAt as DateTime.Utc), + request: null, + startedBy: starterOf(settled, input.messages), + }; + if (taskFailure !== undefined) + return { ...base, state: "failed", since: iso(taskFailure), request: null, startedBy: "other" }; + + const unread = unreadCompletedRuns(input).toSorted( + (left, right) => millis(left.completedAt!) - millis(right.completedAt!), + ); + const first = unread[0]; + if (first !== undefined) + return { + ...base, + state: "answered", + since: iso(first.completedAt!), + request: null, + startedBy: unread.some((run) => starterOf(run, input.messages) === "person") + ? "person" + : "other", + }; + + return { ...base, state: "quiet", since: null, request: null, startedBy: null }; +} + +/** Buddy's thread on this server: GEN3's one thread, GEN4's thread of the shared server. */ +const buddyThreadId = Effect.fn("LazurioBuddySignal.threadId")(function* () { + if (isBuddySurfaceEnvironment(yield* HostProcess.Environment)) + return { kind: "found", threadId: BUDDY_THREAD_ID } as const; + const buddyFolder = yield* sharedBuddyFolder; + if (buddyFolder === undefined) return { kind: "absent", error: "not-buddy" } as const; + const found = yield* findSharedBuddyThread(buddyFolder).pipe(Effect.option); + const thread = Option.getOrUndefined(found)?.thread ?? null; + return thread === null + ? ({ kind: "absent", error: "no-thread" } as const) + : ({ kind: "found", threadId: thread.id } as const); +}); + +/** The signal as the route answers it, or why there is none. */ +export const readBuddySignal = Effect.fn("LazurioBuddySignal.read")(function* () { + const target = yield* buddyThreadId(); + if (target.kind === "absent") return target; + const orchestrator = yield* OrchestratorV2; + const records = yield* orchestrator + .getThreadRecords(target.threadId, ["runs", "runtimeRequests"], { + runtimeRequestStatuses: ["pending"], + }) + .pipe(Effect.option); + if (Option.isNone(records) || records.value.thread.deletedAt !== null) + return { kind: "absent", error: "no-thread" } as const; + const { thread, runs, runtimeRequests } = records.value; + const pending = runtimeRequests.filter((request) => request.status === "pending"); + const { nodes } = + pending.length === 0 + ? { nodes: [] } + : yield* orchestrator + .getThreadRecords(target.threadId, ["nodes"], { + nodeIds: pending.map((request) => request.nodeId), + }) + .pipe(Effect.orDie); + const runOfNode = new Map(nodes.map((node) => [node.id, node.runId])); + const interrupted = runs.filter((run) => run.status === "interrupted").map((run) => run.id); + const { turnItems } = + interrupted.length === 0 + ? { turnItems: [] } + : yield* orchestrator + .getThreadRecords(target.threadId, ["turnItems"], { + turnItemTypes: ["run_interrupt_request"], + turnItemRunIds: interrupted, + }) + .pipe(Effect.orDie); + const failures = yield* Effect.serviceOption(BuddyTaskFailures); + const failedTaskRuns = Option.isNone(failures) + ? [] + : yield* failures.value.failedAt(target.threadId); + const input = { + lastVisitedAt: thread.lastVisitedAt, + runs, + pendingRequests: pending.map((request) => ({ + kind: request.kind, + createdAt: request.createdAt, + runId: runOfNode.get(request.nodeId) ?? null, + })), + stopRequested: new Set(turnItems.flatMap((item) => (item.runId === null ? [] : [item.runId]))), + failedTaskRuns, + }; + const messageIds = buddySignalMessageIds(input); + const messages = + messageIds.length === 0 + ? new Map() + : yield* orchestrator.getMessagesById(target.threadId, messageIds).pipe(Effect.orDie); + return { + kind: "signal", + signal: buddySignalOf({ ...input, threadId: target.threadId, messages }), + } as const; +}); + +export const layer = HttpRouter.use((router) => + Effect.gen(function* () { + // A handler sees only the request's context, so it takes its services from the layer's. + const orchestrator = yield* OrchestratorV2; + const config = yield* ServerConfig.ServerConfig; + const projects = yield* ProjectStoreV2; + const failures = yield* Effect.serviceOption(BuddyTaskFailures); + yield* router.add( + "GET", + BUDDY_SIGNAL_ROUTE, + Effect.gen(function* () { + yield* authenticateOverlayRoute(AuthOrchestrationReadScope); + const answer = yield* readBuddySignal().pipe( + Effect.provideService(OrchestratorV2, orchestrator), + Effect.provideService(ServerConfig.ServerConfig, config), + Effect.provideService(ProjectStoreV2, projects), + Option.isSome(failures) + ? Effect.provideService(BuddyTaskFailures, failures.value) + : (effect) => effect, + ); + return answer.kind === "absent" + ? HttpServerResponse.jsonUnsafe({ error: answer.error }, { status: 404 }) + : HttpServerResponse.jsonUnsafe(answer.signal, { + headers: { "cache-control": "no-store" }, + }); + }), + ); + }), +); diff --git a/apps/server/src/lazurio/buddySignalRead.test.ts b/apps/server/src/lazurio/buddySignalRead.test.ts new file mode 100644 index 0000000000000..9b12d3e2fc659 --- /dev/null +++ b/apps/server/src/lazurio/buddySignalRead.test.ts @@ -0,0 +1,253 @@ +// @effect-diagnostics nodeBuiltinImport:off - Buddy's folder is a temporary folder under a temporary home. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, it } from "@effect/vitest"; +import { + AuthOrchestrationReadScope, + AuthTerminalReadScope, + ProjectId, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; +import { HttpRouter } from "effect/http"; + +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import { OrchestratorV2 } from "../orchestration-v2/Orchestrator.ts"; +import { ProjectStoreV2 } from "../orchestration-v2/ProjectStore.ts"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; + +import * as ServerConfig from "../config.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import { type AgentCliConnection, agentCliFailure } from "./agentCliConnection.ts"; +import * as BuddySignal from "./buddySignal.ts"; +import { addProject, layerBuddyServer } from "./buddySurface.testkit.ts"; +import { followBuddySignal } from "./buddySurfaceCli.ts"; +import { layerManagedConfig, makeFolderDirectories } from "./managedFolder.testkit.ts"; +import { resolveBuddyThread } from "./buddySurfaceThread.ts"; + +const makeShared = () => { + const home = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "buddy-signal-home-")); + const folder = NodePath.join(home, "Lazurio"); + const buddy = NodePath.join(home, "Buddy"); + NodeFS.mkdirSync(folder); + NodeFS.mkdirSync(buddy); + return { home, folder, buddy }; +}; +const withShared = (effect: Effect.Effect, folder: string, buddy: string) => + Effect.flatMap(ServerConfig.ServerConfig, (config) => + effect.pipe( + Effect.provideService(ServerConfig.ServerConfig, { + ...config, + lazurioFolder: folder, + lazurioBuddyFolder: buddy, + }), + ), + ); + +describe("reading the sign by Buddy's avatar", () => { + it.live("says why there is none, and is quiet only for a thread that exists", () => { + const { home, folder, buddy } = makeShared(); + return Effect.gen(function* () { + // No Buddy on this server, then his project without his thread: never quiet. + assert.deepStrictEqual(yield* BuddySignal.readBuddySignal(), { + kind: "absent", + error: "not-buddy", + }); + yield* addProject(ProjectId.make("project:buddy"), buddy); + yield* (yield* ServerSettings.ServerSettingsService).updateSettings({ + projectSettingsOverrides: { + [ProjectId.make("project:buddy")]: { + defaultModelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "luna" }, + }, + }, + }); + assert.deepStrictEqual(yield* withShared(BuddySignal.readBuddySignal(), folder, buddy), { + kind: "absent", + error: "no-thread", + }); + const { threadId } = yield* withShared(resolveBuddyThread(), folder, buddy); + assert.deepStrictEqual(yield* withShared(BuddySignal.readBuddySignal(), folder, buddy), { + kind: "signal", + signal: { + version: 1, + state: "quiet", + threadId, + since: null, + request: null, + startedBy: null, + }, + }); + }).pipe( + Effect.provide(Layer.merge(layerBuddyServer, ServerSettings.layerTest())), + Effect.scoped, + Effect.provideService(HostProcess.Environment, { HOME: home }), + ); + }); + + it.live("on Buddy's own server, before his one thread exists, is no-thread", () => { + const { home } = makeShared(); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* BuddySignal.readBuddySignal(), { + kind: "absent", + error: "no-thread", + }); + }).pipe( + Effect.provide(Layer.merge(layerBuddyServer, ServerSettings.layerTest())), + Effect.scoped, + Effect.provideService(HostProcess.Environment, { HOME: home, T3CODE_LAZURIO_BUDDY: "1" }), + ); + }); +}); + +describe("following the sign", () => { + const signal = (state: string, startedBy: string | null) => ({ + version: 1, + state, + threadId: "buddy-thread", + since: state === "quiet" ? null : "2026-11-18T20:02:00.000Z", + request: null, + startedBy, + }); + /** A connection whose route answers `answers` in turn and whose shell sends `updates`. */ + const connection = ( + answers: Array<{ status: number; body: unknown }>, + updates: number, + ): AgentCliConnection => + ({ + getRoute: () => Effect.sync(() => answers.shift() ?? { status: 500, body: undefined }), + subscribeShell: () => + Stream.fromIterable( + Array.from({ length: updates }, () => ({ + kind: "thread.updated" as const, + thread: { id: ThreadId.make("buddy-thread") }, + })), + ), + }) as unknown as AgentCliConnection; + + it.effect( + "prints each change, startedBy too, and unavailable when it cannot read, never silence", + () => + Effect.gen(function* () { + const lines: Array = []; + const connects = [ + // Starts quiet, an unchanged reading prints nothing, then others' and the person's answers. + connection( + [ + { status: 200, body: signal("quiet", null) }, + { status: 200, body: signal("quiet", null) }, + { status: 200, body: signal("answered", "other") }, + { status: 200, body: signal("answered", "person") }, + ], + 3, + ), + // The session expired. + connection([{ status: 401, body: undefined }], 0), + // Back, before his first visit. + connection([{ status: 404, body: { error: "no-thread" } }], 0), + ]; + const fiber = yield* followBuddySignal( + (use) => { + const next = connects.shift(); + return next === undefined + ? Effect.fail(agentCliFailure("server_not_running", "T3 Code is not running.")) + : use(next); + }, + (line) => Effect.sync(() => lines.push(JSON.parse(line))), + ).pipe(Effect.forkChild); + // Each reconnect waits at most half a minute. + for (let step = 0; step < 6; step += 1) yield* TestClock.adjust("30 seconds"); + const states = lines.map((line) => { + const { state, startedBy, reason } = line as Record; + return state === "unavailable" ? `unavailable:${reason}` : `${state}:${startedBy}`; + }); + assert.deepStrictEqual(states.slice(0, 7), [ + "quiet:null", + "answered:other", + "answered:person", + "unavailable:connection_lost", + "unavailable:unauthorized", + "unavailable:no_thread", + "unavailable:connection_lost", + ]); + assert.include(states.slice(7), "unavailable:server_not_running"); + yield* Fiber.interrupt(fiber); + }), + ); + + it.effect("ends with a failure on a server without Buddy", () => + Effect.gen(function* () { + const lines: Array = []; + const exit = yield* followBuddySignal( + (use) => use(connection([{ status: 404, body: { error: "not-buddy" } }], 0)), + (line) => Effect.sync(() => lines.push(line)), + ).pipe(Effect.exit); + assert.strictEqual(exit._tag, "Failure"); + assert.deepStrictEqual( + lines.map((line) => JSON.parse(line).reason), + ["not_buddy"], + ); + }), + ); +}); + +const { folder: authFolder } = makeFolderDirectories(); +const layerAuth = EnvironmentAuth.layer.pipe( + Layer.provide(ManagedPolicy.layerTest()), + Layer.provideMerge(SqlitePersistence.layerMemory), + Layer.provide(ServerSecretStore.layer), + Layer.provide(ServerEnvironment.layerIdentity), + Layer.provideMerge(layerManagedConfig(authFolder)), +); + +it.layer(layerAuth.pipe(Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), { + excludeTestServices: true, +})("the sign's route", (it) => { + it.effect("needs a session that may read threads, and names a server without Buddy", () => + Effect.gen(function* () { + const auth = yield* EnvironmentAuth.EnvironmentAuth; + const config = yield* ServerConfig.ServerConfig; + // Refused before it reads anything, and without Buddy it reads no thread. + const handler = HttpRouter.toWebHandler( + BuddySignal.layer.pipe( + Layer.provideMerge(Layer.succeed(EnvironmentAuth.EnvironmentAuth, auth)), + Layer.provideMerge(Layer.mock(OrchestratorV2)({})), + Layer.provideMerge(Layer.mock(ProjectStoreV2)({})), + Layer.provideMerge(ServerConfig.layer(config)), + Layer.provideMerge(NodeHttpPlatform.layer), + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + { disableLogger: true }, + ); + yield* Effect.addFinalizer(() => Effect.promise(() => handler.dispose())); + const reader = yield* auth.issueSession({ scopes: [AuthOrchestrationReadScope], label: "t" }); + const terminal = yield* auth.issueSession({ scopes: [AuthTerminalReadScope], label: "t" }); + const get = (token?: string) => + handler.handler( + new Request(`http://localhost${BuddySignal.BUDDY_SIGNAL_ROUTE}`, { + headers: token ? { authorization: `Bearer ${token}` } : {}, + }), + ); + yield* Effect.promise(async () => { + assert.strictEqual((await get()).status, 401); + assert.strictEqual((await get(terminal.token)).status, 403); + const answer = await get(reader.token); + assert.strictEqual(answer.status, 404); + assert.deepStrictEqual(await answer.json(), { error: "not-buddy" }); + }); + }), + ); +}); diff --git a/apps/server/src/lazurio/buddySurfaceCli.ts b/apps/server/src/lazurio/buddySurfaceCli.ts index 23cccb5ad5a54..ef1ddf8e2e828 100644 --- a/apps/server/src/lazurio/buddySurfaceCli.ts +++ b/apps/server/src/lazurio/buddySurfaceCli.ts @@ -1,16 +1,26 @@ /** - * Lazurio overlay (root decision 0210, plan DEV-6676): `t3 lazurio buddy thread`, which names the - * thread to post a wake into: Buddy's one thread (buddySurfaceThread.ts). It runs as an agent - * command (agentCli.ts): `--json` prints `{"threadId", "projectId"}` or `{"error": {"code", - * "message"}}`, and the exit code is 0, 1 for a refusal, 3 when T3 Code is not running. + * Lazurio overlay (root decision 0210): `t3 lazurio buddy thread` names Buddy's one thread + * (buddySurfaceThread.ts) and `t3 lazurio buddy signal` the sign by his avatar (buddySignal.ts). + * `--json` prints the answer or `{"error": {"code", "message"}}`; exit 1 refused, 3 not running. */ +import * as Cause from "effect/Cause"; +import * as Console from "effect/Console"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; -import { Command } from "effect/cli"; +import * as Stream from "effect/Stream"; +import { Command, Flag } from "effect/cli"; import { agentCommandFlags, runAgentCommand } from "./agentCli.ts"; -import { agentCliFailure } from "./agentCliConnection.ts"; +import { + type AgentCliConnection, + type AgentCliFailure, + agentCliFailure, + isAgentCliFailure, + withAgentCliConnection, +} from "./agentCliConnection.ts"; +import { BUDDY_SIGNAL_ROUTE, BuddySignal } from "./buddySignal.ts"; import { BUDDY_THREAD_ROUTE, BuddyThread } from "./buddySurfaceThread.ts"; const decodeBuddyThread = Schema.decodeUnknownOption(BuddyThread); @@ -44,7 +54,143 @@ const threadCommand = Command.make("thread", agentCommandFlags).pipe( ), ); +const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +const decodeBuddySignal = Schema.decodeUnknownOption(BuddySignal); + +/** Buddy's signal now; a refusal fails with its route code (`not_buddy`, `no_thread`). */ +const readBuddySignal = (connection: Pick) => + connection.getRoute(BUDDY_SIGNAL_ROUTE).pipe( + Effect.flatMap(({ status, body }) => { + const signal = decodeBuddySignal(body); + if (status === 200 && Option.isSome(signal)) return Effect.succeed(signal.value); + if (status === 401 || status === 403) + return Effect.fail( + agentCliFailure("unauthorized", `T3 Code refused the session (HTTP ${status}).`), + ); + const refusal = Option.getOrUndefined(decodeRefusal(body)); + return Effect.fail( + agentCliFailure( + refusal?.error.replaceAll("-", "_") ?? "request_failed", + refusal?.message ?? `T3 Code answered HTTP ${status}.`, + ), + ); + }), + ); + +/** The line `--follow` prints while it cannot say Buddy's state: never a quiet one. */ +const unavailableLine = (failure: AgentCliFailure) => ({ + version: 1 as const, + state: "unavailable" as const, + reason: failure.code, + message: failure.message, +}); + +const FOLLOW_SESSION_TTL = Duration.hours(1); +// Scheduled tasks change no thread, so their failures are read on this beat. +const FOLLOW_REREAD = Duration.seconds(60); +const FOLLOW_RECONNECT = [1, 2, 5, 10, 30].map((seconds) => Duration.seconds(seconds)); + +/** + * Print Buddy's signal and then a line whenever any of it changes, `startedBy` and `since` included. + * While it cannot read it (T3 down, the session refused, the stream dropped) it prints + * `unavailable` with the reason and reconnects; only a server without Buddy ends it. + */ +export const followBuddySignal = Effect.fn("LazurioBuddySignal.follow")(function* ( + connect: ( + use: (connection: AgentCliConnection) => Effect.Effect, + ) => Effect.Effect, + print: (line: string) => Effect.Effect, +) { + let last: string | undefined; + let threadId: string | undefined; + const emit = (line: unknown) => { + const text = encodeJson(line); + if (text === last) return Effect.void; + last = text; + return print(text); + }; + let attempt = 0; + while (true) { + const exit = yield* Effect.exit( + connect((connection) => { + const read = readBuddySignal(connection).pipe( + Effect.flatMap((signal) => { + threadId = signal.threadId; + attempt = 0; + return emit(signal); + }), + // Before his first visit his thread does not exist yet: say so and keep following. + Effect.catchIf( + (failure) => failure.code === "no_thread", + (failure) => { + threadId = undefined; + return emit(unavailableLine(failure)); + }, + ), + ); + return read.pipe( + Effect.andThen( + Stream.merge( + connection + .subscribeShell({}) + .pipe( + Stream.filter( + (item) => + item.kind === "thread.updated" && + (threadId === undefined || item.thread.id === threadId), + ), + ), + // The beat after the first reading; the shell's end ends the connection. + Stream.tick(FOLLOW_REREAD).pipe(Stream.drop(1)), + { haltStrategy: "left" }, + ).pipe(Stream.runForEach(() => read)), + ), + Effect.andThen( + Effect.fail(agentCliFailure("connection_lost", "T3 Code ended the stream.")), + ), + ); + }), + ); + if (exit._tag === "Success") return; + const failure = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); + const reported = isAgentCliFailure(failure) + ? failure + : agentCliFailure("request_failed", "Reading Buddy's signal failed."); + yield* emit(unavailableLine(reported)); + if (reported.code === "not_buddy") return yield* reported; + yield* Effect.sleep(FOLLOW_RECONNECT[Math.min(attempt, FOLLOW_RECONNECT.length - 1)]!); + attempt += 1; + } +}); + +const signalCommand = Command.make("signal", { + ...agentCommandFlags, + follow: Flag.Boolean("follow").pipe( + Flag.withDescription( + "Keep printing a JSON line whenever the signal changes, and `unavailable` while it cannot be read.", + ), + Flag.withDefault(false), + ), +}).pipe( + Command.withDescription( + "Print the sign by Buddy's avatar: whether he waits for the person, failed, or has answered.", + ), + Command.withHandler((flags) => + flags.follow + ? followBuddySignal( + (use) => withAgentCliConnection(flags, use, { sessionTtl: FOLLOW_SESSION_TTL }), + Console.log, + ) + : runAgentCommand( + flags, + readBuddySignal, + (signal) => + `Buddy is ${signal.state}${signal.since === null ? "" : ` since ${signal.since}`}.`, + ), + ), +); + export const buddyCommand = Command.make("buddy").pipe( Command.withDescription("Buddy's T3 Code."), - Command.withSubcommands([threadCommand]), + Command.withSubcommands([threadCommand, signalCommand]), ); diff --git a/apps/server/src/lazurio/buddyTaskFailures.test.ts b/apps/server/src/lazurio/buddyTaskFailures.test.ts new file mode 100644 index 0000000000000..1ee69da16f1b8 --- /dev/null +++ b/apps/server/src/lazurio/buddyTaskFailures.test.ts @@ -0,0 +1,235 @@ +// @effect-diagnostics nodeBuiltinImport:off - the server's state is a temporary folder. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, it } from "@effect/vitest"; +import { MessageId, RunId, ScheduledTaskId, ThreadId } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; + +import * as ServerConfig from "../config.ts"; +import { ScheduledTaskService } from "../scheduledTasks/ScheduledTaskService.ts"; +import { buddySignalOf } from "./buddySignal.ts"; +import { + BuddyTaskFailures, + failuresOf, + layer as layerBuddyTaskFailures, + recordTaskFailures, +} from "./buddyTaskFailures.ts"; + +const THREAD = ThreadId.make("buddy"); +const at = (minute: number) => `2026-11-18T20:${String(minute).padStart(2, "0")}:00.000Z`; +const task = ( + runCount: number, + lastRunStatus: "failed" | "succeeded" | "running" | "never", + lastRunAt = at(4), +) => ({ + id: ScheduledTaskId.make("task-1"), + threadId: THREAD, + lastRunStatus, + lastRunAt, + runCount, +}); + +describe("when a task bound to Buddy's thread last failed", () => { + it("is when the list first reported the failed run, not the run's start, and an edit keeps it", () => { + // Started at :04, reported failed at :07. + const failed = recordTaskFailures({}, [task(1, "failed")], at(7)); + assert.deepStrictEqual(failuresOf(failed, THREAD), [DateTime.makeUnsafe(at(7))]); + // Saving or disabling the task at :09 reports the list again with the same run. + const edited = recordTaskFailures(failed, [task(1, "failed")], at(9)); + assert.deepStrictEqual(failuresOf(edited, THREAD), [DateTime.makeUnsafe(at(7))]); + }); + + it("keeps a failure through a later start and success until a completed turn clears it", () => { + // Failed at :05; the next occurrence starts at :10 (running) and succeeds at :12. + const failed = recordTaskFailures({}, [task(1, "failed")], at(5)); + const running = recordTaskFailures(failed, [task(1, "running")], at(10)); + const succeeded = recordTaskFailures(running, [task(2, "succeeded")], at(12)); + assert.deepStrictEqual(failuresOf(succeeded, THREAD), [DateTime.makeUnsafe(at(5))]); + const completed = (minute: number) => ({ + id: RunId.make(`run-${minute}`), + ordinal: minute, + status: "completed" as const, + userMessageId: MessageId.make(`m${minute}`), + completedAt: DateTime.makeUnsafe(at(minute)), + }); + const state = (runs: ReadonlyArray>) => + buddySignalOf({ + threadId: THREAD, + lastVisitedAt: DateTime.makeUnsafe(at(59)), + runs, + pendingRequests: [], + stopRequested: new Set(), + failedTaskRuns: failuresOf(succeeded, THREAD), + messages: new Map(), + }).state; + // No Buddy turn completed after :05: still failed. + assert.strictEqual(state([completed(3)]), "failed"); + // A Buddy turn completed at :15 clears it. + assert.strictEqual(state([completed(3), completed(15)]), "quiet"); + }); + + it("dates a later failed run anew, and drops a deleted task's failure", () => { + const first = recordTaskFailures({}, [task(1, "failed")], at(5)); + const again = recordTaskFailures(first, [task(2, "failed", at(10))], at(12)); + assert.deepStrictEqual(failuresOf(again, THREAD), [DateTime.makeUnsafe(at(12))]); + assert.deepStrictEqual(recordTaskFailures(again, [], at(15)), {}); + }); + + it("dates a failure the record does not know by the reading that finds it, also after a start", () => { + // Started at :04, a Buddy turn completed at :05, failed at :07; the server starts at :30 + // with no record: the failure is dated :30, after the turn, so the sign shows it. + const seen = recordTaskFailures({}, [task(1, "failed", at(4))], at(30)); + const failedTaskRuns = failuresOf(seen, THREAD); + assert.deepStrictEqual(failedTaskRuns, [DateTime.makeUnsafe(at(30))]); + const sign = buddySignalOf({ + threadId: THREAD, + lastVisitedAt: DateTime.makeUnsafe(at(6)), + runs: [ + { + id: RunId.make("run-1"), + ordinal: 1, + status: "completed", + userMessageId: MessageId.make("m1"), + completedAt: DateTime.makeUnsafe(at(5)), + }, + ], + pendingRequests: [], + stopRequested: new Set(), + failedTaskRuns, + messages: new Map(), + }); + assert.strictEqual(sign.state, "failed"); + // A failure the file already holds keeps its time across the restart. + const kept = recordTaskFailures( + { "task-1": { threadId: THREAD, runCount: 1, failedAt: at(7) } }, + [task(1, "failed", at(4))], + at(30), + ); + assert.deepStrictEqual(failuresOf(kept, THREAD), [DateTime.makeUnsafe(at(7))]); + }); + + it("raises a stale failure once, on the first start without a record, until the next completed turn", () => { + // The known cost (Lazurio/t3code#232): failed at :04, cleared by a turn at :05, and this + // version first starts at :30 with no record. The failure is dated :30 and shows ... + const completed = (ordinal: number, minute: number) => ({ + id: RunId.make(`run-${ordinal}`), + ordinal, + status: "completed" as const, + userMessageId: MessageId.make(`m${ordinal}`), + completedAt: DateTime.makeUnsafe(at(minute)), + }); + const state = (runs: ReadonlyArray>, recordedAt: number) => + buddySignalOf({ + threadId: THREAD, + lastVisitedAt: DateTime.makeUnsafe(at(59)), + runs, + pendingRequests: [], + stopRequested: new Set(), + failedTaskRuns: failuresOf( + recordTaskFailures({}, [task(1, "failed", at(3))], at(recordedAt)), + THREAD, + ), + messages: new Map(), + }).state; + assert.strictEqual(state([completed(1, 5)], 30), "failed"); + // ... only until the next completed turn, at :31. + assert.strictEqual(state([completed(1, 5), completed(2, 31)], 30), "quiet"); + // A later start reads the record back, so it raises nothing again. + const recorded = recordTaskFailures({}, [task(1, "failed", at(3))], at(30)); + const restarted = recordTaskFailures(recorded, [task(1, "failed", at(3))], at(45)); + assert.deepStrictEqual(failuresOf(restarted, THREAD), [DateTime.makeUnsafe(at(30))]); + }); + + it("belongs only to the thread the task is bound to", () => { + const other = recordTaskFailures( + {}, + [{ ...task(1, "failed"), threadId: ThreadId.make("other") }], + at(7), + ); + assert.deepStrictEqual(failuresOf(other, THREAD), []); + }); +}); + +const unused = () => Effect.die("unused in this test"); +const tasksListing = (tasks: ReadonlyArray>) => + Layer.succeed( + ScheduledTaskService, + ScheduledTaskService.of({ + list: () => Effect.succeed({ tasks } as never), + subscribeList: () => Stream.empty, + upsert: unused, + setEnabled: unused, + delete: unused, + deleteForThread: unused, + moveThreadTasks: unused, + runNow: unused, + rotateWebhookToken: unused, + listWebhookDeliveries: unused, + getWebhookDelivery: unused, + triggerWebhook: unused, + }), + ); + +describe("the record of a Buddy server's failed tasks", () => { + it.effect("is kept in the server's state, and a restart reads it back", () => { + const base = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "buddy-task-failures-")); + const failedAt = (tasks: ReadonlyArray>) => + Effect.flatMap(BuddyTaskFailures, (failures) => failures.failedAt(THREAD)).pipe( + Effect.provide( + layerBuddyTaskFailures.pipe( + Layer.provide(tasksListing(tasks)), + Layer.provide(ServerConfig.layerTest(base, base)), + ), + ), + Effect.provideService(HostProcess.Environment, { T3CODE_LAZURIO_BUDDY: "1" }), + Effect.provide(NodeServices.layer), + Effect.scoped, + ); + return Effect.gen(function* () { + // The start at :30 finds a failure the record does not know: dated by that reading. + yield* TestClock.setTime(DateTime.toEpochMillis(DateTime.makeUnsafe(at(30)))); + assert.deepStrictEqual(yield* failedAt([task(1, "failed", at(3))]), [ + DateTime.makeUnsafe(at(30)), + ]); + const file = NodeFS.readdirSync(base, { recursive: true }).find((entry) => + String(entry).endsWith("buddy-task-failures.json"), + ); + assert.isDefined(file); + // A restart at :40 keeps the recorded time. + yield* TestClock.setTime(DateTime.toEpochMillis(DateTime.makeUnsafe(at(40)))); + assert.deepStrictEqual(yield* failedAt([task(1, "failed", at(3))]), [ + DateTime.makeUnsafe(at(30)), + ]); + }).pipe( + Effect.ensuring(Effect.sync(() => NodeFS.rmSync(base, { recursive: true, force: true }))), + ); + }); + + it.effect("watches nothing on a server without Buddy", () => + Effect.gen(function* () { + const base = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "buddy-task-failures-")); + const failures = yield* Effect.flatMap(BuddyTaskFailures, (service) => + service.failedAt(THREAD), + ).pipe( + Effect.provide( + layerBuddyTaskFailures.pipe( + Layer.provide(tasksListing([task(1, "failed")])), + Layer.provide(ServerConfig.layerTest(base, base)), + ), + ), + Effect.provideService(HostProcess.Environment, {}), + Effect.provide(NodeServices.layer), + Effect.scoped, + ); + assert.deepStrictEqual(failures, []); + NodeFS.rmSync(base, { recursive: true, force: true }); + }), + ); +}); diff --git a/apps/server/src/lazurio/buddyTaskFailures.ts b/apps/server/src/lazurio/buddyTaskFailures.ts new file mode 100644 index 0000000000000..c90c68306b656 --- /dev/null +++ b/apps/server/src/lazurio/buddyTaskFailures.ts @@ -0,0 +1,127 @@ +/** + * Lazurio overlay (plan LZD-0033): when each task last failed, for the sign by Buddy's avatar. + * A task keeps only its run's start and an `updatedAt` that edits move, so a failed run is + * recorded when the list reports a new `runCount`, which only a finished run changes. + */ + +import type { ScheduledTask, ThreadId } from "@t3tools/contracts"; +import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +import * as HostProcess from "@t3tools/shared/HostProcess"; + +import * as ServerConfig from "../config.ts"; +import { ScheduledTaskService } from "../scheduledTasks/ScheduledTaskService.ts"; +import { isBuddySurfaceEnvironment } from "./buddySurface.ts"; +import { sharedBuddyFolder } from "./buddySurfaceThread.ts"; + +const TaskFailure = Schema.Struct({ + threadId: Schema.NullOr(Schema.String), + runCount: Schema.Number, + failedAt: Schema.String, +}); +const TaskFailuresFile = Schema.Record(Schema.String, TaskFailure); +export type TaskFailures = typeof TaskFailuresFile.Type; + +type TaskRow = Pick; + +/** + * The failures after one reading of the task list at `now`: a failed run not recorded before is + * dated `now`, also on the first reading after a start; a recorded one stays through a later + * start or success, since only a completed Buddy turn clears it (the signal compares the times). + * A deleted task's failure goes with it. + */ +export function recordTaskFailures( + recorded: TaskFailures, + tasks: ReadonlyArray, + now: string, +): TaskFailures { + const next: Record = {}; + for (const task of tasks) { + const known = recorded[task.id]; + if (task.lastRunStatus === "failed" && known?.runCount !== task.runCount) + next[task.id] = { threadId: task.threadId, runCount: task.runCount, failedAt: now }; + else if (known !== undefined) next[task.id] = { ...known, threadId: task.threadId }; + } + return next; +} + +const sameFailures = (left: TaskFailures, right: TaskFailures) => + Object.keys(left).length === Object.keys(right).length && + Object.entries(left).every(([id, failure]) => { + const other = right[id]; + return ( + other !== undefined && + other.threadId === failure.threadId && + other.runCount === failure.runCount && + other.failedAt === failure.failedAt + ); + }); + +/** When each task bound to `threadId` last failed. */ +export const failuresOf = ( + recorded: TaskFailures, + threadId: ThreadId, +): ReadonlyArray => + Object.values(recorded).flatMap((failure) => + failure.threadId === threadId ? [DateTime.makeUnsafe(failure.failedAt)] : [], + ); + +export class BuddyTaskFailures extends Context.Service< + BuddyTaskFailures, + { readonly failedAt: (threadId: ThreadId) => Effect.Effect> } +>()("t3/lazurio/buddyTaskFailures") {} + +const make = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { stateDir } = yield* ServerConfig.ServerConfig; + const filePath = path.join(stateDir, "lazurio", "buddy-task-failures.json"); + const decode = Schema.decodeUnknownOption(Schema.fromJsonString(TaskFailuresFile)); + const encode = Schema.encodeEffect(Schema.fromJsonString(TaskFailuresFile)); + const stored = yield* fs.readFileString(filePath).pipe( + Effect.map((text) => Option.getOrElse(decode(text), () => ({}))), + Effect.orElseSucceed(() => ({})), + ); + const state = yield* Ref.make(stored); + const tasks = yield* Effect.serviceOption(ScheduledTaskService); + // Only a server with Buddy's thread watches its tasks. + const buddy = + isBuddySurfaceEnvironment(yield* HostProcess.Environment) || + (yield* sharedBuddyFolder) !== undefined; + if (Option.isSome(tasks) && buddy) { + const record = Effect.fn("BuddyTaskFailures.record")( + function* (list: { readonly tasks: ReadonlyArray }) { + const now = DateTime.formatIso(yield* DateTime.now); + const before = yield* Ref.get(state); + const after = recordTaskFailures(before, list.tasks, now); + yield* Ref.set(state, after); + if (sameFailures(before, after)) return; + yield* fs.makeDirectory(path.dirname(filePath), { recursive: true, mode: 0o700 }); + yield* writeFileStringAtomically({ filePath, contents: `${yield* encode(after)}\n` }); + }, + Effect.ignoreCause({ log: true }), + ); + // The first reading is the start's, before any change is reported. + yield* tasks.value.list().pipe(Effect.flatMap(record), Effect.ignoreCause({ log: true })); + yield* tasks.value + .subscribeList() + .pipe(Stream.runForEach(record), Effect.ignoreCause({ log: true }), Effect.forkScoped); + } + return BuddyTaskFailures.of({ + failedAt: Effect.fn("BuddyTaskFailures.failedAt")(function* (threadId) { + return failuresOf(yield* Ref.get(state), threadId); + }), + }); +}); + +export const layer = Layer.effect(BuddyTaskFailures, make); diff --git a/apps/server/src/lazurio/overlay.ts b/apps/server/src/lazurio/overlay.ts index ca632c8b142d5..94ba5de04e69f 100644 --- a/apps/server/src/lazurio/overlay.ts +++ b/apps/server/src/lazurio/overlay.ts @@ -8,8 +8,10 @@ import * as Layer from "effect/Layer"; import * as ActingSenders from "./actingSenders.ts"; import * as ActingSendersRoute from "./actingSendersRoute.ts"; import * as AgentCliSendState from "./agentCliSendState.ts"; +import * as BuddySignal from "./buddySignal.ts"; import * as BuddySurfaceDays from "./buddySurfaceDays.ts"; import * as BuddySurfaceThreadRoute from "./buddySurfaceThreadRoute.ts"; +import * as BuddyTaskFailures from "./buddyTaskFailures.ts"; import * as EnvironmentBrowser from "./environmentBrowser.ts"; import * as ModelRecommendationRoute from "./modelRecommendationRoute.ts"; import * as ProviderRefresh from "./providerRefresh.ts"; @@ -27,6 +29,7 @@ export const layer = Layer.mergeAll( ModelRecommendationRoute.layer, ProviderRefreshRoute.layer.pipe(Layer.provide(ProviderRefresh.layer)), BuddySurfaceThreadRoute.layer, + BuddySignal.layer.pipe(Layer.provide(BuddyTaskFailures.layer)), AgentCliSendState.layer, Layer.mergeAll(BuddySurfaceDays.layer, ActingSendersRoute.layer).pipe( Layer.provide(ActingSenders.layer), diff --git a/scripts/lazurio-release-contract.test.mjs b/scripts/lazurio-release-contract.test.mjs index e5c5da049e82b..61ecf9dcc5a7e 100644 --- a/scripts/lazurio-release-contract.test.mjs +++ b/scripts/lazurio-release-contract.test.mjs @@ -2222,7 +2222,7 @@ NodeTest.test("a thread's tabs live in its holder file behind an overlay route", ); NodeAssert.match( overlay, - /\nexport const layer = Layer\.mergeAll\(\s*ThreadTabsRoute\.layer\.pipe\(Layer\.provide\(environmentBrowser\)\),\s*ModelRecommendationRoute\.layer,\s*ProviderRefreshRoute\.layer\.pipe\(Layer\.provide\(ProviderRefresh\.layer\)\),\s*BuddySurfaceThreadRoute\.layer,\s*AgentCliSendState\.layer,\s*Layer\.mergeAll\(BuddySurfaceDays\.layer, ActingSendersRoute\.layer\)\.pipe\(\s*Layer\.provide\(ActingSenders\.layer\),?\s*\),?\s*\)\.pipe\(Layer\.provide\(threadTabs\)\);/, + /\nexport const layer = Layer\.mergeAll\(\s*ThreadTabsRoute\.layer\.pipe\(Layer\.provide\(environmentBrowser\)\),\s*ModelRecommendationRoute\.layer,\s*ProviderRefreshRoute\.layer\.pipe\(Layer\.provide\(ProviderRefresh\.layer\)\),\s*BuddySurfaceThreadRoute\.layer,\s*BuddySignal\.layer\.pipe\(Layer\.provide\(BuddyTaskFailures\.layer\)\),\s*AgentCliSendState\.layer,\s*Layer\.mergeAll\(BuddySurfaceDays\.layer, ActingSendersRoute\.layer\)\.pipe\(\s*Layer\.provide\(ActingSenders\.layer\),?\s*\),?\s*\)\.pipe\(Layer\.provide\(threadTabs\)\);/, ); NodeAssert.match( overlay,