diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts index 51fb7ea56..a03d31462 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts @@ -14,12 +14,13 @@ import { import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; import * as ManagedRuntime from "effect/ManagedRuntime"; import * as Metric from "effect/Metric"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; -import { describe, expect, it } from "vite-plus/test"; +import { describe, expect, it, vi } from "vite-plus/test"; import { PersistenceSqlError } from "../../persistence/Errors.ts"; import { OrchestrationCommandReceiptRepositoryLive } from "../../persistence/Layers/OrchestrationCommandReceipts.ts"; @@ -713,6 +714,111 @@ describe("OrchestrationEngine", () => { await system.dispose(); }); + it("logs a command that waited in line, with what the wait went to", async () => { + type StoredEvent = + ReturnType extends Effect.Effect + ? A + : never; + const events: StoredEvent[] = []; + let nextSequence = 1; + // The first thread's write is slow, so the command behind it waits. + const slowStore: OrchestrationEventStoreShape = { + append(event) { + const save = Effect.sync(() => { + const savedEvent = { ...event, sequence: nextSequence } as StoredEvent; + nextSequence += 1; + events.push(savedEvent); + return savedEvent; + }); + return event.commandId === CommandId.make("cmd-slow-first") + ? Effect.sleep("1200 millis").pipe(Effect.andThen(save)) + : save; + }, + readFromSequence(sequenceExclusive) { + return Stream.fromIterable(events.filter((event) => event.sequence > sequenceExclusive)); + }, + readThreadEvents() { + return Effect.succeed([]); + }, + readAll() { + return Stream.fromIterable(events); + }, + }; + const warnings: Array> = []; + const captureWarnings = Logger.make(({ message, logLevel }) => { + if (logLevel === "Warn") warnings.push(Array.isArray(message) ? message : [message]); + }); + const runtime = ManagedRuntime.make( + OrchestrationEngineLive.pipe( + Layer.provide(OrchestrationProjectionSnapshotQueryLive), + Layer.provide(OrchestrationProjectionPipelineLive), + Layer.provide(Layer.succeed(OrchestrationEventStore, slowStore)), + Layer.provide(OrchestrationCommandReceiptRepositoryLive), + Layer.provide(RepositoryIdentityResolverLive), + Layer.provide(SqlitePersistenceMemory), + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-orchestration-engine-test-" }), + ), + Layer.provideMerge(NodeServices.layer), + Layer.provideMerge(Logger.layer([captureWarnings], { mergeWithExisting: false })), + ), + ); + const engine = await runtime.runPromise(Effect.service(OrchestrationEngineService)); + const createdAt = now(); + const createThread = (commandId: string, threadId: string) => + engine.dispatch({ + type: "thread.create", + commandId: CommandId.make(commandId), + threadId: ThreadId.make(threadId), + projectId: asProjectId("project-wait"), + title: threadId, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5-codex" }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: null, + createdAt, + }); + + await runtime.runPromise( + engine.dispatch({ + type: "project.create", + commandId: CommandId.make("cmd-project-wait-create"), + projectId: asProjectId("project-wait"), + title: "Wait Project", + workspaceRoot: "/tmp/project-wait", + defaultModelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }, + createdAt, + }), + ); + await Promise.all([ + runtime.runPromise(createThread("cmd-slow-first", "thread-slow-first")), + runtime.runPromise(createThread("cmd-waits-behind", "thread-waits-behind")), + ]); + + // Written once its turn is over, just after its answer. + const waited = await vi.waitFor(() => { + const found = warnings.find( + (message) => message[0] === "orchestration command waited in line", + ); + expect(found).toBeDefined(); + return found; + }); + expect(waited?.[1]).toMatchObject({ + commandId: "cmd-waits-behind", + commandType: "thread.create", + commandsAhead: 1, + }); + const detail = waited?.[1] as { waitMs: number; aheadWorkMs: number }; + expect(detail.waitMs).toBeGreaterThanOrEqual(1_000); + // The wait went to the command ahead, not to a held main loop. + expect(detail.aheadWorkMs).toBeGreaterThanOrEqual(1_000); + await runtime.dispose(); + }); + it("records command ack duration using the first committed event type", async () => { const system = await createOrchestrationSystem(); const { engine } = system; diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts index 9fa654611..fb28ac333 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts @@ -5,6 +5,7 @@ import type { ThreadId, } from "@threadlines/contracts"; import { OrchestrationCommand } from "@threadlines/contracts"; +import { performance, type EventLoopUtilization } from "node:perf_hooks"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; @@ -53,8 +54,38 @@ interface CommandEnvelope { command: OrchestrationCommand; result: Deferred.Deferred<{ sequence: number }, OrchestrationDispatchError>; startedAtMs: number; + /** The engine's counters when the command joined the line. */ + joined: CommandLineMark; } +/** + * Every command runs in one line, one at a time. These counters, read when a + * command joins the line and again when its turn comes, say what its wait + * went to: the commands ahead of it, or a main loop held by other work. + * Times are on the monotonic clock, so a clock change can't skew them. + */ +interface CommandLineMark { + readonly atMs: number; + readonly commandsDone: number; + /** Includes what the running command had done by then. */ + readonly commandWorkMs: number; + readonly loop: EventLoopUtilization; +} + +interface CommandWait { + readonly waitMs: number; + readonly commandsAhead: number; + /** Time the line spent on other commands while this one waited. */ + readonly aheadWorkMs: number; + /** The share of the wait the main loop was busy, with that work or any other. */ + readonly loopBusyShare: number; +} + +/** A wait this long is written to the log, which outlives the trace files. */ +const SLOW_COMMAND_WAIT_MS = 1_000; +/** At most one such warning this often; a backed-up line would write hundreds. */ +const SLOW_COMMAND_WAIT_LOG_INTERVAL_MS = 5_000; + function commandToAggregateRef(command: OrchestrationCommand): { readonly aggregateKind: "project" | "thread"; readonly aggregateId: ProjectId | ThreadId; @@ -88,6 +119,45 @@ const makeOrchestrationEngine = Effect.gen(function* () { let commandReadModel = createEmptyReadModel(yield* nowIso); const commandQueue = yield* Queue.unbounded(); + let commandsDone = 0; + let commandWorkMs = 0; + /** When the command now running was picked up; null between commands. */ + let commandRunningSinceMs: number | null = null; + let slowWaitLoggedAtMs = Number.NEGATIVE_INFINITY; + let slowWaitsSinceLog = 0; + const markCommandLine = (): CommandLineMark => { + const atMs = performance.now(); + return { + atMs, + commandsDone, + commandWorkMs: + commandWorkMs + (commandRunningSinceMs === null ? 0 : atMs - commandRunningSinceMs), + loop: performance.eventLoopUtilization(), + }; + }; + const readCommandWait = (joined: CommandLineMark, pickedUpAtMs: number): CommandWait => ({ + waitMs: Math.round(Math.max(0, pickedUpAtMs - joined.atMs)), + commandsAhead: commandsDone - joined.commandsDone, + aheadWorkMs: Math.round(Math.max(0, commandWorkMs - joined.commandWorkMs)), + loopBusyShare: + Math.round(performance.eventLoopUtilization(joined.loop).utilization * 100) / 100, + }); + // Written once the command's turn is over, so the log can never hold it up. + const logSlowWait = (command: OrchestrationCommand, wait: CommandWait, nowMs: number) => { + if (wait.waitMs < SLOW_COMMAND_WAIT_MS) return Effect.void; + slowWaitsSinceLog += 1; + if (nowMs - slowWaitLoggedAtMs < SLOW_COMMAND_WAIT_LOG_INTERVAL_MS) return Effect.void; + slowWaitLoggedAtMs = nowMs; + const slowWaitsSinceLastWarning = slowWaitsSinceLog; + slowWaitsSinceLog = 0; + return Effect.logWarning("orchestration command waited in line", { + commandType: command.type, + commandId: command.commandId, + ...wait, + // This one included. + slowWaitsSinceLastWarning, + }).pipe(Effect.catchCause(() => Effect.void)); + }; const eventPubSub = yield* PubSub.unbounded(); const projectEventsOntoReadModel = ( @@ -105,6 +175,17 @@ const makeOrchestrationEngine = Effect.gen(function* () { const processEnvelope = (envelope: CommandEnvelope): Effect.Effect => { const dispatchStartSequence = commandReadModel.snapshotSequence; let processingStartedAtMs = 0; + let pickedUpAtMs = 0; + let wait: CommandWait | null = null; + // The command's whole turn in the line, failure handling included. + const endTurn = Effect.suspend(() => { + if (wait === null) return Effect.void; + const endedAtMs = performance.now(); + commandsDone += 1; + commandWorkMs += endedAtMs - pickedUpAtMs; + commandRunningSinceMs = null; + return logSlowWait(envelope.command, wait, endedAtMs); + }); const aggregateRef = commandToAggregateRef(envelope.command); const baseMetricAttributes = { commandType: envelope.command.type, @@ -128,11 +209,19 @@ const makeOrchestrationEngine = Effect.gen(function* () { return Effect.exit( Effect.gen(function* () { processingStartedAtMs = yield* Clock.currentTimeMillis; + pickedUpAtMs = performance.now(); + commandRunningSinceMs = pickedUpAtMs; + const currentWait = readCommandWait(envelope.joined, pickedUpAtMs); + wait = currentWait; yield* Effect.annotateCurrentSpan({ "orchestration.command_id": envelope.command.commandId, "orchestration.command_type": envelope.command.type, "orchestration.aggregate_kind": aggregateRef.aggregateKind, "orchestration.aggregate_id": aggregateRef.aggregateId, + "orchestration.queue_wait_ms": currentWait.waitMs, + "orchestration.queue_commands_ahead": currentWait.commandsAhead, + "orchestration.queue_ahead_work_ms": currentWait.aheadWorkMs, + "orchestration.queue_loop_busy_share": currentWait.loopBusyShare, }); const existingReceipt = yield* commandReceiptRepository.getByCommandId({ @@ -222,6 +311,10 @@ const makeOrchestrationEngine = Effect.gen(function* () { ).pipe( Effect.flatMap((exit) => Effect.gen(function* () { + const processingMs = Math.max( + 0, + (yield* Clock.currentTimeMillis) - processingStartedAtMs, + ); const outcome = Exit.isSuccess(exit) ? "success" : Cause.hasInterruptsOnly(exit.cause) @@ -232,7 +325,7 @@ const makeOrchestrationEngine = Effect.gen(function* () { orchestrationCommandDuration, metricAttributes(baseMetricAttributes), ), - Duration.millis(Math.max(0, (yield* Clock.currentTimeMillis) - processingStartedAtMs)), + Duration.millis(processingMs), ); yield* Metric.update( Metric.withAttributes( @@ -283,6 +376,7 @@ const makeOrchestrationEngine = Effect.gen(function* () { yield* Deferred.fail(envelope.result, error); }), ), + Effect.ensuring(endTurn), ); }; @@ -308,6 +402,7 @@ const makeOrchestrationEngine = Effect.gen(function* () { command, result, startedAtMs: yield* Clock.currentTimeMillis, + joined: markCommandLine(), }); return yield* Deferred.await(result); }); diff --git a/apps/web/src/components/ChatView.agents.browser.tsx b/apps/web/src/components/ChatView.agents.browser.tsx index 99e120dc9..d00656693 100644 --- a/apps/web/src/components/ChatView.agents.browser.tsx +++ b/apps/web/src/components/ChatView.agents.browser.tsx @@ -838,17 +838,19 @@ describe("ChatView (full app): agent browser tabs and rooms", () => { { timeout: 8_000, interval: 16 }, ); // The server took the answer, but the thread has not caught up yet, so - // the card is still there. Its buttons stay off: a second answer in - // that gap would be refused as an invite already answered. + // the card is still there. It says what it is doing in place of the + // buttons: a second answer in that gap would be refused as an invite + // already answered. await new Promise((resolve) => setTimeout(resolve, 300)); expect(document.querySelector('[data-testid="agent-invite-panel"]')).toBe(panel); + expect(panel.querySelector('[data-testid="agent-invite-answering"]')?.textContent).toBe( + "Asking GPT-6 Astra for a review…", + ); expect( - [...panel.querySelectorAll("button")] - .filter((button) => - ["Not now", "Add to thread", "Review only"].includes(button.textContent ?? ""), - ) - .map((button) => button.disabled), - ).toEqual([true, true, true]); + [...panel.querySelectorAll("button")].filter((button) => + ["Not now", "Add to thread", "Review only"].includes(button.textContent ?? ""), + ), + ).toEqual([]); } finally { await mounted.cleanup(); } diff --git a/apps/web/src/components/chat/ComposerAgentInvitePanel.tsx b/apps/web/src/components/chat/ComposerAgentInvitePanel.tsx index ac095ce6a..187e0bae8 100644 --- a/apps/web/src/components/chat/ComposerAgentInvitePanel.tsx +++ b/apps/web/src/components/chat/ComposerAgentInvitePanel.tsx @@ -1,6 +1,6 @@ import type { RoomAgentInviteChoice, ScopedThreadRef } from "@threadlines/contracts"; import { ChevronRightIcon } from "lucide-react"; -import { memo, useState } from "react"; +import { memo, useState, useSyncExternalStore } from "react"; import { readEnvironmentApi } from "~/environmentApi"; import { cn, newCommandId } from "~/lib/utils"; @@ -10,11 +10,38 @@ import { failureNotice, toastManager } from "../ui/toast"; import { addComputerNotConnectedNotice } from "../computerConnectionNotices"; /** - * Invites answered from this window. The panel stays up until the thread's - * update takes it away, and it can be left and opened again meanwhile; its - * buttons stay off for an answered invite either way. + * Invites answered from this window, and how, from the click on. The panel + * stays up until the thread's update takes it away, and it can be left and + * opened again meanwhile, even while the answer is still on its way: every + * copy of it follows this, so none offers the choices again. */ -const answeredInviteIds = new Set(); +const answeredInvites = new Map(); +const answeredInviteListeners = new Set<() => void>(); + +function setInviteAnswer(requestId: string, choice: RoomAgentInviteChoice | null): void { + if (choice === null) answeredInvites.delete(requestId); + else answeredInvites.set(requestId, choice); + for (const listener of answeredInviteListeners) listener(); +} + +function subscribeInviteAnswers(listener: () => void): () => void { + answeredInviteListeners.add(listener); + return () => { + answeredInviteListeners.delete(listener); + }; +} + +/** What the panel says while an answer is on its way, in place of the buttons. */ +function answeringLabel(choice: RoomAgentInviteChoice, invite: PendingRoomInvite): string { + switch (choice) { + case "teammate": + return `Adding ${invite.toName} to the thread…`; + case "review": + return `Asking ${invite.toName} for a review…`; + case "decline": + return `Telling ${invite.fromName} not now…`; + } +} /** * An agent asks to bring in another agent for a review @@ -27,7 +54,10 @@ export const ComposerAgentInvitePanel = memo(function ComposerAgentInvitePanel(p invite: PendingRoomInvite; }) { const { invite } = props; - const [responding, setResponding] = useState(() => answeredInviteIds.has(invite.requestId)); + const responding = useSyncExternalStore( + subscribeInviteAnswers, + () => answeredInvites.get(invite.requestId) ?? null, + ); const [showRequest, setShowRequest] = useState(false); const respond = async (choice: RoomAgentInviteChoice) => { @@ -39,7 +69,7 @@ export const ComposerAgentInvitePanel = memo(function ComposerAgentInvitePanel(p }); return; } - setResponding(true); + setInviteAnswer(invite.requestId, choice); try { await api.orchestration.dispatchCommand({ type: "thread.agent-invite.respond", @@ -49,12 +79,11 @@ export const ComposerAgentInvitePanel = memo(function ComposerAgentInvitePanel(p choice, createdAt: new Date().toISOString(), }); - // Answered: the buttons stay off until the thread's update takes the - // panel away, which can land a moment after the reply. A second click - // in that gap would be refused as an invite already answered. - answeredInviteIds.add(invite.requestId); + // Answered: the panel says so until the thread's update takes it away, + // which can land a moment after the reply. A second click in that gap + // would be refused as an invite already answered. } catch (error) { - setResponding(false); + setInviteAnswer(invite.requestId, null); toastManager.add( failureNotice({ title: `Couldn't bring in ${invite.toName}`, @@ -117,21 +146,35 @@ export const ComposerAgentInvitePanel = memo(function ComposerAgentInvitePanel(p

) : null} -
- {choices.map(({ choice, label }, index) => ( - - ))} -
+ {responding !== null ? ( + // The row keeps the buttons' height, so the panel doesn't shift. +

+ {answeringLabel(responding, invite)} +

+ ) : ( +
+ {choices.map(({ choice, label }, index) => ( + + ))} +
+ )} ); });