From de14009c99f8e78f4295352064fa8d10b7136437 Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 20 Nov 2026 13:48:07 -0500 Subject: [PATCH] feat: commands record what they waited on, and the invite card says what it's doing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An invite answer once waited ~3.9s before its command started, and nothing on disk said why: the trace had rotated out, and neither the queue wait nor short freezes are saved. Server: every orchestration command runs in one line. Each command now records, when its turn comes, how long it waited, how many commands ran ahead of it, how much of the wait went to their work, and how busy the main loop was (trace span attributes `orchestration.queue_*`). A wait of 1s or more also writes "orchestration command waited in line" to the log, which outlives the traces, at most once per 5s with a count. Times use the monotonic clock, work ahead counts only what ran after the command joined (failure handling included), and the warning is written after the command's turn so it can never hold it up. Web: after the user answers an agent's invite, the card replaces its buttons with what is happening ("Asking X for a review…") until the thread update removes it. The answer is shared from the click on, so a card left and reopened while the answer is on its way shows the same line instead of the choices. --- .../Layers/OrchestrationEngine.test.ts | 108 +++++++++++++++++- .../Layers/OrchestrationEngine.ts | 97 +++++++++++++++- .../components/ChatView.agents.browser.tsx | 18 +-- .../chat/ComposerAgentInvitePanel.tsx | 97 +++++++++++----- 4 files changed, 283 insertions(+), 37 deletions(-) diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts index 51fb7ea56..a03d31462 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts @@ -14,12 +14,13 @@ import { import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Logger from "effect/Logger"; import * as ManagedRuntime from "effect/ManagedRuntime"; import * as Metric from "effect/Metric"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; -import { describe, expect, it } from "vite-plus/test"; +import { describe, expect, it, vi } from "vite-plus/test"; import { PersistenceSqlError } from "../../persistence/Errors.ts"; import { OrchestrationCommandReceiptRepositoryLive } from "../../persistence/Layers/OrchestrationCommandReceipts.ts"; @@ -713,6 +714,111 @@ describe("OrchestrationEngine", () => { await system.dispose(); }); + it("logs a command that waited in line, with what the wait went to", async () => { + type StoredEvent = + ReturnType extends Effect.Effect + ? A + : never; + const events: StoredEvent[] = []; + let nextSequence = 1; + // The first thread's write is slow, so the command behind it waits. + const slowStore: OrchestrationEventStoreShape = { + append(event) { + const save = Effect.sync(() => { + const savedEvent = { ...event, sequence: nextSequence } as StoredEvent; + nextSequence += 1; + events.push(savedEvent); + return savedEvent; + }); + return event.commandId === CommandId.make("cmd-slow-first") + ? Effect.sleep("1200 millis").pipe(Effect.andThen(save)) + : save; + }, + readFromSequence(sequenceExclusive) { + return Stream.fromIterable(events.filter((event) => event.sequence > sequenceExclusive)); + }, + readThreadEvents() { + return Effect.succeed([]); + }, + readAll() { + return Stream.fromIterable(events); + }, + }; + const warnings: Array> = []; + const captureWarnings = Logger.make(({ message, logLevel }) => { + if (logLevel === "Warn") warnings.push(Array.isArray(message) ? message : [message]); + }); + const runtime = ManagedRuntime.make( + OrchestrationEngineLive.pipe( + Layer.provide(OrchestrationProjectionSnapshotQueryLive), + Layer.provide(OrchestrationProjectionPipelineLive), + Layer.provide(Layer.succeed(OrchestrationEventStore, slowStore)), + Layer.provide(OrchestrationCommandReceiptRepositoryLive), + Layer.provide(RepositoryIdentityResolverLive), + Layer.provide(SqlitePersistenceMemory), + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-orchestration-engine-test-" }), + ), + Layer.provideMerge(NodeServices.layer), + Layer.provideMerge(Logger.layer([captureWarnings], { mergeWithExisting: false })), + ), + ); + const engine = await runtime.runPromise(Effect.service(OrchestrationEngineService)); + const createdAt = now(); + const createThread = (commandId: string, threadId: string) => + engine.dispatch({ + type: "thread.create", + commandId: CommandId.make(commandId), + threadId: ThreadId.make(threadId), + projectId: asProjectId("project-wait"), + title: threadId, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5-codex" }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: null, + createdAt, + }); + + await runtime.runPromise( + engine.dispatch({ + type: "project.create", + commandId: CommandId.make("cmd-project-wait-create"), + projectId: asProjectId("project-wait"), + title: "Wait Project", + workspaceRoot: "/tmp/project-wait", + defaultModelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }, + createdAt, + }), + ); + await Promise.all([ + runtime.runPromise(createThread("cmd-slow-first", "thread-slow-first")), + runtime.runPromise(createThread("cmd-waits-behind", "thread-waits-behind")), + ]); + + // Written once its turn is over, just after its answer. + const waited = await vi.waitFor(() => { + const found = warnings.find( + (message) => message[0] === "orchestration command waited in line", + ); + expect(found).toBeDefined(); + return found; + }); + expect(waited?.[1]).toMatchObject({ + commandId: "cmd-waits-behind", + commandType: "thread.create", + commandsAhead: 1, + }); + const detail = waited?.[1] as { waitMs: number; aheadWorkMs: number }; + expect(detail.waitMs).toBeGreaterThanOrEqual(1_000); + // The wait went to the command ahead, not to a held main loop. + expect(detail.aheadWorkMs).toBeGreaterThanOrEqual(1_000); + await runtime.dispose(); + }); + it("records command ack duration using the first committed event type", async () => { const system = await createOrchestrationSystem(); const { engine } = system; diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts index 9fa654611..fb28ac333 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts @@ -5,6 +5,7 @@ import type { ThreadId, } from "@threadlines/contracts"; import { OrchestrationCommand } from "@threadlines/contracts"; +import { performance, type EventLoopUtilization } from "node:perf_hooks"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as DateTime from "effect/DateTime"; @@ -53,8 +54,38 @@ interface CommandEnvelope { command: OrchestrationCommand; result: Deferred.Deferred<{ sequence: number }, OrchestrationDispatchError>; startedAtMs: number; + /** The engine's counters when the command joined the line. */ + joined: CommandLineMark; } +/** + * Every command runs in one line, one at a time. These counters, read when a + * command joins the line and again when its turn comes, say what its wait + * went to: the commands ahead of it, or a main loop held by other work. + * Times are on the monotonic clock, so a clock change can't skew them. + */ +interface CommandLineMark { + readonly atMs: number; + readonly commandsDone: number; + /** Includes what the running command had done by then. */ + readonly commandWorkMs: number; + readonly loop: EventLoopUtilization; +} + +interface CommandWait { + readonly waitMs: number; + readonly commandsAhead: number; + /** Time the line spent on other commands while this one waited. */ + readonly aheadWorkMs: number; + /** The share of the wait the main loop was busy, with that work or any other. */ + readonly loopBusyShare: number; +} + +/** A wait this long is written to the log, which outlives the trace files. */ +const SLOW_COMMAND_WAIT_MS = 1_000; +/** At most one such warning this often; a backed-up line would write hundreds. */ +const SLOW_COMMAND_WAIT_LOG_INTERVAL_MS = 5_000; + function commandToAggregateRef(command: OrchestrationCommand): { readonly aggregateKind: "project" | "thread"; readonly aggregateId: ProjectId | ThreadId; @@ -88,6 +119,45 @@ const makeOrchestrationEngine = Effect.gen(function* () { let commandReadModel = createEmptyReadModel(yield* nowIso); const commandQueue = yield* Queue.unbounded(); + let commandsDone = 0; + let commandWorkMs = 0; + /** When the command now running was picked up; null between commands. */ + let commandRunningSinceMs: number | null = null; + let slowWaitLoggedAtMs = Number.NEGATIVE_INFINITY; + let slowWaitsSinceLog = 0; + const markCommandLine = (): CommandLineMark => { + const atMs = performance.now(); + return { + atMs, + commandsDone, + commandWorkMs: + commandWorkMs + (commandRunningSinceMs === null ? 0 : atMs - commandRunningSinceMs), + loop: performance.eventLoopUtilization(), + }; + }; + const readCommandWait = (joined: CommandLineMark, pickedUpAtMs: number): CommandWait => ({ + waitMs: Math.round(Math.max(0, pickedUpAtMs - joined.atMs)), + commandsAhead: commandsDone - joined.commandsDone, + aheadWorkMs: Math.round(Math.max(0, commandWorkMs - joined.commandWorkMs)), + loopBusyShare: + Math.round(performance.eventLoopUtilization(joined.loop).utilization * 100) / 100, + }); + // Written once the command's turn is over, so the log can never hold it up. + const logSlowWait = (command: OrchestrationCommand, wait: CommandWait, nowMs: number) => { + if (wait.waitMs < SLOW_COMMAND_WAIT_MS) return Effect.void; + slowWaitsSinceLog += 1; + if (nowMs - slowWaitLoggedAtMs < SLOW_COMMAND_WAIT_LOG_INTERVAL_MS) return Effect.void; + slowWaitLoggedAtMs = nowMs; + const slowWaitsSinceLastWarning = slowWaitsSinceLog; + slowWaitsSinceLog = 0; + return Effect.logWarning("orchestration command waited in line", { + commandType: command.type, + commandId: command.commandId, + ...wait, + // This one included. + slowWaitsSinceLastWarning, + }).pipe(Effect.catchCause(() => Effect.void)); + }; const eventPubSub = yield* PubSub.unbounded(); const projectEventsOntoReadModel = ( @@ -105,6 +175,17 @@ const makeOrchestrationEngine = Effect.gen(function* () { const processEnvelope = (envelope: CommandEnvelope): Effect.Effect => { const dispatchStartSequence = commandReadModel.snapshotSequence; let processingStartedAtMs = 0; + let pickedUpAtMs = 0; + let wait: CommandWait | null = null; + // The command's whole turn in the line, failure handling included. + const endTurn = Effect.suspend(() => { + if (wait === null) return Effect.void; + const endedAtMs = performance.now(); + commandsDone += 1; + commandWorkMs += endedAtMs - pickedUpAtMs; + commandRunningSinceMs = null; + return logSlowWait(envelope.command, wait, endedAtMs); + }); const aggregateRef = commandToAggregateRef(envelope.command); const baseMetricAttributes = { commandType: envelope.command.type, @@ -128,11 +209,19 @@ const makeOrchestrationEngine = Effect.gen(function* () { return Effect.exit( Effect.gen(function* () { processingStartedAtMs = yield* Clock.currentTimeMillis; + pickedUpAtMs = performance.now(); + commandRunningSinceMs = pickedUpAtMs; + const currentWait = readCommandWait(envelope.joined, pickedUpAtMs); + wait = currentWait; yield* Effect.annotateCurrentSpan({ "orchestration.command_id": envelope.command.commandId, "orchestration.command_type": envelope.command.type, "orchestration.aggregate_kind": aggregateRef.aggregateKind, "orchestration.aggregate_id": aggregateRef.aggregateId, + "orchestration.queue_wait_ms": currentWait.waitMs, + "orchestration.queue_commands_ahead": currentWait.commandsAhead, + "orchestration.queue_ahead_work_ms": currentWait.aheadWorkMs, + "orchestration.queue_loop_busy_share": currentWait.loopBusyShare, }); const existingReceipt = yield* commandReceiptRepository.getByCommandId({ @@ -222,6 +311,10 @@ const makeOrchestrationEngine = Effect.gen(function* () { ).pipe( Effect.flatMap((exit) => Effect.gen(function* () { + const processingMs = Math.max( + 0, + (yield* Clock.currentTimeMillis) - processingStartedAtMs, + ); const outcome = Exit.isSuccess(exit) ? "success" : Cause.hasInterruptsOnly(exit.cause) @@ -232,7 +325,7 @@ const makeOrchestrationEngine = Effect.gen(function* () { orchestrationCommandDuration, metricAttributes(baseMetricAttributes), ), - Duration.millis(Math.max(0, (yield* Clock.currentTimeMillis) - processingStartedAtMs)), + Duration.millis(processingMs), ); yield* Metric.update( Metric.withAttributes( @@ -283,6 +376,7 @@ const makeOrchestrationEngine = Effect.gen(function* () { yield* Deferred.fail(envelope.result, error); }), ), + Effect.ensuring(endTurn), ); }; @@ -308,6 +402,7 @@ const makeOrchestrationEngine = Effect.gen(function* () { command, result, startedAtMs: yield* Clock.currentTimeMillis, + joined: markCommandLine(), }); return yield* Deferred.await(result); }); diff --git a/apps/web/src/components/ChatView.agents.browser.tsx b/apps/web/src/components/ChatView.agents.browser.tsx index 99e120dc9..d00656693 100644 --- a/apps/web/src/components/ChatView.agents.browser.tsx +++ b/apps/web/src/components/ChatView.agents.browser.tsx @@ -838,17 +838,19 @@ describe("ChatView (full app): agent browser tabs and rooms", () => { { timeout: 8_000, interval: 16 }, ); // The server took the answer, but the thread has not caught up yet, so - // the card is still there. Its buttons stay off: a second answer in - // that gap would be refused as an invite already answered. + // the card is still there. It says what it is doing in place of the + // buttons: a second answer in that gap would be refused as an invite + // already answered. await new Promise((resolve) => setTimeout(resolve, 300)); expect(document.querySelector('[data-testid="agent-invite-panel"]')).toBe(panel); + expect(panel.querySelector('[data-testid="agent-invite-answering"]')?.textContent).toBe( + "Asking GPT-6 Astra for a review…", + ); expect( - [...panel.querySelectorAll("button")] - .filter((button) => - ["Not now", "Add to thread", "Review only"].includes(button.textContent ?? ""), - ) - .map((button) => button.disabled), - ).toEqual([true, true, true]); + [...panel.querySelectorAll("button")].filter((button) => + ["Not now", "Add to thread", "Review only"].includes(button.textContent ?? ""), + ), + ).toEqual([]); } finally { await mounted.cleanup(); } diff --git a/apps/web/src/components/chat/ComposerAgentInvitePanel.tsx b/apps/web/src/components/chat/ComposerAgentInvitePanel.tsx index ac095ce6a..187e0bae8 100644 --- a/apps/web/src/components/chat/ComposerAgentInvitePanel.tsx +++ b/apps/web/src/components/chat/ComposerAgentInvitePanel.tsx @@ -1,6 +1,6 @@ import type { RoomAgentInviteChoice, ScopedThreadRef } from "@threadlines/contracts"; import { ChevronRightIcon } from "lucide-react"; -import { memo, useState } from "react"; +import { memo, useState, useSyncExternalStore } from "react"; import { readEnvironmentApi } from "~/environmentApi"; import { cn, newCommandId } from "~/lib/utils"; @@ -10,11 +10,38 @@ import { failureNotice, toastManager } from "../ui/toast"; import { addComputerNotConnectedNotice } from "../computerConnectionNotices"; /** - * Invites answered from this window. The panel stays up until the thread's - * update takes it away, and it can be left and opened again meanwhile; its - * buttons stay off for an answered invite either way. + * Invites answered from this window, and how, from the click on. The panel + * stays up until the thread's update takes it away, and it can be left and + * opened again meanwhile, even while the answer is still on its way: every + * copy of it follows this, so none offers the choices again. */ -const answeredInviteIds = new Set(); +const answeredInvites = new Map(); +const answeredInviteListeners = new Set<() => void>(); + +function setInviteAnswer(requestId: string, choice: RoomAgentInviteChoice | null): void { + if (choice === null) answeredInvites.delete(requestId); + else answeredInvites.set(requestId, choice); + for (const listener of answeredInviteListeners) listener(); +} + +function subscribeInviteAnswers(listener: () => void): () => void { + answeredInviteListeners.add(listener); + return () => { + answeredInviteListeners.delete(listener); + }; +} + +/** What the panel says while an answer is on its way, in place of the buttons. */ +function answeringLabel(choice: RoomAgentInviteChoice, invite: PendingRoomInvite): string { + switch (choice) { + case "teammate": + return `Adding ${invite.toName} to the thread…`; + case "review": + return `Asking ${invite.toName} for a review…`; + case "decline": + return `Telling ${invite.fromName} not now…`; + } +} /** * An agent asks to bring in another agent for a review @@ -27,7 +54,10 @@ export const ComposerAgentInvitePanel = memo(function ComposerAgentInvitePanel(p invite: PendingRoomInvite; }) { const { invite } = props; - const [responding, setResponding] = useState(() => answeredInviteIds.has(invite.requestId)); + const responding = useSyncExternalStore( + subscribeInviteAnswers, + () => answeredInvites.get(invite.requestId) ?? null, + ); const [showRequest, setShowRequest] = useState(false); const respond = async (choice: RoomAgentInviteChoice) => { @@ -39,7 +69,7 @@ export const ComposerAgentInvitePanel = memo(function ComposerAgentInvitePanel(p }); return; } - setResponding(true); + setInviteAnswer(invite.requestId, choice); try { await api.orchestration.dispatchCommand({ type: "thread.agent-invite.respond", @@ -49,12 +79,11 @@ export const ComposerAgentInvitePanel = memo(function ComposerAgentInvitePanel(p choice, createdAt: new Date().toISOString(), }); - // Answered: the buttons stay off until the thread's update takes the - // panel away, which can land a moment after the reply. A second click - // in that gap would be refused as an invite already answered. - answeredInviteIds.add(invite.requestId); + // Answered: the panel says so until the thread's update takes it away, + // which can land a moment after the reply. A second click in that gap + // would be refused as an invite already answered. } catch (error) { - setResponding(false); + setInviteAnswer(invite.requestId, null); toastManager.add( failureNotice({ title: `Couldn't bring in ${invite.toName}`, @@ -117,21 +146,35 @@ export const ComposerAgentInvitePanel = memo(function ComposerAgentInvitePanel(p

) : null} -
- {choices.map(({ choice, label }, index) => ( - - ))} -
+ {responding !== null ? ( + // The row keeps the buttons' height, so the panel doesn't shift. +

+ {answeringLabel(responding, invite)} +

+ ) : ( +
+ {choices.map(({ choice, label }, index) => ( + + ))} +
+ )} ); });