diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts
index 51fb7ea56..a03d31462 100644
--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts
+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts
@@ -14,12 +14,13 @@ import {
import * as NodeServices from "@effect/platform-node/NodeServices";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
+import * as Logger from "effect/Logger";
import * as ManagedRuntime from "effect/ManagedRuntime";
import * as Metric from "effect/Metric";
import * as Option from "effect/Option";
import * as Queue from "effect/Queue";
import * as Stream from "effect/Stream";
-import { describe, expect, it } from "vite-plus/test";
+import { describe, expect, it, vi } from "vite-plus/test";
import { PersistenceSqlError } from "../../persistence/Errors.ts";
import { OrchestrationCommandReceiptRepositoryLive } from "../../persistence/Layers/OrchestrationCommandReceipts.ts";
@@ -713,6 +714,111 @@ describe("OrchestrationEngine", () => {
await system.dispose();
});
+ it("logs a command that waited in line, with what the wait went to", async () => {
+ type StoredEvent =
+ ReturnType extends Effect.Effect
+ ? A
+ : never;
+ const events: StoredEvent[] = [];
+ let nextSequence = 1;
+ // The first thread's write is slow, so the command behind it waits.
+ const slowStore: OrchestrationEventStoreShape = {
+ append(event) {
+ const save = Effect.sync(() => {
+ const savedEvent = { ...event, sequence: nextSequence } as StoredEvent;
+ nextSequence += 1;
+ events.push(savedEvent);
+ return savedEvent;
+ });
+ return event.commandId === CommandId.make("cmd-slow-first")
+ ? Effect.sleep("1200 millis").pipe(Effect.andThen(save))
+ : save;
+ },
+ readFromSequence(sequenceExclusive) {
+ return Stream.fromIterable(events.filter((event) => event.sequence > sequenceExclusive));
+ },
+ readThreadEvents() {
+ return Effect.succeed([]);
+ },
+ readAll() {
+ return Stream.fromIterable(events);
+ },
+ };
+ const warnings: Array> = [];
+ const captureWarnings = Logger.make(({ message, logLevel }) => {
+ if (logLevel === "Warn") warnings.push(Array.isArray(message) ? message : [message]);
+ });
+ const runtime = ManagedRuntime.make(
+ OrchestrationEngineLive.pipe(
+ Layer.provide(OrchestrationProjectionSnapshotQueryLive),
+ Layer.provide(OrchestrationProjectionPipelineLive),
+ Layer.provide(Layer.succeed(OrchestrationEventStore, slowStore)),
+ Layer.provide(OrchestrationCommandReceiptRepositoryLive),
+ Layer.provide(RepositoryIdentityResolverLive),
+ Layer.provide(SqlitePersistenceMemory),
+ Layer.provideMerge(
+ ServerConfig.layerTest(process.cwd(), { prefix: "t3-orchestration-engine-test-" }),
+ ),
+ Layer.provideMerge(NodeServices.layer),
+ Layer.provideMerge(Logger.layer([captureWarnings], { mergeWithExisting: false })),
+ ),
+ );
+ const engine = await runtime.runPromise(Effect.service(OrchestrationEngineService));
+ const createdAt = now();
+ const createThread = (commandId: string, threadId: string) =>
+ engine.dispatch({
+ type: "thread.create",
+ commandId: CommandId.make(commandId),
+ threadId: ThreadId.make(threadId),
+ projectId: asProjectId("project-wait"),
+ title: threadId,
+ modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5-codex" },
+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
+ runtimeMode: "approval-required",
+ branch: null,
+ worktreePath: null,
+ createdAt,
+ });
+
+ await runtime.runPromise(
+ engine.dispatch({
+ type: "project.create",
+ commandId: CommandId.make("cmd-project-wait-create"),
+ projectId: asProjectId("project-wait"),
+ title: "Wait Project",
+ workspaceRoot: "/tmp/project-wait",
+ defaultModelSelection: {
+ instanceId: ProviderInstanceId.make("codex"),
+ model: "gpt-5-codex",
+ },
+ createdAt,
+ }),
+ );
+ await Promise.all([
+ runtime.runPromise(createThread("cmd-slow-first", "thread-slow-first")),
+ runtime.runPromise(createThread("cmd-waits-behind", "thread-waits-behind")),
+ ]);
+
+ // Written once its turn is over, just after its answer.
+ const waited = await vi.waitFor(() => {
+ const found = warnings.find(
+ (message) => message[0] === "orchestration command waited in line",
+ );
+ expect(found).toBeDefined();
+ return found;
+ });
+ expect(waited?.[1]).toMatchObject({
+ commandId: "cmd-waits-behind",
+ commandType: "thread.create",
+ commandsAhead: 1,
+ });
+ const detail = waited?.[1] as { waitMs: number; aheadWorkMs: number };
+ expect(detail.waitMs).toBeGreaterThanOrEqual(1_000);
+ // The wait went to the command ahead, not to a held main loop.
+ expect(detail.aheadWorkMs).toBeGreaterThanOrEqual(1_000);
+ await runtime.dispose();
+ });
+
it("records command ack duration using the first committed event type", async () => {
const system = await createOrchestrationSystem();
const { engine } = system;
diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts
index 9fa654611..fb28ac333 100644
--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts
+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts
@@ -5,6 +5,7 @@ import type {
ThreadId,
} from "@threadlines/contracts";
import { OrchestrationCommand } from "@threadlines/contracts";
+import { performance, type EventLoopUtilization } from "node:perf_hooks";
import * as Cause from "effect/Cause";
import * as Clock from "effect/Clock";
import * as DateTime from "effect/DateTime";
@@ -53,8 +54,38 @@ interface CommandEnvelope {
command: OrchestrationCommand;
result: Deferred.Deferred<{ sequence: number }, OrchestrationDispatchError>;
startedAtMs: number;
+ /** The engine's counters when the command joined the line. */
+ joined: CommandLineMark;
}
+/**
+ * Every command runs in one line, one at a time. These counters, read when a
+ * command joins the line and again when its turn comes, say what its wait
+ * went to: the commands ahead of it, or a main loop held by other work.
+ * Times are on the monotonic clock, so a clock change can't skew them.
+ */
+interface CommandLineMark {
+ readonly atMs: number;
+ readonly commandsDone: number;
+ /** Includes what the running command had done by then. */
+ readonly commandWorkMs: number;
+ readonly loop: EventLoopUtilization;
+}
+
+interface CommandWait {
+ readonly waitMs: number;
+ readonly commandsAhead: number;
+ /** Time the line spent on other commands while this one waited. */
+ readonly aheadWorkMs: number;
+ /** The share of the wait the main loop was busy, with that work or any other. */
+ readonly loopBusyShare: number;
+}
+
+/** A wait this long is written to the log, which outlives the trace files. */
+const SLOW_COMMAND_WAIT_MS = 1_000;
+/** At most one such warning this often; a backed-up line would write hundreds. */
+const SLOW_COMMAND_WAIT_LOG_INTERVAL_MS = 5_000;
+
function commandToAggregateRef(command: OrchestrationCommand): {
readonly aggregateKind: "project" | "thread";
readonly aggregateId: ProjectId | ThreadId;
@@ -88,6 +119,45 @@ const makeOrchestrationEngine = Effect.gen(function* () {
let commandReadModel = createEmptyReadModel(yield* nowIso);
const commandQueue = yield* Queue.unbounded();
+ let commandsDone = 0;
+ let commandWorkMs = 0;
+ /** When the command now running was picked up; null between commands. */
+ let commandRunningSinceMs: number | null = null;
+ let slowWaitLoggedAtMs = Number.NEGATIVE_INFINITY;
+ let slowWaitsSinceLog = 0;
+ const markCommandLine = (): CommandLineMark => {
+ const atMs = performance.now();
+ return {
+ atMs,
+ commandsDone,
+ commandWorkMs:
+ commandWorkMs + (commandRunningSinceMs === null ? 0 : atMs - commandRunningSinceMs),
+ loop: performance.eventLoopUtilization(),
+ };
+ };
+ const readCommandWait = (joined: CommandLineMark, pickedUpAtMs: number): CommandWait => ({
+ waitMs: Math.round(Math.max(0, pickedUpAtMs - joined.atMs)),
+ commandsAhead: commandsDone - joined.commandsDone,
+ aheadWorkMs: Math.round(Math.max(0, commandWorkMs - joined.commandWorkMs)),
+ loopBusyShare:
+ Math.round(performance.eventLoopUtilization(joined.loop).utilization * 100) / 100,
+ });
+ // Written once the command's turn is over, so the log can never hold it up.
+ const logSlowWait = (command: OrchestrationCommand, wait: CommandWait, nowMs: number) => {
+ if (wait.waitMs < SLOW_COMMAND_WAIT_MS) return Effect.void;
+ slowWaitsSinceLog += 1;
+ if (nowMs - slowWaitLoggedAtMs < SLOW_COMMAND_WAIT_LOG_INTERVAL_MS) return Effect.void;
+ slowWaitLoggedAtMs = nowMs;
+ const slowWaitsSinceLastWarning = slowWaitsSinceLog;
+ slowWaitsSinceLog = 0;
+ return Effect.logWarning("orchestration command waited in line", {
+ commandType: command.type,
+ commandId: command.commandId,
+ ...wait,
+ // This one included.
+ slowWaitsSinceLastWarning,
+ }).pipe(Effect.catchCause(() => Effect.void));
+ };
const eventPubSub = yield* PubSub.unbounded();
const projectEventsOntoReadModel = (
@@ -105,6 +175,17 @@ const makeOrchestrationEngine = Effect.gen(function* () {
const processEnvelope = (envelope: CommandEnvelope): Effect.Effect => {
const dispatchStartSequence = commandReadModel.snapshotSequence;
let processingStartedAtMs = 0;
+ let pickedUpAtMs = 0;
+ let wait: CommandWait | null = null;
+ // The command's whole turn in the line, failure handling included.
+ const endTurn = Effect.suspend(() => {
+ if (wait === null) return Effect.void;
+ const endedAtMs = performance.now();
+ commandsDone += 1;
+ commandWorkMs += endedAtMs - pickedUpAtMs;
+ commandRunningSinceMs = null;
+ return logSlowWait(envelope.command, wait, endedAtMs);
+ });
const aggregateRef = commandToAggregateRef(envelope.command);
const baseMetricAttributes = {
commandType: envelope.command.type,
@@ -128,11 +209,19 @@ const makeOrchestrationEngine = Effect.gen(function* () {
return Effect.exit(
Effect.gen(function* () {
processingStartedAtMs = yield* Clock.currentTimeMillis;
+ pickedUpAtMs = performance.now();
+ commandRunningSinceMs = pickedUpAtMs;
+ const currentWait = readCommandWait(envelope.joined, pickedUpAtMs);
+ wait = currentWait;
yield* Effect.annotateCurrentSpan({
"orchestration.command_id": envelope.command.commandId,
"orchestration.command_type": envelope.command.type,
"orchestration.aggregate_kind": aggregateRef.aggregateKind,
"orchestration.aggregate_id": aggregateRef.aggregateId,
+ "orchestration.queue_wait_ms": currentWait.waitMs,
+ "orchestration.queue_commands_ahead": currentWait.commandsAhead,
+ "orchestration.queue_ahead_work_ms": currentWait.aheadWorkMs,
+ "orchestration.queue_loop_busy_share": currentWait.loopBusyShare,
});
const existingReceipt = yield* commandReceiptRepository.getByCommandId({
@@ -222,6 +311,10 @@ const makeOrchestrationEngine = Effect.gen(function* () {
).pipe(
Effect.flatMap((exit) =>
Effect.gen(function* () {
+ const processingMs = Math.max(
+ 0,
+ (yield* Clock.currentTimeMillis) - processingStartedAtMs,
+ );
const outcome = Exit.isSuccess(exit)
? "success"
: Cause.hasInterruptsOnly(exit.cause)
@@ -232,7 +325,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {
orchestrationCommandDuration,
metricAttributes(baseMetricAttributes),
),
- Duration.millis(Math.max(0, (yield* Clock.currentTimeMillis) - processingStartedAtMs)),
+ Duration.millis(processingMs),
);
yield* Metric.update(
Metric.withAttributes(
@@ -283,6 +376,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {
yield* Deferred.fail(envelope.result, error);
}),
),
+ Effect.ensuring(endTurn),
);
};
@@ -308,6 +402,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {
command,
result,
startedAtMs: yield* Clock.currentTimeMillis,
+ joined: markCommandLine(),
});
return yield* Deferred.await(result);
});
diff --git a/apps/web/src/components/ChatView.agents.browser.tsx b/apps/web/src/components/ChatView.agents.browser.tsx
index 99e120dc9..d00656693 100644
--- a/apps/web/src/components/ChatView.agents.browser.tsx
+++ b/apps/web/src/components/ChatView.agents.browser.tsx
@@ -838,17 +838,19 @@ describe("ChatView (full app): agent browser tabs and rooms", () => {
{ timeout: 8_000, interval: 16 },
);
// The server took the answer, but the thread has not caught up yet, so
- // the card is still there. Its buttons stay off: a second answer in
- // that gap would be refused as an invite already answered.
+ // the card is still there. It says what it is doing in place of the
+ // buttons: a second answer in that gap would be refused as an invite
+ // already answered.
await new Promise((resolve) => setTimeout(resolve, 300));
expect(document.querySelector('[data-testid="agent-invite-panel"]')).toBe(panel);
+ expect(panel.querySelector('[data-testid="agent-invite-answering"]')?.textContent).toBe(
+ "Asking GPT-6 Astra for a review…",
+ );
expect(
- [...panel.querySelectorAll("button")]
- .filter((button) =>
- ["Not now", "Add to thread", "Review only"].includes(button.textContent ?? ""),
- )
- .map((button) => button.disabled),
- ).toEqual([true, true, true]);
+ [...panel.querySelectorAll("button")].filter((button) =>
+ ["Not now", "Add to thread", "Review only"].includes(button.textContent ?? ""),
+ ),
+ ).toEqual([]);
} finally {
await mounted.cleanup();
}
diff --git a/apps/web/src/components/chat/ComposerAgentInvitePanel.tsx b/apps/web/src/components/chat/ComposerAgentInvitePanel.tsx
index ac095ce6a..187e0bae8 100644
--- a/apps/web/src/components/chat/ComposerAgentInvitePanel.tsx
+++ b/apps/web/src/components/chat/ComposerAgentInvitePanel.tsx
@@ -1,6 +1,6 @@
import type { RoomAgentInviteChoice, ScopedThreadRef } from "@threadlines/contracts";
import { ChevronRightIcon } from "lucide-react";
-import { memo, useState } from "react";
+import { memo, useState, useSyncExternalStore } from "react";
import { readEnvironmentApi } from "~/environmentApi";
import { cn, newCommandId } from "~/lib/utils";
@@ -10,11 +10,38 @@ import { failureNotice, toastManager } from "../ui/toast";
import { addComputerNotConnectedNotice } from "../computerConnectionNotices";
/**
- * Invites answered from this window. The panel stays up until the thread's
- * update takes it away, and it can be left and opened again meanwhile; its
- * buttons stay off for an answered invite either way.
+ * Invites answered from this window, and how, from the click on. The panel
+ * stays up until the thread's update takes it away, and it can be left and
+ * opened again meanwhile, even while the answer is still on its way: every
+ * copy of it follows this, so none offers the choices again.
*/
-const answeredInviteIds = new Set();
+const answeredInvites = new Map();
+const answeredInviteListeners = new Set<() => void>();
+
+function setInviteAnswer(requestId: string, choice: RoomAgentInviteChoice | null): void {
+ if (choice === null) answeredInvites.delete(requestId);
+ else answeredInvites.set(requestId, choice);
+ for (const listener of answeredInviteListeners) listener();
+}
+
+function subscribeInviteAnswers(listener: () => void): () => void {
+ answeredInviteListeners.add(listener);
+ return () => {
+ answeredInviteListeners.delete(listener);
+ };
+}
+
+/** What the panel says while an answer is on its way, in place of the buttons. */
+function answeringLabel(choice: RoomAgentInviteChoice, invite: PendingRoomInvite): string {
+ switch (choice) {
+ case "teammate":
+ return `Adding ${invite.toName} to the thread…`;
+ case "review":
+ return `Asking ${invite.toName} for a review…`;
+ case "decline":
+ return `Telling ${invite.fromName} not now…`;
+ }
+}
/**
* An agent asks to bring in another agent for a review
@@ -27,7 +54,10 @@ export const ComposerAgentInvitePanel = memo(function ComposerAgentInvitePanel(p
invite: PendingRoomInvite;
}) {
const { invite } = props;
- const [responding, setResponding] = useState(() => answeredInviteIds.has(invite.requestId));
+ const responding = useSyncExternalStore(
+ subscribeInviteAnswers,
+ () => answeredInvites.get(invite.requestId) ?? null,
+ );
const [showRequest, setShowRequest] = useState(false);
const respond = async (choice: RoomAgentInviteChoice) => {
@@ -39,7 +69,7 @@ export const ComposerAgentInvitePanel = memo(function ComposerAgentInvitePanel(p
});
return;
}
- setResponding(true);
+ setInviteAnswer(invite.requestId, choice);
try {
await api.orchestration.dispatchCommand({
type: "thread.agent-invite.respond",
@@ -49,12 +79,11 @@ export const ComposerAgentInvitePanel = memo(function ComposerAgentInvitePanel(p
choice,
createdAt: new Date().toISOString(),
});
- // Answered: the buttons stay off until the thread's update takes the
- // panel away, which can land a moment after the reply. A second click
- // in that gap would be refused as an invite already answered.
- answeredInviteIds.add(invite.requestId);
+ // Answered: the panel says so until the thread's update takes it away,
+ // which can land a moment after the reply. A second click in that gap
+ // would be refused as an invite already answered.
} catch (error) {
- setResponding(false);
+ setInviteAnswer(invite.requestId, null);
toastManager.add(
failureNotice({
title: `Couldn't bring in ${invite.toName}`,
@@ -117,21 +146,35 @@ export const ComposerAgentInvitePanel = memo(function ComposerAgentInvitePanel(p
) : null}
-
- {choices.map(({ choice, label }, index) => (
-
- ))}
-
+ {responding !== null ? (
+ // The row keeps the buttons' height, so the panel doesn't shift.
+
+ {answeringLabel(responding, invite)}
+
+ ) : (
+
+ {choices.map(({ choice, label }, index) => (
+
+ ))}
+
+ )}
);
});