Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 107 additions & 1 deletion apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,13 @@ import {
import * as NodeServices from "@effect/platform-node/NodeServices";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Logger from "effect/Logger";
import * as ManagedRuntime from "effect/ManagedRuntime";
import * as Metric from "effect/Metric";
import * as Option from "effect/Option";
import * as Queue from "effect/Queue";
import * as Stream from "effect/Stream";
import { describe, expect, it } from "vite-plus/test";
import { describe, expect, it, vi } from "vite-plus/test";

import { PersistenceSqlError } from "../../persistence/Errors.ts";
import { OrchestrationCommandReceiptRepositoryLive } from "../../persistence/Layers/OrchestrationCommandReceipts.ts";
Expand Down Expand Up @@ -713,6 +714,111 @@ describe("OrchestrationEngine", () => {
await system.dispose();
});

it("logs a command that waited in line, with what the wait went to", async () => {
type StoredEvent =
ReturnType<OrchestrationEventStoreShape["append"]> extends Effect.Effect<infer A, any, any>
? A
: never;
const events: StoredEvent[] = [];
let nextSequence = 1;
// The first thread's write is slow, so the command behind it waits.
const slowStore: OrchestrationEventStoreShape = {
append(event) {
const save = Effect.sync(() => {
const savedEvent = { ...event, sequence: nextSequence } as StoredEvent;
nextSequence += 1;
events.push(savedEvent);
return savedEvent;
});
return event.commandId === CommandId.make("cmd-slow-first")
? Effect.sleep("1200 millis").pipe(Effect.andThen(save))
: save;
},
readFromSequence(sequenceExclusive) {
return Stream.fromIterable(events.filter((event) => event.sequence > sequenceExclusive));
},
readThreadEvents() {
return Effect.succeed([]);
},
readAll() {
return Stream.fromIterable(events);
},
};
const warnings: Array<ReadonlyArray<unknown>> = [];
const captureWarnings = Logger.make(({ message, logLevel }) => {
if (logLevel === "Warn") warnings.push(Array.isArray(message) ? message : [message]);
});
const runtime = ManagedRuntime.make(
OrchestrationEngineLive.pipe(
Layer.provide(OrchestrationProjectionSnapshotQueryLive),
Layer.provide(OrchestrationProjectionPipelineLive),
Layer.provide(Layer.succeed(OrchestrationEventStore, slowStore)),
Layer.provide(OrchestrationCommandReceiptRepositoryLive),
Layer.provide(RepositoryIdentityResolverLive),
Layer.provide(SqlitePersistenceMemory),
Layer.provideMerge(
ServerConfig.layerTest(process.cwd(), { prefix: "t3-orchestration-engine-test-" }),
),
Layer.provideMerge(NodeServices.layer),
Layer.provideMerge(Logger.layer([captureWarnings], { mergeWithExisting: false })),
),
);
const engine = await runtime.runPromise(Effect.service(OrchestrationEngineService));
const createdAt = now();
const createThread = (commandId: string, threadId: string) =>
engine.dispatch({
type: "thread.create",
commandId: CommandId.make(commandId),
threadId: ThreadId.make(threadId),
projectId: asProjectId("project-wait"),
title: threadId,
modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5-codex" },
interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
runtimeMode: "approval-required",
branch: null,
worktreePath: null,
createdAt,
});

await runtime.runPromise(
engine.dispatch({
type: "project.create",
commandId: CommandId.make("cmd-project-wait-create"),
projectId: asProjectId("project-wait"),
title: "Wait Project",
workspaceRoot: "/tmp/project-wait",
defaultModelSelection: {
instanceId: ProviderInstanceId.make("codex"),
model: "gpt-5-codex",
},
createdAt,
}),
);
await Promise.all([
runtime.runPromise(createThread("cmd-slow-first", "thread-slow-first")),
runtime.runPromise(createThread("cmd-waits-behind", "thread-waits-behind")),
]);

// Written once its turn is over, just after its answer.
const waited = await vi.waitFor(() => {
const found = warnings.find(
(message) => message[0] === "orchestration command waited in line",
);
expect(found).toBeDefined();
return found;
});
expect(waited?.[1]).toMatchObject({
commandId: "cmd-waits-behind",
commandType: "thread.create",
commandsAhead: 1,
});
const detail = waited?.[1] as { waitMs: number; aheadWorkMs: number };
expect(detail.waitMs).toBeGreaterThanOrEqual(1_000);
// The wait went to the command ahead, not to a held main loop.
expect(detail.aheadWorkMs).toBeGreaterThanOrEqual(1_000);
await runtime.dispose();
});

it("records command ack duration using the first committed event type", async () => {
const system = await createOrchestrationSystem();
const { engine } = system;
Expand Down
97 changes: 96 additions & 1 deletion apps/server/src/orchestration/Layers/OrchestrationEngine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type {
ThreadId,
} from "@threadlines/contracts";
import { OrchestrationCommand } from "@threadlines/contracts";
import { performance, type EventLoopUtilization } from "node:perf_hooks";
import * as Cause from "effect/Cause";
import * as Clock from "effect/Clock";
import * as DateTime from "effect/DateTime";
Expand Down Expand Up @@ -53,8 +54,38 @@ interface CommandEnvelope {
command: OrchestrationCommand;
result: Deferred.Deferred<{ sequence: number }, OrchestrationDispatchError>;
startedAtMs: number;
/** The engine's counters when the command joined the line. */
joined: CommandLineMark;
}

/**
* Every command runs in one line, one at a time. These counters, read when a
* command joins the line and again when its turn comes, say what its wait
* went to: the commands ahead of it, or a main loop held by other work.
* Times are on the monotonic clock, so a clock change can't skew them.
*/
interface CommandLineMark {
readonly atMs: number;
readonly commandsDone: number;
/** Includes what the running command had done by then. */
readonly commandWorkMs: number;
readonly loop: EventLoopUtilization;
}

interface CommandWait {
readonly waitMs: number;
readonly commandsAhead: number;
/** Time the line spent on other commands while this one waited. */
readonly aheadWorkMs: number;
/** The share of the wait the main loop was busy, with that work or any other. */
readonly loopBusyShare: number;
}

/** A wait this long is written to the log, which outlives the trace files. */
const SLOW_COMMAND_WAIT_MS = 1_000;
/** At most one such warning this often; a backed-up line would write hundreds. */
const SLOW_COMMAND_WAIT_LOG_INTERVAL_MS = 5_000;

function commandToAggregateRef(command: OrchestrationCommand): {
readonly aggregateKind: "project" | "thread";
readonly aggregateId: ProjectId | ThreadId;
Expand Down Expand Up @@ -88,6 +119,45 @@ const makeOrchestrationEngine = Effect.gen(function* () {
let commandReadModel = createEmptyReadModel(yield* nowIso);

const commandQueue = yield* Queue.unbounded<CommandEnvelope>();
let commandsDone = 0;
let commandWorkMs = 0;
/** When the command now running was picked up; null between commands. */
let commandRunningSinceMs: number | null = null;
let slowWaitLoggedAtMs = Number.NEGATIVE_INFINITY;
let slowWaitsSinceLog = 0;
const markCommandLine = (): CommandLineMark => {
const atMs = performance.now();
return {
atMs,
commandsDone,
commandWorkMs:
commandWorkMs + (commandRunningSinceMs === null ? 0 : atMs - commandRunningSinceMs),
loop: performance.eventLoopUtilization(),
};
};
const readCommandWait = (joined: CommandLineMark, pickedUpAtMs: number): CommandWait => ({
waitMs: Math.round(Math.max(0, pickedUpAtMs - joined.atMs)),
commandsAhead: commandsDone - joined.commandsDone,
aheadWorkMs: Math.round(Math.max(0, commandWorkMs - joined.commandWorkMs)),
loopBusyShare:
Math.round(performance.eventLoopUtilization(joined.loop).utilization * 100) / 100,
});
// Written once the command's turn is over, so the log can never hold it up.
const logSlowWait = (command: OrchestrationCommand, wait: CommandWait, nowMs: number) => {
if (wait.waitMs < SLOW_COMMAND_WAIT_MS) return Effect.void;
slowWaitsSinceLog += 1;
if (nowMs - slowWaitLoggedAtMs < SLOW_COMMAND_WAIT_LOG_INTERVAL_MS) return Effect.void;
slowWaitLoggedAtMs = nowMs;
const slowWaitsSinceLastWarning = slowWaitsSinceLog;
slowWaitsSinceLog = 0;
return Effect.logWarning("orchestration command waited in line", {
commandType: command.type,
commandId: command.commandId,
...wait,
// This one included.
slowWaitsSinceLastWarning,
}).pipe(Effect.catchCause(() => Effect.void));
};
const eventPubSub = yield* PubSub.unbounded<OrchestrationEvent>();

const projectEventsOntoReadModel = (
Expand All @@ -105,6 +175,17 @@ const makeOrchestrationEngine = Effect.gen(function* () {
const processEnvelope = (envelope: CommandEnvelope): Effect.Effect<void> => {
const dispatchStartSequence = commandReadModel.snapshotSequence;
let processingStartedAtMs = 0;
let pickedUpAtMs = 0;
let wait: CommandWait | null = null;
// The command's whole turn in the line, failure handling included.
const endTurn = Effect.suspend(() => {
if (wait === null) return Effect.void;
const endedAtMs = performance.now();
commandsDone += 1;
commandWorkMs += endedAtMs - pickedUpAtMs;
commandRunningSinceMs = null;
return logSlowWait(envelope.command, wait, endedAtMs);
});
const aggregateRef = commandToAggregateRef(envelope.command);
const baseMetricAttributes = {
commandType: envelope.command.type,
Expand All @@ -128,11 +209,19 @@ const makeOrchestrationEngine = Effect.gen(function* () {
return Effect.exit(
Effect.gen(function* () {
processingStartedAtMs = yield* Clock.currentTimeMillis;
pickedUpAtMs = performance.now();
commandRunningSinceMs = pickedUpAtMs;
const currentWait = readCommandWait(envelope.joined, pickedUpAtMs);
wait = currentWait;
yield* Effect.annotateCurrentSpan({
"orchestration.command_id": envelope.command.commandId,
"orchestration.command_type": envelope.command.type,
"orchestration.aggregate_kind": aggregateRef.aggregateKind,
"orchestration.aggregate_id": aggregateRef.aggregateId,
"orchestration.queue_wait_ms": currentWait.waitMs,
"orchestration.queue_commands_ahead": currentWait.commandsAhead,
"orchestration.queue_ahead_work_ms": currentWait.aheadWorkMs,
"orchestration.queue_loop_busy_share": currentWait.loopBusyShare,
});

const existingReceipt = yield* commandReceiptRepository.getByCommandId({
Expand Down Expand Up @@ -222,6 +311,10 @@ const makeOrchestrationEngine = Effect.gen(function* () {
).pipe(
Effect.flatMap((exit) =>
Effect.gen(function* () {
const processingMs = Math.max(
0,
(yield* Clock.currentTimeMillis) - processingStartedAtMs,
);
const outcome = Exit.isSuccess(exit)
? "success"
: Cause.hasInterruptsOnly(exit.cause)
Expand All @@ -232,7 +325,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {
orchestrationCommandDuration,
metricAttributes(baseMetricAttributes),
),
Duration.millis(Math.max(0, (yield* Clock.currentTimeMillis) - processingStartedAtMs)),
Duration.millis(processingMs),
);
yield* Metric.update(
Metric.withAttributes(
Expand Down Expand Up @@ -283,6 +376,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {
yield* Deferred.fail(envelope.result, error);
}),
),
Effect.ensuring(endTurn),
);
};

Expand All @@ -308,6 +402,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {
command,
result,
startedAtMs: yield* Clock.currentTimeMillis,
joined: markCommandLine(),
});
return yield* Deferred.await(result);
});
Expand Down
18 changes: 10 additions & 8 deletions apps/web/src/components/ChatView.agents.browser.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -838,17 +838,19 @@ describe("ChatView (full app): agent browser tabs and rooms", () => {
{ timeout: 8_000, interval: 16 },
);
// The server took the answer, but the thread has not caught up yet, so
// the card is still there. Its buttons stay off: a second answer in
// that gap would be refused as an invite already answered.
// the card is still there. It says what it is doing in place of the
// buttons: a second answer in that gap would be refused as an invite
// already answered.
await new Promise((resolve) => setTimeout(resolve, 300));
expect(document.querySelector('[data-testid="agent-invite-panel"]')).toBe(panel);
expect(panel.querySelector('[data-testid="agent-invite-answering"]')?.textContent).toBe(
"Asking GPT-6 Astra for a review…",
);
expect(
[...panel.querySelectorAll<HTMLButtonElement>("button")]
.filter((button) =>
["Not now", "Add to thread", "Review only"].includes(button.textContent ?? ""),
)
.map((button) => button.disabled),
).toEqual([true, true, true]);
[...panel.querySelectorAll<HTMLButtonElement>("button")].filter((button) =>
["Not now", "Add to thread", "Review only"].includes(button.textContent ?? ""),
),
).toEqual([]);
} finally {
await mounted.cleanup();
}
Expand Down
Loading
Loading