Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 79 additions & 0 deletions apps/server/src/orchestration-v2/NotificationMailbox.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ import {
latestProviderTurnForAttempt,
type MessageId,
type OrchestrationV2ConversationMessage,
type OrchestrationV2Run,
type OrchestrationV2Subagent,
type OrchestrationV2ThreadProjection,
type OrchestrationV2TurnItem,
} from "@t3tools/contracts";

export function isBufferedProviderNotification(
Expand Down Expand Up @@ -35,3 +38,79 @@ export function isUndeliveredMailboxSteer(
"completed")
);
}

type DelegatedCompletionRecords = Pick<
OrchestrationV2ThreadProjection,
"messages" | "runs" | "providerTurns"
>;

/** The run's claimed delegated completion still owes a wake: none dispatched, or its steer went unread. */
export function pendingDelegatedCompletionWake(
projection: DelegatedCompletionRecords,
run: Pick<OrchestrationV2Run, "delegatedCompletion">,
) {
const delivery = run.delegatedCompletion?.delivery;
if (run.delegatedCompletion?.disposition !== "open" || delivery == null) return undefined;
const dispatched = projection.messages.some((message) => message.id === delivery.messageId);
return dispatched && !isUndeliveredMailboxSteer(projection, delivery.messageId)
? undefined
: delivery;
}

const TERMINAL_TASK_STATUSES: ReadonlyArray<OrchestrationV2Subagent["status"]> = [
"completed",
"failed",
"cancelled",
"interrupted",
];

/**
* Finished delegated task items whose completion wake has not started its run: claimed into a
* pending wake, or left pending behind a running wake, whose end reserves them a successor. A run
* without its provider thread is never offered a wake, so its tasks hold nothing.
*/
export function delegatedTaskItemsAwaitingWake<Item extends OrchestrationV2TurnItem>(
projection: DelegatedCompletionRecords & {
readonly providerThreads: ReadonlyArray<{ readonly id: string }>;
readonly subagents: ReadonlyArray<
Pick<
OrchestrationV2Subagent,
"id" | "runId" | "origin" | "status" | "completionWake" | "completionDelivery"
>
>;
readonly turnItems: ReadonlyArray<Item>;
},
): ReadonlyArray<Item> {
const taskIds = new Set<string>();
for (const run of projection.runs) {
if (
run.providerThreadId === null ||
!projection.providerThreads.some((thread) => thread.id === run.providerThreadId)
) {
continue;
}
for (const taskId of pendingDelegatedCompletionWake(projection, run)?.taskIds ?? []) {
taskIds.add(taskId);
}
if (run.delegatedCompletion?.disposition !== "open" || run.delegatedCompletion.delivery == null)
continue;
// Mirrors the successor reservation: a live run's blocking wait takes settled_only results.
const runIsLive = ["preparing", "starting", "running"].includes(run.status);
for (const task of projection.subagents) {
if (
task.runId === run.id &&
task.origin === "app_owned" &&
task.completionDelivery?.state === "pending" &&
TERMINAL_TASK_STATUSES.includes(task.status) &&
(!runIsLive || task.completionWake === "always")
) {
taskIds.add(task.id);
}
}
}
if (taskIds.size === 0) return [];
return projection.turnItems.filter(
(item) =>
item.type === "subagent" && item.origin === "app_owned" && taskIds.has(item.subagentId),
);
}
33 changes: 21 additions & 12 deletions apps/server/src/orchestration-v2/Orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ import {
import {
isBufferedProviderNotification,
isUndeliveredMailboxSteer,
pendingDelegatedCompletionWake,
} from "./NotificationMailbox.ts";
import { EventSinkV2 } from "./EventSink.ts";
import {
Expand Down Expand Up @@ -2023,18 +2024,26 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
messageIds: [delivery.messageId],
});
const message = messages.messages[0];
if (message !== undefined) {
if (message.delegatedCompletion === undefined || message.runId === null) return;
const runs = yield* projectionStore.getThreadRecords(threadId, ["runs"], {
runIds: [message.runId],
});
const run = runs.runs[0];
const turns = yield* projectionStore.getThreadRecords(threadId, ["providerTurns"], {
providerTurnAttemptIds: run?.activeAttemptId == null ? [] : [run.activeAttemptId],
});
if (!isUndeliveredMailboxSteer({ ...messages, ...runs, ...turns }, delivery.messageId))
return;
}
const runs =
message?.delegatedCompletion === undefined || message.runId === null
? []
: (yield* projectionStore.getThreadRecords(threadId, ["runs"], {
runIds: [message.runId],
})).runs;
const attemptId = runs[0]?.activeAttemptId;
const providerTurns =
attemptId == null
? []
: (yield* projectionStore.getThreadRecords(threadId, ["providerTurns"], {
providerTurnAttemptIds: [attemptId],
})).providerTurns;
if (
pendingDelegatedCompletionWake(
{ messages: messages.messages, runs, providerTurns },
parentRun,
) === undefined
)
return;
const projection = yield* projectionStore.getThreadRecords(threadId, ["providerThreads"], {
providerThreadIds: parentRun.providerThreadId === null ? [] : [parentRun.providerThreadId],
});
Expand Down
238 changes: 238 additions & 0 deletions apps/server/src/orchestration-v2/ProjectionStore.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import {
MessageId,
type ModelSelection,
type OrchestrationV2ProviderThread,
type OrchestrationV2Run,
type OrchestrationV2Subagent,
type OrchestrationV2TurnItem,
type OrchestrationV2ThreadHistoryPage,
type OrchestrationV2ThreadShellSnapshot,
Expand Down Expand Up @@ -2034,6 +2036,238 @@ const threadWorktreePathsIncludeArchivedOnRequest = Effect.gen(function* () {
);
});

const delegatedCompletionWakeHoldsEveryShell = Effect.gen(function* () {
const store = yield* ProjectionStore.ProjectionStoreV2;
const now = yield* DateTime.now;
const threadId = yield* addRolledBackRecoveryCandidate("delegated-wake-hold");
const providerThreadId = ProviderThreadId.make("provider-thread:delegated-wake-hold");
yield* store.apply({
id: EventId.make("event:delegated-wake-hold:provider-thread"),
type: "provider-thread.updated",
threadId,
occurredAt: now,
payload: {
id: providerThreadId,
appThreadId: threadId,
driver,
providerInstanceId,
providerSessionId: null,
ownerNodeId: null,
nativeThreadRef: null,
nativeConversationHeadRef: null,
lastRunOrdinal: null,
handoffIds: [],
forkedFrom: null,
status: "active",
firstRunOrdinal: 1,
createdAt: now,
updatedAt: now,
},
});
const taskId = NodeId.make("node:delegated-wake-hold:task");
const wakeMessageId = MessageId.make("message:delegated-wake-hold:wake");
const parentRun = {
...(yield* store.getThreadProjection(threadId)).runs[0]!,
id: RunId.make("run:delegated-wake-hold:parent"),
ordinal: 2,
providerThreadId,
userMessageId: MessageId.make("message:delegated-wake-hold:parent"),
rootNodeId: NodeId.make("node:delegated-wake-hold:parent"),
status: "completed" as const,
completedAt: now,
delegatedCompletion: {
disposition: "open" as const,
nextGeneration: 2,
delivery: { generation: 1, messageId: wakeMessageId, taskIds: [taskId] },
},
};
const applyRun = (name: string, run: OrchestrationV2Run) =>
store.apply({
id: EventId.make(`event:delegated-wake-hold:${name}`),
type: "run.updated",
threadId,
runId: run.id,
driver,
providerInstanceId,
occurredAt: now,
payload: run,
});
yield* applyRun("parent", parentRun);
const itemId = TurnItemId.make("item:delegated-wake-hold:task");
yield* store.apply({
id: EventId.make("event:delegated-wake-hold:item"),
type: "turn-item.updated",
threadId,
runId: parentRun.id,
nodeId: taskId,
driver,
occurredAt: now,
payload: {
id: itemId,
type: "subagent",
threadId,
runId: parentRun.id,
nodeId: taskId,
providerThreadId,
providerTurnId: null,
nativeItemRef: null,
parentItemId: null,
ordinal: 10,
status: "completed",
title: "Delegated task",
startedAt: now,
completedAt: now,
updatedAt: now,
subagentId: taskId,
origin: "app_owned",
driver,
providerInstanceId,
childThreadId: null,
prompt: "Do the work",
result: "done",
},
});
const applyTask = (
name: string,
policy: Pick<OrchestrationV2Subagent, "completionWake" | "completionDelivery">,
) =>
store.apply({
id: EventId.make(`event:delegated-wake-hold:${name}`),
type: "subagent.updated",
threadId,
runId: parentRun.id,
nodeId: taskId,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id: taskId,
threadId,
runId: parentRun.id,
parentNodeId: parentRun.rootNodeId,
origin: "app_owned",
createdBy: "agent",
driver,
providerInstanceId,
providerThreadId: null,
childThreadId: null,
nativeTaskRef: null,
prompt: "Do the work",
title: "Delegated task",
model: null,
...policy,
status: "completed",
result: "done",
startedAt: now,
completedAt: now,
updatedAt: now,
},
});
yield* applyTask("task", {
completionWake: "always",
completionDelivery: { state: "claimed", observedByRunId: null },
});
const heldTaskIds = Effect.gen(function* () {
yield* assertShellIndexCoversShells();
const taskIds = (
shell:
| {
readonly pendingBackgroundTasks?:
| ReadonlyArray<{ readonly taskId: string }>
| undefined;
}
| null
| undefined,
) => (shell?.pendingBackgroundTasks ?? []).map((task) => task.taskId);
const producers = [
taskIds(yield* store.getThreadShell(threadId)),
taskIds((yield* store.getShellSnapshot()).threads.find((row) => row.id === threadId)),
taskIds(
ProjectionStore.threadShellFromProjection(yield* store.getThreadProjection(threadId)),
),
taskIds((yield* store.getSettlementCandidates(threadId))[0]),
taskIds((yield* store.getAgentAwarenessSnapshot(threadId)).threads[0]),
];
for (const held of producers) assert.deepEqual(held, producers[0]);
return producers[0];
});

assert.deepEqual(yield* heldTaskIds, [itemId]);

// No provider thread means no wake is ever offered.
yield* applyRun("missing-provider-thread", {
...parentRun,
providerThreadId: ProviderThreadId.make("provider-thread:delegated-wake-hold:missing"),
});
assert.deepEqual(yield* heldTaskIds, []);
yield* applyRun("parent-restored", parentRun);

const wakeRun = {
...parentRun,
id: RunId.make("run:delegated-wake-hold:wake"),
ordinal: 3,
userMessageId: wakeMessageId,
rootNodeId: NodeId.make("node:delegated-wake-hold:wake"),
delegatedCompletion: { disposition: "open" as const, nextGeneration: 1, delivery: null },
};
yield* applyRun("wake-run", wakeRun);
yield* store.apply({
id: EventId.make("event:delegated-wake-hold:wake-message"),
type: "message.updated",
threadId,
runId: wakeRun.id,
occurredAt: now,
payload: {
id: wakeMessageId,
threadId,
runId: wakeRun.id,
nodeId: null,
role: "user",
text: "Delegated task finished",
attachments: [],
streaming: false,
createdBy: "agent",
creationSource: "server",
delegatedCompletion: { parentRunId: parentRun.id, generation: 1, taskIds: [taskId] },
createdAt: now,
updatedAt: now,
},
});
assert.deepEqual(yield* heldTaskIds, []);

// A wake steered into a run that ended before reading it is offered again.
yield* applyRun("wake-run-steered", {
...wakeRun,
userMessageId: MessageId.make("message:delegated-wake-hold:steered-into"),
});
assert.deepEqual(yield* heldTaskIds, [itemId]);
yield* applyRun("wake-run-read", { ...wakeRun, status: "completed" });
assert.deepEqual(yield* heldTaskIds, []);

// A sibling that finished behind that wake waits for the successor its end reserves.
yield* applyTask("task-pending", {
completionWake: "always",
completionDelivery: { state: "pending", observedByRunId: null },
});
assert.deepEqual(yield* heldTaskIds, [itemId]);

// A live parent's blocking wait takes a settled_only result instead.
yield* applyTask("task-settled-only", {
completionWake: "settled_only",
completionDelivery: { state: "pending", observedByRunId: null },
});
assert.deepEqual(yield* heldTaskIds, [itemId]);
yield* applyRun("parent-live", { ...parentRun, status: "running", completedAt: null });
assert.deepEqual(yield* heldTaskIds, []);

// With no wake outstanding, nothing reserves one.
yield* applyRun("parent-settled", {
...parentRun,
delegatedCompletion: { disposition: "open", nextGeneration: 2, delivery: null },
});
assert.deepEqual(yield* heldTaskIds, []);
});

const commandReadsSkipSettledLeafNodes = Effect.gen(function* () {
const store = yield* ProjectionStore.ProjectionStoreV2;
const sql = yield* SqlClient.SqlClient;
Expand Down Expand Up @@ -3736,6 +3970,10 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
"keeps restart-cancelled work through a stale run.updated",
() => restartCancelledWorkSurvivesStaleRunUpdate,
);
it.effect(
"every shell holds a finished delegated task until its completion wake starts",
() => delegatedCompletionWakeHoldsEveryShell,
);
it.effect("records restart-cancelled work without regressing a run that completed since", () =>
Effect.gen(function* () {
const store = yield* ProjectionStore.ProjectionStoreV2;
Expand Down
Loading
Loading