Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 109 additions & 4 deletions apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1777,7 +1777,9 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
string,
Record<string, { status: string; result: string | null }>
>();
const state: { roster: ReadonlyArray<unknown> | undefined } = { roster: undefined };
const state: { roster: OrchestrationV2ProviderThread["pendingBackgroundTasks"] } = {
roster: undefined,
};
let wake = yield* Deferred.make<void>();
yield* runtime.events.pipe(
Stream.tap((event) =>
Expand Down Expand Up @@ -1826,7 +1828,11 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
yield* until(check);
});
});
return { calls, callsById, callsAtChildTurnStart, childTurns, state, until };
// A reported subagent stays listed until the wake that reads its report, which these replays never start.
const rosterSettled = () =>
state.roster !== undefined &&
state.roster.every((task) => callsById.get(task.taskId)?.status !== "running");
return { calls, callsById, callsAtChildTurnStart, childTurns, state, until, rosterSettled };
});
const launchedAndRunning = [
...backgroundLaunch(CHILD),
Expand Down Expand Up @@ -2006,7 +2012,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
() =>
watch.calls.get("Again")?.status === "completed" &&
watch.calls.get("Sleep")?.status === "completed" &&
watch.state.roster?.length === 0,
watch.rosterSettled(),
);
assert.deepEqual(Object.fromEntries(watch.calls), {
Sleep: { status: "completed", result: "CHILD_OK" },
Expand All @@ -2015,6 +2021,105 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
}),
);

it.effect.each(["dispatched", "failed", "cleared", "stopped", "cancelled"] as const)(
"lists a finished background subagent until its wake's continuation is %s",
(outcome) =>
Effect.gen(function* () {
const offered =
yield* Deferred.make<ProviderContinuationRequests.ProviderContinuationRequest>();
const gate = makeProviderReplayGate(["cancel"]);
const { runtime, thread } = yield* resumed(
[
...launchedAndRunning,
event("session.execution.succeeded", { sessionID: SESSION }),
event("session.execution.succeeded", { sessionID: CHILD }),
childReport("msg_report", "CHILD_OK"),
...(outcome === "cancelled"
? [
event(
"session.inbox.cancelled",
{ sessionID: SESSION, inboxID: "msg_report" },
"cancel",
),
]
: [event("session.execution.started", { sessionID: SESSION })]),
...(outcome === "stopped"
? [
out("session.interrupt", { sessionID: SESSION }),
reply("session.interrupt", { interrupted: true }),
]
: []),
],
outcome === "cancelled" ? { replayGate: gate } : undefined,
).pipe(
Effect.provideService(ProviderContinuationRequests.ProviderContinuationRequests, {
offer: (request) => Deferred.succeed(offered, request).pipe(Effect.asVoid),
take: Effect.never,
}),
);
const rosters: Array<ReadonlyArray<string>> = [];
const watch = yield* watchCalls(
{
...runtime,
events: runtime.events.pipe(
Stream.tap((event) =>
Effect.sync(() => {
if (
event.type === "provider_thread.updated" &&
event.providerThread.id === thread.id
)
rosters.push(
(event.providerThread.pendingBackgroundTasks ?? []).map(
(task) => task.taskId,
),
);
}),
),
),
},
thread,
);
yield* runtime.startTurn(withLineage(thread));
yield* watch.until(() => watch.calls.get("Sleep")?.status === "completed");
assert.deepEqual(watch.callsById.get("call-background"), {
status: "completed",
result: "CHILD_OK",
});
const listed = rosters.findIndex((roster) => roster.includes("call-background"));
// The held task matches the running call, so the roster published at launch stays current.
const held = () => {
assert.deepEqual(rosters.at(-1), ["call-background"]);
assert.isFalse(rosters.slice(listed).some((roster) => roster.length === 0));
};
held();
if (outcome === "cancelled") {
gate.release("cancel");
} else {
const offer = yield* Deferred.await(offered);
held();
if (outcome === "dispatched" || outcome === "failed") {
const dispatched = yield* Effect.exit(
offer.dispatchIfCurrent!(
outcome === "dispatched" ? Effect.void : Effect.fail("dispatch failed"),
),
);
assert.strictEqual(Exit.isSuccess(dispatched), outcome === "dispatched");
} else if (outcome === "cleared") {
yield* offer.clearIfCurrent!();
} else {
yield* runtime.interruptTurn({
providerThread: thread,
providerTurnId: yield* providerTurnId,
requestRuntimeRestart: true,
});
yield* watch.until(() => rosters.at(-1)?.length === 0);
assert.isTrue(Option.isNone(yield* offer.dispatchIfCurrent!(Effect.void)));
}
}
yield* watch.until(() => watch.state.roster?.length === 0);
}),
);

it.effect("keeps a call to a finished subagent for its own run's report", () =>
Effect.gen(function* () {
const { runtime, thread } = yield* resumed([
Expand Down Expand Up @@ -2145,7 +2250,7 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
yield* runtime.startTurn(withLineage(thread));
yield* watch.until(
() =>
watch.state.roster?.length === 0 &&
watch.rosterSettled() &&
Object.keys(expected).every((id) => watch.callsById.get(id)?.status === "completed"),
);
assert.deepEqual(Object.fromEntries(watch.callsById), expected);
Expand Down
119 changes: 119 additions & 0 deletions apps/server/src/orchestration-v2/ProjectionRecovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1357,6 +1357,125 @@ it.effect.each([
}).pipe(Effect.provide(layerRecoveryTest)),
);

it.effect.each(["completed", "running"] as const)(
"cuts off and wakes for a listed native subagent only while its item is open (%s)",
(status) =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
const sql = yield* SqlClient.SqlClient;
const now = yield* DateTime.now;
const threadId = yield* createThread(`listed-subagent-${status}`);
const providerThreadId = ProviderThreadId.make(`provider-thread:${threadId}`);
const providerSessionId = ProviderSessionId.make(`session:${threadId}`);
const subagentId = NodeId.make(`node:${threadId}:subagent`);
yield* projections.apply({
id: EventId.make(`event:${threadId}:session`),
type: "provider-session.attached",
threadId,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id: providerSessionId,
driver,
providerInstanceId,
status: "ready",
cwd: "/workspace",
model: modelSelection.model,
capabilities: CodexProviderCapabilitiesV2,
createdAt: now,
updatedAt: now,
lastError: null,
},
});
const settledRun = yield* createRun(threadId, "completed", { providerThreadId });
yield* projections.apply({
id: EventId.make(`event:${threadId}:roster`),
type: "provider-thread.updated",
threadId,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id: providerThreadId,
appThreadId: threadId,
ownerNodeId: null,
driver,
providerInstanceId,
providerSessionId,
nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" },
nativeConversationHeadRef: null,
status: "idle",
firstRunOrdinal: 1,
lastRunOrdinal: 1,
handoffIds: [],
forkedFrom: null,
createdAt: now,
updatedAt: now,
pendingBackgroundTasks: [
{ taskId: "call-review", description: "Review", kind: "subagent" },
],
},
});
yield* projections.apply({
id: EventId.make(`event:${threadId}:subagent-item`),
type: "turn-item.updated",
threadId,
occurredAt: now,
payload: {
id: TurnItemId.make(`item:${threadId}:subagent`),
threadId,
runId: settledRun,
nodeId: subagentId,
providerThreadId,
providerTurnId: null,
nativeItemRef: { driver, nativeId: "call-review", strength: "strong" },
parentItemId: null,
ordinal: 1,
type: "subagent",
title: "Review",
subagentId,
origin: "provider_native",
driver,
providerInstanceId,
childThreadId: null,
prompt: "review src",
status,
result: status === "completed" ? "LGTM" : null,
startedAt: now,
completedAt: status === "completed" ? now : null,
updatedAt: now,
},
});

const recoveryState = yield* projections.getRuntimeRecoveryProjection(threadId);
assert.deepEqual(
(recoveryState.settledRosterSubagents ?? []).map((item) => item.nativeItemRef?.nativeId),
status === "completed" ? ["call-review"] : [],
);

yield* recovery.reconcile("startup");

const effects = yield* sql<{ readonly effect_id: string }>`
SELECT effect_id FROM orchestration_v2_effect_outbox WHERE thread_id = ${threadId}
`;
const projection = yield* projections.getThreadProjection(threadId);
if (status === "completed") {
assert.deepEqual(effects, []);
assert.equal(projection.runs[0]?.status, "completed");
assert.deepEqual(projection.runs[0]?.restartCancelledBackgroundWork ?? [], []);
} else {
assert.deepEqual(effects, [{ effect_id: `effect:restart-continuation:${settledRun}` }]);
assert.deepEqual(
(projection.runs[0]?.restartCancelledBackgroundWork ?? []).map((work) => work.kind),
["subagent"],
);
}
assert.deepEqual(projection.providerThreads[0]?.pendingBackgroundTasks ?? [], []);
}).pipe(Effect.provide(layerRecoveryTest)),
);

it.effect("wakes a settled run whose native subagent ran on its own provider thread", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
Expand Down
29 changes: 28 additions & 1 deletion apps/server/src/orchestration-v2/ProjectionStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,10 @@ export type ProjectionRuntimeRecoveryState = Pick<
| "runtimeRequests"
| "messages"
| "turnItems"
>;
> & {
/** Ended subagent items whose task the provider thread still lists; `turnItems` holds only open ones. */
readonly settledRosterSubagents?: OrchestrationV2ThreadProjection["turnItems"];
};

export type ProjectionPendingUserInputs = Pick<
OrchestrationV2ThreadProjection,
Expand Down Expand Up @@ -5911,6 +5914,27 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
)
ORDER BY item.ordinal ASC, item.turn_item_id ASC
`,
// Ended subagents still listed while their report waits for its wake. CROSS JOIN keeps
// the roster outermost, so items are read only when a subagent is listed.
settledRosterSubagents: sql<PayloadRow>`
SELECT item.payload_json
FROM orchestration_v2_projection_provider_threads AS roster
CROSS JOIN json_each(
CASE WHEN json_valid(roster.payload_json) THEN roster.payload_json ELSE '{}' END,
'$.pendingBackgroundTasks'
) AS task
CROSS JOIN orchestration_v2_projection_turn_items AS item
WHERE roster.thread_id = ${threadId}
AND json_extract(task.value, '$.kind') = 'subagent'
AND item.thread_id = ${threadId}
AND item.type = 'subagent'
AND item.status IN ('completed', 'failed', 'cancelled', 'interrupted')
AND json_extract(
CASE WHEN json_valid(item.payload_json) THEN item.payload_json ELSE '{}' END,
'$.nativeItemRef.nativeId'
) = json_extract(task.value, '$.taskId')
ORDER BY item.ordinal ASC, item.turn_item_id ASC
`,
};
const sections = Object.entries(queries).map(
([section, query]) => sql`
Expand All @@ -5935,6 +5959,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
runtimeRequests,
messages,
turnItems,
settledRosterSubagents,
] = yield* Effect.all([
decodeRows(decodeRunPayload, threadId)(readRows("runs")),
decodeRows(decodeRunAttemptPayload, threadId)(readRows("attempts")),
Expand All @@ -5946,6 +5971,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
decodeRows(decodeRuntimeRequestPayload, threadId)(readRows("runtimeRequests")),
decodeRows(decodeMessagePayload, threadId)(readRows("messages")),
decodeRows(decodeTurnItemPayload, threadId)(readRows("turnItems")),
decodeRows(decodeTurnItemPayload, threadId)(readRows("settledRosterSubagents")),
]);
return {
thread,
Expand All @@ -5959,6 +5985,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
runtimeRequests,
messages,
turnItems,
settledRosterSubagents,
} satisfies ProjectionRuntimeRecoveryState;
},
(effect, threadId) =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import {
mergeRestartCancelledBackgroundWork,
latestStartedRun,
neverStarted,
rosterTaskEnded,
} from "./RestartBackgroundNote.ts";

export class ProviderRuntimeRecoveryError extends Schema.TaggedError<ProviderRuntimeRecoveryError>()(
Expand Down Expand Up @@ -922,7 +923,8 @@ export const make = Effect.gen(function* () {
if (
cancelledBackgroundNativeIds.has(task.taskId) ||
(task.parentTaskId !== undefined &&
cancelledBackgroundNativeIds.has(task.parentTaskId))
cancelledBackgroundNativeIds.has(task.parentTaskId)) ||
rosterTaskEnded(task, providerThread.id, projection)
)
continue;
cancelledBackgroundNativeIds.add(task.taskId);
Expand Down
28 changes: 28 additions & 0 deletions apps/server/src/orchestration-v2/RestartBackgroundNote.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import {
isDeliveredProviderTurn,
isOrchestrationV2WorkActive,
type NodeId,
type OrchestrationV2ConversationMessage,
type OrchestrationV2PendingBackgroundTask,
Expand Down Expand Up @@ -132,6 +133,33 @@ export function cancelledRosterTaskWork(task: OrchestrationV2PendingBackgroundTa
};
}

type RosterItem = Pick<
OrchestrationV2TurnItem,
"type" | "providerThreadId" | "nativeItemRef" | "status"
>;

/** A listed subagent whose own item already ended was waiting only for its report's wake, not cut off. */
export function rosterTaskEnded(
task: OrchestrationV2PendingBackgroundTask,
providerThreadId: OrchestrationV2ProviderThread["id"],
projection: {
readonly turnItems: ReadonlyArray<RosterItem>;
readonly settledRosterSubagents?: ReadonlyArray<RosterItem> | undefined;
},
): boolean {
if (task.kind !== "subagent" || task.parentTaskId !== undefined) return false;
const ownItem = (item: RosterItem) =>
item.type === "subagent" &&
item.providerThreadId === providerThreadId &&
item.nativeItemRef?.nativeId === task.taskId;
return (
(projection.settledRosterSubagents ?? []).some(
(item) => ownItem(item) && !isOrchestrationV2WorkActive(item.status),
) &&
!projection.turnItems.some((item) => ownItem(item) && isOrchestrationV2WorkActive(item.status))
);
}

const MAX_NOTE_ENTRIES = 10;

/** Why background work stopped: a T3 server restart, or the provider process exiting under a live server. */
Expand Down
Loading
Loading