Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2164,6 +2164,62 @@ effectIt.effect(
}),
);

it("finds abandoned turns in a long history reading each record a bounded number of times", () => {
let reads = 0;
const counted = <T extends object>(record: T): T =>
new Proxy(record, {
get: (target, key, receiver) => {
reads += 1;
return Reflect.get(target, key, receiver);
},
});
const providerThreadId = ProviderThreadId.make("provider-thread-long-history");
const endings = ["failed", "interrupted", "cancelled"] as const;
const runs = [];
const attempts = [];
const providerTurns = [];
for (let index = 0; index < 2_000; index += 1) {
// Every tenth run ended without completing, after a steering restart.
const status = index % 10 === 0 ? endings[(index / 10) % endings.length]! : "completed";
const attemptIds = status === "completed" ? [`a${index}`] : [`a${index}-steered`, `a${index}`];
runs.push({
id: `r${index}`,
ordinal: index + 1,
status,
providerThreadId,
activeAttemptId: attemptIds.at(-1),
});
for (const attemptId of attemptIds) {
attempts.push({ id: attemptId, runId: `r${index}` });
const running = attemptId === "a10-steered" || attemptId === "a20" || attemptId === "a21";
providerTurns.push({
id: `t-${attemptId}`,
providerThreadId,
runAttemptId: attemptId,
status: running ? "running" : "completed",
});
}
}
const projection = {
runs: runs.map(counted),
attempts: attempts.map(counted),
providerTurns: providerTurns.map(counted),
} as unknown as Parameters<typeof ProviderTurnStart.abandonedProviderTurns>[0];

const abandoned = ProviderTurnStart.abandonedProviderTurns(
projection,
{ ordinal: 2_001 },
providerThreadId,
);

expect(abandoned.map(({ turn, run }) => [turn.id, run.id])).toEqual([
["t-a10-steered", "r10"],
["t-a20", "r20"],
]);
const records = runs.length + attempts.length + providerTurns.length;
expect(reads).toBeLessThanOrEqual(4 * records);
});

effectIt.effect("sends new steering text when restarting a manual continuation", () =>
Effect.gen(function* () {
const harness = makeLocalCommandHarness({
Expand Down
57 changes: 33 additions & 24 deletions apps/server/src/orchestration-v2/ProviderTurnStartService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,38 @@ const NON_COMPLETING_TERMINAL_RUN_STATUSES: ReadonlySet<OrchestrationV2Run["stat
"rolled_back",
]);

export function abandonedProviderTurns(
projection: Pick<OrchestrationV2ThreadProjection, "runs" | "attempts" | "providerTurns">,
run: Pick<OrchestrationV2Run, "ordinal">,
providerThreadId: OrchestrationV2ProviderThread["id"],
) {
const attemptsByRun = Map.groupBy(projection.attempts, (attempt) => attempt.runId);
const abandonedRuns = new Map<string | null, OrchestrationV2Run>();
for (const previous of projection.runs) {
if (
previous.ordinal >= run.ordinal ||
previous.providerThreadId !== providerThreadId ||
!NON_COMPLETING_TERMINAL_RUN_STATUSES.has(previous.status)
) {
continue;
}
if (previous.activeAttemptId !== null) {
abandonedRuns.set(previous.activeAttemptId, previous);
}
for (const attempt of attemptsByRun.get(previous.id) ?? []) {
abandonedRuns.set(attempt.id, previous);
}
}
return projection.providerTurns.flatMap((turn) => {
const previous = abandonedRuns.get(turn.runAttemptId);
return turn.providerThreadId === providerThreadId &&
turn.status === "running" &&
previous !== undefined
? [{ turn, run: previous }]
: [];
});
}

export const isProviderTurnStartError = Schema.is(ProviderTurnStartError);
const isProviderAdapterProtocolError = Schema.is(ProviderAdapter.ProviderAdapterProtocolError);
const isProviderAdapterOpenSessionError = Schema.is(
Expand Down Expand Up @@ -799,30 +831,7 @@ export const layer: Layer.Layer<
// running when the agent ignored Stop or an interrupt timed out. Every attempt
// of such a run owns a possible abandoned turn, including attempts a steering
// restart superseded, and the session cannot be reused until those turns stop.
const abandonedRuns = new Map<string | null, OrchestrationV2Run>();
for (const previous of projection.runs) {
if (
previous.ordinal >= run.ordinal ||
previous.providerThreadId !== providerThread.id ||
!NON_COMPLETING_TERMINAL_RUN_STATUSES.has(previous.status)
) {
continue;
}
if (previous.activeAttemptId !== null) {
abandonedRuns.set(previous.activeAttemptId, previous);
}
for (const attempt of projection.attempts) {
if (attempt.runId === previous.id) abandonedRuns.set(attempt.id, previous);
}
}
const abandonedTurns = projection.providerTurns.flatMap((turn) => {
const previous = abandonedRuns.get(turn.runAttemptId);
return turn.providerThreadId === providerThread.id &&
turn.status === "running" &&
previous !== undefined
? [{ turn, run: previous }]
: [];
});
const abandonedTurns = abandonedProviderTurns(projection, run, providerThread.id);
const abandonedTurn = abandonedTurns.at(-1)?.turn;
if (abandonedTurn !== undefined) {
const liveSession = yield* providerSessions.get(providerSessionId);
Expand Down
Loading