Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -190,9 +190,17 @@ describe("orchestrator MCP toolkit", () => {
Effect.gen(function* () {
const cwd = yield* checkpointWorkspace("delegated-task-status");
const rawTranscript = yield* readDelegatedTaskStatusTranscript(result);
const transcript = yield* CodexOrchestratorReplayHarness.decodeTranscript(
materializeReplayTranscriptWorkspace(rawTranscript, cwd),
);
const workspaceTranscript = materializeReplayTranscriptWorkspace(rawTranscript, cwd);
// The recorder stopped Codex after its last frame, but this test keeps
// the parent turn running past the recording. Replaying that exit would
// end the session and fail the parent run.
const transcript = yield* CodexOrchestratorReplayHarness.decodeTranscript({
...workspaceTranscript,
entries:
workspaceTranscript.entries.at(-1)?.type === "runtime_exit"
? workspaceTranscript.entries.slice(0, -1)
: workspaceTranscript.entries,
});
const layerOrchestrator = ProviderReplayHarness.layerProviderReplay(
{
name: "delegated-task-status/codex",
Expand Down
311 changes: 290 additions & 21 deletions apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts

Large diffs are not rendered by default.

95 changes: 80 additions & 15 deletions apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,9 @@ function codexMcpToolOutput(

// A hung MCP resource read must not hold run ingestion open.
const MCP_APP_CAPTURE_TIMEOUT = "20 seconds";
// Reading the in-memory thread list is cheap; even an error reply proves liveness.
const CODEX_LIVENESS_PROBE_INTERVAL = "1 minute";
const CODEX_LIVENESS_PROBE_TIMEOUT = "1 minute";

function codexMcpAppResourceUri(item: CodexDynamicToolItem): string | undefined {
if (item.type !== "mcpToolCall") return undefined;
Expand Down Expand Up @@ -1746,7 +1749,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
model: input.modelSelection.model,
now,
});
const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
const events = yield* Queue.unbounded<
ProviderAdapter.ProviderAdapterV2Event,
ProviderAdapter.ProviderAdapterEventStreamError
>();
const rateLimitSnapshot = yield* Ref.make<CodexRateLimitSnapshot | undefined>(undefined);
const limitedTurnItems = yield* Ref.make(
new Map<ProviderThreadId, Extract<OrchestrationV2TurnItem, { type: "error" }>>(),
Expand Down Expand Up @@ -3835,6 +3841,31 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
},
);

// Shared by interrupt and lost app-server paths. Persistent dynamic tools
// end too, because the turn itself is ending.
const terminalizeTrackedTurnWork = Effect.fn("CodexAdapterV2.terminalizeTrackedTurnWork")(
function* (
context: ActiveCodexTurnContext,
status: "interrupted" | "failed",
completedAt: DateTime.Utc,
) {
yield* terminalizeRunningCommandItems(
context,
context.nativeTurnId,
status,
completedAt,
);
yield* terminalizeRunningDynamicTools(
context,
context.nativeTurnId,
status,
completedAt,
true,
);
yield* terminalizeMcpAppCaptures(context, context.nativeTurnId);
},
);

// The plan coalescer builds these artifacts on every streamed update.
const buildProposedPlanArtifacts = Effect.fnUntraced(function* (input: {
readonly context: ActiveCodexTurnContext;
Expand Down Expand Up @@ -6723,6 +6754,53 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
),
};

// Drain buffered events before failing every run sharing this process.
const unresponsive = Effect.gen(function* () {
while (true) {
yield* Effect.sleep(CODEX_LIVENESS_PROBE_INTERVAL);
const answered = yield* client
.request("thread/loaded/list", { limit: 1 })
.pipe(
Effect.catchTags({ CodexAppServerRequestError: () => Effect.void }),
Effect.timeoutOption(CODEX_LIVENESS_PROBE_TIMEOUT),
);
if (Option.isNone(answered)) {
return `Codex stopped responding: it did not answer a liveness check within ${CODEX_LIVENESS_PROBE_TIMEOUT}.`;
}
}
});
// No item/completed will arrive for work the process took with it,
// including background commands of turns that already completed.
const failLostTurnWork = turnTerminalizationPermit.withPermits(1)(
Effect.gen(function* () {
const completedAt = yield* DateTime.now;
for (const context of [
...(yield* Ref.get(activeTurns)).values(),
...(yield* Ref.get(settledTurns)).values(),
]) {
yield* terminalizeTrackedTurnWork(context, "failed", completedAt);
}
}),
);
yield* Effect.raceFirst(
client.terminated,
// A probe that fails with a transport error is the termination above.
unresponsive.pipe(Effect.catch(() => Effect.never)),
).pipe(
Effect.tap(() => failLostTurnWork),
Effect.flatMap((cause) =>
Queue.fail(
events,
new ProviderAdapter.ProviderAdapterEventStreamError({
driver: CODEX_PROVIDER,
providerSessionId: input.providerSessionId,
cause,
}),
),
),
Effect.forkIn(scope),
);

const runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime = {
instanceId: adapterOptions.instanceId,
driver: CODEX_PROVIDER,
Expand Down Expand Up @@ -7397,20 +7475,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
const context = target.context;
if ((yield* Ref.get(settledTurns)).get(context.nativeTurnId) !== context)
continue;
yield* terminalizeRunningCommandItems(
context,
context.nativeTurnId,
"interrupted",
completedAt,
);
yield* terminalizeRunningDynamicTools(
context,
context.nativeTurnId,
"interrupted",
completedAt,
true,
);
yield* terminalizeMcpAppCaptures(context, context.nativeTurnId);
yield* terminalizeTrackedTurnWork(context, "interrupted", completedAt);
yield* Ref.update(runningCommandItemsByTurn, (current) => {
const updated = new Map(current);
updated.delete(context.nativeTurnId);
Expand Down
66 changes: 63 additions & 3 deletions apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,8 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
readonly hasPendingBackgroundWork?: Effect.Effect<boolean>;
readonly hasPendingBackgroundWorkForThread?: Effect.Effect<boolean>;
readonly backgroundTasksEndWithProcess?: boolean;
readonly hangSessionScopeClose?: boolean;
/** "first" wedges only the first session opened. */
readonly hangSessionScopeClose?: boolean | "first";
readonly startTurn?: Effect.Effect<void>;
readonly beforeUnload?: Effect.Effect<void>;
readonly spawnBeforeOpen?: boolean;
Expand Down Expand Up @@ -392,7 +393,11 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
});
if (options.spawnBeforeOpen !== true) {
yield* countClose;
if (options.hangSessionScopeClose === true) yield* hangClose;
if (
options.hangSessionScopeClose === true ||
(options.hangSessionScopeClose === "first" && (yield* Ref.get(state)).openCount === 1)
)
yield* hangClose;
}

return {
Expand Down Expand Up @@ -478,7 +483,7 @@ function layerTest(input: {
readonly hasPendingBackgroundWork?: Effect.Effect<boolean>;
readonly hasPendingBackgroundWorkForThread?: Effect.Effect<boolean>;
readonly backgroundTasksEndWithProcess?: boolean;
readonly hangSessionScopeClose?: boolean;
readonly hangSessionScopeClose?: boolean | "first";
readonly startTurn?: Effect.Effect<void>;
readonly beforeUnload?: Effect.Effect<void>;
readonly spawnBeforeOpen?: boolean;
Expand Down Expand Up @@ -2747,6 +2752,61 @@ it.effect("ProviderSessionManagerV2 release retries leave a replacement session
}),
);

it.effect.each(["open", "closed"] as const)(
"ProviderSessionManagerV2 preserves a %s replacement's status after a late release",
(replacement) =>
Effect.gen(function* () {
const state = yield* Ref.make(emptyState);
const scopeCloseReached = yield* Deferred.make<void>();
yield* Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const threadId = ThreadId.make("thread-provider-session-manager-failed-replacement");
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId,
});
yield* eventSink.write({
events: [
yield* makeThreadCreatedEvent({ idAllocator, threadId, now: yield* DateTime.now }),
],
});
yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });

const release = yield* manager
.release({
providerSessionId,
reason: "runtime_error",
})
.pipe(Effect.forkChild);
yield* Deferred.await(scopeCloseReached);

yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
if (replacement === "closed") yield* manager.close(providerSessionId);
const expected = (yield* projectionStore.getThreadProjection(threadId)).providerSessions.at(
-1,
);
yield* TestClock.adjust("30 seconds");
yield* Fiber.join(release);
assert.deepEqual(
(yield* projectionStore.getThreadProjection(threadId)).providerSessions.at(-1),
expected,
);
}).pipe(
Effect.provide(
layerTest({
state,
idleTimeoutMs: 60_000,
hangSessionScopeClose: "first",
scopeCloseReached,
}),
),
);
}),
);

// regression: #15048 runs got stuck after a failed session release
it.effect("ProviderSessionManagerV2 keeps each failed release's cleanup", () =>
Effect.gen(function* () {
Expand Down
78 changes: 46 additions & 32 deletions apps/server/src/orchestration-v2/ProviderSessionManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,10 @@ export const layerWithOptions = (
}),
);
const sessions = yield* Ref.make(new Map<string, LiveSessionEntry>());
// The latest session opened under each id, kept until its release records it.
const latestOpened = yield* Ref.make(
new Map<string, ProviderAdapter.ProviderAdapterV2SessionRuntime>(),
);
// Retry per released entry; reusing ids must not lose cleanup for earlier threads.
const releaseRecordRetries = yield* FiberSet.make();
const nextSubscriberId = yield* Ref.make(0);
Expand Down Expand Up @@ -815,43 +819,46 @@ export const layerWithOptions = (
{ discard: true },
);

const writeReleaseRecords = (input: {
readonly entry: LiveSessionEntry;
readonly reason: ProviderSessionReleaseReason;
readonly detail?: string;
readonly releasedBackgroundTasks: ReadonlyArray<ReleasedBackgroundTasks>;
readonly releasedAt: DateTime.Utc;
readonly replaced: boolean;
}) =>
Effect.all(
[
input.replaced
? Effect.succeed(Exit.void)
: Effect.exit(writeReleasedSessionEvents(input)),
Effect.exit(
writeReleasedRuntimeRequestEvents(input).pipe(
input.entry.requestEventPermit.withPermits(1),
const writeReleaseRecords = Effect.fn("ProviderSessionManager.writeReleaseRecords")(
function* (input: {
readonly entry: LiveSessionEntry;
readonly reason: ProviderSessionReleaseReason;
readonly detail?: string;
readonly releasedBackgroundTasks: ReadonlyArray<ReleasedBackgroundTasks>;
readonly releasedAt: DateTime.Utc;
}) {
// The open lock protects status ownership, including after a replacement closes.
const key = sessionKey(input.entry.runtime.providerSessionId);
const ownsStatus = (yield* Ref.get(latestOpened)).get(key) === input.entry.runtime;
yield* Effect.all(
[
ownsStatus
? Effect.exit(writeReleasedSessionEvents(input))
: Effect.succeed(Exit.void),
Effect.exit(
writeReleasedRuntimeRequestEvents(input).pipe(
input.entry.requestEventPermit.withPermits(1),
),
),
),
Effect.exit(writeReleasedBackgroundWorkEvents(input.releasedBackgroundTasks)),
],
{ concurrency: 1 },
).pipe(Effect.flatMap(Exit.asVoidAll));
Effect.exit(writeReleasedBackgroundWorkEvents(input.releasedBackgroundTasks)),
],
{ concurrency: 1 },
).pipe(Effect.flatMap(Exit.asVoidAll));
if (!ownsStatus) return;
yield* Ref.update(latestOpened, (current) => {
const updated = new Map(current);
updated.delete(key);
return updated;
});
},
);

// Retry after removal from the live map; later releases cannot clear stale ready sessions and approvals.
const retryReleaseRecords = (
input: Omit<Parameters<typeof writeReleaseRecords>[0], "replaced">,
) => {
const retryReleaseRecords = (input: Parameters<typeof writeReleaseRecords>[0]) => {
const providerSessionId = input.entry.runtime.providerSessionId;
const attempt = Effect.gen(function* () {
const exit = yield* Effect.exit(
sessionOpen.withLock(
providerSessionId,
Effect.gen(function* () {
const replaced = (yield* Ref.get(sessions)).has(sessionKey(providerSessionId));
yield* writeReleaseRecords({ ...input, replaced });
}),
),
sessionOpen.withLock(providerSessionId, writeReleaseRecords(input)),
);
if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) return yield* exit;
yield* Effect.logWarning("orchestration-v2.provider-session-release-records-failed", {
Expand Down Expand Up @@ -971,6 +978,8 @@ export const layerWithOptions = (
readonly cancelIdleFiber?: boolean;
readonly onlyIfIdleGeneration?: number;
readonly gracefulSubscribers?: boolean;
/** Set by callers inside `open`, which already hold this session's open lock. */
readonly openLockHeld?: boolean;
}) =>
Effect.acquireUseRelease(
removeLiveEntry(input),
Expand Down Expand Up @@ -1011,7 +1020,9 @@ export const layerWithOptions = (
releasedAt,
};
const recorded = yield* Effect.exit(
writeReleaseRecords({ ...records, replaced: false }),
input.openLockHeld === true
? writeReleaseRecords(records)
: sessionOpen.withLock(input.providerSessionId, writeReleaseRecords(records)),
);
if (Exit.isFailure(recorded)) {
yield* retryReleaseRecords(records);
Expand Down Expand Up @@ -2191,6 +2202,7 @@ export const layerWithOptions = (
updated.set(key, entry);
return updated;
});
yield* Ref.update(latestOpened, (current) => new Map(current).set(key, runtime));
yield* dropReservation;
yield* withActivityError(
input.providerSessionId,
Expand All @@ -2209,11 +2221,13 @@ export const layerWithOptions = (
providerSessionId: input.providerSessionId,
reason: "manual_shutdown",
detail: "The provider session start was interrupted.",
openLockHeld: true,
}
: {
providerSessionId: input.providerSessionId,
reason: "runtime_error",
detail: "Failed to persist the provider-session attachment.",
openLockHeld: true,
},
).pipe(logReleaseFailure(input.providerSessionId)),
),
Expand Down
Loading
Loading