diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 863f2279ef2a3..d18fa09f10301 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -1334,6 +1334,10 @@ describe("AcpAdapterV2", () => { ); it.live("fails an end_turn the outcome check rejects and restarts before the next turn", () => Effect.gen(function* () { + const checking = yield* Deferred.make(); + const releaseCheck = yield* Deferred.make(); + const cancelled = yield* Deferred.make(); + const idAllocator = yield* IdAllocator.IdAllocatorV2; const instanceId = ProviderInstanceId.make("acp-test-prompt-outcome"); const threadId = ThreadId.make("thread-acp-prompt-outcome"); const quota = { @@ -1343,6 +1347,12 @@ describe("AcpAdapterV2", () => { retryable: true, resetAt: "2030-01-01T00:00:00.000Z", } as const; + const poisoned = { + class: "provider_error", + message: "Saved history is unusable.", + code: "INVALID_THINKING_SIGNATURE", + retryable: true, + } as const; // The runtime ordinal of each prompt, and the session each check watched. const promptRuntimes: Array = []; const promptTexts: Array = []; @@ -1356,14 +1366,27 @@ describe("AcpAdapterV2", () => { watchPromptOutcome: (sessionId) => Effect.sync(() => { watchedSessions.push(sessionId); - const failed = watchedSessions.length === 1; - return () => Effect.succeed(failed ? quota : undefined); + const outcome = + watchedSessions.length === 1 + ? { failure: quota, threadDisposition: "reusable" as const } + : watchedSessions.length >= 3 + ? { failure: poisoned, threadDisposition: "broken" as const } + : undefined; + const interrupted = watchedSessions.length === 4; + return () => + interrupted + ? Deferred.succeed(checking, undefined).pipe( + Effect.andThen(Deferred.await(releaseCheck)), + Effect.as(outcome), + ) + : Effect.succeed(outcome); }), makeRuntime: makeMockRuntime({ childProcessSpawner: yield* ChildProcessSpawner.ChildProcessSpawner, mockAgentPath: yield* bundledMockAgentPath, wrapRuntime: (runtime, runtimeOrdinal) => ({ ...runtime, + cancel: Deferred.succeed(cancelled, undefined).pipe(Effect.asVoid), prompt: (payload) => Effect.sync(() => { promptRuntimes.push(runtimeOrdinal); @@ -1407,19 +1430,20 @@ describe("AcpAdapterV2", () => { ordinal, }), ); - const events = yield* runtime.events.pipe( - Stream.takeUntil((event) => event.type === "turn.terminal"), - Stream.runCollect, + return Array.from( + yield* runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + ), ); - return events.at(-1); }); - const failed = yield* runTurn(1); + const failed = (yield* runTurn(1)).at(-1); if (failed?.type !== "turn.terminal" || failed.status !== "failed") { return assert.fail("expected a failed terminal"); } assert.deepEqual(failed.failure, quota); - const completed = yield* runTurn(2); + const completed = (yield* runTurn(2)).at(-1); assert.equal(completed?.type === "turn.terminal" ? completed.status : null, "completed"); assert.deepEqual(promptRuntimes, [1, 2], "the next turn runs on a fresh runtime"); // The fresh runtime has no record of what reached the agent, so it sends T3's notes again. @@ -1427,6 +1451,61 @@ describe("AcpAdapterV2", () => { assert.include(promptTexts[1] ?? "", "T3 Code interaction mode: Default"); assert.equal(watchedSessions.length, 2); assert.equal(watchedSessions[0], watchedSessions[1]); + + // Poisoned native history marks the thread broken; its persisted error row is never resumed. + const poisonedEvents = yield* runTurn(3); + const broken = poisonedEvents.at(-1); + if (broken?.type !== "turn.terminal" || broken.status !== "failed") { + return assert.fail("expected a failed terminal"); + } + assert.deepEqual(broken.failure, poisoned); + assert.equal(broken.threadDisposition, "broken"); + const brokenThread = poisonedEvents.findLast( + (event) => event.type === "provider_thread.updated", + ); + assert.equal( + brokenThread?.type === "provider_thread.updated" + ? brokenThread.providerThread.status + : null, + "error", + ); + const resumed = yield* Effect.result( + runtime.resumeThread({ + threadId, + providerThread: { ...providerThread, status: "error" }, + modelSelection, + runtimePolicy, + }), + ); + assert.equal( + resumed._tag === "Failure" ? resumed.failure._tag : null, + "ProviderAdapterResumeThreadError", + ); + + const turn = yield* Effect.forkChild(runTurn(4)); + yield* Deferred.await(checking); + const stop = yield* Effect.forkChild( + runtime.interruptTurn({ + providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: acpScopedNativeId(instanceId, `${watchedSessions[3]}:turn:4`), + }), + }), + ); + yield* Deferred.await(cancelled); + yield* Deferred.succeed(releaseCheck, undefined); + const interruptedEvents = yield* Fiber.join(turn); + yield* Fiber.join(stop); + const interrupted = interruptedEvents.at(-1); + assert.equal( + interrupted?.type === "turn.terminal" ? interrupted.status : null, + "interrupted", + ); + assert.equal( + interrupted?.type === "turn.terminal" ? interrupted.threadDisposition : null, + "broken", + ); }).pipe(Effect.provide(layerTest)), ); it.live("retains returned usage when Stop finalizes before the prompt callback", () => diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index f023fe5aa8697..aa03399bc760e 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -2705,6 +2705,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const deliveryProviderThread = canResumeAcrossInstances ? { ...queuedProviderThread, + status: activeProviderThread.status, nativeThreadRef: activeProviderThread.nativeThreadRef, nativeConversationHeadRef: activeProviderThread.nativeConversationHeadRef, nativeMetadata: activeProviderThread.nativeMetadata, @@ -2896,7 +2897,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const providerThread: OrchestrationV2ProviderThread = { ...deliveryProviderThread, providerSessionId, - status: "not_loaded", + status: deliveryProviderThread.status === "error" ? "error" : "not_loaded", firstRunOrdinal: queuedProviderThread.firstRunOrdinal ?? ordinal, lastRunOrdinal: ordinal, handoffIds: appendContextHandoffId( @@ -8673,7 +8674,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : null; const providerThread: OrchestrationV2ProviderThread = { ...ensuredProviderThread, - status: "active", + status: ensuredProviderThread.status === "error" ? "error" : "active", firstRunOrdinal: ensuredProviderThread.firstRunOrdinal ?? ordinal, lastRunOrdinal: ordinal, handoffIds: [ diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 95595c3a0a2ae..bebc96e8b287b 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -2916,11 +2916,20 @@ effectIt.effect("does not mistake a failed state read for a superseded run", () }), ); -effectIt.effect( - "signs out the existing native provider before opening the newly selected provider", - () => +effectIt.effect.each(["not_loaded", "error"] as const)( + "signs out the existing native provider before opening the newly selected provider with status %s", + (providerThreadStatus) => Effect.gen(function* () { const harness = makeLocalCommandHarness({ text: "/logout", previousNativeSession: true }); + const initial = harness.projection(); + harness.setProjection({ + ...initial, + providerThreads: initial.providerThreads.map((thread) => + thread.id === "new-provider-thread" + ? { ...thread, status: providerThreadStatus } + : thread, + ), + }); yield* harness.start; yield* harness.start; @@ -2946,6 +2955,9 @@ effectIt.effect( ]); expect(projection.providerTurns).toEqual([]); expect(projection.checkpoints).toEqual([]); + expect( + projection.providerThreads.find((thread) => thread.id === "new-provider-thread")?.status, + ).toBe(providerThreadStatus); }), ); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 7764bca5cca0c..3bb8e2385eaa2 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -686,7 +686,12 @@ export const layer: Layer.Layer< }, providerThreadUpdate: { ...providerThread, - status: providerThread.nativeThreadRef === null ? "not_loaded" : "idle", + status: + providerThread.status === "error" + ? "error" + : providerThread.nativeThreadRef === null + ? "not_loaded" + : "idle", updatedAt: now, }, }); @@ -842,6 +847,12 @@ export const layer: Layer.Layer< run, checkpointScope, }); + const brokenNativeThread = providerThread.status === "error"; + // Opened without the broken native id, a session that was not live starts fresh. + const opensFreshSession = + brokenNativeThread && + providerThread.nativeThreadRef !== null && + !(yield* providerSessions.isLive(providerSessionId)); const sessionResult = yield* Effect.result( providerSessions.open({ threadId: projection.thread.id, @@ -851,7 +862,7 @@ export const layer: Layer.Layer< ...(existingSessionProjection === undefined ? {} : { resumeFromSession: existingSessionProjection }), - ...(providerThread.nativeThreadRef?.nativeId == null + ...(providerThread.nativeThreadRef?.nativeId == null || brokenNativeThread ? {} : { initialNativeThreadId: providerThread.nativeThreadRef.nativeId }), ...(providerThread.nativeMetadata?.itemIdentityVersion === undefined @@ -1135,6 +1146,7 @@ export const layer: Layer.Layer< const createdAt = yield* DateTime.now; const history = yield* projectionStore.getTurnStartHistory(input.threadId); const boundary = contextBoundaryOrdinal({ runs: projection.runs, turnItems: history }); + const from = Math.max(1, boundary); const handoff = yield* contextHandoffService.prepareProviderHandoff({ threadId: projection.thread.id, targetRunId: run.id, @@ -1143,7 +1155,7 @@ export const layer: Layer.Layer< toProviderThreadId: providerThread.id, fromProviderInstanceId: providerThread.providerInstanceId, toProviderInstanceId: run.providerInstanceId, - coveredRunOrdinals: { from: 1, to: Math.max(1, run.ordinal - 1) }, + coveredRunOrdinals: { from, to: Math.max(from, run.ordinal - 1) }, strategy: "full_thread_summary", runs: projection.runs, items: [ @@ -1501,7 +1513,10 @@ export const layer: Layer.Layer< providerSessionId, // Clear the native ref to force a fresh session while preserving row identity. existingProviderThread: { ...providerThread, nativeThreadRef: null }, - forceNewNativeThread: true, + // A fresh session that never loaded the broken history needs no replacement. + ...(opensFreshSession && resumed._tag === "Failure" + ? {} + : { forceNewNativeThread: true }), }), ); if (replacement === undefined) return undefined; diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index 289e99b414990..ea092f6580381 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -58,6 +58,7 @@ import * as EffectOutbox from "../EffectOutbox.ts"; import * as ProjectionReplay from "./ProjectionReplay.ts"; import * as ProjectionStore from "../ProjectionStore.ts"; import * as ProviderAdapterRegistry from "../ProviderAdapterRegistry.ts"; +import * as ProviderSessionManager from "../ProviderSessionManager.ts"; import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; import { CLAUDE_MODEL_SELECTION, @@ -116,6 +117,7 @@ function makeTestAdapter(input: { >; readonly failure?: ReturnType; readonly failedRunOrdinals?: ReadonlySet; + readonly brokenRunOrdinals?: ReadonlySet; readonly interruptedRunOrdinals?: ReadonlySet; readonly cancelledRunOrdinals?: ReadonlySet; readonly holdRunOrdinal?: number; @@ -131,6 +133,8 @@ function makeTestAdapter(input: { readonly backgroundTasksEndWithProcess?: true; readonly backgroundWork?: Ref.Ref; readonly closedSessions?: Ref.Ref>; + /** Like ACP: opening resumes the initial native id, and forcing a new thread restarts the runtime. */ + readonly runtimeStarts?: Ref.Ref>; // ACP cannot truncate, so its rollback answers with a fresh, empty native session. readonly rollback?: "truncates" | "replaces_native_thread"; }): ProviderAdapter.ProviderAdapterV2["Service"] { @@ -142,6 +146,12 @@ function makeTestAdapter(input: { openSession: (sessionInput) => Effect.gen(function* () { const events = yield* PubSub.unbounded(); + if (input.runtimeStarts !== undefined) { + yield* Ref.update(input.runtimeStarts, (starts) => [ + ...starts, + sessionInput.initialNativeThreadId ?? "new", + ]); + } if (input.closedSessions !== undefined) { const closedSessions = input.closedSessions; yield* Effect.addFinalizer(() => @@ -195,6 +205,9 @@ function makeTestAdapter(input: { }), ensureThread: (threadInput) => Effect.gen(function* () { + if (threadInput.forceNewNativeThread && input.runtimeStarts !== undefined) { + yield* Ref.update(input.runtimeStarts, (starts) => [...starts, "replacement"]); + } const createdAt = yield* DateTime.now; const generation = input.nativeThreadGeneration === undefined @@ -227,6 +240,7 @@ function makeTestAdapter(input: { resumeThread: ({ providerThread }) => Effect.gen(function* () { if ( + (input.brokenRunOrdinals !== undefined && providerThread.status === "error") || input.failResume || (input.failResumeOnce !== undefined && (yield* Ref.getAndSet(input.failResumeOnce, false))) @@ -425,7 +439,9 @@ function makeTestAdapter(input: { }), } : { status: terminalStatus, failure: null }), - threadDisposition: "reusable", + threadDisposition: input.brokenRunOrdinals?.has(turnInput.runOrdinal) + ? "broken" + : "reusable", }, ]; const reportedUsage = input.tokenUsageByRunOrdinal?.[turnInput.runOrdinal]; @@ -441,6 +457,17 @@ function makeTestAdapter(input: { }, }); } + if (input.brokenRunOrdinals?.has(turnInput.runOrdinal)) { + yield* PubSub.publish(events, { + type: "provider_thread.updated", + driver: input.driver, + providerThread: { + ...turnInput.providerThread, + status: "error", + updatedAt: eventTime, + }, + }); + } for (const event of providerEvents) { yield* PubSub.publish(events, event); } @@ -2059,7 +2086,8 @@ describe("orchestration v2 provider switching", () => { it.live.each( ( [ - ["failed", false, false, false], + ["failed", false, false, false, false], + ["failed", false, false, false, true], ["failed", true, true, true], ["interrupted", false, true, false], ["interrupted", true, false, true], @@ -2067,13 +2095,13 @@ describe("orchestration v2 provider switching", () => { ["cancelled", true, true, false], ] as const ).map( - ([status, queued, returning, native]) => + ([status, queued, returning, native, broken]) => [ - `${status} context ${queued ? "through the queue" : "immediately"} to ${returning ? "a returning" : "a new"} provider via ${native ? "native history" : "text"}`, - { status, queued, returning, native }, + `${status}${broken ? " broken" : ""} context ${queued ? "through the queue" : "immediately"} to ${returning ? "a returning" : "a new"} provider via ${native ? "native history" : "text"}`, + { status, queued, returning, native, broken }, ] as const, ), - )("hands off %s", ([, { status, queued, returning, native }]) => + )("hands off %s", ([, { status, queued, returning, native, broken }]) => Effect.gen(function* () { const cwd = yield* checkpointWorkspace(`handoff-${status}-${queued}-${returning}`); const capturedTurns = yield* Ref.make>([]); @@ -2083,6 +2111,7 @@ describe("orchestration v2 provider switching", () => { const sourceOrdinal = returning ? 2 : 1; const sourceSelection = returning ? CLAUDE_MODEL_SELECTION : CODEX_MODEL_SELECTION; const targetSelection = returning ? CODEX_MODEL_SELECTION : CLAUDE_MODEL_SELECTION; + const generation = yield* Ref.make(0); const originalPrompt = "Keep the release marker violet and preserve the existing API."; const partialResponse = "I checked the API and found the release configuration."; const layerRegistry = ProviderAdapterRegistry.layerFromAdapters( @@ -2105,6 +2134,12 @@ describe("orchestration v2 provider switching", () => { ...(modelSelection.instanceId === sourceSelection.instanceId ? { failedRunOrdinals: new Set(status === "failed" ? [sourceOrdinal] : []), + ...(broken + ? { + brokenRunOrdinals: new Set([sourceOrdinal]), + nativeThreadGeneration: generation, + } + : {}), interruptedRunOrdinals: new Set(status === "interrupted" ? [sourceOrdinal] : []), cancelledRunOrdinals: new Set(status === "cancelled" ? [sourceOrdinal] : []), holdRunOrdinal: sourceOrdinal, @@ -2218,8 +2253,17 @@ describe("orchestration v2 provider switching", () => { yield* dispatch("back", "Finish the remaining work", sourceSelection); yield* waitForRun(sourceOrdinal + 2, "completed"); const back = (yield* Ref.get(capturedTurns)).at(-1)!; - assert.notInclude(back.text, originalPrompt); - assert.notInclude(back.text, partialResponse); + if (broken) { + const source = (yield* Ref.get(capturedTurns)).find( + (turn) => turn.modelSelection.instanceId === sourceSelection.instanceId, + )!; + assert.notEqual(back.nativeThreadId, source.nativeThreadId); + assert.include(back.text, originalPrompt); + assert.include(back.text, partialResponse); + } else { + assert.notInclude(back.text, originalPrompt); + assert.notInclude(back.text, partialResponse); + } }).pipe( Effect.provide( ProviderReplayHarness.layerWithRegistry( @@ -3605,15 +3649,23 @@ describe("orchestration v2 provider switching", () => { }), ); - it.live.each(["immediately", "behind a running turn"] as const)( + it.live.each([ + "immediately", + "behind a running turn", + "and recovers it after it breaks", + "and recovers it after it breaks and the server restarts", + ] as const)( "starts a fresh provider session %s and never hands earlier runs to a later provider", (variant) => Effect.gen(function* () { - const queued = variant !== "immediately"; - const key = queued ? "queued" : "immediate"; + const queued = variant === "behind a running turn"; + const restarted = variant === "and recovers it after it breaks and the server restarts"; + const broken = restarted || variant === "and recovers it after it breaks"; + const key = queued ? "queued" : restarted ? "restarted" : broken ? "broken" : "immediate"; const cwd = yield* checkpointWorkspace(`fresh-context-${key}`); const capturedTurns = yield* Ref.make>([]); const closedSessions = yield* Ref.make>([]); + const runtimeStarts = yield* Ref.make>([]); // A queued start has nobody to refuse, so a busy session must stay up. const backgroundWork = yield* Ref.make(queued); const started = yield* Deferred.make(); @@ -3627,11 +3679,18 @@ describe("orchestration v2 provider switching", () => { responseByRunOrdinal: {}, capturedTurns, closedSessions, + runtimeStarts, backgroundWork, nativeThreadGeneration: yield* Ref.make(0), ...(queued ? { holdFirstTurn: started, releaseFirstTurn: release, holdRunOrdinal: 2 } : {}), + ...(broken + ? { + failedRunOrdinals: new Set([3]), + brokenRunOrdinals: new Set([3]), + } + : {}), }), makeTestAdapter({ instanceId: CODEX_MODEL_SELECTION.instanceId, @@ -3725,6 +3784,22 @@ describe("orchestration v2 provider switching", () => { // Claude hosts one native thread per session, so an idle old one is closed. assert.deepEqual(yield* Ref.get(closedSessions), queued ? [] : [firstSession.id]); yield* Ref.set(backgroundWork, false); + let startsBeforeRecovery = 0; + if (broken) { + if (restarted) { + const brokenProjection = yield* orchestrator.getThreadProjection(freshThreadId); + const brokenThread = brokenProjection.providerThreads.find( + (thread) => thread.id === brokenProjection.thread.activeProviderThreadId, + )!; + assert.equal(brokenThread.status, "error"); + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + yield* sessions.close(brokenThread.providerSessionId!); + } + startsBeforeRecovery = (yield* Ref.get(runtimeStarts)).length; + yield* send("epsilon", CLAUDE_MODEL_SELECTION); + yield* worker.drain(); + yield* waitForIdle(freshThreadId); + } yield* send("delta", CODEX_MODEL_SELECTION); yield* worker.drain(); yield* waitForIdle(freshThreadId); @@ -3753,6 +3828,27 @@ describe("orchestration v2 provider switching", () => { if (queued) { assert.equal(turnFor("gamma").providerThreadId, turnFor("beta").providerThreadId); } + if (broken) { + const recovered = turnFor("epsilon"); + assert.equal(recovered.providerThreadId, turnFor("beta").providerThreadId); + assert.notEqual(recovered.nativeThreadId, turnFor("beta").nativeThreadId); + assert.include(recovered.text, "Prompt beta"); + assert.notInclude(recovered.text, "Prompt alpha"); + assert.notInclude(recovered.text, "Prompt zero"); + // The broken native id is never reopened; a reopened session starts fresh only once. + assert.deepEqual( + (yield* Ref.get(runtimeStarts)).slice(startsBeforeRecovery), + restarted ? ["new"] : ["replacement"], + ); + const recoveredRun = projection.runs.find( + (run) => run.userMessageId === `message:fresh-context:${key}:epsilon`, + )!; + assert.equal( + projection.contextHandoffs.find((handoff) => handoff.targetRunId === recoveredRun.id)! + .coveredRunOrdinals.from, + 3, + ); + } const codexTurn = turnFor("delta"); assert.notEqual(codexTurn.providerThreadId, turnFor("zero").providerThreadId); assert.include(codexTurn.text, "Prompt beta"); @@ -3886,6 +3982,8 @@ describe("orchestration v2 provider switching", () => { responseByRunOrdinal: {}, capturedTurns, nativeThreadGeneration: yield* Ref.make(0), + failedRunOrdinals: new Set([5]), + brokenRunOrdinals: new Set([5]), }), ]); const threadId = ThreadId.make("thread:scheduled-fresh-user-chat"); @@ -3933,6 +4031,7 @@ describe("orchestration v2 provider switching", () => { ["compaction", false], ["scheduled-after-compaction", true], ["user-third", false], + ["user-fourth", false], ] as const) { if (name === "compaction") { // The user's run compacts, so later dispatches no longer read the first scheduled run's message. @@ -3983,6 +4082,21 @@ describe("orchestration v2 provider switching", () => { // The scheduled run's fresh session must not replace the user's conversation. assert.equal(turnFor("user-second")!.nativeThreadId, turnFor("user-first")!.nativeThreadId); assert.equal(turnFor("user-third")!.nativeThreadId, turnFor("user-first")!.nativeThreadId); + // The user's session broke; recovery reads the whole thread, so scheduled runs do not cut it. + const recovered = turnFor("user-fourth")!; + assert.equal(recovered.providerThreadId, turnFor("user-first")!.providerThreadId); + assert.notEqual(recovered.nativeThreadId, turnFor("user-first")!.nativeThreadId); + assert.include(recovered.text, "user-first"); + assert.include(recovered.text, "user-third"); + const projection = yield* orchestrator.getThreadProjection(threadId); + const recoveredRun = projection.runs.find( + (run) => run.userMessageId === "message:scheduled-fresh-user-chat:user-fourth", + )!; + assert.equal( + projection.contextHandoffs.find((handoff) => handoff.targetRunId === recoveredRun.id)! + .coveredRunOrdinals.from, + 1, + ); }).pipe( Effect.provide( ProviderReplayHarness.layerWithRegistry( @@ -4119,6 +4233,7 @@ describe("orchestration v2 provider switching", () => { it.live.each([ ["automatic", "Claude"], ["queue.resume", "compatible Codex"], + ["queue.resume", "broken compatible Codex"], ] as const)( "delivers a queued PR update on the current provider through %s (%s)", ([delivery, target]) => @@ -4127,7 +4242,9 @@ describe("orchestration v2 provider switching", () => { const capturedTurns = yield* Ref.make>([]); const started = yield* Deferred.make(); const release = yield* Deferred.make(); - const compatible = target === "compatible Codex"; + const compatible = target !== "Claude"; + const broken = target === "broken compatible Codex"; + const generation = yield* Ref.make(0); const selection = { ...(compatible ? { ...CODEX_MODEL_SELECTION, instanceId: ProviderInstanceId.make("codex-alternate") } @@ -4139,7 +4256,7 @@ describe("orchestration v2 provider switching", () => { ...CodexProviderCapabilitiesV2, context: { ...CodexProviderCapabilitiesV2.context, - canConsumeHandoffSummaries: false, + canConsumeHandoffSummaries: broken, }, } : ClaudeProviderCapabilitiesV2; @@ -4154,6 +4271,9 @@ describe("orchestration v2 provider switching", () => { holdFirstTurn: started, releaseFirstTurn: release, failedRunOrdinals: new Set([1]), + ...(broken + ? { brokenRunOrdinals: new Set([1]), nativeThreadGeneration: generation } + : {}), failure: makeProviderFailure({ message: "Usage limit reached", class: "usage_limit", @@ -4167,6 +4287,9 @@ describe("orchestration v2 provider switching", () => { modelSelection: selection, responseByRunOrdinal: {}, capturedTurns, + ...(broken + ? { brokenRunOrdinals: new Set(), nativeThreadGeneration: generation } + : {}), }), ]; const layerRegistry = compatible @@ -4304,7 +4427,12 @@ describe("orchestration v2 provider switching", () => { assert.equal(executed.providerThreadId, turns[1]!.providerThreadId); assert.equal(executed.nativeThreadId, turns[1]!.nativeThreadId); } - if (compatible) { + if (broken) { + assert.notEqual(executed.nativeThreadId, turns[0]!.nativeThreadId); + assert.isFalse(executed.nativeThreadHasTurns); + assert.include(executed.text, "Original work"); + } + if (compatible && !broken) { assert.equal(executed.nativeThreadId, turns[0]!.nativeThreadId); assert.isEmpty(after.contextHandoffs); } diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts index 52e96caa08aea..eb0695948b30c 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts @@ -6,6 +6,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import type * as EffectAcpSchema from "effect-acp/compat"; +import type { OrchestrationV2ProviderFailure } from "@t3tools/contracts"; import { watchAntigravityPromptOutcome } from "./AntigravityTurnOutcome.ts"; @@ -32,10 +33,19 @@ const RATE_LIMIT_ERROR = errorPayload( const INVALID_ARGUMENT_ERROR = errorPayload( "Error 400, Message: Request contains an invalid argument., Status: INVALID_ARGUMENT, Details: []", ); +// Vertex Claude rejects a resumed history whose saved thinking block lost its signature. +const MISSING_SIGNATURE_ERROR = errorPayload( + 'Error 400, Message: {"type":"error","error":{"type":"invalid_request_error","message":"messages.1.content.0.thinking.signature: Field required"},"request_id":"req_vrtx_test"}, Status: INVALID_ARGUMENT, Details: []', +); +const INVALID_SIGNATURE_ERROR = errorPayload( + 'Error 400, Message: {"type":"error","error":{"type":"invalid_request_error","message":"messages.3.content.0: Invalid `signature` in `thinking` block"}}, Status: INVALID_ARGUMENT, Details: []', +); // The model's reply quoting the agent's quota notice is an ordinary planner response. const QUOTED_NOTICE = Buffer.from( "Usage Limit Reached\n\nYou have reached your current quota for this period. Your limit will reset in 1 hour.", ); +const reusable = (failure: OrchestrationV2ProviderFailure) => + ({ failure, threadDisposition: "reusable" }) as const; describe("watchAntigravityPromptOutcome", () => { it.effect("fails only turns whose own steps end in an upstream error", () => @@ -83,13 +93,13 @@ describe("watchAntigravityPromptOutcome", () => { [USER_INPUT, input], [ERROR_MESSAGE, QUOTA_ERROR], ]), - { + reusable({ class: "usage_limit", message: "You have reached your Antigravity quota for this period.", code: "RESOURCE_EXHAUSTED", retryable: true, resetAt: "1970-01-01T01:00:00.000Z", - }, + }), ); // The earlier error step is outside this turn. assert.isUndefined( @@ -109,26 +119,35 @@ describe("watchAntigravityPromptOutcome", () => { [USER_INPUT, input], [ERROR_MESSAGE, RATE_LIMIT_ERROR], ]), - { + reusable({ class: "usage_limit", message: "Resource has been exhausted (e.g. check quota).", code: "RESOURCE_EXHAUSTED", retryable: true, resetAt: "1970-01-01T00:00:42.000Z", - }, + }), ); assert.deepEqual( yield* turn([ [USER_INPUT, input], [ERROR_MESSAGE, INVALID_ARGUMENT_ERROR], ]), - { + reusable({ class: "provider_error", message: "Request contains an invalid argument.", code: "INVALID_ARGUMENT", retryable: false, - }, + }), ); + // A saved thinking block without a valid signature fails every later prompt. + for (const error of [MISSING_SIGNATURE_ERROR, INVALID_SIGNATURE_ERROR]) { + const outcome = yield* turn([ + [USER_INPUT, input], + [ERROR_MESSAGE, error], + ]); + assert.equal(outcome?.failure.code, "INVALID_THINKING_SIGNATURE"); + assert.equal(outcome?.threadDisposition, "broken"); + } // A completed tool keeps its usage, and the turn still fails. assert.equal( (yield* turn( @@ -142,7 +161,7 @@ describe("watchAntigravityPromptOutcome", () => { stopReason: "end_turn", usage: { inputTokens: 40, outputTokens: 5, totalTokens: 45 }, }, - ))?.class, + ))?.failure.class, "usage_limit", ); assert.isUndefined( @@ -152,12 +171,15 @@ describe("watchAntigravityPromptOutcome", () => { ]), ); // The agent maps the SDK's QUOTA_EXHAUSTED stop reason to `refusal`. - assert.deepEqual(yield* turn([], { stopReason: "refusal" }), { - class: "usage_limit", - message: "You have reached your Antigravity quota for this period.", - code: "QUOTA_EXHAUSTED", - retryable: true, - }); + assert.deepEqual( + yield* turn([], { stopReason: "refusal" }), + reusable({ + class: "usage_limit", + message: "You have reached your Antigravity quota for this period.", + code: "QUOTA_EXHAUSTED", + retryable: true, + }), + ); }).pipe(Effect.provide(NodeServices.layer)), ); @@ -193,19 +215,22 @@ describe("watchAntigravityPromptOutcome", () => { const linked = yield* watchAntigravityPromptOutcome(profileDirectory, own); append(peer, ERROR_MESSAGE, QUOTA_ERROR); assert.isUndefined(yield* linked({ stopReason: "end_turn" })); - assert.equal((yield* linked({ stopReason: "refusal" }))?.class, "usage_limit"); + assert.equal((yield* linked({ stopReason: "refusal" }))?.failure.class, "usage_limit"); // An unreadable log gives no evidence, but the agent's typed refusal still fails the turn. yield* fs.remove(path.join(conversations, `${own}.db`)); yield* fs.writeFileString(path.join(conversations, `${own}.db`), "not a database"); const corrupt = yield* watchAntigravityPromptOutcome(profileDirectory, own); assert.isUndefined(yield* corrupt({ stopReason: "end_turn" })); - assert.deepEqual(yield* corrupt({ stopReason: "refusal" }), { - class: "usage_limit", - message: "You have reached your Antigravity quota for this period.", - code: "QUOTA_EXHAUSTED", - retryable: true, - }); + assert.deepEqual( + yield* corrupt({ stopReason: "refusal" }), + reusable({ + class: "usage_limit", + message: "You have reached your Antigravity quota for this period.", + code: "QUOTA_EXHAUSTED", + retryable: true, + }), + ); }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts index 41e25f2e417e1..2cbe4d62b3a4b 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts @@ -20,6 +20,10 @@ const MAX_ERROR_PAYLOAD_BYTES = 16_384; // The Go harness's rendering of a Google API error, as the agent's api_errors.py reads it. const API_ERROR = /Error (\d{3}), Message: (.+?), Status: ([A-Z_]+)/s; const QUOTA_EXHAUSTED = /\breason:QUOTA_EXHAUSTED\b/; +// Claude rejects a saved thinking block without a valid signature. It sits in the native +// history, so every later prompt to that session fails the same way. +const INVALID_THINKING_SIGNATURE = + /\bmessages\.\d+\.content\.\d+(?:\.thinking\.signature: Field required|: Invalid `signature` in `thinking` block)/; const RESET_TIMESTAMP = /\bquotaResetTimeStamp:([^\s\]]+)/; const RESET_DELAY = /\bquotaResetDelay:((?:\d+(?:\.\d+)?(?:ns|us|µs|μs|ms|h|m|s))+)(?=[\s\]])/; // google.rpc.RetryInfo, which plain rate limits carry instead of a quota reset. @@ -106,7 +110,12 @@ function turnFailure( stopReason: EffectAcpSchema.StopReason, step: LastStep | undefined, now: DateTime.Utc, -): OrchestrationV2ProviderFailure | undefined { +): + | { + readonly failure: OrchestrationV2ProviderFailure; + readonly threadDisposition: "reusable" | "broken"; + } + | undefined { const errorText = step?.type === ERROR_STEP_TYPE ? new TextDecoder().decode(step.payload ?? new Uint8Array()) @@ -115,30 +124,50 @@ function turnFailure( if (errorText === undefined && stopReason !== "refusal") return undefined; const text = errorText ?? ""; const apiError = API_ERROR.exec(text); + const reusable = (failure: OrchestrationV2ProviderFailure) => + ({ failure, threadDisposition: "reusable" }) as const; if (stopReason === "refusal" || QUOTA_EXHAUSTED.test(text)) { - return makeProviderFailure({ - class: "usage_limit", - message: "You have reached your Antigravity quota for this period.", - code: apiError?.[3] ?? "QUOTA_EXHAUSTED", - retryable: true, - resetAt: resetAtFrom(text, now) ?? null, - }); + return reusable( + makeProviderFailure({ + class: "usage_limit", + message: "You have reached your Antigravity quota for this period.", + code: apiError?.[3] ?? "QUOTA_EXHAUSTED", + retryable: true, + resetAt: resetAtFrom(text, now) ?? null, + }), + ); + } + if (INVALID_THINKING_SIGNATURE.test(text)) { + return { + failure: makeProviderFailure({ + class: "provider_error", + message: + "Antigravity's saved conversation can no longer be sent to the model. The next message starts a fresh Antigravity session with this thread's context.", + code: "INVALID_THINKING_SIGNATURE", + retryable: true, + }), + threadDisposition: "broken", + }; } if (apiError === null) { - return makeProviderFailure({ - class: "provider_error", - message: "Antigravity stopped this turn because of an error.", - }); + return reusable( + makeProviderFailure({ + class: "provider_error", + message: "Antigravity stopped this turn because of an error.", + }), + ); } const [, httpStatus, message, status] = apiError; const rateLimited = httpStatus === "429" || status === "RESOURCE_EXHAUSTED"; - return makeProviderFailure({ - class: rateLimited ? "usage_limit" : "provider_error", - message: message!.trim(), - code: status ?? null, - retryable: rateLimited || Number(httpStatus) >= 500, - resetAt: resetAtFrom(text, now) ?? null, - }); + return reusable( + makeProviderFailure({ + class: rateLimited ? "usage_limit" : "provider_error", + message: message!.trim(), + code: status ?? null, + retryable: rateLimited || Number(httpStatus) >= 500, + resetAt: resetAtFrom(text, now) ?? null, + }), + ); } const conversationSegments = ["antigravity-acp", "conversations"]; diff --git a/packages/provider-acp-registry/src/server/adapter.ts b/packages/provider-acp-registry/src/server/adapter.ts index 105778db7e876..4fd90c1fa79b8 100644 --- a/packages/provider-acp-registry/src/server/adapter.ts +++ b/packages/provider-acp-registry/src/server/adapter.ts @@ -34,6 +34,7 @@ import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import type * as Scope from "effect/Scope"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import type * as EffectAcpSchema from "effect-acp/compat"; import * as EffectAcpErrors from "effect-acp/errors"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; @@ -257,7 +258,20 @@ export const makeAcpRegistryAdapterV2 = Effect.fn("makeAcpRegistryAdapterV2")(fu : {}), // glm-acp-agent is the one agent whose per-turn usage a live probe has verified. ...(registryAgentId === "glm-acp-agent" ? { reportsPerTurnPromptUsage: true } : {}), - ...(watchKimiPromptOutcome === undefined ? {} : { watchPromptOutcome: watchKimiPromptOutcome }), + ...(watchKimiPromptOutcome === undefined + ? {} + : { + watchPromptOutcome: (sessionId: string) => + Effect.map( + watchKimiPromptOutcome(sessionId), + (check) => (result: EffectAcpSchema.PromptResponse) => + Effect.map(check(result), (failure) => + failure === undefined + ? undefined + : { failure, threadDisposition: "reusable" as const }, + ), + ), + }), ...(registryAgentId === "mistral-vibe" ? { registerExtensions: registerMistralVibeAcpExtensions } : {}), diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index 696ee6b30d8d7..2a5a081ed0d63 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -339,6 +339,11 @@ export function acpPerTurnPromptUsage( }; } +interface AcpPromptOutcomeFailure { + readonly failure: OrchestrationV2ProviderFailure; + readonly threadDisposition: "reusable" | "broken"; +} + export interface AcpAdapterV2Flavor { readonly buildAttachmentPrompt?: (input: { readonly text: string; @@ -353,14 +358,13 @@ export interface AcpAdapterV2Flavor { /** * Runs before `session/prompt`. The returned check sees a prompt the agent ended * without an error or cancel, and reports a failure its stop reason hid. A reported - * failure restarts the runtime before the next turn. + * failure restarts the runtime before the next turn. A `broken` thread's native + * history is unusable, so the next turn falls back to a fresh session. */ readonly watchPromptOutcome?: ( sessionId: string, ) => Effect.Effect< - ( - result: EffectAcpSchema.PromptResponse, - ) => Effect.Effect + (result: EffectAcpSchema.PromptResponse) => Effect.Effect >; readonly requiresSignIn?: (cause: unknown) => boolean; /** @@ -8042,10 +8046,14 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( context: ActiveAcpTurn, status: "completed" | "interrupted" | "failed" | "cancelled", failure?: OrchestrationV2ProviderFailure, - options?: { readonly childTurnsContinue?: boolean }, + options?: { + readonly childTurnsContinue?: boolean; + readonly threadDisposition?: "reusable" | "broken"; + }, ): Effect.fn.Return { if (context.finalized) return; const settledStatus = context.interrupted ? "interrupted" : status; + const threadDisposition = options?.threadDisposition ?? "reusable"; context.finalizedStatus = settledStatus; context.finalized = true; const watchdog = context.inactivityWatchdog; @@ -8144,7 +8152,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( const updatedProviderThread: OrchestrationV2ProviderThread = { ...context.input.providerThread, providerSessionId: input.providerSessionId, - status: "active", + status: threadDisposition === "broken" ? "error" : "active", lastRunOrdinal: context.input.runOrdinal, firstRunOrdinal: context.input.providerThread.firstRunOrdinal ?? context.input.runOrdinal, @@ -8188,7 +8196,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( retry: context.providerRetry.retry, retryStartedAt: context.providerRetry.startedAt, }), - threadDisposition: "reusable", + threadDisposition, } : { type: "turn.terminal", @@ -8198,7 +8206,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( runOrdinal: context.input.runOrdinal, status: settledStatus, failure: null, - threadDisposition: "reusable", + threadDisposition, }, ); const subagentsRequiringCarryover = [...context.subagents.values()].filter( @@ -8943,7 +8951,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( yield* Ref.set(runtimeRestartRequired, true); } const failure = - hiddenFailure ?? + hiddenFailure?.failure ?? (context.interrupted ? undefined : stopReasonFailure(result.stopReason, result.error)); @@ -8969,6 +8977,9 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( // background tools and subagents keep reporting to this run. yield* finalizeTurn(context, "failed", failure, { childTurnsContinue: hasSurvivingBackgroundWork(context), + ...(hiddenFailure === undefined + ? {} + : { threadDisposition: hiddenFailure.threadDisposition }), }); return; } @@ -9251,6 +9262,13 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( readonly modelSelection?: ModelSelection; readonly runtimePolicy?: ProviderAdapter.ProviderAdapterV2RuntimePolicy; }) { + if (threadInput.providerThread.status === "error") { + // A broken turn left this native history unusable; the caller recovers in a fresh session. + return yield* new ProviderAdapter.ProviderAdapterProtocolError({ + driver, + detail: "The native session history is unusable.", + }); + } return yield* runtimeTransitionPermit.withPermit( Effect.gen(function* () { yield* awaitRuntimeTeardown();