Repository navigation
fix(server): recover Codex threads after encrypted history rejection #36643
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
2cca8a9
57afa34
b5ed144
05bc566
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,4 @@ | ||
| import { classifyCodexManagedError } from "../provider/CodexManagedErrors.ts"; | ||
| import { modelSelectionsEqual } from "@t3tools/shared/model"; | ||
| import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences"; | ||
| import { expandAssistantCitationsForProvider } from "@t3tools/shared/assistantCitations"; | ||
|
|
@@ -1530,6 +1531,71 @@ export const layer: Layer.Layer< | |
| handoff.delivery?.nativeThreadId === providerThread.nativeThreadRef?.nativeId && | ||
| handoff.delivery?.status === "pending", | ||
| ); | ||
| const rejectedHistory = yield* Effect.gen(function* () { | ||
| if (session.driver !== "codex") return false; | ||
| const acceptedAttemptIds = new Set( | ||
| projection.providerTurns.map((turn) => turn.runAttemptId), | ||
| ); | ||
| const previous = projection.runs.findLast( | ||
| (source) => | ||
| source.providerThreadId === providerThread.id && | ||
| source.ordinal < run.ordinal && | ||
| source.activeAttemptId !== null && | ||
| acceptedAttemptIds.has(source.activeAttemptId), | ||
| ); | ||
| if ( | ||
| previous === undefined || | ||
| (previous.status !== "failed" && | ||
| previous.status !== "interrupted" && | ||
| previous.status !== "cancelled") | ||
| ) | ||
| return false; | ||
| const previousAttempt = projection.attempts.find( | ||
| (source) => source.id === previous.activeAttemptId, | ||
| ); | ||
| if (previousAttempt === undefined) return false; | ||
| if (previousAttempt.nativeThreadId !== undefined) { | ||
| if (previousAttempt.nativeThreadId !== providerThread.nativeThreadRef?.nativeId) | ||
| return false; | ||
| } else if ( | ||
| projection.contextHandoffs.some( | ||
| (handoff) => | ||
| handoff.toProviderThreadId === providerThread.id && | ||
| handoff.strategy === "full_thread_summary" && | ||
| handoff.delivery?.nativeThreadId === providerThread.nativeThreadRef?.nativeId && | ||
| handoff.coveredRunOrdinals.to >= previous.ordinal, | ||
| ) | ||
| ) | ||
| return false; | ||
| // The native resume RPC can succeed while persisted input still rejects every turn. | ||
| return (yield* projectionStore.getTurnStartHistory(input.threadId, [previous.id])).some( | ||
| (item) => | ||
| item.type === "error" && | ||
| item.nodeId === previous.rootNodeId && | ||
| (item.failure.code === "invalid_encrypted_content" || | ||
| classifyCodexManagedError(item.failure.message)?.code === | ||
| "invalid_encrypted_content"), | ||
| ); | ||
| }); | ||
| // A fresh session would orphan background work still running on the native thread. | ||
| const hasPendingWork = (thread: OrchestrationV2ProviderThread) => | ||
| session.hasPendingBackgroundWorkForThread?.(thread, { includeSubagents: true }) ?? | ||
| session.hasPendingBackgroundWork ?? | ||
| Effect.succeed(false); | ||
| const failForPendingWork = loadFromProvider( | ||
| Effect.fail( | ||
| new ProviderAdapter.ProviderAdapterTurnStartError({ | ||
| driver: session.driver, | ||
| threadId: projection.thread.id, | ||
| providerThreadId: providerThread.id, | ||
| runId, | ||
| cause: new ContextHandoffBackgroundWorkError(), | ||
| }), | ||
| ), | ||
| ); | ||
| if (rejectedHistory && (yield* hasPendingWork(providerThread))) { | ||
| return yield* failForPendingWork; | ||
| } | ||
| // A provider that cannot truncate rolls back into a fresh native session holding none of the kept turns. | ||
| const nativeSessionLacksKeptTurns = | ||
| !session.providerSession.capabilities.checkpointing.providerCanRollbackConversation && | ||
|
|
@@ -1544,7 +1610,7 @@ export const layer: Layer.Layer< | |
| handoff.strategy === "manual_context", | ||
| ); | ||
| const resumed = yield* Effect.result( | ||
| uncertainDelivery || nativeSessionLacksKeptTurns | ||
| uncertainDelivery || nativeSessionLacksKeptTurns || rejectedHistory | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Check for an active native writer before forcing a replacement on rejected history. On the rejected-history path, the synthetic failure skips Before forcing a native replacement, require the same active-writer check that Codex
|
||
| ? Effect.fail( | ||
| new ProviderAdapter.ProviderAdapterTurnStartError({ | ||
| driver: session.driver, | ||
|
|
@@ -1553,7 +1619,9 @@ export const layer: Layer.Layer< | |
| runId, | ||
| cause: uncertainDelivery | ||
| ? "Uncertain native history injection" | ||
| : "Native session holds none of the kept conversation", | ||
| : nativeSessionLacksKeptTurns | ||
| ? "Native session holds none of the kept conversation" | ||
| : "Rejected native history", | ||
| }), | ||
| ) | ||
| : session.resumeThread({ | ||
|
|
@@ -1595,26 +1663,7 @@ export const layer: Layer.Layer< | |
| Effect.catchTags({ ContextHandoffBudgetError: () => Effect.succeed(false) }), | ||
| ); | ||
| if (!fitsFresh) return resumed.success; | ||
| const hasPendingWork = yield* ( | ||
| session.hasPendingBackgroundWorkForThread?.(resumed.success, { | ||
| includeSubagents: true, | ||
| }) ?? | ||
| session.hasPendingBackgroundWork ?? | ||
| Effect.succeed(false) | ||
| ); | ||
| if (hasPendingWork) { | ||
| return yield* loadFromProvider( | ||
| Effect.fail( | ||
| new ProviderAdapter.ProviderAdapterTurnStartError({ | ||
| driver: session.driver, | ||
| threadId: projection.thread.id, | ||
| providerThreadId: providerThread.id, | ||
| runId, | ||
| cause: new ContextHandoffBackgroundWorkError(), | ||
| }), | ||
| ), | ||
| ); | ||
| } | ||
| if (yield* hasPendingWork(resumed.success)) return yield* failForPendingWork; | ||
| } | ||
| resumedHandoffContext = undefined; | ||
| resumedRetryHandoff = undefined; | ||
|
|
@@ -1636,9 +1685,11 @@ export const layer: Layer.Layer< | |
| ? "uncertain_history_delivery" | ||
| : nativeSessionLacksKeptTurns | ||
| ? "native_session_lacks_kept_turns" | ||
| : resumed._tag === "Success" | ||
| ? "handoff_budget_exhausted" | ||
| : "resume_failed", | ||
| : rejectedHistory | ||
| ? "rejected_native_history" | ||
| : resumed._tag === "Success" | ||
| ? "handoff_budget_exhausted" | ||
| : "resume_failed", | ||
| errorTag: resumed._tag === "Failure" ? resumed.failure._tag : "ContextHandoffBudgetError", | ||
| }); | ||
| const replacement = yield* loadFromProvider( | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Scope the recovery error match to the previous run's active attempt.
rejectedHistoryreads every error item on the previous run's root node. Each attempt writes its own failure item, and earlier attempts' items are kept. If a run has an earlier attempt that receivedinvalid_encrypted_content, and its latest attempt then fails for another reason on the same native thread, this check still returnstrue. The session is then reset to a fresh native session with portable history, although the latest attempt did not reject the native history.Match the error to
previous.activeAttemptId, which is already resolved aspreviousAttempt. Use the attempt-owned provider turn or equivalent attempt identity on the error item. Confirm the item field inpackages/provider-core/src/server/failure.tsbefore writing the filter.