From e1d0997a5bf826bd892b1c26dc3416e36ace4860 Mon Sep 17 00:00:00 2001 From: Nick Spreen <12631527+yspreen@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:10:18 +0900 Subject: [PATCH 1/8] fix(server): usage-limit recovery survives notification wakes that did no work A run that a notification started and that completed at or before the newest usage-limit failure's resetAt no longer replaces that failure as the thread's latest run. Arming, the limit-resume dispatch, manual Resume and Stop all find the limited run through the same rule. Reapplies #31219 on current main. --- .../src/orchestration-v2/Orchestrator.ts | 48 +++- .../orchestration-v2/ProjectionStore.test.ts | 238 +++++++++++++++++- .../src/orchestration-v2/ProjectionStore.ts | 144 ++++++++++- .../src/orchestration-v2/runtimeLayer.test.ts | 65 ++++- .../threadHistoryPaging.test.ts | 84 +++++++ .../orchestration-v2/threadHistoryPaging.ts | 56 +++++ .../src/state/threadExecution.test.ts | 29 +++ .../src/state/threadExecution.ts | 5 +- .../shared/src/orchestrationV2ThreadError.ts | 44 +++- 9 files changed, 694 insertions(+), 19 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 819aa90c03b98..68b546b531366 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -1732,7 +1732,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio "contextTransfers", ], { - turnItemTypes: ["user_message", "error"], + turnItemTypes: ["user_message", "error", "notification"], messageRoles: ["user"], omitSettledLeafNodes: true, // Commands here read only undelivered or active tasks; a dispatcher thread keeps @@ -4529,7 +4529,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const projection = yield* loadProjectionForCommand( command, ["runs", "runtimeRequests", "turnItems"], - { turnItemTypes: ["error"] }, + { turnItemTypes: ["user_message", "error", "notification"] }, ); const run = usageLimitBlockedRun(projection.runs, projection.turnItems, null); const failure = latestRootProviderFailure(run, projection.turnItems); @@ -6898,7 +6898,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio source === undefined || (source.status !== "interrupted" && !(source.status === "failed" && limited?.class === "usage_limit")) || - latestExecutedRun(projection.runs)?.id !== source.id || + ( + usageLimitBlockedRun(projection.runs, projection.turnItems, null) ?? + latestExecutedRun(projection.runs) + )?.id !== source.id || projection.thread.archivedAt !== null || projection.thread.deletedAt !== null || projection.runtimeRequests.some((request) => request.status === "pending") @@ -12730,14 +12733,37 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const { runs, subagents } = input.projection; let { thread } = input.projection; let recovery = thread.limitRecovery; - const latest = latestExecutedRun(runs); - if (latest?.status === "failed") { - const { turnItems } = yield* projectionStore - .getThreadRecords(thread.id, ["turnItems"], { - turnItemTypes: ["error"], - turnItemRunIds: [latest.id], - }) + // Stop must also disarm a limited run that completed notification wakes hide. Only the + // latest failure's errors and the inputs of runs completed before its reset can matter. + const readItems = (filter: Parameters[2]) => + projectionStore + .getThreadRecords(thread.id, ["turnItems"], filter) .pipe(mapDispatchError(input.command)); + const failed = latestExecutedRun(runs.filter((run) => run.status === "failed")); + const errors = + failed === null + ? [] + : (yield* readItems({ turnItemTypes: ["error"], turnItemRunIds: [failed.id] })).turnItems; + const resetAt = latestRootProviderFailure(failed, errors)?.resetAt; + const resetMs = resetAt == null ? Number.NaN : Date.parse(resetAt); + const wakeRunIds = runs + .filter( + (run) => + run.status === "completed" && + run.completedAt !== null && + DateTime.toEpochMillis(run.completedAt) <= resetMs, + ) + .map((run) => run.id); + const inputs = + wakeRunIds.length === 0 + ? [] + : (yield* readItems({ + turnItemTypes: ["user_message", "notification"], + turnItemRunIds: wakeRunIds, + })).turnItems; + const turnItems = [...errors, ...inputs]; + const latest = usageLimitBlockedRun(runs, turnItems, null) ?? latestExecutedRun(runs); + if (latest?.status === "failed") { const failure = latestRootProviderFailure(latest, turnItems); if ( failure?.class === "usage_limit" && @@ -15285,7 +15311,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const projection = yield* loadProjectionForCommand( command, ["runs", "turnItems", "providerSessions", "messages"], - { turnItemTypes: ["error"], messageRoles: ["user"] }, + { turnItemTypes: ["user_message", "error", "notification"], messageRoles: ["user"] }, ); if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) { return yield* new OrchestratorDispatchError({ diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 2c667bf43679f..6062e0e3c911e 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -1,6 +1,7 @@ import * as NodeSqlite from "node:sqlite"; import { assert, it, vi } from "@effect/vitest"; +import { usageLimitBlockedRun } from "@t3tools/shared/orchestrationV2ThreadError"; import { EnvironmentId, EventId, @@ -8545,7 +8546,131 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }, }); yield* assertSummary("Plan limit reached.", "usage_limit"); + // A subagent that died on the same limit starts a notification wake that completes + // before the reset. It did no work, so the failed run stays the latest. + const wakeRunId = RunId.make("run:limit-shell:wake"); + const wakeNodeId = NodeId.make("node:limit-shell:wake"); + yield* store.apply({ + id: EventId.make("event:limit-shell:wake"), + type: "run.created", + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + driver, + providerInstanceId, + occurredAt: now, + payload: { + ...original, + id: wakeRunId, + ordinal: original.ordinal + 4, + rootNodeId: wakeNodeId, + userMessageId: MessageId.make("message:limit-shell:wake"), + status: "completed", + startedAt: DateTime.add(now, { minutes: 1 }), + completedAt: DateTime.add(now, { minutes: 1 }), + }, + }); + yield* store.apply({ + id: EventId.make("event:limit-shell:wake-notification"), + type: "turn-item.updated", + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + driver, + occurredAt: now, + payload: { + id: TurnItemId.make("limit-shell:wake-notification"), + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 4, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "notification", + source: { kind: "delegated_task", taskIds: [] }, + outcome: "failed", + summary: "Subagent failed", + }, + }); + yield* assertSummary("Plan limit reached.", "usage_limit"); + const wakeProjection = yield* store.getThreadProjection(threadId); + assert.equal( + ProjectionStore.threadShellFromProjection(wakeProjection).latestRunId, + original.id, + ); + assert.equal( + (yield* store.getShellSnapshot()).threads.find((row) => row.id === threadId)!.latestRunId, + original.id, + ); + // Near misses: a wake completed after the reset, or a run a user message started, did work. + const assertLatest = Effect.fnUntraced(function* (runId: RunId) { + const projection = yield* store.getThreadProjection(threadId); + assert.equal(ProjectionStore.threadShellFromProjection(projection).latestRunId, runId); + assert.equal( + (yield* store.getShellSnapshot()).threads.find((row) => row.id === threadId)!.latestRunId, + runId, + ); + }); + const setWakeCompletedAt = (completedAt: DateTime.Utc) => + store.apply({ + id: EventId.make(`event:limit-shell:wake-at-${DateTime.formatIso(completedAt)}`), + type: "run.updated", + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + driver, + occurredAt: now, + payload: { + ...wakeProjection.runs.find((run) => run.id === wakeRunId)!, + completedAt, + }, + }); + yield* setWakeCompletedAt(DateTime.makeUnsafe("2099-01-02T00:00:00.000Z")); + yield* assertLatest(wakeRunId); + yield* setWakeCompletedAt(DateTime.add(now, { minutes: 1 })); + yield* assertLatest(original.id); + yield* store.apply({ + id: EventId.make("event:limit-shell:wake-user-start"), + type: "turn-item.updated", + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + driver, + occurredAt: now, + payload: { + createdBy: "user", + creationSource: "web", + id: TurnItemId.make("limit-shell:wake-user-start"), + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 3, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "user_message", + messageId: MessageId.make("message:limit-shell:wake"), + inputIntent: "turn_start", + text: "a real request", + attachments: [], + }, + }); + yield* assertLatest(wakeRunId); const sql = yield* SqlClient.SqlClient; + yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${wakeRunId}`; yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${aheadRunId}`; yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${queuedRunId}`; yield* assertSummary("Plan limit reached.", "usage_limit"); @@ -9207,7 +9332,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { (detail) => /^SEARCH (?:item|error) /.test(detail) && !detail.includes("(turn_item_id=?)"), ); - assert.lengthOf(errorLookups, marker.includes("blocking") ? 2 : 1, marker); + assert.lengthOf(errorLookups, marker.includes("blocking") ? 3 : 2, marker); for (const errorLookup of errorLookups) { assert.include(errorLookup, "turn_items_failed_error_idx"); assert.include(errorLookup, "(thread_id=? AND run_id=? AND node_id=?)"); @@ -9216,6 +9341,117 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }), ); + it.effect("snapshot windows keep the limited run that notification wakes hide", () => + Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + const threadId = yield* addRolledBackRecoveryCandidate("limit-window"); + const original = (yield* store.getThreadProjection(threadId)).runs[0]!; + const now = yield* DateTime.now; + const base = { threadId, providerThreadId: null, providerTurnId: null, nativeItemRef: null }; + yield* store.apply({ + id: EventId.make("event:limit-window:error"), + type: "turn-item.updated", + threadId, + occurredAt: now, + payload: { + ...base, + id: TurnItemId.make("limit-window:error"), + runId: original.id, + nodeId: original.rootNodeId, + parentItemId: null, + ordinal: 2, + status: "failed", + title: "Usage limit reached", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "error", + failure: { + class: "usage_limit", + message: "Plan limit reached.", + resetAt: "2099-01-01T00:00:00.000Z", + code: "usageLimitExceeded", + retryable: null, + }, + }, + }); + yield* store.apply({ + id: EventId.make("event:limit-window:failed"), + type: "run.updated", + threadId, + occurredAt: now, + payload: { ...original, status: "failed", completedAt: now }, + }); + for (let index = 0; index < 6; index++) { + const runId = RunId.make(`run:limit-window:wake-${index}`); + const nodeId = NodeId.make(`node:limit-window:wake-${index}`); + const at = DateTime.add(now, { seconds: index + 1 }); + yield* store.apply({ + id: EventId.make(`event:limit-window:wake-${index}`), + type: "run.created", + threadId, + runId, + nodeId, + driver, + providerInstanceId, + occurredAt: at, + payload: { + ...original, + id: runId, + ordinal: original.ordinal + index + 1, + rootNodeId: nodeId, + userMessageId: MessageId.make(`message:limit-window:wake-${index}`), + status: "completed", + startedAt: at, + completedAt: at, + }, + }); + yield* store.apply({ + id: EventId.make(`event:limit-window:wake-${index}:notification`), + type: "turn-item.updated", + threadId, + runId, + nodeId, + driver, + occurredAt: at, + payload: { + ...base, + id: TurnItemId.make(`limit-window:wake-${index}`), + runId, + nodeId, + parentItemId: null, + ordinal: 10 + index, + status: "completed", + title: null, + startedAt: at, + completedAt: at, + updatedAt: at, + type: "notification", + source: { kind: "delegated_task", taskIds: [] }, + outcome: "failed", + summary: "Subagent failed", + }, + }); + } + + const snapshot = yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 3 }); + const bounded = buildBoundedThreadProjection({ + projection: snapshot.projection, + snapshotSequence: snapshot.snapshotSequence, + }).projection; + for (const projection of [snapshot.projection, bounded]) { + assert.equal( + usageLimitBlockedRun(projection.runs, projection.turnItems, null)?.id, + original.id, + ); + } + assert.notInclude( + snapshot.projection.visibleTurnItems.map((row) => String(row.sourceItemId)), + "limit-window:error", + ); + }), + ); + it.effect("selects the latest waiting secret only from active runs", () => Effect.gen(function* () { const projectionStore = yield* ProjectionStore.ProjectionStoreV2; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 5b8a636cc24f1..5b94a8b2ddb05 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1241,6 +1241,7 @@ const shellIndexLineage = (row: { ...(row.side_chat === null ? {} : { sideChat: row.side_chat === 1 }), }); +const LIMIT_EVIDENCE_MAX_RUNS = 200; const encodeIdList = Schema.encodeSync(Schema.fromJsonString(Schema.Array(Schema.String))); // Empty filters still scan SQLite rows; short-circuit them, including turnItemTypes: []. @@ -4708,6 +4709,50 @@ export const layer: Layer.Layer = ), ); + // The newest failed run of each thread and its root error, for isLimitedWake. + const latestFailureJoins = sql` + LEFT JOIN orchestration_v2_projection_runs latest_failure ON latest_failure.run_id = ( + SELECT failed.run_id FROM orchestration_v2_projection_runs failed + WHERE failed.thread_id = t.thread_id AND failed.status = 'failed' + ORDER BY failed.completed_at IS NULL DESC, failed.completed_at DESC, + failed.ordinal DESC, failed.run_id DESC + LIMIT 1 + ) + LEFT JOIN orchestration_v2_projection_turn_items latest_failure_error + ON latest_failure_error.turn_item_id = ( + SELECT error.turn_item_id FROM orchestration_v2_projection_turn_items error + WHERE error.thread_id = t.thread_id + AND error.run_id = latest_failure.run_id + AND error.node_id IS json_extract(latest_failure.payload_json, '$.rootNodeId') + AND error.type = 'error' AND error.status = 'failed' + ORDER BY error.updated_at DESC, error.ordinal DESC, error.turn_item_id DESC + LIMIT 1 + ) + `; + // Mirrors the wake rule of usageLimitBlockedRun: a run that a notification started and + // that completed before the newest usage-limit failure resets did no work, so it must + // not replace that failure as the latest run. + const isLimitedWake = (alias: string) => { + const run = sql.literal(alias); + return sql`( + ${run}.status = 'completed' + AND COALESCE( + json_extract(latest_failure_error.payload_json, '$.failure.class') = 'usage_limit' + AND julianday(${run}.completed_at) <= julianday( + json_extract(latest_failure_error.payload_json, '$.failure.resetAt') + ), 0 + ) AND ( + SELECT wake.type FROM orchestration_v2_projection_turn_items wake + WHERE wake.run_id = ${run}.run_id + AND wake.node_id IS NOT NULL + AND wake.node_id IS json_extract(${run}.payload_json, '$.rootNodeId') + AND wake.type IN ('user_message', 'notification') + ORDER BY wake.ordinal + LIMIT 1 + ) IS 'notification' + )`; + }; + const getLimitRecoveryCandidates = Effect.fn("ProjectionStore.getLimitRecoveryCandidates")( function* (options: Parameters[0]) { const rows = yield* sql<{ @@ -4731,6 +4776,7 @@ export const layer: Layer.Layer = LIMIT 1 ) AS last_error FROM orchestration_v2_projection_threads t + ${latestFailureJoins} INNER JOIN orchestration_v2_projection_runs r ON r.run_id = ( SELECT latest.run_id FROM orchestration_v2_projection_runs latest WHERE latest.thread_id = t.thread_id @@ -4739,6 +4785,7 @@ export const layer: Layer.Layer = latest.status = 'cancelled' AND json_extract(latest.payload_json, '$.startedAt') IS NULL ) + AND NOT ${isLimitedWake("latest")} -- latestExecutedRun: the run that ended last (runRanAfter). ORDER BY latest.completed_at IS NULL DESC, latest.completed_at DESC, latest.ordinal DESC, latest.run_id DESC @@ -7776,6 +7823,99 @@ export const layer: Layer.Layer = ), ); + // A window reads only the newest rows, so an older usage-limited run and the notification + // starts of the wakes after it can be missing. Clients need them (usageLimitBlockedRun) + // to resume it. They join runs and turnItems, never visibleTurnItems. + const withLimitEvidence = Effect.fn("ProjectionStore.withLimitEvidence")(function* ( + threadId: ThreadId, + projection: OrchestrationV2ThreadProjection, + ) { + const rows = yield* sql<{ readonly kind: string; readonly id: string }>` + WITH blocked AS ( + SELECT r.run_id, r.ordinal, item.turn_item_id AS error_id + FROM orchestration_v2_projection_threads t + ${latestFailureJoins} + INNER JOIN orchestration_v2_projection_runs r ON r.run_id = ( + SELECT latest.run_id FROM orchestration_v2_projection_runs latest + WHERE latest.thread_id = t.thread_id + AND latest.status <> 'queued' + AND NOT ( + latest.status = 'cancelled' + AND json_extract(latest.payload_json, '$.startedAt') IS NULL + ) + AND NOT ${isLimitedWake("latest")} + ORDER BY latest.completed_at IS NULL DESC, latest.completed_at DESC, + latest.ordinal DESC, latest.run_id DESC + LIMIT 1 + ) AND r.status = 'failed' + INNER JOIN orchestration_v2_projection_turn_items item ON item.turn_item_id = ( + SELECT error.turn_item_id FROM orchestration_v2_projection_turn_items error + WHERE error.thread_id = t.thread_id AND error.run_id = r.run_id + AND error.type = 'error' AND error.status = 'failed' + AND error.node_id IS json_extract(r.payload_json, '$.rootNodeId') + ORDER BY error.updated_at DESC, error.ordinal DESC, error.turn_item_id DESC + LIMIT 1 + ) + WHERE t.thread_id = ${threadId} + AND EXISTS ( + SELECT 1 FROM orchestration_v2_projection_turn_items limited + WHERE limited.thread_id = t.thread_id + AND limited.type = 'error' AND limited.status = 'failed' + AND json_extract(limited.payload_json, '$.failure.class') = 'usage_limit' + ) + AND json_extract(item.payload_json, '$.failure.class') = 'usage_limit' + ), later AS ( + -- Runs omitted here are not loaded at all, which leaves the result unchanged. + SELECT run.run_id, run.payload_json + FROM orchestration_v2_projection_runs run, blocked + WHERE run.thread_id = ${threadId} AND run.ordinal > blocked.ordinal + ORDER BY run.ordinal DESC + LIMIT ${LIMIT_EVIDENCE_MAX_RUNS} + ) + SELECT 'run' AS kind, run_id AS id FROM blocked + UNION ALL SELECT 'item', error_id FROM blocked + UNION ALL SELECT 'run', run_id FROM later + UNION ALL + SELECT 'item', start.turn_item_id + FROM later + CROSS JOIN orchestration_v2_projection_turn_items start ON start.turn_item_id = ( + SELECT first_input.turn_item_id FROM orchestration_v2_projection_turn_items first_input + WHERE first_input.run_id = later.run_id + AND first_input.node_id IS json_extract(later.payload_json, '$.rootNodeId') + AND first_input.type IN ('user_message', 'notification') + ORDER BY first_input.ordinal + LIMIT 1 + ) + WHERE start.type = 'notification' + `; + const loadedRunIds = new Set(projection.runs.map((run) => run.id)); + const loadedItemIds = new Set(projection.turnItems.map((item) => item.id)); + const runIds = rows.flatMap((row) => + row.kind === "run" && !loadedRunIds.has(row.id) ? [row.id] : [], + ); + const itemIds = rows.flatMap((row) => + row.kind === "item" && !loadedItemIds.has(row.id) ? [row.id] : [], + ); + if (runIds.length === 0 && itemIds.length === 0) return projection; + const runRows = yield* sql` + SELECT payload_json FROM orchestration_v2_projection_runs + WHERE thread_id = ${threadId} + AND run_id IN (SELECT value FROM json_each(${encodeIdList(runIds)})) + `; + const itemRows = yield* sql` + SELECT payload_json FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} + AND turn_item_id IN (SELECT value FROM json_each(${encodeIdList(itemIds)})) + `; + const runs = yield* decodeRows(decodeRunPayload, threadId)(runRows); + const turnItems = yield* decodeRows(decodeTurnItemPayload, threadId)(itemRows); + return { + ...projection, + runs: [...projection.runs, ...runs].toSorted((left, right) => left.ordinal - right.ordinal), + turnItems: [...projection.turnItems, ...turnItems], + }; + }); + const getThreadSnapshotWindow: ProjectionStoreV2Shape["getThreadSnapshotWindow"] = ( threadId, options, @@ -7844,7 +7984,7 @@ export const layer: Layer.Layer = return { schemaVersion: ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION, snapshotSequence: rows[0]?.snapshot_sequence ?? 0, - projection: windowProjection, + projection: yield* withLimitEvidence(threadId, windowProjection), }; }), ) @@ -8119,6 +8259,7 @@ export const layer: Layer.Layer = ORDER BY r.ordinal DESC, r.run_id DESC LIMIT 1 ) + ${latestFailureJoins} LEFT JOIN orchestration_v2_projection_runs presented ON presented.run_id = ${presentedShellRunId(sql.literal("t.thread_id"))} LEFT JOIN orchestration_v2_projection_runs blocked ON blocked.run_id = ( @@ -8130,6 +8271,7 @@ export const layer: Layer.Layer = candidate.status = 'cancelled' AND json_extract(candidate.payload_json, '$.startedAt') IS NULL ) + AND NOT ${isLimitedWake("candidate")} -- latestExecutedRun: the run that ended last (runRanAfter). ORDER BY candidate.completed_at IS NULL DESC, candidate.completed_at DESC, candidate.ordinal DESC, candidate.run_id DESC diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 82fd3381e20d9..616f4bc23abb5 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -12215,9 +12215,11 @@ it.layer(layerTest)("usage-limit recovery", (it) => { "independent-patches", "expired-snooze", "wake", + "wake-hidden", "stop", "stop-before-arm", "stop-before-arm-mismatched-recovery", + "stop-before-arm-wake-hidden", ] as const)("guards a scheduled usage-limit continuation against %s", (scenario) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -12321,6 +12323,61 @@ it.layer(layerTest)("usage-limit recovery", (it) => { }, ], }); + const hiddenByWake = scenario === "wake-hidden" || scenario === "stop-before-arm-wake-hidden"; + if (hiddenByWake) { + // A subagent that died on the same limit starts a notification wake that completes + // before the reset. The failed run stays the latest, so it is armed and resumes. + const wakeRunId = RunId.make(`recovery:wake-run:${scenario}`); + const wakeNodeId = NodeId.make(`recovery:wake-node:${scenario}`); + yield* events.write({ + events: [ + { + id: EventId.make(`recovery:wake-run-created:${scenario}`), + type: "run.created", + threadId, + runId: wakeRunId, + providerInstanceId: modelSelection.instanceId, + occurredAt: now, + payload: { + ...run, + id: wakeRunId, + ordinal: run.ordinal + 1, + rootNodeId: wakeNodeId, + userMessageId: MessageId.make(`recovery:wake-message:${scenario}`), + status: "completed", + startedAt: now, + completedAt: now, + }, + }, + { + id: EventId.make(`recovery:wake-notification:${scenario}`), + type: "turn-item.updated", + threadId, + occurredAt: now, + payload: { + id: TurnItemId.make(`recovery:wake-notification:${scenario}`), + type: "notification", + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + source: { kind: "delegated_task", taskIds: [] }, + outcome: "failed", + summary: "Subagent failed", + }, + }, + ], + }); + } if (scenario === "queued-resume") { const queuedRun = projection.runs[1]!; yield* events.write({ @@ -12406,7 +12463,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => { limitRecovery: arm.limitRecovery, }; const stopBeforeArm = - scenario === "stop-before-arm" || scenario === "stop-before-arm-mismatched-recovery"; + scenario === "stop-before-arm" || + scenario === "stop-before-arm-mismatched-recovery" || + scenario === "stop-before-arm-wake-hidden"; if (stopBeforeArm) { if (scenario === "stop-before-arm-mismatched-recovery") { const current = yield* orchestrator.getThreadProjection(threadId); @@ -12585,7 +12644,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => { }); assert.lengthOf( (yield* orchestrator.getThreadProjection(threadId)).runs, - scenario === "queued-resume" ? 2 : 1, + scenario === "queued-resume" || hiddenByWake ? 2 : 1, ); yield* TestClock.adjust("1 minute"); const resume = limitRecoveryCommand( @@ -12827,6 +12886,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => { after.runs, before.runs.length + (scenario === "resume" || + scenario === "wake-hidden" || scenario === "resume-started" || scenario === "resume-untagged" || stopBeforeArm || @@ -12840,6 +12900,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => { after.messages, before.messages.length + (scenario === "resume" || + scenario === "wake-hidden" || scenario === "resume-started" || scenario === "resume-untagged" || stopBeforeArm || diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts index 8d81cbfadda7b..505d5a61b43e3 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts @@ -9,6 +9,7 @@ import { type OrchestrationV2TurnItem, } from "@t3tools/contracts"; import { describe, expect, it } from "@effect/vitest"; +import { usageLimitBlockedRun } from "@t3tools/shared/orchestrationV2ThreadError"; import * as DateTime from "effect/DateTime"; import { @@ -709,6 +710,89 @@ describe("threadHistoryPaging", () => { expect(bounded.hasMoreHistory).toBe(true); }); + it("keeps a limited run resumable when later notification wakes outgrow the window", () => { + const limitedNode = NodeId.make("node-limited"); + const limited = { + id: RunId.make("run-limited"), + ordinal: 1, + status: "failed", + startedAt: NOW, + completedAt: NOW, + rootNodeId: limitedNode, + }; + const limitError = { + ...makeRow(0).item, + id: TurnItemId.make("limit-error"), + type: "error", + runId: limited.id, + nodeId: limitedNode, + status: "failed", + failure: { + class: "usage_limit", + message: "Plan limit reached.", + code: "usageLimitExceeded", + retryable: null, + resetAt: "2026-06-20T05:00:00.000Z", + }, + } as unknown as OrchestrationV2TurnItem; + const wakes = Array.from({ length: 6 }, (_, index) => { + const nodeId = NodeId.make(`node-wake-${index}`); + const run = { + ...limited, + id: RunId.make(`run-wake-${index}`), + ordinal: index + 2, + status: "completed", + completedAt: DateTime.add(NOW, { seconds: index + 1 }), + rootNodeId: nodeId, + }; + const row = makeRow(index + 1); + const item = { + ...row.item, + id: TurnItemId.make(`wake-notification-${index}`), + type: "notification", + runId: run.id, + nodeId, + source: { kind: "delegated_task", taskIds: [] }, + outcome: "failed", + summary: "Subagent failed", + detail: "d".repeat(50_000), + } as unknown as OrchestrationV2TurnItem; + return { run, row: { ...row, sourceItemId: item.id, item } }; + }); + const rows = [{ ...makeRow(0), sourceItemId: limitError.id, item: limitError }].concat( + wakes.map(({ row }) => row), + ); + const full = { + ...makeProjection(rows), + runs: [limited, ...wakes.map(({ run }) => run)], + } as unknown as ReturnType; + const bounded = buildBoundedThreadProjection({ + projection: full, + snapshotSequence: 1, + policy: { maxItems: 2, maxEncodedBytes: 10_000_000 }, + }); + + expect(bounded.projection.visibleTurnItems).toHaveLength(2); + expect( + usageLimitBlockedRun(bounded.projection.runs, bounded.projection.turnItems, null)?.id, + ).toBe(limited.id); + // Retained wake starts are stubs: a 50 KB notification detail must not reach the snapshot. + const visibleIds = new Set(bounded.projection.visibleTurnItems.map((row) => row.sourceItemId)); + const retained = bounded.projection.turnItems.filter((item) => !visibleIds.has(item.id)); + expect(retained.length).toBeGreaterThan(0); + expect(JSON.stringify(retained).length).toBeLessThan(10_000); + + // Evidence past a quarter of the budget is dropped, so the snapshot stays within it. + const tight = buildBoundedThreadProjection({ + projection: full, + snapshotSequence: 1, + policy: { maxItems: 2, maxEncodedBytes: 8_000 }, + }); + expect( + usageLimitBlockedRun(tight.projection.runs, tight.projection.turnItems, null), + ).toBeNull(); + }); + it("charges local row.item duplication when measuring bounded timeline bytes", () => { const rows = Array.from({ length: 8 }, (_, index) => makeRow(index, { outputBytes: 2_000 })); const full = makeProjection(rows); diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.ts index cf9a101bba44c..db515123773ab 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.ts @@ -5,6 +5,7 @@ import type { ThreadId, TurnItemId, } from "@t3tools/contracts"; +import { usageLimitBlockedRun } from "@t3tools/shared/orchestrationV2ThreadError"; import * as Schema from "effect/Schema"; /** @@ -558,10 +559,59 @@ function isRetainedDependency( ); } +// usageLimitBlockedRun needs the limited run's root error and the notification that started each +// later run, and a long chain of wakes pushes both out of the window. Starts are kept as stubs +// without their unbounded detail, and dropped past a quarter of the budget (clients then show +// the plain latest run). +function limitRecoveryItems( + projection: Pick, + maxEncodedBytes: number, +): ReadonlyMap { + const items = new Map(); + const blocked = usageLimitBlockedRun(projection.runs, projection.turnItems, null); + if (blocked === null) return items; + const laterRoots = new Set(); + for (const run of projection.runs) { + if (run.ordinal > blocked.ordinal && run.rootNodeId !== null) { + laterRoots.add(String(run.rootNodeId)); + } + } + if (laterRoots.size === 0) return items; + const firstInputs = new Map(); + for (const item of projection.turnItems) { + if (item.type === "error" && item.runId === blocked.id && item.nodeId === blocked.rootNodeId) { + items.set(String(item.id), item); + } else if ( + (item.type === "user_message" || item.type === "notification") && + item.nodeId !== null && + laterRoots.has(String(item.nodeId)) + ) { + const first = firstInputs.get(String(item.nodeId)); + if (first === undefined || item.ordinal < first.ordinal) { + firstInputs.set(String(item.nodeId), item); + } + } + } + let startBytes = 0; + const starts: Array = []; + for (const item of firstInputs.values()) { + if (item.type !== "notification") continue; + const { detail: _detail, ...rest } = item; + const stub = { ...rest, title: null, summary: item.summary.slice(0, 120).trimEnd() }; + startBytes += bytesOfJson(stub); + starts.push(stub); + } + if (startBytes <= maxEncodedBytes / 4) { + for (const stub of starts) items.set(String(stub.id), stub); + } + return items; +} + /** Retain small requests outside the visible window; later result pages still need them for attempt/run reducers. */ function retainedInterruptRequestTurnItems( projection: OrchestrationV2ThreadProjection, visible: ReadonlyArray, + limitItems: ReadonlyMap, ): OrchestrationV2TurnItem[] { const visibleLocalIds = new Set(); for (const row of visible) { @@ -581,6 +631,9 @@ function retainedInterruptRequestTurnItems( } retained.push(item); } + for (const [id, item] of limitItems) { + if (!visibleLocalIds.has(id)) retained.push(item); + } return retained; } @@ -691,6 +744,7 @@ export function buildBoundedThreadProjection(input: { }; const latestLocalTurnOrdinal = computeLatestLocalTurnOrdinal(input.projection.turnItems); + const limitItems = limitRecoveryItems(controlProjection, policy.maxEncodedBytes); const dependencyReserve = (() => { let reserve = 0; const pendingRequestIds = pendingRequestIdsOf(controlProjection); @@ -699,6 +753,7 @@ export function buildBoundedThreadProjection(input: { reserve += bytesOfJson(item); } } + for (const item of limitItems.values()) reserve += bytesOfJson(item); return reserve; })(); const controlBytes = bytesOfJson({ @@ -725,6 +780,7 @@ export function buildBoundedThreadProjection(input: { const dependencyTurnItems = retainedInterruptRequestTurnItems( controlProjection, visibleTurnItems, + limitItems, ); const turnItemById = new Map(); for (const item of windowTurnItems) { diff --git a/packages/client-runtime/src/state/threadExecution.test.ts b/packages/client-runtime/src/state/threadExecution.test.ts index ea6de929b4cb7..50c9d4c12417e 100644 --- a/packages/client-runtime/src/state/threadExecution.test.ts +++ b/packages/client-runtime/src/state/threadExecution.test.ts @@ -272,6 +272,35 @@ describe("thread execution presentation", () => { lastErrorClass: null, }); expect(deriveResumableRunId(providerError, deriveThreadRuntime(providerError))).toBeNull(); + + // A notification wake that completes before the reset did no work and keeps the limit + // in place. One that completes after the reset is a real run and clears it. + const resetAt = "2026-07-28T11:00:00.000Z"; + const limited = { ...item, failure: { ...item.failure, resetAt } }; + const wake = { + ...run("wake", 2, "completed"), + rootNodeId: NodeId.make("wake-root"), + completedAt: DateTime.makeUnsafe("2026-07-28T10:30:00.000Z"), + }; + const notification = { + ...item, + id: TurnItemId.make("wake-notification"), + runId: wake.id, + nodeId: wake.rootNodeId, + type: "notification" as const, + status: "completed" as const, + source: { kind: "delegated_task" as const, taskIds: [] }, + outcome: "failed" as const, + summary: "Subagent failed", + }; + const hidden = { ...v2Projection, runs: [failed, wake], turnItems: [limited, notification] }; + expect(deriveThreadRuntime(hidden)).toMatchObject({ + status: "failed", + lastErrorClass: "usage_limit", + }); + expect(deriveResumableRunId(hidden, deriveThreadRuntime(hidden))).toBe(failed.id); + const afterReset = { ...wake, completedAt: DateTime.makeUnsafe("2026-07-28T11:30:00.000Z") }; + expect(usageLimitBlockedRun([failed, afterReset], hidden.turnItems, null)).toBeNull(); }); it("keeps live activity attached to an executing run when a newer run is queued", () => { diff --git a/packages/client-runtime/src/state/threadExecution.ts b/packages/client-runtime/src/state/threadExecution.ts index 0dec62f735d27..15797886456c7 100644 --- a/packages/client-runtime/src/state/threadExecution.ts +++ b/packages/client-runtime/src/state/threadExecution.ts @@ -3,6 +3,7 @@ import { latestRootProviderFailure, latestUnheldRun, threadErrorSummary, + usageLimitBlockedRun, usageLimitRunPresentedAsLatest, } from "@t3tools/shared/orchestrationV2ThreadError"; import { @@ -320,7 +321,9 @@ export function deriveResumableRunId( projection: Pick, runtime: Pick | null, ): RunId | null { - const run = latestExecutedRun(projection.runs); + const run = + usageLimitBlockedRun(projection.runs, projection.turnItems, null) ?? + latestExecutedRun(projection.runs); if (run?.status === "interrupted") return run.id; return run?.status === "failed" && runtime?.lastErrorClass === "usage_limit" && diff --git a/packages/shared/src/orchestrationV2ThreadError.ts b/packages/shared/src/orchestrationV2ThreadError.ts index 141d73f402540..2440d8eb54f33 100644 --- a/packages/shared/src/orchestrationV2ThreadError.ts +++ b/packages/shared/src/orchestrationV2ThreadError.ts @@ -6,10 +6,14 @@ import type { import * as DateTime from "effect/DateTime"; type ErrorTurnItem = Extract; -/** Only error items are read, so callers may pass items stripped of other fields. */ +/** Error items and the first input of each run are read, so callers may pass items stripped of other fields. */ type FailureScanTurnItem = | ErrorTurnItem - | { readonly type: Exclude }; + | { + readonly type: Exclude; + readonly nodeId?: OrchestrationV2TurnItem["nodeId"]; + readonly ordinal?: number; + }; export function latestRootProviderFailure( run: OrchestrationV2Run | null, @@ -95,12 +99,46 @@ export function runRanAfter( return end(run) === end(other) ? run.ordinal > other.ordinal : end(run) > end(other); } +/** + * Background subagents that die on a usage limit each start a root run from a notification. + * Those runs finish within milliseconds, before the limit resets, and have no work to show. + * They must not replace the limited run, or the thread is never armed or resumed. + * Items of types other than user_message, notification and error may be omitted. + */ +function runsWithoutLimitedWakes( + runs: ReadonlyArray, + turnItems: ReadonlyArray, +): ReadonlyArray { + const failed = latestExecutedRun(runs.filter((run) => run.status === "failed")); + const failure = latestRootProviderFailure(failed, turnItems); + if (failure?.class !== "usage_limit" || failure.resetAt == null) return runs; + const resetMs = Date.parse(failure.resetAt); + if (!Number.isFinite(resetMs)) return runs; + const firstInputs = new Map(); + for (const item of turnItems) { + if (item.nodeId == null || item.ordinal == null) continue; + if (item.type !== "user_message" && item.type !== "notification") continue; + const first = firstInputs.get(item.nodeId); + if (first === undefined || item.ordinal < first.ordinal) { + firstInputs.set(item.nodeId, { type: item.type, ordinal: item.ordinal }); + } + } + return runs.filter( + (run) => + run.status !== "completed" || + run.completedAt === null || + DateTime.toEpochMillis(run.completedAt) > resetMs || + run.rootNodeId === null || + firstInputs.get(run.rootNodeId)?.type !== "notification", + ); +} + export function usageLimitBlockedRun( runs: ReadonlyArray, turnItems: ReadonlyArray, sessionError: string | null, ): OrchestrationV2Run | null { - const executed = latestExecutedRun(runs); + const executed = latestExecutedRun(runsWithoutLimitedWakes(runs, turnItems)); if (executed?.status !== "failed") return null; const summary = threadErrorSummary(latestRootProviderFailure(executed, turnItems), sessionError); return summary.lastErrorClass === "usage_limit" ? executed : null; From a48acaed8483b0bb7fe59dcd2b1577d432cbb0e2 Mon Sep 17 00:00:00 2001 From: Nick Spreen <12631527+yspreen@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:18:41 +0900 Subject: [PATCH 2/8] fix(client-runtime): paging back shows the limit error and wake notifications a snapshot retained --- .../src/state/threadHistoryMerge.test.ts | 37 +++++++++++++++++++ .../src/state/threadHistoryMerge.ts | 13 +++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/packages/client-runtime/src/state/threadHistoryMerge.test.ts b/packages/client-runtime/src/state/threadHistoryMerge.test.ts index 95f2b01e7347e..59133a277d1e8 100644 --- a/packages/client-runtime/src/state/threadHistoryMerge.test.ts +++ b/packages/client-runtime/src/state/threadHistoryMerge.test.ts @@ -233,6 +233,43 @@ describe("threadHistoryMerge", () => { expect(merged.visibleTurnItems).toEqual([]); }); + it("shows the limit error and wake notification a snapshot retained for recovery", () => { + const recent = row(5); + const error = { + ...row(0).item, + type: "error" as const, + status: "failed" as const, + title: "Usage limit reached", + failure: { class: "usage_limit" as const, message: "Plan limit reached" }, + }; + const notification = { + ...row(1).item, + type: "notification" as const, + source: { kind: "delegated_task" as const, taskIds: [] }, + outcome: "failed" as const, + summary: "Subagent failed", + detail: "full detail", + }; + const { detail: _detail, ...stub } = notification; + const coldOpen = { + ...v2Projection, + turnItems: [error, stub, recent.item], + visibleTurnItems: [{ ...recent, position: 0 }], + } as never; + + const merged = mergeOlderHistoryIntoProjection(coldOpen, [ + { ...row(0), item: error }, + { ...row(1), item: notification }, + ] as never); + + expect(merged.visibleTurnItems.map((entry) => entry.item)).toEqual([ + error, + notification, + recent.item, + ]); + expect(merged.turnItems).toEqual([error, notification, recent.item]); + }); + it("retains merged older rows when a live turn-item update arrives", () => { const recent = [row(2), row(3)]; const projection = { diff --git a/packages/client-runtime/src/state/threadHistoryMerge.ts b/packages/client-runtime/src/state/threadHistoryMerge.ts index ee9174e8fbe81..a27f90407ee44 100644 --- a/packages/client-runtime/src/state/threadHistoryMerge.ts +++ b/packages/client-runtime/src/state/threadHistoryMerge.ts @@ -122,11 +122,14 @@ export function mergeOlderHistoryIntoProjection( (row.visibility === "local" || row.sourceThreadId === projection.thread.id) ) { const currentItem = turnItemById.get(String(row.sourceItemId)); - // Hidden live items stay hidden. Snapshots also retain interrupt requests and - // pending request cards outside the visible window; their history page shows them. + // Hidden live items stay hidden. Snapshots also retain interrupt requests, pending + // request cards, and the error and wake notifications that usage-limit recovery reads + // outside the visible window; their history page shows them. if (currentItem !== undefined) { + const recovery = currentItem.type === "error" || currentItem.type === "notification"; if ( - (currentItem.type !== "run_interrupt_request" && + (!recovery && + currentItem.type !== "run_interrupt_request" && currentItem.type !== "approval_request" && currentItem.type !== "user_input_request") || !isOrchestrationV2TurnItemVisible({ @@ -137,7 +140,9 @@ export function mergeOlderHistoryIntoProjection( }) ) continue; - row = { ...row, item: currentItem }; + // Retained notifications are stubs without detail; the page row has the full item. + if (recovery) turnItemById.set(String(row.sourceItemId), row.item); + else row = { ...row, item: currentItem }; } } pageKeys.add(key); From 2c0aaadc07fe901b55b018ac6d64b0ea725577da Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 18 Nov 2026 08:21:31 +0000 Subject: [PATCH 3/8] test(server): drop bounded-window assertion covered by threadHistoryPaging Co-Authored-By: Claude Opus 5.5 --- .../orchestration-v2/ProjectionStore.test.ts | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 6062e0e3c911e..adffd4c6fc8b8 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -9434,19 +9434,13 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }); } - const snapshot = yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 3 }); - const bounded = buildBoundedThreadProjection({ - projection: snapshot.projection, - snapshotSequence: snapshot.snapshotSequence, - }).projection; - for (const projection of [snapshot.projection, bounded]) { - assert.equal( - usageLimitBlockedRun(projection.runs, projection.turnItems, null)?.id, - original.id, - ); - } + const { projection } = yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 3 }); + assert.equal( + usageLimitBlockedRun(projection.runs, projection.turnItems, null)?.id, + original.id, + ); assert.notInclude( - snapshot.projection.visibleTurnItems.map((row) => String(row.sourceItemId)), + projection.visibleTurnItems.map((row) => String(row.sourceItemId)), "limit-window:error", ); }), From fab462ef7e07951e242190b4f85dac2989797733 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 18 Nov 2026 08:46:50 +0000 Subject: [PATCH 4/8] fix(server): continuations and user runs never count as usage-limit wakes A manual Resume starts from a system notification, so a successful resume that finished before the reset was hidden as a no-op wake and the limit stayed on screen. Wake candidates are now one shared rule: completed, not a continuation, ended after the failure and before its reset. The SQL mirror, Stop, and bounded history all use it. usageLimitHoldsQueue compared the limited run with the unfiltered latest run, so a wake let Manual Resume drain the queue into a limited provider. Snapshot windows and bounded history now keep the first input of every loaded wake candidate, user messages included, instead of only notification starts of runs after the limited one by ordinal (capped at 200 runs). Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/Orchestrator.ts | 26 ++---- .../src/orchestration-v2/ProjectionStore.ts | 88 +++++++++---------- .../threadHistoryPaging.test.ts | 27 ++++++ .../orchestration-v2/threadHistoryPaging.ts | 44 ++++++---- .../src/state/threadExecution.test.ts | 3 + .../src/state/threadExecution.ts | 7 +- .../src/state/threadHistoryMerge.ts | 11 ++- .../shared/src/orchestrationV2ThreadError.ts | 56 +++++++++--- 8 files changed, 160 insertions(+), 102 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 68b546b531366..bbfc9a54cbbb2 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -9,9 +9,11 @@ import type { import { latestExecutedRun, latestRootProviderFailure, + latestRunIgnoringLimitedWakes, orchestrationV2WorktreePending, runRanAfter, usageLimitBlockedRun, + usageLimitWakeCandidates, } from "@t3tools/shared/orchestrationV2ThreadError"; import { threadIssueKeysEqual } from "@t3tools/contracts"; import { canonicalizeGitCafePullRequestUrl } from "@t3tools/source-control-definitions/registry"; @@ -1071,7 +1073,9 @@ function usageLimitHoldsQueue( latestSession?.lastError ?? null, ); if (limited === null) return false; - if (latestExecutedRun(projection.runs)?.id === limited.id) return true; + if (latestRunIgnoringLimitedWakes(projection.runs, projection.turnItems)?.id === limited.id) { + return true; + } if (now === undefined) return false; const resetAt = latestRootProviderFailure(limited, projection.turnItems)?.resetAt; return resetAt != null && Date.parse(resetAt) > DateTime.toEpochMillis(now); @@ -6898,10 +6902,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio source === undefined || (source.status !== "interrupted" && !(source.status === "failed" && limited?.class === "usage_limit")) || - ( - usageLimitBlockedRun(projection.runs, projection.turnItems, null) ?? - latestExecutedRun(projection.runs) - )?.id !== source.id || + latestRunIgnoringLimitedWakes(projection.runs, projection.turnItems)?.id !== source.id || projection.thread.archivedAt !== null || projection.thread.deletedAt !== null || projection.runtimeRequests.some((request) => request.status === "pending") @@ -12734,7 +12735,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio let { thread } = input.projection; let recovery = thread.limitRecovery; // Stop must also disarm a limited run that completed notification wakes hide. Only the - // latest failure's errors and the inputs of runs completed before its reset can matter. + // latest failure's errors and the inputs of its wake candidates can matter. const readItems = (filter: Parameters[2]) => projectionStore .getThreadRecords(thread.id, ["turnItems"], filter) @@ -12744,16 +12745,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio failed === null ? [] : (yield* readItems({ turnItemTypes: ["error"], turnItemRunIds: [failed.id] })).turnItems; - const resetAt = latestRootProviderFailure(failed, errors)?.resetAt; - const resetMs = resetAt == null ? Number.NaN : Date.parse(resetAt); - const wakeRunIds = runs - .filter( - (run) => - run.status === "completed" && - run.completedAt !== null && - DateTime.toEpochMillis(run.completedAt) <= resetMs, - ) - .map((run) => run.id); + const wakeRunIds = usageLimitWakeCandidates(runs, errors).map((run) => run.id); const inputs = wakeRunIds.length === 0 ? [] @@ -12762,7 +12754,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio turnItemRunIds: wakeRunIds, })).turnItems; const turnItems = [...errors, ...inputs]; - const latest = usageLimitBlockedRun(runs, turnItems, null) ?? latestExecutedRun(runs); + const latest = latestRunIgnoringLimitedWakes(runs, turnItems); if (latest?.status === "failed") { const failure = latestRootProviderFailure(latest, turnItems); if ( diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 5b94a8b2ddb05..1e355c5a2eb5e 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1241,7 +1241,6 @@ const shellIndexLineage = (row: { ...(row.side_chat === null ? {} : { sideChat: row.side_chat === 1 }), }); -const LIMIT_EVIDENCE_MAX_RUNS = 200; const encodeIdList = Schema.encodeSync(Schema.fromJsonString(Schema.Array(Schema.String))); // Empty filters still scan SQLite rows; short-circuit them, including turnItemTypes: []. @@ -4729,10 +4728,10 @@ export const layer: Layer.Layer = LIMIT 1 ) `; - // Mirrors the wake rule of usageLimitBlockedRun: a run that a notification started and - // that completed before the newest usage-limit failure resets did no work, so it must - // not replace that failure as the latest run. - const isLimitedWake = (alias: string) => { + // Mirrors usageLimitWakeCandidates: a completed run, not a continuation, that ended + // before the newest usage-limit failure resets. Runs that ended before the failure match + // too; they can never be the latest run, so selections come out the same. + const isWakeCandidate = (alias: string) => { const run = sql.literal(alias); return sql`( ${run}.status = 'completed' @@ -4741,17 +4740,28 @@ export const layer: Layer.Layer = AND julianday(${run}.completed_at) <= julianday( json_extract(latest_failure_error.payload_json, '$.failure.resetAt') ), 0 - ) AND ( - SELECT wake.type FROM orchestration_v2_projection_turn_items wake - WHERE wake.run_id = ${run}.run_id - AND wake.node_id IS NOT NULL - AND wake.node_id IS json_extract(${run}.payload_json, '$.rootNodeId') - AND wake.type IN ('user_message', 'notification') - ORDER BY wake.ordinal - LIMIT 1 - ) IS 'notification' + ) + AND json_extract(${run}.payload_json, '$.manualContinuationOfRunId') IS NULL + AND json_extract(${run}.payload_json, '$.usageLimitContinuationOfRunId') IS NULL + AND json_extract(${run}.payload_json, '$.restartContinuationOfRunId') IS NULL )`; }; + const firstRootInput = (alias: string, column: "type" | "turn_item_id") => { + const run = sql.literal(alias); + return sql`( + SELECT ${sql.literal(`wake.${column}`)} FROM orchestration_v2_projection_turn_items wake + WHERE wake.run_id = ${run}.run_id + AND wake.node_id IS NOT NULL + AND wake.node_id IS json_extract(${run}.payload_json, '$.rootNodeId') + AND wake.type IN ('user_message', 'notification') + ORDER BY wake.ordinal + LIMIT 1 + )`; + }; + // A wake candidate that a notification started did no work, so it must not replace the + // usage-limit failure as the latest run (usageLimitBlockedRun). + const isLimitedWake = (alias: string) => + sql`(${isWakeCandidate(alias)} AND ${firstRootInput(alias, "type")} IS 'notification')`; const getLimitRecoveryCandidates = Effect.fn("ProjectionStore.getLimitRecoveryCandidates")( function* (options: Parameters[0]) { @@ -7823,16 +7833,16 @@ export const layer: Layer.Layer = ), ); - // A window reads only the newest rows, so an older usage-limited run and the notification - // starts of the wakes after it can be missing. Clients need them (usageLimitBlockedRun) - // to resume it. They join runs and turnItems, never visibleTurnItems. + // A window reads only the newest rows, so an older usage-limited run and the first input + // of each wake candidate it loaded can be missing. Clients need them (usageLimitBlockedRun) + // to classify wakes and resume the run. They join runs and turnItems, never visibleTurnItems. const withLimitEvidence = Effect.fn("ProjectionStore.withLimitEvidence")(function* ( threadId: ThreadId, projection: OrchestrationV2ThreadProjection, ) { - const rows = yield* sql<{ readonly kind: string; readonly id: string }>` + const rows = yield* sql<{ readonly kind: string; readonly id: string | null }>` WITH blocked AS ( - SELECT r.run_id, r.ordinal, item.turn_item_id AS error_id + SELECT r.run_id, item.turn_item_id AS error_id FROM orchestration_v2_projection_threads t ${latestFailureJoins} INNER JOIN orchestration_v2_projection_runs r ON r.run_id = ( @@ -7856,45 +7866,31 @@ export const layer: Layer.Layer = ORDER BY error.updated_at DESC, error.ordinal DESC, error.turn_item_id DESC LIMIT 1 ) + -- The blocked run, when there is one, is the newest failed run. WHERE t.thread_id = ${threadId} - AND EXISTS ( - SELECT 1 FROM orchestration_v2_projection_turn_items limited - WHERE limited.thread_id = t.thread_id - AND limited.type = 'error' AND limited.status = 'failed' - AND json_extract(limited.payload_json, '$.failure.class') = 'usage_limit' - ) + AND json_extract(latest_failure_error.payload_json, '$.failure.class') = 'usage_limit' AND json_extract(item.payload_json, '$.failure.class') = 'usage_limit' - ), later AS ( - -- Runs omitted here are not loaded at all, which leaves the result unchanged. - SELECT run.run_id, run.payload_json - FROM orchestration_v2_projection_runs run, blocked - WHERE run.thread_id = ${threadId} AND run.ordinal > blocked.ordinal - ORDER BY run.ordinal DESC - LIMIT ${LIMIT_EVIDENCE_MAX_RUNS} ) SELECT 'run' AS kind, run_id AS id FROM blocked UNION ALL SELECT 'item', error_id FROM blocked - UNION ALL SELECT 'run', run_id FROM later UNION ALL - SELECT 'item', start.turn_item_id - FROM later - CROSS JOIN orchestration_v2_projection_turn_items start ON start.turn_item_id = ( - SELECT first_input.turn_item_id FROM orchestration_v2_projection_turn_items first_input - WHERE first_input.run_id = later.run_id - AND first_input.node_id IS json_extract(later.payload_json, '$.rootNodeId') - AND first_input.type IN ('user_message', 'notification') - ORDER BY first_input.ordinal - LIMIT 1 - ) - WHERE start.type = 'notification' + SELECT 'item', ${firstRootInput("run", "turn_item_id")} + FROM orchestration_v2_projection_threads t + ${latestFailureJoins} + INNER JOIN orchestration_v2_projection_runs run ON run.thread_id = t.thread_id + WHERE t.thread_id = ${threadId} + AND run.run_id IN (SELECT value FROM json_each(${encodeIdList( + projection.runs.map((run) => run.id), + )})) + AND ${isWakeCandidate("run")} `; const loadedRunIds = new Set(projection.runs.map((run) => run.id)); const loadedItemIds = new Set(projection.turnItems.map((item) => item.id)); const runIds = rows.flatMap((row) => - row.kind === "run" && !loadedRunIds.has(row.id) ? [row.id] : [], + row.kind === "run" && row.id !== null && !loadedRunIds.has(row.id) ? [row.id] : [], ); const itemIds = rows.flatMap((row) => - row.kind === "item" && !loadedItemIds.has(row.id) ? [row.id] : [], + row.kind === "item" && row.id !== null && !loadedItemIds.has(row.id) ? [row.id] : [], ); if (runIds.length === 0 && itemIds.length === 0) return projection; const runRows = yield* sql` diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts index 505d5a61b43e3..31471e051d749 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts @@ -791,6 +791,33 @@ describe("threadHistoryPaging", () => { expect( usageLimitBlockedRun(tight.projection.runs, tight.projection.turnItems, null), ).toBeNull(); + + // A user message started the run after the limit, so it did work even though only its + // later notification and the limit error are in the window. + const userStart = { + ...limitError, + id: TurnItemId.make("user-start"), + type: "user_message", + runId: wakes[5]!.run.id, + nodeId: wakes[5]!.run.rootNodeId, + ordinal: 0, + text: "a real request", + attachments: [], + } as unknown as OrchestrationV2TurnItem; + const userRows = [ + { ...makeRow(0), position: -1, sourceItemId: userStart.id, item: userStart }, + rows[0]!, + rows[6]!, + ]; + const userRun = buildBoundedThreadProjection({ + projection: { + ...makeProjection(userRows), + runs: [limited, wakes[5]!.run], + } as unknown as ReturnType, + snapshotSequence: 1, + policy: { maxItems: 2, maxEncodedBytes: 10_000_000 }, + }).projection; + expect(usageLimitBlockedRun(userRun.runs, userRun.turnItems, null)).toBeNull(); }); it("charges local row.item duplication when measuring bounded timeline bytes", () => { diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.ts index db515123773ab..0aeb2161f7bd0 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.ts @@ -5,7 +5,10 @@ import type { ThreadId, TurnItemId, } from "@t3tools/contracts"; -import { usageLimitBlockedRun } from "@t3tools/shared/orchestrationV2ThreadError"; +import { + usageLimitBlockedRun, + usageLimitWakeCandidates, +} from "@t3tools/shared/orchestrationV2ThreadError"; import * as Schema from "effect/Schema"; /** @@ -559,32 +562,34 @@ function isRetainedDependency( ); } -// usageLimitBlockedRun needs the limited run's root error and the notification that started each -// later run, and a long chain of wakes pushes both out of the window. Starts are kept as stubs -// without their unbounded detail, and dropped past a quarter of the budget (clients then show +// usageLimitBlockedRun needs the limited run's root error and the first input of each wake +// candidate, and a long chain of wakes pushes both out of the window. Inputs are kept as stubs +// without their unbounded text, and dropped past a quarter of the budget (clients then show // the plain latest run). function limitRecoveryItems( projection: Pick, maxEncodedBytes: number, ): ReadonlyMap { const items = new Map(); - const blocked = usageLimitBlockedRun(projection.runs, projection.turnItems, null); - if (blocked === null) return items; - const laterRoots = new Set(); - for (const run of projection.runs) { - if (run.ordinal > blocked.ordinal && run.rootNodeId !== null) { - laterRoots.add(String(run.rootNodeId)); - } + const candidateRoots = new Set(); + for (const run of usageLimitWakeCandidates(projection.runs, projection.turnItems)) { + if (run.rootNodeId !== null) candidateRoots.add(String(run.rootNodeId)); } - if (laterRoots.size === 0) return items; + if (candidateRoots.size === 0) return items; + const blocked = usageLimitBlockedRun(projection.runs, projection.turnItems, null); const firstInputs = new Map(); for (const item of projection.turnItems) { - if (item.type === "error" && item.runId === blocked.id && item.nodeId === blocked.rootNodeId) { + if ( + item.type === "error" && + blocked !== null && + item.runId === blocked.id && + item.nodeId === blocked.rootNodeId + ) { items.set(String(item.id), item); } else if ( (item.type === "user_message" || item.type === "notification") && item.nodeId !== null && - laterRoots.has(String(item.nodeId)) + candidateRoots.has(String(item.nodeId)) ) { const first = firstInputs.get(String(item.nodeId)); if (first === undefined || item.ordinal < first.ordinal) { @@ -595,9 +600,14 @@ function limitRecoveryItems( let startBytes = 0; const starts: Array = []; for (const item of firstInputs.values()) { - if (item.type !== "notification") continue; - const { detail: _detail, ...rest } = item; - const stub = { ...rest, title: null, summary: item.summary.slice(0, 120).trimEnd() }; + let stub: OrchestrationV2TurnItem; + if (item.type === "notification") { + const { detail: _detail, ...rest } = item; + stub = { ...rest, title: null, summary: item.summary.slice(0, 120).trimEnd() }; + } else if (item.type === "user_message") { + const { context: _context, ...rest } = item; + stub = { ...rest, text: item.text.slice(0, 120), attachments: [] }; + } else continue; startBytes += bytesOfJson(stub); starts.push(stub); } diff --git a/packages/client-runtime/src/state/threadExecution.test.ts b/packages/client-runtime/src/state/threadExecution.test.ts index 50c9d4c12417e..bd982d396efbf 100644 --- a/packages/client-runtime/src/state/threadExecution.test.ts +++ b/packages/client-runtime/src/state/threadExecution.test.ts @@ -301,6 +301,9 @@ describe("thread execution presentation", () => { expect(deriveResumableRunId(hidden, deriveThreadRuntime(hidden))).toBe(failed.id); const afterReset = { ...wake, completedAt: DateTime.makeUnsafe("2026-07-28T11:30:00.000Z") }; expect(usageLimitBlockedRun([failed, afterReset], hidden.turnItems, null)).toBeNull(); + // Manual Resume also starts from a notification; once it completes, it did the work. + const resumed = { ...wake, manualContinuationOfRunId: failed.id }; + expect(usageLimitBlockedRun([failed, resumed], hidden.turnItems, null)).toBeNull(); }); it("keeps live activity attached to an executing run when a newer run is queued", () => { diff --git a/packages/client-runtime/src/state/threadExecution.ts b/packages/client-runtime/src/state/threadExecution.ts index 15797886456c7..98df9f0dd2d4a 100644 --- a/packages/client-runtime/src/state/threadExecution.ts +++ b/packages/client-runtime/src/state/threadExecution.ts @@ -1,9 +1,8 @@ import { - latestExecutedRun, latestRootProviderFailure, + latestRunIgnoringLimitedWakes, latestUnheldRun, threadErrorSummary, - usageLimitBlockedRun, usageLimitRunPresentedAsLatest, } from "@t3tools/shared/orchestrationV2ThreadError"; import { @@ -321,9 +320,7 @@ export function deriveResumableRunId( projection: Pick, runtime: Pick | null, ): RunId | null { - const run = - usageLimitBlockedRun(projection.runs, projection.turnItems, null) ?? - latestExecutedRun(projection.runs); + const run = latestRunIgnoringLimitedWakes(projection.runs, projection.turnItems); if (run?.status === "interrupted") return run.id; return run?.status === "failed" && runtime?.lastErrorClass === "usage_limit" && diff --git a/packages/client-runtime/src/state/threadHistoryMerge.ts b/packages/client-runtime/src/state/threadHistoryMerge.ts index a27f90407ee44..b2e601b672b62 100644 --- a/packages/client-runtime/src/state/threadHistoryMerge.ts +++ b/packages/client-runtime/src/state/threadHistoryMerge.ts @@ -123,10 +123,13 @@ export function mergeOlderHistoryIntoProjection( ) { const currentItem = turnItemById.get(String(row.sourceItemId)); // Hidden live items stay hidden. Snapshots also retain interrupt requests, pending - // request cards, and the error and wake notifications that usage-limit recovery reads - // outside the visible window; their history page shows them. + // request cards, and the error and run inputs that usage-limit recovery reads outside + // the visible window; their history page shows them. if (currentItem !== undefined) { - const recovery = currentItem.type === "error" || currentItem.type === "notification"; + const recovery = + currentItem.type === "error" || + currentItem.type === "notification" || + currentItem.type === "user_message"; if ( (!recovery && currentItem.type !== "run_interrupt_request" && @@ -140,7 +143,7 @@ export function mergeOlderHistoryIntoProjection( }) ) continue; - // Retained notifications are stubs without detail; the page row has the full item. + // Retained inputs are stubs without their full text; the page row has the full item. if (recovery) turnItemById.set(String(row.sourceItemId), row.item); else row = { ...row, item: currentItem }; } diff --git a/packages/shared/src/orchestrationV2ThreadError.ts b/packages/shared/src/orchestrationV2ThreadError.ts index 2440d8eb54f33..859224bc5c7cb 100644 --- a/packages/shared/src/orchestrationV2ThreadError.ts +++ b/packages/shared/src/orchestrationV2ThreadError.ts @@ -102,18 +102,42 @@ export function runRanAfter( /** * Background subagents that die on a usage limit each start a root run from a notification. * Those runs finish within milliseconds, before the limit resets, and have no work to show. - * They must not replace the limited run, or the thread is never armed or resumed. - * Items of types other than user_message, notification and error may be omitted. + * These are the runs that may be such wakes: completed after the latest usage-limit failure + * and before its reset. A continuation the user or recovery started is never a wake. */ -function runsWithoutLimitedWakes( +export function usageLimitWakeCandidates( runs: ReadonlyArray, turnItems: ReadonlyArray, ): ReadonlyArray { const failed = latestExecutedRun(runs.filter((run) => run.status === "failed")); const failure = latestRootProviderFailure(failed, turnItems); - if (failure?.class !== "usage_limit" || failure.resetAt == null) return runs; + if (failed === null || failure?.class !== "usage_limit" || failure.resetAt == null) return []; const resetMs = Date.parse(failure.resetAt); - if (!Number.isFinite(resetMs)) return runs; + if (!Number.isFinite(resetMs)) return []; + return runs.filter( + (run) => + run.status === "completed" && + run.completedAt !== null && + run.rootNodeId !== null && + DateTime.toEpochMillis(run.completedAt) <= resetMs && + runRanAfter(run, failed) && + run.manualContinuationOfRunId === undefined && + run.usageLimitContinuationOfRunId === undefined && + run.restartContinuationOfRunId === undefined, + ); +} + +/** + * Drops the wake candidates a notification started, so they cannot replace the limited run + * and leave the thread never armed or resumed. Items of types other than user_message, + * notification and error may be omitted. + */ +function runsWithoutLimitedWakes( + runs: ReadonlyArray, + turnItems: ReadonlyArray, +): ReadonlyArray { + const candidates = usageLimitWakeCandidates(runs, turnItems); + if (candidates.length === 0) return runs; const firstInputs = new Map(); for (const item of turnItems) { if (item.nodeId == null || item.ordinal == null) continue; @@ -123,14 +147,20 @@ function runsWithoutLimitedWakes( firstInputs.set(item.nodeId, { type: item.type, ordinal: item.ordinal }); } } - return runs.filter( - (run) => - run.status !== "completed" || - run.completedAt === null || - DateTime.toEpochMillis(run.completedAt) > resetMs || - run.rootNodeId === null || - firstInputs.get(run.rootNodeId)?.type !== "notification", + const wakes = new Set( + candidates.filter( + (run) => run.rootNodeId !== null && firstInputs.get(run.rootNodeId)?.type === "notification", + ), ); + return wakes.size === 0 ? runs : runs.filter((run) => !wakes.has(run)); +} + +/** The latest executed run, not counting notification wakes that did no work before a reset. */ +export function latestRunIgnoringLimitedWakes( + runs: ReadonlyArray, + turnItems: ReadonlyArray, +): OrchestrationV2Run | null { + return latestExecutedRun(runsWithoutLimitedWakes(runs, turnItems)); } export function usageLimitBlockedRun( @@ -138,7 +168,7 @@ export function usageLimitBlockedRun( turnItems: ReadonlyArray, sessionError: string | null, ): OrchestrationV2Run | null { - const executed = latestExecutedRun(runsWithoutLimitedWakes(runs, turnItems)); + const executed = latestRunIgnoringLimitedWakes(runs, turnItems); if (executed?.status !== "failed") return null; const summary = threadErrorSummary(latestRootProviderFailure(executed, turnItems), sessionError); return summary.lastErrorClass === "usage_limit" ? executed : null; From 99bd045e3c89e06c5638fabab25315d5546c1727 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 18 Nov 2026 09:06:41 +0000 Subject: [PATCH 5/8] perf(server): read wake inputs only for usage-limit wake candidates Every command read, queue start and recovery arm loaded all of a thread's notifications, each with its full detail, to classify wakes. They now read the user message and notification inputs of the wake candidates only, which exist just between a usage-limit failure and its reset. Bounded history no longer keeps a conversation message because a recovery stub names it; that message bypassed the size budget. Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/Orchestrator.ts | 41 ++++++++++++++++--- .../threadHistoryPaging.test.ts | 4 ++ .../orchestration-v2/threadHistoryPaging.ts | 6 ++- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index bbfc9a54cbbb2..756fc7def634e 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -1713,6 +1713,34 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio return false; }); + // Only wake candidates' inputs decide usage-limit recovery, and a dispatcher thread keeps + // thousands of notifications, so they are read for those runs alone. + const withWakeInputs = < + R extends { + readonly runs: ReadonlyArray; + readonly turnItems: ReadonlyArray; + }, + >( + threadId: ThreadId, + records: R, + ) => { + const runIds = usageLimitWakeCandidates(records.runs, records.turnItems).map((run) => run.id); + if (runIds.length === 0) return Effect.succeed(records); + const loaded = new Set(records.turnItems.map((item) => item.id)); + return projectionStore + .getThreadRecords(threadId, ["turnItems"], { + turnItemTypes: ["user_message", "notification"], + turnItemRunIds: runIds, + }) + .pipe( + Effect.map(({ turnItems }): R => ({ + ...records, + turnItems: [...records.turnItems, ...turnItems.filter((item) => !loaded.has(item.id))], + })), + Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })), + ); + }; + // Runs in every command transaction; avoid reading thousands of finished leaf nodes. const readCommandProjection = ( threadId: ThreadId, @@ -1736,7 +1764,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio "contextTransfers", ], { - turnItemTypes: ["user_message", "error", "notification"], + turnItemTypes: ["user_message", "error"], messageRoles: ["user"], omitSettledLeafNodes: true, // Commands here read only undelivered or active tasks; a dispatcher thread keeps @@ -1750,6 +1778,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }, ) .pipe( + Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })), + Effect.flatMap((records) => withWakeInputs(threadId, records)), Effect.map((records): OrchestrationV2ThreadProjection => ({ ...records, checkpoints: [], @@ -1758,7 +1788,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio visibleTurnItems: [], updatedAt: records.thread.updatedAt, })), - Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })), ); // Command reads omit settled tasks, but a completion notice counts its whole cohort. @@ -4533,8 +4562,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const projection = yield* loadProjectionForCommand( command, ["runs", "runtimeRequests", "turnItems"], - { turnItemTypes: ["user_message", "error", "notification"] }, - ); + { turnItemTypes: ["error"] }, + ).pipe(Effect.flatMap((records) => withWakeInputs(command.threadId, records))); const run = usageLimitBlockedRun(projection.runs, projection.turnItems, null); const failure = latestRootProviderFailure(run, projection.turnItems); const resetMs = Date.parse(command.limitRecovery.resetAt); @@ -15303,8 +15332,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const projection = yield* loadProjectionForCommand( command, ["runs", "turnItems", "providerSessions", "messages"], - { turnItemTypes: ["user_message", "error", "notification"], messageRoles: ["user"] }, - ); + { turnItemTypes: ["error"], messageRoles: ["user"] }, + ).pipe(Effect.flatMap((records) => withWakeInputs(command.threadId, records))); if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) { return yield* new OrchestratorDispatchError({ commandId: command.commandId, diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts index 31471e051d749..47e1bcc7e6317 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts @@ -801,6 +801,7 @@ describe("threadHistoryPaging", () => { runId: wakes[5]!.run.id, nodeId: wakes[5]!.run.rootNodeId, ordinal: 0, + messageId: "message-user-start", text: "a real request", attachments: [], } as unknown as OrchestrationV2TurnItem; @@ -813,11 +814,14 @@ describe("threadHistoryPaging", () => { projection: { ...makeProjection(userRows), runs: [limited, wakes[5]!.run], + messages: [{ id: "message-user-start", runId: null, text: "p".repeat(50_000) }], } as unknown as ReturnType, snapshotSequence: 1, policy: { maxItems: 2, maxEncodedBytes: 10_000_000 }, }).projection; expect(usageLimitBlockedRun(userRun.runs, userRun.turnItems, null)).toBeNull(); + // The stub classifies the run; it must not pull the full prompt past the budget. + expect(userRun.messages).toEqual([]); }); it("charges local row.item duplication when measuring bounded timeline bytes", () => { diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.ts index 0aeb2161f7bd0..3d07e981aa369 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.ts @@ -840,7 +840,11 @@ export function buildBoundedThreadProjection(input: { ...controlProjection, plans, contextHandoffs, - messages: messagesForBoundedProjection(controlProjection, turnItems), + // Recovery stubs only classify runs; their full messages would bypass the budget. + messages: messagesForBoundedProjection( + controlProjection, + turnItems.filter((item) => limitItems.get(String(item.id)) !== item), + ), turnItems, visibleTurnItems, }; From 536d156a193fed01d11b943421b38e09134a1eaf Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 18 Nov 2026 09:29:53 +0000 Subject: [PATCH 6/8] fix(server): a run on another provider is never a usage-limit wake Wake candidates were not scoped to the limited run's provider. After a Claude limit, a notification-started Codex run that completed before the Claude reset was dropped as a wake, so the stale Claude failure came back as the latest run and Resume target. The shared rule and its SQL mirror now require the limited run's provider. Co-Authored-By: Claude Opus 5.5 --- .../orchestration-v2/ProjectionStore.test.ts | 20 +++++++++++-------- .../src/orchestration-v2/ProjectionStore.ts | 8 +++++--- .../src/state/threadExecution.test.ts | 3 +++ .../shared/src/orchestrationV2ThreadError.ts | 8 ++++---- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index adffd4c6fc8b8..48a3efc1b6cc9 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -8618,23 +8618,27 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { runId, ); }); - const setWakeCompletedAt = (completedAt: DateTime.Utc) => + const updateWake = (tag: string, patch: Partial<(typeof wakeProjection.runs)[number]>) => store.apply({ - id: EventId.make(`event:limit-shell:wake-at-${DateTime.formatIso(completedAt)}`), + id: EventId.make(`event:limit-shell:wake-${tag}`), type: "run.updated", threadId, runId: wakeRunId, nodeId: wakeNodeId, driver, occurredAt: now, - payload: { - ...wakeProjection.runs.find((run) => run.id === wakeRunId)!, - completedAt, - }, + payload: { ...wakeProjection.runs.find((run) => run.id === wakeRunId)!, ...patch }, }); - yield* setWakeCompletedAt(DateTime.makeUnsafe("2099-01-02T00:00:00.000Z")); + yield* updateWake("after-reset", { + completedAt: DateTime.makeUnsafe("2099-01-02T00:00:00.000Z"), + }); + yield* assertLatest(wakeRunId); + // A wake on another provider ran there, so it is the latest run. + yield* updateWake("other-provider", { + providerInstanceId: ProviderInstanceId.make("other-provider"), + }); yield* assertLatest(wakeRunId); - yield* setWakeCompletedAt(DateTime.add(now, { minutes: 1 })); + yield* updateWake("before-reset", {}); yield* assertLatest(original.id); yield* store.apply({ id: EventId.make("event:limit-shell:wake-user-start"), diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 1e355c5a2eb5e..f15e0845a1ef6 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -4728,13 +4728,15 @@ export const layer: Layer.Layer = LIMIT 1 ) `; - // Mirrors usageLimitWakeCandidates: a completed run, not a continuation, that ended - // before the newest usage-limit failure resets. Runs that ended before the failure match - // too; they can never be the latest run, so selections come out the same. + // Mirrors usageLimitWakeCandidates: a completed run on the limited provider, not a + // continuation, that ended before the newest usage-limit failure resets. Runs that ended + // before the failure match too; they can never be latest, so selections come out the same. const isWakeCandidate = (alias: string) => { const run = sql.literal(alias); return sql`( ${run}.status = 'completed' + AND json_extract(${run}.payload_json, '$.providerInstanceId') + IS json_extract(latest_failure.payload_json, '$.providerInstanceId') AND COALESCE( json_extract(latest_failure_error.payload_json, '$.failure.class') = 'usage_limit' AND julianday(${run}.completed_at) <= julianday( diff --git a/packages/client-runtime/src/state/threadExecution.test.ts b/packages/client-runtime/src/state/threadExecution.test.ts index bd982d396efbf..ec7615fc5bf4f 100644 --- a/packages/client-runtime/src/state/threadExecution.test.ts +++ b/packages/client-runtime/src/state/threadExecution.test.ts @@ -304,6 +304,9 @@ describe("thread execution presentation", () => { // Manual Resume also starts from a notification; once it completes, it did the work. const resumed = { ...wake, manualContinuationOfRunId: failed.id }; expect(usageLimitBlockedRun([failed, resumed], hidden.turnItems, null)).toBeNull(); + // A wake on another provider ran there; the limited provider's failure stays behind it. + const otherProvider = { ...wake, providerInstanceId: ProviderInstanceId.make("other") }; + expect(usageLimitBlockedRun([failed, otherProvider], hidden.turnItems, null)).toBeNull(); }); it("keeps live activity attached to an executing run when a newer run is queued", () => { diff --git a/packages/shared/src/orchestrationV2ThreadError.ts b/packages/shared/src/orchestrationV2ThreadError.ts index 859224bc5c7cb..7c7092e6cc1e2 100644 --- a/packages/shared/src/orchestrationV2ThreadError.ts +++ b/packages/shared/src/orchestrationV2ThreadError.ts @@ -100,10 +100,9 @@ export function runRanAfter( } /** - * Background subagents that die on a usage limit each start a root run from a notification. - * Those runs finish within milliseconds, before the limit resets, and have no work to show. - * These are the runs that may be such wakes: completed after the latest usage-limit failure - * and before its reset. A continuation the user or recovery started is never a wake. + * Subagents that die on a usage limit start notification runs that finish before the reset + * with no work to show. These candidates ran on the limited provider after its latest failure + * and before the reset; continuations and other providers' runs are never wakes. */ export function usageLimitWakeCandidates( runs: ReadonlyArray, @@ -119,6 +118,7 @@ export function usageLimitWakeCandidates( run.status === "completed" && run.completedAt !== null && run.rootNodeId !== null && + run.providerInstanceId === failed.providerInstanceId && DateTime.toEpochMillis(run.completedAt) <= resetMs && runRanAfter(run, failed) && run.manualContinuationOfRunId === undefined && From 6d9f8c65049ff2cd53316e4d00045d8849f1de16 Mon Sep 17 00:00:00 2001 From: Nick Spreen <12631527+yspreen@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:33:01 +0900 Subject: [PATCH 7/8] fix(server): bounded recovery evidence stays compact and matches the selectors - The wake SQL mirror now applies the same "ran after the failure" bound as usageLimitWakeCandidates, so evidence reads skip inputs of older runs. - Evidence reads drop notification detail and user-message text in SQL instead of decoding them first. - The shell's blocked-failure lookup skips silent compaction runs like the in-memory shell. - Bounded snapshots keep only the latest root error, and recovery-only stubs no longer pull full conversation messages into the response. - Paging back keeps a live update that is newer than the page row. - Antigravity reset timestamps are normalized to millisecond precision. --- .../orchestration-v2/ProjectionStore.test.ts | 87 +++++++++++++++++++ .../src/orchestration-v2/ProjectionStore.ts | 21 ++++- .../threadHistoryPaging.test.ts | 10 ++- .../orchestration-v2/threadHistoryPaging.ts | 11 +-- .../provider/acp/AntigravityTurnOutcome.ts | 3 +- .../src/state/threadHistoryMerge.test.ts | 11 ++- .../src/state/threadHistoryMerge.ts | 7 +- 7 files changed, 134 insertions(+), 16 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 48a3efc1b6cc9..8500b5278324e 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -8609,6 +8609,57 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { (yield* store.getShellSnapshot()).threads.find((row) => row.id === threadId)!.latestRunId, original.id, ); + // A silent compaction that completes after the limit is not work either, in both shells. + const compactRunId = RunId.make("run:limit-shell:compaction"); + const compactMessageId = MessageId.make("message:limit-shell:compaction"); + yield* store.apply({ + id: EventId.make("event:limit-shell:compaction-message"), + type: "message.updated", + threadId, + occurredAt: now, + payload: { + createdBy: "agent", + creationSource: "server", + id: compactMessageId, + threadId, + runId: compactRunId, + nodeId: null, + role: "user", + text: "/compact", + attachments: [], + streaming: false, + createdAt: now, + updatedAt: now, + }, + }); + yield* store.apply({ + id: EventId.make("event:limit-shell:compaction"), + type: "run.created", + threadId, + runId: compactRunId, + nodeId: NodeId.make("node:limit-shell:compaction"), + driver, + providerInstanceId, + occurredAt: now, + payload: { + ...original, + id: compactRunId, + ordinal: original.ordinal + 5, + rootNodeId: NodeId.make("node:limit-shell:compaction"), + userMessageId: compactMessageId, + status: "completed", + startedAt: DateTime.add(now, { minutes: 2 }), + completedAt: DateTime.add(now, { minutes: 2 }), + }, + }); + const compactProjection = yield* store.getThreadProjection(threadId); + const compactShells = [ + ProjectionStore.threadShellFromProjection(compactProjection), + (yield* store.getShellSnapshot()).threads.find((row) => row.id === threadId)!, + ]; + for (const shell of compactShells) assert.equal(shell.lastErrorClass, "usage_limit"); + const compactSql = yield* SqlClient.SqlClient; + yield* compactSql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${compactRunId}`; // Near misses: a wake completed after the reset, or a run a user message started, did work. const assertLatest = Effect.fnUntraced(function* (runId: RunId) { const projection = yield* store.getThreadProjection(threadId); @@ -9434,6 +9485,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { source: { kind: "delegated_task", taskIds: [] }, outcome: "failed", summary: "Subagent failed", + detail: "d".repeat(5_000), }, }); } @@ -9443,6 +9495,41 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { usageLimitBlockedRun(projection.runs, projection.turnItems, null)?.id, original.id, ); + // A one-row window loads the last wake's run but not its notification. The notification + // is read without its detail. + const lastWake = RunId.make("run:limit-window:wake-5"); + const lastNode = NodeId.make("node:limit-window:wake-5"); + yield* store.apply({ + id: EventId.make("event:limit-window:wake-5:command"), + type: "turn-item.updated", + threadId, + runId: lastWake, + nodeId: lastNode, + driver, + occurredAt: now, + payload: { + ...base, + id: TurnItemId.make("limit-window:wake-5:command"), + runId: lastWake, + nodeId: lastNode, + parentItemId: null, + ordinal: 100, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "command_execution", + input: "true", + output: "", + exitCode: 0, + }, + }); + const narrow = (yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 1 })).projection; + const notification = narrow.turnItems.find((item) => item.id === "limit-window:wake-5"); + assert.equal(notification?.type, "notification"); + assert.notProperty(notification!, "detail"); + assert.equal(usageLimitBlockedRun(narrow.runs, narrow.turnItems, null)?.id, original.id); assert.notInclude( projection.visibleTurnItems.map((row) => String(row.sourceItemId)), "limit-window:error", diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index f15e0845a1ef6..6ccf76f3f5e96 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -4729,8 +4729,8 @@ export const layer: Layer.Layer = ) `; // Mirrors usageLimitWakeCandidates: a completed run on the limited provider, not a - // continuation, that ended before the newest usage-limit failure resets. Runs that ended - // before the failure match too; they can never be latest, so selections come out the same. + // continuation, that ended after the newest usage-limit failure (runRanAfter) and before + // it resets. const isWakeCandidate = (alias: string) => { const run = sql.literal(alias); return sql`( @@ -4743,6 +4743,11 @@ export const layer: Layer.Layer = json_extract(latest_failure_error.payload_json, '$.failure.resetAt') ), 0 ) + AND ( + ${run}.completed_at > latest_failure.completed_at + OR (${run}.completed_at = latest_failure.completed_at + AND ${run}.ordinal > latest_failure.ordinal) + ) AND json_extract(${run}.payload_json, '$.manualContinuationOfRunId') IS NULL AND json_extract(${run}.payload_json, '$.usageLimitContinuationOfRunId') IS NULL AND json_extract(${run}.payload_json, '$.restartContinuationOfRunId') IS NULL @@ -7900,8 +7905,17 @@ export const layer: Layer.Layer = WHERE thread_id = ${threadId} AND run_id IN (SELECT value FROM json_each(${encodeIdList(runIds)})) `; + // Recovery reads only an input's identity, root and ordinal, so its text never leaves SQL. const itemRows = yield* sql` - SELECT payload_json FROM orchestration_v2_projection_turn_items + SELECT CASE type + WHEN 'notification' THEN json_remove(payload_json, '$.detail') + WHEN 'user_message' THEN json_remove( + json_set(payload_json, '$.text', substr(json_extract(payload_json, '$.text'), 1, 120)), + '$.context' + ) + ELSE payload_json + END AS payload_json + FROM orchestration_v2_projection_turn_items WHERE thread_id = ${threadId} AND turn_item_id IN (SELECT value FROM json_each(${encodeIdList(itemIds)})) `; @@ -8269,6 +8283,7 @@ export const layer: Layer.Layer = candidate.status = 'cancelled' AND json_extract(candidate.payload_json, '$.startedAt') IS NULL ) + AND NOT ${isSilentShellRun("candidate")} AND NOT ${isLimitedWake("candidate")} -- latestExecutedRun: the run that ended last (runRanAfter). ORDER BY candidate.completed_at IS NULL DESC, candidate.completed_at DESC, diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts index 47e1bcc7e6317..eb2766dd51586 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts @@ -782,15 +782,21 @@ describe("threadHistoryPaging", () => { expect(retained.length).toBeGreaterThan(0); expect(JSON.stringify(retained).length).toBeLessThan(10_000); - // Evidence past a quarter of the budget is dropped, so the snapshot stays within it. + // Evidence past a quarter of the budget is dropped, so the snapshot stays within it. The + // visible rows carry no detail here, so they fit the budget on their own. + const slimRows = rows.map((row) => ({ ...row, item: { ...row.item, detail: "d" } })); const tight = buildBoundedThreadProjection({ - projection: full, + projection: { + ...makeProjection(slimRows as unknown as typeof rows), + runs: full.runs, + } as unknown as ReturnType, snapshotSequence: 1, policy: { maxItems: 2, maxEncodedBytes: 8_000 }, }); expect( usageLimitBlockedRun(tight.projection.runs, tight.projection.turnItems, null), ).toBeNull(); + expect(boundedTimelineEncodedBytes(tight.projection)).toBeLessThanOrEqual(8_000); // A user message started the run after the limit, so it did work even though only its // later notification and the limit error are in the window. diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.ts index 3d07e981aa369..e17949618c255 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.ts @@ -6,6 +6,7 @@ import type { TurnItemId, } from "@t3tools/contracts"; import { + latestRootProviderFailure, usageLimitBlockedRun, usageLimitWakeCandidates, } from "@t3tools/shared/orchestrationV2ThreadError"; @@ -562,7 +563,7 @@ function isRetainedDependency( ); } -// usageLimitBlockedRun needs the limited run's root error and the first input of each wake +// usageLimitBlockedRun needs the limited run's latest root error and the first input of each wake // candidate, and a long chain of wakes pushes both out of the window. Inputs are kept as stubs // without their unbounded text, and dropped past a quarter of the budget (clients then show // the plain latest run). @@ -577,14 +578,10 @@ function limitRecoveryItems( } if (candidateRoots.size === 0) return items; const blocked = usageLimitBlockedRun(projection.runs, projection.turnItems, null); + const failure = latestRootProviderFailure(blocked, projection.turnItems); const firstInputs = new Map(); for (const item of projection.turnItems) { - if ( - item.type === "error" && - blocked !== null && - item.runId === blocked.id && - item.nodeId === blocked.rootNodeId - ) { + if (item.type === "error" && failure !== null && item.failure === failure) { items.set(String(item.id), item); } else if ( (item.type === "user_message" || item.type === "notification") && diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts index 41e25f2e417e1..9e4d7eb39e370 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts @@ -92,7 +92,8 @@ const lastStepAfter = function resetAtFrom(text: string, now: DateTime.Utc): string | undefined { // makeProviderFailure drops a timestamp that does not parse, so fall back to the delay. const timestamp = RESET_TIMESTAMP.exec(text)?.[1]; - if (timestamp !== undefined && Option.isSome(DateTime.make(timestamp))) return timestamp; + const parsed = timestamp === undefined ? Option.none() : DateTime.make(timestamp); + if (Option.isSome(parsed)) return DateTime.formatIso(parsed.value); const delay = RESET_DELAY.exec(text)?.[1] ?? RETRY_DELAY.exec(text)?.[1]; if (delay === undefined) return undefined; let delayMs = 0; diff --git a/packages/client-runtime/src/state/threadHistoryMerge.test.ts b/packages/client-runtime/src/state/threadHistoryMerge.test.ts index 59133a277d1e8..e3da7ee9ebf28 100644 --- a/packages/client-runtime/src/state/threadHistoryMerge.test.ts +++ b/packages/client-runtime/src/state/threadHistoryMerge.test.ts @@ -255,7 +255,7 @@ describe("threadHistoryMerge", () => { ...v2Projection, turnItems: [error, stub, recent.item], visibleTurnItems: [{ ...recent, position: 0 }], - } as never; + } as unknown as typeof v2Projection; const merged = mergeOlderHistoryIntoProjection(coldOpen, [ { ...row(0), item: error }, @@ -268,6 +268,15 @@ describe("threadHistoryMerge", () => { recent.item, ]); expect(merged.turnItems).toEqual([error, notification, recent.item]); + + // A live update that arrived after the page was fetched stays. + const live = { ...error, updatedAt: DateTime.makeUnsafe("2026-06-20T00:00:05.000Z") }; + const kept = mergeOlderHistoryIntoProjection( + { ...coldOpen, turnItems: [live, recent.item] } as never, + [{ ...row(0), item: error }] as never, + ); + expect(kept.turnItems).toContainEqual(live); + expect(kept.visibleTurnItems.map((entry) => entry.item)).toContainEqual(live); }); it("retains merged older rows when a live turn-item update arrives", () => { diff --git a/packages/client-runtime/src/state/threadHistoryMerge.ts b/packages/client-runtime/src/state/threadHistoryMerge.ts index b2e601b672b62..778527acc4200 100644 --- a/packages/client-runtime/src/state/threadHistoryMerge.ts +++ b/packages/client-runtime/src/state/threadHistoryMerge.ts @@ -2,6 +2,7 @@ import type { OrchestrationV2ProjectedTurnItem, OrchestrationV2ThreadProjection, } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import { isOrchestrationV2TurnItemVisible } from "@t3tools/shared/orchestrationV2Timeline"; export type ThreadHistoryMeta = { @@ -143,8 +144,10 @@ export function mergeOlderHistoryIntoProjection( }) ) continue; - // Retained inputs are stubs without their full text; the page row has the full item. - if (recovery) turnItemById.set(String(row.sourceItemId), row.item); + // Retained inputs are stubs without their full text, so an equally recent page row + // restores it. A live update newer than the page stays. + if (recovery && DateTime.isGreaterThanOrEqualTo(row.item.updatedAt, currentItem.updatedAt)) + turnItemById.set(String(row.sourceItemId), row.item); else row = { ...row, item: currentItem }; } } From 57c89881f28229c791c2546b5785adefe530d1e1 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 18 Nov 2026 09:55:10 +0000 Subject: [PATCH 8/8] fix(server): a notification run that did work is never a usage-limit wake A completed notification-started run before the reset was taken as proof that it did no work, so a successful notification turn (for example after switching to another model on the same provider) brought back the old usage-limit failure and its Resume target. A wake now also needs evidence: its run holds no item besides its inputs and Stop's interrupt signal. The shared rule, its SQL mirror, the command reads and Stop (which load every item of notification-started candidates), and bounded history (no bare notification start for a run that did work) all apply it. Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/Orchestrator.ts | 68 ++++++----- .../orchestration-v2/ProjectionStore.test.ts | 107 +++++++++++++----- .../src/orchestration-v2/ProjectionStore.ts | 16 ++- .../orchestration-v2/threadHistoryPaging.ts | 12 +- .../src/state/threadExecution.test.ts | 9 ++ .../shared/src/orchestrationV2ThreadError.ts | 44 +++++-- 6 files changed, 174 insertions(+), 82 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 756fc7def634e..3c23e7c10efbc 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -10,6 +10,7 @@ import { latestExecutedRun, latestRootProviderFailure, latestRunIgnoringLimitedWakes, + notificationStartedWakeCandidates, orchestrationV2WorktreePending, runRanAfter, usageLimitBlockedRun, @@ -1714,32 +1715,34 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); // Only wake candidates' inputs decide usage-limit recovery, and a dispatcher thread keeps - // thousands of notifications, so they are read for those runs alone. - const withWakeInputs = < + // thousands of notifications, so they are read for those runs alone. A notification-started + // candidate counts as a wake only without other items, so all of its items are read. + const withWakeInputs = Effect.fn("Orchestrator.withWakeInputs")(function* < R extends { readonly runs: ReadonlyArray; readonly turnItems: ReadonlyArray; }, - >( - threadId: ThreadId, - records: R, - ) => { + >(threadId: ThreadId, records: R) { + const read = (filter: ProjectionRecordFilter) => + projectionStore + .getThreadRecords(threadId, ["turnItems"], filter) + .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))); const runIds = usageLimitWakeCandidates(records.runs, records.turnItems).map((run) => run.id); - if (runIds.length === 0) return Effect.succeed(records); + if (runIds.length === 0) return records; + const inputs = (yield* read({ + turnItemTypes: ["user_message", "notification"], + turnItemRunIds: runIds, + })).turnItems; const loaded = new Set(records.turnItems.map((item) => item.id)); - return projectionStore - .getThreadRecords(threadId, ["turnItems"], { - turnItemTypes: ["user_message", "notification"], - turnItemRunIds: runIds, - }) - .pipe( - Effect.map(({ turnItems }): R => ({ - ...records, - turnItems: [...records.turnItems, ...turnItems.filter((item) => !loaded.has(item.id))], - })), - Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })), - ); - }; + let turnItems = [...records.turnItems, ...inputs.filter((item) => !loaded.has(item.id))]; + const started = notificationStartedWakeCandidates(records.runs, turnItems).map((run) => run.id); + if (started.length > 0) { + const known = new Set(turnItems.map((item) => item.id)); + const all = (yield* read({ turnItemRunIds: started })).turnItems; + turnItems = [...turnItems, ...all.filter((item) => !known.has(item.id))]; + } + return { ...records, turnItems }; + }); // Runs in every command transaction; avoid reading thousands of finished leaf nodes. const readCommandProjection = ( @@ -12764,25 +12767,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio let { thread } = input.projection; let recovery = thread.limitRecovery; // Stop must also disarm a limited run that completed notification wakes hide. Only the - // latest failure's errors and the inputs of its wake candidates can matter. - const readItems = (filter: Parameters[2]) => - projectionStore - .getThreadRecords(thread.id, ["turnItems"], filter) - .pipe(mapDispatchError(input.command)); + // latest failure's errors and its wake candidates' items can matter. const failed = latestExecutedRun(runs.filter((run) => run.status === "failed")); const errors = failed === null ? [] - : (yield* readItems({ turnItemTypes: ["error"], turnItemRunIds: [failed.id] })).turnItems; - const wakeRunIds = usageLimitWakeCandidates(runs, errors).map((run) => run.id); - const inputs = - wakeRunIds.length === 0 - ? [] - : (yield* readItems({ - turnItemTypes: ["user_message", "notification"], - turnItemRunIds: wakeRunIds, - })).turnItems; - const turnItems = [...errors, ...inputs]; + : (yield* projectionStore + .getThreadRecords(thread.id, ["turnItems"], { + turnItemTypes: ["error"], + turnItemRunIds: [failed.id], + }) + .pipe(mapDispatchError(input.command))).turnItems; + const { turnItems } = yield* withWakeInputs(thread.id, { runs, turnItems: errors }).pipe( + mapDispatchError(input.command), + ); const latest = latestRunIgnoringLimitedWakes(runs, turnItems); if (latest?.status === "failed") { const failure = latestRootProviderFailure(latest, turnItems); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 8500b5278324e..4ace3a071ac72 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -8691,6 +8691,42 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { yield* assertLatest(wakeRunId); yield* updateWake("before-reset", {}); yield* assertLatest(original.id); + // A wake that ran a command did work. + yield* store.apply({ + id: EventId.make("event:limit-shell:wake-command"), + type: "turn-item.updated", + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + driver, + occurredAt: now, + payload: { + id: TurnItemId.make("limit-shell:wake-command"), + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 5, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "command_execution", + input: "true", + output: "", + exitCode: 0, + }, + }); + yield* assertLatest(wakeRunId); + yield* (yield* SqlClient.SqlClient)` + DELETE FROM orchestration_v2_projection_turn_items + WHERE turn_item_id = ${"limit-shell:wake-command"} + `; + yield* assertLatest(original.id); yield* store.apply({ id: EventId.make("event:limit-shell:wake-user-start"), type: "turn-item.updated", @@ -9495,41 +9531,56 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { usageLimitBlockedRun(projection.runs, projection.turnItems, null)?.id, original.id, ); - // A one-row window loads the last wake's run but not its notification. The notification - // is read without its detail. + // A one-row window shows a later notice of the last wake but not its start. The start is + // read without its detail, and the wake still holds only inputs. const lastWake = RunId.make("run:limit-window:wake-5"); const lastNode = NodeId.make("node:limit-window:wake-5"); - yield* store.apply({ - id: EventId.make("event:limit-window:wake-5:command"), - type: "turn-item.updated", - threadId, - runId: lastWake, - nodeId: lastNode, - driver, - occurredAt: now, - payload: { - ...base, - id: TurnItemId.make("limit-window:wake-5:command"), + const addToLastWake = (id: string, item: Record) => + store.apply({ + id: EventId.make(`event:${id}`), + type: "turn-item.updated", + threadId, runId: lastWake, nodeId: lastNode, - parentItemId: null, - ordinal: 100, - status: "completed", - title: null, - startedAt: now, - completedAt: now, - updatedAt: now, - type: "command_execution", - input: "true", - output: "", - exitCode: 0, - }, + driver, + occurredAt: now, + payload: { + ...base, + id: TurnItemId.make(id), + runId: lastWake, + nodeId: lastNode, + parentItemId: null, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + ...item, + } as never, + }); + yield* addToLastWake("limit-window:wake-5:notice", { + ordinal: 100, + type: "notification", + source: { kind: "delegated_task", taskIds: [] }, + outcome: "failed", + summary: "Another subagent failed", }); const narrow = (yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 1 })).projection; - const notification = narrow.turnItems.find((item) => item.id === "limit-window:wake-5"); - assert.equal(notification?.type, "notification"); - assert.notProperty(notification!, "detail"); + const start = narrow.turnItems.find((item) => item.id === "limit-window:wake-5"); + assert.equal(start?.type, "notification"); + assert.notProperty(start!, "detail"); assert.equal(usageLimitBlockedRun(narrow.runs, narrow.turnItems, null)?.id, original.id); + // Once the wake runs a command it did work, so it stays the latest run. + yield* addToLastWake("limit-window:wake-5:command", { + ordinal: 101, + type: "command_execution", + input: "true", + output: "", + exitCode: 0, + }); + const worked = (yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 1 })).projection; + assert.isUndefined(worked.turnItems.find((item) => item.id === "limit-window:wake-5")); + assert.isNull(usageLimitBlockedRun(worked.runs, worked.turnItems, null)); assert.notInclude( projection.visibleTurnItems.map((row) => String(row.sourceItemId)), "limit-window:error", diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 6ccf76f3f5e96..cd599e287c83e 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -4765,10 +4765,18 @@ export const layer: Layer.Layer = LIMIT 1 )`; }; - // A wake candidate that a notification started did no work, so it must not replace the - // usage-limit failure as the latest run (usageLimitBlockedRun). + const holdsNoWork = (alias: string) => + sql`NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_turn_items work + WHERE work.run_id = ${sql.literal(alias)}.run_id + -- NO_WORK_ITEM_TYPES + AND work.type NOT IN ('user_message', 'notification', 'run_interrupt_request') + )`; + // A wake candidate that a notification started and that holds nothing but its inputs did + // no work, so it must not replace the usage-limit failure (usageLimitBlockedRun). const isLimitedWake = (alias: string) => - sql`(${isWakeCandidate(alias)} AND ${firstRootInput(alias, "type")} IS 'notification')`; + sql`(${isWakeCandidate(alias)} AND ${firstRootInput(alias, "type")} IS 'notification' + AND ${holdsNoWork(alias)})`; const getLimitRecoveryCandidates = Effect.fn("ProjectionStore.getLimitRecoveryCandidates")( function* (options: Parameters[0]) { @@ -7890,6 +7898,8 @@ export const layer: Layer.Layer = projection.runs.map((run) => run.id), )})) AND ${isWakeCandidate("run")} + -- A notification start alone would mark a run that did work as a wake. + AND (${firstRootInput("run", "type")} IS 'user_message' OR ${holdsNoWork("run")}) `; const loadedRunIds = new Set(projection.runs.map((run) => run.id)); const loadedItemIds = new Set(projection.turnItems.map((item) => item.id)); diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.ts index e17949618c255..6e39bc1067ecd 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.ts @@ -7,6 +7,7 @@ import type { } from "@t3tools/contracts"; import { latestRootProviderFailure, + NO_WORK_ITEM_TYPES, usageLimitBlockedRun, usageLimitWakeCandidates, } from "@t3tools/shared/orchestrationV2ThreadError"; @@ -563,10 +564,9 @@ function isRetainedDependency( ); } -// usageLimitBlockedRun needs the limited run's latest root error and the first input of each wake -// candidate, and a long chain of wakes pushes both out of the window. Inputs are kept as stubs -// without their unbounded text, and dropped past a quarter of the budget (clients then show -// the plain latest run). +// usageLimitBlockedRun needs the limited run's root error and each wake candidate's first input +// (a notification start only for runs without work), which long wake chains push out of the +// window. Inputs are kept as stubs and dropped past a quarter of the budget. function limitRecoveryItems( projection: Pick, maxEncodedBytes: number, @@ -580,7 +580,9 @@ function limitRecoveryItems( const blocked = usageLimitBlockedRun(projection.runs, projection.turnItems, null); const failure = latestRootProviderFailure(blocked, projection.turnItems); const firstInputs = new Map(); + const worked = new Set(); for (const item of projection.turnItems) { + if (item.runId !== null && !NO_WORK_ITEM_TYPES.has(item.type)) worked.add(String(item.runId)); if (item.type === "error" && failure !== null && item.failure === failure) { items.set(String(item.id), item); } else if ( @@ -599,6 +601,8 @@ function limitRecoveryItems( for (const item of firstInputs.values()) { let stub: OrchestrationV2TurnItem; if (item.type === "notification") { + // Without its other items, a notification start would mark a run that did work as a wake. + if (item.runId !== null && worked.has(String(item.runId))) continue; const { detail: _detail, ...rest } = item; stub = { ...rest, title: null, summary: item.summary.slice(0, 120).trimEnd() }; } else if (item.type === "user_message") { diff --git a/packages/client-runtime/src/state/threadExecution.test.ts b/packages/client-runtime/src/state/threadExecution.test.ts index ec7615fc5bf4f..e9dc1723f5ea4 100644 --- a/packages/client-runtime/src/state/threadExecution.test.ts +++ b/packages/client-runtime/src/state/threadExecution.test.ts @@ -307,6 +307,15 @@ describe("thread execution presentation", () => { // A wake on another provider ran there; the limited provider's failure stays behind it. const otherProvider = { ...wake, providerInstanceId: ProviderInstanceId.make("other") }; expect(usageLimitBlockedRun([failed, otherProvider], hidden.turnItems, null)).toBeNull(); + // A notification run that answered did work; it replaces the limited run as on main. + const answer = { + ...notification, + id: TurnItemId.make("wake-answer"), + type: "assistant_message", + }; + expect( + usageLimitBlockedRun([failed, wake], [...hidden.turnItems, answer as never], null), + ).toBeNull(); }); it("keeps live activity attached to an executing run when a newer run is queued", () => { diff --git a/packages/shared/src/orchestrationV2ThreadError.ts b/packages/shared/src/orchestrationV2ThreadError.ts index 7c7092e6cc1e2..7305b14220ab2 100644 --- a/packages/shared/src/orchestrationV2ThreadError.ts +++ b/packages/shared/src/orchestrationV2ThreadError.ts @@ -6,11 +6,12 @@ import type { import * as DateTime from "effect/DateTime"; type ErrorTurnItem = Extract; -/** Error items and the first input of each run are read, so callers may pass items stripped of other fields. */ +/** Error items, run inputs and the run of other items are read, so callers may pass stripped items. */ type FailureScanTurnItem = | ErrorTurnItem | { readonly type: Exclude; + readonly runId?: OrchestrationV2TurnItem["runId"]; readonly nodeId?: OrchestrationV2TurnItem["nodeId"]; readonly ordinal?: number; }; @@ -127,17 +128,20 @@ export function usageLimitWakeCandidates( ); } -/** - * Drops the wake candidates a notification started, so they cannot replace the limited run - * and leave the thread never armed or resumed. Items of types other than user_message, - * notification and error may be omitted. - */ -function runsWithoutLimitedWakes( +/** Inputs, and the interrupt signal Stop leaves on the latest run, are not work. */ +export const NO_WORK_ITEM_TYPES: ReadonlySet = new Set([ + "user_message", + "notification", + "run_interrupt_request", +]); + +/** The wake candidates a notification started; only those without any other item are wakes. */ +export function notificationStartedWakeCandidates( runs: ReadonlyArray, turnItems: ReadonlyArray, ): ReadonlyArray { const candidates = usageLimitWakeCandidates(runs, turnItems); - if (candidates.length === 0) return runs; + if (candidates.length === 0) return candidates; const firstInputs = new Map(); for (const item of turnItems) { if (item.nodeId == null || item.ordinal == null) continue; @@ -147,11 +151,27 @@ function runsWithoutLimitedWakes( firstInputs.set(item.nodeId, { type: item.type, ordinal: item.ordinal }); } } - const wakes = new Set( - candidates.filter( - (run) => run.rootNodeId !== null && firstInputs.get(run.rootNodeId)?.type === "notification", - ), + return candidates.filter( + (run) => run.rootNodeId !== null && firstInputs.get(run.rootNodeId)?.type === "notification", ); +} + +/** + * Drops notification-started candidates that hold no work item, so they cannot replace the + * limited run. Callers must pass every item + * of notificationStartedWakeCandidates; other runs need only user_message, notification, error. + */ +function runsWithoutLimitedWakes( + runs: ReadonlyArray, + turnItems: ReadonlyArray, +): ReadonlyArray { + const started = notificationStartedWakeCandidates(runs, turnItems); + if (started.length === 0) return runs; + const worked = new Set(); + for (const item of turnItems) { + if (item.runId != null && !NO_WORK_ITEM_TYPES.has(item.type)) worked.add(item.runId); + } + const wakes = new Set(started.filter((run) => !worked.has(run.id))); return wakes.size === 0 ? runs : runs.filter((run) => !wakes.has(run)); }