diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 819aa90c03b98..3c23e7c10efbc 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -9,9 +9,12 @@ import type { import { latestExecutedRun, latestRootProviderFailure, + latestRunIgnoringLimitedWakes, + notificationStartedWakeCandidates, orchestrationV2WorktreePending, runRanAfter, usageLimitBlockedRun, + usageLimitWakeCandidates, } from "@t3tools/shared/orchestrationV2ThreadError"; import { threadIssueKeysEqual } from "@t3tools/contracts"; import { canonicalizeGitCafePullRequestUrl } from "@t3tools/source-control-definitions/registry"; @@ -1071,7 +1074,9 @@ function usageLimitHoldsQueue( latestSession?.lastError ?? null, ); if (limited === null) return false; - if (latestExecutedRun(projection.runs)?.id === limited.id) return true; + if (latestRunIgnoringLimitedWakes(projection.runs, projection.turnItems)?.id === limited.id) { + return true; + } if (now === undefined) return false; const resetAt = latestRootProviderFailure(limited, projection.turnItems)?.resetAt; return resetAt != null && Date.parse(resetAt) > DateTime.toEpochMillis(now); @@ -1709,6 +1714,36 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio return false; }); + // Only wake candidates' inputs decide usage-limit recovery, and a dispatcher thread keeps + // thousands of notifications, so they are read for those runs alone. A notification-started + // candidate counts as a wake only without other items, so all of its items are read. + const withWakeInputs = Effect.fn("Orchestrator.withWakeInputs")(function* < + R extends { + readonly runs: ReadonlyArray; + readonly turnItems: ReadonlyArray; + }, + >(threadId: ThreadId, records: R) { + const read = (filter: ProjectionRecordFilter) => + projectionStore + .getThreadRecords(threadId, ["turnItems"], filter) + .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))); + const runIds = usageLimitWakeCandidates(records.runs, records.turnItems).map((run) => run.id); + if (runIds.length === 0) return records; + const inputs = (yield* read({ + turnItemTypes: ["user_message", "notification"], + turnItemRunIds: runIds, + })).turnItems; + const loaded = new Set(records.turnItems.map((item) => item.id)); + let turnItems = [...records.turnItems, ...inputs.filter((item) => !loaded.has(item.id))]; + const started = notificationStartedWakeCandidates(records.runs, turnItems).map((run) => run.id); + if (started.length > 0) { + const known = new Set(turnItems.map((item) => item.id)); + const all = (yield* read({ turnItemRunIds: started })).turnItems; + turnItems = [...turnItems, ...all.filter((item) => !known.has(item.id))]; + } + return { ...records, turnItems }; + }); + // Runs in every command transaction; avoid reading thousands of finished leaf nodes. const readCommandProjection = ( threadId: ThreadId, @@ -1746,6 +1781,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }, ) .pipe( + Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })), + Effect.flatMap((records) => withWakeInputs(threadId, records)), Effect.map((records): OrchestrationV2ThreadProjection => ({ ...records, checkpoints: [], @@ -1754,7 +1791,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio visibleTurnItems: [], updatedAt: records.thread.updatedAt, })), - Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })), ); // Command reads omit settled tasks, but a completion notice counts its whole cohort. @@ -4530,7 +4566,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio command, ["runs", "runtimeRequests", "turnItems"], { turnItemTypes: ["error"] }, - ); + ).pipe(Effect.flatMap((records) => withWakeInputs(command.threadId, records))); const run = usageLimitBlockedRun(projection.runs, projection.turnItems, null); const failure = latestRootProviderFailure(run, projection.turnItems); const resetMs = Date.parse(command.limitRecovery.resetAt); @@ -6898,7 +6934,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio source === undefined || (source.status !== "interrupted" && !(source.status === "failed" && limited?.class === "usage_limit")) || - latestExecutedRun(projection.runs)?.id !== source.id || + latestRunIgnoringLimitedWakes(projection.runs, projection.turnItems)?.id !== source.id || projection.thread.archivedAt !== null || projection.thread.deletedAt !== null || projection.runtimeRequests.some((request) => request.status === "pending") @@ -12730,14 +12766,23 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const { runs, subagents } = input.projection; let { thread } = input.projection; let recovery = thread.limitRecovery; - const latest = latestExecutedRun(runs); + // Stop must also disarm a limited run that completed notification wakes hide. Only the + // latest failure's errors and its wake candidates' items can matter. + const failed = latestExecutedRun(runs.filter((run) => run.status === "failed")); + const errors = + failed === null + ? [] + : (yield* projectionStore + .getThreadRecords(thread.id, ["turnItems"], { + turnItemTypes: ["error"], + turnItemRunIds: [failed.id], + }) + .pipe(mapDispatchError(input.command))).turnItems; + const { turnItems } = yield* withWakeInputs(thread.id, { runs, turnItems: errors }).pipe( + mapDispatchError(input.command), + ); + const latest = latestRunIgnoringLimitedWakes(runs, turnItems); if (latest?.status === "failed") { - const { turnItems } = yield* projectionStore - .getThreadRecords(thread.id, ["turnItems"], { - turnItemTypes: ["error"], - turnItemRunIds: [latest.id], - }) - .pipe(mapDispatchError(input.command)); const failure = latestRootProviderFailure(latest, turnItems); if ( failure?.class === "usage_limit" && @@ -15286,7 +15331,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio command, ["runs", "turnItems", "providerSessions", "messages"], { turnItemTypes: ["error"], messageRoles: ["user"] }, - ); + ).pipe(Effect.flatMap((records) => withWakeInputs(command.threadId, records))); if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) { return yield* new OrchestratorDispatchError({ commandId: command.commandId, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 2c667bf43679f..4ace3a071ac72 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -1,6 +1,7 @@ import * as NodeSqlite from "node:sqlite"; import { assert, it, vi } from "@effect/vitest"; +import { usageLimitBlockedRun } from "@t3tools/shared/orchestrationV2ThreadError"; import { EnvironmentId, EventId, @@ -8545,7 +8546,222 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }, }); yield* assertSummary("Plan limit reached.", "usage_limit"); + // A subagent that died on the same limit starts a notification wake that completes + // before the reset. It did no work, so the failed run stays the latest. + const wakeRunId = RunId.make("run:limit-shell:wake"); + const wakeNodeId = NodeId.make("node:limit-shell:wake"); + yield* store.apply({ + id: EventId.make("event:limit-shell:wake"), + type: "run.created", + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + driver, + providerInstanceId, + occurredAt: now, + payload: { + ...original, + id: wakeRunId, + ordinal: original.ordinal + 4, + rootNodeId: wakeNodeId, + userMessageId: MessageId.make("message:limit-shell:wake"), + status: "completed", + startedAt: DateTime.add(now, { minutes: 1 }), + completedAt: DateTime.add(now, { minutes: 1 }), + }, + }); + yield* store.apply({ + id: EventId.make("event:limit-shell:wake-notification"), + type: "turn-item.updated", + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + driver, + occurredAt: now, + payload: { + id: TurnItemId.make("limit-shell:wake-notification"), + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 4, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "notification", + source: { kind: "delegated_task", taskIds: [] }, + outcome: "failed", + summary: "Subagent failed", + }, + }); + yield* assertSummary("Plan limit reached.", "usage_limit"); + const wakeProjection = yield* store.getThreadProjection(threadId); + assert.equal( + ProjectionStore.threadShellFromProjection(wakeProjection).latestRunId, + original.id, + ); + assert.equal( + (yield* store.getShellSnapshot()).threads.find((row) => row.id === threadId)!.latestRunId, + original.id, + ); + // A silent compaction that completes after the limit is not work either, in both shells. + const compactRunId = RunId.make("run:limit-shell:compaction"); + const compactMessageId = MessageId.make("message:limit-shell:compaction"); + yield* store.apply({ + id: EventId.make("event:limit-shell:compaction-message"), + type: "message.updated", + threadId, + occurredAt: now, + payload: { + createdBy: "agent", + creationSource: "server", + id: compactMessageId, + threadId, + runId: compactRunId, + nodeId: null, + role: "user", + text: "/compact", + attachments: [], + streaming: false, + createdAt: now, + updatedAt: now, + }, + }); + yield* store.apply({ + id: EventId.make("event:limit-shell:compaction"), + type: "run.created", + threadId, + runId: compactRunId, + nodeId: NodeId.make("node:limit-shell:compaction"), + driver, + providerInstanceId, + occurredAt: now, + payload: { + ...original, + id: compactRunId, + ordinal: original.ordinal + 5, + rootNodeId: NodeId.make("node:limit-shell:compaction"), + userMessageId: compactMessageId, + status: "completed", + startedAt: DateTime.add(now, { minutes: 2 }), + completedAt: DateTime.add(now, { minutes: 2 }), + }, + }); + const compactProjection = yield* store.getThreadProjection(threadId); + const compactShells = [ + ProjectionStore.threadShellFromProjection(compactProjection), + (yield* store.getShellSnapshot()).threads.find((row) => row.id === threadId)!, + ]; + for (const shell of compactShells) assert.equal(shell.lastErrorClass, "usage_limit"); + const compactSql = yield* SqlClient.SqlClient; + yield* compactSql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${compactRunId}`; + // Near misses: a wake completed after the reset, or a run a user message started, did work. + const assertLatest = Effect.fnUntraced(function* (runId: RunId) { + const projection = yield* store.getThreadProjection(threadId); + assert.equal(ProjectionStore.threadShellFromProjection(projection).latestRunId, runId); + assert.equal( + (yield* store.getShellSnapshot()).threads.find((row) => row.id === threadId)!.latestRunId, + runId, + ); + }); + const updateWake = (tag: string, patch: Partial<(typeof wakeProjection.runs)[number]>) => + store.apply({ + id: EventId.make(`event:limit-shell:wake-${tag}`), + type: "run.updated", + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + driver, + occurredAt: now, + payload: { ...wakeProjection.runs.find((run) => run.id === wakeRunId)!, ...patch }, + }); + yield* updateWake("after-reset", { + completedAt: DateTime.makeUnsafe("2099-01-02T00:00:00.000Z"), + }); + yield* assertLatest(wakeRunId); + // A wake on another provider ran there, so it is the latest run. + yield* updateWake("other-provider", { + providerInstanceId: ProviderInstanceId.make("other-provider"), + }); + yield* assertLatest(wakeRunId); + yield* updateWake("before-reset", {}); + yield* assertLatest(original.id); + // A wake that ran a command did work. + yield* store.apply({ + id: EventId.make("event:limit-shell:wake-command"), + type: "turn-item.updated", + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + driver, + occurredAt: now, + payload: { + id: TurnItemId.make("limit-shell:wake-command"), + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 5, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "command_execution", + input: "true", + output: "", + exitCode: 0, + }, + }); + yield* assertLatest(wakeRunId); + yield* (yield* SqlClient.SqlClient)` + DELETE FROM orchestration_v2_projection_turn_items + WHERE turn_item_id = ${"limit-shell:wake-command"} + `; + yield* assertLatest(original.id); + yield* store.apply({ + id: EventId.make("event:limit-shell:wake-user-start"), + type: "turn-item.updated", + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + driver, + occurredAt: now, + payload: { + createdBy: "user", + creationSource: "web", + id: TurnItemId.make("limit-shell:wake-user-start"), + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 3, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + type: "user_message", + messageId: MessageId.make("message:limit-shell:wake"), + inputIntent: "turn_start", + text: "a real request", + attachments: [], + }, + }); + yield* assertLatest(wakeRunId); const sql = yield* SqlClient.SqlClient; + yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${wakeRunId}`; yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${aheadRunId}`; yield* sql`DELETE FROM orchestration_v2_projection_runs WHERE run_id = ${queuedRunId}`; yield* assertSummary("Plan limit reached.", "usage_limit"); @@ -9207,7 +9423,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { (detail) => /^SEARCH (?:item|error) /.test(detail) && !detail.includes("(turn_item_id=?)"), ); - assert.lengthOf(errorLookups, marker.includes("blocking") ? 2 : 1, marker); + assert.lengthOf(errorLookups, marker.includes("blocking") ? 3 : 2, marker); for (const errorLookup of errorLookups) { assert.include(errorLookup, "turn_items_failed_error_idx"); assert.include(errorLookup, "(thread_id=? AND run_id=? AND node_id=?)"); @@ -9216,6 +9432,162 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { }), ); + it.effect("snapshot windows keep the limited run that notification wakes hide", () => + Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + const threadId = yield* addRolledBackRecoveryCandidate("limit-window"); + const original = (yield* store.getThreadProjection(threadId)).runs[0]!; + const now = yield* DateTime.now; + const base = { threadId, providerThreadId: null, providerTurnId: null, nativeItemRef: null }; + yield* store.apply({ + id: EventId.make("event:limit-window:error"), + type: "turn-item.updated", + threadId, + occurredAt: now, + payload: { + ...base, + id: TurnItemId.make("limit-window:error"), + runId: original.id, + nodeId: original.rootNodeId, + parentItemId: null, + ordinal: 2, + status: "failed", + title: "Usage limit reached", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "error", + failure: { + class: "usage_limit", + message: "Plan limit reached.", + resetAt: "2099-01-01T00:00:00.000Z", + code: "usageLimitExceeded", + retryable: null, + }, + }, + }); + yield* store.apply({ + id: EventId.make("event:limit-window:failed"), + type: "run.updated", + threadId, + occurredAt: now, + payload: { ...original, status: "failed", completedAt: now }, + }); + for (let index = 0; index < 6; index++) { + const runId = RunId.make(`run:limit-window:wake-${index}`); + const nodeId = NodeId.make(`node:limit-window:wake-${index}`); + const at = DateTime.add(now, { seconds: index + 1 }); + yield* store.apply({ + id: EventId.make(`event:limit-window:wake-${index}`), + type: "run.created", + threadId, + runId, + nodeId, + driver, + providerInstanceId, + occurredAt: at, + payload: { + ...original, + id: runId, + ordinal: original.ordinal + index + 1, + rootNodeId: nodeId, + userMessageId: MessageId.make(`message:limit-window:wake-${index}`), + status: "completed", + startedAt: at, + completedAt: at, + }, + }); + yield* store.apply({ + id: EventId.make(`event:limit-window:wake-${index}:notification`), + type: "turn-item.updated", + threadId, + runId, + nodeId, + driver, + occurredAt: at, + payload: { + ...base, + id: TurnItemId.make(`limit-window:wake-${index}`), + runId, + nodeId, + parentItemId: null, + ordinal: 10 + index, + status: "completed", + title: null, + startedAt: at, + completedAt: at, + updatedAt: at, + type: "notification", + source: { kind: "delegated_task", taskIds: [] }, + outcome: "failed", + summary: "Subagent failed", + detail: "d".repeat(5_000), + }, + }); + } + + const { projection } = yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 3 }); + assert.equal( + usageLimitBlockedRun(projection.runs, projection.turnItems, null)?.id, + original.id, + ); + // A one-row window shows a later notice of the last wake but not its start. The start is + // read without its detail, and the wake still holds only inputs. + const lastWake = RunId.make("run:limit-window:wake-5"); + const lastNode = NodeId.make("node:limit-window:wake-5"); + const addToLastWake = (id: string, item: Record) => + store.apply({ + id: EventId.make(`event:${id}`), + type: "turn-item.updated", + threadId, + runId: lastWake, + nodeId: lastNode, + driver, + occurredAt: now, + payload: { + ...base, + id: TurnItemId.make(id), + runId: lastWake, + nodeId: lastNode, + parentItemId: null, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + ...item, + } as never, + }); + yield* addToLastWake("limit-window:wake-5:notice", { + ordinal: 100, + type: "notification", + source: { kind: "delegated_task", taskIds: [] }, + outcome: "failed", + summary: "Another subagent failed", + }); + const narrow = (yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 1 })).projection; + const start = narrow.turnItems.find((item) => item.id === "limit-window:wake-5"); + assert.equal(start?.type, "notification"); + assert.notProperty(start!, "detail"); + assert.equal(usageLimitBlockedRun(narrow.runs, narrow.turnItems, null)?.id, original.id); + // Once the wake runs a command it did work, so it stays the latest run. + yield* addToLastWake("limit-window:wake-5:command", { + ordinal: 101, + type: "command_execution", + input: "true", + output: "", + exitCode: 0, + }); + const worked = (yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 1 })).projection; + assert.isUndefined(worked.turnItems.find((item) => item.id === "limit-window:wake-5")); + assert.isNull(usageLimitBlockedRun(worked.runs, worked.turnItems, null)); + assert.notInclude( + projection.visibleTurnItems.map((row) => String(row.sourceItemId)), + "limit-window:error", + ); + }), + ); + it.effect("selects the latest waiting secret only from active runs", () => Effect.gen(function* () { const projectionStore = yield* ProjectionStore.ProjectionStoreV2; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 5b8a636cc24f1..cd599e287c83e 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -4708,6 +4708,76 @@ export const layer: Layer.Layer = ), ); + // The newest failed run of each thread and its root error, for isLimitedWake. + const latestFailureJoins = sql` + LEFT JOIN orchestration_v2_projection_runs latest_failure ON latest_failure.run_id = ( + SELECT failed.run_id FROM orchestration_v2_projection_runs failed + WHERE failed.thread_id = t.thread_id AND failed.status = 'failed' + ORDER BY failed.completed_at IS NULL DESC, failed.completed_at DESC, + failed.ordinal DESC, failed.run_id DESC + LIMIT 1 + ) + LEFT JOIN orchestration_v2_projection_turn_items latest_failure_error + ON latest_failure_error.turn_item_id = ( + SELECT error.turn_item_id FROM orchestration_v2_projection_turn_items error + WHERE error.thread_id = t.thread_id + AND error.run_id = latest_failure.run_id + AND error.node_id IS json_extract(latest_failure.payload_json, '$.rootNodeId') + AND error.type = 'error' AND error.status = 'failed' + ORDER BY error.updated_at DESC, error.ordinal DESC, error.turn_item_id DESC + LIMIT 1 + ) + `; + // Mirrors usageLimitWakeCandidates: a completed run on the limited provider, not a + // continuation, that ended after the newest usage-limit failure (runRanAfter) and before + // it resets. + const isWakeCandidate = (alias: string) => { + const run = sql.literal(alias); + return sql`( + ${run}.status = 'completed' + AND json_extract(${run}.payload_json, '$.providerInstanceId') + IS json_extract(latest_failure.payload_json, '$.providerInstanceId') + AND COALESCE( + json_extract(latest_failure_error.payload_json, '$.failure.class') = 'usage_limit' + AND julianday(${run}.completed_at) <= julianday( + json_extract(latest_failure_error.payload_json, '$.failure.resetAt') + ), 0 + ) + AND ( + ${run}.completed_at > latest_failure.completed_at + OR (${run}.completed_at = latest_failure.completed_at + AND ${run}.ordinal > latest_failure.ordinal) + ) + AND json_extract(${run}.payload_json, '$.manualContinuationOfRunId') IS NULL + AND json_extract(${run}.payload_json, '$.usageLimitContinuationOfRunId') IS NULL + AND json_extract(${run}.payload_json, '$.restartContinuationOfRunId') IS NULL + )`; + }; + const firstRootInput = (alias: string, column: "type" | "turn_item_id") => { + const run = sql.literal(alias); + return sql`( + SELECT ${sql.literal(`wake.${column}`)} FROM orchestration_v2_projection_turn_items wake + WHERE wake.run_id = ${run}.run_id + AND wake.node_id IS NOT NULL + AND wake.node_id IS json_extract(${run}.payload_json, '$.rootNodeId') + AND wake.type IN ('user_message', 'notification') + ORDER BY wake.ordinal + LIMIT 1 + )`; + }; + const holdsNoWork = (alias: string) => + sql`NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_turn_items work + WHERE work.run_id = ${sql.literal(alias)}.run_id + -- NO_WORK_ITEM_TYPES + AND work.type NOT IN ('user_message', 'notification', 'run_interrupt_request') + )`; + // A wake candidate that a notification started and that holds nothing but its inputs did + // no work, so it must not replace the usage-limit failure (usageLimitBlockedRun). + const isLimitedWake = (alias: string) => + sql`(${isWakeCandidate(alias)} AND ${firstRootInput(alias, "type")} IS 'notification' + AND ${holdsNoWork(alias)})`; + const getLimitRecoveryCandidates = Effect.fn("ProjectionStore.getLimitRecoveryCandidates")( function* (options: Parameters[0]) { const rows = yield* sql<{ @@ -4731,6 +4801,7 @@ export const layer: Layer.Layer = LIMIT 1 ) AS last_error FROM orchestration_v2_projection_threads t + ${latestFailureJoins} INNER JOIN orchestration_v2_projection_runs r ON r.run_id = ( SELECT latest.run_id FROM orchestration_v2_projection_runs latest WHERE latest.thread_id = t.thread_id @@ -4739,6 +4810,7 @@ export const layer: Layer.Layer = latest.status = 'cancelled' AND json_extract(latest.payload_json, '$.startedAt') IS NULL ) + AND NOT ${isLimitedWake("latest")} -- latestExecutedRun: the run that ended last (runRanAfter). ORDER BY latest.completed_at IS NULL DESC, latest.completed_at DESC, latest.ordinal DESC, latest.run_id DESC @@ -7776,6 +7848,96 @@ export const layer: Layer.Layer = ), ); + // A window reads only the newest rows, so an older usage-limited run and the first input + // of each wake candidate it loaded can be missing. Clients need them (usageLimitBlockedRun) + // to classify wakes and resume the run. They join runs and turnItems, never visibleTurnItems. + const withLimitEvidence = Effect.fn("ProjectionStore.withLimitEvidence")(function* ( + threadId: ThreadId, + projection: OrchestrationV2ThreadProjection, + ) { + const rows = yield* sql<{ readonly kind: string; readonly id: string | null }>` + WITH blocked AS ( + SELECT r.run_id, item.turn_item_id AS error_id + FROM orchestration_v2_projection_threads t + ${latestFailureJoins} + INNER JOIN orchestration_v2_projection_runs r ON r.run_id = ( + SELECT latest.run_id FROM orchestration_v2_projection_runs latest + WHERE latest.thread_id = t.thread_id + AND latest.status <> 'queued' + AND NOT ( + latest.status = 'cancelled' + AND json_extract(latest.payload_json, '$.startedAt') IS NULL + ) + AND NOT ${isLimitedWake("latest")} + ORDER BY latest.completed_at IS NULL DESC, latest.completed_at DESC, + latest.ordinal DESC, latest.run_id DESC + LIMIT 1 + ) AND r.status = 'failed' + INNER JOIN orchestration_v2_projection_turn_items item ON item.turn_item_id = ( + SELECT error.turn_item_id FROM orchestration_v2_projection_turn_items error + WHERE error.thread_id = t.thread_id AND error.run_id = r.run_id + AND error.type = 'error' AND error.status = 'failed' + AND error.node_id IS json_extract(r.payload_json, '$.rootNodeId') + ORDER BY error.updated_at DESC, error.ordinal DESC, error.turn_item_id DESC + LIMIT 1 + ) + -- The blocked run, when there is one, is the newest failed run. + WHERE t.thread_id = ${threadId} + AND json_extract(latest_failure_error.payload_json, '$.failure.class') = 'usage_limit' + AND json_extract(item.payload_json, '$.failure.class') = 'usage_limit' + ) + SELECT 'run' AS kind, run_id AS id FROM blocked + UNION ALL SELECT 'item', error_id FROM blocked + UNION ALL + SELECT 'item', ${firstRootInput("run", "turn_item_id")} + FROM orchestration_v2_projection_threads t + ${latestFailureJoins} + INNER JOIN orchestration_v2_projection_runs run ON run.thread_id = t.thread_id + WHERE t.thread_id = ${threadId} + AND run.run_id IN (SELECT value FROM json_each(${encodeIdList( + projection.runs.map((run) => run.id), + )})) + AND ${isWakeCandidate("run")} + -- A notification start alone would mark a run that did work as a wake. + AND (${firstRootInput("run", "type")} IS 'user_message' OR ${holdsNoWork("run")}) + `; + const loadedRunIds = new Set(projection.runs.map((run) => run.id)); + const loadedItemIds = new Set(projection.turnItems.map((item) => item.id)); + const runIds = rows.flatMap((row) => + row.kind === "run" && row.id !== null && !loadedRunIds.has(row.id) ? [row.id] : [], + ); + const itemIds = rows.flatMap((row) => + row.kind === "item" && row.id !== null && !loadedItemIds.has(row.id) ? [row.id] : [], + ); + if (runIds.length === 0 && itemIds.length === 0) return projection; + const runRows = yield* sql` + SELECT payload_json FROM orchestration_v2_projection_runs + WHERE thread_id = ${threadId} + AND run_id IN (SELECT value FROM json_each(${encodeIdList(runIds)})) + `; + // Recovery reads only an input's identity, root and ordinal, so its text never leaves SQL. + const itemRows = yield* sql` + SELECT CASE type + WHEN 'notification' THEN json_remove(payload_json, '$.detail') + WHEN 'user_message' THEN json_remove( + json_set(payload_json, '$.text', substr(json_extract(payload_json, '$.text'), 1, 120)), + '$.context' + ) + ELSE payload_json + END AS payload_json + FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} + AND turn_item_id IN (SELECT value FROM json_each(${encodeIdList(itemIds)})) + `; + const runs = yield* decodeRows(decodeRunPayload, threadId)(runRows); + const turnItems = yield* decodeRows(decodeTurnItemPayload, threadId)(itemRows); + return { + ...projection, + runs: [...projection.runs, ...runs].toSorted((left, right) => left.ordinal - right.ordinal), + turnItems: [...projection.turnItems, ...turnItems], + }; + }); + const getThreadSnapshotWindow: ProjectionStoreV2Shape["getThreadSnapshotWindow"] = ( threadId, options, @@ -7844,7 +8006,7 @@ export const layer: Layer.Layer = return { schemaVersion: ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION, snapshotSequence: rows[0]?.snapshot_sequence ?? 0, - projection: windowProjection, + projection: yield* withLimitEvidence(threadId, windowProjection), }; }), ) @@ -8119,6 +8281,7 @@ export const layer: Layer.Layer = ORDER BY r.ordinal DESC, r.run_id DESC LIMIT 1 ) + ${latestFailureJoins} LEFT JOIN orchestration_v2_projection_runs presented ON presented.run_id = ${presentedShellRunId(sql.literal("t.thread_id"))} LEFT JOIN orchestration_v2_projection_runs blocked ON blocked.run_id = ( @@ -8130,6 +8293,8 @@ export const layer: Layer.Layer = candidate.status = 'cancelled' AND json_extract(candidate.payload_json, '$.startedAt') IS NULL ) + AND NOT ${isSilentShellRun("candidate")} + AND NOT ${isLimitedWake("candidate")} -- latestExecutedRun: the run that ended last (runRanAfter). ORDER BY candidate.completed_at IS NULL DESC, candidate.completed_at DESC, candidate.ordinal DESC, candidate.run_id DESC diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 82fd3381e20d9..616f4bc23abb5 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -12215,9 +12215,11 @@ it.layer(layerTest)("usage-limit recovery", (it) => { "independent-patches", "expired-snooze", "wake", + "wake-hidden", "stop", "stop-before-arm", "stop-before-arm-mismatched-recovery", + "stop-before-arm-wake-hidden", ] as const)("guards a scheduled usage-limit continuation against %s", (scenario) => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; @@ -12321,6 +12323,61 @@ it.layer(layerTest)("usage-limit recovery", (it) => { }, ], }); + const hiddenByWake = scenario === "wake-hidden" || scenario === "stop-before-arm-wake-hidden"; + if (hiddenByWake) { + // A subagent that died on the same limit starts a notification wake that completes + // before the reset. The failed run stays the latest, so it is armed and resumes. + const wakeRunId = RunId.make(`recovery:wake-run:${scenario}`); + const wakeNodeId = NodeId.make(`recovery:wake-node:${scenario}`); + yield* events.write({ + events: [ + { + id: EventId.make(`recovery:wake-run-created:${scenario}`), + type: "run.created", + threadId, + runId: wakeRunId, + providerInstanceId: modelSelection.instanceId, + occurredAt: now, + payload: { + ...run, + id: wakeRunId, + ordinal: run.ordinal + 1, + rootNodeId: wakeNodeId, + userMessageId: MessageId.make(`recovery:wake-message:${scenario}`), + status: "completed", + startedAt: now, + completedAt: now, + }, + }, + { + id: EventId.make(`recovery:wake-notification:${scenario}`), + type: "turn-item.updated", + threadId, + occurredAt: now, + payload: { + id: TurnItemId.make(`recovery:wake-notification:${scenario}`), + type: "notification", + threadId, + runId: wakeRunId, + nodeId: wakeNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "completed", + title: null, + startedAt: now, + completedAt: now, + updatedAt: now, + source: { kind: "delegated_task", taskIds: [] }, + outcome: "failed", + summary: "Subagent failed", + }, + }, + ], + }); + } if (scenario === "queued-resume") { const queuedRun = projection.runs[1]!; yield* events.write({ @@ -12406,7 +12463,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => { limitRecovery: arm.limitRecovery, }; const stopBeforeArm = - scenario === "stop-before-arm" || scenario === "stop-before-arm-mismatched-recovery"; + scenario === "stop-before-arm" || + scenario === "stop-before-arm-mismatched-recovery" || + scenario === "stop-before-arm-wake-hidden"; if (stopBeforeArm) { if (scenario === "stop-before-arm-mismatched-recovery") { const current = yield* orchestrator.getThreadProjection(threadId); @@ -12585,7 +12644,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => { }); assert.lengthOf( (yield* orchestrator.getThreadProjection(threadId)).runs, - scenario === "queued-resume" ? 2 : 1, + scenario === "queued-resume" || hiddenByWake ? 2 : 1, ); yield* TestClock.adjust("1 minute"); const resume = limitRecoveryCommand( @@ -12827,6 +12886,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => { after.runs, before.runs.length + (scenario === "resume" || + scenario === "wake-hidden" || scenario === "resume-started" || scenario === "resume-untagged" || stopBeforeArm || @@ -12840,6 +12900,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => { after.messages, before.messages.length + (scenario === "resume" || + scenario === "wake-hidden" || scenario === "resume-started" || scenario === "resume-untagged" || stopBeforeArm || diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts index 8d81cbfadda7b..eb2766dd51586 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.test.ts @@ -9,6 +9,7 @@ import { type OrchestrationV2TurnItem, } from "@t3tools/contracts"; import { describe, expect, it } from "@effect/vitest"; +import { usageLimitBlockedRun } from "@t3tools/shared/orchestrationV2ThreadError"; import * as DateTime from "effect/DateTime"; import { @@ -709,6 +710,126 @@ describe("threadHistoryPaging", () => { expect(bounded.hasMoreHistory).toBe(true); }); + it("keeps a limited run resumable when later notification wakes outgrow the window", () => { + const limitedNode = NodeId.make("node-limited"); + const limited = { + id: RunId.make("run-limited"), + ordinal: 1, + status: "failed", + startedAt: NOW, + completedAt: NOW, + rootNodeId: limitedNode, + }; + const limitError = { + ...makeRow(0).item, + id: TurnItemId.make("limit-error"), + type: "error", + runId: limited.id, + nodeId: limitedNode, + status: "failed", + failure: { + class: "usage_limit", + message: "Plan limit reached.", + code: "usageLimitExceeded", + retryable: null, + resetAt: "2026-06-20T05:00:00.000Z", + }, + } as unknown as OrchestrationV2TurnItem; + const wakes = Array.from({ length: 6 }, (_, index) => { + const nodeId = NodeId.make(`node-wake-${index}`); + const run = { + ...limited, + id: RunId.make(`run-wake-${index}`), + ordinal: index + 2, + status: "completed", + completedAt: DateTime.add(NOW, { seconds: index + 1 }), + rootNodeId: nodeId, + }; + const row = makeRow(index + 1); + const item = { + ...row.item, + id: TurnItemId.make(`wake-notification-${index}`), + type: "notification", + runId: run.id, + nodeId, + source: { kind: "delegated_task", taskIds: [] }, + outcome: "failed", + summary: "Subagent failed", + detail: "d".repeat(50_000), + } as unknown as OrchestrationV2TurnItem; + return { run, row: { ...row, sourceItemId: item.id, item } }; + }); + const rows = [{ ...makeRow(0), sourceItemId: limitError.id, item: limitError }].concat( + wakes.map(({ row }) => row), + ); + const full = { + ...makeProjection(rows), + runs: [limited, ...wakes.map(({ run }) => run)], + } as unknown as ReturnType; + const bounded = buildBoundedThreadProjection({ + projection: full, + snapshotSequence: 1, + policy: { maxItems: 2, maxEncodedBytes: 10_000_000 }, + }); + + expect(bounded.projection.visibleTurnItems).toHaveLength(2); + expect( + usageLimitBlockedRun(bounded.projection.runs, bounded.projection.turnItems, null)?.id, + ).toBe(limited.id); + // Retained wake starts are stubs: a 50 KB notification detail must not reach the snapshot. + const visibleIds = new Set(bounded.projection.visibleTurnItems.map((row) => row.sourceItemId)); + const retained = bounded.projection.turnItems.filter((item) => !visibleIds.has(item.id)); + expect(retained.length).toBeGreaterThan(0); + expect(JSON.stringify(retained).length).toBeLessThan(10_000); + + // Evidence past a quarter of the budget is dropped, so the snapshot stays within it. The + // visible rows carry no detail here, so they fit the budget on their own. + const slimRows = rows.map((row) => ({ ...row, item: { ...row.item, detail: "d" } })); + const tight = buildBoundedThreadProjection({ + projection: { + ...makeProjection(slimRows as unknown as typeof rows), + runs: full.runs, + } as unknown as ReturnType, + snapshotSequence: 1, + policy: { maxItems: 2, maxEncodedBytes: 8_000 }, + }); + expect( + usageLimitBlockedRun(tight.projection.runs, tight.projection.turnItems, null), + ).toBeNull(); + expect(boundedTimelineEncodedBytes(tight.projection)).toBeLessThanOrEqual(8_000); + + // A user message started the run after the limit, so it did work even though only its + // later notification and the limit error are in the window. + const userStart = { + ...limitError, + id: TurnItemId.make("user-start"), + type: "user_message", + runId: wakes[5]!.run.id, + nodeId: wakes[5]!.run.rootNodeId, + ordinal: 0, + messageId: "message-user-start", + text: "a real request", + attachments: [], + } as unknown as OrchestrationV2TurnItem; + const userRows = [ + { ...makeRow(0), position: -1, sourceItemId: userStart.id, item: userStart }, + rows[0]!, + rows[6]!, + ]; + const userRun = buildBoundedThreadProjection({ + projection: { + ...makeProjection(userRows), + runs: [limited, wakes[5]!.run], + messages: [{ id: "message-user-start", runId: null, text: "p".repeat(50_000) }], + } as unknown as ReturnType, + snapshotSequence: 1, + policy: { maxItems: 2, maxEncodedBytes: 10_000_000 }, + }).projection; + expect(usageLimitBlockedRun(userRun.runs, userRun.turnItems, null)).toBeNull(); + // The stub classifies the run; it must not pull the full prompt past the budget. + expect(userRun.messages).toEqual([]); + }); + it("charges local row.item duplication when measuring bounded timeline bytes", () => { const rows = Array.from({ length: 8 }, (_, index) => makeRow(index, { outputBytes: 2_000 })); const full = makeProjection(rows); diff --git a/apps/server/src/orchestration-v2/threadHistoryPaging.ts b/apps/server/src/orchestration-v2/threadHistoryPaging.ts index cf9a101bba44c..6e39bc1067ecd 100644 --- a/apps/server/src/orchestration-v2/threadHistoryPaging.ts +++ b/apps/server/src/orchestration-v2/threadHistoryPaging.ts @@ -5,6 +5,12 @@ import type { ThreadId, TurnItemId, } from "@t3tools/contracts"; +import { + latestRootProviderFailure, + NO_WORK_ITEM_TYPES, + usageLimitBlockedRun, + usageLimitWakeCandidates, +} from "@t3tools/shared/orchestrationV2ThreadError"; import * as Schema from "effect/Schema"; /** @@ -558,10 +564,65 @@ function isRetainedDependency( ); } +// usageLimitBlockedRun needs the limited run's root error and each wake candidate's first input +// (a notification start only for runs without work), which long wake chains push out of the +// window. Inputs are kept as stubs and dropped past a quarter of the budget. +function limitRecoveryItems( + projection: Pick, + maxEncodedBytes: number, +): ReadonlyMap { + const items = new Map(); + const candidateRoots = new Set(); + for (const run of usageLimitWakeCandidates(projection.runs, projection.turnItems)) { + if (run.rootNodeId !== null) candidateRoots.add(String(run.rootNodeId)); + } + if (candidateRoots.size === 0) return items; + const blocked = usageLimitBlockedRun(projection.runs, projection.turnItems, null); + const failure = latestRootProviderFailure(blocked, projection.turnItems); + const firstInputs = new Map(); + const worked = new Set(); + for (const item of projection.turnItems) { + if (item.runId !== null && !NO_WORK_ITEM_TYPES.has(item.type)) worked.add(String(item.runId)); + if (item.type === "error" && failure !== null && item.failure === failure) { + items.set(String(item.id), item); + } else if ( + (item.type === "user_message" || item.type === "notification") && + item.nodeId !== null && + candidateRoots.has(String(item.nodeId)) + ) { + const first = firstInputs.get(String(item.nodeId)); + if (first === undefined || item.ordinal < first.ordinal) { + firstInputs.set(String(item.nodeId), item); + } + } + } + let startBytes = 0; + const starts: Array = []; + for (const item of firstInputs.values()) { + let stub: OrchestrationV2TurnItem; + if (item.type === "notification") { + // Without its other items, a notification start would mark a run that did work as a wake. + if (item.runId !== null && worked.has(String(item.runId))) continue; + const { detail: _detail, ...rest } = item; + stub = { ...rest, title: null, summary: item.summary.slice(0, 120).trimEnd() }; + } else if (item.type === "user_message") { + const { context: _context, ...rest } = item; + stub = { ...rest, text: item.text.slice(0, 120), attachments: [] }; + } else continue; + startBytes += bytesOfJson(stub); + starts.push(stub); + } + if (startBytes <= maxEncodedBytes / 4) { + for (const stub of starts) items.set(String(stub.id), stub); + } + return items; +} + /** Retain small requests outside the visible window; later result pages still need them for attempt/run reducers. */ function retainedInterruptRequestTurnItems( projection: OrchestrationV2ThreadProjection, visible: ReadonlyArray, + limitItems: ReadonlyMap, ): OrchestrationV2TurnItem[] { const visibleLocalIds = new Set(); for (const row of visible) { @@ -581,6 +642,9 @@ function retainedInterruptRequestTurnItems( } retained.push(item); } + for (const [id, item] of limitItems) { + if (!visibleLocalIds.has(id)) retained.push(item); + } return retained; } @@ -691,6 +755,7 @@ export function buildBoundedThreadProjection(input: { }; const latestLocalTurnOrdinal = computeLatestLocalTurnOrdinal(input.projection.turnItems); + const limitItems = limitRecoveryItems(controlProjection, policy.maxEncodedBytes); const dependencyReserve = (() => { let reserve = 0; const pendingRequestIds = pendingRequestIdsOf(controlProjection); @@ -699,6 +764,7 @@ export function buildBoundedThreadProjection(input: { reserve += bytesOfJson(item); } } + for (const item of limitItems.values()) reserve += bytesOfJson(item); return reserve; })(); const controlBytes = bytesOfJson({ @@ -725,6 +791,7 @@ export function buildBoundedThreadProjection(input: { const dependencyTurnItems = retainedInterruptRequestTurnItems( controlProjection, visibleTurnItems, + limitItems, ); const turnItemById = new Map(); for (const item of windowTurnItems) { @@ -774,7 +841,11 @@ export function buildBoundedThreadProjection(input: { ...controlProjection, plans, contextHandoffs, - messages: messagesForBoundedProjection(controlProjection, turnItems), + // Recovery stubs only classify runs; their full messages would bypass the budget. + messages: messagesForBoundedProjection( + controlProjection, + turnItems.filter((item) => limitItems.get(String(item.id)) !== item), + ), turnItems, visibleTurnItems, }; diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts index 41e25f2e417e1..9e4d7eb39e370 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts @@ -92,7 +92,8 @@ const lastStepAfter = function resetAtFrom(text: string, now: DateTime.Utc): string | undefined { // makeProviderFailure drops a timestamp that does not parse, so fall back to the delay. const timestamp = RESET_TIMESTAMP.exec(text)?.[1]; - if (timestamp !== undefined && Option.isSome(DateTime.make(timestamp))) return timestamp; + const parsed = timestamp === undefined ? Option.none() : DateTime.make(timestamp); + if (Option.isSome(parsed)) return DateTime.formatIso(parsed.value); const delay = RESET_DELAY.exec(text)?.[1] ?? RETRY_DELAY.exec(text)?.[1]; if (delay === undefined) return undefined; let delayMs = 0; diff --git a/packages/client-runtime/src/state/threadExecution.test.ts b/packages/client-runtime/src/state/threadExecution.test.ts index ea6de929b4cb7..e9dc1723f5ea4 100644 --- a/packages/client-runtime/src/state/threadExecution.test.ts +++ b/packages/client-runtime/src/state/threadExecution.test.ts @@ -272,6 +272,50 @@ describe("thread execution presentation", () => { lastErrorClass: null, }); expect(deriveResumableRunId(providerError, deriveThreadRuntime(providerError))).toBeNull(); + + // A notification wake that completes before the reset did no work and keeps the limit + // in place. One that completes after the reset is a real run and clears it. + const resetAt = "2026-07-28T11:00:00.000Z"; + const limited = { ...item, failure: { ...item.failure, resetAt } }; + const wake = { + ...run("wake", 2, "completed"), + rootNodeId: NodeId.make("wake-root"), + completedAt: DateTime.makeUnsafe("2026-07-28T10:30:00.000Z"), + }; + const notification = { + ...item, + id: TurnItemId.make("wake-notification"), + runId: wake.id, + nodeId: wake.rootNodeId, + type: "notification" as const, + status: "completed" as const, + source: { kind: "delegated_task" as const, taskIds: [] }, + outcome: "failed" as const, + summary: "Subagent failed", + }; + const hidden = { ...v2Projection, runs: [failed, wake], turnItems: [limited, notification] }; + expect(deriveThreadRuntime(hidden)).toMatchObject({ + status: "failed", + lastErrorClass: "usage_limit", + }); + expect(deriveResumableRunId(hidden, deriveThreadRuntime(hidden))).toBe(failed.id); + const afterReset = { ...wake, completedAt: DateTime.makeUnsafe("2026-07-28T11:30:00.000Z") }; + expect(usageLimitBlockedRun([failed, afterReset], hidden.turnItems, null)).toBeNull(); + // Manual Resume also starts from a notification; once it completes, it did the work. + const resumed = { ...wake, manualContinuationOfRunId: failed.id }; + expect(usageLimitBlockedRun([failed, resumed], hidden.turnItems, null)).toBeNull(); + // A wake on another provider ran there; the limited provider's failure stays behind it. + const otherProvider = { ...wake, providerInstanceId: ProviderInstanceId.make("other") }; + expect(usageLimitBlockedRun([failed, otherProvider], hidden.turnItems, null)).toBeNull(); + // A notification run that answered did work; it replaces the limited run as on main. + const answer = { + ...notification, + id: TurnItemId.make("wake-answer"), + type: "assistant_message", + }; + expect( + usageLimitBlockedRun([failed, wake], [...hidden.turnItems, answer as never], null), + ).toBeNull(); }); it("keeps live activity attached to an executing run when a newer run is queued", () => { diff --git a/packages/client-runtime/src/state/threadExecution.ts b/packages/client-runtime/src/state/threadExecution.ts index 0dec62f735d27..98df9f0dd2d4a 100644 --- a/packages/client-runtime/src/state/threadExecution.ts +++ b/packages/client-runtime/src/state/threadExecution.ts @@ -1,6 +1,6 @@ import { - latestExecutedRun, latestRootProviderFailure, + latestRunIgnoringLimitedWakes, latestUnheldRun, threadErrorSummary, usageLimitRunPresentedAsLatest, @@ -320,7 +320,7 @@ export function deriveResumableRunId( projection: Pick, runtime: Pick | null, ): RunId | null { - const run = latestExecutedRun(projection.runs); + const run = latestRunIgnoringLimitedWakes(projection.runs, projection.turnItems); if (run?.status === "interrupted") return run.id; return run?.status === "failed" && runtime?.lastErrorClass === "usage_limit" && diff --git a/packages/client-runtime/src/state/threadHistoryMerge.test.ts b/packages/client-runtime/src/state/threadHistoryMerge.test.ts index 95f2b01e7347e..e3da7ee9ebf28 100644 --- a/packages/client-runtime/src/state/threadHistoryMerge.test.ts +++ b/packages/client-runtime/src/state/threadHistoryMerge.test.ts @@ -233,6 +233,52 @@ describe("threadHistoryMerge", () => { expect(merged.visibleTurnItems).toEqual([]); }); + it("shows the limit error and wake notification a snapshot retained for recovery", () => { + const recent = row(5); + const error = { + ...row(0).item, + type: "error" as const, + status: "failed" as const, + title: "Usage limit reached", + failure: { class: "usage_limit" as const, message: "Plan limit reached" }, + }; + const notification = { + ...row(1).item, + type: "notification" as const, + source: { kind: "delegated_task" as const, taskIds: [] }, + outcome: "failed" as const, + summary: "Subagent failed", + detail: "full detail", + }; + const { detail: _detail, ...stub } = notification; + const coldOpen = { + ...v2Projection, + turnItems: [error, stub, recent.item], + visibleTurnItems: [{ ...recent, position: 0 }], + } as unknown as typeof v2Projection; + + const merged = mergeOlderHistoryIntoProjection(coldOpen, [ + { ...row(0), item: error }, + { ...row(1), item: notification }, + ] as never); + + expect(merged.visibleTurnItems.map((entry) => entry.item)).toEqual([ + error, + notification, + recent.item, + ]); + expect(merged.turnItems).toEqual([error, notification, recent.item]); + + // A live update that arrived after the page was fetched stays. + const live = { ...error, updatedAt: DateTime.makeUnsafe("2026-06-20T00:00:05.000Z") }; + const kept = mergeOlderHistoryIntoProjection( + { ...coldOpen, turnItems: [live, recent.item] } as never, + [{ ...row(0), item: error }] as never, + ); + expect(kept.turnItems).toContainEqual(live); + expect(kept.visibleTurnItems.map((entry) => entry.item)).toContainEqual(live); + }); + it("retains merged older rows when a live turn-item update arrives", () => { const recent = [row(2), row(3)]; const projection = { diff --git a/packages/client-runtime/src/state/threadHistoryMerge.ts b/packages/client-runtime/src/state/threadHistoryMerge.ts index ee9174e8fbe81..778527acc4200 100644 --- a/packages/client-runtime/src/state/threadHistoryMerge.ts +++ b/packages/client-runtime/src/state/threadHistoryMerge.ts @@ -2,6 +2,7 @@ import type { OrchestrationV2ProjectedTurnItem, OrchestrationV2ThreadProjection, } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import { isOrchestrationV2TurnItemVisible } from "@t3tools/shared/orchestrationV2Timeline"; export type ThreadHistoryMeta = { @@ -122,11 +123,17 @@ export function mergeOlderHistoryIntoProjection( (row.visibility === "local" || row.sourceThreadId === projection.thread.id) ) { const currentItem = turnItemById.get(String(row.sourceItemId)); - // Hidden live items stay hidden. Snapshots also retain interrupt requests and - // pending request cards outside the visible window; their history page shows them. + // Hidden live items stay hidden. Snapshots also retain interrupt requests, pending + // request cards, and the error and run inputs that usage-limit recovery reads outside + // the visible window; their history page shows them. if (currentItem !== undefined) { + const recovery = + currentItem.type === "error" || + currentItem.type === "notification" || + currentItem.type === "user_message"; if ( - (currentItem.type !== "run_interrupt_request" && + (!recovery && + currentItem.type !== "run_interrupt_request" && currentItem.type !== "approval_request" && currentItem.type !== "user_input_request") || !isOrchestrationV2TurnItemVisible({ @@ -137,7 +144,11 @@ export function mergeOlderHistoryIntoProjection( }) ) continue; - row = { ...row, item: currentItem }; + // Retained inputs are stubs without their full text, so an equally recent page row + // restores it. A live update newer than the page stays. + if (recovery && DateTime.isGreaterThanOrEqualTo(row.item.updatedAt, currentItem.updatedAt)) + turnItemById.set(String(row.sourceItemId), row.item); + else row = { ...row, item: currentItem }; } } pageKeys.add(key); diff --git a/packages/shared/src/orchestrationV2ThreadError.ts b/packages/shared/src/orchestrationV2ThreadError.ts index 141d73f402540..7305b14220ab2 100644 --- a/packages/shared/src/orchestrationV2ThreadError.ts +++ b/packages/shared/src/orchestrationV2ThreadError.ts @@ -6,10 +6,15 @@ import type { import * as DateTime from "effect/DateTime"; type ErrorTurnItem = Extract; -/** Only error items are read, so callers may pass items stripped of other fields. */ +/** Error items, run inputs and the run of other items are read, so callers may pass stripped items. */ type FailureScanTurnItem = | ErrorTurnItem - | { readonly type: Exclude }; + | { + readonly type: Exclude; + readonly runId?: OrchestrationV2TurnItem["runId"]; + readonly nodeId?: OrchestrationV2TurnItem["nodeId"]; + readonly ordinal?: number; + }; export function latestRootProviderFailure( run: OrchestrationV2Run | null, @@ -95,12 +100,95 @@ export function runRanAfter( return end(run) === end(other) ? run.ordinal > other.ordinal : end(run) > end(other); } +/** + * Subagents that die on a usage limit start notification runs that finish before the reset + * with no work to show. These candidates ran on the limited provider after its latest failure + * and before the reset; continuations and other providers' runs are never wakes. + */ +export function usageLimitWakeCandidates( + runs: ReadonlyArray, + turnItems: ReadonlyArray, +): ReadonlyArray { + const failed = latestExecutedRun(runs.filter((run) => run.status === "failed")); + const failure = latestRootProviderFailure(failed, turnItems); + if (failed === null || failure?.class !== "usage_limit" || failure.resetAt == null) return []; + const resetMs = Date.parse(failure.resetAt); + if (!Number.isFinite(resetMs)) return []; + return runs.filter( + (run) => + run.status === "completed" && + run.completedAt !== null && + run.rootNodeId !== null && + run.providerInstanceId === failed.providerInstanceId && + DateTime.toEpochMillis(run.completedAt) <= resetMs && + runRanAfter(run, failed) && + run.manualContinuationOfRunId === undefined && + run.usageLimitContinuationOfRunId === undefined && + run.restartContinuationOfRunId === undefined, + ); +} + +/** Inputs, and the interrupt signal Stop leaves on the latest run, are not work. */ +export const NO_WORK_ITEM_TYPES: ReadonlySet = new Set([ + "user_message", + "notification", + "run_interrupt_request", +]); + +/** The wake candidates a notification started; only those without any other item are wakes. */ +export function notificationStartedWakeCandidates( + runs: ReadonlyArray, + turnItems: ReadonlyArray, +): ReadonlyArray { + const candidates = usageLimitWakeCandidates(runs, turnItems); + if (candidates.length === 0) return candidates; + const firstInputs = new Map(); + for (const item of turnItems) { + if (item.nodeId == null || item.ordinal == null) continue; + if (item.type !== "user_message" && item.type !== "notification") continue; + const first = firstInputs.get(item.nodeId); + if (first === undefined || item.ordinal < first.ordinal) { + firstInputs.set(item.nodeId, { type: item.type, ordinal: item.ordinal }); + } + } + return candidates.filter( + (run) => run.rootNodeId !== null && firstInputs.get(run.rootNodeId)?.type === "notification", + ); +} + +/** + * Drops notification-started candidates that hold no work item, so they cannot replace the + * limited run. Callers must pass every item + * of notificationStartedWakeCandidates; other runs need only user_message, notification, error. + */ +function runsWithoutLimitedWakes( + runs: ReadonlyArray, + turnItems: ReadonlyArray, +): ReadonlyArray { + const started = notificationStartedWakeCandidates(runs, turnItems); + if (started.length === 0) return runs; + const worked = new Set(); + for (const item of turnItems) { + if (item.runId != null && !NO_WORK_ITEM_TYPES.has(item.type)) worked.add(item.runId); + } + const wakes = new Set(started.filter((run) => !worked.has(run.id))); + return wakes.size === 0 ? runs : runs.filter((run) => !wakes.has(run)); +} + +/** The latest executed run, not counting notification wakes that did no work before a reset. */ +export function latestRunIgnoringLimitedWakes( + runs: ReadonlyArray, + turnItems: ReadonlyArray, +): OrchestrationV2Run | null { + return latestExecutedRun(runsWithoutLimitedWakes(runs, turnItems)); +} + export function usageLimitBlockedRun( runs: ReadonlyArray, turnItems: ReadonlyArray, sessionError: string | null, ): OrchestrationV2Run | null { - const executed = latestExecutedRun(runs); + const executed = latestRunIgnoringLimitedWakes(runs, turnItems); if (executed?.status !== "failed") return null; const summary = threadErrorSummary(latestRootProviderFailure(executed, turnItems), sessionError); return summary.lastErrorClass === "usage_limit" ? executed : null;