From 51d41aa34921b2b0d6f3975baaa1637dfac83692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 08:36:52 +0000 Subject: [PATCH 01/23] fix(server): cascade parent lifecycle to subagent descendants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preserve independent archives, newer choices and caller limits through replayable cascades, and finish committed cascades before replying Stop the walk at children moved to another project Co-authored-by: Julius Marminge Co-authored-by: maria-rcks Co-authored-by: Claude Opus 5.5 > 🤖 Generated by GPT 6.1 Sol in Codex via T3 Code. --- .../orchestration-v2/CommandReceiptStore.ts | 69 +++++ .../src/orchestration-v2/EffectOutbox.ts | 17 ++ .../src/orchestration-v2/EffectWorker.ts | 19 ++ .../src/orchestration-v2/Orchestrator.ts | 74 ++++- .../src/orchestration-v2/ProjectionStore.ts | 18 ++ .../ProviderTurnControlService.test.ts | 1 + .../ThreadManagementService.ts | 99 +++++++ .../src/orchestration-v2/runtimeLayer.test.ts | 261 +++++++++++++++++- .../OrchestrationCommandReceipts.ts | 27 ++ .../src/relay/AgentAwarenessRelay.test.ts | 1 + docs/user/thread-sidebar.md | 2 + 11 files changed, 585 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/CommandReceiptStore.ts b/apps/server/src/orchestration-v2/CommandReceiptStore.ts index f39de342b5526..7c2b12cdcfec9 100644 --- a/apps/server/src/orchestration-v2/CommandReceiptStore.ts +++ b/apps/server/src/orchestration-v2/CommandReceiptStore.ts @@ -31,9 +31,19 @@ export class CommandReceiptStoreReadError extends Schema.TaggedError()( + "CommandReceiptStoreLifecycleError", + { threadId: ThreadId, cause: Schema.Defect() }, +) { + override get message(): string { + return `Failed to read orchestration V2 lifecycle receipt for thread ${this.threadId}.`; + } +} + export const CommandReceiptStoreV2Error = Schema.Union([ CommandReceiptStoreWriteError, CommandReceiptStoreReadError, + CommandReceiptStoreLifecycleError, ]); export type CommandReceiptStoreV2Error = typeof CommandReceiptStoreV2Error.Type; @@ -65,6 +75,9 @@ export type ProjectCommandReceiptV2 = typeof ProjectCommandReceiptV2.Type; type AnyCommandReceiptV2 = CommandReceiptV2 | ProjectCommandReceiptV2; export interface CommandReceiptStoreV2Shape { + readonly getLatestLifecycle: ( + threadId: ThreadId, + ) => Effect.Effect; readonly insertIfAbsent: ( receipt: AnyCommandReceiptV2, ) => Effect.Effect; @@ -137,6 +150,13 @@ const layerBase: Layer.Layer< const receipts = yield* OrchestrationCommandReceipts.OrchestrationCommandReceiptRepository; return CommandReceiptStoreV2.of({ + getLatestLifecycle: (threadId) => + receipts.getLatestLifecycle({ threadId }).pipe( + Effect.flatMap((receipt) => + Option.isSome(receipt) ? fromApplicationReceipt(receipt.value) : Effect.succeed(null), + ), + Effect.mapError((cause) => new CommandReceiptStoreLifecycleError({ threadId, cause })), + ), insertIfAbsent: (receipt) => receipts.insertIfAbsent(toApplicationReceipt(receipt)).pipe( Effect.mapError( @@ -200,3 +220,52 @@ const layerBase: Layer.Layer< export const layer = layerBase.pipe(Layer.provide(OrchestrationCommandReceipts.layer)); export const layerFromApplicationReceipts = layerBase; + +/** Rejections are permanent, so a cascade retry takes the first unused or accepted suffix. */ +export const cascadeCommandId = Effect.fn("CommandReceiptStore.cascadeCommandId")(function* ( + receipts: CommandReceiptStoreV2Shape, + base: CommandId, +) { + for (let retry = 0; retry < 32; retry++) { + const id = retry === 0 ? base : CommandId.make(`${base}:retry:${retry}`); + const receipt = yield* receipts.getByCommandId(id); + if (Option.isNone(receipt) || receipt.value.status === "accepted") return id; + } + return undefined; +}); + +/** A lifecycle choice made after the cascade's source command, other than `ownCommandId`. */ +export const supersedesCascade = ( + latest: CommandReceiptV2 | null, + source: CommandReceiptV2, + ownCommandId: CommandId, +) => + latest !== null && + latest.resultSequence > source.resultSequence && + latest.commandId !== ownCommandId; + +/** Whether a cascade from an archive of `rootThreadId`, along `path`, archived this thread. */ +export const archivedByCascade = Effect.fn("CommandReceiptStore.archivedByCascade")(function* ( + receipts: CommandReceiptStoreV2Shape, + latest: CommandReceiptV2 | null, + rootThreadId: ThreadId, + path: ReadonlyArray, +) { + if (latest?.commandType !== "thread.archive") return false; + let rootCommandId: string = latest.commandId; + for (const threadId of path.toReversed()) { + const segment = `:subagent:${threadId}`; + if (!rootCommandId.endsWith(segment)) rootCommandId = rootCommandId.replace(/:retry:\d+$/, ""); + if (!rootCommandId.endsWith(segment)) return false; + rootCommandId = rootCommandId.slice(0, -segment.length); + } + const root = Option.getOrNull(yield* receipts.getByCommandId(CommandId.make(rootCommandId))); + const valid = root?.status === "accepted" && root.threadId === rootThreadId; + // Derive the IDs again, so a user ID only shaped like a cascade retry is not counted. + let derived = valid && root.commandType === "thread.archive" ? root.commandId : undefined; + for (const threadId of path) { + if (derived === undefined) return false; + derived = yield* cascadeCommandId(receipts, CommandId.make(`${derived}:subagent:${threadId}`)); + } + return derived === latest.commandId && (root?.resultSequence ?? 0) <= latest.resultSequence; +}); diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index b495293bc2228..d7706873936a8 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -12,7 +12,9 @@ import { ProviderUserInputAnswers, ProviderThreadId, ProviderTurnId, + ProviderInteractionMode, RunId, + RuntimeMode, RuntimeRequestId, ThreadId, } from "@t3tools/contracts"; @@ -31,6 +33,13 @@ import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import { forkParked } from "../serverActivation.ts"; +import type { CommandReceiptV2 } from "./CommandReceiptStore.ts"; + +/** Set while a cascade dispatches a descendant's lifecycle command; delete needs no source. */ +export const SubagentCascade = Context.Reference< + | { readonly source: CommandReceiptV2 | undefined; readonly path: ReadonlyArray } + | undefined +>("t3/orchestration-v2/SubagentCascade", { defaultValue: () => undefined }); export const OrchestrationEffectRequestV2 = Schema.Union([ Schema.Struct({ @@ -127,6 +136,13 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ attachmentIds: Schema.Array(Schema.String), skipReferenced: Schema.optional(Schema.Boolean), }), + Schema.Struct({ + type: Schema.Literal("subagent-threads.cascade"), + operation: Schema.Literals(["thread.archive", "thread.unarchive", "thread.delete"]), + limit: Schema.optional( + Schema.Struct({ runtimeMode: RuntimeMode, interactionMode: ProviderInteractionMode }), + ), + }), Schema.Struct({ type: Schema.Literal("thread-title.generate"), kind: Schema.Union([ @@ -150,6 +166,7 @@ export const REPLAY_SAFE_EFFECT_TYPES_AFTER_PROCESS_LOSS = [ "terminal.cleanup", "preview.cleanup", "attachment.cleanup", + "subagent-threads.cascade", "thread-title.generate", "delegated-tasks.stop", ] as const satisfies ReadonlyArray; diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index c637446d78827..d7791f67c67f4 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -30,6 +30,7 @@ import * as ProviderTurnStartService from "./ProviderTurnStartService.ts"; import * as RuntimeRequestService from "./RuntimeRequestService.ts"; import * as ThreadTitleRegenerationService from "./ThreadTitleRegenerationService.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; +import { DispatchModeLimit } from "./DispatchModeLimit.ts"; import * as ServerSettings from "../serverSettings.ts"; import { continueRestartedRun } from "./RestartContinuation.ts"; @@ -680,6 +681,24 @@ export const layerExecutor: Layer.Layer< ), ); } + case "subagent-threads.cascade": + return threads + .cascadeSubagentLifecycle({ + commandId: effect.commandId, + threadId: effect.threadId, + operation: effect.request.operation, + }) + .pipe( + Effect.provideService(DispatchModeLimit, effect.request.limit), + Effect.mapError( + (cause) => + new OrchestrationEffectExecutionError({ + effectId: effect.id, + effectType: effect.request.type, + cause, + }), + ), + ); case "thread-title.generate": return threadTitleRegeneration .execute({ diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 00939843c13f1..3c833942c5bb3 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -128,7 +128,12 @@ import { providerTurnCapabilities, resolveMessageDispatchIntent, } from "./CommandPolicy.ts"; -import { CommandReceiptStoreV2, type CommandReceiptV2 } from "./CommandReceiptStore.ts"; +import { + archivedByCascade, + CommandReceiptStoreV2, + type CommandReceiptV2, + supersedesCascade, +} from "./CommandReceiptStore.ts"; import { ContextHandoffServiceV2, readForkHistoryItems, @@ -476,6 +481,9 @@ export interface OrchestratorV2Shape { readonly getThreadProjection: ( threadId: ThreadId, ) => Effect.Effect; + readonly getSubagentChildThreads: ( + ...args: Parameters + ) => Effect.Effect, OrchestratorV2Error>; readonly getCheckpointContext: ( threadId: ThreadId, ) => Effect.Effect; @@ -1893,6 +1901,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .findThread(threadId) .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))); + // A cascade walks the whole subtree itself, so its own commands start none. + const subagentCascadeEffects = Effect.fn("Orchestrator.subagentCascadeEffects")(function* ( + { commandId, threadId }: { readonly commandId: CommandId; readonly threadId: ThreadId }, + operation: "thread.archive" | "thread.unarchive" | "thread.delete", + ) { + if (yield* EffectOutbox.SubagentCascade) return []; + if ((yield* projectionStore.getSubagentChildThreads(threadId)).length === 0) return []; + // The outbox schema persists only the caller's runtime and interaction modes. + const limit = yield* DispatchModeLimit; + const request = { type: "subagent-threads.cascade", operation, limit } as const; + const id = `effect:${commandId}:subagent-threads.cascade`; + return [{ id, commandId, threadId, request } satisfies PendingOrchestrationEffectV2]; + }); + /** Side chats live inside their parent, so deleting or archiving the parent takes them too. */ const liveSideChatIds = Effect.fn("Orchestrator.liveSideChatIds")( function* (threadId: ThreadId) { @@ -5600,6 +5622,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ]); } + if (command.type === "thread.archive" || command.type === "thread.unarchive") { + const cascade = yield* subagentCascadeEffects(command, command.type).pipe( + mapDispatchError(command), + ); + yield* Ref.update(effects, (existing) => [...existing, ...cascade]); + } + if (command.type === "thread.metadata.update" && command.regenerateTitle === true) { yield* Ref.update(effects, (existing) => [ ...existing, @@ -15650,7 +15679,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (cause) => new OrchestratorProjectionError({ threadId: deletion.threadId, cause }), ), ); - return yield* mapDispatchError(command)( + const plan = yield* mapDispatchError(command)( planThreadDeletion({ command: deletion, projection, @@ -15661,6 +15690,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio idAllocator, }), ); + const cascade = yield* subagentCascadeEffects(deletion, deletion.type).pipe( + mapDispatchError(command), + ); + return { ...plan, effects: [...plan.effects, ...cascade] }; }); // Checked under the thread lock: a message sent since the client looked keeps the thread. if (command.onlyIfUnused === true) { @@ -16129,6 +16162,38 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ); } + // Decided under the child's lock, so a newer independent choice on it always wins. + const cascade = yield* EffectOutbox.SubagentCascade; + if ( + cascade !== undefined && + (command.type === "thread.archive" || + command.type === "thread.unarchive" || + command.type === "thread.delete") + ) { + const { source, path } = cascade; + const read = mapDispatchError(command); + const thread = yield* projectionStore.getThread(command.threadId).pipe(read); + const parentThreadId = thread.lineage.parentThreadId; + const parent = + parentThreadId === null + ? null + : yield* projectionStore.getThread(parentThreadId).pipe(read); + // A child moved to another project since the walk found it is no longer this subtree's. + let applies = parent?.projectId === thread.projectId; + if (applies && source !== undefined && command.type !== "thread.delete") { + const latest = yield* commandReceipts.getLatestLifecycle(command.threadId).pipe(read); + applies = + thread.deletedAt === null && + (command.type === "thread.archive" + ? thread.archivedAt === null && !supersedesCascade(latest, source, command.commandId) + : thread.archivedAt !== null && + (yield* archivedByCascade(commandReceipts, latest, source.threadId, path).pipe( + read, + ))); + } + if (!applies) return { sequence: source?.resultSequence ?? 0, storedEvents: [] }; + } + const plan = yield* dispatchOnce(command, sideChatIds).pipe( Effect.tap((planned) => refuseAboveRuntimeModeCeiling(command, planned.events)), Effect.filterOrFail( @@ -17042,6 +17107,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))); }, ), + getSubagentChildThreads: Effect.fn("OrchestratorV2.getSubagentChildThreads")((threadId) => + projectionStore + .getSubagentChildThreads(threadId) + .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))), + ), getThreadProjection: Effect.fn("OrchestratorV2.getThreadProjection")(function (threadId) { return projectionStore .getThreadProjection(threadId) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index ba91ba94d2520..0c101d3122a20 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -670,6 +670,10 @@ export interface ProjectionStoreV2Shape { readonly findThread: ( threadId: ThreadId, ) => Effect.Effect; + /** Direct subagents, archived and deleted included. */ + readonly getSubagentChildThreads: ( + threadId: ThreadId, + ) => Effect.Effect, ProjectionStoreV2Error>; readonly getLimitRecoveryCandidates: (options: { readonly now: DateTime.Utc; readonly autoResume: boolean; @@ -5992,6 +5996,19 @@ export const layer: Layer.Layer = const findThread: ProjectionStoreV2Shape["findThread"] = Effect.fn( "ProjectionStore.findThread", )(selectThread); + // Subagents inherit the parent's immutable project. Use its index to bound the scan. + const getSubagentChildThreads: ProjectionStoreV2Shape["getSubagentChildThreads"] = (threadId) => + sql` + SELECT child.payload_json FROM orchestration_v2_projection_threads AS child + JOIN orchestration_v2_projection_threads AS parent + ON parent.thread_id = ${threadId} AND child.project_id = parent.project_id + WHERE json_extract(child.payload_json, '$.lineage.parentThreadId') = ${threadId} + AND json_extract(child.payload_json, '$.lineage.relationshipToParent') = 'subagent' + ORDER BY child.created_at ASC, child.thread_id ASC + `.pipe( + Effect.mapError((cause) => new ProjectionStoreReadError({ threadId, cause })), + Effect.flatMap(decodeRows(decodeThreadPayload, threadId)), + ); const requireThread = Effect.fn("ProjectionStore.requireThread")(function* ( threadId: ThreadId, @@ -9842,6 +9859,7 @@ export const layer: Layer.Layer = getThreadLinkMetadata: readOnReader(getThreadLinkMetadata), getThreadWorktreePaths: readOnReader(getThreadWorktreePaths), getOpenRemoteDelegatedTasks: getOpenRemoteDelegatedTasks.pipe(NodeSqliteClient.readOnly), + getSubagentChildThreads: readOnReader(getSubagentChildThreads), getThreadProjection: readOnReader(getThreadProjection), getTurnStartContext: readOnReader(getTurnStartContext), getTurnStartHistory: readOnReader(getTurnStartHistory), diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts index 51c38efa8a41a..8a84f64f76aa1 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts @@ -147,6 +147,7 @@ function layerProjectionStore( getThreadShell: () => Effect.die("unused getThreadShell"), getThreadTitles: () => Effect.die("unused getThreadTitles"), getThreadLinkMetadata: () => Effect.die("unused getThreadLinkMetadata"), + getSubagentChildThreads: () => Effect.die("unused getSubagentChildThreads"), findGuestShareThreadId: () => Effect.die("unused findGuestShareThreadId"), getThread: () => Effect.die("unused getThread"), findThread: () => Effect.die("unused findThread"), diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 45b5e9effa8cf..b0866c42cee52 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -60,6 +60,7 @@ import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; +import * as EffectOutbox from "./EffectOutbox.ts"; import * as CommandReceiptStore from "./CommandReceiptStore.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; @@ -552,6 +553,14 @@ export interface ThreadManagementServiceShape { void, Orchestrator.OrchestratorV2Error | ThreadManagementProviderInstanceDisabledError >; + readonly cascadeSubagentLifecycle: (input: { + readonly threadId: ThreadId; + readonly commandId: CommandId; + readonly operation: "thread.archive" | "thread.unarchive" | "thread.delete"; + }) => Effect.Effect< + void, + Orchestrator.OrchestratorV2Error | CommandReceiptStore.CommandReceiptStoreV2Error + >; readonly getThreadEventSequence: Orchestrator.OrchestratorV2["Service"]["getThreadEventSequence"]; readonly recoverDelegatedTask: Orchestrator.OrchestratorV2["Service"]["recoverDelegatedTask"]; readonly cancelDelegatedTask: Orchestrator.OrchestratorV2["Service"]["cancelDelegatedTask"]; @@ -844,8 +853,34 @@ const make = Effect.gen(function* () { ensureCommandTranscripts(command).pipe( Effect.andThen(assertDispatchTargetEnabled(command)), Effect.andThen(orchestrator.dispatch(command)), + Effect.tap((result) => cascadeBeforeReply(command, result)), ); + // Clients read archive snapshots as soon as this returns; the outbox effect only backs it up. + // A replay or a no-op (an unused-only delete that kept the thread) leaves it to that effect. + const cascadeBeforeReply = ( + command: OrchestrationV2ServerCommand, + result: Orchestrator.OrchestratorV2DispatchResult, + ) => + (command.type === "thread.archive" || + command.type === "thread.unarchive" || + command.type === "thread.delete") && + result.replayed !== true && + result.storedEvents.length > 0 + ? cascadeSubagentLifecycle({ + commandId: command.commandId, + threadId: command.threadId, + operation: command.type, + }).pipe( + Effect.catch((cause) => + Effect.logWarning("Subagent lifecycle cascade left to the effect worker", { + threadId: command.threadId, + cause, + }), + ), + ) + : Effect.void; + const getProjectThreadRecords: ThreadManagementServiceShape["getProjectThreadRecords"] = ( input, fields, @@ -1774,6 +1809,69 @@ const make = Effect.gen(function* () { if (failures[0] !== undefined) return yield* failures[0]; }); + const cascadeSubagentLifecycle: ThreadManagementServiceShape["cascadeSubagentLifecycle"] = + Effect.fn("ThreadManagementService.cascadeSubagentLifecycle")(function* (input) { + const receipt = Option.getOrUndefined(yield* receipts.getByCommandId(input.commandId)); + // Side chats deleted with their parent have no receipt of their own. + if (input.operation !== "thread.delete" && receipt?.status !== "accepted") return; + const source = input.operation === "thread.delete" ? undefined : receipt; + const visit: ( + thread: { + readonly id: ThreadId; + readonly deletedAt: unknown; + readonly projectId?: unknown; + }, + commandId: CommandId, + path: ReadonlyArray, + ) => ReturnType = Effect.fn( + "ThreadManagementService.visitSubagentCascade", + )(function* (thread, commandId, path) { + if (path.length > 0 && thread.deletedAt === null) { + const dispatched = yield* orchestrator + .dispatch({ type: input.operation, commandId, threadId: thread.id }) + .pipe( + Effect.provideService(EffectOutbox.SubagentCascade, { source, path }), + Effect.as(true), + Effect.catchTags({ + OrchestratorThreadAboveModeLimitError: () => Effect.succeed(false), + }), + ); + if (!dispatched) return; + } + // Holds each thread's lock while visiting its subagents, so locks nest parent before child. + yield* threadDispatch.withLock( + thread.id, + Effect.gen(function* () { + // A deleted or moved thread, one above the caller's limit or a newer choice keeps its subtree. + const limit = yield* DispatchModeLimit; + const current = (yield* orchestrator.getThreadRecords(thread.id, [])).thread; + if (path.length > 0 && current.projectId !== thread.projectId) return; + if (limit && exceededDispatchModeLimit(limit, current)) return; + if (source && current.deletedAt !== null) return; + const latest = source ? yield* receipts.getLatestLifecycle(thread.id) : null; + if (source && CommandReceiptStore.supersedesCascade(latest, source, commandId)) return; + const children = yield* orchestrator.getSubagentChildThreads(thread.id); + const [, failures] = yield* Effect.partition(children, (child) => + Effect.gen(function* () { + const base = CommandId.make(`${commandId}:subagent:${child.id}`); + const childCommandId = yield* CommandReceiptStore.cascadeCommandId(receipts, base); + if (childCommandId !== undefined) + return yield* visit(child, childCommandId, [...path, child.id]); + return yield* new Orchestrator.OrchestratorDispatchError({ + commandId: base, + commandType: input.operation, + cause: "All 32 subagent lifecycle command IDs were rejected.", + }); + }), + ); + // Finish healthy siblings before reporting a failure to the durable worker. + if (failures[0] !== undefined) return yield* failures[0]; + }), + ); + }); + yield* visit({ id: input.threadId, deletedAt: null }, input.commandId, []); + }); + return ThreadManagementService.of({ ensureLegacyTranscript, searchThreadStream: (input) => @@ -1874,6 +1972,7 @@ const make = Effect.gen(function* () { settleThread, interruptThread, stopDelegatedTasks, + cascadeSubagentLifecycle, stopWorkflow, forceStopThread, awaitBackgroundTasks, diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index c899eb774bfc3..86a4ca0f7b0d8 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -6,7 +6,7 @@ import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderS import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { assert, it } from "@effect/vitest"; +import { assert, describe, it } from "@effect/vitest"; import { vi } from "vite-plus/test"; import { type ApplicationStoredEvent, @@ -65,6 +65,8 @@ import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore import * as ProjectEnrichmentService from "../project/ProjectEnrichmentService.ts"; import * as ProjectService from "../project/ProjectService.ts"; import * as ServerSettings from "../serverSettings.ts"; +import * as Receipts from "./CommandReceiptStore.ts"; +import { DispatchModeLimit } from "./DispatchModeLimit.ts"; import * as McpSessionRegistryTestkit from "../mcp/McpSessionRegistry.testkit.ts"; import * as ModelManifest from "../provider/ModelManifest.ts"; import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; @@ -357,6 +359,7 @@ const layerTest = Layer.mergeAll( ProjectStore.layer, ProjectionStore.layer, EffectOutbox.layer, + Receipts.layer, ThreadCommandExecutor.layer, ).pipe( Layer.provide(McpSessionRegistryTestkit.layer), @@ -14112,3 +14115,259 @@ it.layer(layerQueueResumeDisabledInstanceTest)( ); }, ); + +describe("subagent lifecycle cascade", () => { + const parent = ThreadId.make("parent"); + const app = ThreadId.make("app"); + const native = ThreadId.make("native"); + const grandchild = ThreadId.make("grandchild:retry:31"); + const fixture = Effect.gen(function* () { + const threads = yield* ThreadManagementService.ThreadManagementService; + const orchestrator = yield* Orchestrator.OrchestratorV2; + const store = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + yield* threads.dispatch({ + type: "thread.create", + commandId: CommandId.make("create:parent"), + threadId: parent, + projectId: ProjectId.make("project"), + title: "Parent", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "agent", + creationSource: "mcp", + }); + const base = yield* store.getThread(parent); + const set = Effect.fn("test.seed")(function* (id: ThreadId, p: Partial) { + const thread = yield* store.getThread(id).pipe(Effect.orElseSucceed(() => base)); + const event = { id: EventId.make(`seed:${id}`), threadId: id, occurredAt: now }; + yield* store.apply({ ...event, type: "thread.created", payload: { ...thread, id, ...p } }); + }); + for (const [id, parentThreadId, relationshipToParent, projectId] of [ + [app, parent, "subagent", base.projectId], + [native, parent, "subagent", base.projectId], + [grandchild, app, "subagent", base.projectId], + [ThreadId.make("fork"), parent, "fork", base.projectId], + [ThreadId.make("fork-child"), ThreadId.make("fork"), "subagent", base.projectId], + [ThreadId.make("foreign"), parent, "subagent", ProjectId.make("foreign")], + ] as const) { + const lineage = { rootThreadId: parent, parentThreadId, relationshipToParent }; + yield* set(id, { lineage, projectId, creationSource: id === native ? "provider" : "mcp" }); + } + const command = ( + operation: "archive" | "unarchive" | "delete", + threadId: ThreadId, + id?: string, + ) => + ({ + type: `thread.${operation}`, + commandId: CommandId.make(id ?? `${operation}:${threadId}`), + threadId, + }) as const; + const run = (operation: "archive" | "unarchive" | "delete", threadId = parent, id?: string) => + threads.dispatch(command(operation, threadId, id)); + // Committed by a process that stopped before its reply, leaving the cascade to the worker. + const commit = (operation: "archive" | "unarchive", threadId = parent, id?: string) => + orchestrator.dispatch(command(operation, threadId, id)); + const state = (field: "archivedAt" | "deletedAt", ...ids: ReadonlyArray) => + Effect.forEach(ids, (id) => store.getThread(ThreadId.make(id))).pipe( + Effect.map((all) => all.map((thread) => (thread[field] === null ? 0 : 1)).join("")), + ); + const archived = (...ids: ReadonlyArray) => state("archivedAt", ...ids); + const outbox = yield* EffectOutbox.EffectOutboxV2; + const cascade = Effect.map( + outbox.get("effect:archive:parent:subagent-threads.cascade"), + Option.getOrThrow, + ); + const drain = (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); + return { threads, now, set, run, commit, state, archived, outbox, cascade, drain }; + }); + const receipt = (id: string) => + Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.getByCommandId(CommandId.make(id))); + it.effect("archives, restores and deletes subagents recursively, but not forks or projects", () => + Effect.gen(function* () { + const f = yield* fixture; + const orchestrator = yield* Orchestrator.OrchestratorV2; + const store = yield* ProjectionStore.ProjectionStoreV2; + // "foreign" moved to another project, with its own subagent, after the walk found it. + const { projectId } = yield* store.getThread(parent); + const foreign = ThreadId.make("foreign"); + yield* f.set(ThreadId.make("foreign-child"), { + lineage: { + rootThreadId: parent, + parentThreadId: foreign, + relationshipToParent: "subagent", + }, + projectId: ProjectId.make("foreign"), + }); + const children = orchestrator.getSubagentChildThreads; + const stale = store.getThread(foreign).pipe( + Effect.orDie, + Effect.map((thread) => ({ ...thread, projectId })), + ); + using _stale = vi + .spyOn(orchestrator, "getSubagentChildThreads") + .mockImplementation((id) => + id === parent ? Effect.zipWith(children(id), stale, (c, t) => [...c, t]) : children(id), + ); + const all = [native, app, grandchild, "fork", "fork-child", "foreign", "foreign-child"]; + for (const operation of ["archive", "unarchive", "delete"] as const) { + // A deleted intermediate must not hide surviving descendants from deletion. + if (operation === "delete") yield* f.set(app, { deletedAt: f.now }); + // Settled before the reply, without the effect worker, so a client refresh sees it. + yield* f.run(operation); + const field = operation === "delete" ? "deletedAt" : "archivedAt"; + assert.equal( + yield* f.state(field, ...all), + operation === "unarchive" ? "0000000" : "1110000", + ); + const effects = yield* f.outbox.listByCommandId( + CommandId.make(`${operation}:parent:subagent:native`), + ); + if (operation !== "unarchive") + assert.isTrue(effects.some((e) => e.request.type === "terminal.cleanup")); + } + }).pipe(Effect.provide(layerTest)), + ); + it.effect("leaves a replayed parent command's descendants to its committed cascade", () => + Effect.gen(function* () { + const f = yield* fixture; + const limit = { runtimeMode: "approval-required", interactionMode: "plan" } as const; + yield* f.set(parent, limit); + yield* f.run("archive").pipe(Effect.provideService(DispatchModeLimit, limit)); + assert.equal(yield* f.archived(parent, native, app), "100"); + // A retry without the caller's limit must not walk the subtree under its own. + yield* f.run("archive"); + assert.equal(yield* f.archived(native, app), "00"); + }).pipe(Effect.provide(layerTest)), + ); + it.effect("restores only what a parent archive cascaded, through independent archives", () => + Effect.gen(function* () { + const f = yield* fixture; + // An earlier independent archive whose ID matches the one the parent cascade derives. + yield* f.run("archive", app, "archive:parent:subagent:app"); + yield* f.drain; + yield* f.run("archive"); + yield* f.drain; + yield* f.run("unarchive"); + yield* f.drain; + assert.equal(yield* f.archived(app, grandchild, native), "110"); + yield* f.run("unarchive", grandchild); + yield* f.run("archive", parent, "archive-owned"); + yield* f.drain; + yield* f.run("unarchive", native); + // An unused retry-shaped user ID is not a cascade retry. + yield* f.run("archive", native, "archive-owned:subagent:native:retry:31"); + yield* f.run("unarchive", parent, "unarchive-superseded"); + yield* f.run("archive", parent, "archive-superseding"); + yield* f.drain; + yield* TestClock.adjust("8 days"); + assert.isAbove(yield* f.outbox.pruneSettled, 0); + yield* f.run("unarchive", parent, "unarchive-latest"); + yield* f.drain; + assert.equal(yield* f.archived(app, native, grandchild), "110"); + }).pipe(Effect.provide(layerTest)), + ); + it.effect("skips a superseded parent action and keeps descendant choices made after it", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.commit("archive"); + // Newer choices whose IDs only look like this cascade's still win. + yield* f.commit("unarchive", parent, "archive:parent:subagent:parent"); + yield* f.drain; + assert.isTrue(Option.isNone(yield* receipt("archive:parent:subagent:app"))); + yield* f.commit("archive", app); + yield* f.drain; + yield* f.commit("unarchive", grandchild); + yield* f.commit("archive", parent, "archive-later"); + yield* f.commit("unarchive", app, "archive-later:subagent:app:retry:31"); + yield* f.drain; + assert.equal(yield* f.archived(parent, native, app, grandchild), "1100"); + yield* f.commit("unarchive", parent, "unarchive-later"); + yield* f.commit("archive", app, "archive-app-later"); + yield* f.drain; + assert.equal(yield* f.archived(parent, native, app, grandchild), "0011"); + yield* f.set(app, { deletedAt: f.now }); + for (const operation of ["archive", "unarchive"] as const) { + yield* f.set(grandchild, { archivedAt: operation === "archive" ? null : f.now }); + yield* f.commit(operation, parent, `${operation}-tombstone`).pipe(Effect.andThen(f.drain)); + assert.equal(yield* f.archived(grandchild), operation === "archive" ? "0" : "1"); + } + // A root tombstoned without its own receipt, like a side chat deleted with its parent. + yield* f.commit("archive", parent, "archive-deleted"); + yield* f.set(parent, { deletedAt: f.now }).pipe(Effect.andThen(f.drain)); + assert.isTrue(Option.isNone(yield* receipt("archive-deleted:subagent:native"))); + }).pipe(Effect.provide(layerTest)), + ); + it.effect.each([31, 32])("bounds child retries at 32 IDs with %i rejected", (count) => + Effect.gen(function* () { + const f = yield* fixture; + const receipts = yield* Receipts.CommandReceiptStoreV2; + const created = Option.getOrThrow(yield* receipt("create:parent")); + for (let retry = 0; retry < count; retry++) { + const id = CommandId.make(`archive:parent:subagent:app${retry ? `:retry:${retry}` : ""}`); + yield* receipts.upsert({ ...created, commandId: id, threadId: app, status: "rejected" }); + } + yield* f.run("archive"); + // Lose only the outbox acknowledgement, after the child commands commit. + vi.spyOn(f.outbox, "succeed").mockImplementationOnce(() => Effect.succeed(true)); + yield* f.drain; + const accepted = yield* receipt("archive:parent:subagent:app:retry:31"); + if (count === 32) { + const { status, attemptCount, lastError } = yield* f.cascade; + assert.deepEqual([status, attemptCount], ["pending", 1]); + assert.include(lastError, "All 32 subagent lifecycle command IDs were rejected."); + } else { + assert.equal(Option.getOrThrow(accepted).status, "accepted"); + assert.equal((yield* f.cascade).status, "running"); + yield* f.run("unarchive", app); + yield* f.drain; + assert.equal((yield* f.outbox.reconcileAfterProcessLoss).requeued, 1); + yield* f.drain; + assert.equal((yield* f.cascade).status, "succeeded"); + assert.deepEqual(yield* receipt("archive:parent:subagent:app:retry:31"), accepted); + } + assert.equal(yield* f.archived(native, app, grandchild), "100"); + assert.isTrue(Option.isNone(yield* receipt("archive:parent:subagent:app:retry:32"))); + }).pipe(Effect.provide(layerTest)), + ); + it.effect.each([ + { type: "thread.runtime-mode.set", runtimeMode: "full-access", alreadyArchived: false }, + { type: "thread.interaction-mode.set", interactionMode: "default", alreadyArchived: true }, + ] as const)("keeps the caller's limit after $type", ({ alreadyArchived, ...raise }) => + Effect.gen(function* () { + const f = yield* fixture; + const commandId = CommandId.make("raise"); + const limit = { runtimeMode: "approval-required", interactionMode: "plan" } as const; + for (const id of [parent, app, native, grandchild]) yield* f.set(id, limit); + if (alreadyArchived) { + // The child's archive was accepted before the process was lost, so recovery replays it. + const receipts = yield* Receipts.CommandReceiptStoreV2; + const created = Option.getOrThrow(yield* receipt("create:parent")); + const id = CommandId.make("archive:parent:subagent:app"); + yield* receipts.upsert({ ...created, commandId: id, threadId: app }); + yield* f.set(app, { archivedAt: f.now }); + } + yield* f.commit("archive").pipe(Effect.provideService(DispatchModeLimit, limit)); + const claimed = yield* f.outbox.claimNext({ workerId: "lost-caller" }); + assert.equal(Option.getOrThrow(claimed).request.type, "subagent-threads.cascade"); + assert.equal((yield* f.outbox.reconcileAfterProcessLoss).requeued, 1); + // The raise commits just before the walk takes app's lock: for dispatch, or after a replay. + const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor; + const { withLock } = executor; + const raised = Effect.orDie(f.threads.dispatch({ ...raise, commandId, threadId: app })); + let requests = 0; + const lock: typeof withLock = (id, effect) => + id === app && ++requests === (alreadyArchived ? 2 : 1) + ? raised.pipe(Effect.andThen(withLock(id, effect))) + : withLock(id, effect); + using _lock = vi.spyOn(executor, "withLock").mockImplementation(lock); + yield* f.drain; + assert.equal(yield* f.archived(native, app, grandchild), alreadyArchived ? "110" : "100"); + assert.equal((yield* f.cascade).status, "succeeded"); + }).pipe(Effect.provide(layerTest)), + ); +}); diff --git a/apps/server/src/persistence/OrchestrationCommandReceipts.ts b/apps/server/src/persistence/OrchestrationCommandReceipts.ts index 4a39061a00cb0..0d20e04c49497 100644 --- a/apps/server/src/persistence/OrchestrationCommandReceipts.ts +++ b/apps/server/src/persistence/OrchestrationCommandReceipts.ts @@ -41,6 +41,13 @@ export class OrchestrationCommandReceiptRepository extends Context.Service< receipt: OrchestrationCommandReceipt, ) => Effect.Effect; + readonly getLatestLifecycle: (input: { + readonly threadId: ThreadId; + }) => Effect.Effect< + Option.Option, + OrchestrationCommandReceiptRepositoryError + >; + readonly getByCommandId: ( input: GetByCommandIdInput, ) => Effect.Effect< @@ -108,6 +115,20 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { `, }); + const findLatestLifecycle = SqlSchema.findOneOption({ + Request: Schema.Struct({ threadId: ThreadId }), + Result: OrchestrationCommandReceipt, + execute: ({ threadId }) => sql` + SELECT command_id AS "commandId", aggregate_kind AS "aggregateKind", + aggregate_id AS "aggregateId", command_type AS "commandType", + accepted_at AS "acceptedAt", result_sequence AS "resultSequence", status, error + FROM orchestration_command_receipts + WHERE aggregate_kind = 'thread' AND aggregate_id = ${threadId} + AND status = 'accepted' AND command_type IN ('thread.archive', 'thread.unarchive', 'thread.delete') + ORDER BY result_sequence DESC LIMIT 1 + `, + }); + const upsert: OrchestrationCommandReceiptRepository["Service"]["upsert"] = (receipt) => upsertReceiptRow(receipt).pipe( Effect.mapError(toPersistenceSqlError("OrchestrationCommandReceiptRepository.upsert:query")), @@ -157,6 +178,12 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { ); return { + getLatestLifecycle: (input) => + findLatestLifecycle(input).pipe( + Effect.mapError( + toPersistenceSqlError("OrchestrationCommandReceiptRepository.getLatestLifecycle:query"), + ), + ), insertIfAbsent, upsert, getByCommandId, diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index d7ad9ed1b1746..012427506f6f1 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -262,6 +262,7 @@ const makeTestRelay = Effect.fnUntraced(function* ( forceStopThread: () => Effect.die("unused forceStopThread"), awaitBackgroundTasks: unused, stopDelegatedTasks: unused, + cascadeSubagentLifecycle: unused, getThreadEventSequence: unused, recoverDelegatedTask: unused, cancelDelegatedTask: unused, diff --git a/docs/user/thread-sidebar.md b/docs/user/thread-sidebar.md index f58f35feb45f5..16c832b438b31 100644 --- a/docs/user/thread-sidebar.md +++ b/docs/user/thread-sidebar.md @@ -281,6 +281,8 @@ fill the screen; close the full-screen view to return to the parent and inspecto Stop on a thread also stops the subagents it delegated to. Before archiving the parent, wait for its delegated tasks to finish or cancel them. +Archiving, restoring or deleting a thread also applies to its subagent descendants, but not to forks. Restoring brings back only what the parent's archive archived, so independent archives and newer changes on a descendant stay as they are. + Subagent threads started by the agent can't take messages; message the parent thread instead. When such a subagent needs an approval or an answer, the parent thread asks for it. From 2e155e2654b2e97805b430215dd7799dfcdcb62d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 09:25:04 +0000 Subject: [PATCH 02/23] fix(server): preserve cascades through upstream lifecycle changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Acquire deletion ownership before cascading thread locks and finish side-chat descendants before lifecycle replies. Retain side-chat archive receipts after outbox pruning. > 🤖 Generated by GPT 6.1 Sol in Codex via T3 Code. --- apps/server/src/orchestration-v2/EventSink.ts | 15 ++ .../src/orchestration-v2/Orchestrator.ts | 7 +- .../ThreadManagementService.ts | 70 ++++++--- .../src/orchestration-v2/runtimeLayer.test.ts | 136 +++++++++++++++++- docs/user/thread-sidebar.md | 2 +- 5 files changed, 200 insertions(+), 30 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 3f6744e576edd..a854b5a992202 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -870,6 +870,21 @@ const layerBase: Layer.Layer< error: null, }; yield* commandReceipts.upsert(receipt); + // Generated side-chat cascades need archive ownership after their outbox entries are pruned. + for (const effect of input.effects) { + if ( + effect.commandId !== input.commandId && + effect.request.type === "subagent-threads.cascade" && + effect.request.operation !== "thread.delete" + ) { + yield* commandReceipts.insertIfAbsent({ + ...receipt, + commandId: effect.commandId, + threadId: effect.threadId, + commandType: effect.request.operation, + }); + } + } const cancelledEffectIds = input.cancelUnsettledEffects === undefined ? [] diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 3c833942c5bb3..cfcd392d20567 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -1906,7 +1906,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio { commandId, threadId }: { readonly commandId: CommandId; readonly threadId: ThreadId }, operation: "thread.archive" | "thread.unarchive" | "thread.delete", ) { - if (yield* EffectOutbox.SubagentCascade) return []; + const cascade = yield* EffectOutbox.SubagentCascade; + if (cascade?.path.at(-1) === threadId) return []; if ((yield* projectionStore.getSubagentChildThreads(threadId)).length === 0) return []; // The outbox schema persists only the caller's runtime and interaction modes. const limit = yield* DispatchModeLimit; @@ -16465,7 +16466,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ), ); - return yield* command.type === "thread.delete" + // A subagent cascade already owns the deletion lock outside its ancestor thread locks. + return yield* command.type === "thread.delete" && + (yield* EffectOutbox.SubagentCascade) === undefined ? threadDispatch.withDeletionLock(cascade) : cascade; } diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index b0866c42cee52..bafb995e45ac2 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -858,28 +858,48 @@ const make = Effect.gen(function* () { // Clients read archive snapshots as soon as this returns; the outbox effect only backs it up. // A replay or a no-op (an unused-only delete that kept the thread) leaves it to that effect. - const cascadeBeforeReply = ( + const cascadeBeforeReply = Effect.fn("ThreadManagementService.cascadeBeforeReply")(function* ( command: OrchestrationV2ServerCommand, result: Orchestrator.OrchestratorV2DispatchResult, - ) => - (command.type === "thread.archive" || - command.type === "thread.unarchive" || - command.type === "thread.delete") && - result.replayed !== true && - result.storedEvents.length > 0 - ? cascadeSubagentLifecycle({ - commandId: command.commandId, - threadId: command.threadId, + sideChatsOnly = false, + ) { + if ( + (command.type !== "thread.archive" && + command.type !== "thread.unarchive" && + command.type !== "thread.delete") || + result.replayed === true + ) + return; + const eventType = + command.type === "thread.archive" + ? "thread.archived" + : command.type === "thread.unarchive" + ? "thread.unarchived" + : "thread.deleted"; + yield* Effect.forEach( + result.storedEvents.filter( + ({ event }) => + event.type === eventType && (!sideChatsOnly || event.threadId !== command.threadId), + ), + ({ event }) => + cascadeSubagentLifecycle({ + commandId: + event.threadId === command.threadId + ? command.commandId + : CommandId.make(`${command.commandId}:side-chat:${event.threadId}`), + threadId: event.threadId, operation: command.type, }).pipe( Effect.catch((cause) => Effect.logWarning("Subagent lifecycle cascade left to the effect worker", { - threadId: command.threadId, + threadId: event.threadId, cause, }), ), - ) - : Effect.void; + ), + { discard: true }, + ); + }); const getProjectThreadRecords: ThreadManagementServiceShape["getProjectThreadRecords"] = ( input, @@ -1827,15 +1847,15 @@ const make = Effect.gen(function* () { "ThreadManagementService.visitSubagentCascade", )(function* (thread, commandId, path) { if (path.length > 0 && thread.deletedAt === null) { - const dispatched = yield* orchestrator - .dispatch({ type: input.operation, commandId, threadId: thread.id }) - .pipe( - Effect.provideService(EffectOutbox.SubagentCascade, { source, path }), - Effect.as(true), - Effect.catchTags({ - OrchestratorThreadAboveModeLimitError: () => Effect.succeed(false), - }), - ); + const command = { type: input.operation, commandId, threadId: thread.id }; + const dispatched = yield* orchestrator.dispatch(command).pipe( + Effect.tap((result) => cascadeBeforeReply(command, result, true)), + Effect.provideService(EffectOutbox.SubagentCascade, { source, path }), + Effect.as(true), + Effect.catchTags({ + OrchestratorThreadAboveModeLimitError: () => Effect.succeed(false), + }), + ); if (!dispatched) return; } // Holds each thread's lock while visiting its subagents, so locks nest parent before child. @@ -1869,7 +1889,11 @@ const make = Effect.gen(function* () { }), ); }); - yield* visit({ id: input.threadId, deletedAt: null }, input.commandId, []); + const cascade = visit({ id: input.threadId, deletedAt: null }, input.commandId, []); + yield* input.operation === "thread.delete" && + (yield* EffectOutbox.SubagentCascade) === undefined + ? threadDispatch.withDeletionLock(cascade) + : cascade; }); return ThreadManagementService.of({ diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 86a4ca0f7b0d8..3bb54449badbc 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14183,7 +14183,41 @@ describe("subagent lifecycle cascade", () => { Option.getOrThrow, ); const drain = (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); - return { threads, now, set, run, commit, state, archived, outbox, cascade, drain }; + const sideChat = Effect.fn("test.seedSideChat")(function* (source: ThreadId, id: ThreadId) { + yield* set(id, { + lineage: { + rootThreadId: parent, + parentThreadId: source, + relationshipToParent: "fork", + sideChat: true, + }, + }); + yield* store.apply({ + id: EventId.make(`fork:${id}`), + type: "context-transfer.created", + threadId: source, + occurredAt: now, + payload: { + id: ContextTransferId.make(`fork:${id}`), + type: "fork", + sourceThreadId: source, + targetThreadId: id, + sourcePoint: { threadId: source }, + basePoint: null, + sourceProviderInstanceId: null, + targetProviderInstanceId: null, + targetRunId: null, + status: "consumed", + resolution: null, + createdBy: "user", + error: null, + createdAt: now, + updatedAt: now, + consumedAt: now, + }, + }); + }); + return { threads, now, set, sideChat, run, commit, state, archived, outbox, cascade, drain }; }); const receipt = (id: string) => Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.getByCommandId(CommandId.make(id))); @@ -14244,6 +14278,95 @@ describe("subagent lifecycle cascade", () => { assert.equal(yield* f.archived(native, app), "00"); }).pipe(Effect.provide(layerTest)), ); + it.effect("holds the deletion lock before every cascade thread lock", () => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const sideChild = ThreadId.make("side-child"); + yield* f.sideChat(app, side); + yield* f.set(sideChild, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor; + const { withLock, withDeletionLock } = executor; + const held = new Set(); + let deleting = false; + const lock: typeof withLock = (id, effect) => + withLock( + id, + Effect.acquireUseRelease( + Effect.sync(() => { + assert.isTrue(deleting, "a cascade thread lock requires the deletion lock"); + held.add(id); + }), + () => effect, + () => Effect.sync(() => held.delete(id)), + ), + ); + const deletion: typeof withDeletionLock = (effect) => + Effect.suspend(() => { + assert.equal(held.size, 0, "the deletion lock precedes all ancestor thread locks"); + assert.isFalse(deleting, "the deletion lock must not be acquired recursively"); + return withDeletionLock( + Effect.acquireUseRelease( + Effect.sync(() => { + deleting = true; + }), + () => effect, + () => + Effect.sync(() => { + deleting = false; + }), + ), + ); + }); + using _lock = vi.spyOn(executor, "withLock").mockImplementation(lock); + using _deletion = vi.spyOn(executor, "withDeletionLock").mockImplementation(deletion); + yield* f.run("delete"); + yield* f.drain; + assert.equal( + yield* f.state("deletedAt", parent, app, native, grandchild, side, sideChild), + "111111", + ); + }).pipe(Effect.provide(layerTest)), + ); + it.effect.each([ + { source: parent, delivery: "reply" }, + { source: parent, delivery: "worker" }, + { source: app, delivery: "reply" }, + { source: app, delivery: "worker" }, + ] as const)( + "cascades side-chat descendants from $source through $delivery", + ({ source, delivery }) => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + const independent = ThreadId.make("side-independent"); + yield* f.sideChat(source, side); + for (const id of [child, independent]) { + yield* f.set(id, { + lineage: { + rootThreadId: parent, + parentThreadId: side, + relationshipToParent: "subagent", + }, + }); + } + yield* f.run("archive", independent); + const apply = (operation: "archive" | "unarchive") => + delivery === "reply" + ? f.run(operation) + : f.commit(operation).pipe(Effect.andThen(f.drain)); + yield* apply("archive"); + assert.equal(yield* f.archived(side, child, independent), "111"); + yield* f.drain; + yield* TestClock.adjust("8 days"); + yield* f.outbox.pruneSettled; + yield* apply("unarchive"); + assert.equal(yield* f.archived(side, child, independent), "001"); + }).pipe(Effect.provide(layerTest)), + ); it.effect("restores only what a parent archive cascaded, through independent archives", () => Effect.gen(function* () { const f = yield* fixture; @@ -14361,13 +14484,18 @@ describe("subagent lifecycle cascade", () => { const raised = Effect.orDie(f.threads.dispatch({ ...raise, commandId, threadId: app })); let requests = 0; const lock: typeof withLock = (id, effect) => - id === app && ++requests === (alreadyArchived ? 2 : 1) - ? raised.pipe(Effect.andThen(withLock(id, effect))) - : withLock(id, effect); + Effect.suspend(() => + id === app && ++requests === (alreadyArchived ? 2 : 1) + ? raised.pipe(Effect.andThen(withLock(id, effect))) + : withLock(id, effect), + ); using _lock = vi.spyOn(executor, "withLock").mockImplementation(lock); yield* f.drain; assert.equal(yield* f.archived(native, app, grandchild), alreadyArchived ? "110" : "100"); assert.equal((yield* f.cascade).status, "succeeded"); + const orchestrator = yield* Orchestrator.OrchestratorV2; + const { type: _type, ...mode } = raise; + assert.include((yield* orchestrator.getThreadRecords(app, [])).thread, mode); }).pipe(Effect.provide(layerTest)), ); }); diff --git a/docs/user/thread-sidebar.md b/docs/user/thread-sidebar.md index 16c832b438b31..024c3976ea56f 100644 --- a/docs/user/thread-sidebar.md +++ b/docs/user/thread-sidebar.md @@ -281,7 +281,7 @@ fill the screen; close the full-screen view to return to the parent and inspecto Stop on a thread also stops the subagents it delegated to. Before archiving the parent, wait for its delegated tasks to finish or cancel them. -Archiving, restoring or deleting a thread also applies to its subagent descendants, but not to forks. Restoring brings back only what the parent's archive archived, so independent archives and newer changes on a descendant stay as they are. +Archiving, restoring or deleting a thread also applies to its subagent descendants, including subagents in its side chats. Ordinary forks stay independent. Restoring brings back only descendants archived by that action, so independent archives and newer changes on a descendant stay as they are. Subagent threads started by the agent can't take messages; message the parent thread instead. When such a subagent needs an approval or an answer, the parent From 94ab3232e0f2c4b4c03e6b86db180fb7e660b332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 09:49:34 +0000 Subject: [PATCH 03/23] fix(server): reject conflicting side-chat cascade receipts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep existing command receipts and roll back the lifecycle transaction when a generated side-chat source ID is already occupied. Cover archive and restore collisions through reply and worker paths. > 🤖 Reviewed until clean by GPT 6.1 Sol 1, GPT 6.1 Sol 2, GPT 6.1 Sol 3, GPT 6.1 Sol 4, CodeRabbit. Findings that reach consensus across the majority were addressed by the generation model. > 🤖 Generated by GPT 6.1 Sol in Codex via T3 Code. --- apps/server/src/orchestration-v2/EventSink.ts | 8 ++- .../src/orchestration-v2/runtimeLayer.test.ts | 66 ++++++++++++++++--- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index a854b5a992202..b207c770a7c28 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -877,12 +877,18 @@ const layerBase: Layer.Layer< effect.request.type === "subagent-threads.cascade" && effect.request.operation !== "thread.delete" ) { - yield* commandReceipts.insertIfAbsent({ + const reserved = yield* commandReceipts.insertIfAbsent({ ...receipt, commandId: effect.commandId, threadId: effect.threadId, commandType: effect.request.operation, }); + if (!reserved) { + return yield* new CommandReceiptStore.CommandReceiptStoreWriteError({ + commandId: effect.commandId, + cause: "Generated side-chat cascade command ID is already in use.", + }); + } } } const cancelledEffectIds = diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 3bb54449badbc..a488e8cbe446d 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14331,13 +14331,21 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); it.effect.each([ - { source: parent, delivery: "reply" }, - { source: parent, delivery: "worker" }, - { source: app, delivery: "reply" }, - { source: app, delivery: "worker" }, + { source: parent, delivery: "reply", operation: "archive", collision: null }, + { source: parent, delivery: "worker", operation: "archive", collision: null }, + { source: app, delivery: "reply", operation: "archive", collision: null }, + { source: app, delivery: "worker", operation: "archive", collision: null }, + { source: parent, delivery: "reply", operation: "archive", collision: "accepted" }, + { source: parent, delivery: "worker", operation: "archive", collision: "accepted" }, + { source: parent, delivery: "reply", operation: "archive", collision: "rejected" }, + { source: parent, delivery: "worker", operation: "archive", collision: "rejected" }, + { source: parent, delivery: "reply", operation: "unarchive", collision: "accepted" }, + { source: parent, delivery: "worker", operation: "unarchive", collision: "accepted" }, + { source: parent, delivery: "reply", operation: "unarchive", collision: "rejected" }, + { source: parent, delivery: "worker", operation: "unarchive", collision: "rejected" }, ] as const)( - "cascades side-chat descendants from $source through $delivery", - ({ source, delivery }) => + "cascades side-chat descendants from $source through $delivery: $operation, collision $collision", + ({ source, delivery, operation, collision }) => Effect.gen(function* () { const f = yield* fixture; const side = ThreadId.make("side"); @@ -14354,10 +14362,48 @@ describe("subagent lifecycle cascade", () => { }); } yield* f.run("archive", independent); - const apply = (operation: "archive" | "unarchive") => - delivery === "reply" - ? f.run(operation) - : f.commit(operation).pipe(Effect.andThen(f.drain)); + const apply = Effect.fn("test.applySideChatLifecycle")(function* ( + operation: "archive" | "unarchive", + ) { + if (delivery === "reply") yield* f.run(operation); + else { + yield* f.commit(operation); + yield* f.drain; + } + }); + if (collision !== null) { + if (operation === "unarchive") yield* apply("archive"); + const before = yield* f.archived(parent, side, child, independent); + const commandId = CommandId.make(`${operation}:${parent}`); + const existing = { + commandId: CommandId.make(`${commandId}:side-chat:${side}`), + threadId: ThreadId.make("fork"), + commandType: "thread.archive", + acceptedAt: f.now, + resultSequence: 0, + status: collision, + error: collision === "rejected" ? "Earlier rejected command." : null, + }; + yield* Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.upsert(existing)); + const events = yield* EventSink.EventSinkV2; + const sequence = yield* events.latestSequence(); + const result = yield* apply(operation).pipe(Effect.flip); + assert.equal(result._tag, "OrchestratorDispatchError"); + yield* f.drain; + assert.equal(yield* f.archived(parent, side, child, independent), before); + assert.deepEqual(Option.getOrThrow(yield* receipt(existing.commandId)), existing); + assert.isTrue(Option.isNone(yield* receipt(commandId))); + assert.equal(yield* events.latestSequence(), sequence); + assert.isTrue( + Option.isNone(yield* f.outbox.get(`effect:${commandId}:subagent-threads.cascade`)), + ); + assert.isTrue( + Option.isNone( + yield* f.outbox.get(`effect:${existing.commandId}:subagent-threads.cascade`), + ), + ); + return; + } yield* apply("archive"); assert.equal(yield* f.archived(side, child, independent), "111"); yield* f.drain; From 27d5b43680bd4ba6dde22c8bbd6101b59d9877a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 14:00:46 +0000 Subject: [PATCH 04/23] fix(server): preserve side-chat cascade ancestry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep the initiating archive sequence, verify the embedded archive receipt, and check the project boundary before walking side-chat subagents in either delivery path. Preserve older delete jobs and collapse duplicate collision cases. > 🤖 Reviewed until clean by GPT 6.2 Astra, GPT 6.2 Sol, CodeRabbit. Findings that reach consensus across the majority were addressed by the generation model. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../orchestration-v2/CommandReceiptStore.ts | 10 +- .../src/orchestration-v2/EffectOutbox.ts | 29 ++++- .../src/orchestration-v2/EffectWorker.ts | 1 + .../src/orchestration-v2/Orchestrator.ts | 116 ++++++++++++++---- .../ThreadManagementService.ts | 101 ++++++++++----- .../src/orchestration-v2/runtimeLayer.test.ts | 94 +++++++++++++- 6 files changed, 277 insertions(+), 74 deletions(-) diff --git a/apps/server/src/orchestration-v2/CommandReceiptStore.ts b/apps/server/src/orchestration-v2/CommandReceiptStore.ts index 7c2b12cdcfec9..a462b1d042dc6 100644 --- a/apps/server/src/orchestration-v2/CommandReceiptStore.ts +++ b/apps/server/src/orchestration-v2/CommandReceiptStore.ts @@ -237,12 +237,9 @@ export const cascadeCommandId = Effect.fn("CommandReceiptStore.cascadeCommandId" /** A lifecycle choice made after the cascade's source command, other than `ownCommandId`. */ export const supersedesCascade = ( latest: CommandReceiptV2 | null, - source: CommandReceiptV2, + cutoff: number, ownCommandId: CommandId, -) => - latest !== null && - latest.resultSequence > source.resultSequence && - latest.commandId !== ownCommandId; +) => latest !== null && latest.resultSequence > cutoff && latest.commandId !== ownCommandId; /** Whether a cascade from an archive of `rootThreadId`, along `path`, archived this thread. */ export const archivedByCascade = Effect.fn("CommandReceiptStore.archivedByCascade")(function* ( @@ -250,6 +247,7 @@ export const archivedByCascade = Effect.fn("CommandReceiptStore.archivedByCascad latest: CommandReceiptV2 | null, rootThreadId: ThreadId, path: ReadonlyArray, + expectedArchiveCommandId?: CommandId, ) { if (latest?.commandType !== "thread.archive") return false; let rootCommandId: string = latest.commandId; @@ -259,6 +257,8 @@ export const archivedByCascade = Effect.fn("CommandReceiptStore.archivedByCascad if (!rootCommandId.endsWith(segment)) return false; rootCommandId = rootCommandId.slice(0, -segment.length); } + if (expectedArchiveCommandId !== undefined && rootCommandId !== expectedArchiveCommandId) + return false; const root = Option.getOrNull(yield* receipts.getByCommandId(CommandId.make(rootCommandId))); const valid = root?.status === "accepted" && root.threadId === rootThreadId; // Derive the IDs again, so a user ID only shaped like a cascade retry is not counted. diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index d7706873936a8..163b0d80eefaf 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -13,6 +13,7 @@ import { ProviderThreadId, ProviderTurnId, ProviderInteractionMode, + ProjectId, RunId, RuntimeMode, RuntimeRequestId, @@ -33,11 +34,27 @@ import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import { forkParked } from "../serverActivation.ts"; -import type { CommandReceiptV2 } from "./CommandReceiptStore.ts"; + +const EmbeddedSideChatCascade = Schema.Struct({ + parentThreadId: ThreadId, + projectId: ProjectId, + cutoff: Schema.optional(NonNegativeInt), + archiveCommandId: Schema.optional(CommandId), +}); +export type EmbeddedSideChatCascade = typeof EmbeddedSideChatCascade.Type; /** Set while a cascade dispatches a descendant's lifecycle command; delete needs no source. */ export const SubagentCascade = Context.Reference< - | { readonly source: CommandReceiptV2 | undefined; readonly path: ReadonlyArray } + | { + readonly source: + | { + readonly threadId: ThreadId; + readonly cutoff: number; + readonly archiveCommandId?: CommandId | undefined; + } + | undefined; + readonly path: ReadonlyArray; + } | undefined >("t3/orchestration-v2/SubagentCascade", { defaultValue: () => undefined }); @@ -139,6 +156,7 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ Schema.Struct({ type: Schema.Literal("subagent-threads.cascade"), operation: Schema.Literals(["thread.archive", "thread.unarchive", "thread.delete"]), + sideChat: Schema.optional(EmbeddedSideChatCascade), limit: Schema.optional( Schema.Struct({ runtimeMode: RuntimeMode, interactionMode: ProviderInteractionMode }), ), @@ -217,6 +235,13 @@ export interface PendingOrchestrationEffectV2 { readonly availableAt?: DateTime.Utc; } +export type SubagentCascadeEffect = PendingOrchestrationEffectV2 & { + readonly request: Extract< + OrchestrationEffectRequestV2, + { readonly type: "subagent-threads.cascade" } + >; +}; + export class EffectOutboxError extends Schema.TaggedError()( "EffectOutboxError", { diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index d7791f67c67f4..428eb2477b772 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -687,6 +687,7 @@ export const layerExecutor: Layer.Layer< commandId: effect.commandId, threadId: effect.threadId, operation: effect.request.operation, + sideChat: effect.request.sideChat, }) .pipe( Effect.provideService(DispatchModeLimit, effect.request.limit), diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index cfcd392d20567..ebbc1d59b102e 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -401,6 +401,7 @@ export type OrchestratorV2Error = typeof OrchestratorV2Error.Type; export interface OrchestratorV2DispatchResult { readonly sequence: number; readonly storedEvents: ReadonlyArray; + readonly subagentCascades?: ReadonlyArray; /** The command id was already accepted; analytics skip it. */ readonly replayed?: true; } @@ -1901,19 +1902,29 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .findThread(threadId) .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))); + const sideChatLifecycleCommandId = (commandId: CommandId, threadId: ThreadId) => + CommandId.make(`${commandId}:side-chat:${threadId}`); + // A cascade walks the whole subtree itself, so its own commands start none. const subagentCascadeEffects = Effect.fn("Orchestrator.subagentCascadeEffects")(function* ( { commandId, threadId }: { readonly commandId: CommandId; readonly threadId: ThreadId }, operation: "thread.archive" | "thread.unarchive" | "thread.delete", + sideChat?: EffectOutbox.EmbeddedSideChatCascade, ) { const cascade = yield* EffectOutbox.SubagentCascade; if (cascade?.path.at(-1) === threadId) return []; if ((yield* projectionStore.getSubagentChildThreads(threadId)).length === 0) return []; - // The outbox schema persists only the caller's runtime and interaction modes. const limit = yield* DispatchModeLimit; - const request = { type: "subagent-threads.cascade", operation, limit } as const; + const request = { + type: "subagent-threads.cascade", + operation, + limit, + ...(sideChat === undefined + ? {} + : { sideChat: { ...sideChat, cutoff: cascade?.source?.cutoff } }), + } as const; const id = `effect:${commandId}:subagent-threads.cascade`; - return [{ id, commandId, threadId, request } satisfies PendingOrchestrationEffectV2]; + return [{ id, commandId, threadId, request } satisfies EffectOutbox.SubagentCascadeEffect]; }); /** Side chats live inside their parent, so deleting or archiving the parent takes them too. */ @@ -4391,6 +4402,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio >, events: Ref.Ref>, effects: Ref.Ref>, + sideChat?: EffectOutbox.EmbeddedSideChatCascade, ) { const thread = yield* projectionStore.getThread(command.threadId).pipe( Effect.mapError( @@ -5624,7 +5636,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } if (command.type === "thread.archive" || command.type === "thread.unarchive") { - const cascade = yield* subagentCascadeEffects(command, command.type).pipe( + const cascade = yield* subagentCascadeEffects(command, command.type, sideChat).pipe( mapDispatchError(command), ); yield* Ref.update(effects, (existing) => [...existing, ...cascade]); @@ -15665,6 +15677,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const now = yield* DateTime.now; const planDeletion = Effect.fn("Orchestrator.planDeletion")(function* ( deletion: typeof command, + sideChat?: EffectOutbox.EmbeddedSideChatCascade, ) { const projection = yield* projectionStore .getThreadRecords(deletion.threadId, [ @@ -15691,10 +15704,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio idAllocator, }), ); - const cascade = yield* subagentCascadeEffects(deletion, deletion.type).pipe( + const cascade = yield* subagentCascadeEffects(deletion, deletion.type, sideChat).pipe( mapDispatchError(command), ); - return { ...plan, effects: [...plan.effects, ...cascade] }; + return { + ...plan, + projectId: projection.thread.projectId, + effects: [...plan.effects, ...cascade], + }; }); // Checked under the thread lock: a message sent since the client looked keeps the thread. if (command.onlyIfUnused === true) { @@ -15710,14 +15727,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } } // The dispatch wrapper holds each side chat's lock, so none can commit after this plan. + const parent = yield* planDeletion(command); const plans = [ - yield* planDeletion(command), + parent, ...(yield* Effect.forEach(sideChatIds, (threadId) => - planDeletion({ - ...command, - commandId: CommandId.make(`${command.commandId}:side-chat:${threadId}`), - threadId, - }), + planDeletion( + { + ...command, + commandId: sideChatLifecycleCommandId(command.commandId, threadId), + threadId, + }, + { parentThreadId: command.threadId, projectId: parent.projectId }, + ), )), ]; return { @@ -15727,8 +15748,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } case "thread.archive": case "thread.unarchive": { - const parentArchivedAt = (yield* findThread(command.threadId))?.archivedAt ?? null; + const parent = yield* findThread(command.threadId); + const parentArchivedAt = parent?.archivedAt ?? null; yield* dispatchThreadMutation(command, events, effects); + if (parent === null || sideChatIds.length === 0) break; + const parentArchive = + command.type === "thread.unarchive" + ? yield* commandReceipts.getLatestLifecycle(parent.id).pipe(mapDispatchError(command)) + : null; // Unarchive restores the side chats this parent's archive took, not ones archived before it. for (const threadId of sideChatIds) { const sideChatArchivedAt = (yield* findThread(threadId))?.archivedAt ?? null; @@ -15747,15 +15774,34 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .pipe(mapDispatchError(command)); if (hasRunningDelegatedTasks(subagents)) continue; } - yield* dispatchThreadMutation( - { - ...command, - commandId: CommandId.make(`${command.commandId}:side-chat:${threadId}`), - threadId, - }, - events, - effects, - ); + const sideChatCommand = { + ...command, + commandId: sideChatLifecycleCommandId(command.commandId, threadId), + threadId, + }; + const derivedArchiveId = + parentArchive?.commandType === "thread.archive" + ? sideChatLifecycleCommandId(parentArchive.commandId, threadId) + : undefined; + const sideChatArchive = + derivedArchiveId === undefined + ? undefined + : Option.getOrUndefined( + yield* commandReceipts + .getByCommandId(derivedArchiveId) + .pipe(mapDispatchError(command)), + ); + yield* dispatchThreadMutation(sideChatCommand, events, effects, { + parentThreadId: parent.id, + projectId: parent.projectId, + archiveCommandId: + sideChatArchive?.status === "accepted" && + sideChatArchive.commandType === "thread.archive" && + sideChatArchive.threadId === threadId && + sideChatArchive.resultSequence === parentArchive?.resultSequence + ? sideChatArchive.commandId + : undefined, + }); } break; } @@ -16186,13 +16232,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio applies = thread.deletedAt === null && (command.type === "thread.archive" - ? thread.archivedAt === null && !supersedesCascade(latest, source, command.commandId) + ? thread.archivedAt === null && + !supersedesCascade(latest, source.cutoff, command.commandId) : thread.archivedAt !== null && - (yield* archivedByCascade(commandReceipts, latest, source.threadId, path).pipe( - read, - ))); + (yield* archivedByCascade( + commandReceipts, + latest, + source.threadId, + path, + source.archiveCommandId, + ).pipe(read))); } - if (!applies) return { sequence: source?.resultSequence ?? 0, storedEvents: [] }; + if (!applies) return { sequence: source?.cutoff ?? 0, storedEvents: [] }; } const plan = yield* dispatchOnce(command, sideChatIds).pipe( @@ -16414,9 +16465,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ); } + const subagentCascades = + committed.committed && + (command.type === "thread.archive" || + command.type === "thread.unarchive" || + command.type === "thread.delete") + ? plan.effects.filter( + (effect): effect is EffectOutbox.SubagentCascadeEffect => + effect.request.type === "subagent-threads.cascade", + ) + : []; return { sequence: committed.receipt.resultSequence, storedEvents: committed.storedEvents, + ...(subagentCascades.length === 0 ? {} : { subagentCascades }), ...(committed.committed ? {} : { replayed: true as const }), } satisfies OrchestratorV2DispatchResult; }, withCommandMetrics); diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index bafb995e45ac2..d5f3e84996f52 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -557,6 +557,7 @@ export interface ThreadManagementServiceShape { readonly threadId: ThreadId; readonly commandId: CommandId; readonly operation: "thread.archive" | "thread.unarchive" | "thread.delete"; + readonly sideChat?: EffectOutbox.EmbeddedSideChatCascade | undefined; }) => Effect.Effect< void, Orchestrator.OrchestratorV2Error | CommandReceiptStore.CommandReceiptStoreV2Error @@ -853,46 +854,26 @@ const make = Effect.gen(function* () { ensureCommandTranscripts(command).pipe( Effect.andThen(assertDispatchTargetEnabled(command)), Effect.andThen(orchestrator.dispatch(command)), - Effect.tap((result) => cascadeBeforeReply(command, result)), + Effect.tap((result) => cascadeBeforeReply(result)), ); // Clients read archive snapshots as soon as this returns; the outbox effect only backs it up. // A replay or a no-op (an unused-only delete that kept the thread) leaves it to that effect. const cascadeBeforeReply = Effect.fn("ThreadManagementService.cascadeBeforeReply")(function* ( - command: OrchestrationV2ServerCommand, result: Orchestrator.OrchestratorV2DispatchResult, - sideChatsOnly = false, ) { - if ( - (command.type !== "thread.archive" && - command.type !== "thread.unarchive" && - command.type !== "thread.delete") || - result.replayed === true - ) - return; - const eventType = - command.type === "thread.archive" - ? "thread.archived" - : command.type === "thread.unarchive" - ? "thread.unarchived" - : "thread.deleted"; yield* Effect.forEach( - result.storedEvents.filter( - ({ event }) => - event.type === eventType && (!sideChatsOnly || event.threadId !== command.threadId), - ), - ({ event }) => + result.subagentCascades ?? [], + ({ commandId, threadId, request }) => cascadeSubagentLifecycle({ - commandId: - event.threadId === command.threadId - ? command.commandId - : CommandId.make(`${command.commandId}:side-chat:${event.threadId}`), - threadId: event.threadId, - operation: command.type, + commandId, + threadId, + operation: request.operation, + sideChat: request.sideChat, }).pipe( Effect.catch((cause) => Effect.logWarning("Subagent lifecycle cascade left to the effect worker", { - threadId: event.threadId, + threadId, cause, }), ), @@ -1829,12 +1810,48 @@ const make = Effect.gen(function* () { if (failures[0] !== undefined) return yield* failures[0]; }); + // Older embedded deletes have only the enclosing parent's lifecycle receipt. + const legacySideChatDeletionParent = Effect.fn( + "ThreadManagementService.legacySideChatDeletionParent", + )(function* (commandId: CommandId, threadId: ThreadId) { + const suffix = `:side-chat:${threadId}`; + if (commandId.length <= suffix.length || !commandId.endsWith(suffix)) return; + const direct = Option.getOrUndefined(yield* receipts.getByCommandId(commandId)); + if ( + direct?.threadId === threadId && + direct.commandType === "thread.delete" && + direct.status === "accepted" + ) + return; + const parentCommandId = CommandId.make(commandId.slice(0, -suffix.length)); + const parent = Option.getOrUndefined(yield* receipts.getByCommandId(parentCommandId)); + if ( + parent?.status === "accepted" && + parent.commandType === "thread.delete" && + parent.threadId !== threadId + ) + return parent.threadId; + }); + const cascadeSubagentLifecycle: ThreadManagementServiceShape["cascadeSubagentLifecycle"] = Effect.fn("ThreadManagementService.cascadeSubagentLifecycle")(function* (input) { - const receipt = Option.getOrUndefined(yield* receipts.getByCommandId(input.commandId)); - // Side chats deleted with their parent have no receipt of their own. + const { sideChat } = input; + if (input.operation === "thread.unarchive" && sideChat && !sideChat.archiveCommandId) return; + const parentThreadId = + sideChat?.parentThreadId ?? + (input.operation === "thread.delete" + ? yield* legacySideChatDeletionParent(input.commandId, input.threadId) + : undefined); + const receipt = + input.operation === "thread.delete" + ? undefined + : Option.getOrUndefined(yield* receipts.getByCommandId(input.commandId)); if (input.operation !== "thread.delete" && receipt?.status !== "accepted") return; - const source = input.operation === "thread.delete" ? undefined : receipt; + const source = receipt && { + threadId: receipt.threadId, + cutoff: sideChat?.cutoff ?? receipt.resultSequence, + archiveCommandId: sideChat?.archiveCommandId, + }; const visit: ( thread: { readonly id: ThreadId; @@ -1849,7 +1866,7 @@ const make = Effect.gen(function* () { if (path.length > 0 && thread.deletedAt === null) { const command = { type: input.operation, commandId, threadId: thread.id }; const dispatched = yield* orchestrator.dispatch(command).pipe( - Effect.tap((result) => cascadeBeforeReply(command, result, true)), + Effect.tap((result) => cascadeBeforeReply(result)), Effect.provideService(EffectOutbox.SubagentCascade, { source, path }), Effect.as(true), Effect.catchTags({ @@ -1866,10 +1883,24 @@ const make = Effect.gen(function* () { const limit = yield* DispatchModeLimit; const current = (yield* orchestrator.getThreadRecords(thread.id, [])).thread; if (path.length > 0 && current.projectId !== thread.projectId) return; + if (path.length === 0 && parentThreadId !== undefined) { + if ( + current.lineage.sideChat !== true || + current.lineage.parentThreadId !== parentThreadId + ) + return; + const { thread: parent } = yield* orchestrator.getThreadRecords(parentThreadId, []); + if ( + current.projectId !== parent.projectId || + (sideChat && current.projectId !== sideChat.projectId) + ) + return; + } if (limit && exceededDispatchModeLimit(limit, current)) return; if (source && current.deletedAt !== null) return; const latest = source ? yield* receipts.getLatestLifecycle(thread.id) : null; - if (source && CommandReceiptStore.supersedesCascade(latest, source, commandId)) return; + if (source && CommandReceiptStore.supersedesCascade(latest, source.cutoff, commandId)) + return; const children = yield* orchestrator.getSubagentChildThreads(thread.id); const [, failures] = yield* Effect.partition(children, (child) => Effect.gen(function* () { @@ -1889,7 +1920,9 @@ const make = Effect.gen(function* () { }), ); }); - const cascade = visit({ id: input.threadId, deletedAt: null }, input.commandId, []); + const walk = visit({ id: input.threadId, deletedAt: null }, input.commandId, []); + const cascade = + parentThreadId === undefined ? walk : threadDispatch.withLock(parentThreadId, walk); yield* input.operation === "thread.delete" && (yield* EffectOutbox.SubagentCascade) === undefined ? threadDispatch.withDeletionLock(cascade) diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index a488e8cbe446d..fb31a9783ec8c 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14170,8 +14170,11 @@ describe("subagent lifecycle cascade", () => { const run = (operation: "archive" | "unarchive" | "delete", threadId = parent, id?: string) => threads.dispatch(command(operation, threadId, id)); // Committed by a process that stopped before its reply, leaving the cascade to the worker. - const commit = (operation: "archive" | "unarchive", threadId = parent, id?: string) => - orchestrator.dispatch(command(operation, threadId, id)); + const commit = ( + operation: "archive" | "unarchive" | "delete", + threadId = parent, + id?: string, + ) => orchestrator.dispatch(command(operation, threadId, id)); const state = (field: "archivedAt" | "deletedAt", ...ids: ReadonlyArray) => Effect.forEach(ids, (id) => store.getThread(ThreadId.make(id))).pipe( Effect.map((all) => all.map((thread) => (thread[field] === null ? 0 : 1)).join("")), @@ -14336,13 +14339,9 @@ describe("subagent lifecycle cascade", () => { { source: app, delivery: "reply", operation: "archive", collision: null }, { source: app, delivery: "worker", operation: "archive", collision: null }, { source: parent, delivery: "reply", operation: "archive", collision: "accepted" }, - { source: parent, delivery: "worker", operation: "archive", collision: "accepted" }, { source: parent, delivery: "reply", operation: "archive", collision: "rejected" }, - { source: parent, delivery: "worker", operation: "archive", collision: "rejected" }, { source: parent, delivery: "reply", operation: "unarchive", collision: "accepted" }, - { source: parent, delivery: "worker", operation: "unarchive", collision: "accepted" }, { source: parent, delivery: "reply", operation: "unarchive", collision: "rejected" }, - { source: parent, delivery: "worker", operation: "unarchive", collision: "rejected" }, ] as const)( "cascades side-chat descendants from $source through $delivery: $operation, collision $collision", ({ source, delivery, operation, collision }) => @@ -14413,6 +14412,89 @@ describe("subagent lifecycle cascade", () => { assert.equal(yield* f.archived(side, child, independent), "001"); }).pipe(Effect.provide(layerTest)), ); + it.effect("preserves a newer descendant restore through a delayed nested side-chat archive", () => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + yield* f.sideChat(app, side); + yield* f.set(child, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + yield* f.run("archive", child, "archive:side-child:independent"); + yield* f.commit("archive"); + yield* f.run("unarchive", child); + yield* f.drain; + assert.equal(yield* f.archived(parent, app, side, child), "1110"); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect.each([ + { delivery: "reply", useParentDerivedId: false }, + { delivery: "worker", useParentDerivedId: false }, + { delivery: "worker", useParentDerivedId: true }, + ] as const)( + "preserves an independent side-chat rearchive through $delivery, parent-derived ID $useParentDerivedId", + ({ delivery, useParentDerivedId }) => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + yield* f.sideChat(parent, side); + yield* f.set(child, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + if (useParentDerivedId) yield* f.run("archive", side, "earlier:side-archive"); + yield* f.run("archive"); + yield* f.run("unarchive", side); + assert.equal(yield* f.archived(side, child), "00"); + yield* TestClock.adjust("1 second"); + yield* f.run( + "archive", + side, + useParentDerivedId ? "archive:parent:side-chat:side" : undefined, + ); + yield* f.drain; + yield* TestClock.adjust("8 days"); + yield* f.outbox.pruneSettled; + if (delivery === "reply") yield* f.run("unarchive"); + else { + yield* f.commit("unarchive"); + yield* f.drain; + } + assert.equal(yield* f.archived(parent, side, child), "001"); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect.each(["reply", "worker", "legacy worker"] as const)( + "leaves a moved side chat's subagent alone during parent deletion through %s", + (delivery) => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + const projectId = ProjectId.make("foreign"); + yield* f.sideChat(parent, side); + yield* f.set(side, { projectId }); + yield* f.set(child, { + projectId, + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + if (delivery === "reply") yield* f.run("delete"); + else { + yield* f.commit("delete"); + if (delivery === "legacy worker") { + yield* f.threads.cascadeSubagentLifecycle({ + threadId: side, + commandId: CommandId.make("delete:parent:side-chat:side"), + operation: "thread.delete", + }); + } else yield* f.drain; + } + assert.equal(yield* f.state("deletedAt", parent, side, child), "110"); + }).pipe(Effect.provide(layerTest)), + ); + it.effect("restores only what a parent archive cascaded, through independent archives", () => Effect.gen(function* () { const f = yield* fixture; From b188651b88be9e91086d9ca0bd48fbbf5fa4026e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 14:18:07 +0000 Subject: [PATCH 05/23] fix(server): roll back colliding side-chat effects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Require a durable outbox row for each newly committed embedded cascade. Keep the occupied row and roll back the enclosing lifecycle transaction. > 🤖 Reviewed until clean by GPT 6.2 Astra, GPT 6.2 Sol, CodeRabbit. Findings that reach consensus across the majority were addressed by the generation model. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../src/orchestration-v2/EffectOutbox.ts | 7 ++++- .../src/orchestration-v2/runtimeLayer.test.ts | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index 163b0d80eefaf..1fbe8dc6ee965 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -482,7 +482,12 @@ export const layer: Layer.Layer = La ${nowIso}, ${nowIso} ) - ON CONFLICT(effect_id) DO NOTHING + ${ + effect.request.type === "subagent-threads.cascade" && + effect.request.sideChat !== undefined + ? sql`` + : sql`ON CONFLICT(effect_id) DO NOTHING` + } `, { concurrency: 1, discard: true }, ); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index fb31a9783ec8c..18ad2a64143eb 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14495,6 +14495,35 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); + it.effect("rolls back a nested side-chat deletion with an occupied recovery effect ID", () => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + yield* f.sideChat(app, side); + yield* f.set(child, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + const nestedId = "delete:parent:subagent:app"; + const effectId = `effect:${nestedId}:side-chat:${side}:subagent-threads.cascade`; + yield* f.outbox.enqueue([ + { + id: effectId, + commandId: CommandId.make("earlier"), + threadId: ThreadId.make("fork"), + request: { type: "terminal.cleanup" }, + availableAt: DateTime.add(f.now, { days: 1 }), + }, + ]); + const occupied = yield* f.outbox.get(effectId); + yield* f.commit("delete"); + yield* f.drain; + assert.equal(yield* f.state("deletedAt", parent, app, side, child), "1000"); + assert.isTrue(Option.isNone(yield* receipt(nestedId))); + assert.deepEqual(yield* f.outbox.get(effectId), occupied); + }).pipe(Effect.provide(layerTest)), + ); + it.effect("restores only what a parent archive cascaded, through independent archives", () => Effect.gen(function* () { const f = yield* fixture; From 924b04e0484b5e35f5a836e1ef0369c3fc81e843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 14:35:17 +0000 Subject: [PATCH 06/23] fix(server): distinguish unused-only side-chat deletes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep the archived parent's ownership after an unused-only delete. Use the journaled deletion to check an ambiguous legacy recovery receipt. > 🤖 Reviewed until clean by GPT 6.2 Astra, GPT 6.2 Sol, CodeRabbit. Findings that reach consensus across the majority were addressed by the generation model. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../orchestration-v2/CommandReceiptStore.ts | 5 +- .../src/orchestration-v2/Orchestrator.ts | 6 +- .../ThreadManagementService.ts | 27 ++++--- .../src/orchestration-v2/runtimeLayer.test.ts | 72 +++++++++++++++++-- .../OrchestrationCommandReceipts.ts | 22 ++++-- 5 files changed, 107 insertions(+), 25 deletions(-) diff --git a/apps/server/src/orchestration-v2/CommandReceiptStore.ts b/apps/server/src/orchestration-v2/CommandReceiptStore.ts index a462b1d042dc6..c59f1643386fe 100644 --- a/apps/server/src/orchestration-v2/CommandReceiptStore.ts +++ b/apps/server/src/orchestration-v2/CommandReceiptStore.ts @@ -77,6 +77,7 @@ type AnyCommandReceiptV2 = CommandReceiptV2 | ProjectCommandReceiptV2; export interface CommandReceiptStoreV2Shape { readonly getLatestLifecycle: ( threadId: ThreadId, + commandTypes?: ReadonlyArray, ) => Effect.Effect; readonly insertIfAbsent: ( receipt: AnyCommandReceiptV2, @@ -150,8 +151,8 @@ const layerBase: Layer.Layer< const receipts = yield* OrchestrationCommandReceipts.OrchestrationCommandReceiptRepository; return CommandReceiptStoreV2.of({ - getLatestLifecycle: (threadId) => - receipts.getLatestLifecycle({ threadId }).pipe( + getLatestLifecycle: (threadId, commandTypes) => + receipts.getLatestLifecycle({ threadId, commandTypes }).pipe( Effect.flatMap((receipt) => Option.isSome(receipt) ? fromApplicationReceipt(receipt.value) : Effect.succeed(null), ), diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index ebbc1d59b102e..35028da67077c 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -15754,7 +15754,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if (parent === null || sideChatIds.length === 0) break; const parentArchive = command.type === "thread.unarchive" - ? yield* commandReceipts.getLatestLifecycle(parent.id).pipe(mapDispatchError(command)) + ? yield* commandReceipts + .getLatestLifecycle(parent.id, ["thread.archive"]) + .pipe(mapDispatchError(command)) : null; // Unarchive restores the side chats this parent's archive took, not ones archived before it. for (const threadId of sideChatIds) { @@ -15780,7 +15782,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio threadId, }; const derivedArchiveId = - parentArchive?.commandType === "thread.archive" + parentArchive !== null ? sideChatLifecycleCommandId(parentArchive.commandId, threadId) : undefined; const sideChatArchive = diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index d5f3e84996f52..84f0b34bba8cb 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -1810,27 +1810,32 @@ const make = Effect.gen(function* () { if (failures[0] !== undefined) return yield* failures[0]; }); - // Older embedded deletes have only the enclosing parent's lifecycle receipt. + // An older embedded delete can share its derived ID with an accepted unused-only close. const legacySideChatDeletionParent = Effect.fn( "ThreadManagementService.legacySideChatDeletionParent", )(function* (commandId: CommandId, threadId: ThreadId) { const suffix = `:side-chat:${threadId}`; if (commandId.length <= suffix.length || !commandId.endsWith(suffix)) return; + const parentCommandId = CommandId.make(commandId.slice(0, -suffix.length)); + const parent = Option.getOrUndefined(yield* receipts.getByCommandId(parentCommandId)); + if ( + parent?.status !== "accepted" || + parent.commandType !== "thread.delete" || + parent.threadId === threadId + ) + return; const direct = Option.getOrUndefined(yield* receipts.getByCommandId(commandId)); if ( direct?.threadId === threadId && direct.commandType === "thread.delete" && direct.status === "accepted" - ) - return; - const parentCommandId = CommandId.make(commandId.slice(0, -suffix.length)); - const parent = Option.getOrUndefined(yield* receipts.getByCommandId(parentCommandId)); - if ( - parent?.status === "accepted" && - parent.commandType === "thread.delete" && - parent.threadId !== threadId - ) - return parent.threadId; + ) { + const deletion = yield* orchestrator + .streamStoredEventsFrom({ threadId, eventType: "thread.deleted" }) + .pipe(Stream.runHead); + if (Option.isSome(deletion) && deletion.value.commandId !== parentCommandId) return; + } + return parent.threadId; }); const cascadeSubagentLifecycle: ThreadManagementServiceShape["cascadeSubagentLifecycle"] = diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 18ad2a64143eb..82e974e2cc920 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14220,7 +14220,41 @@ describe("subagent lifecycle cascade", () => { }, }); }); - return { threads, now, set, sideChat, run, commit, state, archived, outbox, cascade, drain }; + const markUsed = (threadId: ThreadId) => + store.apply({ + id: EventId.make(`use:${threadId}`), + type: "message.updated", + threadId, + occurredAt: now, + payload: { + id: MessageId.make(`use:${threadId}`), + threadId, + runId: null, + nodeId: null, + role: "user", + text: "This thread has a message.", + attachments: [], + streaming: false, + createdBy: "user", + creationSource: "web", + createdAt: now, + updatedAt: now, + }, + }); + return { + threads, + now, + set, + sideChat, + run, + commit, + state, + archived, + outbox, + cascade, + drain, + markUsed, + }; }); const receipt = (id: string) => Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.getByCommandId(CommandId.make(id))); @@ -14408,6 +14442,22 @@ describe("subagent lifecycle cascade", () => { yield* f.drain; yield* TestClock.adjust("8 days"); yield* f.outbox.pruneSettled; + if (source === parent && delivery === "reply") { + yield* f.markUsed(parent); + yield* f.threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make("rename:archived-parent"), + threadId: parent, + title: "Renamed parent", + }); + const kept = yield* f.threads.dispatch({ + type: "thread.delete", + commandId: CommandId.make("unused-only:parent"), + threadId: parent, + onlyIfUnused: true, + }); + assert.isEmpty(kept.storedEvents); + } yield* apply("unarchive"); assert.equal(yield* f.archived(side, child, independent), "001"); }).pipe(Effect.provide(layerTest)), @@ -14466,7 +14516,7 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); - it.effect.each(["reply", "worker", "legacy worker"] as const)( + it.effect.each(["reply", "worker", "legacy worker", "legacy worker after unused close"] as const)( "leaves a moved side chat's subagent alone during parent deletion through %s", (delivery) => Effect.gen(function* () { @@ -14480,16 +14530,28 @@ describe("subagent lifecycle cascade", () => { projectId, lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, }); + const commandId = CommandId.make("delete:parent:side-chat:side"); + if (delivery === "legacy worker after unused close") { + yield* f.markUsed(side); + const kept = yield* f.threads.dispatch({ + type: "thread.delete", + commandId, + threadId: side, + onlyIfUnused: true, + }); + assert.isEmpty(kept.storedEvents); + } if (delivery === "reply") yield* f.run("delete"); else { yield* f.commit("delete"); - if (delivery === "legacy worker") { + if (delivery === "worker") yield* f.drain; + else { yield* f.threads.cascadeSubagentLifecycle({ threadId: side, - commandId: CommandId.make("delete:parent:side-chat:side"), + commandId, operation: "thread.delete", }); - } else yield* f.drain; + } } assert.equal(yield* f.state("deletedAt", parent, side, child), "110"); }).pipe(Effect.provide(layerTest)), diff --git a/apps/server/src/persistence/OrchestrationCommandReceipts.ts b/apps/server/src/persistence/OrchestrationCommandReceipts.ts index 0d20e04c49497..5c3c0118a4bea 100644 --- a/apps/server/src/persistence/OrchestrationCommandReceipts.ts +++ b/apps/server/src/persistence/OrchestrationCommandReceipts.ts @@ -30,6 +30,17 @@ export const GetByCommandIdInput = Schema.Struct({ }); export type GetByCommandIdInput = typeof GetByCommandIdInput.Type; +const LifecycleCommandType = Schema.Literals([ + "thread.archive", + "thread.unarchive", + "thread.delete", +]); +export type LifecycleCommandType = typeof LifecycleCommandType.Type; +const GetLatestLifecycleInput = Schema.Struct({ + threadId: ThreadId, + commandTypes: Schema.optional(Schema.Array(LifecycleCommandType)), +}); + export class OrchestrationCommandReceiptRepository extends Context.Service< OrchestrationCommandReceiptRepository, { @@ -41,9 +52,9 @@ export class OrchestrationCommandReceiptRepository extends Context.Service< receipt: OrchestrationCommandReceipt, ) => Effect.Effect; - readonly getLatestLifecycle: (input: { - readonly threadId: ThreadId; - }) => Effect.Effect< + readonly getLatestLifecycle: ( + input: typeof GetLatestLifecycleInput.Type, + ) => Effect.Effect< Option.Option, OrchestrationCommandReceiptRepositoryError >; @@ -116,15 +127,16 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { }); const findLatestLifecycle = SqlSchema.findOneOption({ - Request: Schema.Struct({ threadId: ThreadId }), + Request: GetLatestLifecycleInput, Result: OrchestrationCommandReceipt, - execute: ({ threadId }) => sql` + execute: ({ threadId, commandTypes }) => sql` SELECT command_id AS "commandId", aggregate_kind AS "aggregateKind", aggregate_id AS "aggregateId", command_type AS "commandType", accepted_at AS "acceptedAt", result_sequence AS "resultSequence", status, error FROM orchestration_command_receipts WHERE aggregate_kind = 'thread' AND aggregate_id = ${threadId} AND status = 'accepted' AND command_type IN ('thread.archive', 'thread.unarchive', 'thread.delete') + ${commandTypes === undefined ? sql`` : sql`AND command_type IN ${sql.in(commandTypes)}`} ORDER BY result_sequence DESC LIMIT 1 `, }); From 6f4140d89dd8d06c718045a3d6840ba2635f60e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 14:46:37 +0000 Subject: [PATCH 07/23] fix(server): keep committed side-chat cascades after no-op close MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compare archive and restore receipts at a live embedded side chat. An unused-only close leaves the committed descendant work pending. > 🤖 Reviewed until clean by GPT 6.2 Astra, GPT 6.2 Sol, CodeRabbit. Findings that reach consensus across the majority were addressed by the generation model. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../ThreadManagementService.ts | 9 ++++- .../src/orchestration-v2/runtimeLayer.test.ts | 35 ++++++++++++------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 84f0b34bba8cb..9d6670dbe8632 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -1903,7 +1903,14 @@ const make = Effect.gen(function* () { } if (limit && exceededDispatchModeLimit(limit, current)) return; if (source && current.deletedAt !== null) return; - const latest = source ? yield* receipts.getLatestLifecycle(thread.id) : null; + const latest = source + ? yield* receipts.getLatestLifecycle( + thread.id, + path.length === 0 && sideChat + ? ["thread.archive", "thread.unarchive"] + : undefined, + ) + : null; if (source && CommandReceiptStore.supersedesCascade(latest, source.cutoff, commandId)) return; const children = yield* orchestrator.getSubagentChildThreads(thread.id); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 82e974e2cc920..cc97f8d9d744b 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14241,6 +14241,24 @@ describe("subagent lifecycle cascade", () => { updatedAt: now, }, }); + const renameAndKeep = Effect.fn("test.renameAndKeep")(function* ( + threadId: ThreadId, + lifecycle: string, + ) { + yield* threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make(`rename:${lifecycle}:${threadId}`), + threadId, + title: `Renamed after ${lifecycle}`, + }); + const kept = yield* threads.dispatch({ + type: "thread.delete", + commandId: CommandId.make(`unused-only:${lifecycle}:${threadId}`), + threadId, + onlyIfUnused: true, + }); + assert.isEmpty(kept.storedEvents); + }); return { threads, now, @@ -14254,6 +14272,7 @@ describe("subagent lifecycle cascade", () => { cascade, drain, markUsed, + renameAndKeep, }; }); const receipt = (id: string) => @@ -14395,12 +14414,14 @@ describe("subagent lifecycle cascade", () => { }); } yield* f.run("archive", independent); + if (source === parent && delivery === "worker") yield* f.markUsed(side); const apply = Effect.fn("test.applySideChatLifecycle")(function* ( operation: "archive" | "unarchive", ) { if (delivery === "reply") yield* f.run(operation); else { yield* f.commit(operation); + if (source === parent) yield* f.renameAndKeep(side, operation); yield* f.drain; } }); @@ -14444,19 +14465,7 @@ describe("subagent lifecycle cascade", () => { yield* f.outbox.pruneSettled; if (source === parent && delivery === "reply") { yield* f.markUsed(parent); - yield* f.threads.dispatch({ - type: "thread.metadata.update", - commandId: CommandId.make("rename:archived-parent"), - threadId: parent, - title: "Renamed parent", - }); - const kept = yield* f.threads.dispatch({ - type: "thread.delete", - commandId: CommandId.make("unused-only:parent"), - threadId: parent, - onlyIfUnused: true, - }); - assert.isEmpty(kept.storedEvents); + yield* f.renameAndKeep(parent, "archive"); } yield* apply("unarchive"); assert.equal(yield* f.archived(side, child, independent), "001"); From 8f373ac8a6976429c1327f84cb4ceb939ed9d26a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 15:07:56 +0000 Subject: [PATCH 08/23] fix(server): verify side-chat deletion and archive provenance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Require the exact deletion event to prove legacy direct-delete ownership. Keep a newer side-chat restore ahead of delayed descendant archival. > 🤖 Reviewed until clean by GPT 6.2 Astra, GPT 6.2 Sol, CodeRabbit. Findings that reach consensus across the majority were addressed by the generation model. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../src/orchestration-v2/Orchestrator.ts | 30 ++++- .../ThreadManagementService.ts | 5 +- .../src/orchestration-v2/runtimeLayer.test.ts | 103 +++++++++++++----- 3 files changed, 104 insertions(+), 34 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 35028da67077c..85d09b05dfc21 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -559,6 +559,10 @@ export interface OrchestratorV2Shape { readonly getThreadEventSequence: ( threadId: ThreadId, ) => Effect.Effect; + readonly hasRecordedThreadDeletion: ( + commandId: CommandId, + threadId: ThreadId, + ) => Effect.Effect; readonly streamStoredEvents: Stream.Stream; /** Events arrive projected for the wire and shared by every subscriber; never mutate them. */ readonly streamStoredEventsFrom: (input?: { @@ -1914,14 +1918,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const cascade = yield* EffectOutbox.SubagentCascade; if (cascade?.path.at(-1) === threadId) return []; if ((yield* projectionStore.getSubagentChildThreads(threadId)).length === 0) return []; + const cutoff = cascade?.source?.cutoff; + // The generated receipt would hide the side chat's newer choice from the cascade worker. + if (sideChat && operation === "thread.archive" && cutoff !== undefined) { + const latest = yield* commandReceipts.getLatestLifecycle(threadId, [ + "thread.archive", + "thread.unarchive", + ]); + if (supersedesCascade(latest, cutoff, commandId)) return []; + } const limit = yield* DispatchModeLimit; const request = { type: "subagent-threads.cascade", operation, limit, - ...(sideChat === undefined - ? {} - : { sideChat: { ...sideChat, cutoff: cascade?.source?.cutoff } }), + ...(sideChat === undefined ? {} : { sideChat: { ...sideChat, cutoff } }), } as const; const id = `effect:${commandId}:subagent-threads.cascade`; return [{ id, commandId, threadId, request } satisfies EffectOutbox.SubagentCascadeEffect]; @@ -17270,6 +17281,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .latestSequence({ threadId }) .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))); }), + hasRecordedThreadDeletion: Effect.fn("OrchestratorV2.hasRecordedThreadDeletion")(function ( + commandId: CommandId, + threadId: ThreadId, + ) { + return eventSink.readByCommandId({ commandId }).pipe( + Stream.filter( + ({ event }) => event.type === "thread.deleted" && event.threadId === threadId, + ), + Stream.runHead, + Effect.map(Option.isSome), + Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })), + ); + }), streamStoredEvents: eventSink.stream().pipe( Stream.mapError( (cause) => diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 9d6670dbe8632..c2fcd16f4ece1 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -1830,10 +1830,7 @@ const make = Effect.gen(function* () { direct.commandType === "thread.delete" && direct.status === "accepted" ) { - const deletion = yield* orchestrator - .streamStoredEventsFrom({ threadId, eventType: "thread.deleted" }) - .pipe(Stream.runHead); - if (Option.isSome(deletion) && deletion.value.commandId !== parentCommandId) return; + if (yield* orchestrator.hasRecordedThreadDeletion(commandId, threadId)) return; } return parent.threadId; }); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index cc97f8d9d744b..5bdce2e10893a 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14259,6 +14259,18 @@ describe("subagent lifecycle cascade", () => { }); assert.isEmpty(kept.storedEvents); }); + const movedSideChat = Effect.fn("test.movedSideChat")(function* () { + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + const projectId = ProjectId.make("foreign"); + yield* sideChat(parent, side); + yield* set(side, { projectId }); + yield* set(child, { + projectId, + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + return { side, child }; + }); return { threads, now, @@ -14273,6 +14285,7 @@ describe("subagent lifecycle cascade", () => { drain, markUsed, renameAndKeep, + movedSideChat, }; }); const receipt = (id: string) => @@ -14471,21 +14484,28 @@ describe("subagent lifecycle cascade", () => { assert.equal(yield* f.archived(side, child, independent), "001"); }).pipe(Effect.provide(layerTest)), ); - it.effect("preserves a newer descendant restore through a delayed nested side-chat archive", () => - Effect.gen(function* () { - const f = yield* fixture; - const side = ThreadId.make("side"); - const child = ThreadId.make("side-child"); - yield* f.sideChat(app, side); - yield* f.set(child, { - lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, - }); - yield* f.run("archive", child, "archive:side-child:independent"); - yield* f.commit("archive"); - yield* f.run("unarchive", child); - yield* f.drain; - assert.equal(yield* f.archived(parent, app, side, child), "1110"); - }).pipe(Effect.provide(layerTest)), + it.effect.each(["descendant", "side chat"] as const)( + "preserves a newer %s restore through a delayed nested side-chat archive", + (restored) => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + yield* f.sideChat(app, side); + yield* f.set(child, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + if (restored === "side chat") { + yield* f.run("archive", side); + yield* f.run("unarchive", child); + } else { + yield* f.run("archive", child, "archive:side-child:independent"); + } + yield* f.commit("archive"); + yield* f.run("unarchive", restored === "side chat" ? side : child); + yield* f.drain; + assert.equal(yield* f.archived(parent, app, side, child), "1110"); + }).pipe(Effect.provide(layerTest)), ); it.effect.each([ @@ -14525,22 +14545,21 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); - it.effect.each(["reply", "worker", "legacy worker", "legacy worker after unused close"] as const)( + it.effect.each([ + "reply", + "worker", + "legacy worker", + "legacy worker after unused close", + "legacy worker after a later deletion", + ] as const)( "leaves a moved side chat's subagent alone during parent deletion through %s", (delivery) => Effect.gen(function* () { const f = yield* fixture; - const side = ThreadId.make("side"); - const child = ThreadId.make("side-child"); - const projectId = ProjectId.make("foreign"); - yield* f.sideChat(parent, side); - yield* f.set(side, { projectId }); - yield* f.set(child, { - projectId, - lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, - }); + const { side, child } = yield* f.movedSideChat(); const commandId = CommandId.make("delete:parent:side-chat:side"); - if (delivery === "legacy worker after unused close") { + const laterDeletion = delivery === "legacy worker after a later deletion"; + if (delivery === "legacy worker after unused close" || laterDeletion) { yield* f.markUsed(side); const kept = yield* f.threads.dispatch({ type: "thread.delete", @@ -14555,6 +14574,18 @@ describe("subagent lifecycle cascade", () => { yield* f.commit("delete"); if (delivery === "worker") yield* f.drain; else { + if (laterDeletion) { + yield* f.commit("delete", side, "later:delete:side"); + const sql = yield* SqlClient.SqlClient; + const pruned = yield* sql` + DELETE FROM orchestration_events + WHERE command_id = 'delete:parent' + AND stream_id = ${side} + AND event_type = 'thread.deleted' + RETURNING sequence + `; + assert.lengthOf(pruned, 1); + } yield* f.threads.cascadeSubagentLifecycle({ threadId: side, commandId, @@ -14563,7 +14594,25 @@ describe("subagent lifecycle cascade", () => { } } assert.equal(yield* f.state("deletedAt", parent, side, child), "110"); - }).pipe(Effect.provide(layerTest)), + }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), + ); + + it.effect("recovers a direct side-chat deletion with a parent-derived command ID", () => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.movedSideChat(); + yield* f.markUsed(parent); + const kept = yield* f.threads.dispatch({ + type: "thread.delete", + commandId: CommandId.make("delete:parent"), + threadId: parent, + onlyIfUnused: true, + }); + assert.isEmpty(kept.storedEvents); + yield* f.commit("delete", side, "delete:parent:side-chat:side"); + yield* f.drain; + assert.equal(yield* f.state("deletedAt", parent, side, child), "011"); + }).pipe(Effect.provide(layerTest)), ); it.effect("rolls back a nested side-chat deletion with an occupied recovery effect ID", () => From 7b6a607e55918e7d1a57d343400b1f0a9ddd6e2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 15:19:40 +0000 Subject: [PATCH 09/23] fix(server): protect side-chat delete and restore recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reject fresh delete cascade collisions and preserve newer side-chat choices when a delayed restore plans descendant work. > 🤖 Reviewed until clean by GPT 6.2 Astra, GPT 6.2 Sol, CodeRabbit. Findings that reach consensus across the majority were addressed by the generation model. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../src/orchestration-v2/EffectOutbox.ts | 3 +- .../src/orchestration-v2/Orchestrator.ts | 2 +- .../src/orchestration-v2/runtimeLayer.test.ts | 42 +++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index 1fbe8dc6ee965..4e0f3ebafce0e 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -484,7 +484,8 @@ export const layer: Layer.Layer = La ) ${ effect.request.type === "subagent-threads.cascade" && - effect.request.sideChat !== undefined + (effect.request.sideChat !== undefined || + effect.request.operation === "thread.delete") ? sql`` : sql`ON CONFLICT(effect_id) DO NOTHING` } diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 85d09b05dfc21..eb9e108dbce19 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -1920,7 +1920,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio if ((yield* projectionStore.getSubagentChildThreads(threadId)).length === 0) return []; const cutoff = cascade?.source?.cutoff; // The generated receipt would hide the side chat's newer choice from the cascade worker. - if (sideChat && operation === "thread.archive" && cutoff !== undefined) { + if (sideChat && operation !== "thread.delete" && cutoff !== undefined) { const latest = yield* commandReceipts.getLatestLifecycle(threadId, [ "thread.archive", "thread.unarchive", diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 5bdce2e10893a..915d74a197319 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14508,6 +14508,27 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); + it.effect("preserves a newer side-chat archive through a delayed nested restore", () => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + yield* f.sideChat(app, side); + yield* f.set(child, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + yield* f.run("archive"); + yield* f.drain; + assert.equal(yield* f.archived(parent, app, side, child), "1111"); + yield* f.commit("unarchive"); + yield* f.commit("unarchive", side); + yield* TestClock.adjust("1 second"); + yield* f.commit("archive", side); + yield* f.drain; + assert.equal(yield* f.archived(parent, app, side, child), "0001"); + }).pipe(Effect.provide(layerTest)), + ); + it.effect.each([ { delivery: "reply", useParentDerivedId: false }, { delivery: "worker", useParentDerivedId: false }, @@ -14644,6 +14665,27 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); + it.effect("rejects a direct side-chat delete whose recovery ID is occupied", () => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.movedSideChat(); + const collidingCommandId = `delete:parent:side-chat:${side}`; + const effectId = `effect:${collidingCommandId}:subagent-threads.cascade`; + yield* f.commit("delete"); + const occupied = yield* f.outbox.get(effectId); + const { request } = Option.getOrThrow(occupied); + assert.equal(request.type, "subagent-threads.cascade"); + const rejected = yield* f.commit("delete", side, collidingCommandId).pipe(Effect.flip); + assert.equal(rejected._tag, "OrchestratorDispatchError"); + assert.isTrue(Option.isNone(yield* receipt(collidingCommandId))); + assert.deepEqual(yield* f.outbox.get(effectId), occupied); + assert.equal(yield* f.state("deletedAt", parent, side, child), "110"); + yield* f.commit("delete", side, "direct:fresh-delete"); + yield* f.drain; + assert.equal(yield* f.state("deletedAt", parent, side, child), "111"); + }).pipe(Effect.provide(layerTest)), + ); + it.effect("restores only what a parent archive cascaded, through independent archives", () => Effect.gen(function* () { const f = yield* fixture; From 4cd83f0928413e2d8c11a6541d394d9309d135a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 15:36:18 +0000 Subject: [PATCH 10/23] fix(server): require durable side-chat lifecycle recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Require a fresh recovery row for every subagent lifecycle cascade. Reject a different operation against a verified generated receipt. > 🤖 Reviewed until clean by GPT 6.2 Astra, GPT 6.2 Sol, CodeRabbit. Findings that reach consensus across the majority were addressed by the generation model. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../src/orchestration-v2/EffectOutbox.ts | 4 +- .../src/orchestration-v2/Orchestrator.ts | 37 ++++++- .../src/orchestration-v2/runtimeLayer.test.ts | 98 +++++++++++++++---- 3 files changed, 116 insertions(+), 23 deletions(-) diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index 4e0f3ebafce0e..a860a3dd4b1cc 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -483,9 +483,7 @@ export const layer: Layer.Layer = La ${nowIso} ) ${ - effect.request.type === "subagent-threads.cascade" && - (effect.request.sideChat !== undefined || - effect.request.operation === "thread.delete") + effect.request.type === "subagent-threads.cascade" ? sql`` : sql`ON CONFLICT(effect_id) DO NOTHING` } diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index eb9e108dbce19..48d27b8ff2240 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -375,9 +375,12 @@ export class OrchestratorCommandIdConflictError extends Schema.TaggedError new OrchestratorProjectionError({ threadId, cause }))); + const sideChatLifecycleSuffix = (threadId: ThreadId) => `:side-chat:${threadId}`; const sideChatLifecycleCommandId = (commandId: CommandId, threadId: ThreadId) => - CommandId.make(`${commandId}:side-chat:${threadId}`); + CommandId.make(`${commandId}${sideChatLifecycleSuffix(threadId)}`); // A cascade walks the whole subtree itself, so its own commands start none. const subagentCascadeEffects = Effect.fn("Orchestrator.subagentCascadeEffects")(function* ( @@ -16117,6 +16121,33 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }; }); + const rejectConflictingSideChatReceipt = Effect.fn( + "OrchestratorV2.rejectConflictingSideChatReceipt", + )(function* (command: OrchestrationV2ServerCommand, receipt: CommandReceiptV2) { + if (receipt.commandType !== "thread.archive" && receipt.commandType !== "thread.unarchive") + return; + const suffix = sideChatLifecycleSuffix(receipt.threadId); + if (receipt.commandId.length <= suffix.length || !receipt.commandId.endsWith(suffix)) return; + const parentId = CommandId.make(receipt.commandId.slice(0, -suffix.length)); + const parent = Option.getOrUndefined( + yield* commandReceipts.getByCommandId(parentId).pipe(mapDispatchError(command)), + ); + if ( + parent?.status === "accepted" && + parent.commandType === receipt.commandType && + parent.resultSequence === receipt.resultSequence && + parent.threadId !== receipt.threadId + ) { + return yield* new OrchestratorCommandIdConflictError({ + commandId: command.commandId, + commandType: command.type, + receiptThreadId: receipt.threadId, + commandThreadId: receipt.threadId, + receiptCommandType: receipt.commandType, + }); + } + }); + const replayCommandReceipt = Effect.fn("orchestrationV2.dispatch.replayReceipt")(function* ( command: OrchestrationV2ServerCommand, receipt: CommandReceiptV2, @@ -16138,6 +16169,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio commandThreadId: dispatchThreadId, }); } + if (receipt.commandType !== command.type) + yield* rejectConflictingSideChatReceipt(command, receipt); const storedEvents = yield* eventSink.readByCommandId({ commandId: command.commandId }).pipe( Stream.runCollect, Effect.map((events): ReadonlyArray => Array.from(events)), @@ -16439,6 +16472,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio commandThreadId: commandThreadId(command), }); } + if (!committed.committed && committed.receipt.commandType !== command.type) + yield* rejectConflictingSideChatReceipt(command, committed.receipt); if ( command.type === "queue.resume" || command.type === "queued-run.start" || diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 915d74a197319..9cc96b613d2c3 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14508,25 +14508,27 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); - it.effect("preserves a newer side-chat archive through a delayed nested restore", () => - Effect.gen(function* () { - const f = yield* fixture; - const side = ThreadId.make("side"); - const child = ThreadId.make("side-child"); - yield* f.sideChat(app, side); - yield* f.set(child, { - lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, - }); - yield* f.run("archive"); - yield* f.drain; - assert.equal(yield* f.archived(parent, app, side, child), "1111"); - yield* f.commit("unarchive"); - yield* f.commit("unarchive", side); - yield* TestClock.adjust("1 second"); - yield* f.commit("archive", side); - yield* f.drain; - assert.equal(yield* f.archived(parent, app, side, child), "0001"); - }).pipe(Effect.provide(layerTest)), + it.effect( + "keeps the descendant archived during a delayed restore after a newer side-chat archive", + () => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + yield* f.sideChat(app, side); + yield* f.set(child, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + yield* f.run("archive"); + yield* f.drain; + assert.equal(yield* f.archived(parent, app, side, child), "1111"); + yield* f.commit("unarchive"); + yield* f.commit("unarchive", side); + yield* TestClock.adjust("1 second"); + yield* f.commit("archive", side); + yield* f.drain; + assert.equal(yield* f.archived(parent, app, side, child), "0001"); + }).pipe(Effect.provide(layerTest)), ); it.effect.each([ @@ -14686,6 +14688,64 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); + it.effect.each(["archive", "unarchive"] as const)( + "rolls back a direct %s with an occupied lifecycle recovery ID", + (operation) => + Effect.gen(function* () { + const f = yield* fixture; + const { side } = yield* f.movedSideChat(); + const fork = ThreadId.make("fork"); + const child = ThreadId.make("fork-child"); + if (operation === "unarchive") { + yield* f.run("archive", fork); + yield* f.drain; + } + const before = yield* f.archived(fork, child); + const collidingId = `delete:parent:side-chat:${side}`; + const effectId = `effect:${collidingId}:subagent-threads.cascade`; + yield* f.commit("delete"); + const occupied = Option.getOrThrow(yield* f.outbox.get(effectId)); + const refused = yield* f.commit(operation, fork, collidingId).pipe(Effect.flip); + assert.equal(refused._tag, "OrchestratorDispatchError"); + assert.equal(yield* f.archived(fork, child), before); + assert.isTrue(Option.isNone(yield* receipt(collidingId))); + assert.deepEqual(Option.getOrThrow(yield* f.outbox.get(effectId)), occupied); + const fresh = yield* f.commit(operation, fork, "direct:fresh-lifecycle"); + const replay = yield* f.commit(operation, fork, "direct:fresh-lifecycle"); + assert.equal(replay.sequence, fresh.sequence); + assert.isTrue(replay.replayed); + yield* f.drain; + assert.equal(yield* f.archived(fork, child), operation === "archive" ? "11" : "00"); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect("rejects a direct delete that reuses a generated archive receipt", () => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + yield* f.sideChat(parent, side); + yield* f.set(child, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + yield* f.run("archive"); + yield* f.drain; + yield* TestClock.adjust("8 days"); + yield* f.outbox.pruneSettled; + const generatedId = `archive:parent:side-chat:${side}`; + const generated = Option.getOrThrow(yield* receipt(generatedId)); + const replay = yield* f.commit("archive", side, generatedId); + assert.equal(replay.sequence, generated.resultSequence); + assert.isTrue(replay.replayed); + const refused = yield* f.commit("delete", side, generatedId).pipe(Effect.flip); + assert.equal(refused._tag, "OrchestratorCommandIdConflictError"); + assert.deepEqual(Option.getOrThrow(yield* receipt(generatedId)), generated); + assert.equal(yield* f.state("deletedAt", side, child), "00"); + yield* f.run("delete", side, "direct:fresh-delete"); + assert.equal(yield* f.state("deletedAt", side, child), "11"); + }).pipe(Effect.provide(layerTest)), + ); + it.effect("restores only what a parent archive cascaded, through independent archives", () => Effect.gen(function* () { const f = yield* fixture; From 641409b149b0c29827e92b2d4a51ccce9ab2b4e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 15:53:15 +0000 Subject: [PATCH 11/23] fix(server): preserve side-chat recovery eligibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skip descendant restore work without a verified archive owner and persist direct delete provenance through event pruning. > 🤖 Reviewed until clean by GPT 6.2 Astra, GPT 6.2 Sol, CodeRabbit. Findings that reach consensus across the majority were addressed by the generation model. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../src/orchestration-v2/EffectOutbox.ts | 1 + .../src/orchestration-v2/EffectWorker.ts | 1 + .../src/orchestration-v2/Orchestrator.ts | 4 + .../ThreadManagementService.ts | 4 +- .../src/orchestration-v2/runtimeLayer.test.ts | 97 ++++++++++++++++--- 5 files changed, 90 insertions(+), 17 deletions(-) diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index a860a3dd4b1cc..17108d7060ac7 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -157,6 +157,7 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ type: Schema.Literal("subagent-threads.cascade"), operation: Schema.Literals(["thread.archive", "thread.unarchive", "thread.delete"]), sideChat: Schema.optional(EmbeddedSideChatCascade), + directDelete: Schema.optional(Schema.Literal(true)), limit: Schema.optional( Schema.Struct({ runtimeMode: RuntimeMode, interactionMode: ProviderInteractionMode }), ), diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index 428eb2477b772..79c7962e9c51b 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -688,6 +688,7 @@ export const layerExecutor: Layer.Layer< threadId: effect.threadId, operation: effect.request.operation, sideChat: effect.request.sideChat, + directDelete: effect.request.directDelete, }) .pipe( Effect.provideService(DispatchModeLimit, effect.request.limit), diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 48d27b8ff2240..1e729de6cac98 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -1921,6 +1921,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ) { const cascade = yield* EffectOutbox.SubagentCascade; if (cascade?.path.at(-1) === threadId) return []; + if (operation === "thread.unarchive" && sideChat && !sideChat.archiveCommandId) return []; if ((yield* projectionStore.getSubagentChildThreads(threadId)).length === 0) return []; const cutoff = cascade?.source?.cutoff; // The generated receipt would hide the side chat's newer choice from the cascade worker. @@ -1936,6 +1937,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio type: "subagent-threads.cascade", operation, limit, + ...(operation === "thread.delete" && sideChat === undefined + ? { directDelete: true as const } + : {}), ...(sideChat === undefined ? {} : { sideChat: { ...sideChat, cutoff } }), } as const; const id = `effect:${commandId}:subagent-threads.cascade`; diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index c2fcd16f4ece1..a31d239c58b63 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -558,6 +558,7 @@ export interface ThreadManagementServiceShape { readonly commandId: CommandId; readonly operation: "thread.archive" | "thread.unarchive" | "thread.delete"; readonly sideChat?: EffectOutbox.EmbeddedSideChatCascade | undefined; + readonly directDelete?: true | undefined; }) => Effect.Effect< void, Orchestrator.OrchestratorV2Error | CommandReceiptStore.CommandReceiptStoreV2Error @@ -870,6 +871,7 @@ const make = Effect.gen(function* () { threadId, operation: request.operation, sideChat: request.sideChat, + directDelete: request.directDelete, }).pipe( Effect.catch((cause) => Effect.logWarning("Subagent lifecycle cascade left to the effect worker", { @@ -1841,7 +1843,7 @@ const make = Effect.gen(function* () { if (input.operation === "thread.unarchive" && sideChat && !sideChat.archiveCommandId) return; const parentThreadId = sideChat?.parentThreadId ?? - (input.operation === "thread.delete" + (input.operation === "thread.delete" && !input.directDelete ? yield* legacySideChatDeletionParent(input.commandId, input.threadId) : undefined); const receipt = diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 9cc96b613d2c3..38541b85d0fe7 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14559,12 +14559,26 @@ describe("subagent lifecycle cascade", () => { yield* f.drain; yield* TestClock.adjust("8 days"); yield* f.outbox.pruneSettled; + const occupied = { + commandId: CommandId.make("unarchive:parent:side-chat:side"), + threadId: ThreadId.make("fork"), + commandType: "thread.archive", + acceptedAt: f.now, + resultSequence: 0, + status: "accepted", + error: null, + } as const; + if (useParentDerivedId) { + yield* Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.upsert(occupied)); + } if (delivery === "reply") yield* f.run("unarchive"); else { yield* f.commit("unarchive"); yield* f.drain; } assert.equal(yield* f.archived(parent, side, child), "001"); + if (useParentDerivedId) + assert.deepEqual(Option.getOrThrow(yield* receipt(occupied.commandId)), occupied); }).pipe(Effect.provide(layerTest)), ); @@ -14620,22 +14634,73 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), ); - it.effect("recovers a direct side-chat deletion with a parent-derived command ID", () => - Effect.gen(function* () { - const f = yield* fixture; - const { side, child } = yield* f.movedSideChat(); - yield* f.markUsed(parent); - const kept = yield* f.threads.dispatch({ - type: "thread.delete", - commandId: CommandId.make("delete:parent"), - threadId: parent, - onlyIfUnused: true, - }); - assert.isEmpty(kept.storedEvents); - yield* f.commit("delete", side, "delete:parent:side-chat:side"); - yield* f.drain; - assert.equal(yield* f.state("deletedAt", parent, side, child), "011"); - }).pipe(Effect.provide(layerTest)), + it.effect.each(["retained legacy event", "pruned fresh event"] as const)( + "recovers a direct side-chat deletion with a parent-derived ID and %s", + (history) => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.movedSideChat(); + yield* f.markUsed(parent); + const kept = yield* f.threads.dispatch({ + type: "thread.delete", + commandId: CommandId.make("delete:parent"), + threadId: parent, + onlyIfUnused: true, + }); + assert.isEmpty(kept.storedEvents); + + if (history === "pruned fresh event") { + const store = yield* ProjectionStore.ProjectionStoreV2; + yield* f.markUsed(side); + const runId = RunId.make("side:queued"); + yield* store.apply({ + id: EventId.make("side:queued"), + type: "run.created", + threadId: side, + runId, + occurredAt: f.now, + payload: { + id: runId, + threadId: side, + ordinal: 1, + providerInstanceId: modelSelection.instanceId, + modelSelection, + providerThreadId: null, + userMessageId: MessageId.make(`use:${side}`), + rootNodeId: null, + activeAttemptId: null, + status: "queued", + requestedAt: f.now, + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }, + }); + } + const commandId = CommandId.make("delete:parent:side-chat:side"); + const deleted = yield* f.commit("delete", side, commandId); + if (history === "pruned fresh event") { + assert.equal(deleted.storedEvents.at(-1)?.event.type, "run.updated"); + const sql = yield* SqlClient.SqlClient; + const pruned = yield* sql` + DELETE FROM orchestration_events + WHERE command_id = ${commandId} + AND stream_id = ${side} + AND event_type = 'thread.deleted' + RETURNING sequence + `; + assert.lengthOf(pruned, 1); + yield* f.drain; + } else { + yield* f.threads.cascadeSubagentLifecycle({ + threadId: side, + commandId, + operation: "thread.delete", + }); + } + assert.equal(yield* f.state("deletedAt", parent, side, child), "011"); + }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), ); it.effect("rolls back a nested side-chat deletion with an occupied recovery effect ID", () => From fa79039da0babd90886d20a43ae15e503485aa98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 16:03:34 +0000 Subject: [PATCH 12/23] fix(server): check side-chat eligibility before scheduling cascades MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Require current archive ownership and the embedding project's scope before reserving descendant work. Recheck the project at delivery. > 🤖 Reviewed until clean by GPT 6.2 Astra, GPT 6.2 Sol, CodeRabbit. Findings that reach consensus across the majority were addressed by the generation model. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../src/orchestration-v2/Orchestrator.ts | 35 +++--- .../src/orchestration-v2/runtimeLayer.test.ts | 107 ++++++++++++++---- 2 files changed, 104 insertions(+), 38 deletions(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 1e729de6cac98..07719905fe358 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -1917,10 +1917,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const subagentCascadeEffects = Effect.fn("Orchestrator.subagentCascadeEffects")(function* ( { commandId, threadId }: { readonly commandId: CommandId; readonly threadId: ThreadId }, operation: "thread.archive" | "thread.unarchive" | "thread.delete", + projectId: ProjectId, sideChat?: EffectOutbox.EmbeddedSideChatCascade, ) { const cascade = yield* EffectOutbox.SubagentCascade; if (cascade?.path.at(-1) === threadId) return []; + if (sideChat && sideChat.projectId !== projectId) return []; if (operation === "thread.unarchive" && sideChat && !sideChat.archiveCommandId) return []; if ((yield* projectionStore.getSubagentChildThreads(threadId)).length === 0) return []; const cutoff = cascade?.source?.cutoff; @@ -5655,9 +5657,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } if (command.type === "thread.archive" || command.type === "thread.unarchive") { - const cascade = yield* subagentCascadeEffects(command, command.type, sideChat).pipe( - mapDispatchError(command), - ); + const cascade = yield* subagentCascadeEffects( + command, + command.type, + thread.projectId, + sideChat, + ).pipe(mapDispatchError(command)); yield* Ref.update(effects, (existing) => [...existing, ...cascade]); } @@ -15723,9 +15728,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio idAllocator, }), ); - const cascade = yield* subagentCascadeEffects(deletion, deletion.type, sideChat).pipe( - mapDispatchError(command), - ); + const cascade = yield* subagentCascadeEffects( + deletion, + deletion.type, + projection.thread.projectId, + sideChat, + ).pipe(mapDispatchError(command)); return { ...plan, projectId: projection.thread.projectId, @@ -15806,19 +15814,16 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio : undefined; const sideChatArchive = derivedArchiveId === undefined - ? undefined - : Option.getOrUndefined( - yield* commandReceipts - .getByCommandId(derivedArchiveId) - .pipe(mapDispatchError(command)), - ); + ? null + : yield* commandReceipts + .getLatestLifecycle(threadId, ["thread.archive"]) + .pipe(mapDispatchError(command)); yield* dispatchThreadMutation(sideChatCommand, events, effects, { parentThreadId: parent.id, projectId: parent.projectId, archiveCommandId: - sideChatArchive?.status === "accepted" && - sideChatArchive.commandType === "thread.archive" && - sideChatArchive.threadId === threadId && + sideChatArchive !== null && + sideChatArchive.commandId === derivedArchiveId && sideChatArchive.resultSequence === parentArchive?.resultSequence ? sideChatArchive.commandId : undefined, diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 38541b85d0fe7..1a0ed198d3f00 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14259,10 +14259,11 @@ describe("subagent lifecycle cascade", () => { }); assert.isEmpty(kept.storedEvents); }); - const movedSideChat = Effect.fn("test.movedSideChat")(function* () { + const sideChatWithChild = Effect.fn("test.sideChatWithChild")(function* ( + projectId = base.projectId, + ) { const side = ThreadId.make("side"); const child = ThreadId.make("side-child"); - const projectId = ProjectId.make("foreign"); yield* sideChat(parent, side); yield* set(side, { projectId }); yield* set(child, { @@ -14285,7 +14286,7 @@ describe("subagent lifecycle cascade", () => { drain, markUsed, renameAndKeep, - movedSideChat, + sideChatWithChild, }; }); const receipt = (id: string) => @@ -14532,12 +14533,13 @@ describe("subagent lifecycle cascade", () => { ); it.effect.each([ - { delivery: "reply", useParentDerivedId: false }, - { delivery: "worker", useParentDerivedId: false }, - { delivery: "worker", useParentDerivedId: true }, + { delivery: "reply", useParentDerivedId: false, pending: false }, + { delivery: "worker", useParentDerivedId: false, pending: false }, + { delivery: "worker", useParentDerivedId: true, pending: false }, + { delivery: "reply", useParentDerivedId: false, pending: true }, ] as const)( - "preserves an independent side-chat rearchive through $delivery, parent-derived ID $useParentDerivedId", - ({ delivery, useParentDerivedId }) => + "preserves an independent side-chat rearchive through $delivery, parent-derived ID $useParentDerivedId, pending $pending", + ({ delivery, useParentDerivedId, pending }) => Effect.gen(function* () { const f = yield* fixture; const side = ThreadId.make("side"); @@ -14548,17 +14550,23 @@ describe("subagent lifecycle cascade", () => { }); if (useParentDerivedId) yield* f.run("archive", side, "earlier:side-archive"); yield* f.run("archive"); - yield* f.run("unarchive", side); - assert.equal(yield* f.archived(side, child), "00"); - yield* TestClock.adjust("1 second"); - yield* f.run( - "archive", - side, - useParentDerivedId ? "archive:parent:side-chat:side" : undefined, - ); - yield* f.drain; - yield* TestClock.adjust("8 days"); - yield* f.outbox.pruneSettled; + if (pending) { + yield* f.commit("unarchive", side); + yield* TestClock.adjust("1 second"); + yield* f.commit("archive", side); + } else { + yield* f.run("unarchive", side); + assert.equal(yield* f.archived(side, child), "00"); + yield* TestClock.adjust("1 second"); + yield* f.run( + "archive", + side, + useParentDerivedId ? "archive:parent:side-chat:side" : undefined, + ); + yield* f.drain; + yield* TestClock.adjust("8 days"); + yield* f.outbox.pruneSettled; + } const occupied = { commandId: CommandId.make("unarchive:parent:side-chat:side"), threadId: ThreadId.make("fork"), @@ -14576,6 +14584,7 @@ describe("subagent lifecycle cascade", () => { yield* f.commit("unarchive"); yield* f.drain; } + if (pending) yield* f.drain; assert.equal(yield* f.archived(parent, side, child), "001"); if (useParentDerivedId) assert.deepEqual(Option.getOrThrow(yield* receipt(occupied.commandId)), occupied); @@ -14593,7 +14602,7 @@ describe("subagent lifecycle cascade", () => { (delivery) => Effect.gen(function* () { const f = yield* fixture; - const { side, child } = yield* f.movedSideChat(); + const { side, child } = yield* f.sideChatWithChild(ProjectId.make("foreign")); const commandId = CommandId.make("delete:parent:side-chat:side"); const laterDeletion = delivery === "legacy worker after a later deletion"; if (delivery === "legacy worker after unused close" || laterDeletion) { @@ -14634,12 +14643,55 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), ); + it.effect.each(["occupied receipt", "pending direct restore"] as const)( + "archives the original parent without planning a foreign side-chat cascade after %s", + (scenario) => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.sideChatWithChild(ProjectId.make("foreign")); + const occupied = { + commandId: CommandId.make(`archive:parent:side-chat:${side}`), + threadId: ThreadId.make("fork"), + commandType: "thread.archive", + acceptedAt: f.now, + resultSequence: 0, + status: "accepted", + error: null, + } as const; + if (scenario === "occupied receipt") { + yield* Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.upsert(occupied)); + } else { + yield* f.run("archive", side); + yield* f.drain; + yield* f.commit("unarchive", side); + } + yield* f.run("archive"); + yield* f.drain; + assert.equal(yield* f.archived(parent, side, child), "110"); + if (scenario === "occupied receipt") + assert.deepEqual(Option.getOrThrow(yield* receipt(occupied.commandId)), occupied); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect("stops a committed archive cascade at a side chat moved before delivery", () => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.sideChatWithChild(); + yield* f.commit("archive"); + const projectId = ProjectId.make("foreign"); + yield* f.set(side, { projectId }); + yield* f.set(child, { projectId }); + yield* f.drain; + assert.equal(yield* f.archived(parent, side, child), "110"); + }).pipe(Effect.provide(layerTest)), + ); + it.effect.each(["retained legacy event", "pruned fresh event"] as const)( "recovers a direct side-chat deletion with a parent-derived ID and %s", (history) => Effect.gen(function* () { const f = yield* fixture; - const { side, child } = yield* f.movedSideChat(); + const { side, child } = yield* f.sideChatWithChild(ProjectId.make("foreign")); yield* f.markUsed(parent); const kept = yield* f.threads.dispatch({ type: "thread.delete", @@ -14735,7 +14787,7 @@ describe("subagent lifecycle cascade", () => { it.effect("rejects a direct side-chat delete whose recovery ID is occupied", () => Effect.gen(function* () { const f = yield* fixture; - const { side, child } = yield* f.movedSideChat(); + const { side, child } = yield* f.sideChatWithChild(); const collidingCommandId = `delete:parent:side-chat:${side}`; const effectId = `effect:${collidingCommandId}:subagent-threads.cascade`; yield* f.commit("delete"); @@ -14748,6 +14800,15 @@ describe("subagent lifecycle cascade", () => { assert.deepEqual(yield* f.outbox.get(effectId), occupied); assert.equal(yield* f.state("deletedAt", parent, side, child), "110"); yield* f.commit("delete", side, "direct:fresh-delete"); + const fresh = Option.getOrThrow( + yield* f.outbox.get("effect:direct:fresh-delete:subagent-threads.cascade"), + ); + assert.equal(fresh.threadId, side); + assert.include(fresh.request, { + type: "subagent-threads.cascade", + operation: "thread.delete", + directDelete: true, + }); yield* f.drain; assert.equal(yield* f.state("deletedAt", parent, side, child), "111"); }).pipe(Effect.provide(layerTest)), @@ -14758,7 +14819,7 @@ describe("subagent lifecycle cascade", () => { (operation) => Effect.gen(function* () { const f = yield* fixture; - const { side } = yield* f.movedSideChat(); + const { side } = yield* f.sideChatWithChild(); const fork = ThreadId.make("fork"); const child = ThreadId.make("fork-child"); if (operation === "unarchive") { From 372a6f8fe1248617239039e917be911ee93d6aab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 16:59:54 +0000 Subject: [PATCH 13/23] fix(server): trace subagent lifecycle reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a span for lifecycle receipt reads and move the existing discovery span to the projection lookup shared by planning and traversal. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../src/orchestration-v2/CommandReceiptStore.ts | 14 ++++++++------ apps/server/src/orchestration-v2/Orchestrator.ts | 9 +++++---- .../server/src/orchestration-v2/ProjectionStore.ts | 7 +++++-- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/apps/server/src/orchestration-v2/CommandReceiptStore.ts b/apps/server/src/orchestration-v2/CommandReceiptStore.ts index c59f1643386fe..24e2d2714453a 100644 --- a/apps/server/src/orchestration-v2/CommandReceiptStore.ts +++ b/apps/server/src/orchestration-v2/CommandReceiptStore.ts @@ -151,13 +151,15 @@ const layerBase: Layer.Layer< const receipts = yield* OrchestrationCommandReceipts.OrchestrationCommandReceiptRepository; return CommandReceiptStoreV2.of({ - getLatestLifecycle: (threadId, commandTypes) => - receipts.getLatestLifecycle({ threadId, commandTypes }).pipe( - Effect.flatMap((receipt) => - Option.isSome(receipt) ? fromApplicationReceipt(receipt.value) : Effect.succeed(null), + getLatestLifecycle: Effect.fn("CommandReceiptStore.getLatestLifecycle")( + (threadId, commandTypes) => + receipts.getLatestLifecycle({ threadId, commandTypes }).pipe( + Effect.flatMap((receipt) => + Option.isSome(receipt) ? fromApplicationReceipt(receipt.value) : Effect.succeed(null), + ), + Effect.mapError((cause) => new CommandReceiptStoreLifecycleError({ threadId, cause })), ), - Effect.mapError((cause) => new CommandReceiptStoreLifecycleError({ threadId, cause })), - ), + ), insertIfAbsent: (receipt) => receipts.insertIfAbsent(toApplicationReceipt(receipt)).pipe( Effect.mapError( diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 07719905fe358..0134275d8ecdb 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -17229,11 +17229,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))); }, ), - getSubagentChildThreads: Effect.fn("OrchestratorV2.getSubagentChildThreads")((threadId) => - projectionStore + // The projection read owns the discovery span. + getSubagentChildThreads: Effect.fnUntraced(function* (threadId: ThreadId) { + return yield* projectionStore .getSubagentChildThreads(threadId) - .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))), - ), + .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))); + }), getThreadProjection: Effect.fn("OrchestratorV2.getThreadProjection")(function (threadId) { return projectionStore .getThreadProjection(threadId) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 0c101d3122a20..7525cc0883ed8 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -5997,7 +5997,9 @@ export const layer: Layer.Layer = "ProjectionStore.findThread", )(selectThread); // Subagents inherit the parent's immutable project. Use its index to bound the scan. - const getSubagentChildThreads: ProjectionStoreV2Shape["getSubagentChildThreads"] = (threadId) => + const getSubagentChildThreads: ProjectionStoreV2Shape["getSubagentChildThreads"] = Effect.fn( + "OrchestratorV2.getSubagentChildThreads", + )((threadId) => sql` SELECT child.payload_json FROM orchestration_v2_projection_threads AS child JOIN orchestration_v2_projection_threads AS parent @@ -6008,7 +6010,8 @@ export const layer: Layer.Layer = `.pipe( Effect.mapError((cause) => new ProjectionStoreReadError({ threadId, cause })), Effect.flatMap(decodeRows(decodeThreadPayload, threadId)), - ); + ), + ); const requireThread = Effect.fn("ProjectionStore.requireThread")(function* ( threadId: ThreadId, From 4f0800da23093aa28cc24643f3761cec83d950da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 16:59:56 +0000 Subject: [PATCH 14/23] test(server): check descendants after a conditional delete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The empty parent reply alone cannot detect a descendant deletion. Assert that the parent and its same-project subagents all survive the unused-only close. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- apps/server/src/orchestration-v2/runtimeLayer.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 1a0ed198d3f00..a6a05482ddb66 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14700,6 +14700,7 @@ describe("subagent lifecycle cascade", () => { onlyIfUnused: true, }); assert.isEmpty(kept.storedEvents); + assert.equal(yield* f.state("deletedAt", parent, app, native, grandchild), "0000"); if (history === "pruned fresh event") { const store = yield* ProjectionStore.ProjectionStoreV2; From 90b5e50acc2398e71d12d5007e28bec2d077e075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 17:18:07 +0000 Subject: [PATCH 15/23] fix(server): preserve subagent cascade receipt ownership MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retry child lifecycle commands when their IDs are occupied, including a conditional delete that left a used child live. Record the source archive with its receipt and keep no-op deletes out of archive ordering so restore respects independent choices. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../orchestration-v2/CommandReceiptStore.ts | 101 ++++++++++--- .../src/orchestration-v2/EffectOutbox.ts | 1 + apps/server/src/orchestration-v2/EventSink.ts | 12 +- .../src/orchestration-v2/Orchestrator.ts | 117 +++++++++------ .../ThreadManagementService.ts | 85 +++++++---- .../src/orchestration-v2/runtimeLayer.test.ts | 134 +++++++++++++++++- apps/server/src/persistence/Migrations.ts | 2 + .../106_CommandReceiptArchiveSource.ts | 9 ++ .../OrchestrationCommandReceipts.ts | 22 ++- .../reconcileV2PreviewMigration.test.ts | 2 + apps/server/src/project/ProjectService.ts | 1 + 11 files changed, 385 insertions(+), 101 deletions(-) create mode 100644 apps/server/src/persistence/Migrations/106_CommandReceiptArchiveSource.ts diff --git a/apps/server/src/orchestration-v2/CommandReceiptStore.ts b/apps/server/src/orchestration-v2/CommandReceiptStore.ts index 24e2d2714453a..7322c87f7e74c 100644 --- a/apps/server/src/orchestration-v2/CommandReceiptStore.ts +++ b/apps/server/src/orchestration-v2/CommandReceiptStore.ts @@ -58,6 +58,7 @@ export const CommandReceiptV2 = Schema.Struct({ resultSequence: NonNegativeInt, status: CommandReceiptV2Status, error: Schema.NullOr(Schema.String), + archiveSourceCommandId: Schema.optional(CommandId), }); export type CommandReceiptV2 = typeof CommandReceiptV2.Type; @@ -72,7 +73,7 @@ export const ProjectCommandReceiptV2 = Schema.Struct({ }); export type ProjectCommandReceiptV2 = typeof ProjectCommandReceiptV2.Type; -type AnyCommandReceiptV2 = CommandReceiptV2 | ProjectCommandReceiptV2; +export type AnyCommandReceiptV2 = CommandReceiptV2 | ProjectCommandReceiptV2; export interface CommandReceiptStoreV2Shape { readonly getLatestLifecycle: ( @@ -85,6 +86,9 @@ export interface CommandReceiptStoreV2Shape { readonly upsert: ( receipt: AnyCommandReceiptV2, ) => Effect.Effect; + readonly getAnyByCommandId: ( + commandId: CommandId, + ) => Effect.Effect, CommandReceiptStoreV2Error>; /** None for project command ids. */ readonly getByCommandId: ( commandId: CommandId, @@ -122,6 +126,23 @@ function fromApplicationReceipt(receipt: OrchestrationCommandReceipts.Orchestrat resultSequence: receipt.resultSequence, status: receipt.status, error: receipt.error, + ...(receipt.archiveSourceCommandId == null + ? {} + : { archiveSourceCommandId: receipt.archiveSourceCommandId }), + }); +} + +function fromProjectApplicationReceipt( + receipt: OrchestrationCommandReceipts.OrchestrationCommandReceipt, +) { + return decodeProjectReceipt({ + commandId: receipt.commandId, + projectId: receipt.aggregateId, + commandType: receipt.commandType, + acceptedAt: receipt.acceptedAt, + resultSequence: receipt.resultSequence, + status: receipt.status, + error: receipt.error, }); } @@ -132,7 +153,11 @@ function toApplicationReceipt( commandId: receipt.commandId, ...("projectId" in receipt ? { aggregateKind: "project" as const, aggregateId: receipt.projectId } - : { aggregateKind: "thread" as const, aggregateId: receipt.threadId }), + : { + aggregateKind: "thread" as const, + aggregateId: receipt.threadId, + archiveSourceCommandId: receipt.archiveSourceCommandId, + }), commandType: receipt.commandType, acceptedAt: DateTime.formatIso(receipt.acceptedAt), resultSequence: receipt.resultSequence, @@ -180,6 +205,19 @@ const layerBase: Layer.Layer< }), ), ), + getAnyByCommandId: Effect.fn("CommandReceiptStore.getAnyByCommandId")((commandId) => + receipts.getByCommandId({ commandId }).pipe( + Effect.flatMap((receipt) => { + if (Option.isNone(receipt)) return Effect.succeedNone; + const decode: Effect.Effect = + receipt.value.aggregateKind === "project" + ? fromProjectApplicationReceipt(receipt.value) + : fromApplicationReceipt(receipt.value); + return decode.pipe(Effect.asSome); + }), + Effect.mapError((cause) => new CommandReceiptStoreReadError({ commandId, cause })), + ), + ), getByCommandId: (commandId) => receipts.getByCommandId({ commandId }).pipe( Effect.flatMap( @@ -204,15 +242,7 @@ const layerBase: Layer.Layer< Effect.flatMap((receipt) => Option.isNone(receipt) || receipt.value.aggregateKind !== "project" ? Effect.succeedNone - : decodeProjectReceipt({ - commandId: receipt.value.commandId, - projectId: receipt.value.aggregateId, - commandType: receipt.value.commandType, - acceptedAt: receipt.value.acceptedAt, - resultSequence: receipt.value.resultSequence, - status: receipt.value.status, - error: receipt.value.error, - }).pipe(Effect.asSome), + : fromProjectApplicationReceipt(receipt.value).pipe(Effect.asSome), ), Effect.mapError((cause) => new CommandReceiptStoreReadError({ commandId, cause })), ), @@ -224,13 +254,18 @@ export const layer = layerBase.pipe(Layer.provide(OrchestrationCommandReceipts.l export const layerFromApplicationReceipts = layerBase; -/** Rejections are permanent, so a cascade retry takes the first unused or accepted suffix. */ -export const cascadeCommandId = Effect.fn("CommandReceiptStore.cascadeCommandId")(function* ( +export const cascadeCommandIdLimit = 32; + +export const cascadeCommandIdAt = (base: CommandId, retry: number) => + retry === 0 ? base : CommandId.make(`${base}:retry:${retry}`); + +/** Older receipts encode cascade ownership in the command ID. */ +const cascadeCommandId = Effect.fn("CommandReceiptStore.cascadeCommandId")(function* ( receipts: CommandReceiptStoreV2Shape, base: CommandId, ) { - for (let retry = 0; retry < 32; retry++) { - const id = retry === 0 ? base : CommandId.make(`${base}:retry:${retry}`); + for (let retry = 0; retry < cascadeCommandIdLimit; retry++) { + const id = cascadeCommandIdAt(base, retry); const receipt = yield* receipts.getByCommandId(id); if (Option.isNone(receipt) || receipt.value.status === "accepted") return id; } @@ -253,8 +288,36 @@ export const archivedByCascade = Effect.fn("CommandReceiptStore.archivedByCascad expectedArchiveCommandId?: CommandId, ) { if (latest?.commandType !== "thread.archive") return false; - let rootCommandId: string = latest.commandId; - for (const threadId of path.toReversed()) { + let archive = latest; + let depth = path.length; + while (depth > 0 && archive.archiveSourceCommandId !== undefined) { + if (archive.archiveSourceCommandId === archive.commandId) return false; + const source = Option.getOrUndefined( + yield* receipts.getByCommandId(archive.archiveSourceCommandId), + ); + if ( + source?.status !== "accepted" || + source.commandType !== "thread.archive" || + source.resultSequence > archive.resultSequence + ) + return false; + // A cascade can pass through an already archived thread without claiming its archive. + const ancestor = + source.threadId === rootThreadId + ? 0 + : depth < 2 + ? -1 + : path.lastIndexOf(source.threadId, depth - 2) + 1; + if (ancestor <= 0 && source.threadId !== rootThreadId) return false; + depth = ancestor; + archive = source; + } + if (depth === 0) + return expectedArchiveCommandId === undefined || archive.commandId === expectedArchiveCommandId; + + const legacyPath = path.slice(0, depth); + let rootCommandId: string = archive.commandId; + for (const threadId of legacyPath.toReversed()) { const segment = `:subagent:${threadId}`; if (!rootCommandId.endsWith(segment)) rootCommandId = rootCommandId.replace(/:retry:\d+$/, ""); if (!rootCommandId.endsWith(segment)) return false; @@ -266,9 +329,9 @@ export const archivedByCascade = Effect.fn("CommandReceiptStore.archivedByCascad const valid = root?.status === "accepted" && root.threadId === rootThreadId; // Derive the IDs again, so a user ID only shaped like a cascade retry is not counted. let derived = valid && root.commandType === "thread.archive" ? root.commandId : undefined; - for (const threadId of path) { + for (const threadId of legacyPath) { if (derived === undefined) return false; derived = yield* cascadeCommandId(receipts, CommandId.make(`${derived}:subagent:${threadId}`)); } - return derived === latest.commandId && (root?.resultSequence ?? 0) <= latest.resultSequence; + return derived === archive.commandId && (root?.resultSequence ?? 0) <= archive.resultSequence; }); diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index 17108d7060ac7..74e748551abc1 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -54,6 +54,7 @@ export const SubagentCascade = Context.Reference< } | undefined; readonly path: ReadonlyArray; + readonly parentArchiveCommandId: CommandId; } | undefined >("t3/orchestration-v2/SubagentCascade", { defaultValue: () => undefined }); diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index b207c770a7c28..7faf661ad2f5a 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -211,6 +211,7 @@ export interface EventSinkV2Shape { readonly acceptedAt: DateTime.Utc; readonly events: ReadonlyArray; readonly effects: ReadonlyArray; + readonly archiveSourceCommandId?: CommandId; readonly cancelUnsettledEffects?: { readonly effectTypes: ReadonlyArray; readonly reason: string; @@ -219,7 +220,7 @@ export interface EventSinkV2Shape { readonly updateDelegatedCompletionOnlyRunId?: RunId; }) => Effect.Effect< { - readonly receipt: CommandReceiptStore.CommandReceiptV2; + readonly receipt: CommandReceiptStore.AnyCommandReceiptV2; readonly storedEvents: ReadonlyArray; readonly committed: boolean; readonly cancelledEffectCount: number; @@ -772,12 +773,13 @@ const layerBase: Layer.Layer< const existingCommandResult = /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced(function* (commandId: CommandId) { - const existing = yield* commandReceipts.getByCommandId(commandId); + const existing = yield* commandReceipts.getAnyByCommandId(commandId); if (Option.isNone(existing)) { return yield* Effect.die( new Error(`Command receipt ${commandId} disappeared during its transaction.`), ); } + if ("projectId" in existing.value) return { receipt: existing.value, storedEvents: [] }; const storedEvents = yield* eventStore.readByCommandId({ commandId }).pipe( Stream.runCollect, Effect.map((events): ReadonlyArray => Array.from(events)), @@ -868,6 +870,9 @@ const layerBase: Layer.Layer< resultSequence: sequence, status: "accepted", error: null, + ...(input.archiveSourceCommandId === undefined + ? {} + : { archiveSourceCommandId: input.archiveSourceCommandId }), }; yield* commandReceipts.upsert(receipt); // Generated side-chat cascades need archive ownership after their outbox entries are pruned. @@ -882,6 +887,9 @@ const layerBase: Layer.Layer< commandId: effect.commandId, threadId: effect.threadId, commandType: effect.request.operation, + ...(effect.request.operation === "thread.archive" + ? { archiveSourceCommandId: effect.commandId } + : {}), }); if (!reserved) { return yield* new CommandReceiptStore.CommandReceiptStoreWriteError({ diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 0134275d8ecdb..37ffbfb352a28 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -129,6 +129,7 @@ import { resolveMessageDispatchIntent, } from "./CommandPolicy.ts"; import { + type AnyCommandReceiptV2, archivedByCascade, CommandReceiptStoreV2, type CommandReceiptV2, @@ -405,6 +406,7 @@ export interface OrchestratorV2DispatchResult { readonly sequence: number; readonly storedEvents: ReadonlyArray; readonly subagentCascades?: ReadonlyArray; + readonly cascadeIdOccupied?: true; /** The command id was already accepted; analytics skip it. */ readonly replayed?: true; } @@ -16219,24 +16221,49 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio "orchestration_v2.thread_id": commandThreadId(command), }); - const existingReceipt = yield* commandReceipts.getByCommandId(command.commandId).pipe( - Effect.mapError( - (cause) => - new OrchestratorDispatchError({ - commandId: command.commandId, - commandType: command.type, - cause, - }), - ), + const cascade = yield* EffectOutbox.SubagentCascade; + const read = mapDispatchError(command); + const cascadeThread = + cascade !== undefined && + (command.type === "thread.archive" || + command.type === "thread.unarchive" || + command.type === "thread.delete") + ? yield* projectionStore.getThread(command.threadId).pipe(read) + : undefined; + if (cascadeThread !== undefined) { + const parentId = cascadeThread.lineage.parentThreadId; + const parent = + parentId === null ? null : yield* projectionStore.getThread(parentId).pipe(read); + if (parent?.projectId !== cascadeThread.projectId) + return { sequence: cascade?.source?.cutoff ?? 0, storedEvents: [] }; + yield* refuseAboveDispatchModeLimit(command, cascadeThread.id, cascadeThread); + } + const occupiedCascadeId = (receipt: AnyCommandReceiptV2) => + cascadeThread !== undefined && + ("projectId" in receipt || + receipt.threadId !== cascadeThread.id || + receipt.status !== "accepted" || + receipt.commandType !== command.type || + (command.type === "thread.delete" && cascadeThread.deletedAt === null) || + (command.type === "thread.archive" && + receipt.archiveSourceCommandId !== undefined && + receipt.archiveSourceCommandId !== cascade?.parentArchiveCommandId)); + + const existingReceipt = yield* read( + cascadeThread === undefined + ? commandReceipts.getByCommandId(command.commandId) + : commandReceipts.getAnyByCommandId(command.commandId), ); - if (Option.isSome(existingReceipt)) { - return yield* replayCommandReceipt(command, existingReceipt.value); + const receipt = existingReceipt.value; + if (occupiedCascadeId(receipt)) + return { sequence: receipt.resultSequence, storedEvents: [], cascadeIdOccupied: true }; + if ("threadId" in receipt) return yield* replayCommandReceipt(command, receipt); } // Recheck under the lock; the user may have raised modes since sender preflight. const limit = yield* DispatchModeLimit; - if (limit !== undefined) { + if (cascadeThread === undefined && limit !== undefined) { // Fork and merge-back sources are unlocked; validate the modes copied by their dispatch. const threadId = commandThreadId(command); const shell = yield* projectionStore @@ -16265,40 +16292,29 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } // Decided under the child's lock, so a newer independent choice on it always wins. - const cascade = yield* EffectOutbox.SubagentCascade; if ( - cascade !== undefined && - (command.type === "thread.archive" || - command.type === "thread.unarchive" || - command.type === "thread.delete") + cascadeThread !== undefined && + cascade?.source !== undefined && + command.type !== "thread.delete" ) { const { source, path } = cascade; - const read = mapDispatchError(command); - const thread = yield* projectionStore.getThread(command.threadId).pipe(read); - const parentThreadId = thread.lineage.parentThreadId; - const parent = - parentThreadId === null - ? null - : yield* projectionStore.getThread(parentThreadId).pipe(read); - // A child moved to another project since the walk found it is no longer this subtree's. - let applies = parent?.projectId === thread.projectId; - if (applies && source !== undefined && command.type !== "thread.delete") { - const latest = yield* commandReceipts.getLatestLifecycle(command.threadId).pipe(read); - applies = - thread.deletedAt === null && - (command.type === "thread.archive" - ? thread.archivedAt === null && - !supersedesCascade(latest, source.cutoff, command.commandId) - : thread.archivedAt !== null && - (yield* archivedByCascade( - commandReceipts, - latest, - source.threadId, - path, - source.archiveCommandId, - ).pipe(read))); - } - if (!applies) return { sequence: source?.cutoff ?? 0, storedEvents: [] }; + const latest = yield* commandReceipts + .getLatestLifecycle(cascadeThread.id, ["thread.archive", "thread.unarchive"]) + .pipe(read); + const applies = + cascadeThread.deletedAt === null && + (command.type === "thread.archive" + ? cascadeThread.archivedAt === null && + !supersedesCascade(latest, source.cutoff, command.commandId) + : cascadeThread.archivedAt !== null && + (yield* archivedByCascade( + commandReceipts, + latest, + source.threadId, + path, + source.archiveCommandId, + ).pipe(read))); + if (!applies) return { sequence: source.cutoff, storedEvents: [] }; } const plan = yield* dispatchOnce(command, sideChatIds).pipe( @@ -16446,6 +16462,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }).pipe(mapDispatchError(command)), ], effects: plan.effects, + ...(command.type === "thread.archive" + ? { archiveSourceCommandId: cascade?.parentArchiveCommandId ?? command.commandId } + : {}), ...(plan.cancelUnsettledEffects === undefined ? {} : { cancelUnsettledEffects: plan.cancelUnsettledEffects }), @@ -16464,6 +16483,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ); + if (!committed.committed && occupiedCascadeId(committed.receipt)) + return { + sequence: committed.receipt.resultSequence, + storedEvents: [], + cascadeIdOccupied: true, + }; + if ("projectId" in committed.receipt) + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "Conflicting receipt is not a thread command receipt.", + }); if (committed.receipt.status === "rejected") { return yield* new OrchestratorCommandPreviouslyRejectedError({ commandId: command.commandId, diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index a31d239c58b63..e5544ade0c8ea 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -1856,29 +1856,54 @@ const make = Effect.gen(function* () { cutoff: sideChat?.cutoff ?? receipt.resultSequence, archiveCommandId: sideChat?.archiveCommandId, }; + const dispatchChild = Effect.fn("ThreadManagementService.dispatchCascadeChild")(function* ( + threadId: ThreadId, + baseId: CommandId, + path: ReadonlyArray, + parentArchiveCommandId: CommandId, + ) { + for (let retry = 0; retry < CommandReceiptStore.cascadeCommandIdLimit; retry++) { + const commandId = CommandReceiptStore.cascadeCommandIdAt(baseId, retry); + const result = yield* orchestrator + .dispatch({ type: input.operation, commandId, threadId }) + .pipe( + Effect.tap(cascadeBeforeReply), + Effect.provideService(EffectOutbox.SubagentCascade, { + source, + path, + parentArchiveCommandId, + }), + Effect.catchTags({ + OrchestratorThreadAboveModeLimitError: () => Effect.undefined, + }), + ); + if (result === undefined) return; + if (!result.cascadeIdOccupied) return commandId; + } + return yield* new Orchestrator.OrchestratorDispatchError({ + commandId: baseId, + commandType: input.operation, + cause: "All 32 subagent lifecycle command IDs were occupied or rejected.", + }); + }); + const visit: ( thread: { readonly id: ThreadId; readonly deletedAt: unknown; readonly projectId?: unknown; }, - commandId: CommandId, + baseId: CommandId, path: ReadonlyArray, + parentArchiveCommandId: CommandId, ) => ReturnType = Effect.fn( "ThreadManagementService.visitSubagentCascade", - )(function* (thread, commandId, path) { - if (path.length > 0 && thread.deletedAt === null) { - const command = { type: input.operation, commandId, threadId: thread.id }; - const dispatched = yield* orchestrator.dispatch(command).pipe( - Effect.tap((result) => cascadeBeforeReply(result)), - Effect.provideService(EffectOutbox.SubagentCascade, { source, path }), - Effect.as(true), - Effect.catchTags({ - OrchestratorThreadAboveModeLimitError: () => Effect.succeed(false), - }), - ); - if (!dispatched) return; - } + )(function* (thread, baseId, path, parentArchiveCommandId) { + const commandId = + path.length > 0 && thread.deletedAt === null + ? yield* dispatchChild(thread.id, baseId, path, parentArchiveCommandId) + : baseId; + if (commandId === undefined) return; // Holds each thread's lock while visiting its subagents, so locks nest parent before child. yield* threadDispatch.withLock( thread.id, @@ -1903,27 +1928,24 @@ const make = Effect.gen(function* () { if (limit && exceededDispatchModeLimit(limit, current)) return; if (source && current.deletedAt !== null) return; const latest = source - ? yield* receipts.getLatestLifecycle( - thread.id, - path.length === 0 && sideChat - ? ["thread.archive", "thread.unarchive"] - : undefined, - ) + ? yield* receipts.getLatestLifecycle(thread.id, [ + "thread.archive", + "thread.unarchive", + ]) : null; if (source && CommandReceiptStore.supersedesCascade(latest, source.cutoff, commandId)) return; + const archiveCommandId = + input.operation === "thread.archive" && + latest?.commandType === "thread.archive" && + latest.commandId === commandId + ? commandId + : parentArchiveCommandId; const children = yield* orchestrator.getSubagentChildThreads(thread.id); const [, failures] = yield* Effect.partition(children, (child) => Effect.gen(function* () { const base = CommandId.make(`${commandId}:subagent:${child.id}`); - const childCommandId = yield* CommandReceiptStore.cascadeCommandId(receipts, base); - if (childCommandId !== undefined) - return yield* visit(child, childCommandId, [...path, child.id]); - return yield* new Orchestrator.OrchestratorDispatchError({ - commandId: base, - commandType: input.operation, - cause: "All 32 subagent lifecycle command IDs were rejected.", - }); + yield* visit(child, base, [...path, child.id], archiveCommandId); }), ); // Finish healthy siblings before reporting a failure to the durable worker. @@ -1931,7 +1953,12 @@ const make = Effect.gen(function* () { }), ); }); - const walk = visit({ id: input.threadId, deletedAt: null }, input.commandId, []); + const walk = visit( + { id: input.threadId, deletedAt: null }, + input.commandId, + [], + input.commandId, + ); const cascade = parentThreadId === undefined ? walk : threadDispatch.withLock(parentThreadId, walk); yield* input.operation === "thread.delete" && diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index a6a05482ddb66..28008bd1faaab 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14931,6 +14931,135 @@ describe("subagent lifecycle cascade", () => { assert.isTrue(Option.isNone(yield* receipt("archive-deleted:subagent:native"))); }).pipe(Effect.provide(layerTest)), ); + it.effect.each(["existing thread", "late project"] as const)( + "skips a %s receipt at the child lifecycle command ID", + (collision) => + Effect.gen(function* () { + const f = yield* fixture; + const commandId = CommandId.make("archive:parent:subagent:app"); + const receipts = yield* Receipts.CommandReceiptStoreV2; + const existing = Option.getOrThrow(yield* receipt("create:parent")); + const projectId = ProjectId.make("other-project"); + const sink = yield* EventSink.EventSinkV2; + const commit = sink.commitCommand; + if (collision === "existing thread") { + yield* f.run("archive", ThreadId.make("fork"), commandId); + } + using _collision = vi.spyOn(sink, "commitCommand").mockImplementation((input) => + collision === "late project" && input.commandId === commandId + ? receipts + .insertIfAbsent({ + commandId, + projectId, + commandType: "project.create", + acceptedAt: existing.acceptedAt, + resultSequence: existing.resultSequence, + status: "accepted", + error: null, + }) + .pipe(Effect.orDie, Effect.andThen(commit(input))) + : commit(input), + ); + yield* f.run("archive"); + assert.equal(yield* f.archived(app, grandchild, native), "111"); + assert.equal(Option.getOrThrow(yield* receipt(`${commandId}:retry:1`)).threadId, app); + if (collision === "late project") { + assert.equal( + Option.getOrThrow(yield* receipts.getProjectByCommandId(commandId)).projectId, + projectId, + ); + } else { + assert.isTrue((yield* f.run("archive", ThreadId.make("fork"), commandId)).replayed); + assert.equal(Option.getOrThrow(yield* receipt(commandId)).threadId, "fork"); + } + }).pipe(Effect.provide(layerTest)), + ); + + it.effect.each(["existing", "late"] as const)( + "deletes a used subagent after a conditional close with a %s receipt", + (timing) => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.markUsed(native); + const conditional = { + type: "thread.delete", + commandId: CommandId.make("delete:parent:subagent:native"), + threadId: native, + onlyIfUnused: true, + } as const; + const receipts = yield* Receipts.CommandReceiptStoreV2; + const sink = yield* EventSink.EventSinkV2; + const commit = sink.commitCommand; + const create = Option.getOrThrow(yield* receipt("create:parent")); + if (timing === "existing") + assert.isEmpty((yield* f.threads.dispatch(conditional)).storedEvents); + const original = Option.getOrElse(yield* receipt(conditional.commandId), () => ({ + ...create, + commandId: conditional.commandId, + commandType: conditional.type, + threadId: native, + })); + using _conditional = vi + .spyOn(sink, "commitCommand") + .mockImplementation((input) => + timing === "late" && input.commandId === conditional.commandId + ? receipts.insertIfAbsent(original).pipe(Effect.orDie, Effect.andThen(commit(input))) + : commit(input), + ); + yield* f.run("delete"); + assert.equal(yield* f.state("deletedAt", parent, native), "11"); + assert.equal( + Option.getOrThrow(yield* receipt(`${conditional.commandId}:retry:1`)).threadId, + native, + ); + const replay = yield* f.threads.dispatch(conditional); + assert.isTrue(replay.replayed); + assert.isEmpty(replay.storedEvents); + assert.deepEqual(Option.getOrThrow(yield* receipt(conditional.commandId)), original); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect.each(["fresh", "legacy"] as const)( + "preserves an independent archive while restoring a %s cascade receipt", + (history) => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.commit("archive"); + yield* f.run("archive", app, "archive:parent:subagent:app"); + yield* f.drain; + assert.equal(yield* f.archived(parent, app, grandchild, native), "1111"); + if (history === "legacy") { + const receipts = yield* Receipts.CommandReceiptStoreV2; + const { archiveSourceCommandId: _source, ...legacy } = Option.getOrThrow( + yield* receipt("archive:parent:subagent:native"), + ); + yield* receipts.upsert(legacy); + } + yield* TestClock.adjust("8 days"); + assert.isAbove(yield* f.outbox.pruneSettled, 0); + yield* f.run("unarchive"); + yield* f.drain; + assert.equal(yield* f.archived(parent, app, grandchild, native), "0110"); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect.each(["before restore", "before delivery"] as const)( + "restores a used intermediate after a conditional close %s", + (when) => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.markUsed(app); + yield* f.run("archive"); + yield* f.drain; + if (when === "before delivery") yield* f.commit("unarchive"); + yield* f.renameAndKeep(app, when); + if (when === "before restore") yield* f.commit("unarchive"); + yield* f.drain; + assert.equal(yield* f.state("deletedAt", app), "0"); + assert.equal(yield* f.archived(app, grandchild), "00"); + }).pipe(Effect.provide(layerTest)), + ); + it.effect.each([31, 32])("bounds child retries at 32 IDs with %i rejected", (count) => Effect.gen(function* () { const f = yield* fixture; @@ -14948,7 +15077,10 @@ describe("subagent lifecycle cascade", () => { if (count === 32) { const { status, attemptCount, lastError } = yield* f.cascade; assert.deepEqual([status, attemptCount], ["pending", 1]); - assert.include(lastError, "All 32 subagent lifecycle command IDs were rejected."); + assert.include( + lastError, + "All 32 subagent lifecycle command IDs were occupied or rejected.", + ); } else { assert.equal(Option.getOrThrow(accepted).status, "accepted"); assert.equal((yield* f.cascade).status, "running"); diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 91b521fdb6036..155e7fa99e89c 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -113,6 +113,7 @@ import Migration0102 from "./Migrations/102_ThreadItemCounts.ts"; import Migration0103 from "./Migrations/103_IssueTrackerAdditionalScopes.ts"; import Migration0104 from "./Migrations/104_ProjectionProjectsTrackedFolders.ts"; import Migration0105 from "./Migrations/105_TaskFolders.ts"; +import Migration0106 from "./Migrations/106_CommandReceiptArchiveSource.ts"; const migrationEntries = [ [1, "OrchestrationEvents", Migration0001], @@ -221,6 +222,7 @@ const migrationEntries = [ [103, "IssueTrackerAdditionalScopes", Migration0103], [104, "ProjectionProjectsTrackedFolders", Migration0104], [105, "TaskFolders", Migration0105], + [106, "CommandReceiptArchiveSource", Migration0106], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/106_CommandReceiptArchiveSource.ts b/apps/server/src/persistence/Migrations/106_CommandReceiptArchiveSource.ts new file mode 100644 index 0000000000000..20ef4eae87278 --- /dev/null +++ b/apps/server/src/persistence/Migrations/106_CommandReceiptArchiveSource.ts @@ -0,0 +1,9 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql` + ALTER TABLE orchestration_command_receipts ADD COLUMN archive_source_command_id TEXT + `; +}); diff --git a/apps/server/src/persistence/OrchestrationCommandReceipts.ts b/apps/server/src/persistence/OrchestrationCommandReceipts.ts index 5c3c0118a4bea..85624fbdb71cf 100644 --- a/apps/server/src/persistence/OrchestrationCommandReceipts.ts +++ b/apps/server/src/persistence/OrchestrationCommandReceipts.ts @@ -22,6 +22,7 @@ export const OrchestrationCommandReceipt = Schema.Struct({ resultSequence: NonNegativeInt, status: Schema.Literals(["accepted", "rejected"]), error: Schema.NullOr(Schema.String), + archiveSourceCommandId: Schema.optional(Schema.NullOr(CommandId)), }); export type OrchestrationCommandReceipt = typeof OrchestrationCommandReceipt.Type; @@ -83,7 +84,8 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { accepted_at, result_sequence, status, - error + error, + archive_source_command_id ) VALUES ( ${receipt.commandId}, @@ -93,7 +95,8 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { ${receipt.acceptedAt}, ${receipt.resultSequence}, ${receipt.status}, - ${receipt.error} + ${receipt.error}, + ${receipt.archiveSourceCommandId ?? null} ) ON CONFLICT (command_id) DO UPDATE SET @@ -103,7 +106,8 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { accepted_at = excluded.accepted_at, result_sequence = excluded.result_sequence, status = excluded.status, - error = excluded.error + error = excluded.error, + archive_source_command_id = excluded.archive_source_command_id `, }); @@ -120,7 +124,8 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { accepted_at AS "acceptedAt", result_sequence AS "resultSequence", status, - error + error, + archive_source_command_id AS "archiveSourceCommandId" FROM orchestration_command_receipts WHERE command_id = ${commandId} `, @@ -132,7 +137,8 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { execute: ({ threadId, commandTypes }) => sql` SELECT command_id AS "commandId", aggregate_kind AS "aggregateKind", aggregate_id AS "aggregateId", command_type AS "commandType", - accepted_at AS "acceptedAt", result_sequence AS "resultSequence", status, error + accepted_at AS "acceptedAt", result_sequence AS "resultSequence", status, error, + archive_source_command_id AS "archiveSourceCommandId" FROM orchestration_command_receipts WHERE aggregate_kind = 'thread' AND aggregate_id = ${threadId} AND status = 'accepted' AND command_type IN ('thread.archive', 'thread.unarchive', 'thread.delete') @@ -158,7 +164,8 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { accepted_at, result_sequence, status, - error + error, + archive_source_command_id ) VALUES ( ${receipt.commandId}, @@ -168,7 +175,8 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { ${receipt.acceptedAt}, ${receipt.resultSequence}, ${receipt.status}, - ${receipt.error} + ${receipt.error}, + ${receipt.archiveSourceCommandId ?? null} ) ON CONFLICT(command_id) DO NOTHING RETURNING command_id diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index d2b0dafab8731..44430ce6abe36 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -85,6 +85,7 @@ describe("V2 preview upgrade", () => { [103, "IssueTrackerAdditionalScopes"], [104, "ProjectionProjectsTrackedFolders"], [105, "TaskFolders"], + [106, "CommandReceiptArchiveSource"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -213,6 +214,7 @@ describe("V2 preview upgrade", () => { [103, "IssueTrackerAdditionalScopes"], [104, "ProjectionProjectsTrackedFolders"], [105, "TaskFolders"], + [106, "CommandReceiptArchiveSource"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/project/ProjectService.ts b/apps/server/src/project/ProjectService.ts index 80fabe6101c05..287879a9d8fcd 100644 --- a/apps/server/src/project/ProjectService.ts +++ b/apps/server/src/project/ProjectService.ts @@ -693,6 +693,7 @@ export const make = Effect.gen(function* () { effects: plan.effects, }); if ( + "projectId" in committed.receipt || committed.receipt.threadId !== command.threadId || committed.receipt.commandType !== command.type ) { From b513d7944bf53f97675571d9e23b9048e344442a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 17:18:09 +0000 Subject: [PATCH 16/23] docs(user): explain the subagent cascade project boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A moved subagent or side chat stops the descendant cascade at its new project. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- docs/user/thread-sidebar.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/thread-sidebar.md b/docs/user/thread-sidebar.md index 024c3976ea56f..4e4a653e4fdc5 100644 --- a/docs/user/thread-sidebar.md +++ b/docs/user/thread-sidebar.md @@ -281,7 +281,7 @@ fill the screen; close the full-screen view to return to the parent and inspecto Stop on a thread also stops the subagents it delegated to. Before archiving the parent, wait for its delegated tasks to finish or cancel them. -Archiving, restoring or deleting a thread also applies to its subagent descendants, including subagents in its side chats. Ordinary forks stay independent. Restoring brings back only descendants archived by that action, so independent archives and newer changes on a descendant stay as they are. +Archiving, restoring or deleting a thread also applies to its subagent descendants, including subagents in its side chats. The cascade stops at a subagent or side chat moved to another project. Ordinary forks stay independent. Restoring brings back only descendants archived by that action, so independent archives and newer changes on a descendant stay as they are. Subagent threads started by the agent can't take messages; message the parent thread instead. When such a subagent needs an approval or an answer, the parent From adc6eb6fe214dd704a4bbeadbc6ca9d901a45543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 17:36:57 +0000 Subject: [PATCH 17/23] fix(server): keep subagent cascade retries on the same path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Advance past restore receipts that predate the initiating command. Keep traversal IDs stable when a child needs no mutation, and preserve the older prefix rules for already-committed outbox jobs. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../orchestration-v2/CommandReceiptStore.ts | 9 +- .../src/orchestration-v2/EffectOutbox.ts | 1 + .../src/orchestration-v2/EffectWorker.ts | 1 + .../src/orchestration-v2/Orchestrator.ts | 4 + .../ThreadManagementService.ts | 40 ++++++--- .../src/orchestration-v2/runtimeLayer.test.ts | 85 +++++++++++++++++++ 6 files changed, 127 insertions(+), 13 deletions(-) diff --git a/apps/server/src/orchestration-v2/CommandReceiptStore.ts b/apps/server/src/orchestration-v2/CommandReceiptStore.ts index 7322c87f7e74c..63e5bfd3eeb0a 100644 --- a/apps/server/src/orchestration-v2/CommandReceiptStore.ts +++ b/apps/server/src/orchestration-v2/CommandReceiptStore.ts @@ -260,7 +260,7 @@ export const cascadeCommandIdAt = (base: CommandId, retry: number) => retry === 0 ? base : CommandId.make(`${base}:retry:${retry}`); /** Older receipts encode cascade ownership in the command ID. */ -const cascadeCommandId = Effect.fn("CommandReceiptStore.cascadeCommandId")(function* ( +export const legacyCascadeCommandId = Effect.fn("CommandReceiptStore.cascadeCommandId")(function* ( receipts: CommandReceiptStoreV2Shape, base: CommandId, ) { @@ -276,7 +276,7 @@ const cascadeCommandId = Effect.fn("CommandReceiptStore.cascadeCommandId")(funct export const supersedesCascade = ( latest: CommandReceiptV2 | null, cutoff: number, - ownCommandId: CommandId, + ownCommandId?: CommandId, ) => latest !== null && latest.resultSequence > cutoff && latest.commandId !== ownCommandId; /** Whether a cascade from an archive of `rootThreadId`, along `path`, archived this thread. */ @@ -331,7 +331,10 @@ export const archivedByCascade = Effect.fn("CommandReceiptStore.archivedByCascad let derived = valid && root.commandType === "thread.archive" ? root.commandId : undefined; for (const threadId of legacyPath) { if (derived === undefined) return false; - derived = yield* cascadeCommandId(receipts, CommandId.make(`${derived}:subagent:${threadId}`)); + derived = yield* legacyCascadeCommandId( + receipts, + CommandId.make(`${derived}:subagent:${threadId}`), + ); } return derived === archive.commandId && (root?.resultSequence ?? 0) <= archive.resultSequence; }); diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index 74e748551abc1..c72a61a6ba2c2 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -159,6 +159,7 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ operation: Schema.Literals(["thread.archive", "thread.unarchive", "thread.delete"]), sideChat: Schema.optional(EmbeddedSideChatCascade), directDelete: Schema.optional(Schema.Literal(true)), + stableSkipPrefix: Schema.optional(Schema.Literal(true)), limit: Schema.optional( Schema.Struct({ runtimeMode: RuntimeMode, interactionMode: ProviderInteractionMode }), ), diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index 79c7962e9c51b..584263f8cc1e1 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -689,6 +689,7 @@ export const layerExecutor: Layer.Layer< operation: effect.request.operation, sideChat: effect.request.sideChat, directDelete: effect.request.directDelete, + stableSkipPrefix: effect.request.stableSkipPrefix, }) .pipe( Effect.provideService(DispatchModeLimit, effect.request.limit), diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 37ffbfb352a28..c53d5b3f362f4 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -1941,6 +1941,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio type: "subagent-threads.cascade", operation, limit, + stableSkipPrefix: true, ...(operation === "thread.delete" && sideChat === undefined ? { directDelete: true as const } : {}), @@ -16245,6 +16246,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio receipt.status !== "accepted" || receipt.commandType !== command.type || (command.type === "thread.delete" && cascadeThread.deletedAt === null) || + (command.type === "thread.unarchive" && + cascade?.source !== undefined && + receipt.resultSequence < cascade.source.cutoff) || (command.type === "thread.archive" && receipt.archiveSourceCommandId !== undefined && receipt.archiveSourceCommandId !== cascade?.parentArchiveCommandId)); diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index e5544ade0c8ea..2c9322c4d8424 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -559,6 +559,7 @@ export interface ThreadManagementServiceShape { readonly operation: "thread.archive" | "thread.unarchive" | "thread.delete"; readonly sideChat?: EffectOutbox.EmbeddedSideChatCascade | undefined; readonly directDelete?: true | undefined; + readonly stableSkipPrefix?: true | undefined; }) => Effect.Effect< void, Orchestrator.OrchestratorV2Error | CommandReceiptStore.CommandReceiptStoreV2Error @@ -872,6 +873,7 @@ const make = Effect.gen(function* () { operation: request.operation, sideChat: request.sideChat, directDelete: request.directDelete, + stableSkipPrefix: request.stableSkipPrefix, }).pipe( Effect.catch((cause) => Effect.logWarning("Subagent lifecycle cascade left to the effect worker", { @@ -1878,7 +1880,8 @@ const make = Effect.gen(function* () { }), ); if (result === undefined) return; - if (!result.cascadeIdOccupied) return commandId; + if (!result.cascadeIdOccupied) + return result.replayed || result.storedEvents.length > 0 ? commandId : null; } return yield* new Orchestrator.OrchestratorDispatchError({ commandId: baseId, @@ -1899,11 +1902,13 @@ const make = Effect.gen(function* () { ) => ReturnType = Effect.fn( "ThreadManagementService.visitSubagentCascade", )(function* (thread, baseId, path, parentArchiveCommandId) { - const commandId = - path.length > 0 && thread.deletedAt === null - ? yield* dispatchChild(thread.id, baseId, path, parentArchiveCommandId) - : baseId; - if (commandId === undefined) return; + const ownCommandId = + path.length === 0 + ? baseId + : thread.deletedAt === null + ? yield* dispatchChild(thread.id, baseId, path, parentArchiveCommandId) + : null; + if (ownCommandId === undefined) return; // Holds each thread's lock while visiting its subagents, so locks nest parent before child. yield* threadDispatch.withLock( thread.id, @@ -1933,18 +1938,33 @@ const make = Effect.gen(function* () { "thread.unarchive", ]) : null; - if (source && CommandReceiptStore.supersedesCascade(latest, source.cutoff, commandId)) + if ( + source && + CommandReceiptStore.supersedesCascade( + latest, + source.cutoff, + ownCommandId ?? undefined, + ) + ) return; const archiveCommandId = input.operation === "thread.archive" && latest?.commandType === "thread.archive" && - latest.commandId === commandId - ? commandId + latest.commandId === ownCommandId + ? ownCommandId : parentArchiveCommandId; const children = yield* orchestrator.getSubagentChildThreads(thread.id); + if (children.length === 0) return; + // Older jobs may have committed descendants under a suffixed skip. + const prefix = + ownCommandId ?? + (input.stableSkipPrefix + ? baseId + : ((yield* CommandReceiptStore.legacyCascadeCommandId(receipts, baseId)) ?? + baseId)); const [, failures] = yield* Effect.partition(children, (child) => Effect.gen(function* () { - const base = CommandId.make(`${commandId}:subagent:${child.id}`); + const base = CommandId.make(`${prefix}:subagent:${child.id}`); yield* visit(child, base, [...path, child.id], archiveCommandId); }), ); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 28008bd1faaab..7c9ee2732347a 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -15060,6 +15060,91 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); + it.effect("restores a child when an earlier restore holds the derived ID", () => + Effect.gen(function* () { + const f = yield* fixture; + const id = "unarchive:parent:subagent:native"; + yield* f.run("archive", native); + yield* f.run("unarchive", native, id); + const original = yield* receipt(id); + yield* f.run("archive"); + yield* f.run("unarchive"); + assert.equal(yield* f.archived(parent, native), "00"); + assert.equal(Option.getOrThrow(yield* receipt(`${id}:retry:1`)).threadId, native); + assert.isTrue((yield* f.run("unarchive", native, id)).replayed); + assert.deepEqual(yield* receipt(id), original); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect.each(["late collision", "legacy rejection"] as const)( + "resumes an archive past a skipped intermediate with a %s", + (scenario) => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.run("archive", app); + yield* f.run("unarchive", grandchild); + const leaf = ThreadId.make("leaf"); + yield* f.set(leaf, { + lineage: { + rootThreadId: parent, + parentThreadId: grandchild, + relationshipToParent: "subagent", + }, + }); + const skippedId = CommandId.make("archive:parent:subagent:app"); + const receipts = yield* Receipts.CommandReceiptStoreV2; + const created = Option.getOrThrow(yield* receipt("create:parent")); + const collision = { + ...created, + commandId: skippedId, + threadId: scenario === "legacy rejection" ? app : ThreadId.make("fork"), + status: scenario === "legacy rejection" ? "rejected" : "accepted", + error: scenario === "legacy rejection" ? "Earlier rejection." : null, + } as const; + yield* f.commit("archive"); + if (scenario === "legacy rejection") { + const sql = yield* SqlClient.SqlClient; + yield* sql` + UPDATE orchestration_v2_effect_outbox + SET payload_json = json_remove(payload_json, '$.stableSkipPrefix') + WHERE effect_id = 'effect:archive:parent:subagent-threads.cascade' + `; + yield* receipts.upsert(collision); + } + const orchestrator = yield* Orchestrator.OrchestratorV2; + const dispatch = orchestrator.dispatch; + let failLeaf = true; + using _failLeaf = vi.spyOn(orchestrator, "dispatch").mockImplementation((command) => { + if (command.type === "thread.archive" && command.threadId === leaf && failLeaf) { + failLeaf = false; + return Effect.fail( + new Orchestrator.OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "Leaf delivery interrupted.", + }), + ); + } + return dispatch(command); + }); + using _lostRetry = vi + .spyOn(f.outbox, "retry") + .mockImplementationOnce(() => Effect.succeed(true)); + yield* f.drain; + assert.equal(yield* f.archived(app, grandchild, leaf), "110"); + const prefix = scenario === "legacy rejection" ? `${skippedId}:retry:1` : skippedId; + const childId = `${prefix}:subagent:${grandchild}`; + const childReceipt = Option.getOrThrow(yield* receipt(childId)); + if (scenario === "late collision") yield* receipts.upsert(collision); + assert.equal((yield* f.outbox.reconcileAfterProcessLoss).requeued, 1); + yield* f.drain; + assert.equal((yield* f.cascade).status, "succeeded"); + assert.equal(yield* f.archived(app, grandchild, leaf), "111"); + assert.deepEqual(Option.getOrThrow(yield* receipt(childId)), childReceipt); + assert.deepEqual(Option.getOrThrow(yield* receipt(skippedId)), collision); + }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), + ); + it.effect.each([31, 32])("bounds child retries at 32 IDs with %i rejected", (count) => Effect.gen(function* () { const f = yield* fixture; From 9d87f242f272cba54a3af1d9b27303462da71f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 17:59:31 +0000 Subject: [PATCH 18/23] fix(server): recover legacy subagent cascade receipts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Advance past stale archives and use a verified descendant receipt when a later rejection changed a legacy traversal ID. Preserve independently archived work and let the durable worker report an unresolved legacy path when its evidence has expired. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../orchestration-v2/CommandReceiptStore.ts | 35 ++++ .../src/orchestration-v2/EffectOutbox.ts | 1 + .../src/orchestration-v2/Orchestrator.ts | 95 ++++++++- .../ThreadManagementService.ts | 41 ++-- .../src/orchestration-v2/runtimeLayer.test.ts | 196 +++++++++++------- 5 files changed, 274 insertions(+), 94 deletions(-) diff --git a/apps/server/src/orchestration-v2/CommandReceiptStore.ts b/apps/server/src/orchestration-v2/CommandReceiptStore.ts index 63e5bfd3eeb0a..9d06f727109be 100644 --- a/apps/server/src/orchestration-v2/CommandReceiptStore.ts +++ b/apps/server/src/orchestration-v2/CommandReceiptStore.ts @@ -272,6 +272,41 @@ export const legacyCascadeCommandId = Effect.fn("CommandReceiptStore.cascadeComm return undefined; }); +export const matchesLegacyCascadePath = Effect.fn("CommandReceiptStore.matchesLegacyCascadePath")( + function* ( + receipts: CommandReceiptStoreV2Shape, + anchorId: CommandId, + path: ReadonlyArray, + completedId: CommandId, + ) { + const retries: Array<{ baseId: CommandId; retry: number }> = []; + let id: string = completedId; + for (const threadId of path.toReversed()) { + const segment = `:subagent:${threadId}`; + if (!id.endsWith(segment)) { + const suffix = /:retry:([1-9][0-9]*)$/.exec(id); + if (suffix === null) return false; + const retry = Number(suffix[1]); + if (retry >= cascadeCommandIdLimit) return false; + id = id.slice(0, suffix.index); + retries.push({ baseId: CommandId.make(id), retry }); + } + if (!id.endsWith(segment)) return false; + id = id.slice(0, -segment.length); + } + if (id !== anchorId) return false; + + // Legacy selection could only reach a suffix after rejecting every earlier ID. + for (const { baseId, retry } of retries) { + for (let previous = 0; previous < retry; previous++) { + const receipt = yield* receipts.getByCommandId(cascadeCommandIdAt(baseId, previous)); + if (Option.isNone(receipt) || receipt.value.status !== "rejected") return false; + } + } + return true; + }, +); + /** A lifecycle choice made after the cascade's source command, other than `ownCommandId`. */ export const supersedesCascade = ( latest: CommandReceiptV2 | null, diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index c72a61a6ba2c2..ed873f14f5aa7 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -55,6 +55,7 @@ export const SubagentCascade = Context.Reference< | undefined; readonly path: ReadonlyArray; readonly parentArchiveCommandId: CommandId; + readonly legacyPath?: { readonly anchorId: CommandId; readonly depth: number } | undefined; } | undefined >("t3/orchestration-v2/SubagentCascade", { defaultValue: () => undefined }); diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index c53d5b3f362f4..05f69631210b9 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -133,6 +133,7 @@ import { archivedByCascade, CommandReceiptStoreV2, type CommandReceiptV2, + matchesLegacyCascadePath, supersedesCascade, } from "./CommandReceiptStore.ts"; import { @@ -407,6 +408,7 @@ export interface OrchestratorV2DispatchResult { readonly storedEvents: ReadonlyArray; readonly subagentCascades?: ReadonlyArray; readonly cascadeIdOccupied?: true; + readonly cascadeReplayedCommandId?: CommandId; /** The command id was already accepted; analytics skip it. */ readonly replayed?: true; } @@ -16210,6 +16212,43 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } satisfies OrchestratorV2DispatchResult; }); + const ownsLegacyCascadeReceipt = Effect.fn("Orchestrator.ownsLegacyCascadeReceipt")(function* ( + command: OrchestrationV2ServerCommand, + threadId: ThreadId, + receipt: CommandReceiptV2, + parentArchiveCommandId: CommandId, + ) { + if (receipt.archiveSourceCommandId !== undefined) + return receipt.archiveSourceCommandId === parentArchiveCommandId; + + const read = mapDispatchError(command); + const directEffect = yield* effectOutbox + .get(`effect:${receipt.commandId}:subagent-threads.cascade`) + .pipe(read); + if (Option.isSome(directEffect)) return false; + + // A direct command would have kept its own effect if an eligible child already existed. + const cutoff = DateTime.subtractDuration( + yield* DateTime.now, + EffectOutbox.SETTLED_EFFECT_RETENTION, + ); + if ( + !DateTime.isLessThan(receipt.acceptedAt, cutoff) && + (yield* eventSink.latestSequence({ threadId }).pipe(read)) <= receipt.resultSequence + ) { + const children = yield* projectionStore.getSubagentChildThreads(threadId).pipe(read); + for (const child of children) { + const sequence = yield* eventSink.latestSequence({ threadId: child.id }).pipe(read); + if (sequence > 0 && sequence <= receipt.resultSequence) return true; + } + } + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "Cannot verify the archive or restore that owns the legacy cascade path.", + }); + }); + /** `sideChatIds` are the live side chats a delete, archive, or unarchive cascades to, read under their locks. */ const dispatchWithReceiptEffect = Effect.fn("orchestrationV2.dispatch.withReceipt")(function* ( command: OrchestrationV2ServerCommand, @@ -16239,6 +16278,49 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio return { sequence: cascade?.source?.cutoff ?? 0, storedEvents: [] }; yield* refuseAboveDispatchModeLimit(command, cascadeThread.id, cascadeThread); } + const legacyLatest = + cascadeThread !== undefined && + cascade?.source !== undefined && + cascade.legacyPath !== undefined + ? yield* commandReceipts + .getLatestLifecycle(cascadeThread.id, ["thread.archive", "thread.unarchive"]) + .pipe(read) + : undefined; + if ( + cascadeThread !== undefined && + cascade?.source !== undefined && + cascade.legacyPath !== undefined && + legacyLatest?.status === "accepted" && + legacyLatest.commandType === command.type && + legacyLatest.commandId !== command.commandId && + legacyLatest.resultSequence > cascade.source.cutoff && + cascadeThread.deletedAt === null && + (command.type === "thread.archive" + ? cascadeThread.archivedAt !== null + : cascadeThread.archivedAt === null) && + (yield* matchesLegacyCascadePath( + commandReceipts, + cascade.legacyPath.anchorId, + cascade.path.slice(cascade.legacyPath.depth), + legacyLatest.commandId, + ).pipe(read)) + ) { + if ( + !(yield* ownsLegacyCascadeReceipt( + command, + cascadeThread.id, + legacyLatest, + cascade.parentArchiveCommandId, + )) + ) + return { sequence: legacyLatest.resultSequence, storedEvents: [] }; + const replay = yield* replayCommandReceipt( + { ...command, commandId: legacyLatest.commandId }, + legacyLatest, + ); + return { ...replay, cascadeReplayedCommandId: legacyLatest.commandId }; + } + const occupiedCascadeId = (receipt: AnyCommandReceiptV2) => cascadeThread !== undefined && ("projectId" in receipt || @@ -16246,9 +16328,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio receipt.status !== "accepted" || receipt.commandType !== command.type || (command.type === "thread.delete" && cascadeThread.deletedAt === null) || - (command.type === "thread.unarchive" && - cascade?.source !== undefined && - receipt.resultSequence < cascade.source.cutoff) || + (cascade?.source !== undefined && receipt.resultSequence < cascade.source.cutoff) || (command.type === "thread.archive" && receipt.archiveSourceCommandId !== undefined && receipt.archiveSourceCommandId !== cascade?.parentArchiveCommandId)); @@ -16302,9 +16382,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio command.type !== "thread.delete" ) { const { source, path } = cascade; - const latest = yield* commandReceipts - .getLatestLifecycle(cascadeThread.id, ["thread.archive", "thread.unarchive"]) - .pipe(read); + const latest = + legacyLatest === undefined + ? yield* commandReceipts + .getLatestLifecycle(cascadeThread.id, ["thread.archive", "thread.unarchive"]) + .pipe(read) + : legacyLatest; const applies = cascadeThread.deletedAt === null && (command.type === "thread.archive" diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 2c9322c4d8424..e0f3a267fee65 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -1858,11 +1858,13 @@ const make = Effect.gen(function* () { cutoff: sideChat?.cutoff ?? receipt.resultSequence, archiveCommandId: sideChat?.archiveCommandId, }; + type LegacyPath = { readonly anchorId: CommandId; readonly depth: number }; const dispatchChild = Effect.fn("ThreadManagementService.dispatchCascadeChild")(function* ( threadId: ThreadId, baseId: CommandId, path: ReadonlyArray, parentArchiveCommandId: CommandId, + legacyPath: LegacyPath | undefined, ) { for (let retry = 0; retry < CommandReceiptStore.cascadeCommandIdLimit; retry++) { const commandId = CommandReceiptStore.cascadeCommandIdAt(baseId, retry); @@ -1874,6 +1876,7 @@ const make = Effect.gen(function* () { source, path, parentArchiveCommandId, + legacyPath, }), Effect.catchTags({ OrchestratorThreadAboveModeLimitError: () => Effect.undefined, @@ -1881,7 +1884,9 @@ const make = Effect.gen(function* () { ); if (result === undefined) return; if (!result.cascadeIdOccupied) - return result.replayed || result.storedEvents.length > 0 ? commandId : null; + return result.replayed || result.storedEvents.length > 0 + ? (result.cascadeReplayedCommandId ?? commandId) + : null; } return yield* new Orchestrator.OrchestratorDispatchError({ commandId: baseId, @@ -1896,17 +1901,22 @@ const make = Effect.gen(function* () { readonly deletedAt: unknown; readonly projectId?: unknown; }, - baseId: CommandId, + parentCommandId: CommandId, path: ReadonlyArray, parentArchiveCommandId: CommandId, + legacyPath?: LegacyPath, ) => ReturnType = Effect.fn( "ThreadManagementService.visitSubagentCascade", - )(function* (thread, baseId, path, parentArchiveCommandId) { + )(function* (thread, parentCommandId, path, parentArchiveCommandId, legacyPath) { + const baseId = + path.length === 0 + ? parentCommandId + : CommandId.make(`${parentCommandId}:subagent:${thread.id}`); const ownCommandId = path.length === 0 ? baseId : thread.deletedAt === null - ? yield* dispatchChild(thread.id, baseId, path, parentArchiveCommandId) + ? yield* dispatchChild(thread.id, baseId, path, parentArchiveCommandId, legacyPath) : null; if (ownCommandId === undefined) return; // Holds each thread's lock while visiting its subagents, so locks nest parent before child. @@ -1956,17 +1966,20 @@ const make = Effect.gen(function* () { const children = yield* orchestrator.getSubagentChildThreads(thread.id); if (children.length === 0) return; // Older jobs may have committed descendants under a suffixed skip. - const prefix = - ownCommandId ?? - (input.stableSkipPrefix - ? baseId - : ((yield* CommandReceiptStore.legacyCascadeCommandId(receipts, baseId)) ?? - baseId)); + const legacyPrefix = + ownCommandId === null && !input.stableSkipPrefix + ? yield* CommandReceiptStore.legacyCascadeCommandId(receipts, baseId) + : baseId; + const prefix = ownCommandId ?? legacyPrefix ?? baseId; + const skippedPath = + ownCommandId === null + ? (legacyPath ?? + (source && legacyPrefix !== baseId + ? { anchorId: parentCommandId, depth: path.length - 1 } + : undefined)) + : undefined; const [, failures] = yield* Effect.partition(children, (child) => - Effect.gen(function* () { - const base = CommandId.make(`${prefix}:subagent:${child.id}`); - yield* visit(child, base, [...path, child.id], archiveCommandId); - }), + visit(child, prefix, [...path, child.id], archiveCommandId, skippedPath), ); // Finish healthy siblings before reporting a failure to the durable worker. if (failures[0] !== undefined) return yield* failures[0]; diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 7c9ee2732347a..99e6f1a9a4b73 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -15060,89 +15060,137 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); - it.effect("restores a child when an earlier restore holds the derived ID", () => - Effect.gen(function* () { - const f = yield* fixture; - const id = "unarchive:parent:subagent:native"; - yield* f.run("archive", native); - yield* f.run("unarchive", native, id); - const original = yield* receipt(id); - yield* f.run("archive"); - yield* f.run("unarchive"); - assert.equal(yield* f.archived(parent, native), "00"); - assert.equal(Option.getOrThrow(yield* receipt(`${id}:retry:1`)).threadId, native); - assert.isTrue((yield* f.run("unarchive", native, id)).replayed); - assert.deepEqual(yield* receipt(id), original); - }).pipe(Effect.provide(layerTest)), - ); - - it.effect.each(["late collision", "legacy rejection"] as const)( - "resumes an archive past a skipped intermediate with a %s", - (scenario) => + it.effect.each(["archive", "unarchive"] as const)( + "applies a child %s when an earlier receipt holds the derived ID", + (operation) => Effect.gen(function* () { const f = yield* fixture; - yield* f.run("archive", app); - yield* f.run("unarchive", grandchild); - const leaf = ThreadId.make("leaf"); - yield* f.set(leaf, { - lineage: { - rootThreadId: parent, - parentThreadId: grandchild, - relationshipToParent: "subagent", - }, - }); - const skippedId = CommandId.make("archive:parent:subagent:app"); - const receipts = yield* Receipts.CommandReceiptStoreV2; - const created = Option.getOrThrow(yield* receipt("create:parent")); - const collision = { - ...created, - commandId: skippedId, - threadId: scenario === "legacy rejection" ? app : ThreadId.make("fork"), - status: scenario === "legacy rejection" ? "rejected" : "accepted", - error: scenario === "legacy rejection" ? "Earlier rejection." : null, - } as const; - yield* f.commit("archive"); - if (scenario === "legacy rejection") { - const sql = yield* SqlClient.SqlClient; - yield* sql` + const id = `${operation}:parent:subagent:native`; + yield* f.run("archive", native, operation === "archive" ? id : undefined); + yield* f.run("unarchive", native, operation === "unarchive" ? id : undefined); + if (operation === "archive") { + const receipts = yield* Receipts.CommandReceiptStoreV2; + const { archiveSourceCommandId: _source, ...legacy } = Option.getOrThrow( + yield* receipt(id), + ); + yield* receipts.upsert(legacy); + } + const original = yield* receipt(id); + yield* f.run("archive"); + if (operation === "unarchive") yield* f.run("unarchive"); + assert.equal(yield* f.archived(parent, native), operation === "archive" ? "11" : "00"); + assert.equal(Option.getOrThrow(yield* receipt(`${id}:retry:1`)).threadId, native); + assert.isTrue((yield* f.run(operation, native, id)).replayed); + assert.deepEqual(yield* receipt(id), original); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect.each([ + "late collision", + "legacy rejection", + "legacy late acceptance", + "legacy late rejection", + "legacy expired rejection", + "legacy direct archive", + ] as const)("resumes an archive past a skipped intermediate with a %s", (scenario) => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.run("archive", app); + yield* f.run("unarchive", grandchild); + const leaf = ThreadId.make("leaf"); + yield* f.set(leaf, { + lineage: { + rootThreadId: parent, + parentThreadId: grandchild, + relationshipToParent: "subagent", + }, + }); + yield* f.threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make("rename:leaf"), + threadId: leaf, + title: "Leaf", + }); + const skippedId = CommandId.make("archive:parent:subagent:app"); + const receipts = yield* Receipts.CommandReceiptStoreV2; + const created = Option.getOrThrow(yield* receipt("create:parent")); + const rejected = scenario.endsWith("rejection") || scenario === "legacy direct archive"; + const collision = { + ...created, + commandId: skippedId, + threadId: rejected ? app : ThreadId.make("fork"), + status: rejected ? "rejected" : "accepted", + error: rejected ? "Earlier rejection." : null, + } as const; + yield* f.commit("archive"); + if (scenario !== "late collision") { + const sql = yield* SqlClient.SqlClient; + yield* sql` UPDATE orchestration_v2_effect_outbox SET payload_json = json_remove(payload_json, '$.stableSkipPrefix') WHERE effect_id = 'effect:archive:parent:subagent-threads.cascade' `; - yield* receipts.upsert(collision); - } - const orchestrator = yield* Orchestrator.OrchestratorV2; - const dispatch = orchestrator.dispatch; - let failLeaf = true; - using _failLeaf = vi.spyOn(orchestrator, "dispatch").mockImplementation((command) => { - if (command.type === "thread.archive" && command.threadId === leaf && failLeaf) { - failLeaf = false; - return Effect.fail( - new Orchestrator.OrchestratorDispatchError({ - commandId: command.commandId, - commandType: command.type, - cause: "Leaf delivery interrupted.", - }), - ); - } - return dispatch(command); + } + if (scenario === "legacy rejection") yield* receipts.upsert(collision); + const prefix = scenario === "legacy rejection" ? `${skippedId}:retry:1` : skippedId; + const childId = `${prefix}:subagent:${grandchild}`; + if (scenario === "legacy direct archive") { + yield* f.commit("archive", grandchild, childId); + const { archiveSourceCommandId: _source, ...legacy } = Option.getOrThrow( + yield* receipt(childId), + ); + yield* receipts.upsert(legacy); + yield* receipts.upsert(collision); + yield* f.threads.cascadeSubagentLifecycle({ + operation: "thread.archive", + threadId: parent, + commandId: CommandId.make("archive:parent"), }); - using _lostRetry = vi - .spyOn(f.outbox, "retry") - .mockImplementationOnce(() => Effect.succeed(true)); - yield* f.drain; assert.equal(yield* f.archived(app, grandchild, leaf), "110"); - const prefix = scenario === "legacy rejection" ? `${skippedId}:retry:1` : skippedId; - const childId = `${prefix}:subagent:${grandchild}`; - const childReceipt = Option.getOrThrow(yield* receipt(childId)); - if (scenario === "late collision") yield* receipts.upsert(collision); - assert.equal((yield* f.outbox.reconcileAfterProcessLoss).requeued, 1); - yield* f.drain; - assert.equal((yield* f.cascade).status, "succeeded"); + assert.deepEqual(Option.getOrThrow(yield* receipt(childId)), legacy); + return; + } + const orchestrator = yield* Orchestrator.OrchestratorV2; + const dispatch = orchestrator.dispatch; + let failLeaf = true; + using _failLeaf = vi.spyOn(orchestrator, "dispatch").mockImplementation((command) => { + if (command.type === "thread.archive" && command.threadId === leaf && failLeaf) { + failLeaf = false; + return Effect.fail( + new Orchestrator.OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "Leaf delivery interrupted.", + }), + ); + } + return dispatch(command); + }); + using _lostRetry = vi + .spyOn(f.outbox, "retry") + .mockImplementationOnce(() => Effect.succeed(true)); + yield* f.drain; + assert.equal(yield* f.archived(app, grandchild, leaf), "110"); + const committed = Option.getOrThrow(yield* receipt(childId)); + const { archiveSourceCommandId: _source, ...legacy } = committed; + const childReceipt = scenario === "late collision" ? committed : legacy; + if (childReceipt === legacy) yield* receipts.upsert(legacy); + if (scenario !== "legacy rejection") yield* receipts.upsert(collision); + if (scenario === "legacy expired rejection") yield* TestClock.adjust("8 days"); + assert.equal((yield* f.outbox.reconcileAfterProcessLoss).requeued, 1); + yield* f.drain; + const recovered = yield* f.cascade; + if (scenario === "legacy expired rejection") { + assert.equal(recovered.status, "pending"); + assert.include(recovered.lastError, "Cannot verify the archive or restore"); + assert.equal(yield* f.archived(app, grandchild, leaf), "110"); + } else { + assert.equal(recovered.status, "succeeded"); assert.equal(yield* f.archived(app, grandchild, leaf), "111"); - assert.deepEqual(Option.getOrThrow(yield* receipt(childId)), childReceipt); - assert.deepEqual(Option.getOrThrow(yield* receipt(skippedId)), collision); - }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), + } + assert.deepEqual(Option.getOrThrow(yield* receipt(childId)), childReceipt); + assert.deepEqual(Option.getOrThrow(yield* receipt(skippedId)), collision); + }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), ); it.effect.each([31, 32])("bounds child retries at 32 IDs with %i rejected", (count) => From 15db1bbd29f075427bbd5a088f1a0d833dfb6613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 18:22:48 +0000 Subject: [PATCH 19/23] fix(server): verify legacy cascade ownership before recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use bounded retained thread history to verify membership before planning. Save proven archive ownership for later restore, and apply the dispatch receipt rules to the final retry segment while retaining legacy checks for skipped ancestors. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../orchestration-v2/CommandReceiptStore.ts | 36 +++- .../src/orchestration-v2/EffectOutbox.ts | 8 +- apps/server/src/orchestration-v2/EventSink.ts | 23 +++ .../server/src/orchestration-v2/EventStore.ts | 4 + .../src/orchestration-v2/Orchestrator.ts | 156 +++++++++++++++--- .../ThreadManagementService.ts | 25 ++- .../src/orchestration-v2/runtimeLayer.test.ts | 136 ++++++++++++++- .../OrchestrationCommandReceipts.ts | 27 +++ 8 files changed, 367 insertions(+), 48 deletions(-) diff --git a/apps/server/src/orchestration-v2/CommandReceiptStore.ts b/apps/server/src/orchestration-v2/CommandReceiptStore.ts index 9d06f727109be..9072d51ffb037 100644 --- a/apps/server/src/orchestration-v2/CommandReceiptStore.ts +++ b/apps/server/src/orchestration-v2/CommandReceiptStore.ts @@ -86,6 +86,11 @@ export interface CommandReceiptStoreV2Shape { readonly upsert: ( receipt: AnyCommandReceiptV2, ) => Effect.Effect; + readonly setArchiveSourceIfMissing: ( + input: Parameters< + OrchestrationCommandReceipts.OrchestrationCommandReceiptRepository["Service"]["setArchiveSourceIfMissing"] + >[0], + ) => Effect.Effect; readonly getAnyByCommandId: ( commandId: CommandId, ) => Effect.Effect, CommandReceiptStoreV2Error>; @@ -205,6 +210,16 @@ const layerBase: Layer.Layer< }), ), ), + setArchiveSourceIfMissing: Effect.fn("CommandReceiptStore.setArchiveSourceIfMissing")( + (input) => + receipts + .setArchiveSourceIfMissing(input) + .pipe( + Effect.mapError( + (cause) => new CommandReceiptStoreWriteError({ commandId: input.commandId, cause }), + ), + ), + ), getAnyByCommandId: Effect.fn("CommandReceiptStore.getAnyByCommandId")((commandId) => receipts.getByCommandId({ commandId }).pipe( Effect.flatMap((receipt) => { @@ -278,10 +293,11 @@ export const matchesLegacyCascadePath = Effect.fn("CommandReceiptStore.matchesLe anchorId: CommandId, path: ReadonlyArray, completedId: CommandId, + occupiesDispatchedId: (receipt: AnyCommandReceiptV2) => boolean, ) { - const retries: Array<{ baseId: CommandId; retry: number }> = []; + const retries: Array<{ baseId: CommandId; retry: number; dispatched: boolean }> = []; let id: string = completedId; - for (const threadId of path.toReversed()) { + for (const [offset, threadId] of path.toReversed().entries()) { const segment = `:subagent:${threadId}`; if (!id.endsWith(segment)) { const suffix = /:retry:([1-9][0-9]*)$/.exec(id); @@ -289,18 +305,24 @@ export const matchesLegacyCascadePath = Effect.fn("CommandReceiptStore.matchesLe const retry = Number(suffix[1]); if (retry >= cascadeCommandIdLimit) return false; id = id.slice(0, suffix.index); - retries.push({ baseId: CommandId.make(id), retry }); + retries.push({ baseId: CommandId.make(id), retry, dispatched: offset === 0 }); } if (!id.endsWith(segment)) return false; id = id.slice(0, -segment.length); } if (id !== anchorId) return false; - // Legacy selection could only reach a suffix after rejecting every earlier ID. - for (const { baseId, retry } of retries) { + for (const { baseId, retry, dispatched } of retries) { for (let previous = 0; previous < retry; previous++) { - const receipt = yield* receipts.getByCommandId(cascadeCommandIdAt(baseId, previous)); - if (Option.isNone(receipt) || receipt.value.status !== "rejected") return false; + const commandId = cascadeCommandIdAt(baseId, previous); + const receipt = yield* dispatched + ? receipts.getAnyByCommandId(commandId) + : receipts.getByCommandId(commandId); + if ( + Option.isNone(receipt) || + (dispatched ? !occupiesDispatchedId(receipt.value) : receipt.value.status !== "rejected") + ) + return false; } } return true; diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index ed873f14f5aa7..99f7d9121edb0 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -55,7 +55,13 @@ export const SubagentCascade = Context.Reference< | undefined; readonly path: ReadonlyArray; readonly parentArchiveCommandId: CommandId; - readonly legacyPath?: { readonly anchorId: CommandId; readonly depth: number } | undefined; + readonly legacyPath?: + | { + readonly anchorId: CommandId; + readonly depth: number; + readonly verified?: true; + } + | undefined; } | undefined >("t3/orchestration-v2/SubagentCascade", { defaultValue: () => undefined }); diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 7faf661ad2f5a..4bdd622ff821a 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -261,6 +261,11 @@ export interface EventSinkV2Shape { readonly latestSequence: (input?: { readonly threadId?: ThreadId; }) => Effect.Effect; + readonly readRetainedThreadHistory: (input: { + readonly threadId: ThreadId; + readonly throughSequence: number; + readonly limit: number; + }) => Effect.Effect | null, EventSinkV2Error>; readonly readByCommandId: (input: { readonly commandId: CommandId; }) => Stream.Stream; @@ -1334,6 +1339,24 @@ const layerBase: Layer.Layer< }), ), ), + readRetainedThreadHistory: Effect.fn("EventSink.readRetainedThreadHistory")( + function* ({ threadId, throughSequence, limit }) { + const prunedThrough = yield* eventStore.prunedThroughSequence; + const events = yield* eventStore + .read({ + threadId, + afterSequence: prunedThrough, + throughSequence, + limit: limit + 1, + }) + .pipe(Stream.runCollect); + if (events.length > limit || (yield* eventStore.prunedThroughSequence) !== prunedThrough) + return null; + return Array.from(events); + }, + (effect, { threadId }) => + effect.pipe(Effect.mapError((cause) => new EventSinkStreamError({ threadId, cause }))), + ), readByCommandId: (input) => eventStore.readByCommandId(input).pipe( Stream.mapError( diff --git a/apps/server/src/orchestration-v2/EventStore.ts b/apps/server/src/orchestration-v2/EventStore.ts index e2125508dee8f..eca49146c283e 100644 --- a/apps/server/src/orchestration-v2/EventStore.ts +++ b/apps/server/src/orchestration-v2/EventStore.ts @@ -69,6 +69,7 @@ export interface EventStoreV2Shape { readonly threadId?: ThreadId; }) => Effect.Effect; readonly latestApplicationSequence: Effect.Effect; + readonly prunedThroughSequence: Effect.Effect; readonly publishCommitted: (events: ReadonlyArray) => Effect.Effect; } @@ -144,6 +145,9 @@ const layerBase: Layer.Layer new EventStoreReadEventsError({ cause })), ), + prunedThroughSequence: applicationEvents.prunedThroughSequence.pipe( + Effect.mapError((cause) => new EventStoreReadEventsError({ cause })), + ), publishCommitted: applicationEvents.publishCommitted, }); }), diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 05f69631210b9..4b274c6b2220b 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -16212,9 +16212,104 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } satisfies OrchestratorV2DispatchResult; }); + type ThreadMembership = Pick; + const sameMembership = (left: ThreadMembership, right: ThreadMembership) => + left.projectId === right.projectId && + left.lineage.parentThreadId === right.lineage.parentThreadId && + left.lineage.relationshipToParent === right.lineage.relationshipToParent; + + const legacyMembershipHistory = Effect.fn("Orchestrator.legacyMembershipHistory")(function* ( + threadId: ThreadId, + throughSequence: number, + ) { + const events = yield* eventSink.readRetainedThreadHistory({ + threadId, + throughSequence, + limit: 256, + }); + return events?.flatMap(({ commandId, event, sequence }) => { + const { payload, type } = event; + return "projectId" in payload && "lineage" in payload + ? [{ commandId, membership: payload, sequence, type }] + : []; + }); + }); + + const provesLegacyChildEligibility = Effect.fn("Orchestrator.provesLegacyChildEligibility")( + function* (thread: OrchestrationV2AppThread, receipt: CommandReceiptV2) { + const history = yield* legacyMembershipHistory(thread.id, receipt.resultSequence); + const lifecycleType = + receipt.commandType === "thread.archive" ? "thread.archived" : "thread.unarchived"; + const lifecycle = history?.find( + (event) => event.commandId === receipt.commandId && event.type === lifecycleType, + ); + if ( + history === undefined || + lifecycle === undefined || + !sameMembership(lifecycle.membership, thread) + ) + return false; + + // The earlier accepted command held this thread's dispatch lock before the lifecycle plan. + let witness = undefined; + for (const event of history.toReversed()) { + if ( + event.sequence >= lifecycle.sequence || + event.commandId === null || + event.commandId === receipt.commandId + ) + continue; + const commandType = + event.type === "thread.metadata-updated" + ? "thread.metadata.update" + : event.type === "thread.archived" + ? "thread.archive" + : event.type === "thread.unarchived" + ? "thread.unarchive" + : undefined; + if (commandType === undefined) continue; + const prior = Option.getOrUndefined(yield* commandReceipts.getByCommandId(event.commandId)); + if ( + prior?.status === "accepted" && + prior.threadId === thread.id && + prior.commandType === commandType && + prior.resultSequence >= event.sequence && + prior.resultSequence < lifecycle.sequence + ) { + witness = event; + break; + } + } + if ( + witness === undefined || + !history.every( + (event) => event.sequence < witness.sequence || sameMembership(event.membership, thread), + ) + ) + return false; + + const children = yield* projectionStore.getSubagentChildThreads(thread.id); + for (const child of children) { + const childHistory = yield* legacyMembershipHistory(child.id, receipt.resultSequence); + if (childHistory === undefined) continue; + const atWitness = childHistory.findLast((event) => event.sequence <= witness.sequence); + if ( + atWitness !== undefined && + sameMembership(atWitness.membership, child) && + childHistory.every( + (event) => + event.sequence <= witness.sequence || sameMembership(event.membership, child), + ) + ) + return true; + } + return false; + }, + ); + const ownsLegacyCascadeReceipt = Effect.fn("Orchestrator.ownsLegacyCascadeReceipt")(function* ( command: OrchestrationV2ServerCommand, - threadId: ThreadId, + thread: OrchestrationV2AppThread, receipt: CommandReceiptV2, parentArchiveCommandId: CommandId, ) { @@ -16227,21 +16322,16 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .pipe(read); if (Option.isSome(directEffect)) return false; - // A direct command would have kept its own effect if an eligible child already existed. + // An eligible child before planning implies a direct command would still have its own effect. const cutoff = DateTime.subtractDuration( yield* DateTime.now, EffectOutbox.SETTLED_EFFECT_RETENTION, ); if ( !DateTime.isLessThan(receipt.acceptedAt, cutoff) && - (yield* eventSink.latestSequence({ threadId }).pipe(read)) <= receipt.resultSequence - ) { - const children = yield* projectionStore.getSubagentChildThreads(threadId).pipe(read); - for (const child of children) { - const sequence = yield* eventSink.latestSequence({ threadId: child.id }).pipe(read); - if (sequence > 0 && sequence <= receipt.resultSequence) return true; - } - } + (yield* provesLegacyChildEligibility(thread, receipt).pipe(read)) + ) + return true; return yield* new OrchestratorDispatchError({ commandId: command.commandId, commandType: command.type, @@ -16278,6 +16368,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio return { sequence: cascade?.source?.cutoff ?? 0, storedEvents: [] }; yield* refuseAboveDispatchModeLimit(command, cascadeThread.id, cascadeThread); } + const occupiedCascadeId = (receipt: AnyCommandReceiptV2) => + cascadeThread !== undefined && + ("projectId" in receipt || + receipt.threadId !== cascadeThread.id || + receipt.status !== "accepted" || + receipt.commandType !== command.type || + (command.type === "thread.delete" && cascadeThread.deletedAt === null) || + (cascade?.source !== undefined && receipt.resultSequence < cascade.source.cutoff) || + (command.type === "thread.archive" && + receipt.archiveSourceCommandId !== undefined && + receipt.archiveSourceCommandId !== cascade?.parentArchiveCommandId)); + const legacyLatest = cascadeThread !== undefined && cascade?.source !== undefined && @@ -16292,7 +16394,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cascade.legacyPath !== undefined && legacyLatest?.status === "accepted" && legacyLatest.commandType === command.type && - legacyLatest.commandId !== command.commandId && + (legacyLatest.commandId !== command.commandId || cascade.legacyPath.verified === true) && legacyLatest.resultSequence > cascade.source.cutoff && cascadeThread.deletedAt === null && (command.type === "thread.archive" @@ -16303,17 +16405,35 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cascade.legacyPath.anchorId, cascade.path.slice(cascade.legacyPath.depth), legacyLatest.commandId, + occupiedCascadeId, ).pipe(read)) ) { if ( !(yield* ownsLegacyCascadeReceipt( command, - cascadeThread.id, + cascadeThread, legacyLatest, cascade.parentArchiveCommandId, )) ) return { sequence: legacyLatest.resultSequence, storedEvents: [] }; + if (command.type === "thread.archive" && legacyLatest.archiveSourceCommandId === undefined) { + const recorded = yield* commandReceipts + .setArchiveSourceIfMissing({ + commandId: legacyLatest.commandId, + threadId: cascadeThread.id, + resultSequence: legacyLatest.resultSequence, + sourceCommandId: cascade.parentArchiveCommandId, + }) + .pipe(read); + if (!recorded) { + const current = Option.getOrUndefined( + yield* commandReceipts.getByCommandId(legacyLatest.commandId).pipe(read), + ); + if (current?.archiveSourceCommandId !== cascade.parentArchiveCommandId) + return { sequence: legacyLatest.resultSequence, storedEvents: [] }; + } + } const replay = yield* replayCommandReceipt( { ...command, commandId: legacyLatest.commandId }, legacyLatest, @@ -16321,18 +16441,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio return { ...replay, cascadeReplayedCommandId: legacyLatest.commandId }; } - const occupiedCascadeId = (receipt: AnyCommandReceiptV2) => - cascadeThread !== undefined && - ("projectId" in receipt || - receipt.threadId !== cascadeThread.id || - receipt.status !== "accepted" || - receipt.commandType !== command.type || - (command.type === "thread.delete" && cascadeThread.deletedAt === null) || - (cascade?.source !== undefined && receipt.resultSequence < cascade.source.cutoff) || - (command.type === "thread.archive" && - receipt.archiveSourceCommandId !== undefined && - receipt.archiveSourceCommandId !== cascade?.parentArchiveCommandId)); - const existingReceipt = yield* read( cascadeThread === undefined ? commandReceipts.getByCommandId(command.commandId) diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index e0f3a267fee65..19eb0a6010408 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -1858,7 +1858,11 @@ const make = Effect.gen(function* () { cutoff: sideChat?.cutoff ?? receipt.resultSequence, archiveCommandId: sideChat?.archiveCommandId, }; - type LegacyPath = { readonly anchorId: CommandId; readonly depth: number }; + type LegacyPath = { + readonly anchorId: CommandId; + readonly depth: number; + readonly verified?: true; + }; const dispatchChild = Effect.fn("ThreadManagementService.dispatchCascadeChild")(function* ( threadId: ThreadId, baseId: CommandId, @@ -1885,7 +1889,10 @@ const make = Effect.gen(function* () { if (result === undefined) return; if (!result.cascadeIdOccupied) return result.replayed || result.storedEvents.length > 0 - ? (result.cascadeReplayedCommandId ?? commandId) + ? { + commandId: result.cascadeReplayedCommandId ?? commandId, + verified: result.cascadeReplayedCommandId !== undefined, + } : null; } return yield* new Orchestrator.OrchestratorDispatchError({ @@ -1912,13 +1919,14 @@ const make = Effect.gen(function* () { path.length === 0 ? parentCommandId : CommandId.make(`${parentCommandId}:subagent:${thread.id}`); - const ownCommandId = + const dispatched = path.length === 0 - ? baseId + ? { commandId: baseId, verified: false } : thread.deletedAt === null ? yield* dispatchChild(thread.id, baseId, path, parentArchiveCommandId, legacyPath) : null; - if (ownCommandId === undefined) return; + if (dispatched === undefined) return; + const ownCommandId = dispatched?.commandId ?? null; // Holds each thread's lock while visiting its subagents, so locks nest parent before child. yield* threadDispatch.withLock( thread.id, @@ -1971,15 +1979,16 @@ const make = Effect.gen(function* () { ? yield* CommandReceiptStore.legacyCascadeCommandId(receipts, baseId) : baseId; const prefix = ownCommandId ?? legacyPrefix ?? baseId; - const skippedPath = - ownCommandId === null + const childLegacyPath = dispatched?.verified + ? { anchorId: prefix, depth: path.length, verified: true as const } + : ownCommandId === null ? (legacyPath ?? (source && legacyPrefix !== baseId ? { anchorId: parentCommandId, depth: path.length - 1 } : undefined)) : undefined; const [, failures] = yield* Effect.partition(children, (child) => - visit(child, prefix, [...path, child.id], archiveCommandId, skippedPath), + visit(child, prefix, [...path, child.id], archiveCommandId, childLegacyPath), ); // Finish healthy siblings before reporting a failure to the durable worker. if (failures[0] !== undefined) return yield* failures[0]; diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 99e6f1a9a4b73..d9e8f637fd16d 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -15090,8 +15090,12 @@ describe("subagent lifecycle cascade", () => { "legacy rejection", "legacy late acceptance", "legacy late rejection", + "legacy renamed rejection", + "legacy deeper rejection", + "legacy dispatched rejection", "legacy expired rejection", "legacy direct archive", + "legacy planning move", ] as const)("resumes an archive past a skipped intermediate with a %s", (scenario) => Effect.gen(function* () { const f = yield* fixture; @@ -15099,6 +15103,7 @@ describe("subagent lifecycle cascade", () => { yield* f.run("unarchive", grandchild); const leaf = ThreadId.make("leaf"); yield* f.set(leaf, { + ...(scenario === "legacy planning move" ? { projectId: ProjectId.make("foreign") } : {}), lineage: { rootThreadId: parent, parentThreadId: grandchild, @@ -15111,10 +15116,41 @@ describe("subagent lifecycle cascade", () => { threadId: leaf, title: "Leaf", }); + const deepest = ThreadId.make("deepest"); + if (scenario === "legacy deeper rejection") { + yield* f.set(deepest, { + lineage: { + rootThreadId: parent, + parentThreadId: leaf, + relationshipToParent: "subagent", + }, + }); + yield* f.threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make("rename:deepest"), + threadId: deepest, + title: "Deepest", + }); + yield* f.threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make("witness:leaf"), + threadId: leaf, + title: "Leaf before archiving", + }); + } + yield* f.threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make("witness:grandchild"), + threadId: grandchild, + title: "Grandchild before archiving", + }); const skippedId = CommandId.make("archive:parent:subagent:app"); const receipts = yield* Receipts.CommandReceiptStoreV2; const created = Option.getOrThrow(yield* receipt("create:parent")); - const rejected = scenario.endsWith("rejection") || scenario === "legacy direct archive"; + const rejected = + scenario.endsWith("rejection") || + scenario === "legacy direct archive" || + scenario === "legacy planning move"; const collision = { ...created, commandId: skippedId, @@ -15133,19 +15169,57 @@ describe("subagent lifecycle cascade", () => { } if (scenario === "legacy rejection") yield* receipts.upsert(collision); const prefix = scenario === "legacy rejection" ? `${skippedId}:retry:1` : skippedId; - const childId = `${prefix}:subagent:${grandchild}`; - if (scenario === "legacy direct archive") { + const childBase = `${prefix}:subagent:${grandchild}`; + const childId = + scenario === "legacy dispatched rejection" ? `${childBase}:retry:1` : childBase; + const acceptedOccupant = { ...created, commandId: CommandId.make(childBase) }; + if (scenario === "legacy dispatched rejection") yield* receipts.upsert(acceptedOccupant); + if (scenario === "legacy direct archive" || scenario === "legacy planning move") { + const store = yield* ProjectionStore.ProjectionStoreV2; + const sink = yield* EventSink.EventSinkV2; + const { projectId } = yield* store.getThread(parent); + const children = store.getSubagentChildThreads; + using _move = vi.spyOn(store, "getSubagentChildThreads").mockImplementation((threadId) => + children(threadId).pipe( + Effect.tap((found) => + scenario === "legacy planning move" && threadId === grandchild && found.length === 0 + ? store.getThread(leaf).pipe( + Effect.flatMap((thread) => + sink.write({ + events: [ + { + id: EventId.make("move:leaf"), + type: "thread.metadata-updated", + threadId: leaf, + occurredAt: f.now, + payload: { ...thread, projectId }, + }, + ], + }), + ), + Effect.orDie, + ) + : Effect.void, + ), + ), + ); yield* f.commit("archive", grandchild, childId); const { archiveSourceCommandId: _source, ...legacy } = Option.getOrThrow( yield* receipt(childId), ); yield* receipts.upsert(legacy); yield* receipts.upsert(collision); - yield* f.threads.cascadeSubagentLifecycle({ + const recovery = f.threads.cascadeSubagentLifecycle({ operation: "thread.archive", threadId: parent, commandId: CommandId.make("archive:parent"), }); + if (scenario === "legacy planning move") { + const error = yield* recovery.pipe(Effect.as(null), Effect.catch(Effect.succeed)); + assert.equal(error?._tag, "OrchestratorDispatchError"); + if (error?._tag === "OrchestratorDispatchError") + assert.include(String(error.cause), "Cannot verify the archive or restore"); + } else yield* recovery; assert.equal(yield* f.archived(app, grandchild, leaf), "110"); assert.deepEqual(Option.getOrThrow(yield* receipt(childId)), legacy); return; @@ -15154,7 +15228,8 @@ describe("subagent lifecycle cascade", () => { const dispatch = orchestrator.dispatch; let failLeaf = true; using _failLeaf = vi.spyOn(orchestrator, "dispatch").mockImplementation((command) => { - if (command.type === "thread.archive" && command.threadId === leaf && failLeaf) { + const interrupted = scenario === "legacy deeper rejection" ? deepest : leaf; + if (command.type === "thread.archive" && command.threadId === interrupted && failLeaf) { failLeaf = false; return Effect.fail( new Orchestrator.OrchestratorDispatchError({ @@ -15170,11 +15245,32 @@ describe("subagent lifecycle cascade", () => { .spyOn(f.outbox, "retry") .mockImplementationOnce(() => Effect.succeed(true)); yield* f.drain; - assert.equal(yield* f.archived(app, grandchild, leaf), "110"); + assert.equal( + yield* f.archived(app, grandchild, leaf), + scenario === "legacy deeper rejection" ? "111" : "110", + ); const committed = Option.getOrThrow(yield* receipt(childId)); const { archiveSourceCommandId: _source, ...legacy } = committed; - const childReceipt = scenario === "late collision" ? committed : legacy; + const childReceipt = + scenario === "late collision" || scenario === "legacy dispatched rejection" + ? committed + : legacy; if (childReceipt === legacy) yield* receipts.upsert(legacy); + const leafId = `${childId}:subagent:${leaf}`; + if (scenario === "legacy deeper rejection") { + const { archiveSourceCommandId: _source, ...legacyLeaf } = Option.getOrThrow( + yield* receipt(leafId), + ); + yield* receipts.upsert(legacyLeaf); + } + if (scenario === "legacy renamed rejection") { + yield* f.threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make("rename:grandchild:after-archive"), + threadId: grandchild, + title: "Grandchild after archiving", + }); + } if (scenario !== "legacy rejection") yield* receipts.upsert(collision); if (scenario === "legacy expired rejection") yield* TestClock.adjust("8 days"); assert.equal((yield* f.outbox.reconcileAfterProcessLoss).requeued, 1); @@ -15188,8 +15284,32 @@ describe("subagent lifecycle cascade", () => { assert.equal(recovered.status, "succeeded"); assert.equal(yield* f.archived(app, grandchild, leaf), "111"); } - assert.deepEqual(Option.getOrThrow(yield* receipt(childId)), childReceipt); + const { archiveSourceCommandId: recoveredSource, ...recoveredReceipt } = Option.getOrThrow( + yield* receipt(childId), + ); + assert.deepEqual(recoveredReceipt, legacy); assert.deepEqual(Option.getOrThrow(yield* receipt(skippedId)), collision); + if (scenario === "legacy dispatched rejection") { + assert.equal(recoveredSource, committed.archiveSourceCommandId); + assert.deepEqual(Option.getOrThrow(yield* receipt(childBase)), acceptedOccupant); + } + if (scenario === "legacy late rejection" || scenario === "legacy deeper rejection") + assert.equal(recoveredSource, "archive:parent"); + if (scenario === "legacy deeper rejection") { + assert.equal(yield* f.archived(deepest), "1"); + yield* TestClock.adjust("8 days"); + assert.isAbove(yield* f.outbox.pruneSettled, 0); + yield* f.run("unarchive"); + yield* f.drain; + assert.equal(yield* f.archived(app, grandchild, leaf, deepest), "1000"); + } + if (scenario === "legacy late rejection") { + yield* TestClock.adjust("8 days"); + assert.isAbove(yield* f.outbox.pruneSettled, 0); + yield* f.run("unarchive"); + yield* f.drain; + assert.equal(yield* f.archived(app, grandchild, leaf), "100"); + } }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), ); diff --git a/apps/server/src/persistence/OrchestrationCommandReceipts.ts b/apps/server/src/persistence/OrchestrationCommandReceipts.ts index 85624fbdb71cf..965b56365159d 100644 --- a/apps/server/src/persistence/OrchestrationCommandReceipts.ts +++ b/apps/server/src/persistence/OrchestrationCommandReceipts.ts @@ -53,6 +53,13 @@ export class OrchestrationCommandReceiptRepository extends Context.Service< receipt: OrchestrationCommandReceipt, ) => Effect.Effect; + readonly setArchiveSourceIfMissing: (input: { + readonly commandId: CommandId; + readonly threadId: ThreadId; + readonly resultSequence: number; + readonly sourceCommandId: CommandId; + }) => Effect.Effect; + readonly getLatestLifecycle: ( input: typeof GetLatestLifecycleInput.Type, ) => Effect.Effect< @@ -198,6 +205,26 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { ); return { + setArchiveSourceIfMissing: Effect.fn( + "OrchestrationCommandReceiptRepository.setArchiveSourceIfMissing", + )(({ commandId, threadId, resultSequence, sourceCommandId }) => + sql<{ readonly command_id: string }>` + UPDATE orchestration_command_receipts + SET archive_source_command_id = ${sourceCommandId} + WHERE command_id = ${commandId} + AND aggregate_kind = 'thread' AND aggregate_id = ${threadId} + AND status = 'accepted' AND command_type = 'thread.archive' + AND result_sequence = ${resultSequence} AND archive_source_command_id IS NULL + RETURNING command_id + `.pipe( + Effect.map((rows) => rows.length === 1), + Effect.mapError( + toPersistenceSqlError( + "OrchestrationCommandReceiptRepository.setArchiveSourceIfMissing:query", + ), + ), + ), + ), getLatestLifecycle: (input) => findLatestLifecycle(input).pipe( Effect.mapError( From eabb7a06488c06ec1969827467706e566fb99bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 18:35:19 +0000 Subject: [PATCH 20/23] fix(server): replay committed leaf restores on recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An exact restore receipt can replay through the existing cutoff and occupancy checks. Preserve archive ownership proof for older archives and alternate command IDs. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../src/orchestration-v2/Orchestrator.ts | 3 +- .../src/orchestration-v2/runtimeLayer.test.ts | 76 +++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 4b274c6b2220b..1f67c87a42d67 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -16394,7 +16394,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio cascade.legacyPath !== undefined && legacyLatest?.status === "accepted" && legacyLatest.commandType === command.type && - (legacyLatest.commandId !== command.commandId || cascade.legacyPath.verified === true) && + (legacyLatest.commandId !== command.commandId || + (command.type === "thread.archive" && cascade.legacyPath.verified === true)) && legacyLatest.resultSequence > cascade.source.cutoff && cascadeThread.deletedAt === null && (command.type === "thread.archive" diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index d9e8f637fd16d..67f419f39096e 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -15313,6 +15313,82 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), ); + it.effect("replays an exact committed leaf during a recovered legacy restore", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.run("archive", app); + yield* f.run("unarchive", grandchild); + const finished = ThreadId.make("restore-a"); + const unfinished = ThreadId.make("restore-b"); + for (const leaf of [finished, unfinished]) { + yield* f.set(leaf, { + lineage: { + rootThreadId: parent, + parentThreadId: grandchild, + relationshipToParent: "subagent", + }, + }); + yield* f.threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make(`rename:${leaf}`), + threadId: leaf, + title: leaf, + }); + } + yield* f.run("archive"); + yield* f.drain; + yield* f.commit("unarchive"); + const effectId = "effect:unarchive:parent:subagent-threads.cascade"; + const sql = yield* SqlClient.SqlClient; + yield* sql` + UPDATE orchestration_v2_effect_outbox + SET payload_json = json_remove(payload_json, '$.stableSkipPrefix') + WHERE effect_id = ${effectId} + `; + + const orchestrator = yield* Orchestrator.OrchestratorV2; + const dispatch = orchestrator.dispatch; + let interrupt = true; + using _interrupt = vi.spyOn(orchestrator, "dispatch").mockImplementation((command) => { + if (command.type === "thread.unarchive" && command.threadId === unfinished && interrupt) { + interrupt = false; + return Effect.fail( + new Orchestrator.OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "Restore delivery interrupted.", + }), + ); + } + return dispatch(command); + }); + using _lostRetry = vi + .spyOn(f.outbox, "retry") + .mockImplementationOnce(() => Effect.succeed(true)); + yield* f.drain; + assert.equal(yield* f.archived(app, grandchild, finished, unfinished), "1001"); + + const skippedId = CommandId.make("unarchive:parent:subagent:app"); + const completedId = `${skippedId}:subagent:${grandchild}:subagent:${finished}`; + const completed = Option.getOrThrow(yield* receipt(completedId)); + const created = Option.getOrThrow(yield* receipt("create:parent")); + const receipts = yield* Receipts.CommandReceiptStoreV2; + yield* receipts.upsert({ + ...created, + commandId: skippedId, + threadId: app, + commandType: "thread.unarchive", + status: "rejected", + error: "Rejected after the partial delivery.", + }); + assert.equal((yield* f.outbox.reconcileAfterProcessLoss).requeued, 1); + yield* f.drain; + assert.equal(Option.getOrThrow(yield* f.outbox.get(effectId)).status, "succeeded"); + assert.equal(yield* f.archived(app, grandchild, finished, unfinished), "1000"); + assert.deepEqual(Option.getOrThrow(yield* receipt(completedId)), completed); + }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), + ); + it.effect.each([31, 32])("bounds child retries at 32 IDs with %i rejected", (count) => Effect.gen(function* () { const f = yield* fixture; From c7d5c7528ef6ebb51e5ea39aa4b2ec5effbe77d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 20:43:18 +0000 Subject: [PATCH 21/23] test(server): consolidate subagent cascade regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Share setup while preserving the distinct recovery and permission paths. > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../src/orchestration-v2/runtimeLayer.test.ts | 1041 +++++++---------- 1 file changed, 392 insertions(+), 649 deletions(-) diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index 67f419f39096e..b4cb857400f97 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14317,7 +14317,23 @@ describe("subagent lifecycle cascade", () => { .mockImplementation((id) => id === parent ? Effect.zipWith(children(id), stale, (c, t) => [...c, t]) : children(id), ); - const all = [native, app, grandchild, "fork", "fork-child", "foreign", "foreign-child"]; + const side = ThreadId.make("side"); + const sideChild = ThreadId.make("side-child"); + yield* f.sideChat(native, side); + yield* f.set(sideChild, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + const all = [ + native, + app, + grandchild, + "fork", + "fork-child", + "foreign", + "foreign-child", + side, + sideChild, + ]; for (const operation of ["archive", "unarchive", "delete"] as const) { // A deleted intermediate must not hide surviving descendants from deletion. if (operation === "delete") yield* f.set(app, { deletedAt: f.now }); @@ -14326,7 +14342,7 @@ describe("subagent lifecycle cascade", () => { const field = operation === "delete" ? "deletedAt" : "archivedAt"; assert.equal( yield* f.state(field, ...all), - operation === "unarchive" ? "0000000" : "1110000", + operation === "unarchive" ? "000000000" : "111000011", ); const effects = yield* f.outbox.listByCommandId( CommandId.make(`${operation}:parent:subagent:native`), @@ -14336,18 +14352,6 @@ describe("subagent lifecycle cascade", () => { } }).pipe(Effect.provide(layerTest)), ); - it.effect("leaves a replayed parent command's descendants to its committed cascade", () => - Effect.gen(function* () { - const f = yield* fixture; - const limit = { runtimeMode: "approval-required", interactionMode: "plan" } as const; - yield* f.set(parent, limit); - yield* f.run("archive").pipe(Effect.provideService(DispatchModeLimit, limit)); - assert.equal(yield* f.archived(parent, native, app), "100"); - // A retry without the caller's limit must not walk the subtree under its own. - yield* f.run("archive"); - assert.equal(yield* f.archived(native, app), "00"); - }).pipe(Effect.provide(layerTest)), - ); it.effect("holds the deletion lock before every cascade thread lock", () => Effect.gen(function* () { const f = yield* fixture; @@ -14400,115 +14404,73 @@ describe("subagent lifecycle cascade", () => { ); }).pipe(Effect.provide(layerTest)), ); - it.effect.each([ - { source: parent, delivery: "reply", operation: "archive", collision: null }, - { source: parent, delivery: "worker", operation: "archive", collision: null }, - { source: app, delivery: "reply", operation: "archive", collision: null }, - { source: app, delivery: "worker", operation: "archive", collision: null }, - { source: parent, delivery: "reply", operation: "archive", collision: "accepted" }, - { source: parent, delivery: "reply", operation: "archive", collision: "rejected" }, - { source: parent, delivery: "reply", operation: "unarchive", collision: "accepted" }, - { source: parent, delivery: "reply", operation: "unarchive", collision: "rejected" }, - ] as const)( - "cascades side-chat descendants from $source through $delivery: $operation, collision $collision", - ({ source, delivery, operation, collision }) => - Effect.gen(function* () { - const f = yield* fixture; - const side = ThreadId.make("side"); - const child = ThreadId.make("side-child"); - const independent = ThreadId.make("side-independent"); - yield* f.sideChat(source, side); - for (const id of [child, independent]) { - yield* f.set(id, { - lineage: { - rootThreadId: parent, - parentThreadId: side, - relationshipToParent: "subagent", - }, - }); - } - yield* f.run("archive", independent); - if (source === parent && delivery === "worker") yield* f.markUsed(side); - const apply = Effect.fn("test.applySideChatLifecycle")(function* ( - operation: "archive" | "unarchive", - ) { - if (delivery === "reply") yield* f.run(operation); - else { - yield* f.commit(operation); - if (source === parent) yield* f.renameAndKeep(side, operation); - yield* f.drain; - } - }); - if (collision !== null) { - if (operation === "unarchive") yield* apply("archive"); - const before = yield* f.archived(parent, side, child, independent); - const commandId = CommandId.make(`${operation}:${parent}`); - const existing = { - commandId: CommandId.make(`${commandId}:side-chat:${side}`), - threadId: ThreadId.make("fork"), - commandType: "thread.archive", - acceptedAt: f.now, - resultSequence: 0, - status: collision, - error: collision === "rejected" ? "Earlier rejected command." : null, - }; - yield* Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.upsert(existing)); - const events = yield* EventSink.EventSinkV2; - const sequence = yield* events.latestSequence(); - const result = yield* apply(operation).pipe(Effect.flip); - assert.equal(result._tag, "OrchestratorDispatchError"); - yield* f.drain; - assert.equal(yield* f.archived(parent, side, child, independent), before); - assert.deepEqual(Option.getOrThrow(yield* receipt(existing.commandId)), existing); - assert.isTrue(Option.isNone(yield* receipt(commandId))); - assert.equal(yield* events.latestSequence(), sequence); - assert.isTrue( - Option.isNone(yield* f.outbox.get(`effect:${commandId}:subagent-threads.cascade`)), - ); - assert.isTrue( - Option.isNone( - yield* f.outbox.get(`effect:${existing.commandId}:subagent-threads.cascade`), - ), - ); - return; - } - yield* apply("archive"); - assert.equal(yield* f.archived(side, child, independent), "111"); - yield* f.drain; - yield* TestClock.adjust("8 days"); - yield* f.outbox.pruneSettled; - if (source === parent && delivery === "reply") { - yield* f.markUsed(parent); - yield* f.renameAndKeep(parent, "archive"); - } - yield* apply("unarchive"); - assert.equal(yield* f.archived(side, child, independent), "001"); - }).pipe(Effect.provide(layerTest)), - ); - it.effect.each(["descendant", "side chat"] as const)( - "preserves a newer %s restore through a delayed nested side-chat archive", - (restored) => - Effect.gen(function* () { - const f = yield* fixture; - const side = ThreadId.make("side"); - const child = ThreadId.make("side-child"); - yield* f.sideChat(app, side); - yield* f.set(child, { + it.effect("recovers an embedded side-chat cascade and restores only its own archives", () => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + const independent = ThreadId.make("side-independent"); + const restored = ThreadId.make("side-restored"); + yield* f.sideChat(app, side); + for (const id of [child, independent, restored]) { + yield* f.set(id, { lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, }); - if (restored === "side chat") { - yield* f.run("archive", side); - yield* f.run("unarchive", child); - } else { - yield* f.run("archive", child, "archive:side-child:independent"); - } - yield* f.commit("archive"); - yield* f.run("unarchive", restored === "side chat" ? side : child); - yield* f.drain; - assert.equal(yield* f.archived(parent, app, side, child), "1110"); - }).pipe(Effect.provide(layerTest)), + } + yield* f.run("archive", independent); + yield* f.run("archive", restored); + yield* f.commit("archive"); + yield* f.run("unarchive", restored); + yield* f.drain; + assert.equal(yield* f.archived(side, child, independent, restored), "1110"); + yield* TestClock.adjust("8 days"); + assert.isAbove(yield* f.outbox.pruneSettled, 0); + const generatedId = "archive:parent:subagent:app:side-chat:side"; + const generated = Option.getOrThrow(yield* receipt(generatedId)); + assert.isTrue( + Option.isNone(yield* f.outbox.get(`effect:${generatedId}:subagent-threads.cascade`)), + ); + const replay = yield* f.commit("archive", side, generatedId); + assert.isTrue(replay.replayed); + assert.equal(replay.sequence, generated.resultSequence); + const refused = yield* f.commit("delete", side, generatedId).pipe(Effect.flip); + assert.equal(refused._tag, "OrchestratorCommandIdConflictError"); + assert.deepEqual(Option.getOrThrow(yield* receipt(generatedId)), generated); + assert.equal(yield* f.state("deletedAt", side, child), "00"); + yield* f.commit("unarchive"); + yield* f.markUsed(app); + yield* f.renameAndKeep(app, "before-delivery"); + yield* f.drain; + assert.equal(yield* f.archived(app, side, child, independent, restored), "00010"); + }).pipe(Effect.provide(layerTest)), ); + it.effect("rolls back a parent restore whose side-chat receipt ID is occupied", () => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.sideChatWithChild(); + yield* f.run("archive"); + const commandId = CommandId.make("unarchive:parent"); + const occupied = { + commandId: CommandId.make("unarchive:parent:side-chat:side"), + threadId: ThreadId.make("fork"), + commandType: "thread.archive", + acceptedAt: f.now, + resultSequence: 0, + status: "rejected", + error: "Earlier rejected command.", + } as const; + yield* Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.upsert(occupied)); + const denied = yield* f.run("unarchive").pipe(Effect.flip); + assert.equal(denied._tag, "OrchestratorDispatchError"); + assert.equal(yield* f.archived(parent, side, child), "111"); + assert.deepEqual(Option.getOrThrow(yield* receipt(occupied.commandId)), occupied); + assert.isTrue(Option.isNone(yield* receipt(commandId))); + assert.isTrue( + Option.isNone(yield* f.outbox.get("effect:unarchive:parent:subagent-threads.cascade")), + ); + }).pipe(Effect.provide(layerTest)), + ); it.effect( "keeps the descendant archived during a delayed restore after a newer side-chat archive", () => @@ -14532,145 +14494,90 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); - it.effect.each([ - { delivery: "reply", useParentDerivedId: false, pending: false }, - { delivery: "worker", useParentDerivedId: false, pending: false }, - { delivery: "worker", useParentDerivedId: true, pending: false }, - { delivery: "reply", useParentDerivedId: false, pending: true }, - ] as const)( - "preserves an independent side-chat rearchive through $delivery, parent-derived ID $useParentDerivedId, pending $pending", - ({ delivery, useParentDerivedId, pending }) => - Effect.gen(function* () { - const f = yield* fixture; - const side = ThreadId.make("side"); - const child = ThreadId.make("side-child"); - yield* f.sideChat(parent, side); - yield* f.set(child, { - lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, - }); - if (useParentDerivedId) yield* f.run("archive", side, "earlier:side-archive"); - yield* f.run("archive"); - if (pending) { - yield* f.commit("unarchive", side); - yield* TestClock.adjust("1 second"); - yield* f.commit("archive", side); - } else { - yield* f.run("unarchive", side); - assert.equal(yield* f.archived(side, child), "00"); - yield* TestClock.adjust("1 second"); - yield* f.run( - "archive", - side, - useParentDerivedId ? "archive:parent:side-chat:side" : undefined, - ); - yield* f.drain; - yield* TestClock.adjust("8 days"); - yield* f.outbox.pruneSettled; - } - const occupied = { - commandId: CommandId.make("unarchive:parent:side-chat:side"), - threadId: ThreadId.make("fork"), - commandType: "thread.archive", - acceptedAt: f.now, - resultSequence: 0, - status: "accepted", - error: null, - } as const; - if (useParentDerivedId) { - yield* Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.upsert(occupied)); - } - if (delivery === "reply") yield* f.run("unarchive"); - else { - yield* f.commit("unarchive"); - yield* f.drain; - } - if (pending) yield* f.drain; - assert.equal(yield* f.archived(parent, side, child), "001"); - if (useParentDerivedId) - assert.deepEqual(Option.getOrThrow(yield* receipt(occupied.commandId)), occupied); - }).pipe(Effect.provide(layerTest)), + it.effect("keeps an independent side-chat rearchive with a parent-derived ID", () => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.sideChatWithChild(); + yield* f.run("archive", side, "earlier:side-archive"); + yield* f.run("archive"); + yield* f.run("unarchive", side); + yield* TestClock.adjust("1 second"); + yield* f.run("archive", side, "archive:parent:side-chat:side"); + yield* f.drain; + yield* TestClock.adjust("8 days"); + yield* f.outbox.pruneSettled; + const occupied = { + commandId: CommandId.make("unarchive:parent:side-chat:side"), + threadId: ThreadId.make("fork"), + commandType: "thread.archive", + acceptedAt: f.now, + resultSequence: 0, + status: "accepted", + error: null, + } as const; + yield* Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.upsert(occupied)); + yield* f.commit("unarchive"); + yield* f.drain; + assert.equal(yield* f.archived(parent, side, child), "001"); + assert.deepEqual(Option.getOrThrow(yield* receipt(occupied.commandId)), occupied); + }).pipe(Effect.provide(layerTest)), ); - it.effect.each([ - "reply", - "worker", - "legacy worker", - "legacy worker after unused close", - "legacy worker after a later deletion", - ] as const)( - "leaves a moved side chat's subagent alone during parent deletion through %s", - (delivery) => - Effect.gen(function* () { - const f = yield* fixture; - const { side, child } = yield* f.sideChatWithChild(ProjectId.make("foreign")); - const commandId = CommandId.make("delete:parent:side-chat:side"); - const laterDeletion = delivery === "legacy worker after a later deletion"; - if (delivery === "legacy worker after unused close" || laterDeletion) { - yield* f.markUsed(side); - const kept = yield* f.threads.dispatch({ - type: "thread.delete", - commandId, - threadId: side, - onlyIfUnused: true, - }); - assert.isEmpty(kept.storedEvents); - } - if (delivery === "reply") yield* f.run("delete"); - else { - yield* f.commit("delete"); - if (delivery === "worker") yield* f.drain; - else { - if (laterDeletion) { - yield* f.commit("delete", side, "later:delete:side"); - const sql = yield* SqlClient.SqlClient; - const pruned = yield* sql` - DELETE FROM orchestration_events - WHERE command_id = 'delete:parent' - AND stream_id = ${side} - AND event_type = 'thread.deleted' - RETURNING sequence - `; - assert.lengthOf(pruned, 1); - } - yield* f.threads.cascadeSubagentLifecycle({ - threadId: side, - commandId, - operation: "thread.delete", - }); - } - } - assert.equal(yield* f.state("deletedAt", parent, side, child), "110"); - }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), - ); + it.effect("leaves a moved side chat's descendants alone during delete and legacy recovery", () => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.sideChatWithChild(ProjectId.make("foreign")); + const archiveId = CommandId.make("archive:parent:side-chat:side"); + const fork = ThreadId.make("fork"); + const receipts = yield* Receipts.CommandReceiptStoreV2; + const created = Option.getOrThrow(yield* receipt("create:parent")); + const occupied = { ...created, commandId: archiveId, threadId: fork }; + yield* receipts.upsert(occupied); + yield* f.run("archive"); + yield* f.drain; + assert.equal(yield* f.archived(parent, side, child), "110"); + assert.deepEqual(Option.getOrThrow(yield* receipt(archiveId)), occupied); - it.effect.each(["occupied receipt", "pending direct restore"] as const)( - "archives the original parent without planning a foreign side-chat cascade after %s", - (scenario) => - Effect.gen(function* () { - const f = yield* fixture; - const { side, child } = yield* f.sideChatWithChild(ProjectId.make("foreign")); - const occupied = { - commandId: CommandId.make(`archive:parent:side-chat:${side}`), - threadId: ThreadId.make("fork"), - commandType: "thread.archive", - acceptedAt: f.now, - resultSequence: 0, - status: "accepted", - error: null, - } as const; - if (scenario === "occupied receipt") { - yield* Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.upsert(occupied)); - } else { - yield* f.run("archive", side); - yield* f.drain; - yield* f.commit("unarchive", side); - } - yield* f.run("archive"); - yield* f.drain; - assert.equal(yield* f.archived(parent, side, child), "110"); - if (scenario === "occupied receipt") - assert.deepEqual(Option.getOrThrow(yield* receipt(occupied.commandId)), occupied); - }).pipe(Effect.provide(layerTest)), + const commandId = CommandId.make("delete:parent:side-chat:side"); + yield* f.markUsed(side); + const kept = yield* f.threads.dispatch({ + type: "thread.delete", + commandId, + threadId: side, + onlyIfUnused: true, + }); + assert.isEmpty(kept.storedEvents); + yield* f.commit("delete"); + yield* f.drain; + assert.equal(yield* f.state("deletedAt", parent, side, child), "110"); + yield* f.markUsed(fork); + const closeId = CommandId.make("close:fork"); + yield* f.threads.dispatch({ + type: "thread.delete", + commandId: closeId, + threadId: fork, + onlyIfUnused: true, + }); + const directId = CommandId.make(`${closeId}:side-chat:${side}`); + yield* f.commit("delete", side, directId); + const sql = yield* SqlClient.SqlClient; + yield* sql` + DELETE FROM orchestration_events + WHERE command_id = 'delete:parent' AND stream_id = ${side} AND event_type = 'thread.deleted' + `; + yield* f.threads.cascadeSubagentLifecycle({ + threadId: side, + commandId, + operation: "thread.delete", + }); + assert.equal(yield* f.state("deletedAt", parent, side, child), "110"); + yield* f.threads.cascadeSubagentLifecycle({ + threadId: side, + commandId: directId, + operation: "thread.delete", + }); + assert.equal(yield* f.state("deletedAt", fork, side, child), "011"); + }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), ); it.effect("stops a committed archive cascade at a side chat moved before delivery", () => @@ -14686,74 +14593,62 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); - it.effect.each(["retained legacy event", "pruned fresh event"] as const)( - "recovers a direct side-chat deletion with a parent-derived ID and %s", - (history) => - Effect.gen(function* () { - const f = yield* fixture; - const { side, child } = yield* f.sideChatWithChild(ProjectId.make("foreign")); - yield* f.markUsed(parent); - const kept = yield* f.threads.dispatch({ - type: "thread.delete", - commandId: CommandId.make("delete:parent"), - threadId: parent, - onlyIfUnused: true, - }); - assert.isEmpty(kept.storedEvents); - assert.equal(yield* f.state("deletedAt", parent, app, native, grandchild), "0000"); + it.effect("recovers a direct side-chat deletion after its delete event is pruned", () => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.sideChatWithChild(ProjectId.make("foreign")); + yield* f.markUsed(parent); + const kept = yield* f.threads.dispatch({ + type: "thread.delete", + commandId: CommandId.make("delete:parent"), + threadId: parent, + onlyIfUnused: true, + }); + assert.isEmpty(kept.storedEvents); + assert.equal(yield* f.state("deletedAt", parent, app, native, grandchild), "0000"); - if (history === "pruned fresh event") { - const store = yield* ProjectionStore.ProjectionStoreV2; - yield* f.markUsed(side); - const runId = RunId.make("side:queued"); - yield* store.apply({ - id: EventId.make("side:queued"), - type: "run.created", - threadId: side, - runId, - occurredAt: f.now, - payload: { - id: runId, - threadId: side, - ordinal: 1, - providerInstanceId: modelSelection.instanceId, - modelSelection, - providerThreadId: null, - userMessageId: MessageId.make(`use:${side}`), - rootNodeId: null, - activeAttemptId: null, - status: "queued", - requestedAt: f.now, - startedAt: null, - completedAt: null, - checkpointId: null, - contextHandoffId: null, - }, - }); - } - const commandId = CommandId.make("delete:parent:side-chat:side"); - const deleted = yield* f.commit("delete", side, commandId); - if (history === "pruned fresh event") { - assert.equal(deleted.storedEvents.at(-1)?.event.type, "run.updated"); - const sql = yield* SqlClient.SqlClient; - const pruned = yield* sql` - DELETE FROM orchestration_events - WHERE command_id = ${commandId} - AND stream_id = ${side} - AND event_type = 'thread.deleted' - RETURNING sequence - `; - assert.lengthOf(pruned, 1); - yield* f.drain; - } else { - yield* f.threads.cascadeSubagentLifecycle({ - threadId: side, - commandId, - operation: "thread.delete", - }); - } - assert.equal(yield* f.state("deletedAt", parent, side, child), "011"); - }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), + const store = yield* ProjectionStore.ProjectionStoreV2; + yield* f.markUsed(side); + const runId = RunId.make("side:queued"); + yield* store.apply({ + id: EventId.make("side:queued"), + type: "run.created", + threadId: side, + runId, + occurredAt: f.now, + payload: { + id: runId, + threadId: side, + ordinal: 1, + providerInstanceId: modelSelection.instanceId, + modelSelection, + providerThreadId: null, + userMessageId: MessageId.make(`use:${side}`), + rootNodeId: null, + activeAttemptId: null, + status: "queued", + requestedAt: f.now, + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }, + }); + const commandId = CommandId.make("delete:parent:side-chat:side"); + const deleted = yield* f.commit("delete", side, commandId); + assert.equal(deleted.storedEvents.at(-1)?.event.type, "run.updated"); + const sql = yield* SqlClient.SqlClient; + const pruned = yield* sql` + DELETE FROM orchestration_events + WHERE command_id = ${commandId} + AND stream_id = ${side} + AND event_type = 'thread.deleted' + RETURNING sequence + `; + assert.lengthOf(pruned, 1); + yield* f.drain; + assert.equal(yield* f.state("deletedAt", parent, side, child), "011"); + }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), ); it.effect("rolls back a nested side-chat deletion with an occupied recovery effect ID", () => @@ -14777,6 +14672,12 @@ describe("subagent lifecycle cascade", () => { }, ]); const occupied = yield* f.outbox.get(effectId); + const directId = `${nestedId}:side-chat:${side}`; + const fork = ThreadId.make("fork"); + const refused = yield* f.commit("archive", fork, directId).pipe(Effect.flip); + assert.equal(refused._tag, "OrchestratorDispatchError"); + assert.equal(yield* f.archived(fork, ThreadId.make("fork-child")), "00"); + assert.isTrue(Option.isNone(yield* receipt(directId))); yield* f.commit("delete"); yield* f.drain; assert.equal(yield* f.state("deletedAt", parent, app, side, child), "1000"); @@ -14785,94 +14686,6 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); - it.effect("rejects a direct side-chat delete whose recovery ID is occupied", () => - Effect.gen(function* () { - const f = yield* fixture; - const { side, child } = yield* f.sideChatWithChild(); - const collidingCommandId = `delete:parent:side-chat:${side}`; - const effectId = `effect:${collidingCommandId}:subagent-threads.cascade`; - yield* f.commit("delete"); - const occupied = yield* f.outbox.get(effectId); - const { request } = Option.getOrThrow(occupied); - assert.equal(request.type, "subagent-threads.cascade"); - const rejected = yield* f.commit("delete", side, collidingCommandId).pipe(Effect.flip); - assert.equal(rejected._tag, "OrchestratorDispatchError"); - assert.isTrue(Option.isNone(yield* receipt(collidingCommandId))); - assert.deepEqual(yield* f.outbox.get(effectId), occupied); - assert.equal(yield* f.state("deletedAt", parent, side, child), "110"); - yield* f.commit("delete", side, "direct:fresh-delete"); - const fresh = Option.getOrThrow( - yield* f.outbox.get("effect:direct:fresh-delete:subagent-threads.cascade"), - ); - assert.equal(fresh.threadId, side); - assert.include(fresh.request, { - type: "subagent-threads.cascade", - operation: "thread.delete", - directDelete: true, - }); - yield* f.drain; - assert.equal(yield* f.state("deletedAt", parent, side, child), "111"); - }).pipe(Effect.provide(layerTest)), - ); - - it.effect.each(["archive", "unarchive"] as const)( - "rolls back a direct %s with an occupied lifecycle recovery ID", - (operation) => - Effect.gen(function* () { - const f = yield* fixture; - const { side } = yield* f.sideChatWithChild(); - const fork = ThreadId.make("fork"); - const child = ThreadId.make("fork-child"); - if (operation === "unarchive") { - yield* f.run("archive", fork); - yield* f.drain; - } - const before = yield* f.archived(fork, child); - const collidingId = `delete:parent:side-chat:${side}`; - const effectId = `effect:${collidingId}:subagent-threads.cascade`; - yield* f.commit("delete"); - const occupied = Option.getOrThrow(yield* f.outbox.get(effectId)); - const refused = yield* f.commit(operation, fork, collidingId).pipe(Effect.flip); - assert.equal(refused._tag, "OrchestratorDispatchError"); - assert.equal(yield* f.archived(fork, child), before); - assert.isTrue(Option.isNone(yield* receipt(collidingId))); - assert.deepEqual(Option.getOrThrow(yield* f.outbox.get(effectId)), occupied); - const fresh = yield* f.commit(operation, fork, "direct:fresh-lifecycle"); - const replay = yield* f.commit(operation, fork, "direct:fresh-lifecycle"); - assert.equal(replay.sequence, fresh.sequence); - assert.isTrue(replay.replayed); - yield* f.drain; - assert.equal(yield* f.archived(fork, child), operation === "archive" ? "11" : "00"); - }).pipe(Effect.provide(layerTest)), - ); - - it.effect("rejects a direct delete that reuses a generated archive receipt", () => - Effect.gen(function* () { - const f = yield* fixture; - const side = ThreadId.make("side"); - const child = ThreadId.make("side-child"); - yield* f.sideChat(parent, side); - yield* f.set(child, { - lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, - }); - yield* f.run("archive"); - yield* f.drain; - yield* TestClock.adjust("8 days"); - yield* f.outbox.pruneSettled; - const generatedId = `archive:parent:side-chat:${side}`; - const generated = Option.getOrThrow(yield* receipt(generatedId)); - const replay = yield* f.commit("archive", side, generatedId); - assert.equal(replay.sequence, generated.resultSequence); - assert.isTrue(replay.replayed); - const refused = yield* f.commit("delete", side, generatedId).pipe(Effect.flip); - assert.equal(refused._tag, "OrchestratorCommandIdConflictError"); - assert.deepEqual(Option.getOrThrow(yield* receipt(generatedId)), generated); - assert.equal(yield* f.state("deletedAt", side, child), "00"); - yield* f.run("delete", side, "direct:fresh-delete"); - assert.equal(yield* f.state("deletedAt", side, child), "11"); - }).pipe(Effect.provide(layerTest)), - ); - it.effect("restores only what a parent archive cascaded, through independent archives", () => Effect.gen(function* () { const f = yield* fixture; @@ -14881,6 +14694,11 @@ describe("subagent lifecycle cascade", () => { yield* f.drain; yield* f.run("archive"); yield* f.drain; + const receipts = yield* Receipts.CommandReceiptStoreV2; + const { archiveSourceCommandId: _source, ...legacy } = Option.getOrThrow( + yield* receipt("archive:parent:subagent:native"), + ); + yield* receipts.upsert(legacy); yield* f.run("unarchive"); yield* f.drain; assert.equal(yield* f.archived(app, grandchild, native), "110"); @@ -14931,170 +14749,85 @@ describe("subagent lifecycle cascade", () => { assert.isTrue(Option.isNone(yield* receipt("archive-deleted:subagent:native"))); }).pipe(Effect.provide(layerTest)), ); - it.effect.each(["existing thread", "late project"] as const)( - "skips a %s receipt at the child lifecycle command ID", - (collision) => - Effect.gen(function* () { - const f = yield* fixture; - const commandId = CommandId.make("archive:parent:subagent:app"); - const receipts = yield* Receipts.CommandReceiptStoreV2; - const existing = Option.getOrThrow(yield* receipt("create:parent")); - const projectId = ProjectId.make("other-project"); - const sink = yield* EventSink.EventSinkV2; - const commit = sink.commitCommand; - if (collision === "existing thread") { - yield* f.run("archive", ThreadId.make("fork"), commandId); - } - using _collision = vi.spyOn(sink, "commitCommand").mockImplementation((input) => - collision === "late project" && input.commandId === commandId - ? receipts - .insertIfAbsent({ - commandId, - projectId, - commandType: "project.create", - acceptedAt: existing.acceptedAt, - resultSequence: existing.resultSequence, - status: "accepted", - error: null, - }) - .pipe(Effect.orDie, Effect.andThen(commit(input))) - : commit(input), - ); - yield* f.run("archive"); - assert.equal(yield* f.archived(app, grandchild, native), "111"); - assert.equal(Option.getOrThrow(yield* receipt(`${commandId}:retry:1`)).threadId, app); - if (collision === "late project") { - assert.equal( - Option.getOrThrow(yield* receipts.getProjectByCommandId(commandId)).projectId, - projectId, - ); - } else { - assert.isTrue((yield* f.run("archive", ThreadId.make("fork"), commandId)).replayed); - assert.equal(Option.getOrThrow(yield* receipt(commandId)).threadId, "fork"); - } - }).pipe(Effect.provide(layerTest)), - ); - - it.effect.each(["existing", "late"] as const)( - "deletes a used subagent after a conditional close with a %s receipt", - (timing) => - Effect.gen(function* () { - const f = yield* fixture; - yield* f.markUsed(native); - const conditional = { - type: "thread.delete", - commandId: CommandId.make("delete:parent:subagent:native"), - threadId: native, - onlyIfUnused: true, - } as const; - const receipts = yield* Receipts.CommandReceiptStoreV2; - const sink = yield* EventSink.EventSinkV2; - const commit = sink.commitCommand; - const create = Option.getOrThrow(yield* receipt("create:parent")); - if (timing === "existing") - assert.isEmpty((yield* f.threads.dispatch(conditional)).storedEvents); - const original = Option.getOrElse(yield* receipt(conditional.commandId), () => ({ - ...create, - commandId: conditional.commandId, - commandType: conditional.type, - threadId: native, - })); - using _conditional = vi - .spyOn(sink, "commitCommand") - .mockImplementation((input) => - timing === "late" && input.commandId === conditional.commandId - ? receipts.insertIfAbsent(original).pipe(Effect.orDie, Effect.andThen(commit(input))) - : commit(input), - ); - yield* f.run("delete"); - assert.equal(yield* f.state("deletedAt", parent, native), "11"); - assert.equal( - Option.getOrThrow(yield* receipt(`${conditional.commandId}:retry:1`)).threadId, - native, - ); - const replay = yield* f.threads.dispatch(conditional); - assert.isTrue(replay.replayed); - assert.isEmpty(replay.storedEvents); - assert.deepEqual(Option.getOrThrow(yield* receipt(conditional.commandId)), original); - }).pipe(Effect.provide(layerTest)), - ); - - it.effect.each(["fresh", "legacy"] as const)( - "preserves an independent archive while restoring a %s cascade receipt", - (history) => - Effect.gen(function* () { - const f = yield* fixture; - yield* f.commit("archive"); - yield* f.run("archive", app, "archive:parent:subagent:app"); - yield* f.drain; - assert.equal(yield* f.archived(parent, app, grandchild, native), "1111"); - if (history === "legacy") { - const receipts = yield* Receipts.CommandReceiptStoreV2; - const { archiveSourceCommandId: _source, ...legacy } = Option.getOrThrow( - yield* receipt("archive:parent:subagent:native"), - ); - yield* receipts.upsert(legacy); - } - yield* TestClock.adjust("8 days"); - assert.isAbove(yield* f.outbox.pruneSettled, 0); - yield* f.run("unarchive"); - yield* f.drain; - assert.equal(yield* f.archived(parent, app, grandchild, native), "0110"); - }).pipe(Effect.provide(layerTest)), + it.effect("retries a child command when a project receipt claims its ID during commit", () => + Effect.gen(function* () { + const f = yield* fixture; + const commandId = CommandId.make("archive:parent:subagent:app"); + const receipts = yield* Receipts.CommandReceiptStoreV2; + const existing = Option.getOrThrow(yield* receipt("create:parent")); + const projectId = ProjectId.make("other-project"); + const sink = yield* EventSink.EventSinkV2; + const commit = sink.commitCommand; + using _collision = vi.spyOn(sink, "commitCommand").mockImplementation((input) => + input.commandId === commandId + ? receipts + .insertIfAbsent({ + commandId, + projectId, + commandType: "project.create", + acceptedAt: existing.acceptedAt, + resultSequence: existing.resultSequence, + status: "accepted", + error: null, + }) + .pipe(Effect.orDie, Effect.andThen(commit(input))) + : commit(input), + ); + yield* f.run("archive"); + assert.equal(yield* f.archived(app, grandchild, native), "111"); + assert.equal(Option.getOrThrow(yield* receipt(`${commandId}:retry:1`)).threadId, app); + assert.equal( + Option.getOrThrow(yield* receipts.getProjectByCommandId(commandId)).projectId, + projectId, + ); + }).pipe(Effect.provide(layerTest)), ); - it.effect.each(["before restore", "before delivery"] as const)( - "restores a used intermediate after a conditional close %s", - (when) => - Effect.gen(function* () { - const f = yield* fixture; - yield* f.markUsed(app); - yield* f.run("archive"); - yield* f.drain; - if (when === "before delivery") yield* f.commit("unarchive"); - yield* f.renameAndKeep(app, when); - if (when === "before restore") yield* f.commit("unarchive"); - yield* f.drain; - assert.equal(yield* f.state("deletedAt", app), "0"); - assert.equal(yield* f.archived(app, grandchild), "00"); - }).pipe(Effect.provide(layerTest)), + it.effect("deletes a used subagent whose earlier conditional close owns the derived ID", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.markUsed(native); + const conditional = { + type: "thread.delete", + commandId: CommandId.make("delete:parent:subagent:native"), + threadId: native, + onlyIfUnused: true, + } as const; + assert.isEmpty((yield* f.threads.dispatch(conditional)).storedEvents); + const original = Option.getOrThrow(yield* receipt(conditional.commandId)); + yield* f.run("delete"); + assert.equal(yield* f.state("deletedAt", parent, native), "11"); + assert.equal( + Option.getOrThrow(yield* receipt(`${conditional.commandId}:retry:1`)).threadId, + native, + ); + const replay = yield* f.threads.dispatch(conditional); + assert.isTrue(replay.replayed); + assert.isEmpty(replay.storedEvents); + assert.deepEqual(Option.getOrThrow(yield* receipt(conditional.commandId)), original); + }).pipe(Effect.provide(layerTest)), ); - it.effect.each(["archive", "unarchive"] as const)( - "applies a child %s when an earlier receipt holds the derived ID", - (operation) => - Effect.gen(function* () { - const f = yield* fixture; - const id = `${operation}:parent:subagent:native`; - yield* f.run("archive", native, operation === "archive" ? id : undefined); - yield* f.run("unarchive", native, operation === "unarchive" ? id : undefined); - if (operation === "archive") { - const receipts = yield* Receipts.CommandReceiptStoreV2; - const { archiveSourceCommandId: _source, ...legacy } = Option.getOrThrow( - yield* receipt(id), - ); - yield* receipts.upsert(legacy); - } - const original = yield* receipt(id); - yield* f.run("archive"); - if (operation === "unarchive") yield* f.run("unarchive"); - assert.equal(yield* f.archived(parent, native), operation === "archive" ? "11" : "00"); - assert.equal(Option.getOrThrow(yield* receipt(`${id}:retry:1`)).threadId, native); - assert.isTrue((yield* f.run(operation, native, id)).replayed); - assert.deepEqual(yield* receipt(id), original); - }).pipe(Effect.provide(layerTest)), + it.effect("retries a child restore when an earlier restore holds its derived ID", () => + Effect.gen(function* () { + const f = yield* fixture; + const id = "unarchive:parent:subagent:native"; + yield* f.run("archive", native); + yield* f.run("unarchive", native, id); + const original = yield* receipt(id); + yield* f.run("archive"); + yield* f.run("unarchive"); + assert.equal(yield* f.archived(parent, native), "00"); + assert.equal(Option.getOrThrow(yield* receipt(`${id}:retry:1`)).threadId, native); + assert.isTrue((yield* f.run("unarchive", native, id)).replayed); + assert.deepEqual(yield* receipt(id), original); + }).pipe(Effect.provide(layerTest)), ); it.effect.each([ "late collision", - "legacy rejection", - "legacy late acceptance", - "legacy late rejection", - "legacy renamed rejection", "legacy deeper rejection", - "legacy dispatched rejection", "legacy expired rejection", - "legacy direct archive", "legacy planning move", ] as const)("resumes an archive past a skipped intermediate with a %s", (scenario) => Effect.gen(function* () { @@ -15147,10 +14880,7 @@ describe("subagent lifecycle cascade", () => { const skippedId = CommandId.make("archive:parent:subagent:app"); const receipts = yield* Receipts.CommandReceiptStoreV2; const created = Option.getOrThrow(yield* receipt("create:parent")); - const rejected = - scenario.endsWith("rejection") || - scenario === "legacy direct archive" || - scenario === "legacy planning move"; + const rejected = scenario !== "late collision"; const collision = { ...created, commandId: skippedId, @@ -15167,14 +14897,12 @@ describe("subagent lifecycle cascade", () => { WHERE effect_id = 'effect:archive:parent:subagent-threads.cascade' `; } - if (scenario === "legacy rejection") yield* receipts.upsert(collision); - const prefix = scenario === "legacy rejection" ? `${skippedId}:retry:1` : skippedId; - const childBase = `${prefix}:subagent:${grandchild}`; - const childId = - scenario === "legacy dispatched rejection" ? `${childBase}:retry:1` : childBase; - const acceptedOccupant = { ...created, commandId: CommandId.make(childBase) }; - if (scenario === "legacy dispatched rejection") yield* receipts.upsert(acceptedOccupant); - if (scenario === "legacy direct archive" || scenario === "legacy planning move") { + const childBase = `${skippedId}:subagent:${grandchild}`; + const childId = scenario === "legacy deeper rejection" ? `${childBase}:retry:1` : childBase; + if (scenario === "legacy deeper rejection") { + yield* receipts.upsert({ ...created, commandId: CommandId.make(childBase) }); + } + if (scenario === "legacy planning move") { const store = yield* ProjectionStore.ProjectionStoreV2; const sink = yield* EventSink.EventSinkV2; const { projectId } = yield* store.getThread(parent); @@ -15182,7 +14910,7 @@ describe("subagent lifecycle cascade", () => { using _move = vi.spyOn(store, "getSubagentChildThreads").mockImplementation((threadId) => children(threadId).pipe( Effect.tap((found) => - scenario === "legacy planning move" && threadId === grandchild && found.length === 0 + threadId === grandchild && found.length === 0 ? store.getThread(leaf).pipe( Effect.flatMap((thread) => sink.write({ @@ -15214,12 +14942,24 @@ describe("subagent lifecycle cascade", () => { threadId: parent, commandId: CommandId.make("archive:parent"), }); - if (scenario === "legacy planning move") { - const error = yield* recovery.pipe(Effect.as(null), Effect.catch(Effect.succeed)); - assert.equal(error?._tag, "OrchestratorDispatchError"); - if (error?._tag === "OrchestratorDispatchError") - assert.include(String(error.cause), "Cannot verify the archive or restore"); - } else yield* recovery; + const error = yield* recovery.pipe(Effect.as(null), Effect.catch(Effect.succeed)); + assert.equal(error?._tag, "OrchestratorDispatchError"); + if (error?._tag === "OrchestratorDispatchError") + assert.include(String(error.cause), "Cannot verify the archive or restore"); + assert.equal(yield* f.archived(app, grandchild, leaf), "110"); + assert.deepEqual(Option.getOrThrow(yield* receipt(childId)), legacy); + + const directId = CommandId.make(childId); + yield* f.outbox.enqueue([ + { + id: `effect:${directId}:subagent-threads.cascade`, + commandId: directId, + threadId: grandchild, + availableAt: f.now, + request: { type: "subagent-threads.cascade", operation: "thread.archive" }, + }, + ]); + yield* recovery; assert.equal(yield* f.archived(app, grandchild, leaf), "110"); assert.deepEqual(Option.getOrThrow(yield* receipt(childId)), legacy); return; @@ -15251,10 +14991,7 @@ describe("subagent lifecycle cascade", () => { ); const committed = Option.getOrThrow(yield* receipt(childId)); const { archiveSourceCommandId: _source, ...legacy } = committed; - const childReceipt = - scenario === "late collision" || scenario === "legacy dispatched rejection" - ? committed - : legacy; + const childReceipt = scenario === "late collision" ? committed : legacy; if (childReceipt === legacy) yield* receipts.upsert(legacy); const leafId = `${childId}:subagent:${leaf}`; if (scenario === "legacy deeper rejection") { @@ -15262,8 +14999,6 @@ describe("subagent lifecycle cascade", () => { yield* receipt(leafId), ); yield* receipts.upsert(legacyLeaf); - } - if (scenario === "legacy renamed rejection") { yield* f.threads.dispatch({ type: "thread.metadata.update", commandId: CommandId.make("rename:grandchild:after-archive"), @@ -15271,7 +15006,7 @@ describe("subagent lifecycle cascade", () => { title: "Grandchild after archiving", }); } - if (scenario !== "legacy rejection") yield* receipts.upsert(collision); + yield* receipts.upsert(collision); if (scenario === "legacy expired rejection") yield* TestClock.adjust("8 days"); assert.equal((yield* f.outbox.reconcileAfterProcessLoss).requeued, 1); yield* f.drain; @@ -15289,13 +15024,8 @@ describe("subagent lifecycle cascade", () => { ); assert.deepEqual(recoveredReceipt, legacy); assert.deepEqual(Option.getOrThrow(yield* receipt(skippedId)), collision); - if (scenario === "legacy dispatched rejection") { - assert.equal(recoveredSource, committed.archiveSourceCommandId); - assert.deepEqual(Option.getOrThrow(yield* receipt(childBase)), acceptedOccupant); - } - if (scenario === "legacy late rejection" || scenario === "legacy deeper rejection") - assert.equal(recoveredSource, "archive:parent"); if (scenario === "legacy deeper rejection") { + assert.equal(recoveredSource, "archive:parent"); assert.equal(yield* f.archived(deepest), "1"); yield* TestClock.adjust("8 days"); assert.isAbove(yield* f.outbox.pruneSettled, 0); @@ -15303,13 +15033,6 @@ describe("subagent lifecycle cascade", () => { yield* f.drain; assert.equal(yield* f.archived(app, grandchild, leaf, deepest), "1000"); } - if (scenario === "legacy late rejection") { - yield* TestClock.adjust("8 days"); - assert.isAbove(yield* f.outbox.pruneSettled, 0); - yield* f.run("unarchive"); - yield* f.drain; - assert.equal(yield* f.archived(app, grandchild, leaf), "100"); - } }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), ); @@ -15389,80 +15112,100 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), ); - it.effect.each([31, 32])("bounds child retries at 32 IDs with %i rejected", (count) => + it.effect("keeps the cascade recoverable when its child retry IDs are exhausted", () => Effect.gen(function* () { const f = yield* fixture; const receipts = yield* Receipts.CommandReceiptStoreV2; const created = Option.getOrThrow(yield* receipt("create:parent")); - for (let retry = 0; retry < count; retry++) { + for (let retry = 0; retry < 32; retry++) { const id = CommandId.make(`archive:parent:subagent:app${retry ? `:retry:${retry}` : ""}`); yield* receipts.upsert({ ...created, commandId: id, threadId: app, status: "rejected" }); } yield* f.run("archive"); - // Lose only the outbox acknowledgement, after the child commands commit. - vi.spyOn(f.outbox, "succeed").mockImplementationOnce(() => Effect.succeed(true)); yield* f.drain; - const accepted = yield* receipt("archive:parent:subagent:app:retry:31"); - if (count === 32) { - const { status, attemptCount, lastError } = yield* f.cascade; - assert.deepEqual([status, attemptCount], ["pending", 1]); - assert.include( - lastError, - "All 32 subagent lifecycle command IDs were occupied or rejected.", - ); - } else { - assert.equal(Option.getOrThrow(accepted).status, "accepted"); - assert.equal((yield* f.cascade).status, "running"); - yield* f.run("unarchive", app); - yield* f.drain; - assert.equal((yield* f.outbox.reconcileAfterProcessLoss).requeued, 1); - yield* f.drain; - assert.equal((yield* f.cascade).status, "succeeded"); - assert.deepEqual(yield* receipt("archive:parent:subagent:app:retry:31"), accepted); - } + const { status, attemptCount, lastError } = yield* f.cascade; + assert.deepEqual([status, attemptCount], ["pending", 1]); + assert.include(lastError, "All 32 subagent lifecycle command IDs were occupied or rejected."); assert.equal(yield* f.archived(native, app, grandchild), "100"); assert.isTrue(Option.isNone(yield* receipt("archive:parent:subagent:app:retry:32"))); }).pipe(Effect.provide(layerTest)), ); - it.effect.each([ - { type: "thread.runtime-mode.set", runtimeMode: "full-access", alreadyArchived: false }, - { type: "thread.interaction-mode.set", interactionMode: "default", alreadyArchived: true }, - ] as const)("keeps the caller's limit after $type", ({ alreadyArchived, ...raise }) => + it.effect("holds the original caller limit when an accepted child receives a mode raise", () => Effect.gen(function* () { const f = yield* fixture; - const commandId = CommandId.make("raise"); + const limited = ThreadId.make("limited"); const limit = { runtimeMode: "approval-required", interactionMode: "plan" } as const; for (const id of [parent, app, native, grandchild]) yield* f.set(id, limit); - if (alreadyArchived) { - // The child's archive was accepted before the process was lost, so recovery replays it. - const receipts = yield* Receipts.CommandReceiptStoreV2; - const created = Option.getOrThrow(yield* receipt("create:parent")); - const id = CommandId.make("archive:parent:subagent:app"); - yield* receipts.upsert({ ...created, commandId: id, threadId: app }); - yield* f.set(app, { archivedAt: f.now }); - } - yield* f.commit("archive").pipe(Effect.provideService(DispatchModeLimit, limit)); + yield* f.set(limited, { + ...limit, + lineage: { rootThreadId: parent, parentThreadId: parent, relationshipToParent: "subagent" }, + }); + const { sequence: cutoff } = yield* f + .commit("archive") + .pipe(Effect.provideService(DispatchModeLimit, limit)); + const orchestrator = yield* Orchestrator.OrchestratorV2; + const archiveId = CommandId.make("archive:parent"); + yield* orchestrator + .dispatch({ + type: "thread.archive", + commandId: CommandId.make("archive:parent:subagent:app"), + threadId: app, + }) + .pipe( + Effect.provideService(EffectOutbox.SubagentCascade, { + source: { threadId: parent, cutoff }, + path: [app], + parentArchiveCommandId: archiveId, + }), + ); + yield* f.run("archive"); + assert.equal(yield* f.archived(native, app, grandchild, limited), "0100"); const claimed = yield* f.outbox.claimNext({ workerId: "lost-caller" }); assert.equal(Option.getOrThrow(claimed).request.type, "subagent-threads.cascade"); assert.equal((yield* f.outbox.reconcileAfterProcessLoss).requeued, 1); - // The raise commits just before the walk takes app's lock: for dispatch, or after a replay. + // Raise the fresh child before dispatch and the archived child after replay. const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor; const { withLock } = executor; - const raised = Effect.orDie(f.threads.dispatch({ ...raise, commandId, threadId: app })); + const raised = Effect.orDie( + f.threads.dispatch({ + type: "thread.interaction-mode.set", + interactionMode: "default", + commandId: CommandId.make("raise:archived"), + threadId: app, + }), + ); + const raiseFresh = Effect.orDie( + f.threads.dispatch({ + type: "thread.runtime-mode.set", + runtimeMode: "full-access", + commandId: CommandId.make("raise:fresh"), + threadId: limited, + }), + ); let requests = 0; + let raisePending = true; const lock: typeof withLock = (id, effect) => - Effect.suspend(() => - id === app && ++requests === (alreadyArchived ? 2 : 1) + Effect.suspend(() => { + if (id === limited && raisePending) { + raisePending = false; + return raiseFresh.pipe(Effect.andThen(withLock(id, effect))); + } + return id === app && ++requests === 2 ? raised.pipe(Effect.andThen(withLock(id, effect))) - : withLock(id, effect), - ); + : withLock(id, effect); + }); using _lock = vi.spyOn(executor, "withLock").mockImplementation(lock); yield* f.drain; - assert.equal(yield* f.archived(native, app, grandchild), alreadyArchived ? "110" : "100"); + assert.equal(yield* f.archived(native, app, grandchild, limited), "1100"); assert.equal((yield* f.cascade).status, "succeeded"); - const orchestrator = yield* Orchestrator.OrchestratorV2; - const { type: _type, ...mode } = raise; - assert.include((yield* orchestrator.getThreadRecords(app, [])).thread, mode); + assert.equal( + (yield* orchestrator.getThreadRecords(limited, [])).thread.runtimeMode, + "full-access", + ); + assert.equal( + (yield* orchestrator.getThreadRecords(app, [])).thread.interactionMode, + "default", + ); }).pipe(Effect.provide(layerTest)), ); }); From 363ff75075b771a2af65539cbed11e4bf4b34075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 20:58:42 +0000 Subject: [PATCH 22/23] test(server): cover cascade recovery boundaries in place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../src/orchestration-v2/runtimeLayer.test.ts | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index b4cb857400f97..eaed00ac330a2 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14418,11 +14418,24 @@ describe("subagent lifecycle cascade", () => { }); } yield* f.run("archive", independent); + const latestSide = ThreadId.make("side-restored-later"); + const latestChild = ThreadId.make("side-restored-later-child"); + yield* f.sideChat(app, latestSide); + yield* f.set(latestChild, { + lineage: { + rootThreadId: parent, + parentThreadId: latestSide, + relationshipToParent: "subagent", + }, + }); + yield* f.run("archive", latestSide); + yield* f.run("unarchive", latestChild); yield* f.run("archive", restored); yield* f.commit("archive"); yield* f.run("unarchive", restored); + yield* f.run("unarchive", latestSide); yield* f.drain; - assert.equal(yield* f.archived(side, child, independent, restored), "1110"); + assert.equal(yield* f.archived(side, child, independent, restored, latestChild), "11100"); yield* TestClock.adjust("8 days"); assert.isAbove(yield* f.outbox.pruneSettled, 0); const generatedId = "archive:parent:subagent:app:side-chat:side"; @@ -14461,13 +14474,18 @@ describe("subagent lifecycle cascade", () => { error: "Earlier rejected command.", } as const; yield* Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.upsert(occupied)); + const events = yield* EventSink.EventSinkV2; + const sequence = yield* events.latestSequence(); const denied = yield* f.run("unarchive").pipe(Effect.flip); + assert.equal(yield* events.latestSequence(), sequence); assert.equal(denied._tag, "OrchestratorDispatchError"); assert.equal(yield* f.archived(parent, side, child), "111"); assert.deepEqual(Option.getOrThrow(yield* receipt(occupied.commandId)), occupied); assert.isTrue(Option.isNone(yield* receipt(commandId))); assert.isTrue( - Option.isNone(yield* f.outbox.get("effect:unarchive:parent:subagent-threads.cascade")), + Option.isNone( + yield* f.outbox.get("effect:unarchive:parent:side-chat:side:subagent-threads.cascade"), + ), ); }).pipe(Effect.provide(layerTest)), ); @@ -14561,10 +14579,12 @@ describe("subagent lifecycle cascade", () => { const directId = CommandId.make(`${closeId}:side-chat:${side}`); yield* f.commit("delete", side, directId); const sql = yield* SqlClient.SqlClient; - yield* sql` + const pruned = yield* sql` DELETE FROM orchestration_events WHERE command_id = 'delete:parent' AND stream_id = ${side} AND event_type = 'thread.deleted' + RETURNING sequence `; + assert.lengthOf(pruned, 1); yield* f.threads.cascadeSubagentLifecycle({ threadId: side, commandId, @@ -15117,9 +15137,16 @@ describe("subagent lifecycle cascade", () => { const f = yield* fixture; const receipts = yield* Receipts.CommandReceiptStoreV2; const created = Option.getOrThrow(yield* receipt("create:parent")); - for (let retry = 0; retry < 32; retry++) { - const id = CommandId.make(`archive:parent:subagent:app${retry ? `:retry:${retry}` : ""}`); - yield* receipts.upsert({ ...created, commandId: id, threadId: app, status: "rejected" }); + for (const [threadId, slots] of [ + [native, 31], + [app, 32], + ] as const) { + for (let retry = 0; retry < slots; retry++) { + const id = CommandId.make( + `archive:parent:subagent:${threadId}${retry ? `:retry:${retry}` : ""}`, + ); + yield* receipts.upsert({ ...created, commandId: id, threadId, status: "rejected" }); + } } yield* f.run("archive"); yield* f.drain; @@ -15127,6 +15154,8 @@ describe("subagent lifecycle cascade", () => { assert.deepEqual([status, attemptCount], ["pending", 1]); assert.include(lastError, "All 32 subagent lifecycle command IDs were occupied or rejected."); assert.equal(yield* f.archived(native, app, grandchild), "100"); + const accepted = Option.getOrThrow(yield* receipt("archive:parent:subagent:native:retry:31")); + assert.include(accepted, { threadId: native, status: "accepted" }); assert.isTrue(Option.isNone(yield* receipt("archive:parent:subagent:app:retry:32"))); }).pipe(Effect.provide(layerTest)), ); From 8765bc5a939b9fe9aaf2d2a2e72abe8f286981f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 21 Nov 2026 21:06:38 +0000 Subject: [PATCH 23/23] test(server): distinguish cascade recovery from independent archives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 Generated by GPT 6.2 Sol in Codex via T3 Code. --- .../src/orchestration-v2/runtimeLayer.test.ts | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index eaed00ac330a2..7b0af603b26cb 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -14512,7 +14512,7 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(layerTest)), ); - it.effect("keeps an independent side-chat rearchive with a parent-derived ID", () => + it.effect("keeps the child archived when its side chat rearchives with a parent-derived ID", () => Effect.gen(function* () { const f = yield* fixture; const { side, child } = yield* f.sideChatWithChild(); @@ -14671,7 +14671,7 @@ describe("subagent lifecycle cascade", () => { }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), ); - it.effect("rolls back a nested side-chat deletion with an occupied recovery effect ID", () => + it.effect("rolls back direct archive and nested delete when their recovery ID is occupied", () => Effect.gen(function* () { const f = yield* fixture; const side = ThreadId.make("side"); @@ -14709,11 +14709,10 @@ describe("subagent lifecycle cascade", () => { it.effect("restores only what a parent archive cascaded, through independent archives", () => Effect.gen(function* () { const f = yield* fixture; - // An earlier independent archive whose ID matches the one the parent cascade derives. + // The newer direct archive has the same ID the parent cascade would derive. + yield* f.commit("archive"); yield* f.run("archive", app, "archive:parent:subagent:app"); yield* f.drain; - yield* f.run("archive"); - yield* f.drain; const receipts = yield* Receipts.CommandReceiptStoreV2; const { archiveSourceCommandId: _source, ...legacy } = Option.getOrThrow( yield* receipt("archive:parent:subagent:native"), @@ -14845,7 +14844,7 @@ describe("subagent lifecycle cascade", () => { ); it.effect.each([ - "late collision", + "fresh late rejection", "legacy deeper rejection", "legacy expired rejection", "legacy planning move", @@ -14854,12 +14853,13 @@ describe("subagent lifecycle cascade", () => { const f = yield* fixture; yield* f.run("archive", app); yield* f.run("unarchive", grandchild); + yield* f.drain; const leaf = ThreadId.make("leaf"); yield* f.set(leaf, { ...(scenario === "legacy planning move" ? { projectId: ProjectId.make("foreign") } : {}), lineage: { rootThreadId: parent, - parentThreadId: grandchild, + parentThreadId: scenario === "fresh late rejection" ? app : grandchild, relationshipToParent: "subagent", }, }); @@ -14900,16 +14900,15 @@ describe("subagent lifecycle cascade", () => { const skippedId = CommandId.make("archive:parent:subagent:app"); const receipts = yield* Receipts.CommandReceiptStoreV2; const created = Option.getOrThrow(yield* receipt("create:parent")); - const rejected = scenario !== "late collision"; const collision = { ...created, commandId: skippedId, - threadId: rejected ? app : ThreadId.make("fork"), - status: rejected ? "rejected" : "accepted", - error: rejected ? "Earlier rejection." : null, + threadId: app, + status: "rejected", + error: "Earlier rejection.", } as const; yield* f.commit("archive"); - if (scenario !== "late collision") { + if (scenario !== "fresh late rejection") { const sql = yield* SqlClient.SqlClient; yield* sql` UPDATE orchestration_v2_effect_outbox @@ -15011,7 +15010,7 @@ describe("subagent lifecycle cascade", () => { ); const committed = Option.getOrThrow(yield* receipt(childId)); const { archiveSourceCommandId: _source, ...legacy } = committed; - const childReceipt = scenario === "late collision" ? committed : legacy; + const childReceipt = scenario === "fresh late rejection" ? committed : legacy; if (childReceipt === legacy) yield* receipts.upsert(legacy); const leafId = `${childId}:subagent:${leaf}`; if (scenario === "legacy deeper rejection") { @@ -15044,6 +15043,10 @@ describe("subagent lifecycle cascade", () => { ); assert.deepEqual(recoveredReceipt, legacy); assert.deepEqual(Option.getOrThrow(yield* receipt(skippedId)), collision); + if (scenario === "fresh late rejection") { + const resumed = Option.getOrThrow(yield* receipt(`${skippedId}:subagent:${leaf}`)); + assert.include(resumed, { threadId: leaf, status: "accepted" }); + } if (scenario === "legacy deeper rejection") { assert.equal(recoveredSource, "archive:parent"); assert.equal(yield* f.archived(deepest), "1");