diff --git a/apps/server/src/orchestration-v2/CommandReceiptStore.ts b/apps/server/src/orchestration-v2/CommandReceiptStore.ts index f39de342b5526..9072d51ffb037 100644 --- a/apps/server/src/orchestration-v2/CommandReceiptStore.ts +++ b/apps/server/src/orchestration-v2/CommandReceiptStore.ts @@ -31,9 +31,19 @@ export class CommandReceiptStoreReadError extends Schema.TaggedError()( + "CommandReceiptStoreLifecycleError", + { threadId: ThreadId, cause: Schema.Defect() }, +) { + override get message(): string { + return `Failed to read orchestration V2 lifecycle receipt for thread ${this.threadId}.`; + } +} + export const CommandReceiptStoreV2Error = Schema.Union([ CommandReceiptStoreWriteError, CommandReceiptStoreReadError, + CommandReceiptStoreLifecycleError, ]); export type CommandReceiptStoreV2Error = typeof CommandReceiptStoreV2Error.Type; @@ -48,6 +58,7 @@ export const CommandReceiptV2 = Schema.Struct({ resultSequence: NonNegativeInt, status: CommandReceiptV2Status, error: Schema.NullOr(Schema.String), + archiveSourceCommandId: Schema.optional(CommandId), }); export type CommandReceiptV2 = typeof CommandReceiptV2.Type; @@ -62,15 +73,27 @@ export const ProjectCommandReceiptV2 = Schema.Struct({ }); export type ProjectCommandReceiptV2 = typeof ProjectCommandReceiptV2.Type; -type AnyCommandReceiptV2 = CommandReceiptV2 | ProjectCommandReceiptV2; +export type AnyCommandReceiptV2 = CommandReceiptV2 | ProjectCommandReceiptV2; export interface CommandReceiptStoreV2Shape { + readonly getLatestLifecycle: ( + threadId: ThreadId, + commandTypes?: ReadonlyArray, + ) => Effect.Effect; readonly insertIfAbsent: ( receipt: AnyCommandReceiptV2, ) => Effect.Effect; readonly upsert: ( receipt: AnyCommandReceiptV2, ) => Effect.Effect; + readonly setArchiveSourceIfMissing: ( + input: Parameters< + OrchestrationCommandReceipts.OrchestrationCommandReceiptRepository["Service"]["setArchiveSourceIfMissing"] + >[0], + ) => Effect.Effect; + readonly getAnyByCommandId: ( + commandId: CommandId, + ) => Effect.Effect, CommandReceiptStoreV2Error>; /** None for project command ids. */ readonly getByCommandId: ( commandId: CommandId, @@ -108,6 +131,23 @@ function fromApplicationReceipt(receipt: OrchestrationCommandReceipts.Orchestrat resultSequence: receipt.resultSequence, status: receipt.status, error: receipt.error, + ...(receipt.archiveSourceCommandId == null + ? {} + : { archiveSourceCommandId: receipt.archiveSourceCommandId }), + }); +} + +function fromProjectApplicationReceipt( + receipt: OrchestrationCommandReceipts.OrchestrationCommandReceipt, +) { + return decodeProjectReceipt({ + commandId: receipt.commandId, + projectId: receipt.aggregateId, + commandType: receipt.commandType, + acceptedAt: receipt.acceptedAt, + resultSequence: receipt.resultSequence, + status: receipt.status, + error: receipt.error, }); } @@ -118,7 +158,11 @@ function toApplicationReceipt( commandId: receipt.commandId, ...("projectId" in receipt ? { aggregateKind: "project" as const, aggregateId: receipt.projectId } - : { aggregateKind: "thread" as const, aggregateId: receipt.threadId }), + : { + aggregateKind: "thread" as const, + aggregateId: receipt.threadId, + archiveSourceCommandId: receipt.archiveSourceCommandId, + }), commandType: receipt.commandType, acceptedAt: DateTime.formatIso(receipt.acceptedAt), resultSequence: receipt.resultSequence, @@ -137,6 +181,15 @@ const layerBase: Layer.Layer< const receipts = yield* OrchestrationCommandReceipts.OrchestrationCommandReceiptRepository; return CommandReceiptStoreV2.of({ + getLatestLifecycle: Effect.fn("CommandReceiptStore.getLatestLifecycle")( + (threadId, commandTypes) => + receipts.getLatestLifecycle({ threadId, commandTypes }).pipe( + Effect.flatMap((receipt) => + Option.isSome(receipt) ? fromApplicationReceipt(receipt.value) : Effect.succeed(null), + ), + Effect.mapError((cause) => new CommandReceiptStoreLifecycleError({ threadId, cause })), + ), + ), insertIfAbsent: (receipt) => receipts.insertIfAbsent(toApplicationReceipt(receipt)).pipe( Effect.mapError( @@ -157,6 +210,29 @@ const layerBase: Layer.Layer< }), ), ), + setArchiveSourceIfMissing: Effect.fn("CommandReceiptStore.setArchiveSourceIfMissing")( + (input) => + receipts + .setArchiveSourceIfMissing(input) + .pipe( + Effect.mapError( + (cause) => new CommandReceiptStoreWriteError({ commandId: input.commandId, cause }), + ), + ), + ), + getAnyByCommandId: Effect.fn("CommandReceiptStore.getAnyByCommandId")((commandId) => + receipts.getByCommandId({ commandId }).pipe( + Effect.flatMap((receipt) => { + if (Option.isNone(receipt)) return Effect.succeedNone; + const decode: Effect.Effect = + receipt.value.aggregateKind === "project" + ? fromProjectApplicationReceipt(receipt.value) + : fromApplicationReceipt(receipt.value); + return decode.pipe(Effect.asSome); + }), + Effect.mapError((cause) => new CommandReceiptStoreReadError({ commandId, cause })), + ), + ), getByCommandId: (commandId) => receipts.getByCommandId({ commandId }).pipe( Effect.flatMap( @@ -181,15 +257,7 @@ const layerBase: Layer.Layer< Effect.flatMap((receipt) => Option.isNone(receipt) || receipt.value.aggregateKind !== "project" ? Effect.succeedNone - : decodeProjectReceipt({ - commandId: receipt.value.commandId, - projectId: receipt.value.aggregateId, - commandType: receipt.value.commandType, - acceptedAt: receipt.value.acceptedAt, - resultSequence: receipt.value.resultSequence, - status: receipt.value.status, - error: receipt.value.error, - }).pipe(Effect.asSome), + : fromProjectApplicationReceipt(receipt.value).pipe(Effect.asSome), ), Effect.mapError((cause) => new CommandReceiptStoreReadError({ commandId, cause })), ), @@ -200,3 +268,130 @@ const layerBase: Layer.Layer< export const layer = layerBase.pipe(Layer.provide(OrchestrationCommandReceipts.layer)); export const layerFromApplicationReceipts = layerBase; + +export const cascadeCommandIdLimit = 32; + +export const cascadeCommandIdAt = (base: CommandId, retry: number) => + retry === 0 ? base : CommandId.make(`${base}:retry:${retry}`); + +/** Older receipts encode cascade ownership in the command ID. */ +export const legacyCascadeCommandId = Effect.fn("CommandReceiptStore.cascadeCommandId")(function* ( + receipts: CommandReceiptStoreV2Shape, + base: CommandId, +) { + for (let retry = 0; retry < cascadeCommandIdLimit; retry++) { + const id = cascadeCommandIdAt(base, retry); + const receipt = yield* receipts.getByCommandId(id); + if (Option.isNone(receipt) || receipt.value.status === "accepted") return id; + } + return undefined; +}); + +export const matchesLegacyCascadePath = Effect.fn("CommandReceiptStore.matchesLegacyCascadePath")( + function* ( + receipts: CommandReceiptStoreV2Shape, + anchorId: CommandId, + path: ReadonlyArray, + completedId: CommandId, + occupiesDispatchedId: (receipt: AnyCommandReceiptV2) => boolean, + ) { + const retries: Array<{ baseId: CommandId; retry: number; dispatched: boolean }> = []; + let id: string = completedId; + for (const [offset, threadId] of path.toReversed().entries()) { + const segment = `:subagent:${threadId}`; + if (!id.endsWith(segment)) { + const suffix = /:retry:([1-9][0-9]*)$/.exec(id); + if (suffix === null) return false; + const retry = Number(suffix[1]); + if (retry >= cascadeCommandIdLimit) return false; + id = id.slice(0, suffix.index); + retries.push({ baseId: CommandId.make(id), retry, dispatched: offset === 0 }); + } + if (!id.endsWith(segment)) return false; + id = id.slice(0, -segment.length); + } + if (id !== anchorId) return false; + + for (const { baseId, retry, dispatched } of retries) { + for (let previous = 0; previous < retry; previous++) { + const commandId = cascadeCommandIdAt(baseId, previous); + const receipt = yield* dispatched + ? receipts.getAnyByCommandId(commandId) + : receipts.getByCommandId(commandId); + if ( + Option.isNone(receipt) || + (dispatched ? !occupiesDispatchedId(receipt.value) : receipt.value.status !== "rejected") + ) + return false; + } + } + return true; + }, +); + +/** A lifecycle choice made after the cascade's source command, other than `ownCommandId`. */ +export const supersedesCascade = ( + latest: CommandReceiptV2 | null, + cutoff: number, + ownCommandId?: CommandId, +) => latest !== null && latest.resultSequence > cutoff && latest.commandId !== ownCommandId; + +/** Whether a cascade from an archive of `rootThreadId`, along `path`, archived this thread. */ +export const archivedByCascade = Effect.fn("CommandReceiptStore.archivedByCascade")(function* ( + receipts: CommandReceiptStoreV2Shape, + latest: CommandReceiptV2 | null, + rootThreadId: ThreadId, + path: ReadonlyArray, + expectedArchiveCommandId?: CommandId, +) { + if (latest?.commandType !== "thread.archive") return false; + let archive = latest; + let depth = path.length; + while (depth > 0 && archive.archiveSourceCommandId !== undefined) { + if (archive.archiveSourceCommandId === archive.commandId) return false; + const source = Option.getOrUndefined( + yield* receipts.getByCommandId(archive.archiveSourceCommandId), + ); + if ( + source?.status !== "accepted" || + source.commandType !== "thread.archive" || + source.resultSequence > archive.resultSequence + ) + return false; + // A cascade can pass through an already archived thread without claiming its archive. + const ancestor = + source.threadId === rootThreadId + ? 0 + : depth < 2 + ? -1 + : path.lastIndexOf(source.threadId, depth - 2) + 1; + if (ancestor <= 0 && source.threadId !== rootThreadId) return false; + depth = ancestor; + archive = source; + } + if (depth === 0) + return expectedArchiveCommandId === undefined || archive.commandId === expectedArchiveCommandId; + + const legacyPath = path.slice(0, depth); + let rootCommandId: string = archive.commandId; + for (const threadId of legacyPath.toReversed()) { + const segment = `:subagent:${threadId}`; + if (!rootCommandId.endsWith(segment)) rootCommandId = rootCommandId.replace(/:retry:\d+$/, ""); + if (!rootCommandId.endsWith(segment)) return false; + rootCommandId = rootCommandId.slice(0, -segment.length); + } + if (expectedArchiveCommandId !== undefined && rootCommandId !== expectedArchiveCommandId) + return false; + const root = Option.getOrNull(yield* receipts.getByCommandId(CommandId.make(rootCommandId))); + const valid = root?.status === "accepted" && root.threadId === rootThreadId; + // Derive the IDs again, so a user ID only shaped like a cascade retry is not counted. + let derived = valid && root.commandType === "thread.archive" ? root.commandId : undefined; + for (const threadId of legacyPath) { + if (derived === undefined) return false; + derived = yield* legacyCascadeCommandId( + receipts, + CommandId.make(`${derived}:subagent:${threadId}`), + ); + } + return derived === archive.commandId && (root?.resultSequence ?? 0) <= archive.resultSequence; +}); diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index b495293bc2228..99f7d9121edb0 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -12,7 +12,10 @@ import { ProviderUserInputAnswers, ProviderThreadId, ProviderTurnId, + ProviderInteractionMode, + ProjectId, RunId, + RuntimeMode, RuntimeRequestId, ThreadId, } from "@t3tools/contracts"; @@ -32,6 +35,37 @@ import * as SqlClient from "effect/sql/SqlClient"; import { forkParked } from "../serverActivation.ts"; +const EmbeddedSideChatCascade = Schema.Struct({ + parentThreadId: ThreadId, + projectId: ProjectId, + cutoff: Schema.optional(NonNegativeInt), + archiveCommandId: Schema.optional(CommandId), +}); +export type EmbeddedSideChatCascade = typeof EmbeddedSideChatCascade.Type; + +/** Set while a cascade dispatches a descendant's lifecycle command; delete needs no source. */ +export const SubagentCascade = Context.Reference< + | { + readonly source: + | { + readonly threadId: ThreadId; + readonly cutoff: number; + readonly archiveCommandId?: CommandId | undefined; + } + | undefined; + readonly path: ReadonlyArray; + readonly parentArchiveCommandId: CommandId; + readonly legacyPath?: + | { + readonly anchorId: CommandId; + readonly depth: number; + readonly verified?: true; + } + | undefined; + } + | undefined +>("t3/orchestration-v2/SubagentCascade", { defaultValue: () => undefined }); + export const OrchestrationEffectRequestV2 = Schema.Union([ Schema.Struct({ type: Schema.Literal("provider-runtime.continue"), @@ -127,6 +161,16 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ attachmentIds: Schema.Array(Schema.String), skipReferenced: Schema.optional(Schema.Boolean), }), + Schema.Struct({ + type: Schema.Literal("subagent-threads.cascade"), + operation: Schema.Literals(["thread.archive", "thread.unarchive", "thread.delete"]), + sideChat: Schema.optional(EmbeddedSideChatCascade), + directDelete: Schema.optional(Schema.Literal(true)), + stableSkipPrefix: Schema.optional(Schema.Literal(true)), + limit: Schema.optional( + Schema.Struct({ runtimeMode: RuntimeMode, interactionMode: ProviderInteractionMode }), + ), + }), Schema.Struct({ type: Schema.Literal("thread-title.generate"), kind: Schema.Union([ @@ -150,6 +194,7 @@ export const REPLAY_SAFE_EFFECT_TYPES_AFTER_PROCESS_LOSS = [ "terminal.cleanup", "preview.cleanup", "attachment.cleanup", + "subagent-threads.cascade", "thread-title.generate", "delegated-tasks.stop", ] as const satisfies ReadonlyArray; @@ -200,6 +245,13 @@ export interface PendingOrchestrationEffectV2 { readonly availableAt?: DateTime.Utc; } +export type SubagentCascadeEffect = PendingOrchestrationEffectV2 & { + readonly request: Extract< + OrchestrationEffectRequestV2, + { readonly type: "subagent-threads.cascade" } + >; +}; + export class EffectOutboxError extends Schema.TaggedError()( "EffectOutboxError", { @@ -440,7 +492,11 @@ export const layer: Layer.Layer = La ${nowIso}, ${nowIso} ) - ON CONFLICT(effect_id) DO NOTHING + ${ + effect.request.type === "subagent-threads.cascade" + ? sql`` + : sql`ON CONFLICT(effect_id) DO NOTHING` + } `, { concurrency: 1, discard: true }, ); diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index c637446d78827..584263f8cc1e1 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -30,6 +30,7 @@ import * as ProviderTurnStartService from "./ProviderTurnStartService.ts"; import * as RuntimeRequestService from "./RuntimeRequestService.ts"; import * as ThreadTitleRegenerationService from "./ThreadTitleRegenerationService.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; +import { DispatchModeLimit } from "./DispatchModeLimit.ts"; import * as ServerSettings from "../serverSettings.ts"; import { continueRestartedRun } from "./RestartContinuation.ts"; @@ -680,6 +681,27 @@ export const layerExecutor: Layer.Layer< ), ); } + case "subagent-threads.cascade": + return threads + .cascadeSubagentLifecycle({ + commandId: effect.commandId, + threadId: effect.threadId, + operation: effect.request.operation, + sideChat: effect.request.sideChat, + directDelete: effect.request.directDelete, + stableSkipPrefix: effect.request.stableSkipPrefix, + }) + .pipe( + Effect.provideService(DispatchModeLimit, effect.request.limit), + Effect.mapError( + (cause) => + new OrchestrationEffectExecutionError({ + effectId: effect.id, + effectType: effect.request.type, + cause, + }), + ), + ); case "thread-title.generate": return threadTitleRegeneration .execute({ diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 3f6744e576edd..4bdd622ff821a 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -211,6 +211,7 @@ export interface EventSinkV2Shape { readonly acceptedAt: DateTime.Utc; readonly events: ReadonlyArray; readonly effects: ReadonlyArray; + readonly archiveSourceCommandId?: CommandId; readonly cancelUnsettledEffects?: { readonly effectTypes: ReadonlyArray; readonly reason: string; @@ -219,7 +220,7 @@ export interface EventSinkV2Shape { readonly updateDelegatedCompletionOnlyRunId?: RunId; }) => Effect.Effect< { - readonly receipt: CommandReceiptStore.CommandReceiptV2; + readonly receipt: CommandReceiptStore.AnyCommandReceiptV2; readonly storedEvents: ReadonlyArray; readonly committed: boolean; readonly cancelledEffectCount: number; @@ -260,6 +261,11 @@ export interface EventSinkV2Shape { readonly latestSequence: (input?: { readonly threadId?: ThreadId; }) => Effect.Effect; + readonly readRetainedThreadHistory: (input: { + readonly threadId: ThreadId; + readonly throughSequence: number; + readonly limit: number; + }) => Effect.Effect | null, EventSinkV2Error>; readonly readByCommandId: (input: { readonly commandId: CommandId; }) => Stream.Stream; @@ -772,12 +778,13 @@ const layerBase: Layer.Layer< const existingCommandResult = /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced(function* (commandId: CommandId) { - const existing = yield* commandReceipts.getByCommandId(commandId); + const existing = yield* commandReceipts.getAnyByCommandId(commandId); if (Option.isNone(existing)) { return yield* Effect.die( new Error(`Command receipt ${commandId} disappeared during its transaction.`), ); } + if ("projectId" in existing.value) return { receipt: existing.value, storedEvents: [] }; const storedEvents = yield* eventStore.readByCommandId({ commandId }).pipe( Stream.runCollect, Effect.map((events): ReadonlyArray => Array.from(events)), @@ -868,8 +875,35 @@ const layerBase: Layer.Layer< resultSequence: sequence, status: "accepted", error: null, + ...(input.archiveSourceCommandId === undefined + ? {} + : { archiveSourceCommandId: input.archiveSourceCommandId }), }; yield* commandReceipts.upsert(receipt); + // Generated side-chat cascades need archive ownership after their outbox entries are pruned. + for (const effect of input.effects) { + if ( + effect.commandId !== input.commandId && + effect.request.type === "subagent-threads.cascade" && + effect.request.operation !== "thread.delete" + ) { + const reserved = yield* commandReceipts.insertIfAbsent({ + ...receipt, + commandId: effect.commandId, + threadId: effect.threadId, + commandType: effect.request.operation, + ...(effect.request.operation === "thread.archive" + ? { archiveSourceCommandId: effect.commandId } + : {}), + }); + if (!reserved) { + return yield* new CommandReceiptStore.CommandReceiptStoreWriteError({ + commandId: effect.commandId, + cause: "Generated side-chat cascade command ID is already in use.", + }); + } + } + } const cancelledEffectIds = input.cancelUnsettledEffects === undefined ? [] @@ -1305,6 +1339,24 @@ const layerBase: Layer.Layer< }), ), ), + readRetainedThreadHistory: Effect.fn("EventSink.readRetainedThreadHistory")( + function* ({ threadId, throughSequence, limit }) { + const prunedThrough = yield* eventStore.prunedThroughSequence; + const events = yield* eventStore + .read({ + threadId, + afterSequence: prunedThrough, + throughSequence, + limit: limit + 1, + }) + .pipe(Stream.runCollect); + if (events.length > limit || (yield* eventStore.prunedThroughSequence) !== prunedThrough) + return null; + return Array.from(events); + }, + (effect, { threadId }) => + effect.pipe(Effect.mapError((cause) => new EventSinkStreamError({ threadId, cause }))), + ), readByCommandId: (input) => eventStore.readByCommandId(input).pipe( Stream.mapError( diff --git a/apps/server/src/orchestration-v2/EventStore.ts b/apps/server/src/orchestration-v2/EventStore.ts index e2125508dee8f..eca49146c283e 100644 --- a/apps/server/src/orchestration-v2/EventStore.ts +++ b/apps/server/src/orchestration-v2/EventStore.ts @@ -69,6 +69,7 @@ export interface EventStoreV2Shape { readonly threadId?: ThreadId; }) => Effect.Effect; readonly latestApplicationSequence: Effect.Effect; + readonly prunedThroughSequence: Effect.Effect; readonly publishCommitted: (events: ReadonlyArray) => Effect.Effect; } @@ -144,6 +145,9 @@ const layerBase: Layer.Layer new EventStoreReadEventsError({ cause })), ), + prunedThroughSequence: applicationEvents.prunedThroughSequence.pipe( + Effect.mapError((cause) => new EventStoreReadEventsError({ cause })), + ), publishCommitted: applicationEvents.publishCommitted, }); }), diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 00939843c13f1..1f67c87a42d67 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -128,7 +128,14 @@ import { providerTurnCapabilities, resolveMessageDispatchIntent, } from "./CommandPolicy.ts"; -import { CommandReceiptStoreV2, type CommandReceiptV2 } from "./CommandReceiptStore.ts"; +import { + type AnyCommandReceiptV2, + archivedByCascade, + CommandReceiptStoreV2, + type CommandReceiptV2, + matchesLegacyCascadePath, + supersedesCascade, +} from "./CommandReceiptStore.ts"; import { ContextHandoffServiceV2, readForkHistoryItems, @@ -370,9 +377,12 @@ export class OrchestratorCommandIdConflictError extends Schema.TaggedError; + readonly subagentCascades?: ReadonlyArray; + readonly cascadeIdOccupied?: true; + readonly cascadeReplayedCommandId?: CommandId; /** The command id was already accepted; analytics skip it. */ readonly replayed?: true; } @@ -476,6 +489,9 @@ export interface OrchestratorV2Shape { readonly getThreadProjection: ( threadId: ThreadId, ) => Effect.Effect; + readonly getSubagentChildThreads: ( + ...args: Parameters + ) => Effect.Effect, OrchestratorV2Error>; readonly getCheckpointContext: ( threadId: ThreadId, ) => Effect.Effect; @@ -550,6 +566,10 @@ export interface OrchestratorV2Shape { readonly getThreadEventSequence: ( threadId: ThreadId, ) => Effect.Effect; + readonly hasRecordedThreadDeletion: ( + commandId: CommandId, + threadId: ThreadId, + ) => Effect.Effect; readonly streamStoredEvents: Stream.Stream; /** Events arrive projected for the wire and shared by every subscriber; never mutate them. */ readonly streamStoredEventsFrom: (input?: { @@ -1893,6 +1913,46 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .findThread(threadId) .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))); + const sideChatLifecycleSuffix = (threadId: ThreadId) => `:side-chat:${threadId}`; + const sideChatLifecycleCommandId = (commandId: CommandId, threadId: ThreadId) => + CommandId.make(`${commandId}${sideChatLifecycleSuffix(threadId)}`); + + // A cascade walks the whole subtree itself, so its own commands start none. + const subagentCascadeEffects = Effect.fn("Orchestrator.subagentCascadeEffects")(function* ( + { commandId, threadId }: { readonly commandId: CommandId; readonly threadId: ThreadId }, + operation: "thread.archive" | "thread.unarchive" | "thread.delete", + projectId: ProjectId, + sideChat?: EffectOutbox.EmbeddedSideChatCascade, + ) { + const cascade = yield* EffectOutbox.SubagentCascade; + if (cascade?.path.at(-1) === threadId) return []; + if (sideChat && sideChat.projectId !== projectId) return []; + if (operation === "thread.unarchive" && sideChat && !sideChat.archiveCommandId) return []; + if ((yield* projectionStore.getSubagentChildThreads(threadId)).length === 0) return []; + const cutoff = cascade?.source?.cutoff; + // The generated receipt would hide the side chat's newer choice from the cascade worker. + if (sideChat && operation !== "thread.delete" && cutoff !== undefined) { + const latest = yield* commandReceipts.getLatestLifecycle(threadId, [ + "thread.archive", + "thread.unarchive", + ]); + if (supersedesCascade(latest, cutoff, commandId)) return []; + } + const limit = yield* DispatchModeLimit; + const request = { + type: "subagent-threads.cascade", + operation, + limit, + stableSkipPrefix: true, + ...(operation === "thread.delete" && sideChat === undefined + ? { directDelete: true as const } + : {}), + ...(sideChat === undefined ? {} : { sideChat: { ...sideChat, cutoff } }), + } as const; + const id = `effect:${commandId}:subagent-threads.cascade`; + return [{ id, commandId, threadId, request } satisfies EffectOutbox.SubagentCascadeEffect]; + }); + /** Side chats live inside their parent, so deleting or archiving the parent takes them too. */ const liveSideChatIds = Effect.fn("Orchestrator.liveSideChatIds")( function* (threadId: ThreadId) { @@ -4368,6 +4428,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio >, events: Ref.Ref>, effects: Ref.Ref>, + sideChat?: EffectOutbox.EmbeddedSideChatCascade, ) { const thread = yield* projectionStore.getThread(command.threadId).pipe( Effect.mapError( @@ -5600,6 +5661,16 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ]); } + if (command.type === "thread.archive" || command.type === "thread.unarchive") { + const cascade = yield* subagentCascadeEffects( + command, + command.type, + thread.projectId, + sideChat, + ).pipe(mapDispatchError(command)); + yield* Ref.update(effects, (existing) => [...existing, ...cascade]); + } + if (command.type === "thread.metadata.update" && command.regenerateTitle === true) { yield* Ref.update(effects, (existing) => [ ...existing, @@ -15635,6 +15706,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const now = yield* DateTime.now; const planDeletion = Effect.fn("Orchestrator.planDeletion")(function* ( deletion: typeof command, + sideChat?: EffectOutbox.EmbeddedSideChatCascade, ) { const projection = yield* projectionStore .getThreadRecords(deletion.threadId, [ @@ -15650,7 +15722,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio (cause) => new OrchestratorProjectionError({ threadId: deletion.threadId, cause }), ), ); - return yield* mapDispatchError(command)( + const plan = yield* mapDispatchError(command)( planThreadDeletion({ command: deletion, projection, @@ -15661,6 +15733,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio idAllocator, }), ); + const cascade = yield* subagentCascadeEffects( + deletion, + deletion.type, + projection.thread.projectId, + sideChat, + ).pipe(mapDispatchError(command)); + return { + ...plan, + projectId: projection.thread.projectId, + effects: [...plan.effects, ...cascade], + }; }); // Checked under the thread lock: a message sent since the client looked keeps the thread. if (command.onlyIfUnused === true) { @@ -15676,14 +15759,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } } // The dispatch wrapper holds each side chat's lock, so none can commit after this plan. + const parent = yield* planDeletion(command); const plans = [ - yield* planDeletion(command), + parent, ...(yield* Effect.forEach(sideChatIds, (threadId) => - planDeletion({ - ...command, - commandId: CommandId.make(`${command.commandId}:side-chat:${threadId}`), - threadId, - }), + planDeletion( + { + ...command, + commandId: sideChatLifecycleCommandId(command.commandId, threadId), + threadId, + }, + { parentThreadId: command.threadId, projectId: parent.projectId }, + ), )), ]; return { @@ -15693,8 +15780,16 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } case "thread.archive": case "thread.unarchive": { - const parentArchivedAt = (yield* findThread(command.threadId))?.archivedAt ?? null; + const parent = yield* findThread(command.threadId); + const parentArchivedAt = parent?.archivedAt ?? null; yield* dispatchThreadMutation(command, events, effects); + if (parent === null || sideChatIds.length === 0) break; + const parentArchive = + command.type === "thread.unarchive" + ? yield* commandReceipts + .getLatestLifecycle(parent.id, ["thread.archive"]) + .pipe(mapDispatchError(command)) + : null; // Unarchive restores the side chats this parent's archive took, not ones archived before it. for (const threadId of sideChatIds) { const sideChatArchivedAt = (yield* findThread(threadId))?.archivedAt ?? null; @@ -15713,15 +15808,31 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .pipe(mapDispatchError(command)); if (hasRunningDelegatedTasks(subagents)) continue; } - yield* dispatchThreadMutation( - { - ...command, - commandId: CommandId.make(`${command.commandId}:side-chat:${threadId}`), - threadId, - }, - events, - effects, - ); + const sideChatCommand = { + ...command, + commandId: sideChatLifecycleCommandId(command.commandId, threadId), + threadId, + }; + const derivedArchiveId = + parentArchive !== null + ? sideChatLifecycleCommandId(parentArchive.commandId, threadId) + : undefined; + const sideChatArchive = + derivedArchiveId === undefined + ? null + : yield* commandReceipts + .getLatestLifecycle(threadId, ["thread.archive"]) + .pipe(mapDispatchError(command)); + yield* dispatchThreadMutation(sideChatCommand, events, effects, { + parentThreadId: parent.id, + projectId: parent.projectId, + archiveCommandId: + sideChatArchive !== null && + sideChatArchive.commandId === derivedArchiveId && + sideChatArchive.resultSequence === parentArchive?.resultSequence + ? sideChatArchive.commandId + : undefined, + }); } break; } @@ -16024,6 +16135,33 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }; }); + const rejectConflictingSideChatReceipt = Effect.fn( + "OrchestratorV2.rejectConflictingSideChatReceipt", + )(function* (command: OrchestrationV2ServerCommand, receipt: CommandReceiptV2) { + if (receipt.commandType !== "thread.archive" && receipt.commandType !== "thread.unarchive") + return; + const suffix = sideChatLifecycleSuffix(receipt.threadId); + if (receipt.commandId.length <= suffix.length || !receipt.commandId.endsWith(suffix)) return; + const parentId = CommandId.make(receipt.commandId.slice(0, -suffix.length)); + const parent = Option.getOrUndefined( + yield* commandReceipts.getByCommandId(parentId).pipe(mapDispatchError(command)), + ); + if ( + parent?.status === "accepted" && + parent.commandType === receipt.commandType && + parent.resultSequence === receipt.resultSequence && + parent.threadId !== receipt.threadId + ) { + return yield* new OrchestratorCommandIdConflictError({ + commandId: command.commandId, + commandType: command.type, + receiptThreadId: receipt.threadId, + commandThreadId: receipt.threadId, + receiptCommandType: receipt.commandType, + }); + } + }); + const replayCommandReceipt = Effect.fn("orchestrationV2.dispatch.replayReceipt")(function* ( command: OrchestrationV2ServerCommand, receipt: CommandReceiptV2, @@ -16045,6 +16183,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio commandThreadId: dispatchThreadId, }); } + if (receipt.commandType !== command.type) + yield* rejectConflictingSideChatReceipt(command, receipt); const storedEvents = yield* eventSink.readByCommandId({ commandId: command.commandId }).pipe( Stream.runCollect, Effect.map((events): ReadonlyArray => Array.from(events)), @@ -16072,6 +16212,133 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } satisfies OrchestratorV2DispatchResult; }); + type ThreadMembership = Pick; + const sameMembership = (left: ThreadMembership, right: ThreadMembership) => + left.projectId === right.projectId && + left.lineage.parentThreadId === right.lineage.parentThreadId && + left.lineage.relationshipToParent === right.lineage.relationshipToParent; + + const legacyMembershipHistory = Effect.fn("Orchestrator.legacyMembershipHistory")(function* ( + threadId: ThreadId, + throughSequence: number, + ) { + const events = yield* eventSink.readRetainedThreadHistory({ + threadId, + throughSequence, + limit: 256, + }); + return events?.flatMap(({ commandId, event, sequence }) => { + const { payload, type } = event; + return "projectId" in payload && "lineage" in payload + ? [{ commandId, membership: payload, sequence, type }] + : []; + }); + }); + + const provesLegacyChildEligibility = Effect.fn("Orchestrator.provesLegacyChildEligibility")( + function* (thread: OrchestrationV2AppThread, receipt: CommandReceiptV2) { + const history = yield* legacyMembershipHistory(thread.id, receipt.resultSequence); + const lifecycleType = + receipt.commandType === "thread.archive" ? "thread.archived" : "thread.unarchived"; + const lifecycle = history?.find( + (event) => event.commandId === receipt.commandId && event.type === lifecycleType, + ); + if ( + history === undefined || + lifecycle === undefined || + !sameMembership(lifecycle.membership, thread) + ) + return false; + + // The earlier accepted command held this thread's dispatch lock before the lifecycle plan. + let witness = undefined; + for (const event of history.toReversed()) { + if ( + event.sequence >= lifecycle.sequence || + event.commandId === null || + event.commandId === receipt.commandId + ) + continue; + const commandType = + event.type === "thread.metadata-updated" + ? "thread.metadata.update" + : event.type === "thread.archived" + ? "thread.archive" + : event.type === "thread.unarchived" + ? "thread.unarchive" + : undefined; + if (commandType === undefined) continue; + const prior = Option.getOrUndefined(yield* commandReceipts.getByCommandId(event.commandId)); + if ( + prior?.status === "accepted" && + prior.threadId === thread.id && + prior.commandType === commandType && + prior.resultSequence >= event.sequence && + prior.resultSequence < lifecycle.sequence + ) { + witness = event; + break; + } + } + if ( + witness === undefined || + !history.every( + (event) => event.sequence < witness.sequence || sameMembership(event.membership, thread), + ) + ) + return false; + + const children = yield* projectionStore.getSubagentChildThreads(thread.id); + for (const child of children) { + const childHistory = yield* legacyMembershipHistory(child.id, receipt.resultSequence); + if (childHistory === undefined) continue; + const atWitness = childHistory.findLast((event) => event.sequence <= witness.sequence); + if ( + atWitness !== undefined && + sameMembership(atWitness.membership, child) && + childHistory.every( + (event) => + event.sequence <= witness.sequence || sameMembership(event.membership, child), + ) + ) + return true; + } + return false; + }, + ); + + const ownsLegacyCascadeReceipt = Effect.fn("Orchestrator.ownsLegacyCascadeReceipt")(function* ( + command: OrchestrationV2ServerCommand, + thread: OrchestrationV2AppThread, + receipt: CommandReceiptV2, + parentArchiveCommandId: CommandId, + ) { + if (receipt.archiveSourceCommandId !== undefined) + return receipt.archiveSourceCommandId === parentArchiveCommandId; + + const read = mapDispatchError(command); + const directEffect = yield* effectOutbox + .get(`effect:${receipt.commandId}:subagent-threads.cascade`) + .pipe(read); + if (Option.isSome(directEffect)) return false; + + // An eligible child before planning implies a direct command would still have its own effect. + const cutoff = DateTime.subtractDuration( + yield* DateTime.now, + EffectOutbox.SETTLED_EFFECT_RETENTION, + ); + if ( + !DateTime.isLessThan(receipt.acceptedAt, cutoff) && + (yield* provesLegacyChildEligibility(thread, receipt).pipe(read)) + ) + return true; + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "Cannot verify the archive or restore that owns the legacy cascade path.", + }); + }); + /** `sideChatIds` are the live side chats a delete, archive, or unarchive cascades to, read under their locks. */ const dispatchWithReceiptEffect = Effect.fn("orchestrationV2.dispatch.withReceipt")(function* ( command: OrchestrationV2ServerCommand, @@ -16084,24 +16351,112 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio "orchestration_v2.thread_id": commandThreadId(command), }); - const existingReceipt = yield* commandReceipts.getByCommandId(command.commandId).pipe( - Effect.mapError( - (cause) => - new OrchestratorDispatchError({ - commandId: command.commandId, - commandType: command.type, - cause, - }), - ), - ); + const cascade = yield* EffectOutbox.SubagentCascade; + const read = mapDispatchError(command); + const cascadeThread = + cascade !== undefined && + (command.type === "thread.archive" || + command.type === "thread.unarchive" || + command.type === "thread.delete") + ? yield* projectionStore.getThread(command.threadId).pipe(read) + : undefined; + if (cascadeThread !== undefined) { + const parentId = cascadeThread.lineage.parentThreadId; + const parent = + parentId === null ? null : yield* projectionStore.getThread(parentId).pipe(read); + if (parent?.projectId !== cascadeThread.projectId) + return { sequence: cascade?.source?.cutoff ?? 0, storedEvents: [] }; + yield* refuseAboveDispatchModeLimit(command, cascadeThread.id, cascadeThread); + } + const occupiedCascadeId = (receipt: AnyCommandReceiptV2) => + cascadeThread !== undefined && + ("projectId" in receipt || + receipt.threadId !== cascadeThread.id || + receipt.status !== "accepted" || + receipt.commandType !== command.type || + (command.type === "thread.delete" && cascadeThread.deletedAt === null) || + (cascade?.source !== undefined && receipt.resultSequence < cascade.source.cutoff) || + (command.type === "thread.archive" && + receipt.archiveSourceCommandId !== undefined && + receipt.archiveSourceCommandId !== cascade?.parentArchiveCommandId)); + + const legacyLatest = + cascadeThread !== undefined && + cascade?.source !== undefined && + cascade.legacyPath !== undefined + ? yield* commandReceipts + .getLatestLifecycle(cascadeThread.id, ["thread.archive", "thread.unarchive"]) + .pipe(read) + : undefined; + if ( + cascadeThread !== undefined && + cascade?.source !== undefined && + cascade.legacyPath !== undefined && + legacyLatest?.status === "accepted" && + legacyLatest.commandType === command.type && + (legacyLatest.commandId !== command.commandId || + (command.type === "thread.archive" && cascade.legacyPath.verified === true)) && + legacyLatest.resultSequence > cascade.source.cutoff && + cascadeThread.deletedAt === null && + (command.type === "thread.archive" + ? cascadeThread.archivedAt !== null + : cascadeThread.archivedAt === null) && + (yield* matchesLegacyCascadePath( + commandReceipts, + cascade.legacyPath.anchorId, + cascade.path.slice(cascade.legacyPath.depth), + legacyLatest.commandId, + occupiedCascadeId, + ).pipe(read)) + ) { + if ( + !(yield* ownsLegacyCascadeReceipt( + command, + cascadeThread, + legacyLatest, + cascade.parentArchiveCommandId, + )) + ) + return { sequence: legacyLatest.resultSequence, storedEvents: [] }; + if (command.type === "thread.archive" && legacyLatest.archiveSourceCommandId === undefined) { + const recorded = yield* commandReceipts + .setArchiveSourceIfMissing({ + commandId: legacyLatest.commandId, + threadId: cascadeThread.id, + resultSequence: legacyLatest.resultSequence, + sourceCommandId: cascade.parentArchiveCommandId, + }) + .pipe(read); + if (!recorded) { + const current = Option.getOrUndefined( + yield* commandReceipts.getByCommandId(legacyLatest.commandId).pipe(read), + ); + if (current?.archiveSourceCommandId !== cascade.parentArchiveCommandId) + return { sequence: legacyLatest.resultSequence, storedEvents: [] }; + } + } + const replay = yield* replayCommandReceipt( + { ...command, commandId: legacyLatest.commandId }, + legacyLatest, + ); + return { ...replay, cascadeReplayedCommandId: legacyLatest.commandId }; + } + const existingReceipt = yield* read( + cascadeThread === undefined + ? commandReceipts.getByCommandId(command.commandId) + : commandReceipts.getAnyByCommandId(command.commandId), + ); if (Option.isSome(existingReceipt)) { - return yield* replayCommandReceipt(command, existingReceipt.value); + const receipt = existingReceipt.value; + if (occupiedCascadeId(receipt)) + return { sequence: receipt.resultSequence, storedEvents: [], cascadeIdOccupied: true }; + if ("threadId" in receipt) return yield* replayCommandReceipt(command, receipt); } // Recheck under the lock; the user may have raised modes since sender preflight. const limit = yield* DispatchModeLimit; - if (limit !== undefined) { + if (cascadeThread === undefined && limit !== undefined) { // Fork and merge-back sources are unlocked; validate the modes copied by their dispatch. const threadId = commandThreadId(command); const shell = yield* projectionStore @@ -16129,6 +16484,35 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ); } + // Decided under the child's lock, so a newer independent choice on it always wins. + if ( + cascadeThread !== undefined && + cascade?.source !== undefined && + command.type !== "thread.delete" + ) { + const { source, path } = cascade; + const latest = + legacyLatest === undefined + ? yield* commandReceipts + .getLatestLifecycle(cascadeThread.id, ["thread.archive", "thread.unarchive"]) + .pipe(read) + : legacyLatest; + const applies = + cascadeThread.deletedAt === null && + (command.type === "thread.archive" + ? cascadeThread.archivedAt === null && + !supersedesCascade(latest, source.cutoff, command.commandId) + : cascadeThread.archivedAt !== null && + (yield* archivedByCascade( + commandReceipts, + latest, + source.threadId, + path, + source.archiveCommandId, + ).pipe(read))); + if (!applies) return { sequence: source.cutoff, storedEvents: [] }; + } + const plan = yield* dispatchOnce(command, sideChatIds).pipe( Effect.tap((planned) => refuseAboveRuntimeModeCeiling(command, planned.events)), Effect.filterOrFail( @@ -16274,6 +16658,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }).pipe(mapDispatchError(command)), ], effects: plan.effects, + ...(command.type === "thread.archive" + ? { archiveSourceCommandId: cascade?.parentArchiveCommandId ?? command.commandId } + : {}), ...(plan.cancelUnsettledEffects === undefined ? {} : { cancelUnsettledEffects: plan.cancelUnsettledEffects }), @@ -16292,6 +16679,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ); + if (!committed.committed && occupiedCascadeId(committed.receipt)) + return { + sequence: committed.receipt.resultSequence, + storedEvents: [], + cascadeIdOccupied: true, + }; + if ("projectId" in committed.receipt) + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "Conflicting receipt is not a thread command receipt.", + }); if (committed.receipt.status === "rejected") { return yield* new OrchestratorCommandPreviouslyRejectedError({ commandId: command.commandId, @@ -16309,6 +16708,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio commandThreadId: commandThreadId(command), }); } + if (!committed.committed && committed.receipt.commandType !== command.type) + yield* rejectConflictingSideChatReceipt(command, committed.receipt); if ( command.type === "queue.resume" || command.type === "queued-run.start" || @@ -16348,9 +16749,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ); } + const subagentCascades = + committed.committed && + (command.type === "thread.archive" || + command.type === "thread.unarchive" || + command.type === "thread.delete") + ? plan.effects.filter( + (effect): effect is EffectOutbox.SubagentCascadeEffect => + effect.request.type === "subagent-threads.cascade", + ) + : []; return { sequence: committed.receipt.resultSequence, storedEvents: committed.storedEvents, + ...(subagentCascades.length === 0 ? {} : { subagentCascades }), ...(committed.committed ? {} : { replayed: true as const }), } satisfies OrchestratorV2DispatchResult; }, withCommandMetrics); @@ -16400,7 +16812,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio ), ), ); - return yield* command.type === "thread.delete" + // A subagent cascade already owns the deletion lock outside its ancestor thread locks. + return yield* command.type === "thread.delete" && + (yield* EffectOutbox.SubagentCascade) === undefined ? threadDispatch.withDeletionLock(cascade) : cascade; } @@ -17042,6 +17456,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))); }, ), + // The projection read owns the discovery span. + getSubagentChildThreads: Effect.fnUntraced(function* (threadId: ThreadId) { + return yield* projectionStore + .getSubagentChildThreads(threadId) + .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))); + }), getThreadProjection: Effect.fn("OrchestratorV2.getThreadProjection")(function (threadId) { return projectionStore .getThreadProjection(threadId) @@ -17133,6 +17553,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .latestSequence({ threadId }) .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))); }), + hasRecordedThreadDeletion: Effect.fn("OrchestratorV2.hasRecordedThreadDeletion")(function ( + commandId: CommandId, + threadId: ThreadId, + ) { + return eventSink.readByCommandId({ commandId }).pipe( + Stream.filter( + ({ event }) => event.type === "thread.deleted" && event.threadId === threadId, + ), + Stream.runHead, + Effect.map(Option.isSome), + Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })), + ); + }), streamStoredEvents: eventSink.stream().pipe( Stream.mapError( (cause) => diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index ba91ba94d2520..7525cc0883ed8 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -670,6 +670,10 @@ export interface ProjectionStoreV2Shape { readonly findThread: ( threadId: ThreadId, ) => Effect.Effect; + /** Direct subagents, archived and deleted included. */ + readonly getSubagentChildThreads: ( + threadId: ThreadId, + ) => Effect.Effect, ProjectionStoreV2Error>; readonly getLimitRecoveryCandidates: (options: { readonly now: DateTime.Utc; readonly autoResume: boolean; @@ -5992,6 +5996,22 @@ export const layer: Layer.Layer = const findThread: ProjectionStoreV2Shape["findThread"] = Effect.fn( "ProjectionStore.findThread", )(selectThread); + // Subagents inherit the parent's immutable project. Use its index to bound the scan. + const getSubagentChildThreads: ProjectionStoreV2Shape["getSubagentChildThreads"] = Effect.fn( + "OrchestratorV2.getSubagentChildThreads", + )((threadId) => + sql` + SELECT child.payload_json FROM orchestration_v2_projection_threads AS child + JOIN orchestration_v2_projection_threads AS parent + ON parent.thread_id = ${threadId} AND child.project_id = parent.project_id + WHERE json_extract(child.payload_json, '$.lineage.parentThreadId') = ${threadId} + AND json_extract(child.payload_json, '$.lineage.relationshipToParent') = 'subagent' + ORDER BY child.created_at ASC, child.thread_id ASC + `.pipe( + Effect.mapError((cause) => new ProjectionStoreReadError({ threadId, cause })), + Effect.flatMap(decodeRows(decodeThreadPayload, threadId)), + ), + ); const requireThread = Effect.fn("ProjectionStore.requireThread")(function* ( threadId: ThreadId, @@ -9842,6 +9862,7 @@ export const layer: Layer.Layer = getThreadLinkMetadata: readOnReader(getThreadLinkMetadata), getThreadWorktreePaths: readOnReader(getThreadWorktreePaths), getOpenRemoteDelegatedTasks: getOpenRemoteDelegatedTasks.pipe(NodeSqliteClient.readOnly), + getSubagentChildThreads: readOnReader(getSubagentChildThreads), getThreadProjection: readOnReader(getThreadProjection), getTurnStartContext: readOnReader(getTurnStartContext), getTurnStartHistory: readOnReader(getTurnStartHistory), diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts index 51c38efa8a41a..8a84f64f76aa1 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts @@ -147,6 +147,7 @@ function layerProjectionStore( getThreadShell: () => Effect.die("unused getThreadShell"), getThreadTitles: () => Effect.die("unused getThreadTitles"), getThreadLinkMetadata: () => Effect.die("unused getThreadLinkMetadata"), + getSubagentChildThreads: () => Effect.die("unused getSubagentChildThreads"), findGuestShareThreadId: () => Effect.die("unused findGuestShareThreadId"), getThread: () => Effect.die("unused getThread"), findThread: () => Effect.die("unused findThread"), diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 45b5e9effa8cf..19eb0a6010408 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -60,6 +60,7 @@ import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; +import * as EffectOutbox from "./EffectOutbox.ts"; import * as CommandReceiptStore from "./CommandReceiptStore.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; @@ -552,6 +553,17 @@ export interface ThreadManagementServiceShape { void, Orchestrator.OrchestratorV2Error | ThreadManagementProviderInstanceDisabledError >; + readonly cascadeSubagentLifecycle: (input: { + readonly threadId: ThreadId; + readonly commandId: CommandId; + readonly operation: "thread.archive" | "thread.unarchive" | "thread.delete"; + readonly sideChat?: EffectOutbox.EmbeddedSideChatCascade | undefined; + readonly directDelete?: true | undefined; + readonly stableSkipPrefix?: true | undefined; + }) => Effect.Effect< + void, + Orchestrator.OrchestratorV2Error | CommandReceiptStore.CommandReceiptStoreV2Error + >; readonly getThreadEventSequence: Orchestrator.OrchestratorV2["Service"]["getThreadEventSequence"]; readonly recoverDelegatedTask: Orchestrator.OrchestratorV2["Service"]["recoverDelegatedTask"]; readonly cancelDelegatedTask: Orchestrator.OrchestratorV2["Service"]["cancelDelegatedTask"]; @@ -844,7 +856,35 @@ const make = Effect.gen(function* () { ensureCommandTranscripts(command).pipe( Effect.andThen(assertDispatchTargetEnabled(command)), Effect.andThen(orchestrator.dispatch(command)), + Effect.tap((result) => cascadeBeforeReply(result)), + ); + + // Clients read archive snapshots as soon as this returns; the outbox effect only backs it up. + // A replay or a no-op (an unused-only delete that kept the thread) leaves it to that effect. + const cascadeBeforeReply = Effect.fn("ThreadManagementService.cascadeBeforeReply")(function* ( + result: Orchestrator.OrchestratorV2DispatchResult, + ) { + yield* Effect.forEach( + result.subagentCascades ?? [], + ({ commandId, threadId, request }) => + cascadeSubagentLifecycle({ + commandId, + threadId, + operation: request.operation, + sideChat: request.sideChat, + directDelete: request.directDelete, + stableSkipPrefix: request.stableSkipPrefix, + }).pipe( + Effect.catch((cause) => + Effect.logWarning("Subagent lifecycle cascade left to the effect worker", { + threadId, + cause, + }), + ), + ), + { discard: true }, ); + }); const getProjectThreadRecords: ThreadManagementServiceShape["getProjectThreadRecords"] = ( input, @@ -1774,6 +1814,201 @@ const make = Effect.gen(function* () { if (failures[0] !== undefined) return yield* failures[0]; }); + // An older embedded delete can share its derived ID with an accepted unused-only close. + const legacySideChatDeletionParent = Effect.fn( + "ThreadManagementService.legacySideChatDeletionParent", + )(function* (commandId: CommandId, threadId: ThreadId) { + const suffix = `:side-chat:${threadId}`; + if (commandId.length <= suffix.length || !commandId.endsWith(suffix)) return; + const parentCommandId = CommandId.make(commandId.slice(0, -suffix.length)); + const parent = Option.getOrUndefined(yield* receipts.getByCommandId(parentCommandId)); + if ( + parent?.status !== "accepted" || + parent.commandType !== "thread.delete" || + parent.threadId === threadId + ) + return; + const direct = Option.getOrUndefined(yield* receipts.getByCommandId(commandId)); + if ( + direct?.threadId === threadId && + direct.commandType === "thread.delete" && + direct.status === "accepted" + ) { + if (yield* orchestrator.hasRecordedThreadDeletion(commandId, threadId)) return; + } + return parent.threadId; + }); + + const cascadeSubagentLifecycle: ThreadManagementServiceShape["cascadeSubagentLifecycle"] = + Effect.fn("ThreadManagementService.cascadeSubagentLifecycle")(function* (input) { + const { sideChat } = input; + if (input.operation === "thread.unarchive" && sideChat && !sideChat.archiveCommandId) return; + const parentThreadId = + sideChat?.parentThreadId ?? + (input.operation === "thread.delete" && !input.directDelete + ? yield* legacySideChatDeletionParent(input.commandId, input.threadId) + : undefined); + const receipt = + input.operation === "thread.delete" + ? undefined + : Option.getOrUndefined(yield* receipts.getByCommandId(input.commandId)); + if (input.operation !== "thread.delete" && receipt?.status !== "accepted") return; + const source = receipt && { + threadId: receipt.threadId, + cutoff: sideChat?.cutoff ?? receipt.resultSequence, + archiveCommandId: sideChat?.archiveCommandId, + }; + type LegacyPath = { + readonly anchorId: CommandId; + readonly depth: number; + readonly verified?: true; + }; + const dispatchChild = Effect.fn("ThreadManagementService.dispatchCascadeChild")(function* ( + threadId: ThreadId, + baseId: CommandId, + path: ReadonlyArray, + parentArchiveCommandId: CommandId, + legacyPath: LegacyPath | undefined, + ) { + for (let retry = 0; retry < CommandReceiptStore.cascadeCommandIdLimit; retry++) { + const commandId = CommandReceiptStore.cascadeCommandIdAt(baseId, retry); + const result = yield* orchestrator + .dispatch({ type: input.operation, commandId, threadId }) + .pipe( + Effect.tap(cascadeBeforeReply), + Effect.provideService(EffectOutbox.SubagentCascade, { + source, + path, + parentArchiveCommandId, + legacyPath, + }), + Effect.catchTags({ + OrchestratorThreadAboveModeLimitError: () => Effect.undefined, + }), + ); + if (result === undefined) return; + if (!result.cascadeIdOccupied) + return result.replayed || result.storedEvents.length > 0 + ? { + commandId: result.cascadeReplayedCommandId ?? commandId, + verified: result.cascadeReplayedCommandId !== undefined, + } + : null; + } + return yield* new Orchestrator.OrchestratorDispatchError({ + commandId: baseId, + commandType: input.operation, + cause: "All 32 subagent lifecycle command IDs were occupied or rejected.", + }); + }); + + const visit: ( + thread: { + readonly id: ThreadId; + readonly deletedAt: unknown; + readonly projectId?: unknown; + }, + parentCommandId: CommandId, + path: ReadonlyArray, + parentArchiveCommandId: CommandId, + legacyPath?: LegacyPath, + ) => ReturnType = Effect.fn( + "ThreadManagementService.visitSubagentCascade", + )(function* (thread, parentCommandId, path, parentArchiveCommandId, legacyPath) { + const baseId = + path.length === 0 + ? parentCommandId + : CommandId.make(`${parentCommandId}:subagent:${thread.id}`); + const dispatched = + path.length === 0 + ? { commandId: baseId, verified: false } + : thread.deletedAt === null + ? yield* dispatchChild(thread.id, baseId, path, parentArchiveCommandId, legacyPath) + : null; + if (dispatched === undefined) return; + const ownCommandId = dispatched?.commandId ?? null; + // Holds each thread's lock while visiting its subagents, so locks nest parent before child. + yield* threadDispatch.withLock( + thread.id, + Effect.gen(function* () { + // A deleted or moved thread, one above the caller's limit or a newer choice keeps its subtree. + const limit = yield* DispatchModeLimit; + const current = (yield* orchestrator.getThreadRecords(thread.id, [])).thread; + if (path.length > 0 && current.projectId !== thread.projectId) return; + if (path.length === 0 && parentThreadId !== undefined) { + if ( + current.lineage.sideChat !== true || + current.lineage.parentThreadId !== parentThreadId + ) + return; + const { thread: parent } = yield* orchestrator.getThreadRecords(parentThreadId, []); + if ( + current.projectId !== parent.projectId || + (sideChat && current.projectId !== sideChat.projectId) + ) + return; + } + if (limit && exceededDispatchModeLimit(limit, current)) return; + if (source && current.deletedAt !== null) return; + const latest = source + ? yield* receipts.getLatestLifecycle(thread.id, [ + "thread.archive", + "thread.unarchive", + ]) + : null; + if ( + source && + CommandReceiptStore.supersedesCascade( + latest, + source.cutoff, + ownCommandId ?? undefined, + ) + ) + return; + const archiveCommandId = + input.operation === "thread.archive" && + latest?.commandType === "thread.archive" && + latest.commandId === ownCommandId + ? ownCommandId + : parentArchiveCommandId; + const children = yield* orchestrator.getSubagentChildThreads(thread.id); + if (children.length === 0) return; + // Older jobs may have committed descendants under a suffixed skip. + const legacyPrefix = + ownCommandId === null && !input.stableSkipPrefix + ? yield* CommandReceiptStore.legacyCascadeCommandId(receipts, baseId) + : baseId; + const prefix = ownCommandId ?? legacyPrefix ?? baseId; + const childLegacyPath = dispatched?.verified + ? { anchorId: prefix, depth: path.length, verified: true as const } + : ownCommandId === null + ? (legacyPath ?? + (source && legacyPrefix !== baseId + ? { anchorId: parentCommandId, depth: path.length - 1 } + : undefined)) + : undefined; + const [, failures] = yield* Effect.partition(children, (child) => + visit(child, prefix, [...path, child.id], archiveCommandId, childLegacyPath), + ); + // Finish healthy siblings before reporting a failure to the durable worker. + if (failures[0] !== undefined) return yield* failures[0]; + }), + ); + }); + const walk = visit( + { id: input.threadId, deletedAt: null }, + input.commandId, + [], + input.commandId, + ); + const cascade = + parentThreadId === undefined ? walk : threadDispatch.withLock(parentThreadId, walk); + yield* input.operation === "thread.delete" && + (yield* EffectOutbox.SubagentCascade) === undefined + ? threadDispatch.withDeletionLock(cascade) + : cascade; + }); + return ThreadManagementService.of({ ensureLegacyTranscript, searchThreadStream: (input) => @@ -1874,6 +2109,7 @@ const make = Effect.gen(function* () { settleThread, interruptThread, stopDelegatedTasks, + cascadeSubagentLifecycle, stopWorkflow, forceStopThread, awaitBackgroundTasks, diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index c899eb774bfc3..7b0af603b26cb 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -6,7 +6,7 @@ import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderS import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { assert, it } from "@effect/vitest"; +import { assert, describe, it } from "@effect/vitest"; import { vi } from "vite-plus/test"; import { type ApplicationStoredEvent, @@ -65,6 +65,8 @@ import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore import * as ProjectEnrichmentService from "../project/ProjectEnrichmentService.ts"; import * as ProjectService from "../project/ProjectService.ts"; import * as ServerSettings from "../serverSettings.ts"; +import * as Receipts from "./CommandReceiptStore.ts"; +import { DispatchModeLimit } from "./DispatchModeLimit.ts"; import * as McpSessionRegistryTestkit from "../mcp/McpSessionRegistry.testkit.ts"; import * as ModelManifest from "../provider/ModelManifest.ts"; import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts"; @@ -357,6 +359,7 @@ const layerTest = Layer.mergeAll( ProjectStore.layer, ProjectionStore.layer, EffectOutbox.layer, + Receipts.layer, ThreadCommandExecutor.layer, ).pipe( Layer.provide(McpSessionRegistryTestkit.layer), @@ -14112,3 +14115,1129 @@ it.layer(layerQueueResumeDisabledInstanceTest)( ); }, ); + +describe("subagent lifecycle cascade", () => { + const parent = ThreadId.make("parent"); + const app = ThreadId.make("app"); + const native = ThreadId.make("native"); + const grandchild = ThreadId.make("grandchild:retry:31"); + const fixture = Effect.gen(function* () { + const threads = yield* ThreadManagementService.ThreadManagementService; + const orchestrator = yield* Orchestrator.OrchestratorV2; + const store = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + yield* threads.dispatch({ + type: "thread.create", + commandId: CommandId.make("create:parent"), + threadId: parent, + projectId: ProjectId.make("project"), + title: "Parent", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "agent", + creationSource: "mcp", + }); + const base = yield* store.getThread(parent); + const set = Effect.fn("test.seed")(function* (id: ThreadId, p: Partial) { + const thread = yield* store.getThread(id).pipe(Effect.orElseSucceed(() => base)); + const event = { id: EventId.make(`seed:${id}`), threadId: id, occurredAt: now }; + yield* store.apply({ ...event, type: "thread.created", payload: { ...thread, id, ...p } }); + }); + for (const [id, parentThreadId, relationshipToParent, projectId] of [ + [app, parent, "subagent", base.projectId], + [native, parent, "subagent", base.projectId], + [grandchild, app, "subagent", base.projectId], + [ThreadId.make("fork"), parent, "fork", base.projectId], + [ThreadId.make("fork-child"), ThreadId.make("fork"), "subagent", base.projectId], + [ThreadId.make("foreign"), parent, "subagent", ProjectId.make("foreign")], + ] as const) { + const lineage = { rootThreadId: parent, parentThreadId, relationshipToParent }; + yield* set(id, { lineage, projectId, creationSource: id === native ? "provider" : "mcp" }); + } + const command = ( + operation: "archive" | "unarchive" | "delete", + threadId: ThreadId, + id?: string, + ) => + ({ + type: `thread.${operation}`, + commandId: CommandId.make(id ?? `${operation}:${threadId}`), + threadId, + }) as const; + const run = (operation: "archive" | "unarchive" | "delete", threadId = parent, id?: string) => + threads.dispatch(command(operation, threadId, id)); + // Committed by a process that stopped before its reply, leaving the cascade to the worker. + const commit = ( + operation: "archive" | "unarchive" | "delete", + threadId = parent, + id?: string, + ) => orchestrator.dispatch(command(operation, threadId, id)); + const state = (field: "archivedAt" | "deletedAt", ...ids: ReadonlyArray) => + Effect.forEach(ids, (id) => store.getThread(ThreadId.make(id))).pipe( + Effect.map((all) => all.map((thread) => (thread[field] === null ? 0 : 1)).join("")), + ); + const archived = (...ids: ReadonlyArray) => state("archivedAt", ...ids); + const outbox = yield* EffectOutbox.EffectOutboxV2; + const cascade = Effect.map( + outbox.get("effect:archive:parent:subagent-threads.cascade"), + Option.getOrThrow, + ); + const drain = (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); + const sideChat = Effect.fn("test.seedSideChat")(function* (source: ThreadId, id: ThreadId) { + yield* set(id, { + lineage: { + rootThreadId: parent, + parentThreadId: source, + relationshipToParent: "fork", + sideChat: true, + }, + }); + yield* store.apply({ + id: EventId.make(`fork:${id}`), + type: "context-transfer.created", + threadId: source, + occurredAt: now, + payload: { + id: ContextTransferId.make(`fork:${id}`), + type: "fork", + sourceThreadId: source, + targetThreadId: id, + sourcePoint: { threadId: source }, + basePoint: null, + sourceProviderInstanceId: null, + targetProviderInstanceId: null, + targetRunId: null, + status: "consumed", + resolution: null, + createdBy: "user", + error: null, + createdAt: now, + updatedAt: now, + consumedAt: now, + }, + }); + }); + const markUsed = (threadId: ThreadId) => + store.apply({ + id: EventId.make(`use:${threadId}`), + type: "message.updated", + threadId, + occurredAt: now, + payload: { + id: MessageId.make(`use:${threadId}`), + threadId, + runId: null, + nodeId: null, + role: "user", + text: "This thread has a message.", + attachments: [], + streaming: false, + createdBy: "user", + creationSource: "web", + createdAt: now, + updatedAt: now, + }, + }); + const renameAndKeep = Effect.fn("test.renameAndKeep")(function* ( + threadId: ThreadId, + lifecycle: string, + ) { + yield* threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make(`rename:${lifecycle}:${threadId}`), + threadId, + title: `Renamed after ${lifecycle}`, + }); + const kept = yield* threads.dispatch({ + type: "thread.delete", + commandId: CommandId.make(`unused-only:${lifecycle}:${threadId}`), + threadId, + onlyIfUnused: true, + }); + assert.isEmpty(kept.storedEvents); + }); + const sideChatWithChild = Effect.fn("test.sideChatWithChild")(function* ( + projectId = base.projectId, + ) { + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + yield* sideChat(parent, side); + yield* set(side, { projectId }); + yield* set(child, { + projectId, + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + return { side, child }; + }); + return { + threads, + now, + set, + sideChat, + run, + commit, + state, + archived, + outbox, + cascade, + drain, + markUsed, + renameAndKeep, + sideChatWithChild, + }; + }); + const receipt = (id: string) => + Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.getByCommandId(CommandId.make(id))); + it.effect("archives, restores and deletes subagents recursively, but not forks or projects", () => + Effect.gen(function* () { + const f = yield* fixture; + const orchestrator = yield* Orchestrator.OrchestratorV2; + const store = yield* ProjectionStore.ProjectionStoreV2; + // "foreign" moved to another project, with its own subagent, after the walk found it. + const { projectId } = yield* store.getThread(parent); + const foreign = ThreadId.make("foreign"); + yield* f.set(ThreadId.make("foreign-child"), { + lineage: { + rootThreadId: parent, + parentThreadId: foreign, + relationshipToParent: "subagent", + }, + projectId: ProjectId.make("foreign"), + }); + const children = orchestrator.getSubagentChildThreads; + const stale = store.getThread(foreign).pipe( + Effect.orDie, + Effect.map((thread) => ({ ...thread, projectId })), + ); + using _stale = vi + .spyOn(orchestrator, "getSubagentChildThreads") + .mockImplementation((id) => + id === parent ? Effect.zipWith(children(id), stale, (c, t) => [...c, t]) : children(id), + ); + const side = ThreadId.make("side"); + const sideChild = ThreadId.make("side-child"); + yield* f.sideChat(native, side); + yield* f.set(sideChild, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + const all = [ + native, + app, + grandchild, + "fork", + "fork-child", + "foreign", + "foreign-child", + side, + sideChild, + ]; + for (const operation of ["archive", "unarchive", "delete"] as const) { + // A deleted intermediate must not hide surviving descendants from deletion. + if (operation === "delete") yield* f.set(app, { deletedAt: f.now }); + // Settled before the reply, without the effect worker, so a client refresh sees it. + yield* f.run(operation); + const field = operation === "delete" ? "deletedAt" : "archivedAt"; + assert.equal( + yield* f.state(field, ...all), + operation === "unarchive" ? "000000000" : "111000011", + ); + const effects = yield* f.outbox.listByCommandId( + CommandId.make(`${operation}:parent:subagent:native`), + ); + if (operation !== "unarchive") + assert.isTrue(effects.some((e) => e.request.type === "terminal.cleanup")); + } + }).pipe(Effect.provide(layerTest)), + ); + it.effect("holds the deletion lock before every cascade thread lock", () => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const sideChild = ThreadId.make("side-child"); + yield* f.sideChat(app, side); + yield* f.set(sideChild, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor; + const { withLock, withDeletionLock } = executor; + const held = new Set(); + let deleting = false; + const lock: typeof withLock = (id, effect) => + withLock( + id, + Effect.acquireUseRelease( + Effect.sync(() => { + assert.isTrue(deleting, "a cascade thread lock requires the deletion lock"); + held.add(id); + }), + () => effect, + () => Effect.sync(() => held.delete(id)), + ), + ); + const deletion: typeof withDeletionLock = (effect) => + Effect.suspend(() => { + assert.equal(held.size, 0, "the deletion lock precedes all ancestor thread locks"); + assert.isFalse(deleting, "the deletion lock must not be acquired recursively"); + return withDeletionLock( + Effect.acquireUseRelease( + Effect.sync(() => { + deleting = true; + }), + () => effect, + () => + Effect.sync(() => { + deleting = false; + }), + ), + ); + }); + using _lock = vi.spyOn(executor, "withLock").mockImplementation(lock); + using _deletion = vi.spyOn(executor, "withDeletionLock").mockImplementation(deletion); + yield* f.run("delete"); + yield* f.drain; + assert.equal( + yield* f.state("deletedAt", parent, app, native, grandchild, side, sideChild), + "111111", + ); + }).pipe(Effect.provide(layerTest)), + ); + it.effect("recovers an embedded side-chat cascade and restores only its own archives", () => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + const independent = ThreadId.make("side-independent"); + const restored = ThreadId.make("side-restored"); + yield* f.sideChat(app, side); + for (const id of [child, independent, restored]) { + yield* f.set(id, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + } + yield* f.run("archive", independent); + const latestSide = ThreadId.make("side-restored-later"); + const latestChild = ThreadId.make("side-restored-later-child"); + yield* f.sideChat(app, latestSide); + yield* f.set(latestChild, { + lineage: { + rootThreadId: parent, + parentThreadId: latestSide, + relationshipToParent: "subagent", + }, + }); + yield* f.run("archive", latestSide); + yield* f.run("unarchive", latestChild); + yield* f.run("archive", restored); + yield* f.commit("archive"); + yield* f.run("unarchive", restored); + yield* f.run("unarchive", latestSide); + yield* f.drain; + assert.equal(yield* f.archived(side, child, independent, restored, latestChild), "11100"); + yield* TestClock.adjust("8 days"); + assert.isAbove(yield* f.outbox.pruneSettled, 0); + const generatedId = "archive:parent:subagent:app:side-chat:side"; + const generated = Option.getOrThrow(yield* receipt(generatedId)); + assert.isTrue( + Option.isNone(yield* f.outbox.get(`effect:${generatedId}:subagent-threads.cascade`)), + ); + const replay = yield* f.commit("archive", side, generatedId); + assert.isTrue(replay.replayed); + assert.equal(replay.sequence, generated.resultSequence); + const refused = yield* f.commit("delete", side, generatedId).pipe(Effect.flip); + assert.equal(refused._tag, "OrchestratorCommandIdConflictError"); + assert.deepEqual(Option.getOrThrow(yield* receipt(generatedId)), generated); + assert.equal(yield* f.state("deletedAt", side, child), "00"); + yield* f.commit("unarchive"); + yield* f.markUsed(app); + yield* f.renameAndKeep(app, "before-delivery"); + yield* f.drain; + assert.equal(yield* f.archived(app, side, child, independent, restored), "00010"); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect("rolls back a parent restore whose side-chat receipt ID is occupied", () => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.sideChatWithChild(); + yield* f.run("archive"); + const commandId = CommandId.make("unarchive:parent"); + const occupied = { + commandId: CommandId.make("unarchive:parent:side-chat:side"), + threadId: ThreadId.make("fork"), + commandType: "thread.archive", + acceptedAt: f.now, + resultSequence: 0, + status: "rejected", + error: "Earlier rejected command.", + } as const; + yield* Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.upsert(occupied)); + const events = yield* EventSink.EventSinkV2; + const sequence = yield* events.latestSequence(); + const denied = yield* f.run("unarchive").pipe(Effect.flip); + assert.equal(yield* events.latestSequence(), sequence); + assert.equal(denied._tag, "OrchestratorDispatchError"); + assert.equal(yield* f.archived(parent, side, child), "111"); + assert.deepEqual(Option.getOrThrow(yield* receipt(occupied.commandId)), occupied); + assert.isTrue(Option.isNone(yield* receipt(commandId))); + assert.isTrue( + Option.isNone( + yield* f.outbox.get("effect:unarchive:parent:side-chat:side:subagent-threads.cascade"), + ), + ); + }).pipe(Effect.provide(layerTest)), + ); + it.effect( + "keeps the descendant archived during a delayed restore after a newer side-chat archive", + () => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + yield* f.sideChat(app, side); + yield* f.set(child, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + yield* f.run("archive"); + yield* f.drain; + assert.equal(yield* f.archived(parent, app, side, child), "1111"); + yield* f.commit("unarchive"); + yield* f.commit("unarchive", side); + yield* TestClock.adjust("1 second"); + yield* f.commit("archive", side); + yield* f.drain; + assert.equal(yield* f.archived(parent, app, side, child), "0001"); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect("keeps the child archived when its side chat rearchives with a parent-derived ID", () => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.sideChatWithChild(); + yield* f.run("archive", side, "earlier:side-archive"); + yield* f.run("archive"); + yield* f.run("unarchive", side); + yield* TestClock.adjust("1 second"); + yield* f.run("archive", side, "archive:parent:side-chat:side"); + yield* f.drain; + yield* TestClock.adjust("8 days"); + yield* f.outbox.pruneSettled; + const occupied = { + commandId: CommandId.make("unarchive:parent:side-chat:side"), + threadId: ThreadId.make("fork"), + commandType: "thread.archive", + acceptedAt: f.now, + resultSequence: 0, + status: "accepted", + error: null, + } as const; + yield* Effect.flatMap(Receipts.CommandReceiptStoreV2, (r) => r.upsert(occupied)); + yield* f.commit("unarchive"); + yield* f.drain; + assert.equal(yield* f.archived(parent, side, child), "001"); + assert.deepEqual(Option.getOrThrow(yield* receipt(occupied.commandId)), occupied); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect("leaves a moved side chat's descendants alone during delete and legacy recovery", () => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.sideChatWithChild(ProjectId.make("foreign")); + const archiveId = CommandId.make("archive:parent:side-chat:side"); + const fork = ThreadId.make("fork"); + const receipts = yield* Receipts.CommandReceiptStoreV2; + const created = Option.getOrThrow(yield* receipt("create:parent")); + const occupied = { ...created, commandId: archiveId, threadId: fork }; + yield* receipts.upsert(occupied); + yield* f.run("archive"); + yield* f.drain; + assert.equal(yield* f.archived(parent, side, child), "110"); + assert.deepEqual(Option.getOrThrow(yield* receipt(archiveId)), occupied); + + const commandId = CommandId.make("delete:parent:side-chat:side"); + yield* f.markUsed(side); + const kept = yield* f.threads.dispatch({ + type: "thread.delete", + commandId, + threadId: side, + onlyIfUnused: true, + }); + assert.isEmpty(kept.storedEvents); + yield* f.commit("delete"); + yield* f.drain; + assert.equal(yield* f.state("deletedAt", parent, side, child), "110"); + yield* f.markUsed(fork); + const closeId = CommandId.make("close:fork"); + yield* f.threads.dispatch({ + type: "thread.delete", + commandId: closeId, + threadId: fork, + onlyIfUnused: true, + }); + const directId = CommandId.make(`${closeId}:side-chat:${side}`); + yield* f.commit("delete", side, directId); + const sql = yield* SqlClient.SqlClient; + const pruned = yield* sql` + DELETE FROM orchestration_events + WHERE command_id = 'delete:parent' AND stream_id = ${side} AND event_type = 'thread.deleted' + RETURNING sequence + `; + assert.lengthOf(pruned, 1); + yield* f.threads.cascadeSubagentLifecycle({ + threadId: side, + commandId, + operation: "thread.delete", + }); + assert.equal(yield* f.state("deletedAt", parent, side, child), "110"); + yield* f.threads.cascadeSubagentLifecycle({ + threadId: side, + commandId: directId, + operation: "thread.delete", + }); + assert.equal(yield* f.state("deletedAt", fork, side, child), "011"); + }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), + ); + + it.effect("stops a committed archive cascade at a side chat moved before delivery", () => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.sideChatWithChild(); + yield* f.commit("archive"); + const projectId = ProjectId.make("foreign"); + yield* f.set(side, { projectId }); + yield* f.set(child, { projectId }); + yield* f.drain; + assert.equal(yield* f.archived(parent, side, child), "110"); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect("recovers a direct side-chat deletion after its delete event is pruned", () => + Effect.gen(function* () { + const f = yield* fixture; + const { side, child } = yield* f.sideChatWithChild(ProjectId.make("foreign")); + yield* f.markUsed(parent); + const kept = yield* f.threads.dispatch({ + type: "thread.delete", + commandId: CommandId.make("delete:parent"), + threadId: parent, + onlyIfUnused: true, + }); + assert.isEmpty(kept.storedEvents); + assert.equal(yield* f.state("deletedAt", parent, app, native, grandchild), "0000"); + + const store = yield* ProjectionStore.ProjectionStoreV2; + yield* f.markUsed(side); + const runId = RunId.make("side:queued"); + yield* store.apply({ + id: EventId.make("side:queued"), + type: "run.created", + threadId: side, + runId, + occurredAt: f.now, + payload: { + id: runId, + threadId: side, + ordinal: 1, + providerInstanceId: modelSelection.instanceId, + modelSelection, + providerThreadId: null, + userMessageId: MessageId.make(`use:${side}`), + rootNodeId: null, + activeAttemptId: null, + status: "queued", + requestedAt: f.now, + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }, + }); + const commandId = CommandId.make("delete:parent:side-chat:side"); + const deleted = yield* f.commit("delete", side, commandId); + assert.equal(deleted.storedEvents.at(-1)?.event.type, "run.updated"); + const sql = yield* SqlClient.SqlClient; + const pruned = yield* sql` + DELETE FROM orchestration_events + WHERE command_id = ${commandId} + AND stream_id = ${side} + AND event_type = 'thread.deleted' + RETURNING sequence + `; + assert.lengthOf(pruned, 1); + yield* f.drain; + assert.equal(yield* f.state("deletedAt", parent, side, child), "011"); + }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), + ); + + it.effect("rolls back direct archive and nested delete when their recovery ID is occupied", () => + Effect.gen(function* () { + const f = yield* fixture; + const side = ThreadId.make("side"); + const child = ThreadId.make("side-child"); + yield* f.sideChat(app, side); + yield* f.set(child, { + lineage: { rootThreadId: parent, parentThreadId: side, relationshipToParent: "subagent" }, + }); + const nestedId = "delete:parent:subagent:app"; + const effectId = `effect:${nestedId}:side-chat:${side}:subagent-threads.cascade`; + yield* f.outbox.enqueue([ + { + id: effectId, + commandId: CommandId.make("earlier"), + threadId: ThreadId.make("fork"), + request: { type: "terminal.cleanup" }, + availableAt: DateTime.add(f.now, { days: 1 }), + }, + ]); + const occupied = yield* f.outbox.get(effectId); + const directId = `${nestedId}:side-chat:${side}`; + const fork = ThreadId.make("fork"); + const refused = yield* f.commit("archive", fork, directId).pipe(Effect.flip); + assert.equal(refused._tag, "OrchestratorDispatchError"); + assert.equal(yield* f.archived(fork, ThreadId.make("fork-child")), "00"); + assert.isTrue(Option.isNone(yield* receipt(directId))); + yield* f.commit("delete"); + yield* f.drain; + assert.equal(yield* f.state("deletedAt", parent, app, side, child), "1000"); + assert.isTrue(Option.isNone(yield* receipt(nestedId))); + assert.deepEqual(yield* f.outbox.get(effectId), occupied); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect("restores only what a parent archive cascaded, through independent archives", () => + Effect.gen(function* () { + const f = yield* fixture; + // The newer direct archive has the same ID the parent cascade would derive. + yield* f.commit("archive"); + yield* f.run("archive", app, "archive:parent:subagent:app"); + yield* f.drain; + const receipts = yield* Receipts.CommandReceiptStoreV2; + const { archiveSourceCommandId: _source, ...legacy } = Option.getOrThrow( + yield* receipt("archive:parent:subagent:native"), + ); + yield* receipts.upsert(legacy); + yield* f.run("unarchive"); + yield* f.drain; + assert.equal(yield* f.archived(app, grandchild, native), "110"); + yield* f.run("unarchive", grandchild); + yield* f.run("archive", parent, "archive-owned"); + yield* f.drain; + yield* f.run("unarchive", native); + // An unused retry-shaped user ID is not a cascade retry. + yield* f.run("archive", native, "archive-owned:subagent:native:retry:31"); + yield* f.run("unarchive", parent, "unarchive-superseded"); + yield* f.run("archive", parent, "archive-superseding"); + yield* f.drain; + yield* TestClock.adjust("8 days"); + assert.isAbove(yield* f.outbox.pruneSettled, 0); + yield* f.run("unarchive", parent, "unarchive-latest"); + yield* f.drain; + assert.equal(yield* f.archived(app, native, grandchild), "110"); + }).pipe(Effect.provide(layerTest)), + ); + it.effect("skips a superseded parent action and keeps descendant choices made after it", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.commit("archive"); + // Newer choices whose IDs only look like this cascade's still win. + yield* f.commit("unarchive", parent, "archive:parent:subagent:parent"); + yield* f.drain; + assert.isTrue(Option.isNone(yield* receipt("archive:parent:subagent:app"))); + yield* f.commit("archive", app); + yield* f.drain; + yield* f.commit("unarchive", grandchild); + yield* f.commit("archive", parent, "archive-later"); + yield* f.commit("unarchive", app, "archive-later:subagent:app:retry:31"); + yield* f.drain; + assert.equal(yield* f.archived(parent, native, app, grandchild), "1100"); + yield* f.commit("unarchive", parent, "unarchive-later"); + yield* f.commit("archive", app, "archive-app-later"); + yield* f.drain; + assert.equal(yield* f.archived(parent, native, app, grandchild), "0011"); + yield* f.set(app, { deletedAt: f.now }); + for (const operation of ["archive", "unarchive"] as const) { + yield* f.set(grandchild, { archivedAt: operation === "archive" ? null : f.now }); + yield* f.commit(operation, parent, `${operation}-tombstone`).pipe(Effect.andThen(f.drain)); + assert.equal(yield* f.archived(grandchild), operation === "archive" ? "0" : "1"); + } + // A root tombstoned without its own receipt, like a side chat deleted with its parent. + yield* f.commit("archive", parent, "archive-deleted"); + yield* f.set(parent, { deletedAt: f.now }).pipe(Effect.andThen(f.drain)); + assert.isTrue(Option.isNone(yield* receipt("archive-deleted:subagent:native"))); + }).pipe(Effect.provide(layerTest)), + ); + it.effect("retries a child command when a project receipt claims its ID during commit", () => + Effect.gen(function* () { + const f = yield* fixture; + const commandId = CommandId.make("archive:parent:subagent:app"); + const receipts = yield* Receipts.CommandReceiptStoreV2; + const existing = Option.getOrThrow(yield* receipt("create:parent")); + const projectId = ProjectId.make("other-project"); + const sink = yield* EventSink.EventSinkV2; + const commit = sink.commitCommand; + using _collision = vi.spyOn(sink, "commitCommand").mockImplementation((input) => + input.commandId === commandId + ? receipts + .insertIfAbsent({ + commandId, + projectId, + commandType: "project.create", + acceptedAt: existing.acceptedAt, + resultSequence: existing.resultSequence, + status: "accepted", + error: null, + }) + .pipe(Effect.orDie, Effect.andThen(commit(input))) + : commit(input), + ); + yield* f.run("archive"); + assert.equal(yield* f.archived(app, grandchild, native), "111"); + assert.equal(Option.getOrThrow(yield* receipt(`${commandId}:retry:1`)).threadId, app); + assert.equal( + Option.getOrThrow(yield* receipts.getProjectByCommandId(commandId)).projectId, + projectId, + ); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect("deletes a used subagent whose earlier conditional close owns the derived ID", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.markUsed(native); + const conditional = { + type: "thread.delete", + commandId: CommandId.make("delete:parent:subagent:native"), + threadId: native, + onlyIfUnused: true, + } as const; + assert.isEmpty((yield* f.threads.dispatch(conditional)).storedEvents); + const original = Option.getOrThrow(yield* receipt(conditional.commandId)); + yield* f.run("delete"); + assert.equal(yield* f.state("deletedAt", parent, native), "11"); + assert.equal( + Option.getOrThrow(yield* receipt(`${conditional.commandId}:retry:1`)).threadId, + native, + ); + const replay = yield* f.threads.dispatch(conditional); + assert.isTrue(replay.replayed); + assert.isEmpty(replay.storedEvents); + assert.deepEqual(Option.getOrThrow(yield* receipt(conditional.commandId)), original); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect("retries a child restore when an earlier restore holds its derived ID", () => + Effect.gen(function* () { + const f = yield* fixture; + const id = "unarchive:parent:subagent:native"; + yield* f.run("archive", native); + yield* f.run("unarchive", native, id); + const original = yield* receipt(id); + yield* f.run("archive"); + yield* f.run("unarchive"); + assert.equal(yield* f.archived(parent, native), "00"); + assert.equal(Option.getOrThrow(yield* receipt(`${id}:retry:1`)).threadId, native); + assert.isTrue((yield* f.run("unarchive", native, id)).replayed); + assert.deepEqual(yield* receipt(id), original); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect.each([ + "fresh late rejection", + "legacy deeper rejection", + "legacy expired rejection", + "legacy planning move", + ] as const)("resumes an archive past a skipped intermediate with a %s", (scenario) => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.run("archive", app); + yield* f.run("unarchive", grandchild); + yield* f.drain; + const leaf = ThreadId.make("leaf"); + yield* f.set(leaf, { + ...(scenario === "legacy planning move" ? { projectId: ProjectId.make("foreign") } : {}), + lineage: { + rootThreadId: parent, + parentThreadId: scenario === "fresh late rejection" ? app : grandchild, + relationshipToParent: "subagent", + }, + }); + yield* f.threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make("rename:leaf"), + threadId: leaf, + title: "Leaf", + }); + const deepest = ThreadId.make("deepest"); + if (scenario === "legacy deeper rejection") { + yield* f.set(deepest, { + lineage: { + rootThreadId: parent, + parentThreadId: leaf, + relationshipToParent: "subagent", + }, + }); + yield* f.threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make("rename:deepest"), + threadId: deepest, + title: "Deepest", + }); + yield* f.threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make("witness:leaf"), + threadId: leaf, + title: "Leaf before archiving", + }); + } + yield* f.threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make("witness:grandchild"), + threadId: grandchild, + title: "Grandchild before archiving", + }); + const skippedId = CommandId.make("archive:parent:subagent:app"); + const receipts = yield* Receipts.CommandReceiptStoreV2; + const created = Option.getOrThrow(yield* receipt("create:parent")); + const collision = { + ...created, + commandId: skippedId, + threadId: app, + status: "rejected", + error: "Earlier rejection.", + } as const; + yield* f.commit("archive"); + if (scenario !== "fresh late rejection") { + const sql = yield* SqlClient.SqlClient; + yield* sql` + UPDATE orchestration_v2_effect_outbox + SET payload_json = json_remove(payload_json, '$.stableSkipPrefix') + WHERE effect_id = 'effect:archive:parent:subagent-threads.cascade' + `; + } + const childBase = `${skippedId}:subagent:${grandchild}`; + const childId = scenario === "legacy deeper rejection" ? `${childBase}:retry:1` : childBase; + if (scenario === "legacy deeper rejection") { + yield* receipts.upsert({ ...created, commandId: CommandId.make(childBase) }); + } + if (scenario === "legacy planning move") { + const store = yield* ProjectionStore.ProjectionStoreV2; + const sink = yield* EventSink.EventSinkV2; + const { projectId } = yield* store.getThread(parent); + const children = store.getSubagentChildThreads; + using _move = vi.spyOn(store, "getSubagentChildThreads").mockImplementation((threadId) => + children(threadId).pipe( + Effect.tap((found) => + threadId === grandchild && found.length === 0 + ? store.getThread(leaf).pipe( + Effect.flatMap((thread) => + sink.write({ + events: [ + { + id: EventId.make("move:leaf"), + type: "thread.metadata-updated", + threadId: leaf, + occurredAt: f.now, + payload: { ...thread, projectId }, + }, + ], + }), + ), + Effect.orDie, + ) + : Effect.void, + ), + ), + ); + yield* f.commit("archive", grandchild, childId); + const { archiveSourceCommandId: _source, ...legacy } = Option.getOrThrow( + yield* receipt(childId), + ); + yield* receipts.upsert(legacy); + yield* receipts.upsert(collision); + const recovery = f.threads.cascadeSubagentLifecycle({ + operation: "thread.archive", + threadId: parent, + commandId: CommandId.make("archive:parent"), + }); + const error = yield* recovery.pipe(Effect.as(null), Effect.catch(Effect.succeed)); + assert.equal(error?._tag, "OrchestratorDispatchError"); + if (error?._tag === "OrchestratorDispatchError") + assert.include(String(error.cause), "Cannot verify the archive or restore"); + assert.equal(yield* f.archived(app, grandchild, leaf), "110"); + assert.deepEqual(Option.getOrThrow(yield* receipt(childId)), legacy); + + const directId = CommandId.make(childId); + yield* f.outbox.enqueue([ + { + id: `effect:${directId}:subagent-threads.cascade`, + commandId: directId, + threadId: grandchild, + availableAt: f.now, + request: { type: "subagent-threads.cascade", operation: "thread.archive" }, + }, + ]); + yield* recovery; + assert.equal(yield* f.archived(app, grandchild, leaf), "110"); + assert.deepEqual(Option.getOrThrow(yield* receipt(childId)), legacy); + return; + } + const orchestrator = yield* Orchestrator.OrchestratorV2; + const dispatch = orchestrator.dispatch; + let failLeaf = true; + using _failLeaf = vi.spyOn(orchestrator, "dispatch").mockImplementation((command) => { + const interrupted = scenario === "legacy deeper rejection" ? deepest : leaf; + if (command.type === "thread.archive" && command.threadId === interrupted && failLeaf) { + failLeaf = false; + return Effect.fail( + new Orchestrator.OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "Leaf delivery interrupted.", + }), + ); + } + return dispatch(command); + }); + using _lostRetry = vi + .spyOn(f.outbox, "retry") + .mockImplementationOnce(() => Effect.succeed(true)); + yield* f.drain; + assert.equal( + yield* f.archived(app, grandchild, leaf), + scenario === "legacy deeper rejection" ? "111" : "110", + ); + const committed = Option.getOrThrow(yield* receipt(childId)); + const { archiveSourceCommandId: _source, ...legacy } = committed; + const childReceipt = scenario === "fresh late rejection" ? committed : legacy; + if (childReceipt === legacy) yield* receipts.upsert(legacy); + const leafId = `${childId}:subagent:${leaf}`; + if (scenario === "legacy deeper rejection") { + const { archiveSourceCommandId: _source, ...legacyLeaf } = Option.getOrThrow( + yield* receipt(leafId), + ); + yield* receipts.upsert(legacyLeaf); + yield* f.threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make("rename:grandchild:after-archive"), + threadId: grandchild, + title: "Grandchild after archiving", + }); + } + yield* receipts.upsert(collision); + if (scenario === "legacy expired rejection") yield* TestClock.adjust("8 days"); + assert.equal((yield* f.outbox.reconcileAfterProcessLoss).requeued, 1); + yield* f.drain; + const recovered = yield* f.cascade; + if (scenario === "legacy expired rejection") { + assert.equal(recovered.status, "pending"); + assert.include(recovered.lastError, "Cannot verify the archive or restore"); + assert.equal(yield* f.archived(app, grandchild, leaf), "110"); + } else { + assert.equal(recovered.status, "succeeded"); + assert.equal(yield* f.archived(app, grandchild, leaf), "111"); + } + const { archiveSourceCommandId: recoveredSource, ...recoveredReceipt } = Option.getOrThrow( + yield* receipt(childId), + ); + assert.deepEqual(recoveredReceipt, legacy); + assert.deepEqual(Option.getOrThrow(yield* receipt(skippedId)), collision); + if (scenario === "fresh late rejection") { + const resumed = Option.getOrThrow(yield* receipt(`${skippedId}:subagent:${leaf}`)); + assert.include(resumed, { threadId: leaf, status: "accepted" }); + } + if (scenario === "legacy deeper rejection") { + assert.equal(recoveredSource, "archive:parent"); + assert.equal(yield* f.archived(deepest), "1"); + yield* TestClock.adjust("8 days"); + assert.isAbove(yield* f.outbox.pruneSettled, 0); + yield* f.run("unarchive"); + yield* f.drain; + assert.equal(yield* f.archived(app, grandchild, leaf, deepest), "1000"); + } + }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), + ); + + it.effect("replays an exact committed leaf during a recovered legacy restore", () => + Effect.gen(function* () { + const f = yield* fixture; + yield* f.run("archive", app); + yield* f.run("unarchive", grandchild); + const finished = ThreadId.make("restore-a"); + const unfinished = ThreadId.make("restore-b"); + for (const leaf of [finished, unfinished]) { + yield* f.set(leaf, { + lineage: { + rootThreadId: parent, + parentThreadId: grandchild, + relationshipToParent: "subagent", + }, + }); + yield* f.threads.dispatch({ + type: "thread.metadata.update", + commandId: CommandId.make(`rename:${leaf}`), + threadId: leaf, + title: leaf, + }); + } + yield* f.run("archive"); + yield* f.drain; + yield* f.commit("unarchive"); + const effectId = "effect:unarchive:parent:subagent-threads.cascade"; + const sql = yield* SqlClient.SqlClient; + yield* sql` + UPDATE orchestration_v2_effect_outbox + SET payload_json = json_remove(payload_json, '$.stableSkipPrefix') + WHERE effect_id = ${effectId} + `; + + const orchestrator = yield* Orchestrator.OrchestratorV2; + const dispatch = orchestrator.dispatch; + let interrupt = true; + using _interrupt = vi.spyOn(orchestrator, "dispatch").mockImplementation((command) => { + if (command.type === "thread.unarchive" && command.threadId === unfinished && interrupt) { + interrupt = false; + return Effect.fail( + new Orchestrator.OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "Restore delivery interrupted.", + }), + ); + } + return dispatch(command); + }); + using _lostRetry = vi + .spyOn(f.outbox, "retry") + .mockImplementationOnce(() => Effect.succeed(true)); + yield* f.drain; + assert.equal(yield* f.archived(app, grandchild, finished, unfinished), "1001"); + + const skippedId = CommandId.make("unarchive:parent:subagent:app"); + const completedId = `${skippedId}:subagent:${grandchild}:subagent:${finished}`; + const completed = Option.getOrThrow(yield* receipt(completedId)); + const created = Option.getOrThrow(yield* receipt("create:parent")); + const receipts = yield* Receipts.CommandReceiptStoreV2; + yield* receipts.upsert({ + ...created, + commandId: skippedId, + threadId: app, + commandType: "thread.unarchive", + status: "rejected", + error: "Rejected after the partial delivery.", + }); + assert.equal((yield* f.outbox.reconcileAfterProcessLoss).requeued, 1); + yield* f.drain; + assert.equal(Option.getOrThrow(yield* f.outbox.get(effectId)).status, "succeeded"); + assert.equal(yield* f.archived(app, grandchild, finished, unfinished), "1000"); + assert.deepEqual(Option.getOrThrow(yield* receipt(completedId)), completed); + }).pipe(Effect.provide(Layer.merge(layerTest, SqlitePersistence.layerMemory))), + ); + + it.effect("keeps the cascade recoverable when its child retry IDs are exhausted", () => + Effect.gen(function* () { + const f = yield* fixture; + const receipts = yield* Receipts.CommandReceiptStoreV2; + const created = Option.getOrThrow(yield* receipt("create:parent")); + for (const [threadId, slots] of [ + [native, 31], + [app, 32], + ] as const) { + for (let retry = 0; retry < slots; retry++) { + const id = CommandId.make( + `archive:parent:subagent:${threadId}${retry ? `:retry:${retry}` : ""}`, + ); + yield* receipts.upsert({ ...created, commandId: id, threadId, status: "rejected" }); + } + } + yield* f.run("archive"); + yield* f.drain; + const { status, attemptCount, lastError } = yield* f.cascade; + assert.deepEqual([status, attemptCount], ["pending", 1]); + assert.include(lastError, "All 32 subagent lifecycle command IDs were occupied or rejected."); + assert.equal(yield* f.archived(native, app, grandchild), "100"); + const accepted = Option.getOrThrow(yield* receipt("archive:parent:subagent:native:retry:31")); + assert.include(accepted, { threadId: native, status: "accepted" }); + assert.isTrue(Option.isNone(yield* receipt("archive:parent:subagent:app:retry:32"))); + }).pipe(Effect.provide(layerTest)), + ); + it.effect("holds the original caller limit when an accepted child receives a mode raise", () => + Effect.gen(function* () { + const f = yield* fixture; + const limited = ThreadId.make("limited"); + const limit = { runtimeMode: "approval-required", interactionMode: "plan" } as const; + for (const id of [parent, app, native, grandchild]) yield* f.set(id, limit); + yield* f.set(limited, { + ...limit, + lineage: { rootThreadId: parent, parentThreadId: parent, relationshipToParent: "subagent" }, + }); + const { sequence: cutoff } = yield* f + .commit("archive") + .pipe(Effect.provideService(DispatchModeLimit, limit)); + const orchestrator = yield* Orchestrator.OrchestratorV2; + const archiveId = CommandId.make("archive:parent"); + yield* orchestrator + .dispatch({ + type: "thread.archive", + commandId: CommandId.make("archive:parent:subagent:app"), + threadId: app, + }) + .pipe( + Effect.provideService(EffectOutbox.SubagentCascade, { + source: { threadId: parent, cutoff }, + path: [app], + parentArchiveCommandId: archiveId, + }), + ); + yield* f.run("archive"); + assert.equal(yield* f.archived(native, app, grandchild, limited), "0100"); + const claimed = yield* f.outbox.claimNext({ workerId: "lost-caller" }); + assert.equal(Option.getOrThrow(claimed).request.type, "subagent-threads.cascade"); + assert.equal((yield* f.outbox.reconcileAfterProcessLoss).requeued, 1); + // Raise the fresh child before dispatch and the archived child after replay. + const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor; + const { withLock } = executor; + const raised = Effect.orDie( + f.threads.dispatch({ + type: "thread.interaction-mode.set", + interactionMode: "default", + commandId: CommandId.make("raise:archived"), + threadId: app, + }), + ); + const raiseFresh = Effect.orDie( + f.threads.dispatch({ + type: "thread.runtime-mode.set", + runtimeMode: "full-access", + commandId: CommandId.make("raise:fresh"), + threadId: limited, + }), + ); + let requests = 0; + let raisePending = true; + const lock: typeof withLock = (id, effect) => + Effect.suspend(() => { + if (id === limited && raisePending) { + raisePending = false; + return raiseFresh.pipe(Effect.andThen(withLock(id, effect))); + } + return id === app && ++requests === 2 + ? raised.pipe(Effect.andThen(withLock(id, effect))) + : withLock(id, effect); + }); + using _lock = vi.spyOn(executor, "withLock").mockImplementation(lock); + yield* f.drain; + assert.equal(yield* f.archived(native, app, grandchild, limited), "1100"); + assert.equal((yield* f.cascade).status, "succeeded"); + assert.equal( + (yield* orchestrator.getThreadRecords(limited, [])).thread.runtimeMode, + "full-access", + ); + assert.equal( + (yield* orchestrator.getThreadRecords(app, [])).thread.interactionMode, + "default", + ); + }).pipe(Effect.provide(layerTest)), + ); +}); diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 91b521fdb6036..155e7fa99e89c 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -113,6 +113,7 @@ import Migration0102 from "./Migrations/102_ThreadItemCounts.ts"; import Migration0103 from "./Migrations/103_IssueTrackerAdditionalScopes.ts"; import Migration0104 from "./Migrations/104_ProjectionProjectsTrackedFolders.ts"; import Migration0105 from "./Migrations/105_TaskFolders.ts"; +import Migration0106 from "./Migrations/106_CommandReceiptArchiveSource.ts"; const migrationEntries = [ [1, "OrchestrationEvents", Migration0001], @@ -221,6 +222,7 @@ const migrationEntries = [ [103, "IssueTrackerAdditionalScopes", Migration0103], [104, "ProjectionProjectsTrackedFolders", Migration0104], [105, "TaskFolders", Migration0105], + [106, "CommandReceiptArchiveSource", Migration0106], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/106_CommandReceiptArchiveSource.ts b/apps/server/src/persistence/Migrations/106_CommandReceiptArchiveSource.ts new file mode 100644 index 0000000000000..20ef4eae87278 --- /dev/null +++ b/apps/server/src/persistence/Migrations/106_CommandReceiptArchiveSource.ts @@ -0,0 +1,9 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql` + ALTER TABLE orchestration_command_receipts ADD COLUMN archive_source_command_id TEXT + `; +}); diff --git a/apps/server/src/persistence/OrchestrationCommandReceipts.ts b/apps/server/src/persistence/OrchestrationCommandReceipts.ts index 4a39061a00cb0..965b56365159d 100644 --- a/apps/server/src/persistence/OrchestrationCommandReceipts.ts +++ b/apps/server/src/persistence/OrchestrationCommandReceipts.ts @@ -22,6 +22,7 @@ export const OrchestrationCommandReceipt = Schema.Struct({ resultSequence: NonNegativeInt, status: Schema.Literals(["accepted", "rejected"]), error: Schema.NullOr(Schema.String), + archiveSourceCommandId: Schema.optional(Schema.NullOr(CommandId)), }); export type OrchestrationCommandReceipt = typeof OrchestrationCommandReceipt.Type; @@ -30,6 +31,17 @@ export const GetByCommandIdInput = Schema.Struct({ }); export type GetByCommandIdInput = typeof GetByCommandIdInput.Type; +const LifecycleCommandType = Schema.Literals([ + "thread.archive", + "thread.unarchive", + "thread.delete", +]); +export type LifecycleCommandType = typeof LifecycleCommandType.Type; +const GetLatestLifecycleInput = Schema.Struct({ + threadId: ThreadId, + commandTypes: Schema.optional(Schema.Array(LifecycleCommandType)), +}); + export class OrchestrationCommandReceiptRepository extends Context.Service< OrchestrationCommandReceiptRepository, { @@ -41,6 +53,20 @@ export class OrchestrationCommandReceiptRepository extends Context.Service< receipt: OrchestrationCommandReceipt, ) => Effect.Effect; + readonly setArchiveSourceIfMissing: (input: { + readonly commandId: CommandId; + readonly threadId: ThreadId; + readonly resultSequence: number; + readonly sourceCommandId: CommandId; + }) => Effect.Effect; + + readonly getLatestLifecycle: ( + input: typeof GetLatestLifecycleInput.Type, + ) => Effect.Effect< + Option.Option, + OrchestrationCommandReceiptRepositoryError + >; + readonly getByCommandId: ( input: GetByCommandIdInput, ) => Effect.Effect< @@ -65,7 +91,8 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { accepted_at, result_sequence, status, - error + error, + archive_source_command_id ) VALUES ( ${receipt.commandId}, @@ -75,7 +102,8 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { ${receipt.acceptedAt}, ${receipt.resultSequence}, ${receipt.status}, - ${receipt.error} + ${receipt.error}, + ${receipt.archiveSourceCommandId ?? null} ) ON CONFLICT (command_id) DO UPDATE SET @@ -85,7 +113,8 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { accepted_at = excluded.accepted_at, result_sequence = excluded.result_sequence, status = excluded.status, - error = excluded.error + error = excluded.error, + archive_source_command_id = excluded.archive_source_command_id `, }); @@ -102,12 +131,29 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { accepted_at AS "acceptedAt", result_sequence AS "resultSequence", status, - error + error, + archive_source_command_id AS "archiveSourceCommandId" FROM orchestration_command_receipts WHERE command_id = ${commandId} `, }); + const findLatestLifecycle = SqlSchema.findOneOption({ + Request: GetLatestLifecycleInput, + Result: OrchestrationCommandReceipt, + execute: ({ threadId, commandTypes }) => sql` + SELECT command_id AS "commandId", aggregate_kind AS "aggregateKind", + aggregate_id AS "aggregateId", command_type AS "commandType", + accepted_at AS "acceptedAt", result_sequence AS "resultSequence", status, error, + archive_source_command_id AS "archiveSourceCommandId" + FROM orchestration_command_receipts + WHERE aggregate_kind = 'thread' AND aggregate_id = ${threadId} + AND status = 'accepted' AND command_type IN ('thread.archive', 'thread.unarchive', 'thread.delete') + ${commandTypes === undefined ? sql`` : sql`AND command_type IN ${sql.in(commandTypes)}`} + ORDER BY result_sequence DESC LIMIT 1 + `, + }); + const upsert: OrchestrationCommandReceiptRepository["Service"]["upsert"] = (receipt) => upsertReceiptRow(receipt).pipe( Effect.mapError(toPersistenceSqlError("OrchestrationCommandReceiptRepository.upsert:query")), @@ -125,7 +171,8 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { accepted_at, result_sequence, status, - error + error, + archive_source_command_id ) VALUES ( ${receipt.commandId}, @@ -135,7 +182,8 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { ${receipt.acceptedAt}, ${receipt.resultSequence}, ${receipt.status}, - ${receipt.error} + ${receipt.error}, + ${receipt.archiveSourceCommandId ?? null} ) ON CONFLICT(command_id) DO NOTHING RETURNING command_id @@ -157,6 +205,32 @@ const makeOrchestrationCommandReceiptRepository = Effect.gen(function* () { ); return { + setArchiveSourceIfMissing: Effect.fn( + "OrchestrationCommandReceiptRepository.setArchiveSourceIfMissing", + )(({ commandId, threadId, resultSequence, sourceCommandId }) => + sql<{ readonly command_id: string }>` + UPDATE orchestration_command_receipts + SET archive_source_command_id = ${sourceCommandId} + WHERE command_id = ${commandId} + AND aggregate_kind = 'thread' AND aggregate_id = ${threadId} + AND status = 'accepted' AND command_type = 'thread.archive' + AND result_sequence = ${resultSequence} AND archive_source_command_id IS NULL + RETURNING command_id + `.pipe( + Effect.map((rows) => rows.length === 1), + Effect.mapError( + toPersistenceSqlError( + "OrchestrationCommandReceiptRepository.setArchiveSourceIfMissing:query", + ), + ), + ), + ), + getLatestLifecycle: (input) => + findLatestLifecycle(input).pipe( + Effect.mapError( + toPersistenceSqlError("OrchestrationCommandReceiptRepository.getLatestLifecycle:query"), + ), + ), insertIfAbsent, upsert, getByCommandId, diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index d2b0dafab8731..44430ce6abe36 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -85,6 +85,7 @@ describe("V2 preview upgrade", () => { [103, "IssueTrackerAdditionalScopes"], [104, "ProjectionProjectsTrackedFolders"], [105, "TaskFolders"], + [106, "CommandReceiptArchiveSource"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -213,6 +214,7 @@ describe("V2 preview upgrade", () => { [103, "IssueTrackerAdditionalScopes"], [104, "ProjectionProjectsTrackedFolders"], [105, "TaskFolders"], + [106, "CommandReceiptArchiveSource"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/project/ProjectService.ts b/apps/server/src/project/ProjectService.ts index 80fabe6101c05..287879a9d8fcd 100644 --- a/apps/server/src/project/ProjectService.ts +++ b/apps/server/src/project/ProjectService.ts @@ -693,6 +693,7 @@ export const make = Effect.gen(function* () { effects: plan.effects, }); if ( + "projectId" in committed.receipt || committed.receipt.threadId !== command.threadId || committed.receipt.commandType !== command.type ) { diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index d7ad9ed1b1746..012427506f6f1 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -262,6 +262,7 @@ const makeTestRelay = Effect.fnUntraced(function* ( forceStopThread: () => Effect.die("unused forceStopThread"), awaitBackgroundTasks: unused, stopDelegatedTasks: unused, + cascadeSubagentLifecycle: unused, getThreadEventSequence: unused, recoverDelegatedTask: unused, cancelDelegatedTask: unused, diff --git a/docs/user/thread-sidebar.md b/docs/user/thread-sidebar.md index f58f35feb45f5..4e4a653e4fdc5 100644 --- a/docs/user/thread-sidebar.md +++ b/docs/user/thread-sidebar.md @@ -281,6 +281,8 @@ fill the screen; close the full-screen view to return to the parent and inspecto Stop on a thread also stops the subagents it delegated to. Before archiving the parent, wait for its delegated tasks to finish or cancel them. +Archiving, restoring or deleting a thread also applies to its subagent descendants, including subagents in its side chats. The cascade stops at a subagent or side chat moved to another project. Ordinary forks stay independent. Restoring brings back only descendants archived by that action, so independent archives and newer changes on a descendant stay as they are. + Subagent threads started by the agent can't take messages; message the parent thread instead. When such a subagent needs an approval or an answer, the parent thread asks for it.