From 4d9c3e28a0d1e62e087fe634f9244a544d45456e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:13:49 -0800 Subject: [PATCH] fix(server): preserve accepted automation runs during recovery Reconcile task command receipts after failed dispatch replies. Recover accepted preparation and eligible bindings, retain webhook replay ownership across completion failures, and cover manual, scheduled, immediate, and coalesced delivery paths. --- .../ThreadLaunchService.test.ts | 205 ++++++++++++++++-- .../scheduledTasks/ScheduledTaskService.ts | 190 ++++++++++++---- .../ScheduledTaskService.webhook.test.ts | 200 ++++++++++++++--- 3 files changed, 499 insertions(+), 96 deletions(-) diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index b251e9f119595..4cd5e9f589d38 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -45,6 +45,7 @@ import * as Stream from "effect/Stream"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; +import * as Tracer from "effect/Tracer"; import { ChildProcessSpawner } from "effect/process"; import * as GitWorkflow from "../git/GitWorkflowService.ts"; @@ -457,20 +458,127 @@ it.effect( }, ); -it.effect.each( - (["new", "existing"] as const).flatMap((target) => - (["user", "agent"] as const).map((createdBy) => ({ target, createdBy })), +it.effect.each([ + ...(["new", "existing"] as const).flatMap((target) => + (["user", "agent"] as const).map((createdBy) => ({ + target, + createdBy, + delivery: "manual" as const, + failure: "none" as const, + })), ), -)( - "attributes $createdBy-configured automations in $target threads without changing their prompt", - ({ target, createdBy }) => { + ...(["manual", "once", "immediate", "coalesced"] as const).flatMap((delivery) => [ + { + target: "new" as const, + createdBy: "user" as const, + delivery, + failure: "before preparation" as const, + }, + { target: "new" as const, createdBy: "user" as const, delivery, failure: "reply" as const }, + { + target: "existing" as const, + createdBy: "user" as const, + delivery, + failure: "reply" as const, + }, + ]), +])( + "attributes $createdBy-configured automations once in $target threads ($delivery, failure: $failure)", + ({ target, createdBy, delivery, failure }) => { const harness = makeHarness(); + const workers = new Map>(); + let failNext = failure !== "none"; + const layerDispatch = Layer.mergeAll( + Layer.effect( + ThreadLaunch.ThreadLaunchService, + Effect.gen(function* () { + const launches = yield* ThreadLaunch.ThreadLaunchService; + const threads = yield* ThreadManagement.ThreadManagementService; + return { + ...launches, + launch: (input: ThreadLaunch.ThreadLaunchInput) => + Effect.gen(function* () { + if (!failNext) return yield* launches.launch(input); + failNext = false; + if (failure === "before preparation") { + const initialMessage = input.initialMessage; + if (initialMessage?.messageId === undefined) + return yield* Effect.die("Expected a scheduled initial message."); + yield* threads + .dispatch({ + type: "thread.create", + commandId: input.commandId, + threadId: ThreadId.make("thread:accepted-automation"), + projectId: input.projectId, + title: input.title, + modelSelection: input.modelSelection, + runtimeMode: input.runtimeMode, + interactionMode: input.interactionMode, + branch: null, + worktreePath: null, + createdBy: input.createdBy, + creationSource: input.creationSource, + initialMessage: { + ...initialMessage, + messageId: initialMessage.messageId, + workspaceStrategy: input.workspaceStrategy, + }, + }) + .pipe( + Effect.mapError( + (cause) => + new ThreadLaunch.ThreadLaunchError({ + commandId: input.commandId, + projectId: input.projectId, + operation: "create-thread", + cause, + }), + ), + ); + } else yield* launches.launch(input); + return yield* new ThreadLaunch.ThreadLaunchError({ + commandId: input.commandId, + projectId: input.projectId, + operation: "update-thread", + cause: new Error("injected accepted reply failure"), + }); + }), + }; + }), + ), + Layer.effect( + ThreadManagement.ThreadManagementService, + Effect.map(ThreadManagement.ThreadManagementService, (threads) => ({ + ...threads, + sendToThread: (input: Parameters[0]) => + Effect.gen(function* () { + const result = yield* threads.sendToThread(input); + if (!failNext) return result; + failNext = false; + return yield* new ThreadManagement.ThreadManagementProjectionLoadError({ + projectId: input.projectId, + threadId: input.threadId, + cause: new Error("injected accepted reply failure"), + }); + }), + })), + ), + ).pipe(Layer.provide(harness.layer)); const layerScheduledTasks = ScheduledTasks.layer.pipe( + Layer.provide(layerDispatch), Layer.provide( Layer.mergeAll( harness.layer, NodeCrypto.layer, - Scheduler.layer, + delivery === "manual" + ? Scheduler.layer + : Layer.succeed(Scheduler.Scheduler, { + register: (name: string, work: Effect.Effect) => + Effect.gen(function* () { + const context = yield* Effect.context(); + workers.set(name, work.pipe(Effect.provideContext(context), Effect.orDie)); + }), + }), Layer.mock(SecretRequests.SecretRequests)({}), ManagedPolicy.layerTest(), Layer.mock(RemoteDelegation.RemoteDelegation)({}), @@ -481,6 +589,7 @@ it.effect.each( const tasks = yield* ScheduledTasks.ScheduledTaskService; const launches = yield* ThreadLaunch.ThreadLaunchService; const threads = yield* ThreadManagement.ThreadManagementService; + const sql = yield* SqlClient.SqlClient; const existing = target === "existing" ? yield* launches.launch( @@ -491,8 +600,24 @@ it.effect.each( id: ScheduledTaskId.make("scheduled-task:attribution"), title: "Daily audit", prompt: "Audit performance and crashes.", - enabled: false, - schedule: { type: "interval", everyMs: 60_000 }, + enabled: delivery !== "manual", + schedule: + delivery === "manual" + ? { type: "interval", everyMs: 60_000 } + : delivery === "once" + ? { + type: "once", + runAt: DateTime.formatIso(DateTime.add(yield* DateTime.now, { seconds: 10 })), + } + : { + type: "webhook", + dedupe: { + replayKey: "{{body}}", + ...(delivery === "coalesced" + ? { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 } + : {}), + }, + }, projectId, threadId: existing?.threadId ?? null, workspaceStrategy: { type: "root" }, @@ -502,8 +627,56 @@ it.effect.each( createdBy, creationSource: createdBy === "agent" ? "mcp" : "web", }); - const result = yield* tasks.runNow({ id: task.id }); - assert.equal(result.task.lastRunStatus, "succeeded"); + const result = delivery === "manual" ? yield* tasks.runNow({ id: task.id }) : null; + if (delivery === "once") { + yield* TestClock.adjust("10 seconds"); + yield* workers.get("scheduled-tasks")!; + const completed = (yield* tasks.list()).tasks[0]; + assert.equal(completed?.lastRunStatus, "succeeded"); + assert.equal(completed?.runCount, 1); + assert.isNull(completed?.nextRunAt); + } else if (delivery !== "manual") { + const finished = yield* Deferred.make(); + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + const end = span.end.bind(span); + span.end = (...args) => { + end(...args); + if ( + span.name === + (delivery === "coalesced" + ? "ScheduledTaskService.dispatchPendingWebhook" + : "ScheduledTaskService.runWebhookDelivery") + ) + Deferred.doneUnsafe(finished, Effect.void); + }; + return span; + }, + }); + const request = { + hookId: task.id, + token: task.webhook!.path.split("/").at(-1)!, + method: "POST", + path: `/api/hooks/${task.id}`, + query: "", + headers: { "content-type": "application/json" }, + body: new TextEncoder().encode('{"event":"same"}'), + bodyText: '{"event":"same"}', + }; + yield* tasks.triggerWebhook(request).pipe(Effect.withTracer(tracer)); + if (delivery === "coalesced") { + yield* TestClock.adjust("10 seconds"); + yield* workers.get("webhook-coalescing")!.pipe(Effect.withTracer(tracer)); + } + yield* Deferred.await(finished); + assert.lengthOf(yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`, 1); + assert.lengthOf(yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`, 0); + const repeated = yield* tasks.triggerWebhook(request); + assert.equal(repeated._tag === "accepted" && repeated.outcome, "duplicate"); + assert.equal((yield* tasks.list()).tasks[0]?.lastRunStatus, "succeeded"); + assert.equal((yield* tasks.list()).tasks[0]?.runCount, 1); + } else assert.equal(result?.task.lastRunStatus, "succeeded"); const projectThreads = yield* threads.listProjectThreads({ projectId, includeSubagents: false, @@ -512,9 +685,15 @@ it.effect.each( projectThreads.find((candidate) => candidate.id === existing?.threadId) ?? projectThreads[0]; assert.isDefined(thread); - assert.equal(result.threadId, thread!.id); - const projection = yield* threads.getThreadProjection(thread!.id); + if (result !== null) assert.equal(result.threadId, thread!.id); + const projection = + failure === "before preparation" + ? yield* preparationSettled(thread!.id) + : yield* threads.getThreadProjection(thread!.id); const wire = yield* encodeThreadProjection(projection); + assert.lengthOf(wire.messages, 1); + assert.lengthOf(wire.runs, 1); + if (failure === "before preparation") assert.notEqual(wire.runs[0]?.status, "preparing"); assert.equal(wire.messages[0]?.text, task.prompt); assert.equal(wire.messages[0]?.scheduledTaskId, task.id); assert.equal(wire.messages[0]?.createdBy, createdBy); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 17840c50bf499..a3b5023935705 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -3,6 +3,7 @@ import { MessageId, PROVIDER_SEND_TURN_MAX_INPUT_CHARS, type ProjectId, + type RunId, ScheduledTask, ScheduledTaskError, ScheduledTaskId, @@ -79,6 +80,11 @@ import { import { describeFailedWebhookFilter, findFailedWebhookFilter } from "./webhookFilter.ts"; import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts"; +interface AcceptedTaskCommand { + readonly thread_id: ThreadId; + readonly command_type: "thread.create" | "message.dispatch"; +} + export const WEBHOOK_ROUTE_PREFIX = "/api/hooks"; const isScheduledTaskError = Schema.is(ScheduledTaskError); const isOrchestratorDispatchError = Schema.is(OrchestratorDispatchError); @@ -883,6 +889,45 @@ export const layer = Layer.effect( ), ); + const readAcceptedTaskCommand = Effect.fn("ScheduledTaskService.readAcceptedTaskCommand")( + function* (commandId: CommandId) { + const [accepted] = yield* sql` + SELECT aggregate_id AS thread_id, command_type FROM orchestration_command_receipts + WHERE command_id = ${commandId} + AND aggregate_kind = 'thread' AND status = 'accepted' + AND command_type IN ('thread.create', 'message.dispatch') + `.pipe(NodeSqliteClient.readOnly); + return accepted; + }, + ); + + const recoverAcceptedTaskCommand = Effect.fn("ScheduledTaskService.recoverAcceptedTaskCommand")( + function* ( + task: ScheduledTask, + commandId: CommandId, + messageId: MessageId, + accepted: AcceptedTaskCommand, + ) { + if (accepted.command_type === "thread.create") { + const [preparing] = yield* sql<{ readonly run_id: RunId }>` + SELECT runs.run_id FROM orchestration_v2_projection_messages AS messages + JOIN orchestration_v2_projection_runs AS runs + ON runs.run_id = messages.run_id AND runs.thread_id = messages.thread_id + WHERE messages.message_id = ${messageId} + AND messages.thread_id = ${accepted.thread_id} AND runs.status = 'preparing' + `.pipe(NodeSqliteClient.readOnly); + if (preparing !== undefined) + yield* threadLaunch.prepareRun({ + commandId, + threadId: accepted.thread_id, + runId: preparing.run_id, + }); + yield* bindThread(task, accepted.thread_id); + } + return accepted.thread_id; + }, + ); + // A dispatch is recorded once the message is accepted; the provider can still // reject the turn after that, so follow the run and record its failure. A one-time // task whose turn completed is spent, so it is removed; its thread stays. @@ -942,7 +987,8 @@ export const layer = Layer.effect( webhook?: { readonly deliveryId: string; readonly prompt: string; - readonly dispatched: Ref.Ref; + readonly dispatched: Ref.Ref; + readonly accepted: AcceptedTaskCommand | undefined; }, ) { // Read before reserving: nothing between the reservation and its release may fail. @@ -1001,13 +1047,15 @@ export const layer = Layer.effect( const reason = active.createdAt !== task.createdAt ? "The task was replaced before this delivery ran." - : active.schedule.type !== "webhook" - ? "The task's trigger changed before this delivery ran." - : webhookPolicyBlock !== undefined - ? webhookPolicyBlock - : !active.enabled - ? "The task was paused before this delivery ran." - : null; + : webhook.accepted !== undefined + ? null + : active.schedule.type !== "webhook" + ? "The task's trigger changed before this delivery ran." + : webhookPolicyBlock !== undefined + ? webhookPolicyBlock + : !active.enabled + ? "The task was paused before this delivery ran." + : null; if (reason !== null) return yield* new WebhookDeliverySkipped({ reason }); } @@ -1076,48 +1124,68 @@ export const layer = Layer.effect( ); // Capture defects and interrupts too so dispatch failures still reach markCompleted. - const result = - active.threadId === null - ? yield* Effect.exit( - threadLaunch - .launch({ - commandId, - projectId: active.projectId, - title: active.title, - modelSelection: active.modelSelection, - runtimeMode: active.runtimeMode, - interactionMode: active.interactionMode, - workspaceStrategy: active.workspaceStrategy, - initialMessage: { - messageId, - scheduledTaskId: active.id, - text: prompt, - attachments: [], - }, - createdBy: active.createdBy, - creationSource: active.creationSource, - }) - .pipe( - Effect.map((launched) => launched.threadId), - Effect.tap((threadId) => - threadMode === "new_thread" ? Effect.void : bindThread(active, threadId), - ), - ), + let result = + webhook?.accepted !== undefined + ? Exit.succeed( + yield* recoverAcceptedTaskCommand(task, commandId, messageId, webhook.accepted), ) - : yield* Effect.exit( + : yield* Effect.gen(function* () { + if (active.threadId === null) + return yield* threadLaunch + .launch({ + commandId, + projectId: active.projectId, + title: active.title, + modelSelection: active.modelSelection, + runtimeMode: active.runtimeMode, + interactionMode: active.interactionMode, + workspaceStrategy: active.workspaceStrategy, + initialMessage: { + messageId, + scheduledTaskId: active.id, + text: prompt, + attachments: [], + }, + createdBy: active.createdBy, + creationSource: active.creationSource, + }) + .pipe( + Effect.map((launched) => launched.threadId), + Effect.tap((threadId) => + threadMode === "new_thread" ? Effect.void : bindThread(active, threadId), + ), + ); // The stored selection only seeds fresh threads. A bound run // follows the thread's current provider, so it passes none. - sendBound(ThreadId.make(active.threadId), active.projectId, 2).pipe( + return yield* sendBound(ThreadId.make(active.threadId), active.projectId, 2).pipe( Effect.as(ThreadId.make(active.threadId)), + ); + }).pipe( + // A failed reply can follow acceptance; keep its claim until a receipt settles it. + Effect.onExit((exit) => + webhook === undefined + ? Effect.void + : Ref.set(webhook.dispatched, Exit.isSuccess(exit) ? true : undefined), ), + Effect.exit, ); + if (Exit.isFailure(result)) { + const accepted = yield* readAcceptedTaskCommand(commandId); + if (accepted === undefined) { + if (webhook !== undefined) yield* Ref.set(webhook.dispatched, false); + } else { + if (webhook !== undefined) yield* Ref.set(webhook.dispatched, true); + result = Exit.succeed( + yield* recoverAcceptedTaskCommand(active, commandId, messageId, accepted), + ); + } + } const runSucceeded = result._tag === "Success"; - if (webhook !== undefined && runSucceeded) yield* Ref.set(webhook.dispatched, true); const completedAt = yield* localNow; const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const); - const lastRunError = runSucceeded ? null : dispatchErrorMessage(result.cause); - const threadId = runSucceeded ? result.value : null; + const lastRunError = Exit.isFailure(result) ? dispatchErrorMessage(result.cause) : null; + const threadId = Exit.isSuccess(result) ? result.value : null; // One transaction, so an edit cannot land between reading the schedule and // writing the next run computed from it. The origin is read first: it reads // files, and the transaction holds SQLite's writer lock. @@ -2031,14 +2099,20 @@ export const layer = Layer.effect( task: ScheduledTask, deliveryId: string, prompt: string, + accepted?: AcceptedTaskCommand, ) { const runOutcome = (outcome: "started" | "skipped" | "failed") => Effect.all([ Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }), Metrics.increment(Metrics.webhookRunsTotal, { outcome }), ]); - const dispatched = yield* Ref.make(false); - return yield* runTask(task, "webhook", { deliveryId, prompt, dispatched }).pipe( + const dispatched = yield* Ref.make(accepted !== undefined); + return yield* runTask(task, "webhook", { + deliveryId, + prompt, + dispatched, + accepted, + }).pipe( Effect.flatMap(({ dispatchFailed }) => dispatchFailed ? runOutcome("failed").pipe( @@ -2060,14 +2134,16 @@ export const layer = Layer.effect( Effect.gen(function* () { const started = yield* Ref.get(dispatched); yield* Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }); - yield* runOutcome(started ? "started" : "failed"); + if (started !== undefined) yield* runOutcome(started ? "started" : "failed"); yield* markDeliveryFailed( deliveryId, - started - ? "The run started, but completion could not be recorded." - : "The run failed to start.", + started === undefined + ? "The run could not be confirmed; its delivery is retained for recovery." + : started + ? "The run started, but completion could not be recorded." + : "The run failed to start.", ); - if (!started) { + if (started === false) { yield* sql .withTransaction( Effect.gen(function* () { @@ -2083,7 +2159,7 @@ export const layer = Layer.effect( yield* releaseFailedDelivery(deliveryId); } // Pending windows retry the same command id to finish bookkeeping. - return !started; + return started === false; }), ), ); @@ -2558,6 +2634,24 @@ export const layer = Layer.effect( yield* remove; continue; } + // A running window may have committed its command before completion failed. + // Read outside runWebhookDelivery so a failed lookup retains the replay claim. + const accepted = + entry.state === "running" + ? yield* readAcceptedTaskCommand( + CommandId.make(`scheduled-task:${taskId}:webhook:${entry.delivery_id}`), + ) + : undefined; + if (accepted !== undefined) { + // Resume accepted work without rendering or dispatching another message. + const task = yield* decodePendingTask(entry.task_json); + yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL + WHERE delivery_id = ${entry.delivery_id}`; + if (!(yield* runWebhookDelivery(task, entry.delivery_id, task.prompt, accepted))) + return; + yield* remove; + continue; + } const prepared = yield* Effect.gen(function* () { // A running row was already sent, so recovery keeps the prompt it was sent with. const current = entry.state === "dispatching" ? yield* findTask(taskId) : null; diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index e06ed817ac4f4..1daf418a6ee4c 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -169,6 +169,9 @@ const withService = ( readonly release: Deferred.Deferred; }; readonly completedLaunches?: Set; + readonly sent?: Array< + Parameters[0] + >; readonly failLaunches?: { remaining: number }; readonly gate?: Deferred.Deferred; readonly gateForTask?: (input: LaunchInput) => Deferred.Deferred | undefined; @@ -240,7 +243,19 @@ const withService = ( namedProjectsRoot: "/projects", isScratchProject: () => Effect.succeed(false), }), - Layer.mock(ThreadManagementService.ThreadManagementService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)( + options.sent === undefined + ? {} + : { + getThreadShell: (id) => + Effect.succeed({ id, projectId: "project-webhook", deletedAt: null } as never), + sendToThread: (input) => + Effect.sync(() => { + options.sent!.push(input); + return {} as never; + }), + }, + ), Layer.mock(SecretRequests.SecretRequests)({ consume: ({ ref }) => { const value = secretsByRef.get(ref); @@ -1067,40 +1082,155 @@ it.effect("a coalesced window dispatches the prompt saved after it opened", () = ); }); -it.effect("retrying a running window keeps the prompt it was dispatched with", () => { - const workers = new Map>(); - return withService( - ({ service, sql, launches }) => - Effect.gen(function* () { - const schedule = { - type: "webhook", - dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 }, - }; - const save = (prompt: string) => - Effect.flatMap(webhookTaskInput({ prompt, schedule }), service.upsert); - const { task } = yield* save("Prompt when the window opened"); - yield* service.triggerWebhook(requestFor(task)); - yield* save("Prompt at dispatch"); - yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks - WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; - yield* TestClock.adjust("10 seconds"); - yield* drainPendingDispatch(workers); - const launched = yield* Queue.take(launches); - assert.equal(launched.initialMessage?.text, "Prompt at dispatch"); - yield* save("Prompt saved after dispatch"); - yield* sql`DROP TRIGGER fail_completion`; - yield* drainPendingDispatch(workers); - assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); - const owner = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!; - const { delivery } = yield* service.getWebhookDelivery({ - id: task.id, - deliveryId: owner.id, - }); - assert.equal(delivery.renderedPrompt, "Prompt at dispatch"); - }), - { workers }, - ); -}); +it.effect.each([ + { receipt: "accepted", paused: false, recovery: "normal" }, + { receipt: "accepted", paused: true, recovery: "normal" }, + { receipt: "missing", paused: true, recovery: "normal" }, + { receipt: "rejected", paused: true, recovery: "normal" }, + { receipt: "accepted", paused: true, recovery: "binding" }, + { receipt: "accepted", paused: true, recovery: "binding edited" }, + { receipt: "accepted", paused: true, recovery: "rendering" }, +] as const)( + "retrying a running window preserves its dispatch and replay claim (receipt: $receipt, paused: $paused, recovery: $recovery)", + ({ receipt, paused, recovery }) => { + const workers = new Map>(); + const completedLaunches = new Set(); + const sent: Array< + Parameters[0] + > = []; + const lostBinding = recovery === "binding" || recovery === "binding edited"; + let frameUnavailable = false; + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const schedule = { + type: "webhook", + dedupe: { + replayKey: "{{body}}", + coalesceKey: "{{body}}", + coalesceWindowSeconds: 10, + }, + }; + const save = ( + prompt: string, + threadMode = lostBinding ? "fresh_session" : "new_thread", + ) => Effect.flatMap(webhookTaskInput({ prompt, schedule, threadMode }), service.upsert); + const { task } = yield* save("Prompt when the window opened"); + const accepted = yield* service.triggerWebhook(requestFor(task)); + if (accepted._tag !== "accepted") throw new Error("Expected an accepted webhook"); + const commandId = `scheduled-task:${task.id}:webhook:${accepted.deliveryId}`; + yield* save("Prompt at dispatch"); + if (receipt === "accepted") { + if (lostBinding) + yield* sql`CREATE TRIGGER fail_binding BEFORE UPDATE ON scheduled_tasks + WHEN NEW.thread_id IS NOT NULL BEGIN SELECT RAISE(FAIL, 'binding unavailable'); END`; + yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks + WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + const launched = yield* Queue.take(launches); + assert.equal(launched.commandId, commandId); + assert.equal(launched.initialMessage?.text, "Prompt at dispatch"); + yield* sql`DROP TRIGGER fail_completion`; + if (lostBinding) { + yield* sql`DROP TRIGGER fail_binding`; + assert.equal((yield* service.list()).tasks[0]!.threadId, null); + } + } else { + // A process can stop after claiming the window but before the command is accepted. + const startedAt = DateTime.formatIso(yield* DateTime.now); + yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'running' + WHERE delivery_id = ${accepted.deliveryId}`; + yield* sql`UPDATE scheduled_tasks SET last_run_status = 'running', + last_run_at = ${startedAt}, last_run_webhook_delivery_id = ${accepted.deliveryId} + WHERE task_id = ${task.id}`; + } + if (receipt !== "missing") { + yield* sql`INSERT INTO orchestration_command_receipts ( + command_id, aggregate_kind, aggregate_id, command_type, accepted_at, + result_sequence, status, error + ) VALUES (${commandId}, 'thread', 'thread-1', 'thread.create', + ${DateTime.formatIso(yield* DateTime.now)}, 1, ${receipt}, + ${receipt === "rejected" ? "The command was rejected." : null})`; + } + yield* save( + "Prompt saved after dispatch", + recovery === "binding edited" + ? "new_thread" + : lostBinding + ? "fresh_session" + : "new_thread", + ); + if (paused) yield* service.setEnabled({ id: task.id, enabled: false }); + frameUnavailable = recovery === "rendering"; + yield* drainPendingDispatch(workers); + frameUnavailable = false; + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); + assert.equal(yield* Queue.size(launches), 0); + const { delivery } = yield* service.getWebhookDelivery({ + id: task.id, + deliveryId: accepted.deliveryId, + acceptAnyOutcome: true, + }); + yield* service.setEnabled({ id: task.id, enabled: true }); + const retry = yield* service.triggerWebhook(requestFor(task)); + assert.equal( + retry._tag === "accepted" && retry.outcome, + receipt === "accepted" ? "duplicate" : "accepted", + ); + if (receipt === "accepted") { + assert.equal(delivery.renderedPrompt, "Prompt at dispatch"); + assert.equal(delivery.outcome, "accepted"); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal( + (yield* service.list()).tasks[0]!.threadId, + recovery === "binding" ? "thread-1" : null, + ); + } else { + assert.equal(delivery.outcome, "dispatch_failed"); + assert.equal(delivery.error, "The task was paused before this delivery ran."); + } + if (receipt !== "accepted") { + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + const retried = yield* Queue.take(launches); + assert.notEqual(retried.commandId, commandId); + assert.equal(retried.initialMessage?.text, "Prompt saved after dispatch"); + } + assert.equal(yield* Queue.size(launches), 0); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + if (lostBinding) { + const bodyText = '{"value":"next occurrence"}'; + const next = yield* service.triggerWebhook( + requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }), + ); + assert.equal(next._tag === "accepted" && next.outcome, "accepted"); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + if (recovery === "binding") { + assert.equal(yield* Queue.size(launches), 0); + assert.equal(sent.length, 1); + assert.equal(sent[0]!.threadId, "thread-1"); + assert.isTrue(sent[0]!.freshContext); + } else { + assert.equal(sent.length, 0); + assert.equal(yield* Queue.size(launches), 1); + assert.notEqual((yield* Queue.take(launches)).commandId, commandId); + } + assert.equal((yield* service.list()).tasks[0]!.runCount, 2); + } + }), + { + workers, + completedLaunches, + ...(lostBinding ? { sent } : {}), + onFrame: () => { + if (frameUnavailable) throw new Error("Frame generation unavailable"); + }, + }, + ); + }, +); it.effect("a coalesced window fails its delivery when the task row cannot be read", () => { const workers = new Map>();