diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index b251e9f119595..4cd5e9f589d38 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -45,6 +45,7 @@ import * as Stream from "effect/Stream"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; +import * as Tracer from "effect/Tracer"; import { ChildProcessSpawner } from "effect/process"; import * as GitWorkflow from "../git/GitWorkflowService.ts"; @@ -457,20 +458,127 @@ it.effect( }, ); -it.effect.each( - (["new", "existing"] as const).flatMap((target) => - (["user", "agent"] as const).map((createdBy) => ({ target, createdBy })), +it.effect.each([ + ...(["new", "existing"] as const).flatMap((target) => + (["user", "agent"] as const).map((createdBy) => ({ + target, + createdBy, + delivery: "manual" as const, + failure: "none" as const, + })), ), -)( - "attributes $createdBy-configured automations in $target threads without changing their prompt", - ({ target, createdBy }) => { + ...(["manual", "once", "immediate", "coalesced"] as const).flatMap((delivery) => [ + { + target: "new" as const, + createdBy: "user" as const, + delivery, + failure: "before preparation" as const, + }, + { target: "new" as const, createdBy: "user" as const, delivery, failure: "reply" as const }, + { + target: "existing" as const, + createdBy: "user" as const, + delivery, + failure: "reply" as const, + }, + ]), +])( + "attributes $createdBy-configured automations once in $target threads ($delivery, failure: $failure)", + ({ target, createdBy, delivery, failure }) => { const harness = makeHarness(); + const workers = new Map>(); + let failNext = failure !== "none"; + const layerDispatch = Layer.mergeAll( + Layer.effect( + ThreadLaunch.ThreadLaunchService, + Effect.gen(function* () { + const launches = yield* ThreadLaunch.ThreadLaunchService; + const threads = yield* ThreadManagement.ThreadManagementService; + return { + ...launches, + launch: (input: ThreadLaunch.ThreadLaunchInput) => + Effect.gen(function* () { + if (!failNext) return yield* launches.launch(input); + failNext = false; + if (failure === "before preparation") { + const initialMessage = input.initialMessage; + if (initialMessage?.messageId === undefined) + return yield* Effect.die("Expected a scheduled initial message."); + yield* threads + .dispatch({ + type: "thread.create", + commandId: input.commandId, + threadId: ThreadId.make("thread:accepted-automation"), + projectId: input.projectId, + title: input.title, + modelSelection: input.modelSelection, + runtimeMode: input.runtimeMode, + interactionMode: input.interactionMode, + branch: null, + worktreePath: null, + createdBy: input.createdBy, + creationSource: input.creationSource, + initialMessage: { + ...initialMessage, + messageId: initialMessage.messageId, + workspaceStrategy: input.workspaceStrategy, + }, + }) + .pipe( + Effect.mapError( + (cause) => + new ThreadLaunch.ThreadLaunchError({ + commandId: input.commandId, + projectId: input.projectId, + operation: "create-thread", + cause, + }), + ), + ); + } else yield* launches.launch(input); + return yield* new ThreadLaunch.ThreadLaunchError({ + commandId: input.commandId, + projectId: input.projectId, + operation: "update-thread", + cause: new Error("injected accepted reply failure"), + }); + }), + }; + }), + ), + Layer.effect( + ThreadManagement.ThreadManagementService, + Effect.map(ThreadManagement.ThreadManagementService, (threads) => ({ + ...threads, + sendToThread: (input: Parameters[0]) => + Effect.gen(function* () { + const result = yield* threads.sendToThread(input); + if (!failNext) return result; + failNext = false; + return yield* new ThreadManagement.ThreadManagementProjectionLoadError({ + projectId: input.projectId, + threadId: input.threadId, + cause: new Error("injected accepted reply failure"), + }); + }), + })), + ), + ).pipe(Layer.provide(harness.layer)); const layerScheduledTasks = ScheduledTasks.layer.pipe( + Layer.provide(layerDispatch), Layer.provide( Layer.mergeAll( harness.layer, NodeCrypto.layer, - Scheduler.layer, + delivery === "manual" + ? Scheduler.layer + : Layer.succeed(Scheduler.Scheduler, { + register: (name: string, work: Effect.Effect) => + Effect.gen(function* () { + const context = yield* Effect.context(); + workers.set(name, work.pipe(Effect.provideContext(context), Effect.orDie)); + }), + }), Layer.mock(SecretRequests.SecretRequests)({}), ManagedPolicy.layerTest(), Layer.mock(RemoteDelegation.RemoteDelegation)({}), @@ -481,6 +589,7 @@ it.effect.each( const tasks = yield* ScheduledTasks.ScheduledTaskService; const launches = yield* ThreadLaunch.ThreadLaunchService; const threads = yield* ThreadManagement.ThreadManagementService; + const sql = yield* SqlClient.SqlClient; const existing = target === "existing" ? yield* launches.launch( @@ -491,8 +600,24 @@ it.effect.each( id: ScheduledTaskId.make("scheduled-task:attribution"), title: "Daily audit", prompt: "Audit performance and crashes.", - enabled: false, - schedule: { type: "interval", everyMs: 60_000 }, + enabled: delivery !== "manual", + schedule: + delivery === "manual" + ? { type: "interval", everyMs: 60_000 } + : delivery === "once" + ? { + type: "once", + runAt: DateTime.formatIso(DateTime.add(yield* DateTime.now, { seconds: 10 })), + } + : { + type: "webhook", + dedupe: { + replayKey: "{{body}}", + ...(delivery === "coalesced" + ? { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 } + : {}), + }, + }, projectId, threadId: existing?.threadId ?? null, workspaceStrategy: { type: "root" }, @@ -502,8 +627,56 @@ it.effect.each( createdBy, creationSource: createdBy === "agent" ? "mcp" : "web", }); - const result = yield* tasks.runNow({ id: task.id }); - assert.equal(result.task.lastRunStatus, "succeeded"); + const result = delivery === "manual" ? yield* tasks.runNow({ id: task.id }) : null; + if (delivery === "once") { + yield* TestClock.adjust("10 seconds"); + yield* workers.get("scheduled-tasks")!; + const completed = (yield* tasks.list()).tasks[0]; + assert.equal(completed?.lastRunStatus, "succeeded"); + assert.equal(completed?.runCount, 1); + assert.isNull(completed?.nextRunAt); + } else if (delivery !== "manual") { + const finished = yield* Deferred.make(); + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + const end = span.end.bind(span); + span.end = (...args) => { + end(...args); + if ( + span.name === + (delivery === "coalesced" + ? "ScheduledTaskService.dispatchPendingWebhook" + : "ScheduledTaskService.runWebhookDelivery") + ) + Deferred.doneUnsafe(finished, Effect.void); + }; + return span; + }, + }); + const request = { + hookId: task.id, + token: task.webhook!.path.split("/").at(-1)!, + method: "POST", + path: `/api/hooks/${task.id}`, + query: "", + headers: { "content-type": "application/json" }, + body: new TextEncoder().encode('{"event":"same"}'), + bodyText: '{"event":"same"}', + }; + yield* tasks.triggerWebhook(request).pipe(Effect.withTracer(tracer)); + if (delivery === "coalesced") { + yield* TestClock.adjust("10 seconds"); + yield* workers.get("webhook-coalescing")!.pipe(Effect.withTracer(tracer)); + } + yield* Deferred.await(finished); + assert.lengthOf(yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`, 1); + assert.lengthOf(yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`, 0); + const repeated = yield* tasks.triggerWebhook(request); + assert.equal(repeated._tag === "accepted" && repeated.outcome, "duplicate"); + assert.equal((yield* tasks.list()).tasks[0]?.lastRunStatus, "succeeded"); + assert.equal((yield* tasks.list()).tasks[0]?.runCount, 1); + } else assert.equal(result?.task.lastRunStatus, "succeeded"); const projectThreads = yield* threads.listProjectThreads({ projectId, includeSubagents: false, @@ -512,9 +685,15 @@ it.effect.each( projectThreads.find((candidate) => candidate.id === existing?.threadId) ?? projectThreads[0]; assert.isDefined(thread); - assert.equal(result.threadId, thread!.id); - const projection = yield* threads.getThreadProjection(thread!.id); + if (result !== null) assert.equal(result.threadId, thread!.id); + const projection = + failure === "before preparation" + ? yield* preparationSettled(thread!.id) + : yield* threads.getThreadProjection(thread!.id); const wire = yield* encodeThreadProjection(projection); + assert.lengthOf(wire.messages, 1); + assert.lengthOf(wire.runs, 1); + if (failure === "before preparation") assert.notEqual(wire.runs[0]?.status, "preparing"); assert.equal(wire.messages[0]?.text, task.prompt); assert.equal(wire.messages[0]?.scheduledTaskId, task.id); assert.equal(wire.messages[0]?.createdBy, createdBy); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 17840c50bf499..a3b5023935705 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -3,6 +3,7 @@ import { MessageId, PROVIDER_SEND_TURN_MAX_INPUT_CHARS, type ProjectId, + type RunId, ScheduledTask, ScheduledTaskError, ScheduledTaskId, @@ -79,6 +80,11 @@ import { import { describeFailedWebhookFilter, findFailedWebhookFilter } from "./webhookFilter.ts"; import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts"; +interface AcceptedTaskCommand { + readonly thread_id: ThreadId; + readonly command_type: "thread.create" | "message.dispatch"; +} + export const WEBHOOK_ROUTE_PREFIX = "/api/hooks"; const isScheduledTaskError = Schema.is(ScheduledTaskError); const isOrchestratorDispatchError = Schema.is(OrchestratorDispatchError); @@ -883,6 +889,45 @@ export const layer = Layer.effect( ), ); + const readAcceptedTaskCommand = Effect.fn("ScheduledTaskService.readAcceptedTaskCommand")( + function* (commandId: CommandId) { + const [accepted] = yield* sql` + SELECT aggregate_id AS thread_id, command_type FROM orchestration_command_receipts + WHERE command_id = ${commandId} + AND aggregate_kind = 'thread' AND status = 'accepted' + AND command_type IN ('thread.create', 'message.dispatch') + `.pipe(NodeSqliteClient.readOnly); + return accepted; + }, + ); + + const recoverAcceptedTaskCommand = Effect.fn("ScheduledTaskService.recoverAcceptedTaskCommand")( + function* ( + task: ScheduledTask, + commandId: CommandId, + messageId: MessageId, + accepted: AcceptedTaskCommand, + ) { + if (accepted.command_type === "thread.create") { + const [preparing] = yield* sql<{ readonly run_id: RunId }>` + SELECT runs.run_id FROM orchestration_v2_projection_messages AS messages + JOIN orchestration_v2_projection_runs AS runs + ON runs.run_id = messages.run_id AND runs.thread_id = messages.thread_id + WHERE messages.message_id = ${messageId} + AND messages.thread_id = ${accepted.thread_id} AND runs.status = 'preparing' + `.pipe(NodeSqliteClient.readOnly); + if (preparing !== undefined) + yield* threadLaunch.prepareRun({ + commandId, + threadId: accepted.thread_id, + runId: preparing.run_id, + }); + yield* bindThread(task, accepted.thread_id); + } + return accepted.thread_id; + }, + ); + // A dispatch is recorded once the message is accepted; the provider can still // reject the turn after that, so follow the run and record its failure. A one-time // task whose turn completed is spent, so it is removed; its thread stays. @@ -942,7 +987,8 @@ export const layer = Layer.effect( webhook?: { readonly deliveryId: string; readonly prompt: string; - readonly dispatched: Ref.Ref; + readonly dispatched: Ref.Ref; + readonly accepted: AcceptedTaskCommand | undefined; }, ) { // Read before reserving: nothing between the reservation and its release may fail. @@ -1001,13 +1047,15 @@ export const layer = Layer.effect( const reason = active.createdAt !== task.createdAt ? "The task was replaced before this delivery ran." - : active.schedule.type !== "webhook" - ? "The task's trigger changed before this delivery ran." - : webhookPolicyBlock !== undefined - ? webhookPolicyBlock - : !active.enabled - ? "The task was paused before this delivery ran." - : null; + : webhook.accepted !== undefined + ? null + : active.schedule.type !== "webhook" + ? "The task's trigger changed before this delivery ran." + : webhookPolicyBlock !== undefined + ? webhookPolicyBlock + : !active.enabled + ? "The task was paused before this delivery ran." + : null; if (reason !== null) return yield* new WebhookDeliverySkipped({ reason }); } @@ -1076,48 +1124,68 @@ export const layer = Layer.effect( ); // Capture defects and interrupts too so dispatch failures still reach markCompleted. - const result = - active.threadId === null - ? yield* Effect.exit( - threadLaunch - .launch({ - commandId, - projectId: active.projectId, - title: active.title, - modelSelection: active.modelSelection, - runtimeMode: active.runtimeMode, - interactionMode: active.interactionMode, - workspaceStrategy: active.workspaceStrategy, - initialMessage: { - messageId, - scheduledTaskId: active.id, - text: prompt, - attachments: [], - }, - createdBy: active.createdBy, - creationSource: active.creationSource, - }) - .pipe( - Effect.map((launched) => launched.threadId), - Effect.tap((threadId) => - threadMode === "new_thread" ? Effect.void : bindThread(active, threadId), - ), - ), + let result = + webhook?.accepted !== undefined + ? Exit.succeed( + yield* recoverAcceptedTaskCommand(task, commandId, messageId, webhook.accepted), ) - : yield* Effect.exit( + : yield* Effect.gen(function* () { + if (active.threadId === null) + return yield* threadLaunch + .launch({ + commandId, + projectId: active.projectId, + title: active.title, + modelSelection: active.modelSelection, + runtimeMode: active.runtimeMode, + interactionMode: active.interactionMode, + workspaceStrategy: active.workspaceStrategy, + initialMessage: { + messageId, + scheduledTaskId: active.id, + text: prompt, + attachments: [], + }, + createdBy: active.createdBy, + creationSource: active.creationSource, + }) + .pipe( + Effect.map((launched) => launched.threadId), + Effect.tap((threadId) => + threadMode === "new_thread" ? Effect.void : bindThread(active, threadId), + ), + ); // The stored selection only seeds fresh threads. A bound run // follows the thread's current provider, so it passes none. - sendBound(ThreadId.make(active.threadId), active.projectId, 2).pipe( + return yield* sendBound(ThreadId.make(active.threadId), active.projectId, 2).pipe( Effect.as(ThreadId.make(active.threadId)), + ); + }).pipe( + // A failed reply can follow acceptance; keep its claim until a receipt settles it. + Effect.onExit((exit) => + webhook === undefined + ? Effect.void + : Ref.set(webhook.dispatched, Exit.isSuccess(exit) ? true : undefined), ), + Effect.exit, ); + if (Exit.isFailure(result)) { + const accepted = yield* readAcceptedTaskCommand(commandId); + if (accepted === undefined) { + if (webhook !== undefined) yield* Ref.set(webhook.dispatched, false); + } else { + if (webhook !== undefined) yield* Ref.set(webhook.dispatched, true); + result = Exit.succeed( + yield* recoverAcceptedTaskCommand(active, commandId, messageId, accepted), + ); + } + } const runSucceeded = result._tag === "Success"; - if (webhook !== undefined && runSucceeded) yield* Ref.set(webhook.dispatched, true); const completedAt = yield* localNow; const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const); - const lastRunError = runSucceeded ? null : dispatchErrorMessage(result.cause); - const threadId = runSucceeded ? result.value : null; + const lastRunError = Exit.isFailure(result) ? dispatchErrorMessage(result.cause) : null; + const threadId = Exit.isSuccess(result) ? result.value : null; // One transaction, so an edit cannot land between reading the schedule and // writing the next run computed from it. The origin is read first: it reads // files, and the transaction holds SQLite's writer lock. @@ -2031,14 +2099,20 @@ export const layer = Layer.effect( task: ScheduledTask, deliveryId: string, prompt: string, + accepted?: AcceptedTaskCommand, ) { const runOutcome = (outcome: "started" | "skipped" | "failed") => Effect.all([ Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }), Metrics.increment(Metrics.webhookRunsTotal, { outcome }), ]); - const dispatched = yield* Ref.make(false); - return yield* runTask(task, "webhook", { deliveryId, prompt, dispatched }).pipe( + const dispatched = yield* Ref.make(accepted !== undefined); + return yield* runTask(task, "webhook", { + deliveryId, + prompt, + dispatched, + accepted, + }).pipe( Effect.flatMap(({ dispatchFailed }) => dispatchFailed ? runOutcome("failed").pipe( @@ -2060,14 +2134,16 @@ export const layer = Layer.effect( Effect.gen(function* () { const started = yield* Ref.get(dispatched); yield* Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }); - yield* runOutcome(started ? "started" : "failed"); + if (started !== undefined) yield* runOutcome(started ? "started" : "failed"); yield* markDeliveryFailed( deliveryId, - started - ? "The run started, but completion could not be recorded." - : "The run failed to start.", + started === undefined + ? "The run could not be confirmed; its delivery is retained for recovery." + : started + ? "The run started, but completion could not be recorded." + : "The run failed to start.", ); - if (!started) { + if (started === false) { yield* sql .withTransaction( Effect.gen(function* () { @@ -2083,7 +2159,7 @@ export const layer = Layer.effect( yield* releaseFailedDelivery(deliveryId); } // Pending windows retry the same command id to finish bookkeeping. - return !started; + return started === false; }), ), ); @@ -2558,6 +2634,24 @@ export const layer = Layer.effect( yield* remove; continue; } + // A running window may have committed its command before completion failed. + // Read outside runWebhookDelivery so a failed lookup retains the replay claim. + const accepted = + entry.state === "running" + ? yield* readAcceptedTaskCommand( + CommandId.make(`scheduled-task:${taskId}:webhook:${entry.delivery_id}`), + ) + : undefined; + if (accepted !== undefined) { + // Resume accepted work without rendering or dispatching another message. + const task = yield* decodePendingTask(entry.task_json); + yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL + WHERE delivery_id = ${entry.delivery_id}`; + if (!(yield* runWebhookDelivery(task, entry.delivery_id, task.prompt, accepted))) + return; + yield* remove; + continue; + } const prepared = yield* Effect.gen(function* () { // A running row was already sent, so recovery keeps the prompt it was sent with. const current = entry.state === "dispatching" ? yield* findTask(taskId) : null; diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts index e06ed817ac4f4..1daf418a6ee4c 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts @@ -169,6 +169,9 @@ const withService = ( readonly release: Deferred.Deferred; }; readonly completedLaunches?: Set; + readonly sent?: Array< + Parameters[0] + >; readonly failLaunches?: { remaining: number }; readonly gate?: Deferred.Deferred; readonly gateForTask?: (input: LaunchInput) => Deferred.Deferred | undefined; @@ -240,7 +243,19 @@ const withService = ( namedProjectsRoot: "/projects", isScratchProject: () => Effect.succeed(false), }), - Layer.mock(ThreadManagementService.ThreadManagementService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)( + options.sent === undefined + ? {} + : { + getThreadShell: (id) => + Effect.succeed({ id, projectId: "project-webhook", deletedAt: null } as never), + sendToThread: (input) => + Effect.sync(() => { + options.sent!.push(input); + return {} as never; + }), + }, + ), Layer.mock(SecretRequests.SecretRequests)({ consume: ({ ref }) => { const value = secretsByRef.get(ref); @@ -1067,40 +1082,155 @@ it.effect("a coalesced window dispatches the prompt saved after it opened", () = ); }); -it.effect("retrying a running window keeps the prompt it was dispatched with", () => { - const workers = new Map>(); - return withService( - ({ service, sql, launches }) => - Effect.gen(function* () { - const schedule = { - type: "webhook", - dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 }, - }; - const save = (prompt: string) => - Effect.flatMap(webhookTaskInput({ prompt, schedule }), service.upsert); - const { task } = yield* save("Prompt when the window opened"); - yield* service.triggerWebhook(requestFor(task)); - yield* save("Prompt at dispatch"); - yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks - WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; - yield* TestClock.adjust("10 seconds"); - yield* drainPendingDispatch(workers); - const launched = yield* Queue.take(launches); - assert.equal(launched.initialMessage?.text, "Prompt at dispatch"); - yield* save("Prompt saved after dispatch"); - yield* sql`DROP TRIGGER fail_completion`; - yield* drainPendingDispatch(workers); - assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); - const owner = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!; - const { delivery } = yield* service.getWebhookDelivery({ - id: task.id, - deliveryId: owner.id, - }); - assert.equal(delivery.renderedPrompt, "Prompt at dispatch"); - }), - { workers }, - ); -}); +it.effect.each([ + { receipt: "accepted", paused: false, recovery: "normal" }, + { receipt: "accepted", paused: true, recovery: "normal" }, + { receipt: "missing", paused: true, recovery: "normal" }, + { receipt: "rejected", paused: true, recovery: "normal" }, + { receipt: "accepted", paused: true, recovery: "binding" }, + { receipt: "accepted", paused: true, recovery: "binding edited" }, + { receipt: "accepted", paused: true, recovery: "rendering" }, +] as const)( + "retrying a running window preserves its dispatch and replay claim (receipt: $receipt, paused: $paused, recovery: $recovery)", + ({ receipt, paused, recovery }) => { + const workers = new Map>(); + const completedLaunches = new Set(); + const sent: Array< + Parameters[0] + > = []; + const lostBinding = recovery === "binding" || recovery === "binding edited"; + let frameUnavailable = false; + return withService( + ({ service, sql, launches }) => + Effect.gen(function* () { + const schedule = { + type: "webhook", + dedupe: { + replayKey: "{{body}}", + coalesceKey: "{{body}}", + coalesceWindowSeconds: 10, + }, + }; + const save = ( + prompt: string, + threadMode = lostBinding ? "fresh_session" : "new_thread", + ) => Effect.flatMap(webhookTaskInput({ prompt, schedule, threadMode }), service.upsert); + const { task } = yield* save("Prompt when the window opened"); + const accepted = yield* service.triggerWebhook(requestFor(task)); + if (accepted._tag !== "accepted") throw new Error("Expected an accepted webhook"); + const commandId = `scheduled-task:${task.id}:webhook:${accepted.deliveryId}`; + yield* save("Prompt at dispatch"); + if (receipt === "accepted") { + if (lostBinding) + yield* sql`CREATE TRIGGER fail_binding BEFORE UPDATE ON scheduled_tasks + WHEN NEW.thread_id IS NOT NULL BEGIN SELECT RAISE(FAIL, 'binding unavailable'); END`; + yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks + WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`; + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + const launched = yield* Queue.take(launches); + assert.equal(launched.commandId, commandId); + assert.equal(launched.initialMessage?.text, "Prompt at dispatch"); + yield* sql`DROP TRIGGER fail_completion`; + if (lostBinding) { + yield* sql`DROP TRIGGER fail_binding`; + assert.equal((yield* service.list()).tasks[0]!.threadId, null); + } + } else { + // A process can stop after claiming the window but before the command is accepted. + const startedAt = DateTime.formatIso(yield* DateTime.now); + yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'running' + WHERE delivery_id = ${accepted.deliveryId}`; + yield* sql`UPDATE scheduled_tasks SET last_run_status = 'running', + last_run_at = ${startedAt}, last_run_webhook_delivery_id = ${accepted.deliveryId} + WHERE task_id = ${task.id}`; + } + if (receipt !== "missing") { + yield* sql`INSERT INTO orchestration_command_receipts ( + command_id, aggregate_kind, aggregate_id, command_type, accepted_at, + result_sequence, status, error + ) VALUES (${commandId}, 'thread', 'thread-1', 'thread.create', + ${DateTime.formatIso(yield* DateTime.now)}, 1, ${receipt}, + ${receipt === "rejected" ? "The command was rejected." : null})`; + } + yield* save( + "Prompt saved after dispatch", + recovery === "binding edited" + ? "new_thread" + : lostBinding + ? "fresh_session" + : "new_thread", + ); + if (paused) yield* service.setEnabled({ id: task.id, enabled: false }); + frameUnavailable = recovery === "rendering"; + yield* drainPendingDispatch(workers); + frameUnavailable = false; + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0); + assert.equal(yield* Queue.size(launches), 0); + const { delivery } = yield* service.getWebhookDelivery({ + id: task.id, + deliveryId: accepted.deliveryId, + acceptAnyOutcome: true, + }); + yield* service.setEnabled({ id: task.id, enabled: true }); + const retry = yield* service.triggerWebhook(requestFor(task)); + assert.equal( + retry._tag === "accepted" && retry.outcome, + receipt === "accepted" ? "duplicate" : "accepted", + ); + if (receipt === "accepted") { + assert.equal(delivery.renderedPrompt, "Prompt at dispatch"); + assert.equal(delivery.outcome, "accepted"); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + assert.equal( + (yield* service.list()).tasks[0]!.threadId, + recovery === "binding" ? "thread-1" : null, + ); + } else { + assert.equal(delivery.outcome, "dispatch_failed"); + assert.equal(delivery.error, "The task was paused before this delivery ran."); + } + if (receipt !== "accepted") { + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + const retried = yield* Queue.take(launches); + assert.notEqual(retried.commandId, commandId); + assert.equal(retried.initialMessage?.text, "Prompt saved after dispatch"); + } + assert.equal(yield* Queue.size(launches), 0); + assert.equal((yield* service.list()).tasks[0]!.runCount, 1); + if (lostBinding) { + const bodyText = '{"value":"next occurrence"}'; + const next = yield* service.triggerWebhook( + requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }), + ); + assert.equal(next._tag === "accepted" && next.outcome, "accepted"); + yield* TestClock.adjust("10 seconds"); + yield* drainPendingDispatch(workers); + if (recovery === "binding") { + assert.equal(yield* Queue.size(launches), 0); + assert.equal(sent.length, 1); + assert.equal(sent[0]!.threadId, "thread-1"); + assert.isTrue(sent[0]!.freshContext); + } else { + assert.equal(sent.length, 0); + assert.equal(yield* Queue.size(launches), 1); + assert.notEqual((yield* Queue.take(launches)).commandId, commandId); + } + assert.equal((yield* service.list()).tasks[0]!.runCount, 2); + } + }), + { + workers, + completedLaunches, + ...(lostBinding ? { sent } : {}), + onFrame: () => { + if (frameUnavailable) throw new Error("Frame generation unavailable"); + }, + }, + ); + }, +); it.effect("a coalesced window fails its delivery when the task row cannot be read", () => { const workers = new Map>();