Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions apps/server/src/orchestration-v2/Orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -772,6 +772,8 @@ function delegatedTasksStopEffect(command: {
}

const WORKSPACE_PREPARATION_INPUT = "Preparing workspace";
/** How long automatic settlement waits for a queued turn to start. */
export const QUEUED_TURN_START_GRACE_MS = 2 * 60 * 1_000;
const RUNTIME_REQUEST_DELIVERY_FAILED_MESSAGE =
"T3 Code could not confirm that your answer reached the agent. If the run stays stuck, stop it and try again.";

Expand Down Expand Up @@ -15463,6 +15465,22 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
cause: `Thread ${command.threadId} changed before automatic settlement.`,
});
}
// The completion reaction starts or holds the queue just after a run ends; settling
// first would cancel an automatic wake it was about to start.
const { runs } = yield* loadProjectionForCommand(command, ["runs"]);
const endedAt = latestExecutedRun(runs)?.completedAt;
if (
endedAt != null &&
runs.some((run) => run.status === "queued") &&
Math.abs(DateTime.toEpochMillis(yield* DateTime.now) - DateTime.toEpochMillis(endedAt)) <
QUEUED_TURN_START_GRACE_MS
) {
return yield* new OrchestratorDispatchError({
commandId: command.commandId,
commandType: command.type,
cause: `Thread ${command.threadId} has a queued run that may still start.`,
});
}
yield* dispatchThreadMutation(
{
type: "thread.settle",
Expand Down
3 changes: 1 addition & 2 deletions apps/server/src/orchestration-v2/ThreadSettlementService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ export interface SettlementPullRequest {
}

const DAY_MS = 24 * 60 * 60 * 1_000;
const QUEUED_TURN_START_GRACE_MS = 2 * 60 * 1_000;

function toMillis(value: DateTime.Utc | null | undefined): number | null {
return value == null ? null : DateTime.toEpochMillis(value);
Expand Down Expand Up @@ -126,7 +125,7 @@ export function threadHasQueuedTurnStart(
const messageAtMs = toMillis(thread.latestUserMessageAt);
if (messageAtMs === null || thread.status === "failed") return false;
const age = nowMs - messageAtMs;
if (Number.isNaN(age) || Math.abs(age) > QUEUED_TURN_START_GRACE_MS) return false;
if (Number.isNaN(age) || Math.abs(age) > Orchestrator.QUEUED_TURN_START_GRACE_MS) return false;
if (thread.latestRunId === null) return true;
return [
toMillis(thread.latestRunRequestedAt),
Expand Down
113 changes: 113 additions & 0 deletions apps/server/src/orchestration-v2/runtimeLayer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4481,6 +4481,119 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
}),
);

// A thread watching a pull request, with a wake queued behind a running turn.
const queuedWatchWake = (prefix: string) =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const threadId = ThreadId.make(`${prefix}-auto-settle`);
yield* orchestrator.dispatch({
type: "thread.create",
createdBy: "user",
creationSource: "web",
commandId: CommandId.make(`${prefix}-create`),
threadId,
projectId: ProjectId.make(`${prefix}-project`),
title: "Queued watch wake",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: process.cwd(),
});
const key = { host: "github.com", repository: "pingdotgg/t3code", number: 7 };
yield* orchestrator.dispatch({
type: "thread.pull-request.watch",
commandId: CommandId.make(`${prefix}-start`),
threadId,
...key,
watching: true,
link: { url: "https://github.com/pingdotgg/t3code/pull/7", source: "agent" },
});
const started = (yield* orchestrator.getThreadShell(threadId))?.pullRequests?.[0]?.watch;
assert.isDefined(started);
yield* orchestrator.dispatch({
type: "message.dispatch",
createdBy: "user",
creationSource: "web",
commandId: CommandId.make(`${prefix}-active`),
threadId,
messageId: MessageId.make(`${prefix}-active`),
text: "Merge it",
attachments: [],
modelSelection,
dispatchMode: { type: "start_immediately" },
});
const wakeMessageId = MessageId.make(prefix);
yield* orchestrator.dispatch({
type: "thread.pull-request-watch.sync",
commandId: CommandId.make(`${prefix}-sync`),
threadId,
...key,
startedAt: started.startedAt,
watch: { ...started, headSha: "merged", wakes: 1 },
wake: {
messageId: wakeMessageId,
text: "Update",
notification: { source: { kind: "monitor" }, outcome: "updated", summary: "#7 merged" },
},
});
const before = yield* orchestrator.getThreadProjection(threadId);
const activeRun = before.runs.find((run) => run.status === "starting");
const wakeRun = before.runs.find((run) => run.userMessageId === wakeMessageId);
assert.isDefined(activeRun);
assert.isDefined(wakeRun);
assert.equal(wakeRun.status, "queued");
assert.notEqual(wakeRun.queueHeld, true);
return { threadId, activeRun, wakeRun };
});

// regression: #29898 settle on merge cancelled a watch wake queued behind the finished run
it.effect("automatic settlement refuses a held wake only within the queued-turn grace", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const eventSink = yield* EventSink.EventSinkV2;
const { threadId, activeRun, wakeRun } = yield* queuedWatchWake("pr-watch-held-wake");
const now = yield* DateTime.now;
// Startup recovery's command prefix skips the completion reaction, as after a restart.
yield* eventSink.write({
commandId: CommandId.make("command:runtime-reconcile:pr-watch-held-wake"),
events: [
{
id: EventId.make("pr-watch-held-wake-completed"),
type: "run.updated",
threadId,
occurredAt: now,
payload: { ...activeRun, status: "completed", completedAt: now },
},
{
id: EventId.make("pr-watch-held-wake-hold"),
type: "run.updated",
threadId,
occurredAt: now,
payload: { ...wakeRun, queueHeld: true },
},
],
});
const autoSettle = (commandId: string) =>
Effect.flatMap(orchestrator.getThreadShell(threadId), (shell) =>
orchestrator.dispatch({
type: "thread.auto-settle",
commandId: CommandId.make(commandId),
threadId,
snapshotAt: shell?.updatedAt ?? now,
}),
);
const early = yield* autoSettle("pr-watch-held-wake-early").pipe(Effect.exit);
assert.equal(early._tag, "Failure");

yield* TestClock.adjust(Orchestrator.QUEUED_TURN_START_GRACE_MS);
yield* autoSettle("pr-watch-held-wake-late");
const after = yield* orchestrator.getThreadProjection(threadId);
assert.isNotNull(after.thread.settledAt);
assert.equal(after.runs.find((run) => run.id === wakeRun.id)?.status, "cancelled");
}),
);

it.effect("archiving ends watches, and an archived thread cannot start one", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
Expand Down
Loading