Skip to content
Open
11 changes: 11 additions & 0 deletions apps/server/src/orchestration-v2/CommandPolicy.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import {
type ChatAttachment,
type ModelSelection,
CommandId,
type OrchestrationV2Command,
OrchestrationV2ProviderCapabilities,
Expand All @@ -16,6 +17,16 @@ import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Schema from "effect/Schema";

// Account choice is preflighted outside the dispatch lock; a raced user action wins.
export const AccountFallback = Context.Reference<
| {
readonly commandId: CommandId;
readonly threadSelection: ModelSelection;
readonly requestedSelection: ModelSelection | undefined;
}
| undefined
>("t3/orchestration-v2/AccountFallback", { defaultValue: () => undefined });

export const SteeringExecutionPolicyV2 = Schema.Literals(["active_steering", "interrupt_restart"]);
export type SteeringExecutionPolicyV2 = typeof SteeringExecutionPolicyV2.Type;

Expand Down
33 changes: 32 additions & 1 deletion apps/server/src/orchestration-v2/Orchestrator.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { AccountFallback } from "./CommandPolicy.ts";
import type {
OrchestrationV2ThreadLinkMetadata,
GuestShareId,
Expand Down Expand Up @@ -6878,6 +6879,25 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
});
return;
}
const fallback = yield* AccountFallback;
const automaticFallback = fallback?.commandId === command.commandId ? fallback : undefined;
if (automaticFallback && command.manualContinuationOfRunId === undefined) {
const mode = resolveMessageDispatchIntent(
projection,
command.dispatchMode,
command.deliveryIntent,
);
if (
((mode.type === "steer_active" || mode.type === "restart_active") &&
command.freshContext !== true) ||
!modelSelectionsEqual(projection.thread.modelSelection, automaticFallback.threadSelection)
) {
command = {
...command,
modelSelection: automaticFallback.requestedSelection ?? projection.thread.modelSelection,
};
}
}
const activeProviderInstanceId = projection.providerThreads.find(
(candidate) => candidate.id === projection.thread.activeProviderThreadId,
)?.providerInstanceId;
Expand All @@ -6901,7 +6921,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
latestExecutedRun(projection.runs)?.id !== source.id ||
projection.thread.archivedAt !== null ||
projection.thread.deletedAt !== null ||
projection.runtimeRequests.some((request) => request.status === "pending")
projection.runtimeRequests.some((request) => request.status === "pending") ||
(automaticFallback !== undefined &&
(projection.thread.settledOverride === "settled" ||
!modelSelectionsEqual(source.modelSelection, automaticFallback.threadSelection) ||
!modelSelectionsEqual(
projection.thread.modelSelection,
automaticFallback.threadSelection,
) ||
(projection.thread.snoozedUntil != null &&
DateTime.toEpochMillis(projection.thread.snoozedUntil) >
DateTime.toEpochMillis(yield* DateTime.now)) ||
(yield* stopReachedRun(command, command.threadId, source.id))))
) {
return yield* new OrchestratorDispatchError({
commandId: command.commandId,
Expand Down
40 changes: 40 additions & 0 deletions apps/server/src/orchestration-v2/ProjectionStore.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8413,6 +8413,8 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
if (lastErrorClass === "usage_limit") {
assert.deepEqual(candidate, {
id: sqlShell.id,
projectId: sqlShell.projectId,
modelSelection: sqlShell.modelSelection,
status: sqlShell.status,
lastErrorClass: sqlShell.lastErrorClass,
usageLimitResetAt: sqlShell.usageLimitResetAt,
Expand Down Expand Up @@ -8592,6 +8594,33 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
(row) => row.id === threadId,
),
);
for (const [since, found] of [
[DateTime.subtract(now, { minutes: 1 }), true],
[DateTime.add(now, { minutes: 1 }), false],
] as const) {
assert.strictEqual(
(yield* store.getLimitRecoveryCandidates({
...recoveryOptions,
switchAccountsSince: since,
})).some((row) => row.id === threadId),
found,
);
}
yield* sql`UPDATE orchestration_v2_projection_turn_items
SET payload_json = json_remove(payload_json, '$.failure.resetAt') WHERE turn_item_id = ${limitItem.id}`;
assert.isTrue(
(yield* store.getLimitRecoveryCandidates({
...recoveryOptions,
switchAccountsSince: DateTime.subtract(now, { minutes: 1 }),
})).some((row) => row.id === threadId),
);
assert.isFalse(
(yield* store.getLimitRecoveryCandidates({ ...recoveryOptions, autoResume: true })).some(
(row) => row.id === threadId,
),
);
yield* sql`UPDATE orchestration_v2_projection_turn_items
SET payload_json = json_set(payload_json, '$.failure.resetAt', ${limitItem.failure.resetAt}) WHERE turn_item_id = ${limitItem.id}`;
const reset = DateTime.makeUnsafe(limitItem.failure.resetAt);
const recovery = {
runId: original.id,
Expand Down Expand Up @@ -8856,6 +8885,17 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
deletedParentChild,
activeParentChild,
];
assert.deepEqual(
(yield* store.getLimitRecoveryCandidates({
now,
autoResume: false,
snooze: false,
switchAccountsSince: DateTime.subtract(now, { minutes: 1 }),
}))
.filter((row) => children.includes(row.id))
.map((row) => row.id),
[activeParentChild],
);
for (const options of [
{ now, autoResume: true, snooze: false },
{ now, autoResume: false, snooze: true },
Expand Down
31 changes: 28 additions & 3 deletions apps/server/src/orchestration-v2/ProjectionStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,8 @@ export type ProjectionRecoveryKind =
export type ProjectionLimitRecoveryCandidate = Pick<
OrchestrationV2ThreadShell,
| "id"
| "projectId"
| "modelSelection"
| "status"
| "lastErrorClass"
| "latestRunId"
Expand Down Expand Up @@ -636,6 +638,8 @@ export interface ProjectionStoreV2Shape {
readonly now: DateTime.Utc;
readonly autoResume: boolean;
readonly snooze: boolean;
/** Also returns limits hit since then whose window has not reset, to move to another account. */
readonly switchAccountsSince?: DateTime.Utc;
}) => Effect.Effect<ReadonlyArray<ProjectionLimitRecoveryCandidate>, ProjectionStoreV2Error>;
/** Epoch ms of the next armed reset or snooze end after `now`, when a waiting thread becomes a candidate. */
readonly getLimitRecoveryWakeAt: (
Expand Down Expand Up @@ -4764,11 +4768,28 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
AND t.archived_at IS NULL
AND json_extract(t.payload_json, '$.settledOverride') IS NOT 'settled'
AND json_extract(item.payload_json, '$.failure.class') = 'usage_limit'
AND json_extract(item.payload_json, '$.failure.resetAt') IS NOT NULL
AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt')))
AND (
(
json_extract(t.payload_json, '$.limitRecovery.runId') IS r.run_id
${booleanInt(options.switchAccountsSince !== undefined)}
AND julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt'))) >= julianday(${DateTime.formatIso(options.switchAccountsSince ?? options.now)})
AND (
json_extract(item.payload_json, '$.failure.resetAt') IS NULL
OR julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(${DateTime.formatIso(options.now)})
)
AND NOT EXISTS (
SELECT 1 FROM orchestration_v2_projection_threads parent
WHERE json_extract(t.payload_json, '$.lineage.relationshipToParent') = 'subagent'
AND parent.thread_id = json_extract(t.payload_json, '$.lineage.parentThreadId')
AND (
json_extract(parent.payload_json, '$.settledOverride') = 'settled'
OR json_extract(parent.payload_json, '$.archivedAt') IS NOT NULL
OR parent.deleted_at IS NOT NULL
)
)
)
OR (
julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt')))
AND json_extract(t.payload_json, '$.limitRecovery.runId') IS r.run_id
AND json_extract(t.payload_json, '$.limitRecovery.resetAt') IS json_extract(item.payload_json, '$.failure.resetAt')
AND json_extract(t.payload_json, '$.limitRecovery.autoResume') = 1
AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) <= julianday(${DateTime.formatIso(options.now)})
Expand All @@ -4778,6 +4799,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
)
)
OR (
julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt')))
AND
(
json_extract(t.payload_json, '$.limitRecovery.runId') IS NOT r.run_id
OR json_extract(t.payload_json, '$.limitRecovery.resetAt') IS NOT json_extract(item.payload_json, '$.failure.resetAt')
Expand Down Expand Up @@ -4816,6 +4839,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
if (summary.lastErrorClass !== "usage_limit") continue;
candidates.push({
id: thread.id,
projectId: thread.projectId,
modelSelection: thread.modelSelection,
status: "failed",
lastErrorClass: summary.lastErrorClass,
usageLimitResetAt: summary.usageLimitResetAt,
Expand Down
115 changes: 115 additions & 0 deletions apps/server/src/orchestration-v2/ThreadMessageIntake.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import * as DocumentText from "../provider/DocumentText.ts";
import { AccountFallback } from "./CommandPolicy.ts";
// @effect-diagnostics nodeBuiltinImport:off
import * as NodeFS from "node:fs";
import * as NodePath from "node:path";
Expand All @@ -10,13 +11,17 @@ import {
EventId,
GuestMessageId,
MessageId,
ProjectId,
ProviderDriverKind,
ProviderInstanceId,
RunId,
RuntimeRequestId,
ThreadId,
TurnItemId,
type OrchestrationV2Command,
type OrchestrationV2ServerCommand,
type OrchestrationV2StoredEvent,
type ServerProvider,
type UserInputAttachments,
} from "@t3tools/contracts";
import * as DateTime from "effect/DateTime";
Expand All @@ -28,6 +33,7 @@ import {
resolveAttachmentPath,
} from "../attachmentStore.ts";
import * as ServerConfig from "../config.ts";
import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
import * as ServerSettings from "../serverSettings.ts";
import {
OrchestratorCommandPreviouslyRejectedError,
Expand All @@ -41,6 +47,9 @@ const layerIntakeTest = ServerConfig.layerTest(process.cwd(), {
}).pipe(
Layer.provideMerge(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))),
Layer.provideMerge(ServerSettings.layerTest()),
Layer.provideMerge(
Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }),
),
);

const layerFailingDispatchHandler =
Expand Down Expand Up @@ -428,6 +437,112 @@ it.effect("sends a guest message with the guest's files, never ones the client n
}).pipe(Effect.provide(layerIntakeTest)),
);

it.effect.each([false, true])(
"routes an exhausted account while preserving manual resume: %s",
(manual) =>
Effect.gen(function* () {
const claude = (id: string, usedPercent: number): ServerProvider => ({
instanceId: ProviderInstanceId.make(id),
driver: ProviderDriverKind.make("claudeAgent"),
enabled: true,
installed: true,
version: null,
status: "ready",
auth: { status: "authenticated" },
checkedAt: "2026-09-13T00:00:00.000Z",
models: [
{ slug: "claude-opus-5-5", name: "Opus 5.5", isCustom: false, capabilities: null },
],
slashCommands: [],
skills: [],
usageLimits: {
checkedAt: "2026-09-13T00:00:00.000Z",
windows: [
{
id: "five_hour",
kind: "session",
label: "Session",
usedPercent,
resetsAt: "2099-01-01T00:00:00.000Z",
},
],
},
});
const captured: OrchestrationV2ServerCommand[] = [];
const selection = {
instanceId: ProviderInstanceId.make("claude_work"),
model: "claude-opus-5-5",
};
yield* dispatchCommand({
type: "message.dispatch",
commandId: CommandId.make("send-limited"),
threadId: ThreadId.make("thread-limited"),
messageId: MessageId.make("message-limited"),
...(manual ? { manualContinuationOfRunId: RunId.make("limited-run") } : {}),
text: "go",
attachments: [],
dispatchMode: { type: "start_immediately" },
createdBy: "user",
creationSource: "web",
}).pipe(
Effect.provide(
Layer.mergeAll(
Layer.mock(ThreadManagementService.ThreadManagementService)({
dispatch: (command) =>
AccountFallback.pipe(
Effect.flatMap((fallback) => {
expect(fallback).toEqual(
manual
? undefined
: {
commandId: command.commandId,
threadSelection: selection,
requestedSelection: undefined,
},
);
return layerFailingDispatchHandler(captured)(command);
}),
),
getThreadRecords: () =>
Effect.succeed({
thread: {
projectId: ProjectId.make("project-limited"),
modelSelection: selection,
},
} as never),
}),
Layer.mock(ProviderRegistry.ProviderRegistry)({
getProviders: Effect.succeed([
claude("claude_work", 100),
claude("claude_personal", 30),
]),
}),
ServerSettings.layerTest({
fallbackToAvailableAccounts: true,
providerInstances: Object.fromEntries(
["claude_work", "claude_personal"].map((id) => [
ProviderInstanceId.make(id),
{ driver: ProviderDriverKind.make("claudeAgent"), enabled: true },
]),
),
}),
),
),
Effect.result,
);
if (manual) {
expect(captured[0]).not.toHaveProperty("modelSelection");
return;
}
expect(captured[0]).toMatchObject({
modelSelection: {
instanceId: "claude_personal",
model: "claude-opus-5-5",
},
});
}).pipe(Effect.provide(layerIntakeTest)),
);

it.effect("refuses a queued edit that keeps a file another edit already deleted", () =>
Effect.gen(function* () {
const config = yield* ServerConfig.ServerConfig;
Expand Down
Loading
Loading