Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions apps/server/src/orchestration-v2/Orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6820,9 +6820,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
) {
// A compacted thread reads every run and turn item, but attempts, nodes, messages and provider
// turns only for its recent runs and the runs this command names. An idle compaction reads
// in full: its busy check below looks for a delegated result's message under any old run.
// A compacted thread reads every run, but attempts, nodes, messages, provider turns and prompts
// only for its recent runs and the runs this command names. An idle compaction reads in full:
// its busy check below looks for a delegated result's message under any old run.
let projection = yield* getProjectionWithPendingEvents(
command.threadId,
events,
Expand Down Expand Up @@ -7094,7 +7094,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
}
const userMessages = projection.messages.filter((message) => message.role === "user");
// The recent runs of a compacted thread can hold only a `/compact` while an older prompt
// exists. Every prompt's turn item is loaded; without one, every user message is checked.
// exists, so when every loaded prompt is one, every user message is checked.
const onlyMaintenanceHistory = [
...userMessages,
...projection.turnItems.flatMap((item) => (item.type === "user_message" ? [item] : [])),
Expand Down
105 changes: 94 additions & 11 deletions apps/server/src/orchestration-v2/ProjectionStore.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -857,7 +857,7 @@ it.effect("idle compaction filters keep the latest run, live work and the newest

// One thread holds every way a run can still matter after a compaction. A run is requested at
// its ordinal in minutes and ends 30 seconds later, unless the table says otherwise.
it.effect("a compacted thread reads every run, and run records only for its recent runs", () =>
it.effect("a compacted thread reads every run, but records and prompts only for recent ones", () =>
Effect.gen(function* () {
const store = yield* ProjectionStore.ProjectionStoreV2;
const threadId = yield* addRolledBackRecoveryCandidate("compacted");
Expand Down Expand Up @@ -903,9 +903,10 @@ it.effect("a compacted thread reads every run, and run records only for its rece
driver,
},
});
for (const [ordinal, status, endedAt] of [
for (const [ordinal, status, endedAt, fresh] of [
[2, "queued", null],
[3, "completed"],
// A fresh-context run starts its own conversation, so its prompt is read for the schedule check.
[3, "completed", undefined, true],
// A held queue ran this old run after the compactions.
[4, "completed", at(11.5)],
[5, "completed"],
Expand Down Expand Up @@ -949,6 +950,7 @@ it.effect("a compacted thread reads every run, and run records only for its rece
completedAt: ended,
checkpointId: null,
contextHandoffId: null,
...(fresh === true ? { freshContext: true } : {}),
},
});
yield* store.apply({
Expand Down Expand Up @@ -1084,19 +1086,44 @@ it.effect("a compacted thread reads every run, and run records only for its rece
})),
);
const all = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
// Every run and every turn item is read, so a model switch is still measured from the last
// prompt. Run 1, the helper's, holds only a command item.
const expected = (recent: ReadonlyArray<number>) => ({
// Every run is read. Turn items are the failed errors of every run, and the prompts of the
// recent runs, of the fresh-context run 3 and of the run a model switch is measured from.
// Run 1, the helper's, holds only a command item.
const expected = (
recent: ReadonlyArray<number>,
prompts: ReadonlyArray<number> = [...recent, 3],
) => ({
runs: 15,
attempts: recent,
nodes: recent,
providerTurns: recent,
messages: recent,
turnItems: [1, ...all],
turnItems: [...new Set([5, ...prompts])].toSorted((a, b) => a - b),
});

// Nothing is compacted yet, so the whole history is current.
assert.deepEqual(yield* read(), expected(all));
// Run 5 failed, and nothing is compacted yet, so the whole history is current.
yield* store.apply({
id: EventId.make("event:compacted:error-5"),
type: "turn-item.updated",
threadId,
runId: runId(5),
nodeId: rootNodeId(5),
driver,
occurredAt: at(5),
payload: {
...item(5, 52, "failed"),
id: TurnItemId.make("item:compacted:error-5"),
type: "error",
failure: {
class: "usage_limit",
message: "Plan limit reached.",
resetAt: "2099-01-01T00:00:00.000Z",
code: "usageLimitExceeded",
retryable: null,
},
},
});
assert.deepEqual(yield* read(), { ...expected(all), turnItems: [1, ...all] });

// The window starts at the compaction's run. Queued run 2 and held run 4 are older but had
// not ended by then.
Expand All @@ -1120,6 +1147,20 @@ it.effect("a compacted thread reads every run, and run records only for its rece
expected([2, 3, 4, 6, 11, 12, 13, 14, 15]),
);

// Only an unfiltered read with runs narrows its prompts; any other read keeps its filter.
const itemRuns = (fields: ReadonlyArray<"runs" | "turnItems">, filter: object) =>
store
.getThreadRecords(threadId, fields, {
sinceLastCompaction: { keepRunIds: [], keepMessageId: messageId(0) },
...filter,
})
.pipe(Effect.map((records) => ordinals(records.turnItems.map((row) => row.runId))));
assert.deepEqual(yield* itemRuns(["runs", "turnItems"], { turnItemRunIds: [runId(5)] }), [5]);
assert.deepEqual(yield* itemRuns(["runs", "turnItems"], { turnItemRunIds: [] }), []);
assert.deepEqual(yield* itemRuns(["runs", "turnItems"], { turnItemRunId: runId(5) }), [5]);
assert.deepEqual(yield* itemRuns(["runs", "turnItems"], { runIds: [runId(15)] }), [1, ...all]);
assert.deepEqual(yield* itemRuns(["turnItems"], {}), [1, ...all]);

// A command also skips the node and the provider thread of each settled task. Tasks ended on
// both sides of the compaction; one ran in the provider thread the thread's own runs use.
const addProviderThread = (id: ProviderThreadId, appThreadId: ThreadId) =>
Expand Down Expand Up @@ -1266,11 +1307,12 @@ it.effect("a compacted thread reads every run, and run records only for its rece
assert.deepEqual(unsettled.providerThreads, ["compacted", "old-pending", "recent-pending"]);
assert.lengthOf(unsettled.rootNodes, all.length);
assert.lengthOf(unsettled.runs, 15);
// Together: every run, every turn item and the provider thread the runs use stay, and the
// windowed records are read for the recent and named runs, less each settled task's node.
// Together: every run and the provider thread the runs use stay, and the windowed records are
// read for the recent and named runs, less each settled task's node.
const recentRuns = ["11", "12", "13", "14", "15", "2", "3", "4", "6"];
assert.deepEqual(both, {
...unsettled,
turnItems: windowed.turnItems,
attempts: recentRuns,
rootNodes: recentRuns,
taskNodes: ["recent-pending"],
Expand All @@ -1287,6 +1329,47 @@ it.effect("a compacted thread reads every run, and run records only for its rece
),
both,
);

// A prompt that clients do not show cannot anchor a model switch. With the recent ones, and
// run 4's, folded into async answers, the switch is measured from run 10, so its prompt is
// read, with run 9's from the same batch.
const storedRuns = (yield* store.getThreadRecords(threadId, ["runs"])).runs;
for (const ordinal of [4, 11, 12, 14, 15]) {
yield* store.apply({
id: EventId.make(`event:compacted:async-run-${ordinal}`),
type: "run.updated",
threadId,
occurredAt: at(ordinal),
payload: {
...storedRuns.find((run) => run.id === runId(ordinal))!,
userMessageId: MessageId.make(`async-answer:${ordinal}`),
},
});
yield* store.apply({
id: EventId.make(`event:compacted:async-${ordinal}`),
type: "turn-item.updated",
threadId,
runId: runId(ordinal),
nodeId: rootNodeId(ordinal),
driver,
occurredAt: at(ordinal),
payload: {
createdBy: "user",
creationSource: "web",
...item(ordinal, ordinal * 10, "completed"),
id: TurnItemId.make(`item:compacted:prompt-${ordinal}`),
type: "user_message",
messageId: MessageId.make(`async-answer:${ordinal}`),
inputIntent: "turn_start",
text: `Answer ${ordinal}`,
attachments: [],
},
});
}
assert.deepEqual(
(yield* read()).turnItems,
expected([2, 4, 11, 12, 13, 14, 15], [2, 3, 4, 9, 10, 11, 12, 13, 14, 15]).turnItems,
);
}).pipe(Effect.provide(layerTest)),
);

Expand Down
103 changes: 95 additions & 8 deletions apps/server/src/orchestration-v2/ProjectionStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,16 @@ import type {
import {
latestRootProviderFailure,
latestUnheldRun,
runRanAfter,
threadErrorSummary,
usageLimitRunPresentedAsLatest,
} from "@t3tools/shared/orchestrationV2ThreadError";
import { threadPullRequestsOf } from "../sourceControl/threadPullRequests.ts";
import {
anchorsModelSwitch,
isModelSwitchCandidate,
isShownUserMessageItem,
} from "./modelSwitch.ts";
import type {
OrchestrationV2AppThread,
OrchestrationV2Checkpoint,
Expand Down Expand Up @@ -346,6 +352,17 @@ export interface ProjectionRuntimeResponseContext {
readonly session: OrchestrationV2ThreadProjection["providerSessions"][number] | undefined;
}

const PROMPT_NARROWING_BLOCKING_FILTERS = [
"runIds",
"runStatuses",
"latestRunLimit",
"turnItemRunId",
"turnItemRunIds",
"turnItemNodeIds",
"turnItemRequestIds",
"turnItemStatuses",
] as const satisfies ReadonlyArray<keyof ProjectionRecordFilter>;

export interface ProjectionRecordFilter {
readonly providerThreadIds?: ReadonlyArray<ProviderThreadId>;
readonly providerTurnIds?: ReadonlyArray<ProviderTurnId>;
Expand Down Expand Up @@ -390,9 +407,10 @@ export interface ProjectionRecordFilter {
*/
readonly omitSettledSubagentRecords?: boolean;
/**
* For a compacted thread: reads attempts, nodes, messages and provider turns only for runs from
* the latest completed compaction's run on, runs that had not ended when that run was
* requested, and the runs named here. Runs and turn items are not narrowed.
* For a compacted thread, reads attempts, nodes, messages and provider turns of the recent runs
* and the runs named here. Turn items are the failed errors of every run and the prompts of the
* recent, fresh-context and model-switch runs. `turnItemTypes` does not apply. A read without `runs`, or with a run or turn-item
* filter, keeps its filtered items. Runs stay whole.
*/
readonly sinceLastCompaction?: {
readonly keepRunIds: ReadonlyArray<RunId>;
Expand Down Expand Up @@ -3646,7 +3664,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
const threadColumn = (ids: ReadonlyArray<unknown> | undefined, column = "thread_id") =>
sql.literal(ids === undefined ? column : `+${column}`);

/** The runs whose records a new message reads. Null when nothing was compacted. */
/** The ids of the runs whose records a new message reads. Null when nothing was compacted. */
const runIdsSinceLastCompaction = Effect.fn("ProjectionStore.runIdsSinceLastCompaction")(
function* (
threadId: ThreadId,
Expand Down Expand Up @@ -3685,10 +3703,67 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
// The compaction's own run is always recent, so no rows means no compaction.
return recent.length === 0
? null
: encodeIdList([...recent.map((row) => row.run_id), ...keep.keepRunIds]);
: [...recent.map((row) => row.run_id), ...keep.keepRunIds];
},
);

/**
* What a command reads of a compacted thread's prompts and errors: the failed errors of every
* run, and the prompts of the recent runs, of fresh-context runs, and of the newest runs back
* to the one `modelSwitchSince` measures from. That one can be older than the compaction.
*/
const commandTurnItems = Effect.fn("ProjectionStore.commandTurnItems")(function* (
threadId: ThreadId,
runs: ReadonlyArray<OrchestrationV2Run>,
recentRunIds: ReadonlyArray<RunId>,
) {
const errors = yield* sql<PayloadRow>`
SELECT payload_json
FROM orchestration_v2_projection_turn_items
INDEXED BY orchestration_v2_projection_turn_items_failed_error_idx
WHERE thread_id = ${threadId} AND type = 'error' AND status = 'failed'
`.pipe(Effect.flatMap(decodeRows(decodeTurnItemPayload, threadId)));
const readPrompts = (runIds: ReadonlyArray<string>) =>
sql<PayloadRow>`
SELECT payload_json
FROM orchestration_v2_projection_turn_items
INDEXED BY orchestration_v2_projection_turn_items_thread_run_idx
WHERE thread_id = ${threadId} AND type = 'user_message'
AND run_id IN (SELECT value FROM json_each(${encodeIdList(runIds)}))
`.pipe(Effect.flatMap(decodeRows(decodeTurnItemPayload, threadId)));
const read = new Set<string>(recentRunIds);
for (const run of runs) if (run.freshContext === true) read.add(run.id);
const prompts = yield* readPrompts([...read]);
const shownMessageIds = new Set<string>();
const showPrompts = (items: ReadonlyArray<OrchestrationV2TurnItem>) => {
for (const item of items) {
if (item.type === "user_message" && isShownUserMessageItem(item)) {
shownMessageIds.add(item.messageId);
}
}
};
showPrompts(prompts);
const newestFirst = runs
.filter(isModelSwitchCandidate)
.toSorted((left, right) => (runRanAfter(left, right) ? -1 : 1));
// The batches double, so a thread with no such run takes few queries.
for (let start = 0, size = 1; start < newestFirst.length; start += size, size *= 2) {
const batch = newestFirst.slice(start, start + size);
const unread = batch.filter((run) => !read.has(run.id));
if (unread.length > 0) {
for (const run of unread) read.add(run.id);
const more = yield* readPrompts(unread.map((run) => run.id));
showPrompts(more);
prompts.push(...more);
}
if (batch.some((run) => anchorsModelSwitch(run, shownMessageIds))) break;
}
return [...errors, ...prompts].toSorted(
(left, right) =>
left.ordinal - right.ordinal || (left.id < right.id ? -1 : left.id > right.id ? 1 : 0),
);
});

const readCanonicalProjection = Effect.fn("ProjectionStore.readCanonicalProjection")(
function* (
threadId: ThreadId,
Expand All @@ -3706,10 +3781,15 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
if (!threadRow) {
return yield* new ProjectionStoreThreadNotFoundError({ threadId });
}
const recentRunIds =
const recentRuns =
window === undefined && filter?.sinceLastCompaction !== undefined
? yield* runIdsSinceLastCompaction(threadId, filter.sinceLastCompaction)
: null;
const recentRunIds = recentRuns === null ? null : encodeIdList(recentRuns);
const readsWholeHistory =
(fields === undefined || fields.includes("runs")) &&
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
PROMPT_NARROWING_BLOCKING_FILTERS.every((key) => filter?.[key] === undefined);
const promptRuns = readsWholeHistory ? recentRuns : null;
const inRecentRuns = (column: string) =>
recentRunIds === null
? sql``
Expand All @@ -3727,6 +3807,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
};
const boundedTurnItemRows: ReadonlyArray<WindowPayloadRow> =
(fields !== undefined && !fields.includes("turnItems")) ||
// A compacted thread reads its items once its runs are known.
promptRuns !== null ||
(window === undefined &&
matchesNothing(
filter?.turnItemRequestIds,
Expand Down Expand Up @@ -3776,7 +3858,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
decodeTurnItemPayload,
threadId,
)(boundedTurnItemRows);
const turnItems = decodedTurnItems.map((item, index) => {
const storedTurnItems = decodedTurnItems.map((item, index) => {
const row = boundedTurnItemRows[index]!;
return row.work_last_id == null ||
row.work_item_count == null ||
Expand All @@ -3799,7 +3881,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
});
const cohortJson = (field: string) =>
JSON.stringify(
turnItems.flatMap((payload) => {
storedTurnItems.flatMap((payload) => {
const value = nullableStringField(payload, field);
return value === null ? [] : [value];
}),
Expand Down Expand Up @@ -4442,12 +4524,17 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
decodeRows(decodeContextHandoffPayload, threadId)(contextHandoffRows),
decodeRows(decodeContextTransferPayload, threadId)(contextTransferRows),
]);
const turnItems =
promptRuns === null || (fields !== undefined && !fields.includes("turnItems"))
? storedTurnItems
: yield* commandTurnItems(threadId, runs, promptRuns);
const messageOrdinals = new Map<string, number>();
// Only user items can name a user message, so a user-only read needs no assistant items.
const userOnly = messages.every((message) => message.role === "user");
// The loaded items already order the messages when they hold every item the lookup reads.
const loadsEveryItemOf = (type: OrchestrationV2TurnItem["type"]) =>
window === undefined &&
promptRuns === null &&
(fields === undefined || fields.includes("turnItems")) &&
filter?.turnItemRequestIds === undefined &&
filter?.turnItemStatuses === undefined &&
Expand Down
2 changes: 1 addition & 1 deletion apps/server/src/orchestration-v2/contextBoundary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ export function isScheduledFreshRun(
run: Pick<OrchestrationV2Run, "freshContext" | "userMessageId">,
turnItems: OrchestrationV2ThreadProjection["turnItems"],
): boolean {
// Turn items, unlike messages, are not narrowed to the runs since the latest compaction.
// A compacted thread's command read loads these prompts for every fresh-context run.
return (
run.freshContext === true &&
turnItems.some(
Expand Down
Loading
Loading