Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 38 additions & 1 deletion apps/server/src/cli/thread.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { assert, it } from "@effect/vitest";
import { AuthSessionId, ProjectId, RunId, ThreadId } from "@t3tools/contracts";
import { AuthSessionId, ProjectId, RunId, ThreadId, TurnItemId } from "@t3tools/contracts";
import * as HostProcess from "@t3tools/shared/HostProcess";
import * as NetService from "@t3tools/shared/Net";
import * as ConfigProvider from "effect/ConfigProvider";
Expand All @@ -23,6 +23,7 @@ import {
persistServerRuntimeState,
} from "../serverRuntimeState.ts";
import {
countObservedToolCalls,
readLatestMessages,
resolveProject,
threadCommand,
Expand Down Expand Up @@ -528,3 +529,39 @@ it.live("resolves a path inside a git worktree of the project's repository to th
assert.include(ambiguous.message, "feature (project-feature)");
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
);

it("counts each native tool item once on the exact local turn", () => {
const tool = {
visibility: "local" as const,
sourceThreadId: threadId,
runId,
type: "command_execution",
itemId: TurnItemId.make("command"),
};
const otherItems = [
{
...tool,
itemId: TurnItemId.make(
`turn-item:provider:codex:native-item:${encodeURIComponent(`workspace-preparation:${runId}`)}`,
),
},
{ ...tool, visibility: "inherited" as const },
{ ...tool, runId: RunId.make("another-run") },
{ ...tool, sourceThreadId: ThreadId.make("another-thread") },
{ ...tool, type: "assistant_message" },
];
assert.equal(countObservedToolCalls(otherItems, threadId, runId), 0);
assert.equal(
countObservedToolCalls(
[
...otherItems,
tool,
tool,
{ ...tool, type: "dynamic_tool", itemId: TurnItemId.make("read") },
],
threadId,
runId,
),
2,
);
});
87 changes: 87 additions & 0 deletions apps/server/src/cli/thread.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import {
isToolLifecycleItemType,
OrchestratorMcpFailure,
OrchestratorMcpThreadListResult,
OrchestratorMcpThreadReadResult,
Expand Down Expand Up @@ -896,6 +897,91 @@ const threadSendCommand = Command.make("send", {
),
);

export function countObservedToolCalls(
items: ReadonlyArray<
Pick<
OrchestratorMcpThreadTimelineItem,
"visibility" | "sourceThreadId" | "runId" | "type" | "itemId"
>
>,
threadId: ThreadId,
runId: RunId,
): number {
// The runtime emits workspace preparation as a synthetic command_execution.
const workspacePreparationItem = `:native-item:${encodeURIComponent(`workspace-preparation:${runId}`)}`;
return new Set(
items
.filter(
(item) =>
item.visibility === "local" &&
item.sourceThreadId === threadId &&
item.runId === runId &&
!item.itemId.endsWith(workspacePreparationItem) &&
(item.type === "dynamic_tool" || isToolLifecycleItemType(item.type)),
)
.map((item) => item.itemId),
).size;
}

const threadToolUsageCommand = Command.make("tool-usage", {
...projectLocationFlags,
thread: threadArgument,
run: Argument.String("run"),
}).pipe(
Command.withDescription("Print the count observed in the native Work Log for a completed run."),
Command.withHandler((flags) =>
Effect.gen(function* () {
const threadId = yield* decodeThreadId(flags.thread);
yield* withThreadTools({ ...flags, json: true }, (tools) =>
Effect.gen(function* () {
let cursor: number | undefined;
const items: Array<OrchestratorMcpThreadTimelineItem> = [];
for (;;) {
const page = yield* tools.call(
"t3_thread_read",
{
threadId,
view: "activity",
limit: READ_PAGE_LIMIT,
maxCharsPerItem: 1,
runLimit: 1,
...(cursor === undefined ? {} : { afterPosition: cursor }),
},
OrchestratorMcpThreadReadResult,
);
const run = page.recentRuns[0];
if (
page.thread.threadId !== threadId ||
run?.runId !== flags.run ||
run.status !== "completed" ||
page.thread.activeRunId !== null
) {
return yield* new ThreadUsageError({
detail: "Tool evidence requires the latest completed run of an idle thread.",
});
}
items.push(...page.items);
if (!page.hasMore) {
return yield* printJson({
source: "t3-work-log",
threadId,
turnId: run.runId,
toolCalls: countObservedToolCalls(items, threadId, run.runId),
});
}
if (page.nextPosition === null || page.nextPosition <= (cursor ?? -1)) {
return yield* new ThreadUsageError({
detail: "Tool evidence timeline is incomplete.",
});
}
cursor = page.nextPosition;
}
}),
);
}).pipe(reportThreadErrors),
),
);

const threadShowCommand = Command.make("show", {
...projectLocationFlags,
thread: threadArgument,
Expand Down Expand Up @@ -957,5 +1043,6 @@ export const threadCommand = Command.make("thread").pipe(
threadStartCommand,
threadSendCommand,
threadShowCommand,
threadToolUsageCommand,
]),
);
67 changes: 67 additions & 0 deletions apps/web/src/components/ManagedRunEvidence.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import { Link } from "@tanstack/react-router";

import type { ManagedRun } from "~/lib/managedRunDelivery";

function formatCount(value: number | null): string {
return value === null ? "Unknown" : value.toLocaleString();
}

export function ManagedRunEvidence({ run }: { run: ManagedRun }) {
const { brief, budget } = run;
const fields = [
["Run ID", run.runId],
["Status", run.status],
["Brief revision", run.revision],
["Workspace", brief.workspaceId],
["Repository", brief.repository],
["Base commit", brief.baseSha],
["Scope", brief.scope],
["Brief tokens", formatCount(brief.tokens)],
["Reserved tokens", formatCount(budget.reserved)],
["Used tokens", formatCount(budget.used)],
["Remaining tokens", formatCount(budget.remaining)],
];
return (
<article className="mt-4 max-w-2xl rounded-lg border p-4">
<h2 className="font-medium">Managed run · {run.ticket}</h2>
<p className="mt-2 text-sm text-muted-foreground">
Read only. Approval and dispatch controls are unavailable here.
</p>
<h3 className="mt-4 text-sm font-medium">Brief and budget</h3>
<dl className="mt-2 grid grid-cols-[max-content_minmax(0,1fr)] gap-x-6 gap-y-2 text-sm">
{fields.map(([label, value]) => (
<div className="contents" key={label}>
<dt className="text-muted-foreground">{label}</dt>
<dd className="whitespace-pre-wrap break-words">{value}</dd>
</div>
))}
</dl>
<h3 className="mt-4 text-sm font-medium">Stages</h3>
{run.stages.length === 0 ? (
<p className="mt-2 text-sm text-muted-foreground">No stages recorded.</p>
) : (
<ul className="mt-2 space-y-2 text-sm">
{run.stages.map((stage) => (
<li className="rounded-md border p-2 break-words" key={stage.stageId}>
<p className="font-medium">
{stage.role} · {stage.status}
</p>
<p>Stage ID {stage.stageId}</p>
<p>Provider instance {stage.providerInstanceId ?? "Unknown"}</p>
<p>Thread {stage.threadId ?? "Unknown"}</p>
<p>Tool calls {formatCount(stage.toolCalls ?? null)}</p>
</li>
))}
</ul>
)}
<div className="mt-4 flex flex-wrap gap-4 text-sm">
<Link className="underline" to="/settings/providers" search={{}}>
Provider Settings
</Link>
<Link className="underline" to="/usage">
Usage
</Link>
</div>
</article>
);
}
74 changes: 74 additions & 0 deletions apps/web/src/lib/managedRunDelivery.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import { describe, expect, it, vi } from "vite-plus/test";

import { parseManagedRun, parseManagedRunId, readManagedRun } from "./managedRunDelivery";

const runId = "547c0437-b190-4c3f-a703-ae2c3d839e27";
const otherId = "547c0437-b190-4c3f-a703-ae2c3d839e28";
const run = {
version: 1,
runId,
ticket: "T3-03",
revision: "a".repeat(64),
status: "approved",
brief: {
workspaceId: "isolated",
repository: "InTheta/t3code",
baseSha: "b".repeat(40),
scope: "Delivery",
tokens: 1000,
},
budget: { reserved: 1000, used: null, remaining: null },
stages: [
{
stageId: `${runId}:lead`,
role: "lead",
status: "queued",
providerInstanceId: null,
threadId: null,
},
],
};

describe("managed-run Delivery", () => {
it("normalizes UUIDs and rejects values that could change the endpoint path", () => {
expect(parseManagedRunId(` ${runId.toUpperCase()} `)).toBe(runId);
expect(parseManagedRunId(`${runId}/extra`)).toBeUndefined();
});

it("preserves unknown metrics and accepts additive sidecar fields", () => {
expect(parseManagedRun({ ...run, nextVersionField: null }, runId)).toMatchObject(run);
});

it("distinguishes an observed zero from missing tool usage", () => {
expect(parseManagedRun(run, runId)?.stages[0]?.toolCalls).toBeUndefined();
const withUsage = (toolCalls: number | null) => ({
...run,
stages: [{ ...run.stages[0], toolCalls }],
});
expect(parseManagedRun(withUsage(null), runId)?.stages[0]?.toolCalls).toBeNull();
expect(parseManagedRun(withUsage(0), runId)?.stages[0]?.toolCalls).toBe(0);
expect(parseManagedRun(withUsage(3), runId)?.stages[0]?.toolCalls).toBe(3);
expect(parseManagedRun(withUsage(-1), runId)).toBeNull();
});

it("rejects mismatched identity and malformed evidence", () => {
expect(parseManagedRun(run, otherId)).toBeNull();
expect(parseManagedRun({ ...run, version: 2 }, runId)).toBeNull();
expect(parseManagedRun({ ...run, budget: { ...run.budget, used: -1 } }, runId)).toBeNull();
expect(parseManagedRun({ ...run, stages: [{ status: "queued" }] }, runId)).toBeNull();
});

it("reads fresh evidence and treats a disabled sidecar as unavailable", async () => {
const { signal } = new AbortController();
using fetch = vi
.spyOn(globalThis, "fetch")
.mockResolvedValueOnce(Response.json(run))
.mockResolvedValueOnce(new Response(null, { status: 404 }));
await expect(readManagedRun(runId, signal)).resolves.toMatchObject(run);
expect(fetch).toHaveBeenCalledWith(`/delivery-sidecar/v1/managed-runs/${runId}`, {
cache: "no-store",
signal,
});
await expect(readManagedRun(runId, signal)).resolves.toBeNull();
});
});
61 changes: 61 additions & 0 deletions apps/web/src/lib/managedRunDelivery.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import * as Schema from "effect/Schema";

const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
const Tokens = Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }));
const ManagedRun = Schema.Struct({
version: Schema.Literal(1),
runId: Schema.String.check(Schema.isPattern(uuid)),
ticket: Schema.String,
revision: Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)),
status: Schema.String,
brief: Schema.Struct({
workspaceId: Schema.String,
repository: Schema.String,
baseSha: Schema.String.check(Schema.isPattern(/^[0-9a-f]{40}$/)),
scope: Schema.String,
tokens: Tokens,
}),
budget: Schema.Struct({
reserved: Tokens,
used: Schema.NullOr(Tokens),
remaining: Schema.NullOr(Tokens),
}),
stages: Schema.Array(
Schema.Struct({
stageId: Schema.String,
role: Schema.String,
status: Schema.String,
providerInstanceId: Schema.NullOr(Schema.String),
threadId: Schema.NullOr(Schema.String),
toolCalls: Schema.optionalKey(Schema.NullOr(Tokens)),
}),
),
});

export type ManagedRun = typeof ManagedRun.Type;
const isManagedRun = Schema.is(ManagedRun);

export function parseManagedRunId(input: unknown): string | undefined {
if (typeof input !== "string") return undefined;
const normalized = input.trim().toLowerCase();
return uuid.test(normalized) ? normalized : undefined;
}

export function parseManagedRun(input: unknown, requestedRunId: string): ManagedRun | null {
return isManagedRun(input) && input.runId === requestedRunId ? input : null;
}

export async function readManagedRun(
runId: string,
signal: AbortSignal,
): Promise<ManagedRun | null> {
try {
const response = await fetch(`/delivery-sidecar/v1/managed-runs/${encodeURIComponent(runId)}`, {
cache: "no-store",
signal,
});
return response.ok ? parseManagedRun(await response.json(), runId) : null;
} catch {
return null;
}
}
Loading
Loading