diff --git a/app/README.md b/app/README.md index 596b7e8..9b62dae 100644 --- a/app/README.md +++ b/app/README.md @@ -25,9 +25,16 @@ browser (React + useAgentChat over WebSocket) endpoint. Connections use the MCP v2 client (`@modelcontextprotocol/client` 2.x, stateless streamable HTTP). Identity travels as `x-user-netid` headers set per connection; changing netid reconnects everything. -- **Models**: Claude Opus 5 by default, with Sonnet 5 and a Workers AI - fallback ("Campus") in the switcher. Claude models require the - `ANTHROPIC_API_KEY` secret; without it every choice falls back to Campus. +- **Models**: Claude Sonnet 5.5 by default, with Haiku 5.5 and Opus 5 in the + switcher. Existing Opus choices are preserved. Saved Sonnet 5 and retired + model choices resolve to Sonnet 5.5. All models require `ANTHROPIC_API_KEY`. + Prompt caching marks the last system block (covering tool definitions) and + the last conversation block with the default five-minute cache TTL. + Reuse requires an unchanged prefix, the same model, and enough prompt tokens. + Check `cacheRead` in `pi.usage` after a follow-up step or turn to verify a hit. + The weekly budget restricts Opus to Sonnet 5.5 after tier one and preserves + Haiku choices. Haiku usage is priced per request, including its higher rates + above 100,000 prompt tokens, before costs are summed for the turn. - **Identity**: Princeton CAS sign-in (`src/server/auth.ts`, using `https://authenticate.princeton.edu/cas`). The Worker validates service tickets through `/p3/serviceValidate` over HTTPS and takes the NetID from diff --git a/app/src/client/lib/store.ts b/app/src/client/lib/store.ts index e9ca7f3..3424846 100644 --- a/app/src/client/lib/store.ts +++ b/app/src/client/lib/store.ts @@ -2,7 +2,8 @@ import { useSyncExternalStore } from "react"; import { COVERED_BY_JUNCTION, DEFAULT_APPS, - PI_MODELS, + DEFAULT_MODEL, + resolveModel, type AppKey, type PiModel, type PiSettings, @@ -33,7 +34,7 @@ export const GENERAL_CHAT_TITLE = "PI general chat"; /** User-tweakable preferences; identity is never stored here. */ export type Prefs = { apps: AppKey[]; model: PiModel }; -const DEFAULT_PREFS: Prefs = { apps: DEFAULT_APPS, model: "claude-opus-5" }; +const DEFAULT_PREFS: Prefs = { apps: DEFAULT_APPS, model: DEFAULT_MODEL }; /** * Stamped on every saved blob, so a one-time migration runs once. Bump it only @@ -96,9 +97,7 @@ function getPrefs(netid: string): Prefs { const stored: StoredPrefs = raw ? JSON.parse(raw) : {}; let merged: Prefs = { ...DEFAULT_PREFS, ...stored }; // A retired model choice (e.g. the old "campus") falls back to default. - if (!PI_MODELS.some((m) => m.value === merged.model)) { - merged.model = DEFAULT_PREFS.model; - } + merged.model = resolveModel(merged.model); if (raw) { merged = migrate(stored, merged); // Written back straight away, stamp and all: a migration that reruns diff --git a/app/src/client/pages/ChatPage.tsx b/app/src/client/pages/ChatPage.tsx index 9dd8ff3..ab08902 100644 --- a/app/src/client/pages/ChatPage.tsx +++ b/app/src/client/pages/ChatPage.tsx @@ -156,7 +156,7 @@ function ChatBody({ // a consent card must not call it on until it has. const [engineNote, setEngineNote] = useState(null); const [gcalReady, setGcalReady] = useState(false); - /** Why answers changed: Sonnet only, or nothing left this week. */ + /** Why answers changed: restricted model choices, or nothing left this week. */ const [usageNote, setUsageNote] = useState(null); const agent = useAgent({ agent: "pi", diff --git a/app/src/server/pi.ts b/app/src/server/pi.ts index 9d747ff..7eaff99 100644 --- a/app/src/server/pi.ts +++ b/app/src/server/pi.ts @@ -2,20 +2,21 @@ import { createAnthropic } from "@ai-sdk/anthropic"; import { Think } from "@cloudflare/think"; import { callable, getAgentByName } from "agents"; import type { AgentMcpOAuthProvider } from "agents/mcp/do-oauth-client-provider"; -import { hasToolCall, tool, type UIMessage } from "ai"; +import { hasToolCall, tool, wrapLanguageModel, type UIMessage } from "ai"; import { z } from "zod"; import { EMPTY_WORKSPACES, workspaceContext, type WorkspaceState } from "../shared/workspaces"; import { handleWorkspaceRequest, WORKSPACE_KEY } from "./workspaces"; import { - TIER2_MODEL, budgetFrom, limitNote, + modelForBudget, usageStatus, type Budget, type Spend, type UsageStatus, } from "../shared/usage-limit"; import { NOTICE_MODEL_ID, noticeModel } from "./notice-model"; +import { promptCache } from "./prompt-cache"; import { addStep, costUsd, @@ -33,10 +34,13 @@ import { import { COVERED_BY_JUNCTION, DEFAULT_ENGINE_BASE, + DEFAULT_MODEL, PI_APPS, + resolveModel, toolOwner, type AppKey, type PiApp, + type PiModel, type PiSettings, } from "../shared/apps"; @@ -59,9 +63,6 @@ export type PiState = { usage?: UsageStatus; }; -const CLAUDE_MODELS = new Set(["claude-opus-5", "claude-sonnet-5"]); -const DEFAULT_CLAUDE_MODEL = "claude-opus-5"; - /** The apps PI may ask a student to switch on. */ const APP_KEYS = PI_APPS.map((a) => a.key) as [AppKey, ...AppKey[]]; @@ -238,7 +239,7 @@ export class Pi extends Think { private turnUsage: TurnUsage | null = null; private turnStartedAt = 0; /** The model getModel() resolved, for pricing the turn. */ - private turnModel = DEFAULT_CLAUDE_MODEL; + private turnModel: string = DEFAULT_MODEL; getDefaultTimezone() { return "America/New_York"; @@ -251,10 +252,8 @@ export class Pi extends Think { } /** The Claude model the student picked in the composer. */ - private selectedModel(): string { - const preference = this.getConfig()?.model ?? DEFAULT_CLAUDE_MODEL; - // Stale configs may still carry the retired "campus" preference. - return CLAUDE_MODELS.has(preference) ? preference : DEFAULT_CLAUDE_MODEL; + private selectedModel(): PiModel { + return resolveModel(this.getConfig()?.model); } private claude(model: string) { @@ -263,7 +262,9 @@ export class Pi extends Think { // No silent fallback model: fail the turn loudly so ops notices. throw new Error("ANTHROPIC_API_KEY is not configured"); } - return createAnthropic({ apiKey: key })(model); + // Every Claude call goes through here, so this is where prompt caching + // lands: both the composer's model and the budget downgrade. + return wrapLanguageModel({ model: createAnthropic({ apiKey: key })(model), middleware: promptCache }); } getSystemPrompt() { @@ -571,11 +572,10 @@ export class Pi extends Think { maxSteps: 1, }; } - // Past the Opus budget, every answer comes from Sonnet whatever the - // composer says, the same way PrincetonCourses steps down. + // Tier two restricts Opus, but keeps a student's cheaper Haiku selection. const selected = this.selectedModel(); - const downgrade = usage?.tier === 2 && selected !== TIER2_MODEL; - this.turnModel = downgrade ? TIER2_MODEL : selected; + const allowed = modelForBudget(selected, usage); + this.turnModel = allowed; const workspaceNote = await (await this.deskStub(settings.netid)).getWorkspaceContext( this.name.slice(`u-${settings.netid}-`.length) ); @@ -640,7 +640,7 @@ export class Pi extends Think { : []; return { ...(inherited ?? {}), - ...(downgrade ? { model: this.claude(TIER2_MODEL) } : {}), + ...(allowed !== selected ? { model: this.claude(allowed) } : {}), // The system prompt is assembled before this hook runs, so a turn that // lost an app can only correct the record here — otherwise the model // reads "Connected TigerApps: TigerJunction" and answers with the diff --git a/app/src/server/prompt-cache.ts b/app/src/server/prompt-cache.ts new file mode 100644 index 0000000..ea5eb2a --- /dev/null +++ b/app/src/server/prompt-cache.ts @@ -0,0 +1,74 @@ +import type { LanguageModelMiddleware } from "ai"; + +/** + * Anthropic prompt caching for every Claude call PI makes. + * + * Each step of a turn resends the tool definitions, the system prompt and the + * whole conversation. Without cache breakpoints Anthropic bills all of it at + * the full input price every time (measured 2026-10-29: zero cache reads on + * the production key). Two breakpoints fix that: + * + * 1. The last system message. Anthropic caches the prefix up to a breakpoint, + * and the prefix is tools, then system, then messages, so this one covers + * the tool definitions and the system prompt. Reused while the entire + * prefix is unchanged, including the student's identity and workspace. + * 2. The last message. The next step (tool result) or the next turn extends + * the conversation, so the previous breakpoint can become a cache read and the + * new last message takes over. Anthropic looks back up to 20 blocks for a + * matching breakpoint. Changes to earlier content or the model, expiry, + * and growth beyond that window can cause a miss. + * + * The default cache TTL is five minutes. Prompts under the model's minimum + * are silently not cached (512 tokens on Sonnet 5.5 and Haiku 5.5). + * Verify reuse through cacheRead in pi.usage rather than assuming a hit. + * https://platform.claude.com/docs/en/build-with-claude/prompt-caching + */ + +type Params = Parameters>[0]["params"]; +export type Prompt = Params["prompt"]; +type Message = Prompt[number]; + +const EPHEMERAL = { cacheControl: { type: "ephemeral" } } as const; + +/** True when the message already carries an Anthropic cache breakpoint, ours or anyone's. */ +function hasBreakpoint(message: Message): boolean { + const anthropic = message.providerOptions?.anthropic as Record | undefined; + return Boolean(anthropic && (anthropic.cacheControl || anthropic.cache_control)); +} + +function withBreakpoint(message: Message): Message { + return { + ...message, + providerOptions: { + ...(message.providerOptions ?? {}), + anthropic: { ...((message.providerOptions?.anthropic as Record) ?? {}), ...EPHEMERAL }, + }, + } as Message; +} + +/** + * The prompt with the two breakpoints above. Pure, so it can be tested + * without a model: never adds a breakpoint a message already has, never + * marks the same message twice, and leaves an empty prompt alone. + */ +export function withCacheBreakpoints(prompt: Prompt): Prompt { + if (prompt.length === 0) return prompt; + const out = [...prompt]; + let lastSystem = -1; + for (let i = 0; i < out.length; i++) { + if (out[i].role === "system") lastSystem = i; + } + if (lastSystem >= 0 && !hasBreakpoint(out[lastSystem])) { + out[lastSystem] = withBreakpoint(out[lastSystem]); + } + const last = out.length - 1; + if (last !== lastSystem && !hasBreakpoint(out[last])) { + out[last] = withBreakpoint(out[last]); + } + return out; +} + +/** Wrap an Anthropic model with `wrapLanguageModel({ model, middleware: promptCache })`. */ +export const promptCache: LanguageModelMiddleware = { + transformParams: async ({ params }) => ({ ...params, prompt: withCacheBreakpoints(params.prompt) }), +}; diff --git a/app/src/server/usage.ts b/app/src/server/usage.ts index ac0f15f..d6ee535 100644 --- a/app/src/server/usage.ts +++ b/app/src/server/usage.ts @@ -6,7 +6,9 @@ * the cost analysis reads (Workers Logs parses JSON lines into fields). */ -/** USD per million tokens, from Anthropic's price list. */ +/** USD per million tokens, verified 2026-11-05. + * https://platform.claude.com/docs/en/about-claude/pricing + */ type Price = { input: number; output: number; @@ -21,7 +23,8 @@ export const MODEL_PRICES: Record = { "claude-opus-5": { input: 5, output: 25, cacheRead: 0.5, cacheWrite5m: 6.25, cacheWrite1h: 10 }, "claude-sonnet-5": { input: 2, output: 10, cacheRead: 0.2, cacheWrite5m: 2.5, cacheWrite1h: 4 }, "claude-opus-5-5": { input: 4, output: 20, cacheRead: 0.2, cacheWrite5m: 5, cacheWrite1h: 8 }, - "claude-sonnet-5-5": { input: 2, output: 10, cacheRead: 0.2, cacheWrite5m: 2.5, cacheWrite1h: 4 }, + "claude-sonnet-5-5": { input: 2, output: 10, cacheRead: 0.1, cacheWrite5m: 2.5, cacheWrite1h: 4 }, + "claude-haiku-5-5": { input: 0.1, output: 0.5, cacheRead: 0.01, cacheWrite5m: 0.125, cacheWrite1h: 0.2 }, }; /** Anthropic's server-side web search: $10 per 1,000 searches. */ @@ -39,8 +42,18 @@ export type TurnUsage = { output: number; reasoning: number; webSearches: number; + /** Tokens from individual requests above Haiku 5.5's 100k prompt threshold. + * Kept separately so a long step does not reprice earlier short steps. + */ + longContext?: TokenCounts; }; +const TOKEN_KINDS = ["input", "cacheRead", "cacheWrite5m", "cacheWrite1h", "output"] as const; +type TokenCounts = Pick; + +const tokenCost = (usage: TokenCounts, price: Price): number => + TOKEN_KINDS.reduce((sum, kind) => sum + usage[kind] * price[kind], 0); + export function emptyTurnUsage(): TurnUsage { return { steps: 0, @@ -91,15 +104,28 @@ export function addStep(total: TurnUsage, usage: StepUsage | undefined): TurnUsa noCache !== undefined ? n(noCache) : Math.max(0, n(usage?.inputTokens) - cacheRead - cacheWrite); + const output = n(usage?.outputTokens); + let longContext = total.longContext; + // The threshold counts all prompt tokens, including reads and writes. + if (input + cacheRead + cacheWrite > 100_000) { + longContext = { + input: (longContext?.input ?? 0) + input, + cacheRead: (longContext?.cacheRead ?? 0) + cacheRead, + cacheWrite5m: (longContext?.cacheWrite5m ?? 0) + cacheWrite - cacheWrite1h, + cacheWrite1h: (longContext?.cacheWrite1h ?? 0) + cacheWrite1h, + output: (longContext?.output ?? 0) + output, + }; + } return { steps: total.steps + 1, input: total.input + input, cacheRead: total.cacheRead + cacheRead, cacheWrite5m: total.cacheWrite5m + cacheWrite - cacheWrite1h, cacheWrite1h: total.cacheWrite1h + cacheWrite1h, - output: total.output + n(usage?.outputTokens), + output: total.output + output, reasoning: total.reasoning + n(usage?.outputTokenDetails?.reasoningTokens), webSearches: total.webSearches + n(raw.server_tool_use?.web_search_requests), + ...(longContext ? { longContext } : {}), }; } @@ -110,12 +136,12 @@ export function addStep(total: TurnUsage, usage: StepUsage | undefined): TurnUsa export function costUsd(model: string, usage: TurnUsage): number | null { const price = MODEL_PRICES[model]; if (!price) return null; - const tokens = - usage.input * price.input + - usage.cacheRead * price.cacheRead + - usage.cacheWrite5m * price.cacheWrite5m + - usage.cacheWrite1h * price.cacheWrite1h + - usage.output * price.output; + let tokens = tokenCost(usage, price); + // Haiku charges 5x the base token rates for each request above 100k. + // The base cost is already counted, so add only the remaining 4x. + if (model === "claude-haiku-5-5" && usage.longContext) { + tokens += 4 * tokenCost(usage.longContext, price); + } return tokens / 1_000_000 + usage.webSearches * WEB_SEARCH_USD; } diff --git a/app/src/shared/apps.ts b/app/src/shared/apps.ts index b5a7334..cdcf3f6 100644 --- a/app/src/shared/apps.ts +++ b/app/src/shared/apps.ts @@ -291,13 +291,21 @@ export function countConnected( export const DEFAULT_ENGINE_BASE = "https://junction-engine.tigerapps.org"; /** Models the switcher offers. */ -export type PiModel = "claude-opus-5" | "claude-sonnet-5"; +export type PiModel = "claude-opus-5" | "claude-sonnet-5-5" | "claude-haiku-5-5"; + +export const DEFAULT_MODEL: PiModel = "claude-sonnet-5-5"; export const PI_MODELS: Array<{ value: PiModel; label: string }> = [ + { value: "claude-sonnet-5-5", label: "Sonnet 5.5, balanced" }, + { value: "claude-haiku-5-5", label: "Haiku 5.5, fastest" }, { value: "claude-opus-5", label: "Opus 5, sharpest" }, - { value: "claude-sonnet-5", label: "Sonnet 5, quicker" }, ]; +/** Shared by persisted browser preferences and the server's model allowlist. */ +export function resolveModel(preference: unknown): PiModel { + return PI_MODELS.find((model) => model.value === preference)?.value ?? DEFAULT_MODEL; +} + /** Settings the client pushes to a Pi agent instance. */ export type PiSettings = { netid: string; @@ -308,5 +316,5 @@ export type PiSettings = { export const DEFAULT_SETTINGS: PiSettings = { netid: "", apps: DEFAULT_APPS, - model: "claude-opus-5", + model: DEFAULT_MODEL, }; diff --git a/app/src/shared/usage-limit.ts b/app/src/shared/usage-limit.ts index 76afb5d..1bd963b 100644 --- a/app/src/shared/usage-limit.ts +++ b/app/src/shared/usage-limit.ts @@ -1,3 +1,5 @@ +import { DEFAULT_MODEL, type PiModel } from "./apps"; + /** * Each student's weekly model budget, in dollars. The tiers are copied from * PrincetonCourses' ask gateway (tiger-junction, apps/ask-gateway/app/ @@ -10,7 +12,7 @@ export type Budget = { /** Spend allowed on any model, Opus included. */ tier1Usd: number; - /** Further spend allowed on Sonnet only, once tier 1 is used up. */ + /** Further spend allowed on Sonnet or Haiku, once tier 1 is used up. */ tier2Usd: number; windowMs: number; }; @@ -19,8 +21,13 @@ export const WEEK_MS = 7 * 24 * 60 * 60 * 1000; export const DEFAULT_BUDGET: Budget = { tier1Usd: 3, tier2Usd: 1, windowMs: WEEK_MS }; -/** The model a student drops to once their Opus budget is spent. */ -export const TIER2_MODEL = "claude-sonnet-5"; +/** The model a student drops to once tier 1 is spent. */ +export const TIER2_MODEL = DEFAULT_MODEL; + +/** Restrict Opus after tier one without upgrading a cheaper Haiku choice. */ +export function modelForBudget(selected: PiModel, usage: UsageStatus | null): PiModel { + return usage?.tier === 2 && selected === "claude-opus-5" ? TIER2_MODEL : selected; +} /** * The dollar amounts can be changed per deploy (Worker vars) without a code @@ -47,7 +54,7 @@ export type Spend = { at: number; usd: number }; export type UsageStatus = { spentUsd: number; limitUsd: number; - /** 1: any model. 2: Opus budget spent, Sonnet only. Exhausted: nothing left. */ + /** 1: any model. 2: Sonnet or Haiku. Exhausted: nothing left. */ tier: 1 | 2 | "exhausted"; blocked: boolean; percentUsed: number; @@ -121,8 +128,8 @@ export function limitNote(status: UsageStatus): string | null { } if (status.tier === 2) { return when - ? `You've used this week's Opus budget, so PI is answering with Sonnet 5 until ${when}.` - : "You've used this week's Opus budget, so PI is answering with Sonnet 5 for now."; + ? `You've used most of this week's PI budget. Sonnet 5.5 and Haiku 5.5 are available until ${when}.` + : "You've used most of this week's PI budget. Sonnet 5.5 and Haiku 5.5 are available for now."; } return null; } diff --git a/app/tests/models.test.ts b/app/tests/models.test.ts new file mode 100644 index 0000000..0b943f8 --- /dev/null +++ b/app/tests/models.test.ts @@ -0,0 +1,28 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { DEFAULT_SETTINGS, PI_MODELS, resolveModel } from "../src/shared/apps.ts"; +import { modelForBudget, usageStatus } from "../src/shared/usage-limit.ts"; + +test("Sonnet 5.5 is the default for new and retired model preferences", () => { + assert.equal(DEFAULT_SETTINGS.model, "claude-sonnet-5-5"); + for (const preference of [undefined, null, "campus", "claude-sonnet-5", "unknown"]) { + assert.equal(resolveModel(preference), "claude-sonnet-5-5"); + } +}); + +test("the picker and model resolver preserve Haiku and existing Opus choices", () => { + assert.ok(PI_MODELS.some((model) => model.value === "claude-haiku-5-5")); + assert.equal(resolveModel("claude-haiku-5-5"), "claude-haiku-5-5"); + assert.equal(resolveModel("claude-opus-5"), "claude-opus-5"); + for (const model of PI_MODELS) assert.equal(resolveModel(model.value), model.value); +}); + +test("tier two downgrades Opus to Sonnet 5.5 without upgrading Haiku", () => { + const status = usageStatus([{ at: 1, usd: 3.25 }], 2); + assert.equal(status.tier, 2); + assert.equal(modelForBudget("claude-opus-5", status), "claude-sonnet-5-5"); + assert.equal(modelForBudget("claude-sonnet-5-5", status), "claude-sonnet-5-5"); + assert.equal(modelForBudget("claude-haiku-5-5", status), "claude-haiku-5-5"); + assert.equal(modelForBudget("claude-opus-5", usageStatus([], 2)), "claude-opus-5"); + assert.equal(modelForBudget("claude-haiku-5-5", null), "claude-haiku-5-5"); +}); diff --git a/app/tests/prompt-cache-wire.test.ts b/app/tests/prompt-cache-wire.test.ts new file mode 100644 index 0000000..c4f9982 --- /dev/null +++ b/app/tests/prompt-cache-wire.test.ts @@ -0,0 +1,143 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createAnthropic } from "@ai-sdk/anthropic"; +import { generateText, streamText, tool, wrapLanguageModel } from "ai"; +import { z } from "zod"; +import { promptCache } from "../src/server/prompt-cache.ts"; +import { PI_MODELS, type PiModel } from "../src/shared/apps.ts"; +import { addStep, emptyTurnUsage } from "../src/server/usage.ts"; + +// The provider's request body, captured by a fake fetch, so the test proves the +// breakpoints reach Anthropic's wire format without touching the network. +async function capture(modelId: PiModel, run: (model: ReturnType>) => Promise) { + let body: Record | null = null; + const anthropic = createAnthropic({ + apiKey: "test", + fetch: async (_url, init) => { + body = JSON.parse(String(init?.body)); + if (body?.stream) { + const usage = { input_tokens: 10, output_tokens: 1, cache_read_input_tokens: 2_000, cache_creation_input_tokens: 1_000 }; + const events = [ + { type: "message_start", message: { + id: "msg_1", type: "message", role: "assistant", model: modelId, + content: [], stop_reason: null, stop_sequence: null, usage, + } }, + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "ok" } }, + { type: "content_block_stop", index: 0 }, + { type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage }, + { type: "message_stop" }, + ]; + return new Response(events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join(""), { + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response( + JSON.stringify({ + id: "msg_1", + type: "message", + role: "assistant", + model: modelId, + content: [{ type: "text", text: "ok" }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 10, output_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + }, + }); + await run(wrapLanguageModel({ model: anthropic(modelId), middleware: promptCache }) as never); + assert.ok(body, "the provider made a request"); + return body as unknown as { + model: string; + stream?: boolean; + temperature?: number; + top_p?: number; + top_k?: number; + system?: Array<{ text: string; cache_control?: { type: string } }>; + messages: Array<{ role: string; content: Array<{ type: string; cache_control?: { type: string } }> }>; + tools?: Array<{ name: string; cache_control?: { type: string } }>; + }; +} + +for (const { value: modelId } of PI_MODELS) { + test(`${modelId}: system prompt and the last user message carry cache_control on the wire`, async () => { + const body = await capture(modelId, (model) => + generateText({ + model, + system: "You are PI, the TigerApps assistant.", + messages: [ + { role: "user", content: "what's due" }, + { role: "assistant", content: "Nothing today." }, + { role: "user", content: "and tomorrow?" }, + ], + }) + ); + assert.equal(body.model, modelId); + assert.equal(body.temperature, undefined); + assert.equal(body.top_p, undefined); + assert.equal(body.top_k, undefined); + assert.deepEqual(body.system?.map((b) => b.cache_control?.type), ["ephemeral"]); + const perMessage = body.messages.map((m) => m.content.map((c) => c.cache_control?.type)); + assert.deepEqual(perMessage, [[undefined], [undefined], ["ephemeral"]]); + }); + + test(`${modelId}: tools are covered by the system breakpoint`, async () => { + const body = await capture(modelId, (model) => + generateText({ + model, + system: "You are PI.", + tools: { find: tool({ description: "find", inputSchema: z.object({ q: z.string() }) }) }, + messages: [{ role: "user", content: "search" }], + }) + ); + assert.equal(body.tools?.length, 1); + assert.equal(body.tools?.[0].cache_control, undefined); + assert.deepEqual(body.system?.map((b) => b.cache_control?.type), ["ephemeral"]); + assert.deepEqual(body.messages.at(-1)?.content.map((c) => c.cache_control?.type), ["ephemeral"]); + }); + + test(`${modelId}: the rolling breakpoint reaches the last tool result`, async () => { + const body = await capture(modelId, (model) => + generateText({ + model, + system: "You are PI.", + tools: { find: tool({ description: "find", inputSchema: z.object({ q: z.string() }) }) }, + messages: [ + { role: "user", content: "check COS 217 and COS 226" }, + { role: "assistant", content: [ + { type: "tool-call", toolCallId: "1", toolName: "find", input: { q: "COS 217" } }, + { type: "tool-call", toolCallId: "2", toolName: "find", input: { q: "COS 226" } }, + ] }, + { role: "tool", content: [ + { type: "tool-result", toolCallId: "1", toolName: "find", output: { type: "text", value: "COS 217" } }, + { type: "tool-result", toolCallId: "2", toolName: "find", output: { type: "text", value: "COS 226" } }, + ] }, + ], + }) + ); + assert.deepEqual(body.system?.map((b) => b.cache_control?.type), ["ephemeral"]); + assert.deepEqual(body.messages.at(-1)?.content.map((c) => [c.type, c.cache_control?.type]), [ + ["tool_result", undefined], + ["tool_result", "ephemeral"], + ]); + assert.equal(body.messages.flatMap((m) => m.content).filter((c) => c.cache_control).length, 1); + }); + + test(`${modelId}: streaming requests preserve caching and reported cache usage`, async () => { + const body = await capture(modelId, async (model) => { + const result = streamText({ model, system: "You are PI.", prompt: "hello" }); + assert.equal(await result.text, "ok"); + const usage = addStep(emptyTurnUsage(), await result.usage); + assert.equal(usage.input, 10); + assert.equal(usage.cacheRead, 2_000); + assert.equal(usage.cacheWrite5m, 1_000); + assert.equal(usage.output, 1); + }); + assert.equal(body.stream, true); + assert.equal(body.model, modelId); + assert.deepEqual(body.system?.map((b) => b.cache_control?.type), ["ephemeral"]); + assert.deepEqual(body.messages.at(-1)?.content.map((c) => c.cache_control?.type), ["ephemeral"]); + }); +} diff --git a/app/tests/prompt-cache.test.ts b/app/tests/prompt-cache.test.ts new file mode 100644 index 0000000..054132e --- /dev/null +++ b/app/tests/prompt-cache.test.ts @@ -0,0 +1,63 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { withCacheBreakpoints, type Prompt } from "../src/server/prompt-cache.ts"; + +const bp = (m: Prompt[number]) => + (m.providerOptions?.anthropic as { cacheControl?: { type: string } } | undefined)?.cacheControl?.type; + +const user = (text: string): Prompt[number] => ({ role: "user", content: [{ type: "text", text }] }); +const assistant = (text: string): Prompt[number] => ({ role: "assistant", content: [{ type: "text", text }] }); + +test("a turn gets a breakpoint on the system prompt and one on the last message", () => { + const out = withCacheBreakpoints([ + { role: "system", content: "You are PI." }, + user("what's due this week"), + assistant("Let me check."), + user("thanks"), + ]); + assert.deepEqual(out.map(bp), ["ephemeral", undefined, undefined, "ephemeral"]); +}); + +test("several system messages: only the last one is marked, so the whole prefix is one cache entry", () => { + const out = withCacheBreakpoints([ + { role: "system", content: "Stable rules." }, + { role: "system", content: "Workspace note." }, + user("hi"), + ]); + assert.deepEqual(out.map(bp), [undefined, "ephemeral", "ephemeral"]); +}); + +test("a tool result at the end of a step is the rolling breakpoint", () => { + const out = withCacheBreakpoints([ + { role: "system", content: "You are PI." }, + user("search"), + { role: "assistant", content: [{ type: "tool-call", toolCallId: "1", toolName: "find", input: {} }] }, + { role: "tool", content: [{ type: "tool-result", toolCallId: "1", toolName: "find", output: { type: "text", value: "x" } }] }, + ]); + assert.deepEqual(out.map(bp), ["ephemeral", undefined, undefined, "ephemeral"]); +}); + +test("existing breakpoints are kept and never doubled, other provider options survive", () => { + const marked: Prompt[number] = { + role: "user", + content: [{ type: "text", text: "pinned" }], + providerOptions: { anthropic: { cacheControl: { type: "ephemeral", ttl: "1h" } }, other: { a: 1 } }, + }; + const out = withCacheBreakpoints([{ role: "system", content: "s" }, marked]); + assert.deepEqual(out[1].providerOptions, marked.providerOptions); + const twice = withCacheBreakpoints(out); + assert.deepEqual(twice, out); +}); + +test("a system-only or empty prompt is left sane", () => { + assert.deepEqual(withCacheBreakpoints([]), []); + const out = withCacheBreakpoints([{ role: "system", content: "s" }]); + assert.deepEqual(out.map(bp), ["ephemeral"]); +}); + +test("the input prompt is not mutated", () => { + const input: Prompt = [{ role: "system", content: "s" }, user("u")]; + const snapshot = JSON.stringify(input); + withCacheBreakpoints(input); + assert.equal(JSON.stringify(input), snapshot); +}); diff --git a/app/tests/usage-limit.test.ts b/app/tests/usage-limit.test.ts index 546c4b7..db8532f 100644 --- a/app/tests/usage-limit.test.ts +++ b/app/tests/usage-limit.test.ts @@ -26,7 +26,7 @@ test("under $3 a student can use any model", () => { assert.equal(limitNote(s), null); }); -test("past $3 it's Sonnet only, until enough old spend ages out", () => { +test("past $3 it's Sonnet or Haiku, until enough old spend ages out", () => { const spends = [ { at: now - 5 * DAY, usd: 2 }, { at: now - 2 * DAY, usd: 1.2 }, @@ -36,7 +36,7 @@ test("past $3 it's Sonnet only, until enough old spend ages out", () => { assert.equal(s.blocked, false); // Dropping the $2 turn leaves $1.20, back under $3. assert.equal(s.resetAt, now - 5 * DAY + WEEK_MS); - assert.match(limitNote(s)!, /^You've used this week's Opus budget, so PI is answering with Sonnet 5 until Fri, Oct 16, 3:10 PM\.$/); + assert.equal(limitNote(s), "You've used most of this week's PI budget. Sonnet 5.5 and Haiku 5.5 are available until Fri, Oct 16, 3:10 PM."); }); test("at $4 PI stops answering until the week rolls over far enough", () => { diff --git a/app/tests/usage.test.ts b/app/tests/usage.test.ts index 41b1976..55e9489 100644 --- a/app/tests/usage.test.ts +++ b/app/tests/usage.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { addStep, costUsd, emptyTurnUsage, pseudonym } from "../src/server/usage.ts"; +import { PI_MODELS } from "../src/shared/apps.ts"; // One Anthropic step as the AI SDK reports it: 1,000 uncached input tokens, // 8,000 read from cache, 2,000 written (500 of them with the 1-hour TTL), @@ -61,6 +62,54 @@ test("an unpriced model reads as unknown, not free", () => { assert.equal(costUsd("claude-mystery-9", addStep(emptyTurnUsage(), step)), null); }); +test("every selectable model has a known nonzero price", () => { + for (const model of PI_MODELS) { + const cost = costUsd(model.value, addStep(emptyTurnUsage(), step)); + assert.ok(cost !== null && cost > 0, model.value); + } +}); + +test("Sonnet 5.5 and Haiku 5.5 price all cache buckets and searches", () => { + const usage = addStep(emptyTurnUsage(), step); + // Sonnet 5.5 cache reads are $0.10/MTok, unlike Sonnet 5's $0.20. + assert.ok(Math.abs(costUsd("claude-sonnet-5-5", usage)! - 0.03155) < 1e-12); + // 100 + 80 + 187.5 + 100 + 150 per million, plus two $0.01 searches. + assert.ok(Math.abs(costUsd("claude-haiku-5-5", usage)! - 0.0206175) < 1e-12); +}); + +test("Haiku's higher rate starts strictly above 100,000 prompt tokens", () => { + const atLimit = addStep(emptyTurnUsage(), { inputTokens: 100_000, outputTokens: 1_000 }); + const overLimit = addStep(emptyTurnUsage(), { inputTokens: 100_001, outputTokens: 1_000 }); + assert.ok(Math.abs(costUsd("claude-haiku-5-5", atLimit)! - 0.0105) < 1e-12); + assert.ok(Math.abs(costUsd("claude-haiku-5-5", overLimit)! - 0.0525005) < 1e-12); +}); + +test("Haiku's prompt threshold includes cache reads and both write TTLs", () => { + const usage = addStep(emptyTurnUsage(), { + inputTokenDetails: { noCacheTokens: 1, cacheReadTokens: 90_000, cacheWriteTokens: 10_000 }, + outputTokens: 1_000, + raw: { + cache_creation: { ephemeral_1h_input_tokens: 4_000 }, + server_tool_use: { web_search_requests: 1 }, + }, + }); + // Higher rates apply to the entire request. Search stays $0.01. + assert.ok(Math.abs(costUsd("claude-haiku-5-5", usage)! - 0.0247505) < 1e-12); +}); + +test("Haiku prices each step independently before summing a turn", () => { + const short = { inputTokens: 60_000, outputTokens: 1_000 }; + const twoShort = addStep(addStep(emptyTurnUsage(), short), short); + // The turn totals 120k input tokens, but neither request crosses the threshold. + assert.ok(Math.abs(costUsd("claude-haiku-5-5", twoShort)! - 0.013) < 1e-12); + const mixed = addStep(twoShort, { inputTokens: 100_001, outputTokens: 1_000 }); + assert.ok(Math.abs(costUsd("claude-haiku-5-5", mixed)! - 0.0655005) < 1e-12); + const shortAfterLong = addStep(mixed, short); + assert.ok(Math.abs(costUsd("claude-haiku-5-5", shortAfterLong)! - 0.0720005) < 1e-12); + // Other models keep their flat rates even for the same long requests. + assert.ok(Math.abs(costUsd("claude-sonnet-5-5", mixed)! - 0.470002) < 1e-12); +}); + test("student ids in the logs are stable, short and not the NetID", async () => { const a = await pseudonym("aj5828", "secret-one"); assert.match(a, /^[0-9a-f]{16}$/); diff --git a/app/wrangler.jsonc b/app/wrangler.jsonc index d75d61e..e6b9d7b 100644 --- a/app/wrangler.jsonc +++ b/app/wrangler.jsonc @@ -23,7 +23,7 @@ // Public OAuth client id from the Google Cloud project; the secret is a Worker secret. "GOOGLE_OAUTH_CLIENT_ID": "548956142814-upcmsgqr9pbafnlsorhimvje1negt5lm.apps.googleusercontent.com", // Weekly model budget per student, in USD (src/shared/usage-limit.ts): - // any model up to tier 1, then Sonnet only for tier 2 more. + // any model up to tier 1, then Sonnet or Haiku for tier 2 more. "USAGE_TIER1_USD": "3", "USAGE_TIER2_USD": "1" },