From f606f57a4eecaef7f7c5d672d1e6e601271e0f58 Mon Sep 17 00:00:00 2001 From: Tejas Kashinath Date: Wed, 30 Sep 2026 00:32:03 +0000 Subject: [PATCH 1/3] feat(project): add LLM-as-a-judge and code-based evaluator TUI wizards `agentcore add` now lists `evaluator` with a submenu for its two leaves, and a bare `add evaluator llm-as-a-judge` or `add evaluator code-based` on a TTY opens its wizard. - llm-as-a-judge asks for a name, level, model, instructions and a rating scale preset. The model step offers Bedrock and OpenResponses, each opening its own model ID input. Bedrock is prefilled with global.anthropic.claude-sonnet-4-6, because the Evaluator service sends a temperature and newer Claude models refuse it. - code-based asks for a name, level and Lambda: scaffold a new one (with a timeout step) or use an existing one by ARN. - Each handler and its wizard build their input through one shared builder, so they refuse the same input with the same message. - RouterScreen keeps command names in one column when a description wraps. - The evaluator leaf descriptions follow the other add commands, and promptPreview moves into the wizard module for both reviews to use. --- command.md | 4 +- src/components/Root.tsx | 10 + src/components/RouterScreen.test.tsx | 19 ++ src/components/RouterScreen.tsx | 29 +- src/components/wizard/fields.tsx | 12 + src/components/wizard/index.ts | 1 + src/components/wizard/wizard.test.tsx | 25 ++ src/core/project/templates/evaluator.ts | 8 +- src/handlers/project/add/add.screen.test.tsx | 1 + .../code-based/code-based.screen.test.tsx | 166 +++++++++++ .../project/add/evaluator/code-based/index.ts | 129 +++++---- .../add/evaluator/code-based/screen.tsx | 226 +++++++++++++++ .../add/evaluator/evaluator.screen.test.tsx | 112 ++++++++ src/handlers/project/add/evaluator/index.ts | 8 +- .../add/evaluator/llm-as-a-judge/index.ts | 120 ++++---- .../llm-as-a-judge.screen.test.tsx | 270 ++++++++++++++++++ .../evaluator/llm-as-a-judge/ratingScales.ts | 4 + .../add/evaluator/llm-as-a-judge/screen.tsx | 269 +++++++++++++++++ src/handlers/project/add/evaluator/shared.ts | 40 +++ .../add/harness/harness.screen.test.tsx | 27 -- src/handlers/project/add/harness/screen.tsx | 21 +- src/handlers/project/add/index.ts | 3 +- src/projectSchemas/evaluator.ts | 3 +- src/testing/renderScreen.tsx | 1 + 24 files changed, 1336 insertions(+), 172 deletions(-) create mode 100644 src/handlers/project/add/evaluator/code-based/code-based.screen.test.tsx create mode 100644 src/handlers/project/add/evaluator/code-based/screen.tsx create mode 100644 src/handlers/project/add/evaluator/evaluator.screen.test.tsx create mode 100644 src/handlers/project/add/evaluator/llm-as-a-judge/llm-as-a-judge.screen.test.tsx create mode 100644 src/handlers/project/add/evaluator/llm-as-a-judge/screen.tsx create mode 100644 src/handlers/project/add/evaluator/shared.ts diff --git a/command.md b/command.md index 9dc95e674f..9241d6fb23 100644 --- a/command.md +++ b/command.md @@ -427,7 +427,7 @@ add a custom evaluator to the current project agentcore add evaluator llm-as-a-judge [options] ``` -add an LLM-as-a-Judge evaluator: another LLM prompted with instructions on how to score a session +add an LLM-as-a-Judge evaluator to the current project **Options** @@ -447,7 +447,7 @@ add an LLM-as-a-Judge evaluator: another LLM prompted with instructions on how t agentcore add evaluator code-based [options] ``` -add a code-based evaluator: scaffold a Python Lambda with custom evaluation logic, or reference an existing Lambda with --lambda-arn +add a code-based evaluator to the current project **Options** diff --git a/src/components/Root.tsx b/src/components/Root.tsx index aceb2282cd..3596fcec1d 100644 --- a/src/components/Root.tsx +++ b/src/components/Root.tsx @@ -139,6 +139,8 @@ import { AddHarnessScreen } from "../handlers/project/add/harness/screen.tsx"; import { AddConfigBundleScreen } from "../handlers/project/add/config-bundle/screen.tsx"; import { AddPaymentManagerScreen } from "../handlers/project/add/payment-manager/screen.tsx"; import { AddPaymentConnectorScreen } from "../handlers/project/add/payment-connector/screen.tsx"; +import { AddLlmAsAJudgeEvaluatorScreen } from "../handlers/project/add/evaluator/llm-as-a-judge/screen.tsx"; +import { AddCodeBasedEvaluatorScreen } from "../handlers/project/add/evaluator/code-based/screen.tsx"; import { ProjectStatusScreen } from "../handlers/project/status/screen.tsx"; import { ProjectRemoveScreen } from "../handlers/project/remove/screen.tsx"; import { HelpScreen, RootScreen } from "../handlers/screen.tsx"; @@ -935,6 +937,14 @@ function RouteTable({ ctx, core }: ScreenProps) { path="agentcore/add/payment-connector" element={} /> + } + /> + } + /> } /> { }); }); +describe("narrow terminals", () => { + test("a description that wraps leaves every command name in the same column", async () => { + const r = renderScreen("/agentcore/add"); + await waitForText(r.lastFrame, "❯ "); + await r.resize(60); + + await waitForText(r.lastFrame, "── cli"); + const lines = r.lastFrame()!.split("\n"); + const nameColumns = lines + .map((line) => /^\s{1,3}(?:❯ )?\s*[a-z][a-z0-9-]*\s{2,}\S/.exec(line)?.[0]) + .filter((row) => row !== undefined) + .map((row) => row.replace("❯", " ").search(/[a-z]/)); + expect(nameColumns.length).toBeGreaterThan(1); + expect(new Set(nameColumns).size).toBe(1); + expect(lines.some((line) => /^\s{20,}\S/.test(line))).toBe(true); + r.unmount(); + }); +}); + describe("filtering", () => { test("typing narrows the options to matches", async () => { const r = renderScreen("/agentcore/harness"); diff --git a/src/components/RouterScreen.tsx b/src/components/RouterScreen.tsx index cc5757a045..cf15bbba31 100644 --- a/src/components/RouterScreen.tsx +++ b/src/components/RouterScreen.tsx @@ -273,16 +273,25 @@ function CommandMenu({ const isHl = row.index === highlight; return ( - {isHl ? `${glyphs.pointer} ` : " "} - - {o.name.padEnd(nameWidth)} - - {o.description} + {/** A wrapped description continues under itself and leaves the name in line. **/} + + {isHl ? `${glyphs.pointer} ` : " "} + + {o.name.padEnd(nameWidth)} + + + + {o.description} + ); }) diff --git a/src/components/wizard/fields.tsx b/src/components/wizard/fields.tsx index f0d65202d4..6ce7166480 100644 --- a/src/components/wizard/fields.tsx +++ b/src/components/wizard/fields.tsx @@ -611,6 +611,18 @@ export function MultiChoiceField({ ); } +// promptPreview keeps a multi-line answer to one review line: its first line, +// cut short, with a count of what follows it. It reads the text exactly as it +// will be saved — no trimming — so a leading blank line or a trailing newline +// left by an extra enter shows up in the review rather than only in the spec. +export function promptPreview(prompt: string): string { + const lines = prompt.split("\n"); + const first = lines[0] ?? ""; + const shown = first.length > 60 ? `${first.slice(0, 59)}…` : first; + const rest = lines.length - 1; + return rest > 0 ? `${shown} (+${rest} more ${rest === 1 ? "line" : "lines"})` : shown; +} + export interface SummaryProps { items: Record; } diff --git a/src/components/wizard/index.ts b/src/components/wizard/index.ts index f45473dc5b..e7efc27440 100644 --- a/src/components/wizard/index.ts +++ b/src/components/wizard/index.ts @@ -12,6 +12,7 @@ export { MultiTextField, Summary, firstIssue, + promptPreview, type Choice, type TextFieldProps, type TextAreaFieldProps, diff --git a/src/components/wizard/wizard.test.tsx b/src/components/wizard/wizard.test.tsx index 4620cf5f44..04a31755c0 100644 --- a/src/components/wizard/wizard.test.tsx +++ b/src/components/wizard/wizard.test.tsx @@ -11,6 +11,7 @@ import { ChoiceField, MultiChoiceField, MultiTextField, + promptPreview, ResourceChoiceField, RevealChoiceField, Summary, @@ -1145,3 +1146,27 @@ describe("Wizard authoring guards", () => { expect((error as Error).message).toBe('duplicate '); }); }); + +describe("promptPreview", () => { + test.each([ + ["a one-line prompt", "You are a pirate.", "You are a pirate."], + [ + "counts the lines after the first", + "You are a pirate.\nAnswer in rhyme.", + "You are a pirate. (+1 more line)", + ], + ["keeps a leading blank line", "\nYou are a pirate.", " (+1 more line)"], + [ + "counts a trailing newline left by enter", + "You are a pirate.\n", + "You are a pirate. (+1 more line)", + ], + [ + "cuts a long first line short", + `${"x".repeat(70)}\ny\nz`, + `${"x".repeat(59)}… (+2 more lines)`, + ], + ])("%s", (_label, prompt, preview) => { + expect(promptPreview(prompt)).toBe(preview); + }); +}); diff --git a/src/core/project/templates/evaluator.ts b/src/core/project/templates/evaluator.ts index d57ccd2524..a961904697 100644 --- a/src/core/project/templates/evaluator.ts +++ b/src/core/project/templates/evaluator.ts @@ -1,11 +1,13 @@ import { FsTreeNode } from "./fsTree"; import type { AssetSource } from "../source"; -import type { Evaluator } from "../../../projectSchemas/evaluator"; +import { + DEFAULT_CODE_BASED_TIMEOUT_SECONDS, + type Evaluator, +} from "../../../projectSchemas/evaluator"; import type { TemplateRenderer, TemplateResolver } from "./types"; import { toPythonPackageName } from "../fsUtils"; import type { ManagedEvaluatorScaffoldInput } from "../../../handlers/project/types"; -const DEFAULT_TIMEOUT = 60; const ASSET_DIR = "evaluators/python-lambda"; function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evaluator { @@ -18,7 +20,7 @@ function buildManagedEvaluatorSpec(input: ManagedEvaluatorScaffoldInput): Evalua managed: { codeLocation: `app/${input.name}`, entrypoint: "lambda_function.handler", - timeoutSeconds: input.timeoutSeconds ?? DEFAULT_TIMEOUT, + timeoutSeconds: input.timeoutSeconds ?? DEFAULT_CODE_BASED_TIMEOUT_SECONDS, additionalPolicies: ["execution-role-policy.json"], }, }, diff --git a/src/handlers/project/add/add.screen.test.tsx b/src/handlers/project/add/add.screen.test.tsx index 9df953028a..c3b2308c16 100644 --- a/src/handlers/project/add/add.screen.test.tsx +++ b/src/handlers/project/add/add.screen.test.tsx @@ -32,6 +32,7 @@ const WITH_SCREENS = [ "config-bundle", "payment-manager", "payment-connector", + "evaluator", ]; describe("project add menu", () => { diff --git a/src/handlers/project/add/evaluator/code-based/code-based.screen.test.tsx b/src/handlers/project/add/evaluator/code-based/code-based.screen.test.tsx new file mode 100644 index 0000000000..468badb576 --- /dev/null +++ b/src/handlers/project/add/evaluator/code-based/code-based.screen.test.tsx @@ -0,0 +1,166 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { + cleanupScreens, + flatFrame, + renderScreen, + waitForFlatText, + waitForText, + type RenderScreenResult, +} from "../../../../../testing"; +import { createGatewayProjectTestHarness } from "../../gateway-test-support"; + +const { cleanup, inProject, projectSpec, run } = + createGatewayProjectTestHarness("add-code-based-wizard"); + +afterEach(cleanup); +afterEach(cleanupScreens); + +const LAMBDA_ARN = "arn:aws:lambda:us-west-2:123456789012:function:refund-policy"; + +async function evaluatorOf(projectRoot: string, name: string) { + return ((await projectSpec(projectRoot)).evaluators ?? []).find( + (evaluator: { name: string }) => evaluator.name === name, + ); +} + +async function reachLambdaStep(screen: RenderScreenResult, name: string): Promise { + await waitForText(screen.lastFrame, "what should this evaluator be called?"); + await screen.write(name); + await screen.press("return"); + await waitForText(screen.lastFrame, "what should it score?"); + await screen.press("return"); + await waitForText(screen.lastFrame, "which Lambda should score it?"); +} + +describe("project add evaluator code-based wizard", () => { + test("scaffolds the same managed evaluator as the flags", async () => { + const projectRoot = await inProject(); + const screen = renderScreen("/agentcore/add/evaluator/code-based"); + await reachLambdaStep(screen, "refund_policy"); + + expect(screen.lastFrame()).toContain("❯ ● scaffold a new Lambda"); + expect(screen.lastFrame()).toContain("Python code in app/refund_policy"); + expect(screen.lastFrame()).toContain("○ timeout"); + await screen.press("return"); + + await waitForText(screen.lastFrame, "how long may it run?"); + expect(screen.lastFrame()).toContain("60"); + await screen.press("backspace"); + await screen.press("backspace"); + await screen.write("120"); + await screen.press("return"); + + await waitForText(screen.lastFrame, "this evaluator will be added to agentcore.json"); + const review = flatFrame(screen.lastFrame); + expect(review).toContain("evaluator refund_policy"); + expect(review).toContain("level SESSION"); + expect(review).toContain("lambda scaffolded in app/refund_policy"); + expect(review).toContain("timeout 120 seconds"); + await screen.press("return"); + + await waitForText(screen.lastFrame, "added evaluator 'refund_policy' to 'TestProject'"); + await waitForFlatText( + screen.lastFrame, + "note: this evaluator returns Pass for every session until you implement app/refund_policy/lambda_function.py", + ); + expect( + await Bun.file(join(projectRoot, "app", "refund_policy", "lambda_function.py")).exists(), + ).toBe(true); + + await run([ + "add", + "evaluator", + "code-based", + "--name", + "flags", + "--level", + "SESSION", + "--timeout-seconds", + "120", + ]); + const flags = await evaluatorOf(projectRoot, "flags"); + expect(await evaluatorOf(projectRoot, "refund_policy")).toEqual({ + ...flags, + name: "refund_policy", + config: { + codeBased: { + managed: { ...flags.config.codeBased.managed, codeLocation: "app/refund_policy" }, + }, + }, + }); + + await screen.press("return"); + await waitForText(screen.lastFrame, "add a custom evaluator to the current project"); + screen.unmount(); + }, 20000); + + test("an existing Lambda skips the timeout and writes an external config", async () => { + const projectRoot = await inProject(); + const screen = renderScreen("/agentcore/add/evaluator/code-based"); + await reachLambdaStep(screen, "refund_policy"); + + await screen.press("down"); + await waitForText(screen.lastFrame, "❯ ● use an existing Lambda"); + expect(screen.lastFrame()).not.toContain("timeout"); + await screen.press("return"); + await waitForText(screen.lastFrame, "Lambda ARN"); + await screen.write(LAMBDA_ARN); + await screen.press("return"); + + await waitForText(screen.lastFrame, "this evaluator will be added to agentcore.json"); + const review = flatFrame(screen.lastFrame); + expect(review.replace(/\s/g, "")).toContain(`lambda${LAMBDA_ARN}`); + expect(review).not.toContain("timeout"); + await screen.press("return"); + + await waitForText(screen.lastFrame, "added evaluator 'refund_policy'"); + expect(screen.lastFrame()).not.toContain("returns Pass"); + expect((await evaluatorOf(projectRoot, "refund_policy")).config).toEqual({ + codeBased: { external: { lambdaArn: LAMBDA_ARN } }, + }); + expect(await Bun.file(join(projectRoot, "app", "refund_policy")).exists()).toBe(false); + screen.unmount(); + }, 20000); + + test.each([ + [ + "a malformed Lambda ARN", + ["down", "return"], + "refund-policy", + "Must be a valid Lambda function ARN", + ], + ["a timeout past 300 seconds", ["return"], "0", "Too big: expected number to be <=300"], + ] as const)("%s keeps the wizard on its step", async (_label, moves, typed, message) => { + await inProject(); + const screen = renderScreen("/agentcore/add/evaluator/code-based"); + await reachLambdaStep(screen, "refund_policy"); + for (const move of moves) await screen.press(move); + await screen.write(typed); + + await screen.press("return"); + + await waitForText(screen.lastFrame, message); + expect(screen.lastFrame()).not.toContain("this evaluator will be added"); + screen.unmount(); + }); + + test("a rejected add reports itself and hands the form back", async () => { + const projectRoot = await inProject(); + await run(["add", "evaluator", "code-based", "--name", "refund_policy", "--level", "SESSION"]); + const screen = renderScreen("/agentcore/add/evaluator/code-based"); + await reachLambdaStep(screen, "refund_policy"); + await screen.press("down"); + await screen.press("return"); + await screen.write(LAMBDA_ARN); + await screen.press("return"); + await waitForText(screen.lastFrame, "this evaluator will be added to agentcore.json"); + await screen.press("return"); + + await waitForFlatText(screen.lastFrame, "a evaluator with name 'refund_policy' already exists"); + await screen.press("escape"); + await waitForText(screen.lastFrame, "this evaluator will be added to agentcore.json"); + expect((await projectSpec(projectRoot)).evaluators).toHaveLength(1); + screen.unmount(); + }, 20000); +}); diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts index ec94fa9b9e..5622679a0e 100644 --- a/src/handlers/project/add/evaluator/code-based/index.ts +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -1,18 +1,65 @@ import z from "zod"; import { createHandler, flag, ProjectKey } from "../../../../../router"; import { InputValidationError } from "../../../../../errors"; +import type { AwsDeploymentTarget } from "../../../../../projectSchemas/aws-targets"; import { EvaluatorSchema, EvaluationLevelSchema } from "../../../../../projectSchemas/evaluator"; import { TagsSchema } from "../../../../../projectSchemas/tags"; -import type { ManagedEvaluatorScaffoldInput } from "../../../types"; +import type { AddResourceInput, ManagedEvaluatorScaffoldInput, Project } from "../../../types"; import { parseJsonFlagWithSchema } from "../../../../utils"; import type { AddProjectResourceConfig } from "../../types"; import { addProjectResource, requireDeployedNameFits } from "../../shared"; +export const TimeoutSecondsSchema = z.number().int().min(1).max(300); + +export type CodeBasedEvaluatorInput = { + name: string; + level: string; + description?: string; + kmsKeyArn?: string; + tags?: Record; +} & ({ lambdaArn: string } | { lambdaArn?: undefined; timeoutSeconds?: number }); + +export function toAddCodeBasedEvaluatorInput( + project: Project, + targets: readonly AwsDeploymentTarget[], + input: CodeBasedEvaluatorInput, +): AddResourceInput { + requireDeployedNameFits("Evaluator", project.name, input.name, "_", 48, targets); + const levelParsed = EvaluationLevelSchema.safeParse(input.level); + if (!levelParsed.success) throw new InputValidationError(z.prettifyError(levelParsed.error)); + + const base = { + name: input.name, + level: levelParsed.data, + description: input.description, + kmsKeyArn: input.kmsKeyArn, + tags: input.tags, + }; + + if (input.lambdaArn !== undefined) { + const parsed = EvaluatorSchema.safeParse({ + ...base, + config: { codeBased: { external: { lambdaArn: input.lambdaArn } } }, + }); + if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); + return { resourceType: "evaluator", resourceConfig: parsed.data }; + } + + const scaffold: ManagedEvaluatorScaffoldInput = { + ...base, + ...(input.timeoutSeconds !== undefined && { timeoutSeconds: input.timeoutSeconds }), + }; + return { resourceType: "evaluator", resourceConfig: { name: scaffold.name }, scaffold }; +} + +export function scaffoldedEvaluatorNote(name: string): string { + return `note: this evaluator returns Pass for every session until you implement app/${name}/lambda_function.py`; +} + export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceConfig) => createHandler({ name: "code-based", - description: - "add a code-based evaluator — scaffold a Python Lambda with custom evaluation logic, or reference an existing Lambda with --lambda-arn", + description: "add a code-based evaluator to the current project", flags: [ flag("name", "the name of the evaluator", z.string().min(1)), flag("level", "what to score: SESSION, TRACE, or TOOL_CALL", z.string().min(1)), @@ -20,7 +67,7 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon flag( "timeout-seconds", "evaluator timeout in seconds (1-300)", - z.number().int().min(1).max(300).optional(), + TimeoutSecondsSchema.optional(), ), flag("description", "a description of what this evaluator measures", z.string().optional()), flag( @@ -32,70 +79,32 @@ export const createAddCodeBasedEvaluatorHandler = (config: AddProjectResourceCon ], handle: async (ctx, flags) => { const project = ctx.require(ProjectKey); - requireDeployedNameFits( - "Evaluator", - project.name, - flags["name"], - "_", - 48, + const lambdaArn = flags["lambda-arn"]; + if (lambdaArn !== undefined && flags["timeout-seconds"] !== undefined) + throw new InputValidationError("--timeout-seconds is not valid with --lambda-arn"); + + const input = toAddCodeBasedEvaluatorInput( + project, await config.projectManager.listTargets(project), + { + name: flags["name"], + level: flags["level"], + description: flags["description"], + kmsKeyArn: flags["kms-key-arn"], + tags: parseJsonFlagWithSchema("tags", flags["tags"], TagsSchema), + ...(lambdaArn !== undefined + ? { lambdaArn } + : { timeoutSeconds: flags["timeout-seconds"] }), + }, ); - const levelParsed = EvaluationLevelSchema.safeParse(flags["level"]); - if (!levelParsed.success) throw new InputValidationError(z.prettifyError(levelParsed.error)); - const level = levelParsed.data; - - const hasLambda = flags["lambda-arn"] !== undefined; - - const tags = parseJsonFlagWithSchema("tags", flags["tags"], TagsSchema); - const base = { - name: flags["name"], - level, - description: flags["description"], - kmsKeyArn: flags["kms-key-arn"], - tags, - }; - - if (hasLambda) { - if (flags["timeout-seconds"] !== undefined) - throw new InputValidationError("--timeout-seconds is not valid with --lambda-arn"); - const parsed = EvaluatorSchema.safeParse({ - ...base, - config: { codeBased: { external: { lambdaArn: flags["lambda-arn"] } } }, - }); - if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); - await addProjectResource( - ctx, - config, - project, - { - resourceType: "evaluator", - resourceConfig: parsed.data, - }, - `added evaluator '${flags["name"]}' to '${project.name}'`, - ); - return; - } - - const scaffold: ManagedEvaluatorScaffoldInput = { - ...base, - ...(flags["timeout-seconds"] !== undefined && { timeoutSeconds: flags["timeout-seconds"] }), - }; await addProjectResource( ctx, config, project, - { - resourceType: "evaluator", - resourceConfig: { name: scaffold.name }, - scaffold, - }, + input, `added evaluator '${flags["name"]}' to '${project.name}'`, - { - notes: [ - `note: this evaluator returns Pass for every session until you implement app/${flags["name"]}/lambda_function.py`, - ], - }, + lambdaArn === undefined ? { notes: [scaffoldedEvaluatorNote(flags["name"])] } : {}, ); }, }); diff --git a/src/handlers/project/add/evaluator/code-based/screen.tsx b/src/handlers/project/add/evaluator/code-based/screen.tsx new file mode 100644 index 0000000000..2355870f7b --- /dev/null +++ b/src/handlers/project/add/evaluator/code-based/screen.tsx @@ -0,0 +1,226 @@ +import { useMemo, useState } from "react"; +import { useQueryClient } from "@tanstack/react-query"; +import { useNavigate } from "react-router"; +import { + ChoiceField, + RevealChoiceField, + Step, + Summary, + TextField, + Wizard, + type Choice, +} from "../../../../../components/wizard"; +import type { AwsDeploymentTarget } from "../../../../../projectSchemas/aws-targets"; +import { + DEFAULT_CODE_BASED_TIMEOUT_SECONDS, + ExternalCodeBasedConfigSchema, + type EvaluationLevel, +} from "../../../../../projectSchemas/evaluator"; +import { ProjectKey } from "../../../../../router"; +import type { ScreenProps } from "../../../../types"; +import type { Project } from "../../../types"; +import { + LoadingFrame, + ProjectGate, + projectQueryKey, + useProjectTargets, +} from "../../../ProjectGate"; +import { EVALUATOR_MENU, EVALUATOR_NAME_HELP, evaluatorNameSchema, LEVEL_CHOICES } from "../shared"; +import { + scaffoldedEvaluatorNote, + TimeoutSecondsSchema, + toAddCodeBasedEvaluatorInput, + type CodeBasedEvaluatorInput, +} from "./index"; + +const BREADCRUMB = ["agentcore", "add", "evaluator", "code-based"]; +const DESCRIPTION = "add a code-based evaluator to the current project"; + +type LambdaSource = "scaffold" | "existing"; + +type CodeBasedFormValues = { + name: string; + level: EvaluationLevel; + lambda: LambdaSource; + lambdaArn: string; + timeoutSeconds: string; +}; + +function lambdaChoices(name: string): Choice[] { + return [ + { + value: "scaffold", + label: "scaffold a new Lambda", + description: `Python code in app/${name || ""}, deployed with the project`, + }, + { + value: "existing", + label: "use an existing Lambda", + description: "a Lambda function deployed outside this project", + }, + ]; +} + +export function toCodeBasedInput(values: CodeBasedFormValues): CodeBasedEvaluatorInput { + const base = { name: values.name, level: values.level }; + return values.lambda === "existing" + ? { ...base, lambdaArn: values.lambdaArn } + : { ...base, timeoutSeconds: Number(values.timeoutSeconds) }; +} + +function summaryOf(values: CodeBasedFormValues): Record { + const scaffolds = values.lambda === "scaffold"; + return { + evaluator: values.name, + level: values.level, + lambda: scaffolds ? `scaffolded in app/${values.name}` : values.lambdaArn, + ...(scaffolds ? { timeout: `${values.timeoutSeconds} seconds` } : {}), + }; +} + +export function AddCodeBasedEvaluatorScreen({ ctx, core }: ScreenProps) { + const navigate = useNavigate(); + return ( + navigate(EVALUATOR_MENU)} + > + {(project) => } + + ); +} + +function AddCodeBasedEvaluatorLoader({ + project, + core, +}: { + project: Project; + core: ScreenProps["core"]; +}) { + const navigate = useNavigate(); + const targets = useProjectTargets(core, project); + + if (targets.data !== undefined) { + return ; + } + + return ( + navigate(EVALUATOR_MENU)} + /> + ); +} + +function AddCodeBasedEvaluatorWizard({ + project, + targets, + core, +}: { + project: Project; + targets: readonly AwsDeploymentTarget[]; + core: ScreenProps["core"]; +}) { + const navigate = useNavigate(); + const queryClient = useQueryClient(); + const [values, setValues] = useState({ + name: "", + level: "SESSION", + lambda: "scaffold", + lambdaArn: "", + timeoutSeconds: String(DEFAULT_CODE_BASED_TIMEOUT_SECONDS), + }); + const set = (update: Partial) => + setValues((current) => ({ ...current, ...update })); + const nameSchema = useMemo( + () => evaluatorNameSchema(project.name, targets), + [project.name, targets], + ); + const scaffolds = values.lambda === "scaffold"; + + return ( + navigate(EVALUATOR_MENU)} + onSubmit={async function* () { + const updated = yield* core.projectManager.addResource( + project, + toAddCodeBasedEvaluatorInput(project, targets, toCodeBasedInput(values)), + ); + queryClient.setQueryData(projectQueryKey(), updated); + return updated; + }} + runningLabel={`adding evaluator ${values.name}…`} + successLabel={`added evaluator '${values.name}' to '${project.name}'`} + successNotes={scaffolds ? [scaffoldedEvaluatorNote(values.name)] : undefined} + successNextSteps={["agentcore deploy"]} + onDone={() => navigate(EVALUATOR_MENU)} + doneLabel="go back" + > + + set({ name })} + required + schema={nameSchema} + live + /> + + + + set({ level })} + /> + + + + set({ lambda })} + input={{ + opensFor: (lambda) => lambda === "existing", + label: "Lambda ARN", + name: "Lambda ARN", + help: "the ARN of the function that scores each evaluation", + placeholder: "arn:aws:lambda:us-west-2:123456789012:function:refund-policy", + value: values.lambdaArn, + onChange: (lambdaArn) => set({ lambdaArn }), + required: true, + schema: ExternalCodeBasedConfigSchema.shape.lambdaArn, + }} + /> + + + {scaffolds && ( + + set({ timeoutSeconds })} + required + number + schema={TimeoutSecondsSchema} + /> + + )} + + + + + + ); +} diff --git a/src/handlers/project/add/evaluator/evaluator.screen.test.tsx b/src/handlers/project/add/evaluator/evaluator.screen.test.tsx new file mode 100644 index 0000000000..1cfc4f8117 --- /dev/null +++ b/src/handlers/project/add/evaluator/evaluator.screen.test.tsx @@ -0,0 +1,112 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + cleanupScreens, + createSilentLogger, + menuEntries, + renderScreen, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, + ttyTestIO, + waitFor, + waitForText, +} from "../../../../testing"; +import { InputValidationError } from "../../../../errors"; +import type { AppIO } from "../../../../io"; +import { createRootHandler } from "../../../index"; +import { createGatewayProjectTestHarness } from "../gateway-test-support"; + +const { cleanup, inProject } = createGatewayProjectTestHarness("add-evaluator-menu"); + +afterEach(cleanup); +afterEach(cleanupScreens); + +const LEAVES = ["llm-as-a-judge", "code-based"]; + +describe("project add evaluator menu", () => { + test("lists both evaluator wizards and esc returns to the add menu", async () => { + await inProject(); + const screen = renderScreen("/agentcore/add/evaluator"); + + await waitForText(screen.lastFrame, "add a custom evaluator to the current project"); + const { screens, cliOnly } = menuEntries(screen.lastFrame()!); + expect(screens).toEqual(LEAVES); + expect(cliOnly).toEqual([]); + await screen.press("escape"); + + await waitForText(screen.lastFrame, "add project resources"); + screen.unmount(); + }); + + test.each(LEAVES)("esc on the first %s step returns to the evaluator menu", async (leaf) => { + await inProject(); + const screen = renderScreen(`/agentcore/add/evaluator/${leaf}`); + + await waitForText(screen.lastFrame, "what should this evaluator be called?"); + await screen.press("escape"); + + await waitForText(screen.lastFrame, "add a custom evaluator to the current project"); + screen.unmount(); + }); +}); + +describe.each(LEAVES)("project add evaluator %s dispatch", (leaf) => { + function buildRoot(io: AppIO) { + return createRootHandler(new TestCoreClient(), { + io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + } + + async function routeError(io: AppIO, args: string[]): Promise { + return buildRoot(io) + .route(["node", "agentcore", "add", "evaluator", leaf, ...args]) + .then( + () => undefined, + (caught: unknown) => caught, + ); + } + + function expectMissingName(error: unknown) { + expect(error).toBeInstanceOf(InputValidationError); + expect((error as Error).message).toContain("required option '--name"); + } + + test("a bare command in a TTY session opens the wizard", async () => { + await inProject(); + const { streams, stdin } = ttyTestIO(); + + const outcome = buildRoot(streams.io) + .route(["node", "agentcore", "add", "evaluator", leaf]) + .then( + () => ({ ok: true as const }), + (error: unknown) => ({ ok: false as const, error }), + ); + let settled = false; + void outcome.finally(() => { + settled = true; + }); + + await waitFor( + () => { + if (!settled) stdin.write("\x03"); + return settled; + }, + 5000, + 150, + ); + expect(await outcome).toEqual({ ok: true }); + expect(streams.stderr()).not.toContain("required option"); + }, 10000); + + test.each<[string, () => AppIO, string[]]>([ + ["a bare command without a TTY", () => testIO().io, []], + ["a user-supplied flag in a TTY", () => ttyTestIO().streams.io, ["--level", "SESSION"]], + ["--json in a TTY", () => ttyTestIO().streams.io, ["--json"]], + ])("%s stays headless", async (_label, io, args) => { + await inProject(); + + expectMissingName(await routeError(io(), args)); + }); +}); diff --git a/src/handlers/project/add/evaluator/index.ts b/src/handlers/project/add/evaluator/index.ts index 1c05ea3be4..50cebf57e7 100644 --- a/src/handlers/project/add/evaluator/index.ts +++ b/src/handlers/project/add/evaluator/index.ts @@ -1,10 +1,14 @@ import { Router } from "../../../../router"; +import { renderTui } from "../../../../tui"; +import type { Core } from "../../../types"; import type { AddProjectResourceConfig } from "../types"; import { createAddLlmAsAJudgeEvaluatorHandler } from "./llm-as-a-judge"; import { createAddCodeBasedEvaluatorHandler } from "./code-based"; -export function createAddEvaluatorHandler(config: AddProjectResourceConfig): Router { - const evaluator = new Router("evaluator", "add a custom evaluator to the current project"); +export function createAddEvaluatorHandler(config: AddProjectResourceConfig, core: Core): Router { + const evaluator = new Router("evaluator", "add a custom evaluator to the current project") + .default(renderTui(core, config.io)) + .supportedTuiCommands("llm-as-a-judge", "code-based"); evaluator.handler(createAddLlmAsAJudgeEvaluatorHandler(config)); evaluator.handler(createAddCodeBasedEvaluatorHandler(config)); return evaluator; diff --git a/src/handlers/project/add/evaluator/llm-as-a-judge/index.ts b/src/handlers/project/add/evaluator/llm-as-a-judge/index.ts index f39f7f4cb1..a794dd18bc 100644 --- a/src/handlers/project/add/evaluator/llm-as-a-judge/index.ts +++ b/src/handlers/project/add/evaluator/llm-as-a-judge/index.ts @@ -2,6 +2,7 @@ import z from "zod"; import { createHandler, flag, ProjectKey } from "../../../../../router"; import { InputValidationError } from "../../../../../errors"; import { SourceResolver } from "../../../../../io"; +import type { AwsDeploymentTarget } from "../../../../../projectSchemas/aws-targets"; import { EvaluatorModelProviderSchema, EvaluatorSchema, @@ -11,20 +12,71 @@ import { type RatingScale, } from "../../../../../projectSchemas/evaluator"; import { TagsSchema } from "../../../../../projectSchemas/tags"; +import type { AddResourceInput, Project } from "../../../types"; import { parseJsonFlagWithSchema } from "../../../../utils"; import type { AddProjectResourceConfig } from "../../types"; import { addProjectResource, requireDeployedNameFits } from "../../shared"; import { + expandRatingScalePreset, isRatingScalePreset, - RATING_SCALE_PRESETS, RATING_SCALE_PRESET_NAMES, } from "./ratingScales"; +export const MODEL_ID_FORMATS: Record = { + Bedrock: + "a Bedrock model ID (e.g. anthropic.claude-3-5-sonnet-20240620-v1:0) or an inference-profile/foundation-model ARN", + OpenResponses: + "an OpenResponses model ID (a non-empty identifier without spaces, e.g. openai.gpt-5.4)", +}; + +export type LlmAsAJudgeEvaluatorInput = { + name: string; + level: string; + modelProvider: EvaluatorModelProvider; + model: string; + instructions: string; + ratingScale: RatingScale; + description?: string; + kmsKeyArn?: string; + tags?: Record; +}; + +export function toAddLlmAsAJudgeEvaluatorInput( + project: Project, + targets: readonly AwsDeploymentTarget[], + input: LlmAsAJudgeEvaluatorInput, +): AddResourceInput { + requireDeployedNameFits("Evaluator", project.name, input.name, "_", 48, targets); + if (!isValidEvaluatorModelId(input.modelProvider, input.model)) + throw new InputValidationError( + `invalid --model "${input.model}": expected ${MODEL_ID_FORMATS[input.modelProvider]}`, + ); + + const parsed = EvaluatorSchema.safeParse({ + name: input.name, + level: input.level, + description: input.description, + config: { + llmAsAJudge: { + // Bedrock is the default and stays implicit so existing Bedrock + // agentcore.json files are unchanged; only OpenResponses is written. + ...(input.modelProvider === "OpenResponses" ? { modelProvider: input.modelProvider } : {}), + model: input.model, + instructions: input.instructions, + ratingScale: input.ratingScale, + }, + }, + kmsKeyArn: input.kmsKeyArn, + tags: input.tags, + }); + if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); + return { resourceType: "evaluator", resourceConfig: parsed.data }; +} + export const createAddLlmAsAJudgeEvaluatorHandler = (config: AddProjectResourceConfig) => createHandler({ name: "llm-as-a-judge", - description: - "add an LLM-as-a-Judge evaluator — another LLM prompted with instructions on how to score a session", + description: "add an LLM-as-a-Judge evaluator to the current project", flags: [ flag("name", "the name of the evaluator", z.string().min(1)), flag("level", "what to score: SESSION, TRACE, or TOOL_CALL", z.string().min(1)), @@ -58,52 +110,26 @@ export const createAddLlmAsAJudgeEvaluatorHandler = (config: AddProjectResourceC ], handle: async (ctx, flags) => { const project = ctx.require(ProjectKey); - requireDeployedNameFits( - "Evaluator", - project.name, - flags["name"], - "_", - 48, - await config.projectManager.listTargets(project), - ); - const modelProvider = resolveModelProvider(flags["model-provider"]); - validateModel(modelProvider, flags["model"]); - const ratingScale = resolveRatingScale(flags["rating-scale"]); - const resolver = new SourceResolver({ stdin: config.io.stdin }); const instructions = await resolver.resolveText("instructions", flags["instructions"]); - const candidate = { - name: flags["name"], - level: flags["level"], - description: flags["description"], - config: { - llmAsAJudge: { - // Bedrock is the default and stays implicit so existing Bedrock - // agentcore.json files are unchanged; only OpenResponses is written. - ...(modelProvider === "OpenResponses" ? { modelProvider } : {}), - model: flags["model"], - instructions, - ratingScale, - }, - }, - kmsKeyArn: flags["kms-key-arn"], - tags: parseJsonFlagWithSchema("tags", flags["tags"], TagsSchema), - }; - - const parsed = EvaluatorSchema.safeParse(candidate); - if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); - await addProjectResource( ctx, config, project, - { - resourceType: "evaluator", - resourceConfig: parsed.data, - }, + toAddLlmAsAJudgeEvaluatorInput(project, await config.projectManager.listTargets(project), { + name: flags["name"], + level: flags["level"], + modelProvider, + model: flags["model"], + instructions, + ratingScale, + description: flags["description"], + kmsKeyArn: flags["kms-key-arn"], + tags: parseJsonFlagWithSchema("tags", flags["tags"], TagsSchema), + }), `added evaluator '${flags["name"]}' to '${project.name}'`, ); }, @@ -119,22 +145,10 @@ function resolveModelProvider(value: string | undefined): EvaluatorModelProvider return parsed.data; } -function validateModel(provider: EvaluatorModelProvider, model: string): void { - if (!isValidEvaluatorModelId(provider, model)) { - throw new InputValidationError( - provider === "Bedrock" - ? `invalid --model "${model}": expected a Bedrock model ID (e.g. anthropic.claude-3-5-sonnet-20240620-v1:0) or an inference-profile/foundation-model ARN` - : `invalid --model "${model}": expected an OpenResponses model ID (a non-empty identifier without spaces, e.g. openai.gpt-5.4)`, - ); - } -} - // A preset name expands to a fresh copy of the shared table; anything else is // treated as an inline JSON rating scale and validated against the schema. function resolveRatingScale(value: string): RatingScale { - if (isRatingScalePreset(value)) { - return structuredClone(RATING_SCALE_PRESETS[value]) as RatingScale; - } + if (isRatingScalePreset(value)) return expandRatingScalePreset(value); if (!value.trim().startsWith("{")) { throw new InputValidationError( `invalid --rating-scale "${value}": expected a preset (${RATING_SCALE_PRESET_NAMES.join(", ")}) or an inline JSON rating scale`, diff --git a/src/handlers/project/add/evaluator/llm-as-a-judge/llm-as-a-judge.screen.test.tsx b/src/handlers/project/add/evaluator/llm-as-a-judge/llm-as-a-judge.screen.test.tsx new file mode 100644 index 0000000000..50d14ed24c --- /dev/null +++ b/src/handlers/project/add/evaluator/llm-as-a-judge/llm-as-a-judge.screen.test.tsx @@ -0,0 +1,270 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { QueryClient } from "@tanstack/react-query"; +import { + cleanupScreens, + flatFrame, + renderScreen, + waitForFlatText, + waitForText, + type RenderScreenResult, +} from "../../../../../testing"; +import { createGatewayProjectTestHarness } from "../../gateway-test-support"; +import { projectQueryKey } from "../../../ProjectGate"; +import type { Project } from "../../../types"; +import { RATING_SCALE_PRESETS } from "./ratingScales"; +import { DEFAULT_JUDGE_MODEL } from "./screen"; + +const { cleanup, inProject, projectSpec, run } = createGatewayProjectTestHarness( + "add-llm-as-a-judge-wizard", +); + +afterEach(cleanup); +afterEach(cleanupScreens); + +const INSTRUCTIONS = "Rate the response: {assistant_turn}"; + +async function evaluatorOf(projectRoot: string, name: string) { + return ((await projectSpec(projectRoot)).evaluators ?? []).find( + (evaluator: { name: string }) => evaluator.name === name, + ); +} + +async function reachModelStep(screen: RenderScreenResult, name: string): Promise { + await waitForText(screen.lastFrame, "what should this evaluator be called?"); + await screen.write(name); + await screen.press("return"); + await waitForText(screen.lastFrame, "what should it score?"); + await screen.press("return"); + await waitForText(screen.lastFrame, "which model should judge?"); +} + +async function finishFromInstructions(screen: RenderScreenResult): Promise { + await waitForText(screen.lastFrame, "how should the judge score it?"); + await screen.write(INSTRUCTIONS); + await screen.press("ctrl+d"); + await waitForText(screen.lastFrame, "which rating scale?"); + await screen.press("return"); + await waitForText(screen.lastFrame, "this evaluator will be added to agentcore.json"); +} + +describe("project add evaluator llm-as-a-judge wizard", () => { + test("adds the same evaluator as the flags", async () => { + const projectRoot = await inProject(); + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false, gcTime: Infinity, staleTime: Infinity } }, + }); + const screen = renderScreen("/agentcore/add/evaluator/llm-as-a-judge", { queryClient }); + + await waitForText(screen.lastFrame, "what should this evaluator be called?"); + await screen.write("helpfulness"); + await screen.press("return"); + + await waitForText(screen.lastFrame, "what should it score?"); + expect(screen.lastFrame()).toContain("❯ ● SESSION"); + expect(screen.lastFrame()).toContain("score each agent response"); + await screen.press("down"); + await waitForText(screen.lastFrame, "❯ ● TRACE"); + await screen.press("return"); + + await waitForText(screen.lastFrame, "which model should judge?"); + expect(screen.lastFrame()).toContain("❯ ● Bedrock"); + expect(screen.lastFrame()).toContain("○ OpenResponses"); + await screen.press("return"); + await waitForText(screen.lastFrame, DEFAULT_JUDGE_MODEL); + await screen.press("return"); + + await waitForText(screen.lastFrame, "how should the judge score it?"); + expect(screen.lastFrame()).toContain("{context}"); + expect(screen.lastFrame()).toContain("[ctrl+d] continue"); + await screen.write("Rate the response: {assistant_turn}"); + await screen.press("return"); + await screen.write("Conversation: {context}"); + await screen.press("ctrl+d"); + + await waitForText(screen.lastFrame, "which rating scale?"); + expect(screen.lastFrame()).toContain( + "❯ ● 1-5-quality numerical · 1 Very Poor, 2 Poor, 3 Fair, 4 Good, 5 Excellent", + ); + expect(screen.lastFrame()).toContain("categorical · pass, fail"); + await screen.press("down"); + await screen.press("down"); + await waitForText(screen.lastFrame, "❯ ● pass-fail"); + await screen.press("return"); + + await waitForText(screen.lastFrame, "this evaluator will be added to agentcore.json"); + const review = flatFrame(screen.lastFrame); + expect(review).toContain("evaluator helpfulness"); + expect(review).toContain("level TRACE"); + expect(review).toContain("provider Bedrock"); + expect(review).toContain(`model ${DEFAULT_JUDGE_MODEL}`); + expect(review).toContain("instructions Rate the response: {assistant_turn} (+1 more line)"); + expect(review).toContain("rating scale pass-fail"); + await screen.press("return"); + + await waitForText(screen.lastFrame, "added evaluator 'helpfulness' to 'TestProject'"); + expect(screen.lastFrame()).toContain("agentcore deploy"); + const added = await evaluatorOf(projectRoot, "helpfulness"); + expect(added).toEqual({ + name: "helpfulness", + level: "TRACE", + config: { + llmAsAJudge: { + model: DEFAULT_JUDGE_MODEL, + instructions: "Rate the response: {assistant_turn}\nConversation: {context}", + ratingScale: RATING_SCALE_PRESETS["pass-fail"], + }, + }, + }); + expect( + queryClient + .getQueryData(projectQueryKey()) + ?.spec.evaluators?.some((evaluator) => evaluator.name === "helpfulness"), + ).toBe(true); + + await run([ + "add", + "evaluator", + "llm-as-a-judge", + "--name", + "flags", + "--level", + "TRACE", + "--model", + DEFAULT_JUDGE_MODEL, + "--instructions", + "Rate the response: {assistant_turn}\nConversation: {context}", + "--rating-scale", + "pass-fail", + ]); + expect({ ...(await evaluatorOf(projectRoot, "flags")), name: "helpfulness" }).toEqual(added); + + await screen.press("return"); + await waitForText(screen.lastFrame, "add a custom evaluator to the current project"); + screen.unmount(); + }, 20000); + + test("OpenResponses takes its own model ID and is written explicitly", async () => { + const projectRoot = await inProject(); + const screen = renderScreen("/agentcore/add/evaluator/llm-as-a-judge"); + await reachModelStep(screen, "judge"); + + await screen.press("down"); + await waitForText(screen.lastFrame, "❯ ● OpenResponses"); + await screen.press("return"); + await waitForText(screen.lastFrame, "an OpenResponses model ID"); + await screen.write("openai.gpt-5.4"); + + /** Each provider keeps its own answer, so Bedrock still opens on its default. **/ + await screen.press("escape"); + await screen.press("up"); + await screen.press("return"); + await waitForText(screen.lastFrame, DEFAULT_JUDGE_MODEL); + await screen.press("escape"); + await screen.press("down"); + await screen.press("return"); + await waitForText(screen.lastFrame, "openai.gpt-5.4"); + await screen.press("return"); + + await finishFromInstructions(screen); + expect(flatFrame(screen.lastFrame)).toContain("provider OpenResponses"); + await screen.press("return"); + + await waitForText(screen.lastFrame, "added evaluator 'judge'"); + expect((await evaluatorOf(projectRoot, "judge")).config.llmAsAJudge).toMatchObject({ + modelProvider: "OpenResponses", + model: "openai.gpt-5.4", + }); + screen.unmount(); + }, 20000); + + test.each([ + /** Bedrock opens on its default ID, so a trailing word breaks it. **/ + ["Bedrock", [], " x", "Must be a Bedrock model ID"], + ["OpenResponses", ["down"], "bad model", "Must be an OpenResponses model ID"], + ] as const)( + "a malformed %s model ID keeps the wizard on the model step", + async (_provider, moves, typed, message) => { + await inProject(); + const screen = renderScreen("/agentcore/add/evaluator/llm-as-a-judge"); + await reachModelStep(screen, "judge"); + for (const move of moves) await screen.press(move); + await screen.press("return"); + await waitForText(screen.lastFrame, "model ID"); + await screen.write(typed); + + await screen.press("return"); + + await waitForText(screen.lastFrame, message); + expect(screen.lastFrame()).toContain("which model should judge?"); + screen.unmount(); + }, + ); + + test("blank instructions are refused", async () => { + await inProject(); + const screen = renderScreen("/agentcore/add/evaluator/llm-as-a-judge"); + await reachModelStep(screen, "judge"); + await screen.press("return"); + await screen.press("return"); + await waitForText(screen.lastFrame, "how should the judge score it?"); + + await screen.press("ctrl+d"); + + await waitForText(screen.lastFrame, "Instructions is required"); + expect(screen.lastFrame()).not.toContain("which rating scale?"); + screen.unmount(); + }); + + test("validates the deployed name against the longest project target", async () => { + const projectRoot = await inProject(); + await writeFile( + join(projectRoot, "agentcore", "aws-targets.json"), + JSON.stringify([{ name: "production", account: "111122223333", region: "us-east-1" }]), + ); + const screen = renderScreen("/agentcore/add/evaluator/llm-as-a-judge"); + + await waitForText(screen.lastFrame, "what should this evaluator be called?"); + await screen.write(`e${"x".repeat(24)}`); + await waitForText(screen.lastFrame, `e${"x".repeat(24)}`); + expect(screen.lastFrame()).not.toContain("The maximum is 48."); + + await screen.write("x"); + await waitForFlatText(screen.lastFrame, "is 49 characters. The maximum is 48."); + expect(flatFrame(screen.lastFrame)).toContain("TestProject_production_"); + screen.unmount(); + }); + + test("a rejected add reports itself and hands the form back", async () => { + const projectRoot = await inProject(); + await run([ + "add", + "evaluator", + "llm-as-a-judge", + "--name", + "judge", + "--level", + "SESSION", + "--model", + DEFAULT_JUDGE_MODEL, + "--instructions", + INSTRUCTIONS, + "--rating-scale", + "pass-fail", + ]); + const screen = renderScreen("/agentcore/add/evaluator/llm-as-a-judge"); + await reachModelStep(screen, "judge"); + await screen.press("return"); + await screen.press("return"); + await finishFromInstructions(screen); + await screen.press("return"); + + await waitForFlatText(screen.lastFrame, "a evaluator with name 'judge' already exists"); + await screen.press("escape"); + await waitForText(screen.lastFrame, "this evaluator will be added to agentcore.json"); + expect(flatFrame(screen.lastFrame)).toContain("evaluator judge"); + expect((await projectSpec(projectRoot)).evaluators).toHaveLength(1); + screen.unmount(); + }, 20000); +}); diff --git a/src/handlers/project/add/evaluator/llm-as-a-judge/ratingScales.ts b/src/handlers/project/add/evaluator/llm-as-a-judge/ratingScales.ts index c75d18dc50..38976176e7 100644 --- a/src/handlers/project/add/evaluator/llm-as-a-judge/ratingScales.ts +++ b/src/handlers/project/add/evaluator/llm-as-a-judge/ratingScales.ts @@ -83,3 +83,7 @@ export const RATING_SCALE_PRESET_NAMES = Object.keys(RATING_SCALE_PRESETS) as Ra export function isRatingScalePreset(value: string): value is RatingScalePreset { return value in RATING_SCALE_PRESETS; } + +export function expandRatingScalePreset(preset: RatingScalePreset): RatingScale { + return structuredClone(RATING_SCALE_PRESETS[preset]) as RatingScale; +} diff --git a/src/handlers/project/add/evaluator/llm-as-a-judge/screen.tsx b/src/handlers/project/add/evaluator/llm-as-a-judge/screen.tsx new file mode 100644 index 0000000000..c5dc243847 --- /dev/null +++ b/src/handlers/project/add/evaluator/llm-as-a-judge/screen.tsx @@ -0,0 +1,269 @@ +import { useMemo, useState } from "react"; +import { useQueryClient } from "@tanstack/react-query"; +import { useNavigate } from "react-router"; +import z from "zod"; +import { + ChoiceField, + promptPreview, + RevealChoiceField, + Step, + Summary, + TextAreaField, + TextField, + Wizard, + type Choice, +} from "../../../../../components/wizard"; +import type { AwsDeploymentTarget } from "../../../../../projectSchemas/aws-targets"; +import { + isValidEvaluatorModelId, + type EvaluationLevel, + type EvaluatorModelProvider, +} from "../../../../../projectSchemas/evaluator"; +import { ProjectKey } from "../../../../../router"; +import type { ScreenProps } from "../../../../types"; +import type { Project } from "../../../types"; +import { + LoadingFrame, + ProjectGate, + projectQueryKey, + useProjectTargets, +} from "../../../ProjectGate"; +import { EVALUATOR_MENU, EVALUATOR_NAME_HELP, evaluatorNameSchema, LEVEL_CHOICES } from "../shared"; +import { + MODEL_ID_FORMATS, + toAddLlmAsAJudgeEvaluatorInput, + type LlmAsAJudgeEvaluatorInput, +} from "./index"; +import { + expandRatingScalePreset, + RATING_SCALE_PRESET_NAMES, + RATING_SCALE_PRESETS, + type RatingScalePreset, +} from "./ratingScales"; + +const BREADCRUMB = ["agentcore", "add", "evaluator", "llm-as-a-judge"]; +const DESCRIPTION = "add an LLM-as-a-Judge evaluator to the current project"; + +const PROVIDER_CHOICES: Choice[] = [ + { + value: "Bedrock", + label: "Bedrock", + description: "an Amazon Bedrock model or inference profile", + }, + { + value: "OpenResponses", + label: "OpenResponses", + description: "an OpenAI model on Bedrock through the OpenResponses API", + }, +]; + +/** The Evaluator service sends a temperature, which Claude models from Opus 4.7 on refuse. **/ +export const DEFAULT_JUDGE_MODEL = "global.anthropic.claude-sonnet-4-6"; + +const MODEL_PLACEHOLDERS: Record = { + Bedrock: DEFAULT_JUDGE_MODEL, + OpenResponses: "openai.gpt-5.4", +}; + +function ratingScaleDescription(preset: RatingScalePreset): string { + const scale = RATING_SCALE_PRESETS[preset]; + return "numerical" in scale + ? `numerical · ${scale.numerical.map((rung) => `${rung.value} ${rung.label}`).join(", ")}` + : `categorical · ${scale.categorical.map((rung) => rung.label).join(", ")}`; +} + +const RATING_SCALE_CHOICES: Choice[] = RATING_SCALE_PRESET_NAMES.map( + (preset) => ({ value: preset, label: preset, description: ratingScaleDescription(preset) }), +); + +type LlmAsAJudgeFormValues = { + name: string; + level: EvaluationLevel; + modelProvider: EvaluatorModelProvider; + models: Record; + instructions: string; + ratingScale: RatingScalePreset; +}; + +export function toLlmAsAJudgeInput(values: LlmAsAJudgeFormValues): LlmAsAJudgeEvaluatorInput { + return { + name: values.name, + level: values.level, + modelProvider: values.modelProvider, + model: values.models[values.modelProvider].trim(), + instructions: values.instructions, + ratingScale: expandRatingScalePreset(values.ratingScale), + }; +} + +function summaryOf(values: LlmAsAJudgeFormValues): Record { + return { + evaluator: values.name, + level: values.level, + provider: values.modelProvider, + model: values.models[values.modelProvider].trim(), + instructions: promptPreview(values.instructions), + "rating scale": values.ratingScale, + }; +} + +function modelSchema(provider: EvaluatorModelProvider): z.ZodType { + return z + .string() + .refine( + (model) => isValidEvaluatorModelId(provider, model.trim()), + `Must be ${MODEL_ID_FORMATS[provider]}`, + ); +} + +export function AddLlmAsAJudgeEvaluatorScreen({ ctx, core }: ScreenProps) { + const navigate = useNavigate(); + return ( + navigate(EVALUATOR_MENU)} + > + {(project) => } + + ); +} + +function AddLlmAsAJudgeEvaluatorLoader({ + project, + core, +}: { + project: Project; + core: ScreenProps["core"]; +}) { + const navigate = useNavigate(); + const targets = useProjectTargets(core, project); + + if (targets.data !== undefined) { + return ; + } + + return ( + navigate(EVALUATOR_MENU)} + /> + ); +} + +function AddLlmAsAJudgeEvaluatorWizard({ + project, + targets, + core, +}: { + project: Project; + targets: readonly AwsDeploymentTarget[]; + core: ScreenProps["core"]; +}) { + const navigate = useNavigate(); + const queryClient = useQueryClient(); + const [values, setValues] = useState({ + name: "", + level: "SESSION", + modelProvider: "Bedrock", + models: { Bedrock: DEFAULT_JUDGE_MODEL, OpenResponses: "" }, + instructions: "", + ratingScale: "1-5-quality", + }); + const set = (update: Partial) => + setValues((current) => ({ ...current, ...update })); + const nameSchema = useMemo( + () => evaluatorNameSchema(project.name, targets), + [project.name, targets], + ); + const provider = values.modelProvider; + + return ( + navigate(EVALUATOR_MENU)} + onSubmit={async function* () { + const updated = yield* core.projectManager.addResource( + project, + toAddLlmAsAJudgeEvaluatorInput(project, targets, toLlmAsAJudgeInput(values)), + ); + queryClient.setQueryData(projectQueryKey(), updated); + return updated; + }} + runningLabel={`adding evaluator ${values.name}…`} + successLabel={`added evaluator '${values.name}' to '${project.name}'`} + successNextSteps={["agentcore deploy"]} + onDone={() => navigate(EVALUATOR_MENU)} + doneLabel="go back" + > + + set({ name })} + required + schema={nameSchema} + live + /> + + + + set({ level })} + /> + + + + set({ modelProvider })} + input={{ + opensFor: () => true, + label: "Model ID", + name: "model ID", + help: MODEL_ID_FORMATS[provider], + placeholder: MODEL_PLACEHOLDERS[provider], + value: values.models[provider], + onChange: (model) => set({ models: { ...values.models, [provider]: model } }), + required: true, + schema: modelSchema(provider), + }} + /> + + + + set({ instructions })} + required + /> + + + + set({ ratingScale })} + /> + + + + + + + ); +} diff --git a/src/handlers/project/add/evaluator/shared.ts b/src/handlers/project/add/evaluator/shared.ts new file mode 100644 index 0000000000..2b3c6ce34c --- /dev/null +++ b/src/handlers/project/add/evaluator/shared.ts @@ -0,0 +1,40 @@ +import type z from "zod"; +import type { Choice } from "../../../../components/wizard"; +import type { AwsDeploymentTarget } from "../../../../projectSchemas/aws-targets"; +import { + EvaluationLevelSchema, + EvaluatorNameSchema, + type EvaluationLevel, +} from "../../../../projectSchemas/evaluator"; +import { requireDeployedNameFits } from "../shared"; + +export const EVALUATOR_MENU = "/agentcore/add/evaluator"; + +export const EVALUATOR_NAME_HELP = + "letters, digits and underscores, starting with a letter; the deployed __ must fit 48 characters"; + +const LEVEL_DESCRIPTIONS: Record = { + SESSION: "score a whole conversation", + TRACE: "score each agent response", + TOOL_CALL: "score each tool call", +}; + +export const LEVEL_CHOICES: Choice[] = EvaluationLevelSchema.options.map( + (level) => ({ value: level, label: level, description: LEVEL_DESCRIPTIONS[level] }), +); + +export function evaluatorNameSchema( + projectName: string, + targets: readonly AwsDeploymentTarget[], +): z.ZodType { + return EvaluatorNameSchema.superRefine((name, ctx) => { + try { + requireDeployedNameFits("Evaluator", projectName, name, "_", 48, targets); + } catch (error) { + ctx.addIssue({ + code: "custom", + message: error instanceof Error ? error.message : String(error), + }); + } + }); +} diff --git a/src/handlers/project/add/harness/harness.screen.test.tsx b/src/handlers/project/add/harness/harness.screen.test.tsx index 21a033b665..2b2cd74a91 100644 --- a/src/handlers/project/add/harness/harness.screen.test.tsx +++ b/src/handlers/project/add/harness/harness.screen.test.tsx @@ -24,7 +24,6 @@ import { DEFAULT_HARNESS_MODEL } from "../../../../projectSchemas/harness"; import { createGatewayProjectTestHarness } from "../gateway-test-support"; import { projectQueryKey } from "../../ProjectGate"; import type { Project } from "../../types"; -import { promptPreview } from "./screen"; const { cleanup, inProject, projectSpec, run } = createGatewayProjectTestHarness("add-harness-wizard"); @@ -60,32 +59,6 @@ async function acceptDefaultModel(screen: RenderScreenResult): Promise { await screen.press("return"); } -// The review line describes the prompt exactly as system-prompt.md will hold -// it, blank lines included, so what the user confirms is what gets written. -describe("promptPreview", () => { - test.each([ - ["a one-line prompt", "You are a pirate.", "You are a pirate."], - [ - "counts the lines after the first", - "You are a pirate.\nAnswer in rhyme.", - "You are a pirate. (+1 more line)", - ], - ["keeps a leading blank line", "\nYou are a pirate.", " (+1 more line)"], - [ - "counts a trailing newline left by enter", - "You are a pirate.\n", - "You are a pirate. (+1 more line)", - ], - [ - "cuts a long first line short", - `${"x".repeat(70)}\ny\nz`, - `${"x".repeat(59)}… (+2 more lines)`, - ], - ])("%s", (_label, prompt, preview) => { - expect(promptPreview(prompt)).toBe(preview); - }); -}); - describe("project add harness wizard", () => { test("collects a name, prompt and model, then scaffolds the same harness as the flags", async () => { const projectRoot = await inProject(); diff --git a/src/handlers/project/add/harness/screen.tsx b/src/handlers/project/add/harness/screen.tsx index 325949d03e..ff8c01720f 100644 --- a/src/handlers/project/add/harness/screen.tsx +++ b/src/handlers/project/add/harness/screen.tsx @@ -2,7 +2,14 @@ import { useMemo, useState } from "react"; import { useQueryClient } from "@tanstack/react-query"; import { useNavigate } from "react-router"; import type z from "zod"; -import { Step, Summary, TextAreaField, TextField, Wizard } from "../../../../components/wizard"; +import { + promptPreview, + Step, + Summary, + TextAreaField, + TextField, + Wizard, +} from "../../../../components/wizard"; import type { AwsDeploymentTarget } from "../../../../projectSchemas/aws-targets"; import { HarnessNameSchema, type HarnessSpecSchema } from "../../../../projectSchemas/harness"; import { ProjectKey } from "../../../../router"; @@ -46,18 +53,6 @@ export function toHarnessInput(values: HarnessFormValues): HarnessSpecInput { }; } -// promptPreview keeps the review to one line: the prompt's first line, cut -// short, with a count of what follows it. It reads the prompt exactly as it -// will be saved — no trimming — so a leading blank line or a trailing newline -// left by an extra enter shows up here rather than only in system-prompt.md. -export function promptPreview(prompt: string): string { - const lines = prompt.split("\n"); - const first = lines[0] ?? ""; - const shown = first.length > 60 ? `${first.slice(0, 59)}…` : first; - const rest = lines.length - 1; - return rest > 0 ? `${shown} (+${rest} more ${rest === 1 ? "line" : "lines"})` : shown; -} - function summaryOf(values: HarnessFormValues): Record { return { harness: values.name, diff --git a/src/handlers/project/add/index.ts b/src/handlers/project/add/index.ts index 2f35df6703..0a1d74cea9 100644 --- a/src/handlers/project/add/index.ts +++ b/src/handlers/project/add/index.ts @@ -39,6 +39,7 @@ export function createAddProjectResourceHandler( "config-bundle", "payment-manager", "payment-connector", + "evaluator", ); projectAdd.default(renderTui(core, config.io)); // withProject first, so it is the outermost wrapper: a resource added outside @@ -56,7 +57,7 @@ export function createAddProjectResourceHandler( projectAdd.handler(createAddRuntimeHandler(config)); projectAdd.handler(createAddOnlineEvalHandler(config)); projectAdd.handler(createAddOnlineInsightHandler(config)); - projectAdd.handler(createAddEvaluatorHandler(config)); + projectAdd.handler(createAddEvaluatorHandler(config, core)); projectAdd.handler(createAddCredentialsHandler(config)); projectAdd.handler(createAddGatewayHandler(config)); projectAdd.handler(createAddGatewayTargetHandler(config)); diff --git a/src/projectSchemas/evaluator.ts b/src/projectSchemas/evaluator.ts index 4dd65d8395..36e647b231 100644 --- a/src/projectSchemas/evaluator.ts +++ b/src/projectSchemas/evaluator.ts @@ -71,10 +71,11 @@ export const LlmAsAJudgeConfigSchema = z.object({ ratingScale: RatingScaleSchema, }); export type LlmAsAJudgeConfig = z.infer; +export const DEFAULT_CODE_BASED_TIMEOUT_SECONDS = 60; export const ManagedCodeBasedConfigSchema = z.object({ codeLocation: z.string().min(1), entrypoint: z.string().min(1).default("lambda_function.handler"), - timeoutSeconds: z.number().int().min(1).max(300).default(60), + timeoutSeconds: z.number().int().min(1).max(300).default(DEFAULT_CODE_BASED_TIMEOUT_SECONDS), additionalPolicies: z.array(z.string().min(1)).optional(), }); export type ManagedCodeBasedConfig = z.infer; diff --git a/src/testing/renderScreen.tsx b/src/testing/renderScreen.tsx index 70c32990db..5558892812 100644 --- a/src/testing/renderScreen.tsx +++ b/src/testing/renderScreen.tsx @@ -129,6 +129,7 @@ export const keys = { down: "", left: "", right: "", + backspace: "\u007F", // The end-of-transmission control character, which Ink reports as ctrl+d. "ctrl+d": "\u0004", } as const; From 94e558560349bde592125157c8fb394cfbae7d81 Mon Sep 17 00:00:00 2001 From: Tejas Kashinath Date: Wed, 30 Sep 2026 00:46:04 +0000 Subject: [PATCH 2/3] refactor(tui): show the line count in promptPreview instead of cutting the first line --- src/components/wizard/fields.tsx | 9 +-------- src/components/wizard/wizard.test.tsx | 15 +++------------ .../llm-as-a-judge/llm-as-a-judge.screen.test.tsx | 2 +- .../project/add/harness/harness.screen.test.tsx | 2 +- 4 files changed, 6 insertions(+), 22 deletions(-) diff --git a/src/components/wizard/fields.tsx b/src/components/wizard/fields.tsx index 6ce7166480..392397b4d0 100644 --- a/src/components/wizard/fields.tsx +++ b/src/components/wizard/fields.tsx @@ -611,16 +611,9 @@ export function MultiChoiceField({ ); } -// promptPreview keeps a multi-line answer to one review line: its first line, -// cut short, with a count of what follows it. It reads the text exactly as it -// will be saved — no trimming — so a leading blank line or a trailing newline -// left by an extra enter shows up in the review rather than only in the spec. export function promptPreview(prompt: string): string { const lines = prompt.split("\n"); - const first = lines[0] ?? ""; - const shown = first.length > 60 ? `${first.slice(0, 59)}…` : first; - const rest = lines.length - 1; - return rest > 0 ? `${shown} (+${rest} more ${rest === 1 ? "line" : "lines"})` : shown; + return lines.length === 1 ? prompt : `${lines[0]} · ${lines.length} lines`; } export interface SummaryProps { diff --git a/src/components/wizard/wizard.test.tsx b/src/components/wizard/wizard.test.tsx index 04a31755c0..45f44ccea0 100644 --- a/src/components/wizard/wizard.test.tsx +++ b/src/components/wizard/wizard.test.tsx @@ -1150,21 +1150,12 @@ describe("Wizard authoring guards", () => { describe("promptPreview", () => { test.each([ ["a one-line prompt", "You are a pirate.", "You are a pirate."], - [ - "counts the lines after the first", - "You are a pirate.\nAnswer in rhyme.", - "You are a pirate. (+1 more line)", - ], - ["keeps a leading blank line", "\nYou are a pirate.", " (+1 more line)"], + ["counts every line", "You are a pirate.\nAnswer in rhyme.", "You are a pirate. · 2 lines"], + ["keeps a leading blank line", "\nYou are a pirate.", " · 2 lines"], [ "counts a trailing newline left by enter", "You are a pirate.\n", - "You are a pirate. (+1 more line)", - ], - [ - "cuts a long first line short", - `${"x".repeat(70)}\ny\nz`, - `${"x".repeat(59)}… (+2 more lines)`, + "You are a pirate. · 2 lines", ], ])("%s", (_label, prompt, preview) => { expect(promptPreview(prompt)).toBe(preview); diff --git a/src/handlers/project/add/evaluator/llm-as-a-judge/llm-as-a-judge.screen.test.tsx b/src/handlers/project/add/evaluator/llm-as-a-judge/llm-as-a-judge.screen.test.tsx index 50d14ed24c..facc024c51 100644 --- a/src/handlers/project/add/evaluator/llm-as-a-judge/llm-as-a-judge.screen.test.tsx +++ b/src/handlers/project/add/evaluator/llm-as-a-judge/llm-as-a-judge.screen.test.tsx @@ -99,7 +99,7 @@ describe("project add evaluator llm-as-a-judge wizard", () => { expect(review).toContain("level TRACE"); expect(review).toContain("provider Bedrock"); expect(review).toContain(`model ${DEFAULT_JUDGE_MODEL}`); - expect(review).toContain("instructions Rate the response: {assistant_turn} (+1 more line)"); + expect(review).toContain("instructions Rate the response: {assistant_turn} · 2 lines"); expect(review).toContain("rating scale pass-fail"); await screen.press("return"); diff --git a/src/handlers/project/add/harness/harness.screen.test.tsx b/src/handlers/project/add/harness/harness.screen.test.tsx index 2b2cd74a91..04f2a5ee65 100644 --- a/src/handlers/project/add/harness/harness.screen.test.tsx +++ b/src/handlers/project/add/harness/harness.screen.test.tsx @@ -101,7 +101,7 @@ describe("project add harness wizard", () => { expect(review).toContain("harness assistant"); expect(review).toContain("provider bedrock"); expect(review).toContain(`model ${DEFAULT_HARNESS_MODEL.modelId}`); - expect(review).toContain("system prompt You are a pirate. (+1 more line)"); + expect(review).toContain("system prompt You are a pirate. · 2 lines"); await screen.press("return"); await waitForText(screen.lastFrame, "added harness 'assistant' to 'TestProject'"); From 4656785ff22db769aad9ee57d6be135b884297ba Mon Sep 17 00:00:00 2001 From: Tejas Kashinath Date: Wed, 30 Sep 2026 01:47:22 +0000 Subject: [PATCH 3/3] fix(project): validate evaluator input before any I/O - llm-as-a-judge checks the deployed name and the model before reading --instructions, so `--instructions -` no longer waits on stdin when a flag is already invalid. - The managed code-based branch validates its whole scaffold input against the evaluator schema before scaffolding. A --name such as ../outside, or a malformed --kms-key-arn, used to scaffold first and fail at the spec write, which could leave files outside the project. - promptPreview caps the first line at 60 characters again. --- src/components/wizard/fields.tsx | 4 +- src/components/wizard/wizard.test.tsx | 2 + .../add/evaluator/code-based/index.test.ts | 26 +++++++++++++ .../project/add/evaluator/code-based/index.ts | 37 ++++++++++--------- .../evaluator/llm-as-a-judge/index.test.ts | 24 ++++++++++++ .../add/evaluator/llm-as-a-judge/index.ts | 33 ++++++++++------- .../add/evaluator/llm-as-a-judge/screen.tsx | 2 +- 7 files changed, 94 insertions(+), 34 deletions(-) diff --git a/src/components/wizard/fields.tsx b/src/components/wizard/fields.tsx index 392397b4d0..e0bfc9408e 100644 --- a/src/components/wizard/fields.tsx +++ b/src/components/wizard/fields.tsx @@ -613,7 +613,9 @@ export function MultiChoiceField({ export function promptPreview(prompt: string): string { const lines = prompt.split("\n"); - return lines.length === 1 ? prompt : `${lines[0]} · ${lines.length} lines`; + const [first = ""] = lines; + const shown = first.length > 60 ? `${first.slice(0, 59)}…` : first; + return lines.length === 1 ? shown : `${shown} · ${lines.length} lines`; } export interface SummaryProps { diff --git a/src/components/wizard/wizard.test.tsx b/src/components/wizard/wizard.test.tsx index 45f44ccea0..db82b4f998 100644 --- a/src/components/wizard/wizard.test.tsx +++ b/src/components/wizard/wizard.test.tsx @@ -1157,6 +1157,8 @@ describe("promptPreview", () => { "You are a pirate.\n", "You are a pirate. · 2 lines", ], + ["cuts a long first line short", `${"x".repeat(70)}\ny`, `${"x".repeat(59)}… · 2 lines`], + ["cuts a long single line short", "x".repeat(70), `${"x".repeat(59)}…`], ])("%s", (_label, prompt, preview) => { expect(promptPreview(prompt)).toBe(preview); }); diff --git a/src/handlers/project/add/evaluator/code-based/index.test.ts b/src/handlers/project/add/evaluator/code-based/index.test.ts index 2b1052557e..fa361ce779 100644 --- a/src/handlers/project/add/evaluator/code-based/index.test.ts +++ b/src/handlers/project/add/evaluator/code-based/index.test.ts @@ -171,6 +171,32 @@ describe("project add evaluator code-based", () => { await expectError(promise, requiredMessage ?? /./, InputValidationError); }); + test.each<[string, string[], string]>([ + ["a path in --name", ["--name", "../outside"], "Must begin with a letter"], + [ + "an invalid --kms-key-arn", + ["--name", "custom_eval", "--kms-key-arn", "not-a-key"], + "Must be a valid KMS key ARN", + ], + ])("rejects %s before scaffolding anything", async (_label, flags, message) => { + const { projectRoot, cleanup } = await initProject(); + cleanups.push(cleanup); + + await expectError( + run(["add", "evaluator", "code-based", "--level", "SESSION", ...flags]), + message, + InputValidationError, + ); + + for (const path of [ + join(projectRoot, "..", "outside"), + join(projectRoot, "outside"), + join(projectRoot, "app", "custom_eval"), + ]) { + expect(await Bun.file(join(path, "lambda_function.py")).exists()).toBe(false); + } + }); + test.each([ ["--metric", "deepeval.FaithfulnessMetric"], ["--model", "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"], diff --git a/src/handlers/project/add/evaluator/code-based/index.ts b/src/handlers/project/add/evaluator/code-based/index.ts index 5622679a0e..ec2a574e43 100644 --- a/src/handlers/project/add/evaluator/code-based/index.ts +++ b/src/handlers/project/add/evaluator/code-based/index.ts @@ -2,9 +2,9 @@ import z from "zod"; import { createHandler, flag, ProjectKey } from "../../../../../router"; import { InputValidationError } from "../../../../../errors"; import type { AwsDeploymentTarget } from "../../../../../projectSchemas/aws-targets"; -import { EvaluatorSchema, EvaluationLevelSchema } from "../../../../../projectSchemas/evaluator"; +import { EvaluatorSchema } from "../../../../../projectSchemas/evaluator"; import { TagsSchema } from "../../../../../projectSchemas/tags"; -import type { AddResourceInput, ManagedEvaluatorScaffoldInput, Project } from "../../../types"; +import type { AddResourceInput, Project } from "../../../types"; import { parseJsonFlagWithSchema } from "../../../../utils"; import type { AddProjectResourceConfig } from "../../types"; import { addProjectResource, requireDeployedNameFits } from "../../shared"; @@ -19,37 +19,38 @@ export type CodeBasedEvaluatorInput = { tags?: Record; } & ({ lambdaArn: string } | { lambdaArn?: undefined; timeoutSeconds?: number }); +const ManagedEvaluatorScaffoldSchema = EvaluatorSchema.pick({ + name: true, + level: true, + description: true, + kmsKeyArn: true, + tags: true, +}).extend({ timeoutSeconds: TimeoutSecondsSchema.optional() }); + export function toAddCodeBasedEvaluatorInput( project: Project, targets: readonly AwsDeploymentTarget[], input: CodeBasedEvaluatorInput, ): AddResourceInput { requireDeployedNameFits("Evaluator", project.name, input.name, "_", 48, targets); - const levelParsed = EvaluationLevelSchema.safeParse(input.level); - if (!levelParsed.success) throw new InputValidationError(z.prettifyError(levelParsed.error)); - - const base = { - name: input.name, - level: levelParsed.data, - description: input.description, - kmsKeyArn: input.kmsKeyArn, - tags: input.tags, - }; if (input.lambdaArn !== undefined) { + const { lambdaArn, ...evaluator } = input; const parsed = EvaluatorSchema.safeParse({ - ...base, - config: { codeBased: { external: { lambdaArn: input.lambdaArn } } }, + ...evaluator, + config: { codeBased: { external: { lambdaArn } } }, }); if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); return { resourceType: "evaluator", resourceConfig: parsed.data }; } - const scaffold: ManagedEvaluatorScaffoldInput = { - ...base, - ...(input.timeoutSeconds !== undefined && { timeoutSeconds: input.timeoutSeconds }), + const parsed = ManagedEvaluatorScaffoldSchema.safeParse(input); + if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); + return { + resourceType: "evaluator", + resourceConfig: { name: parsed.data.name }, + scaffold: parsed.data, }; - return { resourceType: "evaluator", resourceConfig: { name: scaffold.name }, scaffold }; } export function scaffoldedEvaluatorNote(name: string): string { diff --git a/src/handlers/project/add/evaluator/llm-as-a-judge/index.test.ts b/src/handlers/project/add/evaluator/llm-as-a-judge/index.test.ts index ad0c3f5323..51c67c399b 100644 --- a/src/handlers/project/add/evaluator/llm-as-a-judge/index.test.ts +++ b/src/handlers/project/add/evaluator/llm-as-a-judge/index.test.ts @@ -204,6 +204,30 @@ describe("project add evaluator llm-as-a-judge", () => { ); }); + test.each<[string, string, string, string]>([ + ["an invalid --model", "x", "not a model", 'invalid --model "not a model"'], + ["a deployed name over the limit", `e${"x".repeat(28)}`, MODEL, "The maximum is 48."], + ])("%s fails before --instructions - reads stdin", async (_label, name, model, message) => { + const { cleanup } = await initProject(); + cleanups.push(cleanup); + const promise = run([ + "add", + "evaluator", + "llm-as-a-judge", + "--name", + name, + "--level", + "SESSION", + "--model", + model, + "--instructions", + "-", + "--rating-scale", + "pass-fail", + ]); + await expectError(promise, message, InputValidationError); + }); + test("accepts a valid OpenResponses model id", async () => { const { projectRoot, cleanup } = await initProject(); cleanups.push(cleanup); diff --git a/src/handlers/project/add/evaluator/llm-as-a-judge/index.ts b/src/handlers/project/add/evaluator/llm-as-a-judge/index.ts index a794dd18bc..dae892bb16 100644 --- a/src/handlers/project/add/evaluator/llm-as-a-judge/index.ts +++ b/src/handlers/project/add/evaluator/llm-as-a-judge/index.ts @@ -2,7 +2,6 @@ import z from "zod"; import { createHandler, flag, ProjectKey } from "../../../../../router"; import { InputValidationError } from "../../../../../errors"; import { SourceResolver } from "../../../../../io"; -import type { AwsDeploymentTarget } from "../../../../../projectSchemas/aws-targets"; import { EvaluatorModelProviderSchema, EvaluatorSchema, @@ -12,7 +11,7 @@ import { type RatingScale, } from "../../../../../projectSchemas/evaluator"; import { TagsSchema } from "../../../../../projectSchemas/tags"; -import type { AddResourceInput, Project } from "../../../types"; +import type { AddResourceInput } from "../../../types"; import { parseJsonFlagWithSchema } from "../../../../utils"; import type { AddProjectResourceConfig } from "../../types"; import { addProjectResource, requireDeployedNameFits } from "../../shared"; @@ -41,17 +40,7 @@ export type LlmAsAJudgeEvaluatorInput = { tags?: Record; }; -export function toAddLlmAsAJudgeEvaluatorInput( - project: Project, - targets: readonly AwsDeploymentTarget[], - input: LlmAsAJudgeEvaluatorInput, -): AddResourceInput { - requireDeployedNameFits("Evaluator", project.name, input.name, "_", 48, targets); - if (!isValidEvaluatorModelId(input.modelProvider, input.model)) - throw new InputValidationError( - `invalid --model "${input.model}": expected ${MODEL_ID_FORMATS[input.modelProvider]}`, - ); - +export function toAddLlmAsAJudgeEvaluatorInput(input: LlmAsAJudgeEvaluatorInput): AddResourceInput { const parsed = EvaluatorSchema.safeParse({ name: input.name, level: input.level, @@ -110,7 +99,16 @@ export const createAddLlmAsAJudgeEvaluatorHandler = (config: AddProjectResourceC ], handle: async (ctx, flags) => { const project = ctx.require(ProjectKey); + requireDeployedNameFits( + "Evaluator", + project.name, + flags["name"], + "_", + 48, + await config.projectManager.listTargets(project), + ); const modelProvider = resolveModelProvider(flags["model-provider"]); + validateModel(modelProvider, flags["model"]); const ratingScale = resolveRatingScale(flags["rating-scale"]); const resolver = new SourceResolver({ stdin: config.io.stdin }); const instructions = await resolver.resolveText("instructions", flags["instructions"]); @@ -119,7 +117,7 @@ export const createAddLlmAsAJudgeEvaluatorHandler = (config: AddProjectResourceC ctx, config, project, - toAddLlmAsAJudgeEvaluatorInput(project, await config.projectManager.listTargets(project), { + toAddLlmAsAJudgeEvaluatorInput({ name: flags["name"], level: flags["level"], modelProvider, @@ -145,6 +143,13 @@ function resolveModelProvider(value: string | undefined): EvaluatorModelProvider return parsed.data; } +function validateModel(provider: EvaluatorModelProvider, model: string): void { + if (!isValidEvaluatorModelId(provider, model)) + throw new InputValidationError( + `invalid --model "${model}": expected ${MODEL_ID_FORMATS[provider]}`, + ); +} + // A preset name expands to a fresh copy of the shared table; anything else is // treated as an inline JSON rating scale and validated against the schema. function resolveRatingScale(value: string): RatingScale { diff --git a/src/handlers/project/add/evaluator/llm-as-a-judge/screen.tsx b/src/handlers/project/add/evaluator/llm-as-a-judge/screen.tsx index c5dc243847..ffdf2fffbd 100644 --- a/src/handlers/project/add/evaluator/llm-as-a-judge/screen.tsx +++ b/src/handlers/project/add/evaluator/llm-as-a-judge/screen.tsx @@ -191,7 +191,7 @@ function AddLlmAsAJudgeEvaluatorWizard({ onSubmit={async function* () { const updated = yield* core.projectManager.addResource( project, - toAddLlmAsAJudgeEvaluatorInput(project, targets, toLlmAsAJudgeInput(values)), + toAddLlmAsAJudgeEvaluatorInput(toLlmAsAJudgeInput(values)), ); queryClient.setQueryData(projectQueryKey(), updated); return updated;