diff --git a/braintrust-api/build.gradle b/braintrust-api/build.gradle index 5d2083d6..3fdd1ae8 100644 --- a/braintrust-api/build.gradle +++ b/braintrust-api/build.gradle @@ -60,6 +60,7 @@ tasks.register('fetchOpenApiSpec', Exec) { // ── Code generation ────────────────────────────────────────────────────────── def generatedSourcesDir = layout.buildDirectory.dir('generated/openapi') +def normalizedSpecFile = openApiSpecDir.map { it.file('spec-normalized.yaml') } openApiGenerate { @@ -71,7 +72,7 @@ openApiGenerate { // at code-generation time). See src/generator/templates for the diffs from upstream. templateDir = "${projectDir}/src/generator/templates/Java" - inputSpec = openApiSpecFile.map { it.asFile.absolutePath } + inputSpec = normalizedSpecFile.map { it.asFile.absolutePath } outputDir = generatedSourcesDir.map { it.asFile.absolutePath } @@ -104,10 +105,6 @@ openApiGenerate { 'code' : 'FunctionDataCode', 'remote_eval' : 'FunctionDataRemoteEval', 'parameters' : 'FunctionDataParameters', - 'function' : 'InlineFunctionRef', - 'function_1' : 'InlineFunction1', - 'function_2' : 'InlineFunction2', - 'function_3' : 'InlineFunction3', 'experiment' : 'CodeBundleLocationExperiment', 'user' : 'ChatMessageUser', ] @@ -125,6 +122,182 @@ openApiGenerate { } +// ── Spec normalization ─────────────────────────────────────────────────────── +// +// The spec repeatedly expresses "a SavedFunctionId, narrowed" as +// +// allOf: +// - $ref: "#/components/schemas/SavedFunctionId" +// - anyOf: [ {title: function, ...}, {title: global, ...} ] +// +// The referenced schema already declares those same branches (plus null), so the inline +// anyOf is redundant. The generator can't represent allOf-over-a-composition, so it flattens +// both sides into one POJO, merging the branches: the surviving `type` enum keeps only the +// last branch's values and the other branch's properties are dropped. The result is a class +// that cannot deserialize real payloads -- e.g. TopicMapFunctionAutomationFunction loses both +// "function" from its type enum and the `id` property, so a topic automation fails to parse. +// Because ProjectAutomationConfig is a oneOf, that single failure throws for the whole +// response, breaking any listing page containing a topic automation. +// +// This collapses the construct back to the plain $ref, which generates the proper anyOf +// wrapper class. It only fires when every inline branch title is already declared by the +// referenced schema, so a genuinely-narrowing allOf is left alone. + +tasks.register('normalizeOpenApiSpec') { + description = 'Collapses redundant allOf-over-$ref-plus-composition nodes before generation.' + group = 'Build' + + def rawSpec = openApiSpecFile + def outSpec = normalizedSpecFile + inputs.file(rawSpec) + outputs.file(outSpec) + + doLast { + def yaml = new org.yaml.snakeyaml.Yaml() + def spec = yaml.load(rawSpec.get().asFile.text) + + // The branches of a component schema's own anyOf/oneOf, or null if it isn't a composition. + def declaredBranches = { String refName -> + def target = spec?.components?.schemas?.get(refName) + return (target?.anyOf ?: target?.oneOf) + } + + // A branch that only widens nullability, e.g. the bare `type: null` arm of + // SavedFunctionId. Collapsing past one of these is acceptable; collapsing past a + // substantive branch is not, because it would silently broaden the accepted shapes. + def isNullishBranch = { branch -> + branch instanceof Map && branch.get('type') == 'null' && + !branch.containsKey('properties') && !branch.containsKey('$ref') + } + + def collapsed = [] + def untitled = [] + + // Component schemas that are plain POJOs — the ones an inline title can clobber. + def componentPojoNames = (spec?.components?.schemas?.findAll { name, schema -> + schema?.containsKey('properties') + }?.keySet() ?: []) as Set + + def toPascalCase = { String s -> + s.split('[_\\-\\s]+').collect { it.capitalize() }.join('') + } + + // Collect the distinct shapes behind each inline title. A title backing one shape can be + // redirected with an inlineSchemaNameMappings entry, even if it appears several times -- + // collapsing identical shapes onto one class is harmless. A title backing several + // different shapes cannot: mappings are keyed by title, so a single entry collapses them + // all and every shape but one comes out wrong. + def shapesByTitle = [:].withDefault { [] as Set } + def collectShapes + collectShapes = { node, path -> + if (node instanceof Map) { + def title = node.get('title') + def isTopLevelComponent = path ==~ /components\.schemas\.[^.]+/ + if (title instanceof String && !title.isEmpty() && !isTopLevelComponent) { + def props = node.get('properties') + def shape = props instanceof Map + ? (props.keySet() as List).sort().join(',') + : "type:${node.get('type')}" + shapesByTitle[title] = shapesByTitle[title] + shape + } + node.each { k, v -> collectShapes(v, path ? "${path}.${k}" : k as String) } + } else if (node instanceof List) { + node.eachWithIndex { v, i -> collectShapes(v, "${path}[${i}]") } + } + } + collectShapes(spec, '') + + def visit + visit = { node, path -> + if (node instanceof Map) { + def allOf = node.get('allOf') + if (allOf instanceof List && allOf.size() == 2) { + def refPart = allOf.find { it instanceof Map && it.containsKey('$ref') } + def compositionPart = allOf.find { + it instanceof Map && (it.containsKey('anyOf') || it.containsKey('oneOf')) + } + if (refPart != null && compositionPart != null) { + def refName = (refPart['$ref'] as String).split('/').last() + def branches = declaredBranches(refName) + def declared = (branches ?: []) + .collect { it instanceof Map ? it.title : null } + .findAll { it != null } as Set + def inline = (compositionPart.anyOf ?: compositionPart.oneOf) + .collect { it instanceof Map ? it.title : null } + .findAll { it != null } as Set + + // Collapse only when the referenced schema covers every inline branch and + // adds nothing substantive of its own. Requiring the two branch sets to + // match exactly would be wrong: the referenced schema legitimately carries + // an extra `type: null` arm, so an exact-match rule would never fire and + // the flattening bug this task exists to fix would come back. + def extraBranches = (branches ?: []).findAll { branch -> + !(branch instanceof Map && branch.title != null + && inline.contains(branch.title)) + } + if (branches != null && !inline.isEmpty() && declared.containsAll(inline) + && extraBranches.every { isNullishBranch(it) }) { + node.remove('allOf') + node.put('$ref', refPart['$ref']) + collapsed << "${path} -> ${refName}" + } + } + } + node.each { k, v -> visit(v, path ? "${path}.${k}" : k as String) } + } else if (node instanceof List) { + node.eachWithIndex { v, i -> visit(v, "${path}[${i}]") } + } + } + visit(spec, '') + + // Second pass, deliberately after the allOf collapse above: that comparison reads the + // branch titles of the referenced schema, so stripping titles first would silently + // disable it. + // + // Drop ambiguous inline titles that would clobber a component POJO. Titles become class + // names, so an inline schema titled `function` overwrites the real `Function` component. + // A name mapping cannot fix it when the title is reused: mappings are keyed by title, so + // one entry collapses every schema sharing it onto a single class and all but one end up + // with the wrong shape. Removing the title lets the generator name each schema from its + // path, which is unique by construction, leaving the component POJO intact. + def stripAmbiguousTitles + stripAmbiguousTitles = { node, path -> + if (node instanceof Map) { + def title = node.get('title') + def isTopLevelComponent = path ==~ /components\.schemas\.[^.]+/ + if (title instanceof String && !title.isEmpty() && !isTopLevelComponent + && shapesByTitle[title].size() > 1 + && componentPojoNames.contains(toPascalCase(title))) { + node.remove('title') + untitled << "${path} (was '${title}')" + } + node.each { k, v -> stripAmbiguousTitles(v, path ? "${path}.${k}" : k as String) } + } else if (node instanceof List) { + node.eachWithIndex { v, i -> stripAmbiguousTitles(v, "${path}[${i}]") } + } + } + stripAmbiguousTitles(spec, '') + + def dumperOptions = new org.yaml.snakeyaml.DumperOptions() + dumperOptions.defaultFlowStyle = org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK + dumperOptions.splitLines = false + def out = outSpec.get().asFile + out.parentFile.mkdirs() + out.text = new org.yaml.snakeyaml.Yaml(dumperOptions).dump(spec) + + if (collapsed.isEmpty()) { + logger.lifecycle('normalizeOpenApiSpec: no redundant allOf nodes found') + } else { + logger.lifecycle("normalizeOpenApiSpec: collapsed ${collapsed.size()} redundant allOf node(s)") + collapsed.each { logger.info(" ${it}") } + } + if (!untitled.isEmpty()) { + logger.lifecycle("normalizeOpenApiSpec: dropped ${untitled.size()} ambiguous inline title(s)") + untitled.each { logger.info(" ${it}") } + } + } +} + // ── Clobber detection ──────────────────────────────────────────────────────── // // The generator names inline anyOf/oneOf variant classes after their `title` @@ -138,7 +311,8 @@ tasks.register('checkClobberedSchemas') { description = 'Fails if any inline schema title would clobber a component schema name.' group = 'Verification' - def specFile = openApiSpecFile + // Check the normalized spec, since that's what generation consumes. + def specFile = normalizedSpecFile // Capture the current mappings at configuration time so they're available at execution time. // inlineSchemaNameMappings is a Gradle MapProperty so .get() is needed to resolve it. def mappedTitles = openApiGenerate.inlineSchemaNameMappings.get().keySet() as Set @@ -214,11 +388,16 @@ tasks.register('checkClobberedSchemas') { // checkClobberedSchemas runs after fetch (needs the spec) but before generation // so it fails fast without wasting time on codegen. tasks.named('checkClobberedSchemas') { + dependsOn tasks.named('normalizeOpenApiSpec') +} + +tasks.named('normalizeOpenApiSpec') { dependsOn tasks.named('fetchOpenApiSpec') } tasks.named('openApiGenerate') { dependsOn tasks.named('checkClobberedSchemas') + dependsOn tasks.named('normalizeOpenApiSpec') } tasks.named('compileJava') { @@ -251,3 +430,7 @@ dependencies { testImplementation "org.junit.jupiter:junit-jupiter-api:${rootProject.ext.junitVersion}" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${rootProject.ext.junitVersion}" } + +test { + useJUnitPlatform() +} diff --git a/braintrust-api/src/generator/templates/Java/libraries/native/api.mustache b/braintrust-api/src/generator/templates/Java/libraries/native/api.mustache index 6d89b929..a34458a8 100644 --- a/braintrust-api/src/generator/templates/Java/libraries/native/api.mustache +++ b/braintrust-api/src/generator/templates/Java/libraries/native/api.mustache @@ -419,10 +419,15 @@ public class {{classname}} { {{/hasVars}} {{^hasVars}} {{#isModel}} + // anyOf/oneOf-typed parameter: unwrap the matched variant and serialize it as a normal + // form-style parameter. Calling toUrlQueryString() here instead drops the parameter, since + // that method has no way to know the parameter is named "{{baseName}}". if ({{paramName}} != null) { - String queryString = {{paramName}}.toUrlQueryString(); - if (queryString != null && !queryString.isBlank()) { - localVarQueryStringJoiner.add(queryString); + Object {{paramName}}QueryValue = {{paramName}}.getActualInstance(); + if ({{paramName}}QueryValue instanceof java.util.Collection {{paramName}}QueryValues) { + localVarQueryParams.addAll(ApiClient.parameterToPairs("multi", "{{baseName}}", {{paramName}}QueryValues)); + } else { + localVarQueryParams.addAll(ApiClient.parameterToPairs("{{baseName}}", {{paramName}}QueryValue)); } } {{/isModel}} diff --git a/braintrust-api/src/generator/templates/PATCHES.md b/braintrust-api/src/generator/templates/PATCHES.md index 2c3c4a8b..265a79d2 100644 --- a/braintrust-api/src/generator/templates/PATCHES.md +++ b/braintrust-api/src/generator/templates/PATCHES.md @@ -46,15 +46,42 @@ Renamed typed getter methods from `getanyOf0Instance()` / `getanyOf1Instance()` the same identifier as the `SchemaType` enum constant (e.g. `getSystemInstance()`, `getWeightedInstance()`), keeping the instance accessor API consistent with the enum. +### Note — `toUrlQueryString` is intentionally left unimplemented for anyOf models +The `toUrlQueryString(String prefix)` body iterates `{{#composedSchemas.oneOf}}`, which is always +empty in an *anyOf* model, so the method falls through to `return null`. This looks like a one-line +fix (swap the tag to `composedSchemas.anyOf`) but is not: + +- anyOf variants have no `baseName`, so the generator emits synthetic `any_of_0` / `any_of_1` + literals — the output would be `any_of_0=` rather than `ids=`. +- Container variants render as `getActualInstance() instanceof List`, which is not legal + Java (the same generics problem Patch 1 above fixes in the deserializer). + +Query parameters are serialized in `api.mustache` instead (see its Patch 1), so nothing calls this +method from the api layer. Leaving it returning `null` is preferable to emitting either garbage +parameter names or code that does not compile. + --- ## api.mustache **Upstream:** `Java/libraries/native/api.mustache` @ v7.14.0 -### Patch 1 — Null-guard for anyOf model query parameters -The upstream template calls `{{paramName}}.toUrlQueryString()` unconditionally for -`isModel` params in the `isExplode/!hasVars` branch. When the parameter is `null` -(e.g. the `ids` parameter on list endpoints), this throws a `NullPointerException`. -Fixed by wrapping the call in `if ({{paramName}} != null)` and checking the result is -non-blank before adding it to the query string joiner. +### Patch 1 — anyOf/oneOf model query parameters are serialized via `parameterToPairs` +In the `isExplode`/`!hasVars`/`isModel` branch, the upstream template serializes the parameter +with `{{paramName}}.toUrlQueryString()`. Two problems: + +1. It calls the method unconditionally, so a `null` parameter (e.g. `ids` on any list endpoint) + throws a `NullPointerException`. +2. More seriously, `toUrlQueryString()` takes no argument, so the wrapper has no way to know the + parameter is named `ids` — and for anyOf models the method returns `null` outright (see the + note under `anyof_model.mustache` below). The generated code then skipped the blank result and + sent the request **unfiltered**, so callers got a successful response over the wrong result + set rather than an error. + +Fixed by null-guarding, unwrapping the matched variant with `getActualInstance()`, and handing it +to the existing `ApiClient.parameterToPairs` helpers — `("multi", name, collection)` for list +variants (repeated `ids=a&ids=b`, which is what the spec documents) and `(name, value)` for +scalars. This reuses the same pair machinery as every other query parameter instead of the +composed-model serialization path. + +Covered by `QueryParameterSerializationTest`, which asserts against the built request URI. diff --git a/braintrust-api/src/test/java/dev/braintrust/openapi/api/QueryParameterSerializationTest.java b/braintrust-api/src/test/java/dev/braintrust/openapi/api/QueryParameterSerializationTest.java new file mode 100644 index 00000000..ee6e7cff --- /dev/null +++ b/braintrust-api/src/test/java/dev/braintrust/openapi/api/QueryParameterSerializationTest.java @@ -0,0 +1,104 @@ +package dev.braintrust.openapi.api; + +import static org.junit.jupiter.api.Assertions.*; + +import dev.braintrust.openapi.ApiClient; +import dev.braintrust.openapi.model.Ids; +import dev.braintrust.openapi.model.UserEmail; +import java.net.URI; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +/** + * Verifies that filters typed as anyOf wrappers reach the wire. + * + *

These assert against the request URI rather than the wrapper's {@code toUrlQueryString} in + * isolation, because the request is what callers actually depend on: a filter that fails to + * serialize produces a successful response over an unfiltered result set, which is worse than an + * error. + */ +public class QueryParameterSerializationTest { + + private static final UUID ID_A = UUID.fromString("11111111-2222-3333-4444-555555555555"); + private static final UUID ID_B = UUID.fromString("66666666-7777-8888-9999-000000000000"); + + /** + * Captures the URI of the request an api call would send, without sending it. The generated + * request interceptor runs before dispatch, so the connection failure afterwards is irrelevant. + */ + private static String capturedUri(java.util.function.Consumer call) { + var captured = new AtomicReference(); + var client = new ApiClient(); + client.updateBaseUri("http://localhost:1"); + client.setRequestInterceptor(builder -> captured.set(builder.build().uri().toString())); + try { + call.accept(client); + } catch (RuntimeException expected) { + // The request never completes; we only care about the URI it was built with. + } + assertNotNull(captured.get(), "no request was built"); + return captured.get(); + } + + private static String query(String uri) { + return URI.create(uri).getQuery(); + } + + @Test + void idsFilter_singleValue_isSentAsQueryParameter() { + var uri = + capturedUri( + client -> + new ProjectAutomationsApi(client) + .getProjectAutomation( + 5, + null, + null, + new Ids(Ids.SchemaType.UUID, ID_A), + null, + null)); + + assertEquals("limit=5&ids=" + ID_A, query(uri)); + } + + /** The spec documents repeating the parameter to pass a list of ids. */ + @Test + void idsFilter_multipleValues_isSentAsRepeatedQueryParameter() { + var uri = + capturedUri( + client -> + new ProjectAutomationsApi(client) + .getProjectAutomation( + null, + null, + null, + new Ids(Ids.SchemaType.List, List.of(ID_A, ID_B)), + null, + null)); + + assertEquals("ids=" + ID_A + "&ids=" + ID_B, query(uri)); + } + + /** The same wrapper machinery backs the scalar string filters on other endpoints. */ + @Test + void emailFilter_isSentAsQueryParameter() { + var uri = + capturedUri( + client -> + new UsersApi(client) + .getUser( + null, + null, + null, + null, + null, + null, + new UserEmail( + UserEmail.SchemaType.String, "a@b.com"), + null)); + + assertEquals("email=a@b.com", query(uri)); + } +} diff --git a/braintrust-api/src/test/java/dev/braintrust/openapi/model/ChatCompletionMessageParamTest.java b/braintrust-api/src/test/java/dev/braintrust/openapi/model/ChatCompletionMessageParamTest.java new file mode 100644 index 00000000..f93759cb --- /dev/null +++ b/braintrust-api/src/test/java/dev/braintrust/openapi/model/ChatCompletionMessageParamTest.java @@ -0,0 +1,60 @@ +package dev.braintrust.openapi.model; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.openapi.JSON; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Verifies that every chat message role survives a read. + * + *

Prompts fetched through {@code PromptsApi} carry these messages, so a role that fails to round + * trip silently changes the prompt a caller renders. + */ +public class ChatCompletionMessageParamTest { + + private static ObjectMapper mapper; + + @BeforeAll + static void setUp() { + mapper = new JSON().getMapper(); + } + + private static void assertRoundTrips(String json) throws Exception { + var parsed = mapper.readValue(json, ChatCompletionMessageParam.class); + assertEquals(mapper.readTree(json), mapper.readTree(mapper.writeValueAsString(parsed))); + } + + @Test + void functionRole_preservesNameAndContent() throws Exception { + assertRoundTrips("{\"role\":\"function\",\"name\":\"get_weather\",\"content\":\"sunny\"}"); + } + + @Test + void systemRole_preservesContent() throws Exception { + assertRoundTrips("{\"role\":\"system\",\"content\":\"be nice\"}"); + } + + @Test + void userRole_preservesStringContent() throws Exception { + assertRoundTrips("{\"role\":\"user\",\"content\":\"hello\"}"); + } + + @Test + void userRole_preservesContentParts() throws Exception { + assertRoundTrips( + "{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"hello\"}]}"); + } + + @Test + void assistantRole_preservesContent() throws Exception { + assertRoundTrips("{\"role\":\"assistant\",\"content\":\"hi there\"}"); + } + + @Test + void toolRole_preservesToolCallId() throws Exception { + assertRoundTrips("{\"role\":\"tool\",\"tool_call_id\":\"call_1\",\"content\":\"result\"}"); + } +} diff --git a/braintrust-api/src/test/java/dev/braintrust/openapi/model/ProjectAutomationConfigTest.java b/braintrust-api/src/test/java/dev/braintrust/openapi/model/ProjectAutomationConfigTest.java new file mode 100644 index 00000000..727e9ba5 --- /dev/null +++ b/braintrust-api/src/test/java/dev/braintrust/openapi/model/ProjectAutomationConfigTest.java @@ -0,0 +1,184 @@ +package dev.braintrust.openapi.model; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.openapi.JSON; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Deserialization tests for {@link ProjectAutomationConfig}, the oneOf covering the automation + * kinds returned by {@code GET /v1/project_automation}. + * + *

A oneOf that fails to match a variant throws, so a single unmatched automation anywhere in a + * listing page breaks the whole call with {@code 0 classes match result, expected 1}. + */ +public class ProjectAutomationConfigTest { + + private static ObjectMapper mapper; + + @BeforeAll + static void setUp() { + mapper = new JSON().getMapper(); + } + + /** A {@code logs} automation, the {@link ProjectAutomationConfigOneOf} variant. */ + private static final String LOGS_JSON = + """ + { + "event_type": "logs", + "btql_filter": "scores.Factuality < 0.5", + "interval_seconds": 300, + "action": { "type": "webhook", "url": "https://example.com/hook" } + } + """; + + /** A {@code topic} automation, the {@link TopicAutomationConfig} variant. */ + private static final String TOPIC_JSON = + """ + { + "event_type": "topic", + "sampling_rate": 1, + "facet_functions": [ + { "type": "global", "name": "Task", "function_type": "facet" }, + { "type": "global", "name": "Sentiment", "function_type": "facet" }, + { "type": "global", "name": "Issues", "function_type": "facet" } + ], + "topic_map_functions": [ + { "function": { "type": "function", "id": "36363b1a-b126-41da-9d28-a91a648b0b4c" } }, + { "function": { "type": "function", "id": "19fe31d1-3b15-4a78-9711-a79d2a40c0df" } }, + { "function": { "type": "function", "id": "dfe599b5-9e36-479a-9d37-42efe516fa84" } } + ], + "scope": { "type": "trace", "idle_seconds": 600 }, + "rerun_seconds": 86400, + "relabel_overlap_seconds": 3600, + "backfill_time_range": "86400s" + } + """; + + /** + * A {@code windowed} automation, the {@link WindowedAutomationConfig} variant. This is the type + * behind scheduled Loop jobs; it was absent from the spec entirely until the pinned ref was + * moved forward, and its absence broke every automation listing page. + */ + private static final String WINDOWED_JSON = + """ + { + "event_type": "windowed", + "status": "active", + "window": { + "window_seconds": 86400, + "schedule": { + "type": "cron", + "cron_expression": "0 9 * * 1", + "timezone": "America/Los_Angeles" + }, + "evaluation_delay_seconds": 0 + }, + "loop": { + "prompt": "say hi to my friends", + "include_trigger_input": false, + "agent_slug": "loop-chat", + "auto_approve_tools": [], + "harness": "codex", + "model": "gpt-5.6-sol" + }, + "actions": [] + } + """; + + @Test + void deserializes_logsVariant() throws Exception { + var config = mapper.readValue(LOGS_JSON, ProjectAutomationConfig.class); + + var logs = assertInstanceOf(ProjectAutomationConfigOneOf.class, config.getActualInstance()); + assertEquals(ProjectAutomationConfigOneOf.EventTypeEnum.LOGS, logs.getEventType()); + assertEquals("scores.Factuality < 0.5", logs.getBtqlFilter()); + } + + /** Pinpoints the failing layer: the variant itself, before any oneOf matching. */ + @Test + void deserializes_topicVariantDirectly() throws Exception { + var topic = mapper.readValue(TOPIC_JSON, TopicAutomationConfig.class); + assertEquals(TopicAutomationConfig.EventTypeEnum.TOPIC, topic.getEventType()); + assertEquals(3, topic.getTopicMapFunctions().size()); + + assertNotNull(topic.getTopicMapFunctions().get(0).getFunction()); + } + + /** + * A topic map function's saved-function reference keeps the id that identifies it. + * + *

Asserted through a round trip of the whole config, since that is what a caller reading an + * automation gets back. + */ + @Test + void topicVariant_preservesTopicMapFunctionIds() throws Exception { + var config = mapper.readValue(TOPIC_JSON, ProjectAutomationConfig.class); + + assertEquals( + mapper.readTree(TOPIC_JSON), mapper.readTree(mapper.writeValueAsString(config))); + } + + @Test + void deserializes_topicVariant() throws Exception { + var config = mapper.readValue(TOPIC_JSON, ProjectAutomationConfig.class); + + var topic = assertInstanceOf(TopicAutomationConfig.class, config.getActualInstance()); + assertEquals(TopicAutomationConfig.EventTypeEnum.TOPIC, topic.getEventType()); + assertEquals(3, topic.getFacetFunctions().size()); + assertEquals(3, topic.getTopicMapFunctions().size()); + } + + @Test + void deserializes_windowedVariant() throws Exception { + var config = mapper.readValue(WINDOWED_JSON, ProjectAutomationConfig.class); + + var windowed = assertInstanceOf(WindowedAutomationConfig.class, config.getActualInstance()); + assertEquals(WindowedAutomationConfig.EventTypeEnum.WINDOWED, windowed.getEventType()); + assertEquals("say hi to my friends", windowed.getLoop().getPrompt()); + } + + /** The whole point: one unmatched automation must not break a page of good ones. */ + @Test + void deserializes_mixedListingPage() throws Exception { + var page = + mapper.readValue( + """ + { "objects": [ + { "id": "11111111-1111-1111-1111-111111111111", + "project_id": "22222222-2222-2222-2222-222222222222", + "user_id": "33333333-3333-3333-3333-333333333333", + "created": "2026-08-19T01:57:15.013Z", + "name": "logs-alert", + "config": %s }, + { "id": "44444444-4444-4444-4444-444444444444", + "project_id": "22222222-2222-2222-2222-222222222222", + "user_id": "33333333-3333-3333-3333-333333333333", + "created": "2026-08-19T01:57:15.013Z", + "name": "topic-discovery", + "config": %s }, + { "id": "55555555-5555-5555-5555-555555555555", + "project_id": "22222222-2222-2222-2222-222222222222", + "user_id": "33333333-3333-3333-3333-333333333333", + "created": "2026-08-19T01:57:15.013Z", + "name": "pattern-discovery", + "config": %s } + ] } + """ + .formatted(LOGS_JSON, TOPIC_JSON, WINDOWED_JSON), + GetProjectAutomation200Response.class); + + assertEquals(3, page.getObjects().size()); + assertInstanceOf( + ProjectAutomationConfigOneOf.class, + page.getObjects().get(0).getConfig().getActualInstance()); + assertInstanceOf( + TopicAutomationConfig.class, + page.getObjects().get(1).getConfig().getActualInstance()); + assertInstanceOf( + WindowedAutomationConfig.class, + page.getObjects().get(2).getConfig().getActualInstance()); + } +} diff --git a/braintrust-api/src/test/java/dev/braintrust/openapi/model/SavedFunctionIdTest.java b/braintrust-api/src/test/java/dev/braintrust/openapi/model/SavedFunctionIdTest.java new file mode 100644 index 00000000..fe8f6dcf --- /dev/null +++ b/braintrust-api/src/test/java/dev/braintrust/openapi/model/SavedFunctionIdTest.java @@ -0,0 +1,81 @@ +package dev.braintrust.openapi.model; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.openapi.JSON; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Verifies that saved function references survive a read. + * + *

A reference identifies a function either by id ({@code {type: function, id, version}}) or by + * global name ({@code {type: global, name, function_type}}). Assertions are written against a JSON + * round trip rather than typed getters, so they describe the payload contract rather than the shape + * of whichever generated class happens to back a variant. + * + *

These references appear in topic automations ({@code topic_map_functions}, {@code + * facet_functions}), online scoring rules ({@code scorers}), and prompt {@code tool_functions}. + */ +public class SavedFunctionIdTest { + + private static ObjectMapper mapper; + + @BeforeAll + static void setUp() { + mapper = new JSON().getMapper(); + } + + private static void assertRoundTrips(String json, Class type) throws Exception { + var parsed = mapper.readValue(json, type); + assertEquals(mapper.readTree(json), mapper.readTree(mapper.writeValueAsString(parsed))); + } + + @Test + void functionReference_preservesIdAndVersion() throws Exception { + assertRoundTrips( + "{\"type\":\"function\",\"id\":\"36363b1a-b126-41da-9d28-a91a648b0b4c\",\"version\":\"v1\"}", + SavedFunctionId.class); + } + + @Test + void functionReference_preservesIdWhenVersionOmitted() throws Exception { + assertRoundTrips( + "{\"type\":\"function\",\"id\":\"36363b1a-b126-41da-9d28-a91a648b0b4c\"}", + SavedFunctionId.class); + } + + @Test + void nullableFunctionReference_preservesId() throws Exception { + assertRoundTrips( + "{\"type\":\"function\",\"id\":\"36363b1a-b126-41da-9d28-a91a648b0b4c\"}", + NullableSavedFunctionId.class); + } + + @Test + void globalReference_preservesNameAndFunctionType() throws Exception { + assertRoundTrips( + "{\"type\":\"global\",\"name\":\"Task\",\"function_type\":\"scorer\"}", + SavedFunctionId.class); + } + + /** A code bundle location references a function positionally rather than by id. */ + @Test + void codeBundleLocation_preservesFunctionIndex() throws Exception { + assertRoundTrips("{\"type\":\"function\",\"index\":2}", CodeBundleLocation.class); + } + + /** An OpenAI tool choice names the function it selects. */ + @Test + void toolChoice_preservesSelectedFunctionName() throws Exception { + assertRoundTrips( + "{\"type\":\"function\",\"function\":{\"name\":\"get_weather\"}}", + OpenAIModelParamsToolChoice.class); + } + + @Test + void toolChoice_preservesStringShorthand() throws Exception { + assertRoundTrips("\"auto\"", OpenAIModelParamsToolChoice.class); + } +} diff --git a/braintrust-sdk/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java b/braintrust-sdk/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java index f28dbf67..06148e99 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java @@ -47,11 +47,15 @@ public List> renderMessages(Map parameters) List> renderedMessages = new ArrayList<>(); for (ChatCompletionMessageParam param : chat.getMessages()) { final String role; + // Optional on most roles, but required on function-role messages -- dropping it there + // renders a message the OpenAI API rejects. + String name = null; final String content = switch (param.getVariantType()) { case System -> { var sys = param.getSystemInstance(); role = "system"; + name = sys.getName(); yield sys.getContent() != null ? extractStringContent(sys.getContent().getActualInstance()) : null; @@ -59,6 +63,7 @@ public List> renderMessages(Map parameters) case ChatMessageUser -> { var user = param.getChatMessageUserInstance(); role = "user"; + name = user.getName(); yield user.getContent() != null ? extractStringContent(user.getContent().getActualInstance()) : null; @@ -66,6 +71,7 @@ public List> renderMessages(Map parameters) case Assistant -> { var asst = param.getAssistantInstance(); role = "assistant"; + name = asst.getName(); yield asst.getContent() != null ? extractStringContent(asst.getContent().getActualInstance()) : null; @@ -77,15 +83,18 @@ public List> renderMessages(Map parameters) ? extractStringContent(tool.getContent().getActualInstance()) : null; } - case InlineFunctionRef -> { - // function-role messages have an index reference, not text content - var fn = param.getInlineFunctionRefInstance(); + case ChatCompletionMessageParamAnyOf -> { + // function-role message; its content is a plain string rather than the + // anyOf wrapper the other roles use + var fn = param.getChatCompletionMessageParamAnyOfInstance(); role = "function"; - yield null; + name = fn.getName(); + yield fn.getContent(); } case Developer -> { var dev = param.getDeveloperInstance(); role = "developer"; + name = dev.getName(); yield dev.getContent() != null ? extractStringContent(dev.getContent().getActualInstance()) : null; @@ -99,6 +108,9 @@ public List> renderMessages(Map parameters) Map rendered = new HashMap<>(); rendered.put("role", role); + if (name != null) { + rendered.put("name", name); + } if (content != null) { rendered.put("content", renderTemplate(content, parameters)); } diff --git a/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java b/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java index c1451191..fc870432 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java @@ -117,6 +117,58 @@ void testRenderMessagesWithParameters() { assertEquals("What's up my friend? My name is Alice", rendered.get(1).get("content")); } + @Test + void testRenderFunctionMessagePreservesRequiredName() { + Map prompt = + Map.of( + "type", + "chat", + "messages", + List.of( + Map.of( + "role", + "function", + "name", + "lookup_weather", + "content", + "Forecast for {{city}}"))); + + BraintrustPrompt braintrustPrompt = + new BraintrustPrompt(promptData(prompt, Map.of("model", "gpt-4o-mini"))); + + assertEquals( + Map.of( + "role", + "function", + "name", + "lookup_weather", + "content", + "Forecast for Boise"), + braintrustPrompt.renderMessages(Map.of("city", "Boise")).get(0)); + } + + @Test + void testRenderMessagePreservesOptionalName() { + Map prompt = + Map.of( + "type", + "chat", + "messages", + List.of( + Map.of( + "role", + "user", + "name", + "example_user", + "content", + "Hello"))); + + BraintrustPrompt braintrustPrompt = + new BraintrustPrompt(promptData(prompt, Map.of("model", "gpt-4o-mini"))); + + assertEquals("example_user", braintrustPrompt.renderMessages(Map.of()).get(0).get("name")); + } + @Test void testRenderMessagesWithList() { Map prompt = diff --git a/examples/api-client/build.gradle b/examples/api-client/build.gradle index 7a5873a8..77f0eabe 100644 --- a/examples/api-client/build.gradle +++ b/examples/api-client/build.gradle @@ -3,13 +3,10 @@ application { } dependencies { - // The OpenAPI-generated client (dev.braintrust.openapi.*) is bundled into the published - // braintrust-sdk jar, so real consumers get it transitively. This example uses a Gradle - // project() dependency, which doesn't expose the embedded classes, so depend on the - // generated client subproject directly for compilation. implementation project(':braintrust-api') + implementation "com.fasterxml.jackson.core:jackson-databind:${rootProject.ext.jacksonVersion}" } run { - description = 'Read projects, experiments, prompts, and datasets via the low-level API client' + description = 'Read various api resources via the low-level API client. This also includes example for Create/Update/Delete, but these mutating operations are intentionally not invoked by default.' } diff --git a/examples/api-client/src/main/java/dev/braintrust/examples/ApiClientExample.java b/examples/api-client/src/main/java/dev/braintrust/examples/ApiClientExample.java index 941eefd5..3f765213 100644 --- a/examples/api-client/src/main/java/dev/braintrust/examples/ApiClientExample.java +++ b/examples/api-client/src/main/java/dev/braintrust/examples/ApiClientExample.java @@ -4,11 +4,24 @@ import dev.braintrust.config.BraintrustConfig; import dev.braintrust.openapi.api.DatasetsApi; import dev.braintrust.openapi.api.ExperimentsApi; +import dev.braintrust.openapi.api.ProjectAutomationsApi; import dev.braintrust.openapi.api.ProjectsApi; import dev.braintrust.openapi.api.PromptsApi; +import dev.braintrust.openapi.model.CreateProjectAutomation; import dev.braintrust.openapi.model.Dataset; import dev.braintrust.openapi.model.Experiment; +import dev.braintrust.openapi.model.PatchProjectAutomation; +import dev.braintrust.openapi.model.PatchProjectAutomationConfig; +import dev.braintrust.openapi.model.Project; +import dev.braintrust.openapi.model.ProjectAutomation; +import dev.braintrust.openapi.model.ProjectAutomationConfig; +import dev.braintrust.openapi.model.ProjectAutomationConfigOneOf; +import dev.braintrust.openapi.model.ProjectAutomationConfigOneOfAction; import dev.braintrust.openapi.model.Prompt; +import dev.braintrust.openapi.model.WindowedAutomationConfigActionsInnerOneOf; +import java.math.BigDecimal; +import java.util.List; +import java.util.UUID; /** * Demonstrates the low-level, OpenAPI-generated Braintrust API client for raw REST access beyond @@ -20,24 +33,30 @@ * 

  *   BRAINTRUST_API_KEY=sk-... ./gradlew :examples:api-client:run
  *
+ * + * NOTE: this example is safe to run. It only reads resources by default. Methods which mutate state + * are included as an example, but they are not invoked. */ public class ApiClientExample { // Cap each listing so the example prints a manageable amount. private static final int LIMIT = 5; + // Name of the automation this example creates and then deletes. + private static final String AUTOMATION_NAME = "java-example-low-factuality-alert"; + + // How far to scan when checking whether AUTOMATION_NAME is already taken. The server rejects + // the spec's project_automation_name filter, so this has to be done client-side. + private static final int AUTOMATION_SCAN_LIMIT = 100; + public static void main(String[] args) { // BraintrustOpenApiClient is an ApiClient with the base URL, bearer auth, and TLS // wired up from the config. Every *Api class takes it in its constructor. var client = BraintrustOpenApiClient.of(BraintrustConfig.fromEnvironment()); // Resolve the org name (login() is a Braintrust helper on top of the generated client) - // and grab the first project to read from. + // and pick the project to read from. var orgName = client.login().orgInfo().get(0).name(); - var project = - new ProjectsApi(client) - .getProject(1, null, null, null, null, null) - .getObjects() - .get(0); + var project = resolveProject(new ProjectsApi(client)); var projectId = project.getId(); System.out.println("Reading project " + project.getName() + " from org " + orgName); @@ -71,5 +90,144 @@ public static void main(String[] args) { for (Dataset d : datasetPage.getObjects()) { System.out.println(" " + d.getName() + " (" + d.getId() + ")"); } + + // ── Project automations ─────────────────────────────────────────────────── + // Automations are the alert and export rules attached to a project. + listAutomations(client); + // automationLifecycle(client, projectId); + } + + /** + * Picks the project to read from, preferring explicit configuration over whatever happens to be + * first in the org: {@code BRAINTRUST_DEFAULT_PROJECT_NAME}, then {@code BRAINTRUST_PROJECT}, + * then the org's first project. + * + *

Read straight from the environment rather than via {@link BraintrustConfig} because {@code + * defaultProjectName()} falls back to a built-in default, so it can't express "unset" and would + * never let the later options apply. + */ + private static Project resolveProject(ProjectsApi projects) { + for (var envVar : List.of("BRAINTRUST_DEFAULT_PROJECT_NAME", "BRAINTRUST_PROJECT")) { + var name = System.getenv(envVar); + if (name == null || name.isBlank()) { + continue; + } + // projectName is a server-side filter, so a match comes back as the only object. + var matches = projects.getProject(1, null, null, null, name, null).getObjects(); + if (matches.isEmpty()) { + throw new IllegalStateException( + "%s is set to \"%s\" but no project by that name exists" + .formatted(envVar, name)); + } + System.out.println("Selected project via " + envVar); + return matches.get(0); + } + + var firstProject = projects.getProject(1, null, null, null, null, null).getObjects(); + if (firstProject.isEmpty()) { + throw new IllegalStateException("this org has no projects to read from"); + } + return firstProject.get(0); + } + + /** + * Lists the org's project automations. + * + *

{@code config} is a oneOf, and a oneOf that matches no variant throws, so an automation + * kind missing from the pinned spec fails the whole page rather than just that row. Keep {@code + * braintrustOpenApiRef} current if a listing starts failing with {@code 0 classes match + * result}. See docs/api-client.md. + */ + private static void listAutomations(BraintrustOpenApiClient client) { + var automations = new ProjectAutomationsApi(client); + System.out.println("\nAutomations:"); + var page = automations.getProjectAutomation(LIMIT, null, null, null, null, null); + for (ProjectAutomation a : page.getObjects()) { + System.out.println(" " + a.getName() + " (" + a.getId() + ")"); + } + } + + /** + * Full create / read / update / delete round trip for a log-alert automation, which POSTs to a + * webhook whenever a low-scoring row lands. + * + *

Not called by default -- it mutates the project. Call it from {@code main} to try the + * write path; it deletes what it creates, so it leaves the project as it found it. + * + *

Note the {@code dev.braintrust.openapi.model} classes are imported by name rather than + * with a wildcard: that package contains a class named {@code System}, which shadows {@code + * java.lang.System} under {@code import ...model.*}. + */ + private static void automationLifecycle(BraintrustOpenApiClient client, UUID projectId) { + var automations = new ProjectAutomationsApi(client); + + // Config is a oneOf over the automation kinds. ProjectAutomationConfigOneOf is the + // "logs" variant; the OneOf1/2/3 siblings are btql_export, retention, and + // environment_update respectively. + // The webhook/slack action variants are shared with windowed automations, hence the + // WindowedAutomationConfigActionsInner* class names. + var action = + new ProjectAutomationConfigOneOfAction( + new WindowedAutomationConfigActionsInnerOneOf() + .type(WindowedAutomationConfigActionsInnerOneOf.TypeEnum.WEBHOOK) + .url("https://example.com/braintrust-hook")); + var logsConfig = + new ProjectAutomationConfigOneOf() + .eventType(ProjectAutomationConfigOneOf.EventTypeEnum.LOGS) + // Fire at most once every 5 minutes for matching rows. + .btqlFilter("scores.Factuality < 0.5") + .intervalSeconds(new BigDecimal(300)) + .action(action); + + // postProjectAutomation is create-or-return: if an automation with this name already + // exists it comes back unmodified, and this method would then update and delete something + // it did not create. Bail out rather than touching pre-existing state. + var existing = + automations + .getProjectAutomation(AUTOMATION_SCAN_LIMIT, null, null, null, null, null) + .getObjects() + .stream() + .filter(a -> AUTOMATION_NAME.equals(a.getName())) + .findFirst(); + if (existing.isPresent()) { + System.out.println( + "\nSkipping lifecycle: an automation named " + + AUTOMATION_NAME + + " already exists (" + + existing.get().getId() + + ")"); + return; + } + + ProjectAutomation created = + automations.postProjectAutomation( + new CreateProjectAutomation() + .projectId(projectId) + .name(AUTOMATION_NAME) + .description("created by examples/api-client") + .config(new ProjectAutomationConfig(logsConfig))); + System.out.println( + "\nCreated automation: " + created.getName() + " (" + created.getId() + ")"); + + // Read back by id. Prefer this over the list endpoint, which can throw -- see + // listAutomations above. + ProjectAutomation fetched = automations.getProjectAutomationId(created.getId()); + System.out.println("Fetched back: " + fetched.getDescription()); + + // Update: tighten the filter and relabel. + ProjectAutomation updated = + automations.patchProjectAutomationId( + created.getId(), + new PatchProjectAutomation() + .description("updated by examples/api-client") + .config( + new PatchProjectAutomationConfig( + logsConfig.btqlFilter( + "scores.Factuality < 0.25")))); + System.out.println("Updated: " + updated.getDescription()); + + // Delete, so the example leaves nothing behind. + automations.deleteProjectAutomationId(created.getId()); + System.out.println("Deleted automation " + created.getId()); } } diff --git a/gradle.properties b/gradle.properties index a0066932..a411a4f9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,7 +11,7 @@ org.gradle.warning.mode=summary braintrustSpecRef=v0.0.11 # braintrust-openapi commit SHA used by braintrust-api -braintrustOpenApiRef=64b79cb9122f50a74eac98ea86c3ec1858c0cdd1 +braintrustOpenApiRef=4481f2e10e5859c930abc844483354101d10a57b # Let Gradle locate local JDKs and download one if needed org.gradle.java.installations.auto-detect=true