diff --git a/docs/user-manual/modules/ROOT/pages/camel-catalog.adoc b/docs/user-manual/modules/ROOT/pages/camel-catalog.adoc index ce268d3ee666a..d97e2614ae783 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-catalog.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-catalog.adoc @@ -191,6 +191,10 @@ options a side names exist, that `unchanged` is only on the body after a step an The values are read from the code that sets the body, and checked against the body types the message history records when the routes run. +The YAML validator of the Camel CLI follows the body through the steps of a route with the body types, and reports a +step that cannot work on it, such as a Map that a `to: file` cannot write (see +xref:camel-jbang-validation.adoc#_what_a_running_integration_saw[Camel CLI validation]). + === Documentation in the Catalog The catalog includes the AsciiDoc documentation pages from all modules. This makes the full diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-validation.adoc b/docs/user-manual/modules/ROOT/pages/camel-jbang-validation.adoc index 5553b58ff3403..26c18f3af01cd 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-jbang-validation.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-validation.adoc @@ -75,8 +75,12 @@ A part of a Java route the parser cannot work out (a value computed at runtime) The YAML validator follows the message body through the steps of each route, and reports a step that cannot work on it: a field read such as `+${body.qty}+` on text that was never unmarshalled, or a Map that a `to: file` cannot write. -Where the file does not tell what the body is, such as after a `platform-http` or `kafka` consumer, or after a call -with `to: http`, it says nothing. +What a component or a data format does with the body comes from the +xref:camel-catalog.adoc#_body_types_in_the_catalog[body types in the catalog]: the file consumer gives the file it +read, the `http` producer leaves the response as bytes or a stream, and `unmarshal: json` makes a Map or a List. +Where they do not tell what the body is, the validator says nothing: after a `platform-http` consumer the body +depends on the request, and the deserializer of a `kafka` consumer can be set on the component, which the route file +does not show. When an integration runs the routes of the file with the message history on, such as with `camel run --dev`, the validator fills these gaps with what the integration saw at the endpoints of its routes: the body each consumer gave, diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/BodyTypeFlow.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/BodyTypeFlow.java index 10ac2e4adcc4c..613847f223027 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/BodyTypeFlow.java +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/BodyTypeFlow.java @@ -29,6 +29,7 @@ import static org.apache.camel.dsl.yaml.validator.RouteGraph.Route; import static org.apache.camel.dsl.yaml.validator.RouteGraph.endpointOf; +import static org.apache.camel.dsl.yaml.validator.RouteGraph.from; import static org.apache.camel.dsl.yaml.validator.RouteGraph.normalize; import static org.apache.camel.dsl.yaml.validator.RouteGraph.routes; import static org.apache.camel.dsl.yaml.validator.RouteGraph.scheme; @@ -63,21 +64,15 @@ final class BodyTypeFlow { /** The REST verbs that carry no body, so a route they send to starts with none. */ private static final Set VERBS_WITHOUT_BODY = Set.of("get", "delete", "head"); - /** The consumers that produce no body of their own, so the message reaching the route has none. */ - private static final Set NO_BODY_CONSUMER = Set.of("timer", "quartz", "scheduler", "cron"); - private BodyTypeFlow() { } - static void check(JsonNode target, NodePath path, List errors) { - check(target, path, errors, Set.of()); - } - /** - * @param known the endpoints the caller knows deliver no body, such as {@code direct:getStock} for the GET - * operation of an OpenAPI specification the route binds to (CAMEL-24844 phase B) + * @param known the endpoints the caller knows deliver no body, such as {@code direct:getStock} for the GET + * operation of an OpenAPI specification the route binds to (CAMEL-24844 phase B) + * @param bodyTypes the body types of the catalog, which say the consumers that give no body (timer, cron) */ - static void check(JsonNode target, NodePath path, List errors, Set known) { + static void check(JsonNode target, NodePath path, List errors, Set known, CatalogBodyTypes bodyTypes) { List routes = routes(target); if (routes.isEmpty()) { return; @@ -106,7 +101,7 @@ static void check(JsonNode target, NodePath path, List errors, Set())) { + if (!certainlyWithoutABody(r, callers, restless, new HashSet<>(), bodyTypes)) { continue; } errors.add(Error.builder() @@ -131,7 +126,7 @@ static void check(JsonNode target, NodePath path, List errors, Set> callers, Set restless, - Set seen) { + Set seen, CatalogBodyTypes bodyTypes) { if (!seen.add(route)) { return false; // a cycle: say nothing } @@ -139,7 +134,9 @@ private static boolean certainlyWithoutABody( if (scheme == null) { return false; } - if (NO_BODY_CONSUMER.contains(scheme)) { + // a consumer that produces no body of its own, as the catalog says: timer, cron, scheduler, quartz + CatalogBodyTypes.Answer out = bodyTypes.consumerOut(route.fromUri(), from(route.node())); + if (out != null && out.none()) { return true; } if (!"direct".equals(scheme) && !"seda".equals(scheme) && !"direct-vm".equals(scheme)) { @@ -155,7 +152,7 @@ private static boolean certainlyWithoutABody( if (setsTheBodyBeforeSendingTo(caller, route)) { return false; } - if (!certainlyWithoutABody(caller, callers, restless, seen)) { + if (!certainlyWithoutABody(caller, callers, restless, seen, bodyTypes)) { return false; } } diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/CatalogBodyTypes.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/CatalogBodyTypes.java new file mode 100644 index 0000000000000..3259582da25f3 --- /dev/null +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/CatalogBodyTypes.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.yaml.validator; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.camel.catalog.CamelCatalog; +import org.apache.camel.tooling.model.BaseOptionModel; +import org.apache.camel.tooling.model.BodyTypeModel; +import org.apache.camel.tooling.model.BodyTypeModel.OptionBodyTypeModel; +import org.apache.camel.tooling.model.ComponentModel; +import org.apache.camel.tooling.model.DataFormatModel; +import org.apache.camel.tooling.model.EipModel; + +import static org.apache.camel.dsl.yaml.validator.RouteGraph.scheme; + +/** + * The body types of the catalog (CAMEL-24961), as the walk of {@link TextBodyFlow} reads them: the body a consumer + * gives, the bodies a producer can send and the body it leaves, the body an unmarshal makes, and the bodies a consumer + * that answers its caller can reply with. + *

+ * A side holds for a step only when the step decides every option the side depends on. A component can be configured as + * well ({@code camel.component.kafka.valueDeserializer}), and the route file does not show that: an option of the + * component counts only when the endpoint sets it, and otherwise the side is not known for the step. A value that is + * only known when the route runs (a placeholder) makes it not known too. + */ +final class CatalogBodyTypes { + + /** The carriers of a payload: the text, the bytes, the stream or the file as read or written. */ + private static final Set PAYLOAD = Set.of("text", "bytes", "stream", "file"); + + /** The carriers of parsed data. */ + private static final Set DATA = Set.of("map", "list"); + + private final CamelCatalog catalog; + private final CamelCatalog fallback; + private final Map> components = new ConcurrentHashMap<>(); + private final Map> dataFormats = new ConcurrentHashMap<>(); + private volatile Map> byModelName; + + /** + * @param catalog the catalog of the validator + * @param fallback the catalog to read the body types of a model from when the other has none, as the catalog of an + * older Camel version does; null for none + */ + CatalogBodyTypes(CamelCatalog catalog, CamelCatalog fallback) { + this.catalog = catalog; + this.fallback = fallback; + } + + /** What a side says for a step: its carriers, and its Java type and format when it has them. */ + record Answer(List carriers, String javaType, String format) { + + /** A payload (text, bytes, a stream or a file), maybe no body as well, as an HTTP response without one. */ + boolean payload() { + return carriers.stream().anyMatch(PAYLOAD::contains) && onlyPayload(); + } + + /** Nothing but a payload or no body: no Map, no List, no object of a class. */ + boolean onlyPayload() { + return !carriers.isEmpty() && carriers.stream().allMatch(c -> PAYLOAD.contains(c) || "null".equals(c)); + } + + /** A Map or a List: an iterator is not one, as it is read once and has no index. */ + boolean data() { + return !carriers.isEmpty() && DATA.containsAll(carriers); + } + + /** No body: the consumer gives none (timer). */ + boolean none() { + return carriers.equals(List.of("null")); + } + + /** The body is left as it was. */ + boolean unchanged() { + return carriers.equals(List.of("unchanged")); + } + + /** The file as read. */ + boolean file() { + return carriers.contains("file"); + } + + /** The data format of the format, as an unmarshal or a marshal names it: json, jacksonXml, csv or yaml. */ + String dataFormat() { + if (format == null) { + return null; + } + return switch (format) { + case "json", "csv", "yaml" -> format; + case "xml" -> "jacksonXml"; + default -> null; + }; + } + + /** The carriers in words, without no body: text, bytes, a stream or a file. */ + String inWords() { + List words = new ArrayList<>(); + for (String carrier : carriers) { + switch (carrier) { + case "null" -> { + // a step that needs text takes no body too, and an HTTP response may have none + } + case "stream" -> words.add("a stream"); + case "file" -> words.add("a file"); + case "map" -> words.add("a Map"); + case "list" -> words.add("a List"); + case "iterator" -> words.add("an iterator"); + case "value" -> words.add("a value"); + case "pojo" -> words.add("an object"); + default -> words.add(carrier); + } + } + if (words.size() < 2) { + return String.join("", words); + } + return String.join(", ", words.subList(0, words.size() - 1)) + " or " + words.get(words.size() - 1); + } + } + + /** The body a consumer gives the route it starts. */ + Answer consumerOut(String uri, JsonNode from) { + return componentSide(uri, from, BodyTypeModel.CONSUMER_OUT); + } + + /** The bodies a consumer that answers its caller can reply with: what the route can end with. */ + Answer consumerReply(String uri, JsonNode from) { + return componentSide(uri, from, BodyTypeModel.CONSUMER_REPLY); + } + + /** The bodies a producer can send or write. */ + Answer producerIn(String uri, JsonNode to) { + return componentSide(uri, to, BodyTypeModel.PRODUCER_IN); + } + + /** The body after a producer. */ + Answer producerOut(String uri, JsonNode to) { + return componentSide(uri, to, BodyTypeModel.PRODUCER_OUT); + } + + /** Whether the component of an endpoint is an HTTP component, by its label: platform-http. */ + boolean http(String uri) { + ComponentModel model = component(scheme(uri)); + return model != null && model.getLabel() != null && Arrays.asList(model.getLabel().split(",")).contains("http"); + } + + /** + * The body after an unmarshal with a data format of the YAML DSL (json, csv...) and its options: the data format of + * the catalog is the one its library option names, such as jackson for json. + */ + Answer unmarshalOut(String key, JsonNode options) { + DataFormatModel model = dataFormat(key, options); + return model != null + ? answer(model.getBodyType(BodyTypeModel.UNMARSHAL_OUT), values(options), Set.of()) : null; + } + + private Answer componentSide(String uri, JsonNode node, String side) { + ComponentModel model = component(scheme(uri)); + if (model == null) { + return null; + } + Set configurable = new HashSet<>(); + for (BaseOptionModel option : model.getComponentOptions()) { + configurable.add(option.getName()); + } + Map set = values(node != null && node.isObject() ? node.get("parameters") : null); + String text = node != null && node.isTextual() ? node.asText() + : node != null && node.isObject() && node.has("uri") ? node.get("uri").asText() : null; + set.putAll(query(text)); + return answer(model.getBodyType(side), set, configurable); + } + + /** + * What a side says for the options a step sets: the body for an option value it names, or its body when the options + * it depends on keep their defaults; null when it is not known for the step. + */ + private static Answer answer(BodyTypeModel side, Map set, Set configurable) { + if (side == null) { + return null; + } + Set options = new LinkedHashSet<>(side.getDependsOn()); + side.getByOption().forEach(o -> options.add(o.getOption())); + for (String option : options) { + String value = set.get(option); + if (value == null ? configurable.contains(option) : value.contains("{{") || value.contains("${")) { + // the component may set it, or its value is only known when the route runs + return null; + } + } + Answer matched = null; + for (OptionBodyTypeModel when : side.getByOption()) { + String value = set.get(when.getOption()); + if (value != null && value.equalsIgnoreCase(when.getValue())) { + if (matched != null && !matched.carriers().equals(when.getCarriers())) { + // two options the step sets say different bodies + return null; + } + matched = new Answer(when.getCarriers(), when.getJavaType(), side.getFormat()); + } + } + if (matched != null) { + return matched; + } + if (side.isUnknown() || side.getDependsOn().stream().anyMatch(set::containsKey)) { + return null; + } + return new Answer(side.getCarriers(), side.getJavaType(), side.getFormat()); + } + + /** The options of a node, by name in camel case: allowNullBody as well for allow-null-body. */ + private static Map values(JsonNode node) { + Map answer = new HashMap<>(); + if (node != null && node.isObject()) { + for (var it = node.fieldNames(); it.hasNext();) { + String key = it.next(); + JsonNode value = node.get(key); + answer.put(camelCase(key), value.isValueNode() ? value.asText() : value.toString()); + } + } + return answer; + } + + /** The options in the query of an endpoint uri: file:inbox?noop=true. */ + private static Map query(String uri) { + Map answer = new HashMap<>(); + int q = uri != null ? uri.indexOf('?') : -1; + if (q < 0) { + return answer; + } + for (String pair : uri.substring(q + 1).split("&")) { + int eq = pair.indexOf('='); + if (eq > 0) { + answer.put(camelCase(pair.substring(0, eq)), pair.substring(eq + 1)); + } + } + return answer; + } + + private static String camelCase(String name) { + if (name.indexOf('-') < 0) { + return name; + } + StringBuilder sb = new StringBuilder(name.length()); + boolean upper = false; + for (char c : name.toCharArray()) { + if (c == '-') { + upper = true; + } else { + sb.append(upper ? Character.toUpperCase(c) : c); + upper = false; + } + } + return sb.toString(); + } + + private ComponentModel component(String scheme) { + if (scheme == null) { + return null; + } + return components.computeIfAbsent(scheme, s -> { + ComponentModel model = catalog.componentModel(s); + if ((model == null || model.getBodyTypes().isEmpty()) && fallback != null) { + ComponentModel other = fallback.componentModel(s); + if (other != null && !other.getBodyTypes().isEmpty()) { + model = other; + } + } + return Optional.ofNullable(model); + }).orElse(null); + } + + private DataFormatModel dataFormat(String key, JsonNode options) { + List names = dataFormatNames().getOrDefault(key, List.of()); + String name = names.size() == 1 ? names.get(0) : null; + if (names.size() > 1) { + // json is jackson, gson, fastjson or jsonb: the library option names it + JsonNode library = options != null && options.isObject() ? options.get("library") : null; + String wanted = library != null && library.isValueNode() ? library.asText() : defaultLibrary(key); + name = names.stream().filter(n -> n.equalsIgnoreCase(wanted)).findFirst().orElse(null); + } + if (name == null) { + return null; + } + return dataFormats.computeIfAbsent(name, n -> { + DataFormatModel model = catalog.dataFormatModel(n); + if ((model == null || model.getBodyTypes().isEmpty()) && fallback != null) { + DataFormatModel other = fallback.dataFormatModel(n); + if (other != null && !other.getBodyTypes().isEmpty()) { + model = other; + } + } + return Optional.ofNullable(model); + }).orElse(null); + } + + /** The default of the library option of a data format of the YAML DSL: Jackson for json. */ + private String defaultLibrary(String key) { + EipModel model = catalog.eipModel(key); + if (model != null) { + for (EipModel.EipOptionModel option : model.getOptions()) { + if ("library".equals(option.getName()) && option.getDefaultValue() != null) { + return option.getDefaultValue().toString(); + } + } + } + return null; + } + + /** The data formats of the catalog by the name the YAML DSL gives them: json to fastjson, gson, jackson, jsonb. */ + private Map> dataFormatNames() { + Map> answer = byModelName; + if (answer == null) { + answer = new HashMap<>(); + for (String name : catalog.findDataFormatNames()) { + DataFormatModel model = catalog.dataFormatModel(name); + if (model != null && model.getModelName() != null) { + answer.computeIfAbsent(model.getModelName(), k -> new ArrayList<>()).add(name); + } + } + byModelName = answer; + } + return answer; + } +} diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/RouteGraph.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/RouteGraph.java index 686d6d1e8a7a1..5e73988acea79 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/RouteGraph.java +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/RouteGraph.java @@ -41,6 +41,15 @@ private RouteGraph() { record Route(String id, String fromUri, JsonNode steps, JsonNode node) { } + /** The from of a route entry, in both the canonical and the short form; null for another entry. */ + static JsonNode from(JsonNode entry) { + if (entry == null || !entry.isObject()) { + return null; + } + JsonNode route = entry.get("route"); + return route != null ? route.get("from") : entry.get("from"); + } + /** The routes of the file, in both the canonical and the short form. */ static List routes(JsonNode target) { List answer = new ArrayList<>(); diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/TextBodyFlow.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/TextBodyFlow.java index 7805e50f33ed6..feb84871f8b09 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/TextBodyFlow.java +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/TextBodyFlow.java @@ -34,6 +34,7 @@ import org.apache.camel.util.MimeTypeHelper; import static org.apache.camel.dsl.yaml.validator.RouteGraph.endpointOf; +import static org.apache.camel.dsl.yaml.validator.RouteGraph.from; import static org.apache.camel.dsl.yaml.validator.RouteGraph.normalize; import static org.apache.camel.dsl.yaml.validator.RouteGraph.scheme; @@ -52,9 +53,13 @@ * at runtime, because no type converter turns a Map into the stream they write: "No type converter available to convert * from type: java.util.LinkedHashMap to the required type: java.io.InputStream". *

- * Where the walk does not know the body - after a consumer other than the file consumer, or after a to: it cannot - * follow - what a running integration of the routes saw there fills the gap, when all the bodies it saw were of one - * kind (CAMEL-25750): the stream cache of an HTTP request, the List a sql call returns. + * What a component or a data format does with the body comes from the body types of the catalog (CAMEL-24961): the body + * a consumer gives (the file as read), what a producer can write (text, bytes, a stream or a file), the body it leaves + * (log keeps it, the http producer gives the response as bytes or a stream), what an unmarshal makes (a Map or a List), + * and what a consumer can reply with. Where the walk does not know the body - after a consumer or a to: the catalog + * does not say, or after a to: it cannot follow - what a running integration of the routes saw there fills the gap, + * when all the bodies it saw were of one kind (CAMEL-25750): the stream cache of an HTTP request, the List a sql call + * returns. */ final class TextBodyFlow { @@ -92,9 +97,6 @@ final class TextBodyFlow { private static final Set SAME_BODY_BLOCKS = Set.of("choice", "when", "otherwise", "filter", "doTry", "doCatch", "doFinally"); - /** Endpoints that leave the body as it is. */ - private static final Set KEEPS_THE_BODY = Set.of("log", "mock"); - /** Steps that leave the body as it is. */ private static final Set KEEPING_STEPS = Set.of("setHeader", "setHeaders", "setProperty", "setVariable", "setVariables", "removeHeader", "removeHeaders", "removeProperty", "removeProperties", "removeVariable", @@ -106,12 +108,6 @@ final class TextBodyFlow { /** $.sku or $['sku']: a jsonpath that reads a field of the body, which is then an object. */ private static final Pattern JSONPATH_FIELD = Pattern.compile("^\\$\\s*(\\.\\s*[A-Za-z_]|\\[\\s*['\"])"); - /** The data formats whose unmarshal makes a Map or a List of the text. */ - private static final Set PARSES = Set.of("json", "jacksonXml", "yaml", "csv"); - - /** The unmarshal options that make something else of the text: an object of a class, or an iterator. */ - private static final Set NOT_A_MAP = Set.of("unmarshalType", "unmarshalTypeName", "lazyLoad"); - /** The Groovy collection methods whose result is data again: a List or a Map. */ private static final Set GIVES_DATA = Set.of("findAll", "collect", "collectEntries", "grep", "sort", "groupBy", "unique"); @@ -125,9 +121,6 @@ final class TextBodyFlow { /** The parameter of a closure: x -> ... */ private static final Pattern CLOSURE_PARAMETER = Pattern.compile("^\\s*([A-Za-z_]\\w*)\\s*->"); - /** The producers that write the body to a file. */ - private static final Set WRITES_A_FILE = Set.of("file", "ftp", "ftps", "sftp"); - /** * The consumers that bring no HTTP control headers, so an http producer after them picks POST for a body: after an * HTTP consumer, the CamelHttpMethod header of the request picks the method instead. @@ -185,21 +178,21 @@ Set readable() { /** * What a walk knows besides the body: the route, the routes of the file, where reports go (null for a route a step - * calls, which reports from its own walk), whether an http producer sends the body with POST, and the verbs of the + * calls, which reports from its own walk), whether an http producer sends the body with POST, the verbs of the * OpenAPI operations a rest-openapi call can be judged by (empty when a REST binding may write the body), with the - * fields their JSON requests need. + * fields their JSON requests need, and the body types of the catalog. */ private record Walk(String routeId, Map byFrom, List errors, Set seen, boolean post, Map> operations, Map>> requestFields, - Map observed) { + Map observed, CatalogBodyTypes bodyTypes) { /** The walk of a route a step calls: for its outcome alone. */ Walk called() { - return new Walk(null, byFrom, null, seen, false, operations, requestFields, observed); + return new Walk(null, byFrom, null, seen, false, operations, requestFields, observed, bodyTypes); } } - static void check(JsonNode target, NodePath path, List errors, ProjectFacts facts) { + static void check(JsonNode target, NodePath path, List errors, ProjectFacts facts, CatalogBodyTypes bodyTypes) { if (target == null || !target.isArray()) { return; } @@ -240,24 +233,26 @@ static void check(JsonNode target, NodePath path, List errors, ProjectFac String uri = endpointOf(from); String consumer = scheme(uri); boolean post = NO_HTTP_HEADERS.contains(consumer) && !setsHttpHeaders(steps); - Body start = consumerText(from); + Body start = uri != null ? consumerBody(uri, from, bodyTypes) : null; if (start == null && uri != null) { // what the consumer gave in a running integration of the route start = observed(observed, ProjectFacts.consumerKey(uri), "from: " + normalize(uri) + " gave"); } Body end = walk(steps, start, at, new Walk( id, byFrom, errors, new HashSet<>(), post, operations, - requestFields, observed)); - // the body at the end of a platform-http route, or of the route of a REST call, is the reply + requestFields, observed, bodyTypes)); + // the body at the end of the route of a consumer that replies with text (platform-http), or of the route of + // a REST call, is the reply boolean rest = uri != null && restReplies.contains(normalize(uri)); - if (("platform-http".equals(consumer) || rest) && end instanceof Parsed parsed && steps != null + CatalogBodyTypes.Answer reply = uri != null && !rest ? bodyTypes.consumerReply(uri, from) : null; + if ((rest || reply != null && reply.onlyPayload()) && end instanceof Parsed parsed && steps != null && steps.isArray() && !steps.isEmpty() && steps.get(steps.size() - 1).fieldNames().hasNext()) { int last = steps.size() - 1; + String caller = rest ? "REST call" : bodyTypes.http(uri) ? "HTTP request" : consumer + " request"; errors.add(error(at.append(last).append(steps.get(last).fieldNames().next()), id, new Need( - "the body at the end of the route is the reply to the " - + (rest ? "REST call" : "HTTP request"), - "as the last step", rest), + "the body at the end of the route is the reply to the " + caller, "as the last step", + rest), parsed)); } } @@ -272,20 +267,27 @@ private static boolean setsHttpHeaders(JsonNode steps) { return steps != null && steps.toString().toLowerCase(Locale.ROOT).contains("camelhttp"); } - private static JsonNode from(JsonNode entry) { - if (entry == null || !entry.isObject()) { + /** + * The body a consumer starts with, as the catalog says: the file as read (file, ftp), text or bytes (aws2-sqs, + * paho-mqtt5), or data (a row of sql); null when it is not known, or neither text nor data. + */ + private static Body consumerBody(String uri, JsonNode from, CatalogBodyTypes bodyTypes) { + CatalogBodyTypes.Answer out = bodyTypes.consumerOut(uri, from); + if (out == null) { return null; } - JsonNode route = entry.get("route"); - return route != null ? route.get("from") : entry.get("from"); + String consumer = "the " + scheme(uri) + " consumer gives it"; + if (out.payload()) { + // the file as read has the format of the files it reads + return out.file() + ? new Text(consumer + (out.carriers().size() > 1 ? " as " + out.inWords() : ""), fileFormat(from), true) + : new Text(consumer + " as " + out.inWords(), out.dataFormat()); + } + return out.data() ? new Parsed(consumer + " as " + out.inWords(), out.dataFormat()) : null; } - /** The body a consumer starts with: the file consumer reads a file, the text as it is; others are not known. */ - private static Text consumerText(JsonNode from) { - String uri = endpointOf(from); - if (!"file".equals(scheme(uri))) { - return null; - } + /** The format of the files a file consumer reads, by the extension it includes. */ + private static String fileFormat(JsonNode from) { JsonNode parameters = from.get("parameters"); String format = null; for (String key : new String[] { "include", "antInclude", "fileName" }) { @@ -294,7 +296,21 @@ private static Text consumerText(JsonNode from) { format = formatOf(parameters.get(key).asText().replace("\\", "")); } } - return new Text("the file consumer gives it", format, true); + return format; + } + + /** + * The body a producer leaves as the catalog says it: text (the response of the http producer) or data (the List of + * a mongodb findAll); null when it is not known, or neither text nor data. + */ + private static Body producerBody(CatalogBodyTypes.Answer out, String origin) { + if (out == null) { + return null; + } + if (out.payload()) { + return new Text(origin + " as " + out.inWords(), out.dataFormat(), out.file()); + } + return out.data() ? new Parsed(origin + " as " + out.inWords(), out.dataFormat()) : null; } /** @@ -329,6 +345,9 @@ private static Body walk(JsonNode steps, Body body, NodePath at, Walk w) { Parsed element = "split".equals(name) ? splitElement(value, parsed) : null; if (need != null) { w.errors().add(error(here, w.routeId(), need, parsed)); + // the step fails at runtime, so what comes after it is not run with this body: one report + body = null; + continue; } else if (element != null) { walk(value.get("steps"), element, here.append("steps"), w); } @@ -385,7 +404,7 @@ private static Body after(String name, JsonNode value, Body body, Walk w) { : null; } case "unmarshal" -> { - return unmarshalled(value); + return unmarshalled(value, w.bodyTypes()); } case "marshal" -> { String format = value != null && value.isObject() && value.fieldNames().hasNext() @@ -404,11 +423,15 @@ private static Body after(String name, JsonNode value, Body body, Walk w) { } case "to", "toD" -> { String uri = endpointOf(value); - String scheme = scheme(uri); - if (scheme != null && KEEPS_THE_BODY.contains(scheme)) { + CatalogBodyTypes.Answer out = uri != null ? w.bodyTypes().producerOut(uri, value) : null; + if (out != null && out.unchanged()) { + // log, a file write return body; } Body returned = calledRoute(name, uri, body, w); + if (returned == null) { + returned = producerBody(out, name + ": " + normalize(uri) + " returns it"); + } if (returned == null && uri != null) { // what the producer returned in a running integration of the route returned = observed(w.observed(), ProjectFacts.producerKey(uri), @@ -512,22 +535,18 @@ private static Text setBodyText(JsonNode value) { return null; } - /** The body after an unmarshal: parsed data for the data formats that make a Map or a List of the text. */ - private static Parsed unmarshalled(JsonNode value) { + /** + * The body after an unmarshal: parsed data for a data format that makes a Map or a List of the text, as the catalog + * says with the options of the step (not with unmarshalType, which makes an object of a class, or lazyLoad, an + * iterator). + */ + private static Parsed unmarshalled(JsonNode value, CatalogBodyTypes bodyTypes) { if (value == null || !value.isObject() || !value.fieldNames().hasNext()) { return null; } String format = value.fieldNames().next(); - if (!PARSES.contains(format)) { - return null; - } - JsonNode options = value.get(format); - for (String option : NOT_A_MAP) { - if (options != null && options.isObject() && options.has(option)) { - return null; - } - } - return new Parsed("unmarshal: " + format + " turns the text into data", format); + CatalogBodyTypes.Answer out = bodyTypes.unmarshalOut(format, value.get(format)); + return out != null && out.data() ? new Parsed("unmarshal: " + format + " turns the text into data", format) : null; } /** @@ -654,8 +673,10 @@ private record Operation(String spec, String id) { } /** - * A to: that needs the body as text: one that writes a file, an http producer that sends it with POST, or a - * rest-openapi call to an operation whose verb sends it (the producer calls with the verb of the specification). + * A to: that needs the body as text: a producer that the catalog says can only send or write text, bytes, a stream + * or a file (file, ftp), an http producer that sends it with POST, or a rest-openapi call to an operation whose + * verb sends it (the producer calls with the verb of the specification). The http method decides whether the http + * producer sends the body, and a REST binding may write it for rest-openapi: they keep their own rule. */ private static Need needsText(String name, JsonNode value, Walk w) { if (!"to".equals(name) && !"toD".equals(name)) { @@ -666,14 +687,6 @@ private static Need needsText(String name, JsonNode value, Walk w) { if (scheme == null) { return null; } - if (WRITES_A_FILE.contains(scheme)) { - return new Need(name + ": " + scheme + " writes the body to a file", "before this step"); - } - if (w.post() && ("http".equals(scheme) || "https".equals(scheme)) && sendsTheBody(uri, value)) { - return new Need( - name + ": " + scheme + " sends the body as the request (a message with a body is sent as a POST)", - "before this step, or set httpMethod: GET when the call sends no body"); - } if ("rest-openapi".equals(scheme)) { Operation operation = operationOf(uri, value); String verb = operation != null @@ -687,8 +700,21 @@ private static Need needsText(String name, JsonNode value, Walk w) { + verb.toUpperCase(Locale.ROOT) + " in " + operation.spec() + ")", "before this step", true, fields); } + return null; } - return null; + CatalogBodyTypes.Answer in = w.bodyTypes().producerIn(uri, value); + if (in == null || !in.onlyPayload()) { + return null; + } + if ("http".equals(scheme) || "https".equals(scheme)) { + return w.post() && sendsTheBody(uri, value) + ? new Need( + name + ": " + scheme + + " sends the body as the request (a message with a body is sent as a POST)", + "before this step, or set httpMethod: GET when the call sends no body") + : null; + } + return new Need(name + ": " + scheme + " needs the body as " + in.inWords(), "before this step"); } /** diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/YamlValidator.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/YamlValidator.java index d51ac3e72cb8a..576624c766dc3 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/YamlValidator.java +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/YamlValidator.java @@ -87,6 +87,9 @@ public class YamlValidator { private final boolean canonical; private final String schemaJson; private CamelCatalog catalog; + /** The catalog of another Camel version than the one of the validator (CAMEL-24711), if given. */ + private final boolean otherCatalog; + private CatalogBodyTypes bodyTypes; private Schema schema; /** * The schema for a Kamelet: the envelope, with its {@code spec.template} being one entry of the route schema. A @@ -121,6 +124,7 @@ public YamlValidator(boolean canonical, String schemaJson, CamelCatalog catalog) this.canonical = canonical; this.schemaJson = schemaJson; this.catalog = catalog; + this.otherCatalog = catalog != null; } public boolean isCanonical() { @@ -848,9 +852,10 @@ private List validate(JsonNode target, ProjectFacts facts) { checkSimpleSyntaxInScripts(camelOnly, new NodePath(PathType.JSON_POINTER), errors); checkDynamicUri(camelOnly, new NodePath(PathType.JSON_POINTER), errors); // where the body comes from, across the routes of the file (CAMEL-24844) - BodyTypeFlow.check(camelOnly, new NodePath(PathType.JSON_POINTER), errors, facts.bodylessEndpoints()); + BodyTypeFlow.check(camelOnly, new NodePath(PathType.JSON_POINTER), errors, facts.bodylessEndpoints(), + bodyTypes()); // a field read on a body that is still text (CAMEL-24844) - TextBodyFlow.check(camelOnly, new NodePath(PathType.JSON_POINTER), errors, facts); + TextBodyFlow.check(camelOnly, new NodePath(PathType.JSON_POINTER), errors, facts, bodyTypes()); } if (canonical) { checkOneOfCardinality(camelOnly, new NodePath(PathType.JSON_POINTER), errors); @@ -1564,6 +1569,17 @@ private CamelCatalog catalog() { return catalog; } + /** + * The body types of the catalog (CAMEL-24961). The catalog of an older Camel version has none, so the body types of + * the validator's own catalog stand in for a model that has none. + */ + private CatalogBodyTypes bodyTypes() { + if (bodyTypes == null) { + bodyTypes = new CatalogBodyTypes(catalog(), otherCatalog ? new DefaultCamelCatalog() : null); + } + return bodyTypes; + } + private void collectProperties(JsonNode node, Set answer, int depth) { if (node == null || node.isMissingNode() || depth > 3) { return; diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/BodyTypeFlowTest.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/BodyTypeFlowTest.java index 076987525d9eb..c2bf215eedd8e 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/BodyTypeFlowTest.java +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/BodyTypeFlowTest.java @@ -93,6 +93,42 @@ public void testATimerAtTheRootOfTheChain() { assertThat(messages(yaml)).anyMatch(m -> m.contains("reads the message body")); } + @Test + public void testTheConsumersWithoutABodyComeFromTheCatalog() { + // CAMEL-24961: cron gives no body, and neither does an ftp consumer that does not download the file + String yaml = """ + - route: + id: nightly + from: + uri: cron:nightly + parameters: + schedule: 0 0 2 * * ? + steps: + - setHeader: + name: sku + jsonpath: $.sku + - route: + id: listing + from: + uri: ftp:files.example.com/inbox?download=false + steps: + - setHeader: + name: sku + jsonpath: $.sku + - route: + id: files + from: + uri: ftp:files.example.com/inbox + steps: + - setHeader: + name: sku + jsonpath: $.sku + """; + assertThat(messages(yaml)).satisfiesExactly( + m -> assertThat(m).contains("route nightly: jsonpath reads the message body"), + m -> assertThat(m).contains("route listing: jsonpath reads the message body")); + } + @Test public void testAPostCarriesABodySoNothingIsSaid() { String yaml = """ diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/TextBodyFlowTest.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/TextBodyFlowTest.java index 976f97598d03e..5c0085afe5a80 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/TextBodyFlowTest.java +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/TextBodyFlowTest.java @@ -16,6 +16,7 @@ */ package org.apache.camel.dsl.yaml.validator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -23,6 +24,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.camel.catalog.DefaultCamelCatalog; +import org.apache.camel.tooling.model.ComponentModel; +import org.apache.camel.tooling.model.DataFormatModel; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -289,7 +293,7 @@ public void testParsedDataWrittenToAFile() { """; assertThat(lines(yaml)).singleElement().satisfies(m -> { assertThat(m).startsWith("Line 12: "); - assertThat(m).contains("route archive: to: file writes the body to a file"); + assertThat(m).contains("route archive: to: file needs the body as text, bytes, a stream or a file"); assertThat(m).contains("the body here is parsed data (a Map or a List), not text"); assertThat(m).contains("unmarshal: json turns the text into data: add marshal: json before this step"); }); @@ -313,7 +317,7 @@ public void testValidateKeepsTheBody() { - to: file:valid """; assertThat(lines(yaml)).singleElement() - .satisfies(m -> assertThat(m).contains("to: file writes the body to a file")); + .satisfies(m -> assertThat(m).contains("to: file needs the body as text, bytes, a stream or a file")); } @Test @@ -818,7 +822,8 @@ public void testTheBodyAProducerReturnedInTheRunningApp() { ProjectFacts seen = observed(ProjectFacts.producerKey("sql:select * from orders"), List.of("java.util.ArrayList"), List.of("collection")); assertThat(lines(yaml, seen)).singleElement().satisfies(m -> { - assertThat(m).contains("to: file writes the body to a file, but the body here is parsed data"); + assertThat(m) + .contains("to: file needs the body as text, bytes, a stream or a file, but the body here is parsed data"); assertThat(m).contains("in the running app, to: sql:select * from orders returned an ArrayList"); // the walk knows no data format for what the running app saw assertThat(m).contains("add marshal with the data format to write (such as marshal: json) before this step"); @@ -866,6 +871,225 @@ public void testWhatTheWalkKnowsComesFirst() { }); } + @Test + public void testTheConsumersOfTheCatalog() { + // CAMEL-24961: what a consumer gives comes from the body types of the catalog + String yaml = """ + - route: + id: sqs + from: + uri: aws2-sqs:orders + steps: + - setHeader: + name: orderId + simple: ${body[orderId]} + - route: + id: mqtt + from: + uri: paho-mqtt5:sensors + steps: + - setBody: + groovy: body.temperature + - route: + id: ftp + from: + uri: ftp:files.example.com/inbox + parameters: + include: .*\\.json + steps: + - setHeader: + name: orderId + simple: ${body[orderId]} + - route: + id: ftpStream + from: + uri: ftp:files.example.com/inbox?streamDownload=true + steps: + - setHeader: + name: orderId + simple: ${body[orderId]} + """; + assertThat(lines(yaml)).satisfiesExactly( + m -> assertThat(m).contains("route sqs: simple reads a field of the body (${body[orderId]})") + .contains("the body here is still text, not parsed data - the aws2-sqs consumer gives it as text"), + m -> assertThat(m).contains("route mqtt: groovy reads fields of the body (body.temperature)") + .contains("the paho-mqtt5 consumer gives it as bytes"), + m -> assertThat(m).contains("route ftp: simple reads a field of the body") + .contains("the body here is the file as it was read (a GenericFile), not parsed data") + .contains("the ftp consumer gives it: add unmarshal: json before this step"), + m -> assertThat(m).contains("route ftpStream: simple reads a field of the body") + .contains("the ftp consumer gives it as a stream")); + } + + @Test + public void testTheProducersOfTheCatalog() { + // the response of the http producer is bytes or a stream, a row of sql is data, and a file write keeps the body + String yaml = """ + - route: + id: price + from: + uri: timer:tick + steps: + - to: https://api.example.com/prices + - setHeader: + name: price + simple: ${body[price]} + - route: + id: rows + from: + uri: sql:select * from orders + steps: + - to: file:orders + - route: + id: archive + from: + uri: file:inbox + steps: + - to: file:backup + - setBody: + groovy: body.items + - route: + id: sensors + from: + uri: file:readings + steps: + - unmarshal: + json: {} + - to: paho-mqtt5:readings + """; + assertThat(lines(yaml)).satisfiesExactly( + m -> assertThat(m).contains("route price: simple reads a field of the body (${body[price]})") + .contains("to: https://api.example.com/prices returns it as bytes or a stream: unmarshal it first"), + m -> assertThat(m).contains("route rows: to: file needs the body as text, bytes, a stream or a file") + .contains("the sql consumer gives it as a Map: add marshal with the data format to write"), + m -> assertThat(m).contains("route archive: groovy reads fields of the body (body.items)") + .contains("the file as it was read (a GenericFile)"), + m -> assertThat(m).contains("route sensors: to: paho-mqtt5 needs the body as text, bytes, a stream or a file") + .contains("unmarshal: json turns the text into data: add marshal: json before this step")); + } + + @Test + public void testTheOptionsTheBodyDependsOn() { + // xslt with output DOM gives a Document, an unmarshal with useList a List, gson is not known, and with download + // false the ftp consumer gives no body + String yaml = """ + - route: + from: + uri: file:in + steps: + - to: xslt:order.xsl?output=DOM + - setBody: + groovy: body.order + - route: + from: + uri: file:in + steps: + - unmarshal: + json: + library: Gson + - to: file:out + - route: + from: + uri: ftp:files.example.com/inbox?download=false + steps: + - setBody: + groovy: body.items + - route: + from: + uri: file:in + steps: + - unmarshal: + json: + useList: true + - to: file:out + """; + assertThat(lines(yaml)).singleElement().satisfies(m -> assertThat(m) + .contains("to: file needs the body as text, bytes, a stream or a file, but the body here is parsed data")); + } + + @Test + public void testWhatTheComponentMayConfigureSaysNothing() { + // the valueDeserializer of kafka and the operation of aws2-sqs can be set on the component, which the route + // does not show; a placeholder is only known when the route runs + String yaml = """ + - route: + from: + uri: kafka:orders + steps: + - setHeader: + name: orderId + simple: ${body[orderId]} + - route: + from: + uri: file:in + steps: + - unmarshal: + json: {} + - to: aws2-sqs:orders + - route: + from: + uri: ftp:files.example.com/inbox?streamDownload={{stream}} + steps: + - setHeader: + name: orderId + simple: ${body[orderId]} + """; + assertThat(lines(yaml)).isEmpty(); + } + + @Test + public void testOneReportForAStepThatFails() { + // the first write fails at runtime, so the second does not run with this body + String yaml = """ + - route: + from: + uri: file:inbox + steps: + - unmarshal: + json: {} + - to: file:archive + - to: file:backup + """; + assertThat(lines(yaml)).singleElement().satisfies(m -> assertThat(m).startsWith("Line 7: ")); + } + + @Test + public void testACatalogWithoutBodyTypes() throws Exception { + // the catalog of an older Camel version has no body types: the validator's own catalog stands in for it + DefaultCamelCatalog older = new DefaultCamelCatalog() { + @Override + public ComponentModel componentModel(String name) { + ComponentModel model = super.componentModel(name); + if (model != null) { + model.setBodyTypes(new LinkedHashMap<>()); + } + return model; + } + + @Override + public DataFormatModel dataFormatModel(String name) { + DataFormatModel model = super.dataFormatModel(name); + if (model != null) { + model.setBodyTypes(new LinkedHashMap<>()); + } + return model; + } + }; + YamlValidator other = new YamlValidator(false, null, older); + other.init(); + String yaml = """ + - route: + from: + uri: file:inbox + steps: + - unmarshal: + json: {} + - to: file:archive + """; + assertThat(YamlValidator.describeAll(yaml, other.validate(yaml))).singleElement() + .satisfies(m -> assertThat(m).contains("to: file needs the body as text, bytes, a stream or a file")); + } + private static ProjectFacts observed(String key, List types, List kinds) { return new ProjectFacts(Set.of(), Map.of(), null, Map.of(), Map.of(key, new ProjectFacts.Observed(types, kinds))); }