From 4da06751f391c709367df692c43c4e7f456442e4 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Mon, 2 Nov 2026 11:03:45 -0800 Subject: [PATCH 01/34] Apply Play routes to root HTTP spans in OpenTelemetry mode Play already decorates its framework span with the matched route, but the underlying HTTP server span also needs that route for OpenTelemetry naming. Apply it to the local root before application code can trigger sampling, using the shared HTTP resource decorator. Keep the existing Play span decoration and API Security route dispatch. --- .../play26/PlayHttpServerDecorator.java | 11 ++++-- ...rverRoutesScalaWithErrorHandlerTest.groovy | 36 ++++++++++++++++--- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java b/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java index 1018b576f3d..7b8e5628986 100644 --- a/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java @@ -153,14 +153,21 @@ protected void doOnRequest( PATH_CACHE.computeIfAbsent( defOption.get().path(), p -> addMissingSlash(p, request.path())); HTTP_RESOURCE_DECORATOR.withRoute(span, request.method(), path, true); + if (Config.get().isTraceOtelSemanticsEnabled()) { + final AgentSpan rootSpan = span.getLocalRootSpan(); + if (rootSpan != null && rootSpan != span) { + // Set the transport span's route before application code can trigger sampling. + HTTP_RESOURCE_DECORATOR.withRoute(rootSpan, request.method(), path, true); + } + } dispatchRoute(span, path); } } } /** - * Play does not set the http.route in the local root span so we need to store it in the context - * for API security + * Store the route in the context for API security, including when legacy semantics leave it only + * on the Play span. */ private void dispatchRoute(final AgentSpan span, final CharSequence route) { try { diff --git a/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy b/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy index 721aed82005..9f517dacfe6 100644 --- a/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy +++ b/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy @@ -3,8 +3,11 @@ package datadog.trace.instrumentation.play27.server.test import datadog.trace.agent.test.asserts.TraceAssert import datadog.trace.agent.test.base.HttpServer import datadog.trace.agent.test.base.HttpServerTest +import datadog.trace.agent.test.naming.TestingOtelHttpSemantics +import datadog.trace.api.Config import datadog.trace.api.DDSpanTypes import datadog.trace.api.DDTags +import datadog.trace.api.KnownTags import datadog.trace.api.config.TraceInstrumentationConfig import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.instrumentation.play26.PlayHttpServerDecorator @@ -73,10 +76,24 @@ class PlayAsyncServerRoutesScalaWithErrorHandlerTest extends AbstractPlayServer2 } } + @Override + Serializable expectedServerSpanRoute(HttpServerTest.ServerEndpoint endpoint) { + Config.get().isTraceOtelSemanticsEnabled() ? routeFor(endpoint) : null + } + + @Override + Map expectedExtraServerTags(HttpServerTest.ServerEndpoint endpoint) { + Config.get().isTraceOtelSemanticsEnabled() ? [:] : [(Tags.HTTP_ROUTE): null] + } + // we set Tags.HTTP_ROUTE with a routes file; override method for this @Override void handlerSpan(TraceAssert trace, HttpServerTest.ServerEndpoint endpoint = SUCCESS) { def expectedQueryTag = expectedQueryTag(endpoint) + def expectedRoute = routeFor(endpoint) + def otelSemantics = Config.get().isTraceOtelSemanticsEnabled() + def requestUrl = endpoint.resolve(address) + def expectedPath = Config.get().isHttpServerRawResource() && supportsRaw() ? requestUrl.rawPath : requestUrl.path trace.span { serviceName expectedServiceName() operationName "play.request" @@ -89,18 +106,29 @@ class PlayAsyncServerRoutesScalaWithErrorHandlerTest extends AbstractPlayServer2 "$Tags.PEER_HOST_IPV4" '127.0.0.1' "$Tags.HTTP_CLIENT_IP" (endpoint == FORWARDED ? endpoint.body : '127.0.0.1') "$Tags.NETWORK_CLIENT_IP" '127.0.0.1' - "$Tags.HTTP_URL" String - "$Tags.HTTP_HOSTNAME" address.host + if (otelSemantics) { + "$Tags.HTTP_URL" null + "$Tags.HTTP_HOSTNAME" address.host + "$KnownTags.SERVER_PORT_NAME" address.port + "$KnownTags.URL_PATH_NAME" expectedPath + "$KnownTags.URL_SCHEME_NAME" address.scheme + } else { + "$Tags.HTTP_URL" String + "$Tags.HTTP_HOSTNAME" address.host + } "$Tags.HTTP_METHOD" String - "$Tags.HTTP_ROUTE" this.routeFor(endpoint) + "$Tags.HTTP_ROUTE" expectedRoute if (endpoint == EXCEPTION || endpoint == CUSTOM_EXCEPTION) { errorTags(endpoint == CUSTOM_EXCEPTION ? TestHttpErrorHandler.CustomRuntimeException : RuntimeException, endpoint.body) } if (endpoint.query) { - "$DDTags.HTTP_QUERY" expectedQueryTag + tag(otelSemantics ? KnownTags.URL_QUERY_NAME : DDTags.HTTP_QUERY, expectedQueryTag) } defaultTags() } } } } + +class PlayAsyncServerRoutesOtelSemanticsForkedTest extends PlayAsyncServerRoutesScalaWithErrorHandlerTest implements TestingOtelHttpSemantics { +} From 8b2fdb8d15cb9d0185283e1c9df5de1a72eadd14 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Mon, 2 Nov 2026 11:03:45 -0800 Subject: [PATCH 02/34] Capture Akka HTTP route templates in OpenTelemetry mode Observe successful path matchers and combine their literal segments and parameter placeholders into a route on the local root HTTP server span. Reuse the shared HTTP resource decorator to set both http.route and the method-and-route resource name without retaining parameter values. Allow the required matcher classes to be transformed and include a forked test that verifies stable routes across different parameter values. --- .../bytebuddy/matcher/ignored_class_name.trie | 4 +- .../AkkaHttpServerInstrumentationTest.groovy | 30 +++++ .../AkkaHttpPathMatcherInstrumentation.java | 93 +++++++++++++ .../akkahttp/AkkaHttpRouteHelper.java | 126 ++++++++++++++++++ 4 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java create mode 100644 dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java diff --git a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie index 7a57990c9f5..e1bb3d199ae 100644 --- a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie +++ b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie @@ -162,7 +162,9 @@ 0 akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 0 akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport$class 0 akka.http.scaladsl.unmarshalling.* -0 akka.http.scaladsl.server.PathMatcher$Matched +# Path matchers supply route templates when OTel HTTP semantics are enabled +0 akka.http.scaladsl.server.PathMatcher* +0 akka.http.scaladsl.server.ImplicitPathMatcherConstruction* 0 akka.http.scaladsl.server.directives.ParameterDirectives$class 0 akka.http.scaladsl.server.directives.ParameterDirectives 0 akka.http.scaladsl.server.directives.BasicDirectives$class diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy index c4619379863..3c5c4155e72 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy @@ -1,5 +1,6 @@ import datadog.trace.agent.test.base.HttpServer import datadog.trace.agent.test.base.HttpServerTest +import datadog.trace.agent.test.base.WithHttpServer import datadog.trace.agent.test.naming.TestingGenericHttpNamingConventions import datadog.trace.agent.test.naming.TestingOtelHttpSemantics import datadog.trace.api.config.TraceInstrumentationConfig @@ -279,6 +280,35 @@ class AkkaHttpServerInstrumentationSyncV1ForkedTest extends AkkaHttpServerInstru class AkkaHttpServerInstrumentationSyncOtelSemanticsForkedTest extends AkkaHttpServerInstrumentationSyncV0ForkedTest implements TestingOtelHttpSemantics { } +class AkkaHttpRouteOtelSemanticsForkedTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0, TestingOtelHttpSemantics { + @Override + HttpServer server() { + new AkkaHttpTestWebServer(AkkaHttpTestWebServer.BindAndHandle()) + } + + def "matched paths produce stable server routes"() { + when: + def response = client.newCall(new Request.Builder().url(address.resolve(path).toString()).build()).execute() + assert response.code() == 200 + response.close() + TEST_WRITER.waitForTraces(1) + + then: + def serverSpan = TEST_WRITER.flatten().find { it.getTag('component')?.toString() == 'akka-http-server' } + serverSpan != null + serverSpan.getTag('http.route') == route + serverSpan.resourceName.toString() == "GET $route" + + where: + path | route + '/success' | '/success' + '/query?some=value' | '/query' + '/injected-id/ping/1' | '/injected-id/ping/{param}' + '/injected-id/ping/2' | '/injected-id/ping/{param}' + '/injected-id/fing/3' | '/injected-id/fing/{param}' + } +} + class AkkaHttpServerInstrumentationAsyncTest extends AkkaHttpServerInstrumentationTest { @Override HttpServer server() { diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java new file mode 100644 index 00000000000..1cd43014118 --- /dev/null +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java @@ -0,0 +1,93 @@ +package datadog.trace.instrumentation.akkahttp; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.extendsClass; +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static java.util.Collections.singletonMap; +import static net.bytebuddy.matcher.ElementMatchers.isBridge; +import static net.bytebuddy.matcher.ElementMatchers.not; +import static net.bytebuddy.matcher.ElementMatchers.returns; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import akka.http.scaladsl.model.Uri; +import akka.http.scaladsl.server.PathMatcher; +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.api.Config; +import datadog.trace.bootstrap.InstrumentationContext; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import java.util.Map; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; + +/** Captures route templates from successful Akka HTTP path matchers. */ +@AutoService(InstrumenterModule.class) +public final class AkkaHttpPathMatcherInstrumentation extends InstrumenterModule.Tracing + implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { + public AkkaHttpPathMatcherInstrumentation() { + super("akka-http", "akka-http-server"); + } + + @Override + public boolean isEnabled() { + return super.isEnabled() && Config.get().isTraceOtelSemanticsEnabled(); + } + + @Override + public String hierarchyMarkerType() { + return "akka.http.scaladsl.server.PathMatcher"; + } + + @Override + public ElementMatcher hierarchyMatcher() { + return extendsClass(named(hierarchyMarkerType())); + } + + @Override + public Map contextStore() { + return singletonMap( + "datadog.trace.bootstrap.instrumentation.api.AgentSpan", + packageName + ".AkkaHttpRouteHelper$Route"); + } + + @Override + public String[] helperClassNames() { + return new String[] { + packageName + ".AkkaHttpRouteHelper", + packageName + ".AkkaHttpRouteHelper$Frame", + packageName + ".AkkaHttpRouteHelper$Route" + }; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + named("apply") + .and(takesArguments(1)) + .and(takesArgument(0, named("akka.http.scaladsl.model.Uri$Path"))) + .and(returns(named("akka.http.scaladsl.server.PathMatcher$Matching"))) + .and(not(isBridge())), + getClass().getName() + "$MatchAdvice"); + } + + public static final class MatchAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static AkkaHttpRouteHelper.Frame enter(@Advice.Argument(0) Uri.Path path) { + return AkkaHttpRouteHelper.enter(path); + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void exit( + @Advice.Enter AkkaHttpRouteHelper.Frame frame, + @Advice.Return PathMatcher.Matching matching, + @Advice.Thrown Throwable throwable) { + AkkaHttpRouteHelper.exit( + frame, + matching, + throwable, + InstrumentationContext.get(AgentSpan.class, AkkaHttpRouteHelper.Route.class)); + } + } +} diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java new file mode 100644 index 00000000000..a7e5ce55890 --- /dev/null +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java @@ -0,0 +1,126 @@ +package datadog.trace.instrumentation.akkahttp; + +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; +import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR; + +import akka.http.scaladsl.model.Uri; +import akka.http.scaladsl.server.PathMatcher; +import datadog.trace.api.KnownTags; +import datadog.trace.bootstrap.ContextStore; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.URIUtils; +import java.util.ArrayList; +import java.util.List; +import scala.Product; + +/** + * Builds templates from matched literals and extracted parameters, retaining no parameter values. + */ +public final class AkkaHttpRouteHelper { + private static final ThreadLocal CURRENT = new ThreadLocal<>(); + + private AkkaHttpRouteHelper() {} + + public static Frame enter(Uri.Path path) { + AgentSpan active = activeSpan(); + if (active == null) { + return null; + } + AgentSpan root = active.getLocalRootSpan(); + if (root == null || root.getTag(KnownTags.URL_PATH_NAME) == null) { + return null; + } + Frame frame = new Frame(path.toString(), root, CURRENT.get()); + CURRENT.set(frame); + return frame; + } + + public static void exit( + Frame frame, + PathMatcher.Matching matching, + Throwable throwable, + ContextStore routes) { + if (frame == null) { + return; + } + if (frame.parent == null) { + CURRENT.remove(); + } else { + CURRENT.set(frame.parent); + } + if (throwable != null || !(matching instanceof PathMatcher.Matched)) { + return; + } + PathMatcher.Matched matched = (PathMatcher.Matched) matching; + String rest = matched.pathRest().toString(); + int consumed = frame.path.length() - rest.length(); + if (consumed <= 0) { + return; + } + String template = frame.template(consumed, matched.extractions()); + if (frame.parent != null) { + if (frame.parent.path.endsWith(frame.path)) { + frame.parent.parts.add( + new Route(frame.parent.path.length() - frame.path.length(), rest, template)); + } + return; + } + Object requestPath = frame.span.getTag(KnownTags.URL_PATH_NAME); + if (requestPath == null || !requestPath.toString().equals(URIUtils.decode(frame.path))) { + Route previous = routes.get(frame.span); + if (previous == null || !previous.remaining.equals(frame.path)) { + return; + } + template = previous.template + template; + } + routes.put(frame.span, new Route(0, rest, template)); + Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); + HTTP_RESOURCE_DECORATOR.withRoute( + frame.span, method == null ? null : method.toString(), template, true); + } + + public static final class Frame { + final String path; + final AgentSpan span; + final Frame parent; + final List parts = new ArrayList<>(); + + Frame(String path, AgentSpan span, Frame parent) { + this.path = path; + this.span = span; + this.parent = parent; + } + + String template(int consumed, Object extractions) { + if (parts.isEmpty()) { + if (extractions instanceof Product && ((Product) extractions).productArity() > 0) { + return path.charAt(0) == '/' ? "/{param}" : "{param}"; + } + return path.substring(0, consumed); + } + StringBuilder template = new StringBuilder(); + int cursor = 0; + for (Route part : parts) { + int end = path.length() - part.remaining.length(); + if (part.offset < cursor || end > consumed) { + continue; + } + template.append(path, cursor, part.offset).append(part.template); + cursor = end; + } + return template.append(path, cursor, consumed).toString(); + } + } + + public static final class Route { + final int offset; + final String remaining; + final String template; + + Route(int offset, String remaining, String template) { + this.offset = offset; + this.remaining = remaining; + this.template = template; + } + } +} From a8673bd8f1913ae688a6eadb423b277fb2d15709 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Mon, 2 Nov 2026 11:03:45 -0800 Subject: [PATCH 03/34] Support nested Netty HTTP span suppression in OpenTelemetry mode Add a scoped context marker for HTTP client spans owned by higher-level instrumentation. Enable suppression only with OpenTelemetry semantics and only when the marker identifies the active request span. When marked, Netty propagates that context without creating another HTTP span. Response, error and close handlers retain the callback context and leave decoration and completion to the instrumentation that owns the span. Unmarked requests and Datadog semantics keep the existing Netty behavior. --- .../decorator/HttpClientDecorator.java | 19 ++++++++++++ .../HttpClientRequestTracingHandler.java | 20 +++++++++++- .../HttpClientResponseTracingHandler.java | 31 ++++++++++++------- 3 files changed, 57 insertions(+), 13 deletions(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java index 3bee2a6afbf..40e3a3d2970 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java @@ -6,6 +6,9 @@ import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR; import datadog.appsec.api.blocking.BlockingException; +import datadog.context.Context; +import datadog.context.ContextKey; +import datadog.context.ContextScope; import datadog.trace.api.Config; import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.KnownTags; @@ -40,6 +43,9 @@ public abstract class HttpClientDecorator extends UriBasedCli private static final Logger log = LoggerFactory.getLogger(HttpClientDecorator.class); + private static final ContextKey CLIENT_SPAN_KEY = + ContextKey.named("dd-http-client-span"); + private static final String DATADOG_META_LANG_HEADER_NAME = "Datadog-Meta-Lang"; private static final String DD_CLIENT_LIBRARY_LANGUAGE_HEADER_NAME = "DD-Client-Library-Language"; @@ -54,6 +60,19 @@ public abstract class HttpClientDecorator extends UriBasedCli private static final boolean APPSEC_RASP_ENABLED = Config.get().isAppSecRaspEnabled(); + /** Suppresses duplicate transport spans when OpenTelemetry semantics are enabled. */ + public static ContextScope suppressNestedClientSpans(final AgentSpan span) { + return OTEL_SEMANTICS ? Context.current().with(CLIENT_SPAN_KEY, span).attach() : null; + } + + /** Returns whether {@code span} owns the HTTP request represented by {@code context}. */ + public static boolean isNestedClientSpanSuppressed(final Context context, final AgentSpan span) { + return OTEL_SEMANTICS + && context != null + && span != null + && context.get(CLIENT_SPAN_KEY) == span; + } + protected abstract String method(REQUEST request); protected abstract URI url(REQUEST request) throws URISyntaxException; diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java index 47134f4a0cf..e53cb117b26 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java @@ -3,6 +3,7 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator.isNestedClientSpanSuppressed; import static datadog.trace.instrumentation.netty41.AttributeKeys.CLIENT_PARENT_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONNECT_PARENT_CONTINUATION_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; @@ -72,8 +73,25 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann } } } + final AgentSpan parent = activeSpan(); + final Context parentContext = Context.current(); + if (isNestedClientSpanSuppressed(parentContext, parent)) { + if (ctx.channel().parent() == null) { + ctx.channel().attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(parent); + } + DECORATE.injectContext(parentContext, request.headers(), SETTER); + ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(parentContext); + try { + ctx.write(msg, prm); + return; + } finally { + if (null != parentScope) { + parentScope.close(); + } + } + } if (ctx.channel().parent() == null) { - ctx.channel().attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(activeSpan()); + ctx.channel().attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(parent); } boolean isSecure = SSL_HANDLER != null && ctx.pipeline().get(SSL_HANDLER) != null; NettyHttpClientDecorator decorate = isSecure ? DECORATE_SECURE : DECORATE; diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java index b80f967a3fb..94443eeff4d 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java @@ -2,6 +2,7 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.noopSpan; +import static datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator.isNestedClientSpanSuppressed; import static datadog.trace.instrumentation.netty41.AttributeKeys.CLIENT_PARENT_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.client.NettyHttpClientDecorator.DECORATE; @@ -29,13 +30,15 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { final AgentSpan parent = parentAttr.get(); final Context storedContext = ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).get(); final AgentSpan span = AgentSpan.fromContext(storedContext); + final boolean clientSpanSuppressed = isNestedClientSpanSuppressed(storedContext, span); + final AgentSpan callbackParent = clientSpanSuppressed ? span : parent; // Set parent context back to maintain the same functionality as getAndSet(parent) if (storedContext != null) { - ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(parent)); + ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(callbackParent)); } - if (span != null) { + if (span != null && !clientSpanSuppressed) { final boolean finishSpan = msg instanceof HttpResponse && (!HttpResponseStatus.SWITCHING_PROTOCOLS.equals(((HttpResponse) msg).status()) @@ -54,8 +57,8 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } } - // We want the callback in the scope of the parent, not the client span - try (final ContextScope scope = activateSpan(parent)) { + // Run the callback under the span that initiated the transport request. + try (final ContextScope scope = activateSpan(callbackParent)) { ctx.fireChannelRead(msg); } } @@ -67,13 +70,15 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E final AgentSpan parent = parentAttr.get(); final Context storedContext = ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).get(); final AgentSpan span = AgentSpan.fromContext(storedContext); + final boolean clientSpanSuppressed = isNestedClientSpanSuppressed(storedContext, span); + final AgentSpan callbackParent = clientSpanSuppressed ? span : parent; // Set parent context back to maintain the same functionality as getAndSet(parent) if (storedContext != null) { - ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(parent)); + ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(callbackParent)); } - if (span != null) { + if (span != null && !clientSpanSuppressed) { // If an exception is passed to this point, it likely means it was unhandled and the // client span won't be finished with a proper response, so we should finish the span here. try (final ContextScope scope = activateSpan(span)) { @@ -82,8 +87,8 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E span.finish(); } } - // We want the callback in the scope of the parent, not the client span - try (final ContextScope scope = activateSpan(parent)) { + // Run the callback under the span that initiated the transport request. + try (final ContextScope scope = activateSpan(callbackParent)) { super.exceptionCaught(ctx, cause); } } @@ -95,20 +100,22 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { final AgentSpan parent = parentAttr.get(); final Context storedContext = ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).get(); final AgentSpan span = AgentSpan.fromContext(storedContext); + final boolean clientSpanSuppressed = isNestedClientSpanSuppressed(storedContext, span); + final AgentSpan callbackParent = clientSpanSuppressed ? span : parent; // Set parent context back to maintain the same functionality as getAndSet(parent) if (storedContext != null) { - ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(parent)); + ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(callbackParent)); } - if (span != null && span != parent) { + if (span != null && span != parent && !clientSpanSuppressed) { try (final ContextScope scope = activateSpan(span)) { DECORATE.beforeFinish(span); span.finish(); } } - // We want the callback in the scope of the parent, not the client span - try (final ContextScope scope = activateSpan(parent)) { + // Run the callback under the span that initiated the transport request. + try (final ContextScope scope = activateSpan(callbackParent)) { super.channelInactive(ctx); } } From 6b3369329a34990341a2d8730c6378a10b07ce49 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Mon, 2 Nov 2026 11:03:45 -0800 Subject: [PATCH 04/34] Emit one Spring WebClient span in OpenTelemetry mode Mark the WebClient span while subscribing so Netty reuses its context instead of creating a duplicate HTTP client span. In OpenTelemetry mode, finish the WebClient span before response or error callbacks, which may wait for that span to complete. Preserve the existing two spans and completion timing in Datadog mode. Update the Spring 5 and Spring 6 client suites to check the mode-dependent span counts and add forked OpenTelemetry coverage for both versions. Skip HEAD response-body consumption only in OpenTelemetry mode in the Spring 5 test helper. --- .../client/TraceWebClientSubscriber.java | 33 +++++++++++++++---- .../client/WebClientTracingFilter.java | 4 ++- .../client/SpringWebfluxHttpClientBase.groovy | 16 +++++---- .../SpringWebfluxHttpClientBasicTest.groovy | 5 +++ ...bfluxHttpClientDoAfterTerminateTest.groovy | 4 +-- ...luxHttpClientDoOnSuccessOrErrorTest.groovy | 4 +-- ...ingWebfluxHttpClientDoOnSuccessTest.groovy | 4 +-- ...gWebfluxHttpClientDoOnTerminateTest.groovy | 4 +-- .../client/SpringWebfluxHttpClientBase.groovy | 7 ++-- .../SpringWebfluxHttpClientBasicTest.groovy | 5 +++ ...bfluxHttpClientDoAfterTerminateTest.groovy | 2 +- ...luxHttpClientDoOnSuccessOrErrorTest.groovy | 2 +- ...ingWebfluxHttpClientDoOnSuccessTest.groovy | 2 +- ...gWebfluxHttpClientDoOnTerminateTest.groovy | 2 +- 14 files changed, 66 insertions(+), 28 deletions(-) diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/TraceWebClientSubscriber.java b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/TraceWebClientSubscriber.java index c0d75bf4999..44c9668e1bd 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/TraceWebClientSubscriber.java +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/TraceWebClientSubscriber.java @@ -5,6 +5,7 @@ import static datadog.trace.instrumentation.springwebflux.client.SpringWebfluxHttpClientDecorator.DECORATE; import datadog.context.ContextScope; +import datadog.trace.api.Config; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import org.reactivestreams.Subscription; import org.springframework.web.reactive.function.client.ClientResponse; @@ -12,6 +13,7 @@ import reactor.util.context.Context; public final class TraceWebClientSubscriber implements CoreSubscriber { + private static final boolean OTEL_SEMANTICS = Config.get().isTraceOtelSemanticsEnabled(); final CoreSubscriber actual; @@ -39,23 +41,30 @@ public void onSubscribe(final Subscription subscription) { @Override public void onNext(final ClientResponse response) { + // Without a transport span, callbacks may wait for this span to finish. + if (OTEL_SEMANTICS) { + finishSpan(response); + } try (final ContextScope scope = activateSpan(parent)) { actual.onNext(response); } finally { - DECORATE.onResponse(span, response); - DECORATE.beforeFinish(span); - span.finish(); + if (!OTEL_SEMANTICS) { + finishSpan(response); + } } } @Override public void onError(final Throwable t) { + if (OTEL_SEMANTICS) { + finishSpan(t); + } try (final ContextScope scope = activateSpan(parent)) { actual.onError(t); } finally { - DECORATE.onError(span, t); - DECORATE.beforeFinish(span); - span.finish(); + if (!OTEL_SEMANTICS) { + finishSpan(t); + } } } @@ -76,4 +85,16 @@ public void onCancel() { public Context currentContext() { return context; } + + private void finishSpan(final ClientResponse response) { + DECORATE.onResponse(span, response); + DECORATE.beforeFinish(span); + span.finish(); + } + + private void finishSpan(final Throwable error) { + DECORATE.onError(span, error); + DECORATE.beforeFinish(span); + span.finish(); + } } diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java index ad84ba4be92..b4b4e4961ac 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java @@ -3,6 +3,7 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator.suppressNestedClientSpans; import static datadog.trace.instrumentation.springwebflux.client.SpringWebfluxHttpClientDecorator.DECORATE; import static datadog.trace.instrumentation.springwebflux.client.SpringWebfluxHttpClientDecorator.HTTP_REQUEST; @@ -54,7 +55,8 @@ public void subscribe(final CoreSubscriber subscriber) { DECORATE.afterStart(span); DECORATE.onRequest(span, request); AgentSpan parent = activeSpan(); - try (final ContextScope scope = activateSpan(span)) { + try (final ContextScope scope = activateSpan(span); + final ContextScope suppressScope = suppressNestedClientSpans(span)) { TraceWebClientSubscriber tracingSubscriber = new TraceWebClientSubscriber(subscriber, span, parent); next.exchange(request).doOnCancel(tracingSubscriber::onCancel).subscribe(tracingSubscriber); diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBase.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBase.groovy index 066f37474e6..ea5b6b3b4c0 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBase.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBase.groovy @@ -3,6 +3,7 @@ package dd.trace.instrumentation.springwebflux.client import datadog.trace.agent.test.asserts.TraceAssert import datadog.trace.agent.test.base.HttpClientTest import datadog.trace.agent.test.naming.TestingGenericHttpNamingConventions +import datadog.trace.api.Config import datadog.trace.api.DDSpanTypes import datadog.trace.api.DDTags import datadog.trace.bootstrap.instrumentation.api.Tags @@ -40,18 +41,21 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes .block() if (hasParent) { - blockUntilChildSpansFinished(callback ? 3 : 2) + blockUntilChildSpansFinished(size(callback ? 2 : 1)) } check() - consumeResponse(response) + consumeResponse(response, method) } - protected static int consumeResponse(ClientResponse response) { + protected static int consumeResponse(ClientResponse response, String method) { int status = response.statusCode().value() Channel channel = responseChannel(response) - response.bodyToMono(Void).block() + // Only skip HEAD body consumption in OpenTelemetry mode. + if (!Config.get().isTraceOtelSemanticsEnabled() || method != "HEAD") { + response.bodyToMono(Void).block() + } // Body completion can occur inside channelRead, before channelReadComplete is fired. channel.eventLoop().submit({} as Runnable).sync() return status @@ -92,7 +96,7 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes def leafParentId = trace.spanAssertCount.get() super.clientSpan(trace, parentSpan, method, renameService, tagQueryString, uri, status, error, exception, ignorePeer, extraTags) - if (!exception) { + if (!exception && !Config.get().isTraceOtelSemanticsEnabled()) { def expectedQuery = tagQueryString ? uri.query : null def expectedUrl = URIUtils.buildURL(uri.scheme, uri.host, uri.port, uri.path) if (expectedQuery != null && !expectedQuery.empty) { @@ -140,7 +144,7 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes @Override int size(int size) { - return size + 1 + return Config.get().isTraceOtelSemanticsEnabled() ? size : size + 1 } boolean testRedirects() { diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBasicTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBasicTest.groovy index 69886df3eaa..a6cd953b133 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBasicTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBasicTest.groovy @@ -1,5 +1,6 @@ package dd.trace.instrumentation.springwebflux.client +import datadog.trace.agent.test.naming.TestingOtelHttpSemantics import org.springframework.web.reactive.function.client.WebClient import spock.lang.Timeout @@ -15,3 +16,7 @@ class SpringWebfluxHttpClientBasicTest extends SpringWebfluxHttpClientBase { void check() { } } + +@Timeout(5) +class SpringWebfluxHttpClientOtelSemanticsForkedTest extends SpringWebfluxHttpClientBasicTest implements TestingOtelHttpSemantics { +} diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy index 8355d53d18d..eb462aa02ba 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy @@ -24,12 +24,12 @@ class SpringWebfluxHttpClientDoAfterTerminateTest extends SpringWebfluxHttpClien .block() if (hasParent) { - blockUntilChildSpansFinished(callback ? 3 : 2) + blockUntilChildSpansFinished(size(callback ? 2 : 1)) } check() - consumeResponse(response) + consumeResponse(response, method) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy index 366e87d6df1..e5f78b8a0c4 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy @@ -24,12 +24,12 @@ abstract class SpringWebfluxHttpClientDoOnSuccessOrErrorTest extends SpringWebfl .block() if (hasParent) { - blockUntilChildSpansFinished(callback ? 3 : 2) + blockUntilChildSpansFinished(size(callback ? 2 : 1)) } check() - consumeResponse(response) + consumeResponse(response, method) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy index d6fbb96bc75..e5542b0dcdf 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy @@ -24,12 +24,12 @@ class SpringWebfluxHttpClientDoOnSuccessTest extends SpringWebfluxHttpClientBase .block() if (hasParent) { - blockUntilChildSpansFinished(callback ? 3 : 2) + blockUntilChildSpansFinished(size(callback ? 2 : 1)) } check() - consumeResponse(response) + consumeResponse(response, method) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy index b6b05def34d..32293622d8b 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy @@ -24,12 +24,12 @@ class SpringWebfluxHttpClientDoOnTerminateTest extends SpringWebfluxHttpClientBa .block() if (hasParent) { - blockUntilChildSpansFinished(callback ? 3 : 2) + blockUntilChildSpansFinished(size(callback ? 2 : 1)) } check() - consumeResponse(response) + consumeResponse(response, method) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBase.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBase.groovy index b419398ca6a..30d80ef78e0 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBase.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBase.groovy @@ -3,6 +3,7 @@ package dd.trace.instrumentation.springwebflux6.client import datadog.trace.agent.test.asserts.TraceAssert import datadog.trace.agent.test.base.HttpClientTest import datadog.trace.agent.test.naming.TestingGenericHttpNamingConventions +import datadog.trace.api.Config import datadog.trace.api.DDSpanTypes import datadog.trace.api.DDTags import datadog.trace.api.naming.SpanNaming @@ -44,7 +45,7 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes .block() if (hasParent) { - blockUntilChildSpansFinished(callback ? 3 : 2) + blockUntilChildSpansFinished(size(callback ? 2 : 1)) } check() @@ -74,7 +75,7 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes Map extraTags = null) { def leafParentId = trace.spanAssertCount.get() super.clientSpan(trace, parentSpan, method, renameService, tagQueryString, uri, status, error, exception, ignorePeer, extraTags) - if (!exception) { + if (!exception && !Config.get().isTraceOtelSemanticsEnabled()) { def expectedQuery = tagQueryString ? uri.query : null def expectedUrl = URIUtils.buildURL(uri.scheme, uri.host, uri.port, uri.path) if (expectedQuery != null && !expectedQuery.empty) { @@ -122,7 +123,7 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes @Override int size(int size) { - return size + 1 + return Config.get().isTraceOtelSemanticsEnabled() ? size : size + 1 } boolean testRedirects() { diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBasicTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBasicTest.groovy index b6510d4f7bc..4ff945bade9 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBasicTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBasicTest.groovy @@ -1,5 +1,6 @@ package dd.trace.instrumentation.springwebflux6.client +import datadog.trace.agent.test.naming.TestingOtelHttpSemantics import org.springframework.web.reactive.function.client.WebClient import spock.lang.Timeout @@ -15,3 +16,7 @@ class SpringWebfluxHttpClientBasicTest extends SpringWebfluxHttpClientBase { void check() { } } + +@Timeout(5) +class SpringWebfluxHttpClientOtelSemanticsForkedTest extends SpringWebfluxHttpClientBasicTest implements TestingOtelHttpSemantics { +} diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy index 36adaa1b9c4..22b3844a67d 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy @@ -30,7 +30,7 @@ class SpringWebfluxHttpClientDoAfterTerminateTest extends SpringWebfluxHttpClien .block() if (hasParent) { - blockUntilChildSpansFinished(callback ? 3 : 2) + blockUntilChildSpansFinished(size(callback ? 2 : 1)) } check() diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy index 625d91084ec..8d8b615d909 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy @@ -30,7 +30,7 @@ abstract class SpringWebfluxHttpClientDoOnSuccessOrErrorTest extends SpringWebfl .block() if (hasParent) { - blockUntilChildSpansFinished(callback ? 3 : 2) + blockUntilChildSpansFinished(size(callback ? 2 : 1)) } check() diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy index be489e8a6fe..6fc3bec70df 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy @@ -30,7 +30,7 @@ class SpringWebfluxHttpClientDoOnSuccessTest extends SpringWebfluxHttpClientBase .block() if (hasParent) { - blockUntilChildSpansFinished(callback ? 3 : 2) + blockUntilChildSpansFinished(size(callback ? 2 : 1)) } check() diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy index c4dc5bd9d2f..cb508b8fbb8 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy @@ -30,7 +30,7 @@ class SpringWebfluxHttpClientDoOnTerminateTest extends SpringWebfluxHttpClientBa .block() if (hasParent) { - blockUntilChildSpansFinished(callback ? 3 : 2) + blockUntilChildSpansFinished(size(callback ? 2 : 1)) } check() From 7e940a74d56047504a3eced09de64444b5c1eadc Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Tue, 3 Nov 2026 09:25:00 +0000 Subject: [PATCH 05/34] Fix WebFlux span expectations in OpenTelemetry mode --- .../bootTest/groovy/SpringWebfluxTest.groovy | 84 ++++++++++++------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/bootTest/groovy/SpringWebfluxTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/bootTest/groovy/SpringWebfluxTest.groovy index 9b49386871c..75fb1275d9f 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/bootTest/groovy/SpringWebfluxTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/bootTest/groovy/SpringWebfluxTest.groovy @@ -77,9 +77,11 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { def traceParent sortSpansByStart() - trace(2) { - clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) + trace(clientSpanCount()) { + traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) + if (!Config.get().isTraceOtelSemanticsEnabled()) { + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) + } } trace(2) { span { @@ -159,9 +161,11 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { sortSpansByStart() def traceParent - trace(2) { - clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) + trace(clientSpanCount()) { + traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) + if (!Config.get().isTraceOtelSemanticsEnabled()) { + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) + } } trace(3) { span { @@ -253,9 +257,11 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { sortSpansByStart() def traceParent - trace(2) { - clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) + trace(clientSpanCount()) { + traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) + if (!Config.get().isTraceOtelSemanticsEnabled()) { + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) + } } trace(3) { span { @@ -301,9 +307,11 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { sortSpansByStart() def traceParent - trace(2) { - clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), 404, true) - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), 404, true) + trace(clientSpanCount()) { + traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), 404, true) + if (!Config.get().isTraceOtelSemanticsEnabled()) { + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), 404, true) + } } trace(2) { span { @@ -357,9 +365,11 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { sortSpansByStart() def traceParent - trace(2) { - clientSpan(it, null, "http.request", "spring-webflux-client", "POST", URI.create(url), 202) - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "POST", URI.create(url), 202) + trace(clientSpanCount()) { + traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "POST", URI.create(url), 202) + if (!Config.get().isTraceOtelSemanticsEnabled()) { + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "POST", URI.create(url), 202) + } } trace(3) { span { @@ -422,9 +432,11 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { sortSpansByStart() def traceParent - trace(2) { - clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), 500) - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), 500) + trace(clientSpanCount()) { + traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), 500) + if (!Config.get().isTraceOtelSemanticsEnabled()) { + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), 500) + } } trace(2) { span { @@ -513,10 +525,12 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(4, SORT_TRACES_BY_START) { def traceParent1, traceParent2 - trace(2) { + trace(clientSpanCount()) { sortSpansByStart() - clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), 307) - traceParent1 = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), 307) + traceParent1 = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), 307) + if (!Config.get().isTraceOtelSemanticsEnabled()) { + traceParent1 = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), 307) + } } trace(2) { @@ -559,10 +573,12 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { } } } - trace(2) { + trace(clientSpanCount()) { sortSpansByStart() - clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(finalUrl)) - traceParent2 = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(finalUrl)) + traceParent2 = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(finalUrl)) + if (!Config.get().isTraceOtelSemanticsEnabled()) { + traceParent2 = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(finalUrl)) + } } trace(2) { sortSpansByStart() @@ -620,9 +636,11 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { sortSpansByStart() responses.eachWithIndex { def response, int i -> def traceParent - trace(2) { - clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) + trace(clientSpanCount()) { + traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) + if (!Config.get().isTraceOtelSemanticsEnabled()) { + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) + } } trace(2) { span { @@ -695,10 +713,12 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { def traceParent sortSpansByStart() - trace(2) { - clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), null, false, false, + trace(clientSpanCount()) { + traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), null, false, false, ["message": "The subscription was cancelled", "event": "cancelled"]) - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), null) + if (!Config.get().isTraceOtelSemanticsEnabled()) { + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), null) + } } trace(2) { span { @@ -819,6 +839,10 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { } } + int clientSpanCount() { + Config.get().isTraceOtelSemanticsEnabled() ? 1 : 2 + } + def clientSpan( TraceAssert trace, Object parentSpan, From e318582ed34d271f24c29f6560f6be43d37d7e20 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Tue, 3 Nov 2026 09:45:02 +0000 Subject: [PATCH 06/34] Avoid unused Akka HTTP route matcher lists --- .../instrumentation/akkahttp/AkkaHttpRouteHelper.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java index a7e5ce55890..27a61c6a03e 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java @@ -60,6 +60,9 @@ public static void exit( String template = frame.template(consumed, matched.extractions()); if (frame.parent != null) { if (frame.parent.path.endsWith(frame.path)) { + if (frame.parent.parts == null) { + frame.parent.parts = new ArrayList<>(); + } frame.parent.parts.add( new Route(frame.parent.path.length() - frame.path.length(), rest, template)); } @@ -83,7 +86,7 @@ public static final class Frame { final String path; final AgentSpan span; final Frame parent; - final List parts = new ArrayList<>(); + List parts; Frame(String path, AgentSpan span, Frame parent) { this.path = path; @@ -92,7 +95,7 @@ public static final class Frame { } String template(int consumed, Object extractions) { - if (parts.isEmpty()) { + if (parts == null) { if (extractions instanceof Product && ((Product) extractions).productArity() > 0) { return path.charAt(0) == '/' ? "/{param}" : "{param}"; } From 98227c1278cb33d5e47883beafc5081943ac0f76 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Tue, 3 Nov 2026 09:47:03 +0000 Subject: [PATCH 07/34] Reuse cached OpenTelemetry semantics flag in Play --- .../trace/instrumentation/play26/PlayHttpServerDecorator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java b/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java index 7b8e5628986..fc621b75de6 100644 --- a/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java @@ -153,7 +153,7 @@ protected void doOnRequest( PATH_CACHE.computeIfAbsent( defOption.get().path(), p -> addMissingSlash(p, request.path())); HTTP_RESOURCE_DECORATOR.withRoute(span, request.method(), path, true); - if (Config.get().isTraceOtelSemanticsEnabled()) { + if (OTEL_SEMANTICS) { final AgentSpan rootSpan = span.getLocalRootSpan(); if (rootSpan != null && rootSpan != span) { // Set the transport span's route before application code can trigger sampling. From 67188e61a9baa3b74d52a1927a9d56cdf2003059 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Tue, 3 Nov 2026 09:48:23 +0000 Subject: [PATCH 08/34] Share Netty HTTP client parent assignment --- .../netty41/client/HttpClientRequestTracingHandler.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java index e53cb117b26..0b5bc4639af 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java @@ -75,10 +75,10 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann } final AgentSpan parent = activeSpan(); final Context parentContext = Context.current(); + if (ctx.channel().parent() == null) { + ctx.channel().attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(parent); + } if (isNestedClientSpanSuppressed(parentContext, parent)) { - if (ctx.channel().parent() == null) { - ctx.channel().attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(parent); - } DECORATE.injectContext(parentContext, request.headers(), SETTER); ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(parentContext); try { @@ -90,9 +90,6 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann } } } - if (ctx.channel().parent() == null) { - ctx.channel().attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(parent); - } boolean isSecure = SSL_HANDLER != null && ctx.pipeline().get(SSL_HANDLER) != null; NettyHttpClientDecorator decorate = isSecure ? DECORATE_SECURE : DECORATE; From 4a1cbc389b709fecd4c07e91e3c358d00f2e4134 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Tue, 3 Nov 2026 09:51:58 +0000 Subject: [PATCH 09/34] Consolidate WebFlux client test completion and assertions --- .../client/SpringWebfluxHttpClientBase.groovy | 18 ++++--- ...bfluxHttpClientDoAfterTerminateTest.groovy | 8 +-- ...luxHttpClientDoOnSuccessOrErrorTest.groovy | 8 +-- ...ingWebfluxHttpClientDoOnSuccessTest.groovy | 8 +-- ...gWebfluxHttpClientDoOnTerminateTest.groovy | 8 +-- .../bootTest/groovy/SpringWebfluxTest.groovy | 53 ++++++------------- .../client/SpringWebfluxHttpClientBase.groovy | 4 ++ ...bfluxHttpClientDoAfterTerminateTest.groovy | 8 +-- ...luxHttpClientDoOnSuccessOrErrorTest.groovy | 8 +-- ...ingWebfluxHttpClientDoOnSuccessTest.groovy | 8 +-- ...gWebfluxHttpClientDoOnTerminateTest.groovy | 8 +-- 11 files changed, 40 insertions(+), 99 deletions(-) diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBase.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBase.groovy index ea5b6b3b4c0..11c86b54dbf 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBase.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBase.groovy @@ -40,13 +40,7 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes } .block() - if (hasParent) { - blockUntilChildSpansFinished(size(callback ? 2 : 1)) - } - - check() - - consumeResponse(response, method) + finishRequest(response, method, hasParent, callback) } protected static int consumeResponse(ClientResponse response, String method) { @@ -73,6 +67,16 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes return field.get(target) } + protected int finishRequest(ClientResponse response, String method, boolean hasParent, Closure callback) { + if (hasParent) { + blockUntilChildSpansFinished(size(callback ? 2 : 1)) + } + + check() + + consumeResponse(response, method) + } + @Override CharSequence component() { return SpringWebfluxHttpClientDecorator.DECORATE.component() diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy index eb462aa02ba..3cefc17f0a2 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy @@ -23,13 +23,7 @@ class SpringWebfluxHttpClientDoAfterTerminateTest extends SpringWebfluxHttpClien } .block() - if (hasParent) { - blockUntilChildSpansFinished(size(callback ? 2 : 1)) - } - - check() - - consumeResponse(response, method) + finishRequest(response, method, hasParent, callback) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy index e5f78b8a0c4..01ebace29ee 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy @@ -23,13 +23,7 @@ abstract class SpringWebfluxHttpClientDoOnSuccessOrErrorTest extends SpringWebfl } .block() - if (hasParent) { - blockUntilChildSpansFinished(size(callback ? 2 : 1)) - } - - check() - - consumeResponse(response, method) + finishRequest(response, method, hasParent, callback) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy index e5542b0dcdf..5cffc4e6b26 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy @@ -23,13 +23,7 @@ class SpringWebfluxHttpClientDoOnSuccessTest extends SpringWebfluxHttpClientBase } .block() - if (hasParent) { - blockUntilChildSpansFinished(size(callback ? 2 : 1)) - } - - check() - - consumeResponse(response, method) + finishRequest(response, method, hasParent, callback) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy index 32293622d8b..2706301f93f 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy @@ -23,13 +23,7 @@ class SpringWebfluxHttpClientDoOnTerminateTest extends SpringWebfluxHttpClientBa } .block() - if (hasParent) { - blockUntilChildSpansFinished(size(callback ? 2 : 1)) - } - - check() - - consumeResponse(response, method) + finishRequest(response, method, hasParent, callback) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/bootTest/groovy/SpringWebfluxTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/bootTest/groovy/SpringWebfluxTest.groovy index 75fb1275d9f..2d820b8a4da 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/bootTest/groovy/SpringWebfluxTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/bootTest/groovy/SpringWebfluxTest.groovy @@ -78,10 +78,7 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { def traceParent sortSpansByStart() trace(clientSpanCount()) { - traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) - if (!Config.get().isTraceOtelSemanticsEnabled()) { - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) - } + traceParent = clientSpans(it, "GET", URI.create(url)) } trace(2) { span { @@ -162,10 +159,7 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { sortSpansByStart() def traceParent trace(clientSpanCount()) { - traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) - if (!Config.get().isTraceOtelSemanticsEnabled()) { - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) - } + traceParent = clientSpans(it, "GET", URI.create(url)) } trace(3) { span { @@ -258,10 +252,7 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { sortSpansByStart() def traceParent trace(clientSpanCount()) { - traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) - if (!Config.get().isTraceOtelSemanticsEnabled()) { - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) - } + traceParent = clientSpans(it, "GET", URI.create(url)) } trace(3) { span { @@ -308,10 +299,7 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { sortSpansByStart() def traceParent trace(clientSpanCount()) { - traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), 404, true) - if (!Config.get().isTraceOtelSemanticsEnabled()) { - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), 404, true) - } + traceParent = clientSpans(it, "GET", URI.create(url), 404, true) } trace(2) { span { @@ -366,10 +354,7 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { sortSpansByStart() def traceParent trace(clientSpanCount()) { - traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "POST", URI.create(url), 202) - if (!Config.get().isTraceOtelSemanticsEnabled()) { - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "POST", URI.create(url), 202) - } + traceParent = clientSpans(it, "POST", URI.create(url), 202) } trace(3) { span { @@ -433,10 +418,7 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { sortSpansByStart() def traceParent trace(clientSpanCount()) { - traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), 500) - if (!Config.get().isTraceOtelSemanticsEnabled()) { - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), 500) - } + traceParent = clientSpans(it, "GET", URI.create(url), 500) } trace(2) { span { @@ -527,10 +509,7 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { trace(clientSpanCount()) { sortSpansByStart() - traceParent1 = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), 307) - if (!Config.get().isTraceOtelSemanticsEnabled()) { - traceParent1 = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), 307) - } + traceParent1 = clientSpans(it, "GET", URI.create(url), 307) } trace(2) { @@ -575,10 +554,7 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { } trace(clientSpanCount()) { sortSpansByStart() - traceParent2 = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(finalUrl)) - if (!Config.get().isTraceOtelSemanticsEnabled()) { - traceParent2 = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(finalUrl)) - } + traceParent2 = clientSpans(it, "GET", URI.create(finalUrl)) } trace(2) { sortSpansByStart() @@ -637,10 +613,7 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { responses.eachWithIndex { def response, int i -> def traceParent trace(clientSpanCount()) { - traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) - if (!Config.get().isTraceOtelSemanticsEnabled()) { - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) - } + traceParent = clientSpans(it, "GET", URI.create(url)) } trace(2) { span { @@ -843,6 +816,14 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { Config.get().isTraceOtelSemanticsEnabled() ? 1 : 2 } + def clientSpans(TraceAssert trace, String method, URI uri, Integer status = 200, boolean error = false) { + def parent = clientSpan(trace, null, "http.request", "spring-webflux-client", method, uri, status, error) + if (!Config.get().isTraceOtelSemanticsEnabled()) { + parent = clientSpan(trace, parent, "netty.client.request", "netty-client", method, uri, status, error) + } + return parent + } + def clientSpan( TraceAssert trace, Object parentSpan, diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBase.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBase.groovy index 30d80ef78e0..b7185c2dcd3 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBase.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBase.groovy @@ -44,6 +44,10 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes } .block() + finishRequest(response, hasParent, callback) + } + + protected int finishRequest(ClientResponse response, boolean hasParent, Closure callback) { if (hasParent) { blockUntilChildSpansFinished(size(callback ? 2 : 1)) } diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy index 22b3844a67d..3ef6d2eb8bc 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy @@ -29,13 +29,7 @@ class SpringWebfluxHttpClientDoAfterTerminateTest extends SpringWebfluxHttpClien } .block() - if (hasParent) { - blockUntilChildSpansFinished(size(callback ? 2 : 1)) - } - - check() - - response.statusCode().value() + finishRequest(response, hasParent, callback) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy index 8d8b615d909..582ef9aed8c 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy @@ -29,13 +29,7 @@ abstract class SpringWebfluxHttpClientDoOnSuccessOrErrorTest extends SpringWebfl } .block() - if (hasParent) { - blockUntilChildSpansFinished(size(callback ? 2 : 1)) - } - - check() - - response.statusCode().value() + finishRequest(response, hasParent, callback) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy index 6fc3bec70df..3b51ff35fca 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy @@ -29,13 +29,7 @@ class SpringWebfluxHttpClientDoOnSuccessTest extends SpringWebfluxHttpClientBase } .block() - if (hasParent) { - blockUntilChildSpansFinished(size(callback ? 2 : 1)) - } - - check() - - response.statusCode().value() + finishRequest(response, hasParent, callback) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy index cb508b8fbb8..6e035bc0240 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy @@ -29,13 +29,7 @@ class SpringWebfluxHttpClientDoOnTerminateTest extends SpringWebfluxHttpClientBa } .block() - if (hasParent) { - blockUntilChildSpansFinished(size(callback ? 2 : 1)) - } - - check() - - response.statusCode().value() + finishRequest(response, hasParent, callback) } @Override From 8aad14d220c5c1d3fa33ad3cb0cd823f501e005f Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Tue, 3 Nov 2026 11:26:44 +0000 Subject: [PATCH 10/34] Close Netty client parent scope in one place Wrap the request handler in a single try/finally so the AWS and suppressed-span early returns no longer repeat the parent scope close, and derive the client span context from the already-captured parent. --- .../HttpClientRequestTracingHandler.java | 84 ++++++++----------- 1 file changed, 37 insertions(+), 47 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java index 0b5bc4639af..60799663f07 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java @@ -60,64 +60,54 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann parentScope = continuation.resume(); } - final HttpRequest request = (HttpRequest) msg; - boolean awsClientCall = request.headers().contains("amz-sdk-invocation-id"); - if (!AWS_LEGACY_TRACING && awsClientCall) { - // avoid creating an extra HTTP client span beneath the AWS client call - try { + try { + final HttpRequest request = (HttpRequest) msg; + boolean awsClientCall = request.headers().contains("amz-sdk-invocation-id"); + if (!AWS_LEGACY_TRACING && awsClientCall) { + // avoid creating an extra HTTP client span beneath the AWS client call ctx.write(msg, prm); return; - } finally { - if (null != parentScope) { - parentScope.close(); - } } - } - final AgentSpan parent = activeSpan(); - final Context parentContext = Context.current(); - if (ctx.channel().parent() == null) { - ctx.channel().attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(parent); - } - if (isNestedClientSpanSuppressed(parentContext, parent)) { - DECORATE.injectContext(parentContext, request.headers(), SETTER); - ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(parentContext); - try { + final AgentSpan parent = activeSpan(); + final Context parentContext = Context.current(); + if (ctx.channel().parent() == null) { + ctx.channel().attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(parent); + } + if (isNestedClientSpanSuppressed(parentContext, parent)) { + DECORATE.injectContext(parentContext, request.headers(), SETTER); + ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(parentContext); ctx.write(msg, prm); return; - } finally { - if (null != parentScope) { - parentScope.close(); - } } - } - boolean isSecure = SSL_HANDLER != null && ctx.pipeline().get(SSL_HANDLER) != null; - NettyHttpClientDecorator decorate = isSecure ? DECORATE_SECURE : DECORATE; + boolean isSecure = SSL_HANDLER != null && ctx.pipeline().get(SSL_HANDLER) != null; + NettyHttpClientDecorator decorate = isSecure ? DECORATE_SECURE : DECORATE; - final AgentSpan span = startSpan(NETTY_CLIENT.toString(), NETTY_CLIENT_REQUEST); - final Context context = Context.current().with(span); - try (final ContextScope scope = activateSpan(span)) { - decorate.afterStart(span); - decorate.onRequest(span, request); + final AgentSpan span = startSpan(NETTY_CLIENT.toString(), NETTY_CLIENT_REQUEST); + final Context context = parentContext.with(span); + try (final ContextScope scope = activateSpan(span)) { + decorate.afterStart(span); + decorate.onRequest(span, request); - SocketAddress socketAddress = ctx.channel().remoteAddress(); - if (socketAddress instanceof InetSocketAddress) { - decorate.onPeerConnection(span, (InetSocketAddress) socketAddress); - } + SocketAddress socketAddress = ctx.channel().remoteAddress(); + if (socketAddress instanceof InetSocketAddress) { + decorate.onPeerConnection(span, (InetSocketAddress) socketAddress); + } - // AWS calls are often signed, so we can't add headers without breaking the signature. - if (!awsClientCall) { - DECORATE.injectContext(Context.current(), request.headers(), SETTER); - } + // AWS calls are often signed, so we can't add headers without breaking the signature. + if (!awsClientCall) { + DECORATE.injectContext(Context.current(), request.headers(), SETTER); + } - ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(context); + ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(context); - try { - ctx.write(msg, prm); - } catch (final Throwable throwable) { - decorate.onError(span, throwable); - decorate.beforeFinish(span); - span.finish(); - throw throwable; + try { + ctx.write(msg, prm); + } catch (final Throwable throwable) { + decorate.onError(span, throwable); + decorate.beforeFinish(span); + span.finish(); + throw throwable; + } } } finally { if (null != parentScope) { From 69379cc3d9712640c049ab334cc7f58c22547a2e Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Tue, 3 Nov 2026 11:26:49 +0000 Subject: [PATCH 11/34] Defer Akka HTTP path rendering until a matcher succeeds Keep the Uri.Path on each frame and render it only when the match succeeds, since most path matcher attempts fail and never need it. Nested matchers also reuse the root span checked by the outermost frame. --- .../akkahttp/AkkaHttpRouteHelper.java | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java index 27a61c6a03e..1a272ebe642 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java @@ -22,6 +22,13 @@ public final class AkkaHttpRouteHelper { private AkkaHttpRouteHelper() {} public static Frame enter(Uri.Path path) { + Frame parent = CURRENT.get(); + if (parent != null) { + // nested matchers share the root span already checked by the outermost frame + Frame frame = new Frame(path, parent.span, parent); + CURRENT.set(frame); + return frame; + } AgentSpan active = activeSpan(); if (active == null) { return null; @@ -30,7 +37,7 @@ public static Frame enter(Uri.Path path) { if (root == null || root.getTag(KnownTags.URL_PATH_NAME) == null) { return null; } - Frame frame = new Frame(path.toString(), root, CURRENT.get()); + Frame frame = new Frame(path, root, null); CURRENT.set(frame); return frame; } @@ -53,25 +60,26 @@ public static void exit( } PathMatcher.Matched matched = (PathMatcher.Matched) matching; String rest = matched.pathRest().toString(); - int consumed = frame.path.length() - rest.length(); + String path = frame.path(); + int consumed = path.length() - rest.length(); if (consumed <= 0) { return; } String template = frame.template(consumed, matched.extractions()); if (frame.parent != null) { - if (frame.parent.path.endsWith(frame.path)) { + String parentPath = frame.parent.path(); + if (parentPath.endsWith(path)) { if (frame.parent.parts == null) { frame.parent.parts = new ArrayList<>(); } - frame.parent.parts.add( - new Route(frame.parent.path.length() - frame.path.length(), rest, template)); + frame.parent.parts.add(new Route(parentPath.length() - path.length(), rest, template)); } return; } Object requestPath = frame.span.getTag(KnownTags.URL_PATH_NAME); - if (requestPath == null || !requestPath.toString().equals(URIUtils.decode(frame.path))) { + if (requestPath == null || !requestPath.toString().equals(URIUtils.decode(path))) { Route previous = routes.get(frame.span); - if (previous == null || !previous.remaining.equals(frame.path)) { + if (previous == null || !previous.remaining.equals(path)) { return; } template = previous.template + template; @@ -83,18 +91,28 @@ public static void exit( } public static final class Frame { - final String path; + final Uri.Path uriPath; final AgentSpan span; final Frame parent; List parts; + private String path; - Frame(String path, AgentSpan span, Frame parent) { - this.path = path; + Frame(Uri.Path uriPath, AgentSpan span, Frame parent) { + this.uriPath = uriPath; this.span = span; this.parent = parent; } + /** Renders the path on demand, as most match attempts fail and never need it. */ + String path() { + if (path == null) { + path = uriPath.toString(); + } + return path; + } + String template(int consumed, Object extractions) { + String path = path(); if (parts == null) { if (extractions instanceof Product && ((Product) extractions).productArity() > 0) { return path.charAt(0) == '/' ? "/{param}" : "{param}"; From 486b6046e4c21c498540db3af64a5971bc381f33 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Tue, 3 Nov 2026 11:35:48 +0000 Subject: [PATCH 12/34] Return early from Netty client callbacks for suppressed spans When the outer client span owns the request, run the callback under it and return, leaving the existing client span handling unchanged instead of threading the suppression flag through each branch. --- .../HttpClientResponseTracingHandler.java | 51 ++++++++++++------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java index 94443eeff4d..c08e0a13787 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java @@ -30,15 +30,20 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { final AgentSpan parent = parentAttr.get(); final Context storedContext = ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).get(); final AgentSpan span = AgentSpan.fromContext(storedContext); - final boolean clientSpanSuppressed = isNestedClientSpanSuppressed(storedContext, span); - final AgentSpan callbackParent = clientSpanSuppressed ? span : parent; + if (isNestedClientSpanSuppressed(storedContext, span)) { + // The outer client span owns this request, so run the callback under it. + try (final ContextScope scope = activateSpan(span)) { + ctx.fireChannelRead(msg); + } + return; + } // Set parent context back to maintain the same functionality as getAndSet(parent) if (storedContext != null) { - ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(callbackParent)); + ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(parent)); } - if (span != null && !clientSpanSuppressed) { + if (span != null) { final boolean finishSpan = msg instanceof HttpResponse && (!HttpResponseStatus.SWITCHING_PROTOCOLS.equals(((HttpResponse) msg).status()) @@ -57,8 +62,8 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { } } - // Run the callback under the span that initiated the transport request. - try (final ContextScope scope = activateSpan(callbackParent)) { + // We want the callback in the scope of the parent, not the client span + try (final ContextScope scope = activateSpan(parent)) { ctx.fireChannelRead(msg); } } @@ -70,15 +75,20 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E final AgentSpan parent = parentAttr.get(); final Context storedContext = ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).get(); final AgentSpan span = AgentSpan.fromContext(storedContext); - final boolean clientSpanSuppressed = isNestedClientSpanSuppressed(storedContext, span); - final AgentSpan callbackParent = clientSpanSuppressed ? span : parent; + if (isNestedClientSpanSuppressed(storedContext, span)) { + // The outer client span owns this request, so run the callback under it. + try (final ContextScope scope = activateSpan(span)) { + super.exceptionCaught(ctx, cause); + } + return; + } // Set parent context back to maintain the same functionality as getAndSet(parent) if (storedContext != null) { - ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(callbackParent)); + ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(parent)); } - if (span != null && !clientSpanSuppressed) { + if (span != null) { // If an exception is passed to this point, it likely means it was unhandled and the // client span won't be finished with a proper response, so we should finish the span here. try (final ContextScope scope = activateSpan(span)) { @@ -87,8 +97,8 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E span.finish(); } } - // Run the callback under the span that initiated the transport request. - try (final ContextScope scope = activateSpan(callbackParent)) { + // We want the callback in the scope of the parent, not the client span + try (final ContextScope scope = activateSpan(parent)) { super.exceptionCaught(ctx, cause); } } @@ -100,22 +110,27 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { final AgentSpan parent = parentAttr.get(); final Context storedContext = ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).get(); final AgentSpan span = AgentSpan.fromContext(storedContext); - final boolean clientSpanSuppressed = isNestedClientSpanSuppressed(storedContext, span); - final AgentSpan callbackParent = clientSpanSuppressed ? span : parent; + if (isNestedClientSpanSuppressed(storedContext, span)) { + // The outer client span owns this request, so run the callback under it. + try (final ContextScope scope = activateSpan(span)) { + super.channelInactive(ctx); + } + return; + } // Set parent context back to maintain the same functionality as getAndSet(parent) if (storedContext != null) { - ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(callbackParent)); + ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(parent)); } - if (span != null && span != parent && !clientSpanSuppressed) { + if (span != null && span != parent) { try (final ContextScope scope = activateSpan(span)) { DECORATE.beforeFinish(span); span.finish(); } } - // Run the callback under the span that initiated the transport request. - try (final ContextScope scope = activateSpan(callbackParent)) { + // We want the callback in the scope of the parent, not the client span + try (final ContextScope scope = activateSpan(parent)) { super.channelInactive(ctx); } } From 30b1b422cd4127b4cdaa95c050dfe6db577ed423 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Tue, 3 Nov 2026 11:35:50 +0000 Subject: [PATCH 13/34] Capture Pekko HTTP route templates in OpenTelemetry mode Mirror the Akka HTTP path matcher instrumentation for Pekko HTTP so successful matchers build a route on the local root HTTP server span, and add a forked test that verifies stable routes across parameters. --- .../PekkoHttpServerInstrumentationTest.groovy | 31 ++++ .../PekkoHttpPathMatcherInstrumentation.java | 93 +++++++++++ .../pekkohttp/PekkoHttpRouteHelper.java | 147 ++++++++++++++++++ 3 files changed, 271 insertions(+) create mode 100644 dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java create mode 100644 dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy index 1034cf83e26..a1e4f3174ce 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy @@ -1,6 +1,8 @@ import datadog.trace.agent.test.base.HttpServer import datadog.trace.agent.test.base.HttpServerTest +import datadog.trace.agent.test.base.WithHttpServer import datadog.trace.agent.test.naming.TestingGenericHttpNamingConventions +import datadog.trace.agent.test.naming.TestingOtelHttpSemantics import datadog.trace.api.config.TraceInstrumentationConfig import datadog.trace.test.util.ThreadUtils import datadog.trace.instrumentation.pekkohttp.PekkoHttpServerDecorator @@ -108,6 +110,35 @@ class PekkoHttpServerInstrumentationSyncV0ForkedTest extends PekkoHttpServerInst class PekkoHttpServerInstrumentationSyncV1ForkedTest extends PekkoHttpServerInstrumentationSyncTest implements TestingGenericHttpNamingConventions.ServerV1 { } +class PekkoHttpRouteOtelSemanticsForkedTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0, TestingOtelHttpSemantics { + @Override + HttpServer server() { + new PekkoHttpTestWebServer(PekkoHttpTestWebServer.BindAndHandle()) + } + + def "matched paths produce stable server routes"() { + when: + def response = client.newCall(new Request.Builder().url(address.resolve(path).toString()).build()).execute() + assert response.code() == 200 + response.close() + TEST_WRITER.waitForTraces(1) + + then: + def serverSpan = TEST_WRITER.flatten().find { it.getTag('component')?.toString() == 'pekko-http-server' } + serverSpan != null + serverSpan.getTag('http.route') == route + serverSpan.resourceName.toString() == "GET $route" + + where: + path | route + '/success' | '/success' + '/query?some=value' | '/query' + '/injected-id/ping/1' | '/injected-id/ping/{param}' + '/injected-id/ping/2' | '/injected-id/ping/{param}' + '/injected-id/fing/3' | '/injected-id/fing/{param}' + } +} + class PekkoHttpServerInstrumentationAsyncTest extends PekkoHttpServerInstrumentationTest { @Override HttpServer server() { diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java new file mode 100644 index 00000000000..d9a9c37fbde --- /dev/null +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java @@ -0,0 +1,93 @@ +package datadog.trace.instrumentation.pekkohttp; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.extendsClass; +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static java.util.Collections.singletonMap; +import static net.bytebuddy.matcher.ElementMatchers.isBridge; +import static net.bytebuddy.matcher.ElementMatchers.not; +import static net.bytebuddy.matcher.ElementMatchers.returns; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.api.Config; +import datadog.trace.bootstrap.InstrumentationContext; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import java.util.Map; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; +import org.apache.pekko.http.scaladsl.model.Uri; +import org.apache.pekko.http.scaladsl.server.PathMatcher; + +/** Captures route templates from successful Pekko HTTP path matchers. */ +@AutoService(InstrumenterModule.class) +public final class PekkoHttpPathMatcherInstrumentation extends InstrumenterModule.Tracing + implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { + public PekkoHttpPathMatcherInstrumentation() { + super("pekko-http", "pekko-http-server"); + } + + @Override + public boolean isEnabled() { + return super.isEnabled() && Config.get().isTraceOtelSemanticsEnabled(); + } + + @Override + public String hierarchyMarkerType() { + return "org.apache.pekko.http.scaladsl.server.PathMatcher"; + } + + @Override + public ElementMatcher hierarchyMatcher() { + return extendsClass(named(hierarchyMarkerType())); + } + + @Override + public Map contextStore() { + return singletonMap( + "datadog.trace.bootstrap.instrumentation.api.AgentSpan", + packageName + ".PekkoHttpRouteHelper$Route"); + } + + @Override + public String[] helperClassNames() { + return new String[] { + packageName + ".PekkoHttpRouteHelper", + packageName + ".PekkoHttpRouteHelper$Frame", + packageName + ".PekkoHttpRouteHelper$Route" + }; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + named("apply") + .and(takesArguments(1)) + .and(takesArgument(0, named("org.apache.pekko.http.scaladsl.model.Uri$Path"))) + .and(returns(named("org.apache.pekko.http.scaladsl.server.PathMatcher$Matching"))) + .and(not(isBridge())), + getClass().getName() + "$MatchAdvice"); + } + + public static final class MatchAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static PekkoHttpRouteHelper.Frame enter(@Advice.Argument(0) Uri.Path path) { + return PekkoHttpRouteHelper.enter(path); + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void exit( + @Advice.Enter PekkoHttpRouteHelper.Frame frame, + @Advice.Return PathMatcher.Matching matching, + @Advice.Thrown Throwable throwable) { + PekkoHttpRouteHelper.exit( + frame, + matching, + throwable, + InstrumentationContext.get(AgentSpan.class, PekkoHttpRouteHelper.Route.class)); + } + } +} diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java new file mode 100644 index 00000000000..1d259c173c6 --- /dev/null +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java @@ -0,0 +1,147 @@ +package datadog.trace.instrumentation.pekkohttp; + +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; +import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR; + +import datadog.trace.api.KnownTags; +import datadog.trace.bootstrap.ContextStore; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.URIUtils; +import java.util.ArrayList; +import java.util.List; +import org.apache.pekko.http.scaladsl.model.Uri; +import org.apache.pekko.http.scaladsl.server.PathMatcher; +import scala.Product; + +/** + * Builds templates from matched literals and extracted parameters, retaining no parameter values. + */ +public final class PekkoHttpRouteHelper { + private static final ThreadLocal CURRENT = new ThreadLocal<>(); + + private PekkoHttpRouteHelper() {} + + public static Frame enter(Uri.Path path) { + Frame parent = CURRENT.get(); + if (parent != null) { + // nested matchers share the root span already checked by the outermost frame + Frame frame = new Frame(path, parent.span, parent); + CURRENT.set(frame); + return frame; + } + AgentSpan active = activeSpan(); + if (active == null) { + return null; + } + AgentSpan root = active.getLocalRootSpan(); + if (root == null || root.getTag(KnownTags.URL_PATH_NAME) == null) { + return null; + } + Frame frame = new Frame(path, root, null); + CURRENT.set(frame); + return frame; + } + + public static void exit( + Frame frame, + PathMatcher.Matching matching, + Throwable throwable, + ContextStore routes) { + if (frame == null) { + return; + } + if (frame.parent == null) { + CURRENT.remove(); + } else { + CURRENT.set(frame.parent); + } + if (throwable != null || !(matching instanceof PathMatcher.Matched)) { + return; + } + PathMatcher.Matched matched = (PathMatcher.Matched) matching; + String rest = matched.pathRest().toString(); + String path = frame.path(); + int consumed = path.length() - rest.length(); + if (consumed <= 0) { + return; + } + String template = frame.template(consumed, matched.extractions()); + if (frame.parent != null) { + String parentPath = frame.parent.path(); + if (parentPath.endsWith(path)) { + if (frame.parent.parts == null) { + frame.parent.parts = new ArrayList<>(); + } + frame.parent.parts.add(new Route(parentPath.length() - path.length(), rest, template)); + } + return; + } + Object requestPath = frame.span.getTag(KnownTags.URL_PATH_NAME); + if (requestPath == null || !requestPath.toString().equals(URIUtils.decode(path))) { + Route previous = routes.get(frame.span); + if (previous == null || !previous.remaining.equals(path)) { + return; + } + template = previous.template + template; + } + routes.put(frame.span, new Route(0, rest, template)); + Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); + HTTP_RESOURCE_DECORATOR.withRoute( + frame.span, method == null ? null : method.toString(), template, true); + } + + public static final class Frame { + final Uri.Path uriPath; + final AgentSpan span; + final Frame parent; + List parts; + private String path; + + Frame(Uri.Path uriPath, AgentSpan span, Frame parent) { + this.uriPath = uriPath; + this.span = span; + this.parent = parent; + } + + /** Renders the path on demand, as most match attempts fail and never need it. */ + String path() { + if (path == null) { + path = uriPath.toString(); + } + return path; + } + + String template(int consumed, Object extractions) { + String path = path(); + if (parts == null) { + if (extractions instanceof Product && ((Product) extractions).productArity() > 0) { + return path.charAt(0) == '/' ? "/{param}" : "{param}"; + } + return path.substring(0, consumed); + } + StringBuilder template = new StringBuilder(); + int cursor = 0; + for (Route part : parts) { + int end = path.length() - part.remaining.length(); + if (part.offset < cursor || end > consumed) { + continue; + } + template.append(path, cursor, part.offset).append(part.template); + cursor = end; + } + return template.append(path, cursor, consumed).toString(); + } + } + + public static final class Route { + final int offset; + final String remaining; + final String template; + + Route(int offset, String remaining, String template) { + this.offset = offset; + this.remaining = remaining; + this.template = template; + } + } +} From 7f38168e418f312152eb316cd49ad39969793c08 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Tue, 3 Nov 2026 11:35:51 +0000 Subject: [PATCH 14/34] Apply Play 2.3 and 2.4 routes to root HTTP spans in OpenTelemetry mode Mirror the Play 2.6 change so the underlying HTTP server span also gets the matched route for OpenTelemetry naming. --- .../play23/PlayHttpServerDecorator.java | 11 +++++++++-- .../play24/PlayHttpServerDecorator.java | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java b/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java index b1589452e46..c447a7fe0e7 100644 --- a/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java @@ -105,14 +105,21 @@ protected void doOnRequest( if (!pathOption.isEmpty()) { final String path = (String) pathOption.get(); HTTP_RESOURCE_DECORATOR.withRoute(span, request.method(), path); + if (OTEL_SEMANTICS) { + final AgentSpan rootSpan = span.getLocalRootSpan(); + if (rootSpan != null && rootSpan != span) { + // Set the transport span's route before application code can trigger sampling. + HTTP_RESOURCE_DECORATOR.withRoute(rootSpan, request.method(), path); + } + } dispatchRoute(span, path); } } } /** - * Play does not set the http.route in the local root span so we need to store it in the context - * for API security + * Store the route in the context for API security, including when legacy semantics leave it only + * on the Play span. */ private void dispatchRoute(final AgentSpan span, final String route) { try { diff --git a/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java b/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java index e5e80e81d05..e7948f90164 100644 --- a/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java @@ -105,14 +105,21 @@ protected void doOnRequest( if (!pathOption.isEmpty()) { final String path = (String) pathOption.get(); HTTP_RESOURCE_DECORATOR.withRoute(span, request.method(), path); + if (OTEL_SEMANTICS) { + final AgentSpan rootSpan = span.getLocalRootSpan(); + if (rootSpan != null && rootSpan != span) { + // Set the transport span's route before application code can trigger sampling. + HTTP_RESOURCE_DECORATOR.withRoute(rootSpan, request.method(), path); + } + } dispatchRoute(span, path); } } } /** - * Play does not set the http.route in the local root span so we need to store it in the context - * for API security + * Store the route in the context for API security, including when legacy semantics leave it only + * on the Play span. */ private void dispatchRoute(final AgentSpan span, final String route) { try { From 46862bbde38dd346771b13aca056453e516891f7 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Tue, 3 Nov 2026 12:44:38 +0000 Subject: [PATCH 15/34] Fix HTTP route templates after rejected Akka and Pekko alternatives --- .../AkkaHttpServerInstrumentationTest.groovy | 1 + .../baseTest/scala/AkkaHttpTestWebServer.scala | 3 +++ .../akkahttp/AkkaHttpRouteHelper.java | 17 ++++++++++++++--- .../PekkoHttpServerInstrumentationTest.groovy | 1 + .../baseTest/scala/PekkoHttpTestWebServer.scala | 3 +++ .../pekkohttp/PekkoHttpRouteHelper.java | 17 ++++++++++++++--- 6 files changed, 36 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy index 3c5c4155e72..90054031741 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy @@ -306,6 +306,7 @@ class AkkaHttpRouteOtelSemanticsForkedTest extends WithHttpServer reject } ~ + path(IntNumber / "accepted") { id => complete(s"$id") } } ~ pathPrefix("injected-id") { path("ping" / IntNumber) { id => val traceId = AgentTracer.activeSpan().getTraceId diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java index 1a272ebe642..6d3c66ea52d 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java @@ -76,15 +76,20 @@ public static void exit( } return; } + Route previous = null; Object requestPath = frame.span.getTag(KnownTags.URL_PATH_NAME); if (requestPath == null || !requestPath.toString().equals(URIUtils.decode(path))) { - Route previous = routes.get(frame.span); - if (previous == null || !previous.remaining.equals(path)) { + previous = routes.get(frame.span); + // A rejected route can retry an earlier unmatched path in another alternative. + while (previous != null && !previous.remaining.equals(path)) { + previous = previous.previous; + } + if (previous == null) { return; } template = previous.template + template; } - routes.put(frame.span, new Route(0, rest, template)); + routes.put(frame.span, new Route(0, rest, template, previous)); Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); HTTP_RESOURCE_DECORATOR.withRoute( frame.span, method == null ? null : method.toString(), template, true); @@ -137,11 +142,17 @@ public static final class Route { final int offset; final String remaining; final String template; + final Route previous; Route(int offset, String remaining, String template) { + this(offset, remaining, template, null); + } + + Route(int offset, String remaining, String template, Route previous) { this.offset = offset; this.remaining = remaining; this.template = template; + this.previous = previous; } } } diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy index a1e4f3174ce..efedbe5961e 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy @@ -136,6 +136,7 @@ class PekkoHttpRouteOtelSemanticsForkedTest extends WithHttpServer reject } ~ + path(IntNumber / "accepted") { id => complete(s"$id") } } ~ pathPrefix("injected-id") { path("ping" / IntNumber) { id => val traceId = AgentTracer.activeSpan().getTraceId diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java index 1d259c173c6..2aa47a023d6 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java @@ -76,15 +76,20 @@ public static void exit( } return; } + Route previous = null; Object requestPath = frame.span.getTag(KnownTags.URL_PATH_NAME); if (requestPath == null || !requestPath.toString().equals(URIUtils.decode(path))) { - Route previous = routes.get(frame.span); - if (previous == null || !previous.remaining.equals(path)) { + previous = routes.get(frame.span); + // A rejected route can retry an earlier unmatched path in another alternative. + while (previous != null && !previous.remaining.equals(path)) { + previous = previous.previous; + } + if (previous == null) { return; } template = previous.template + template; } - routes.put(frame.span, new Route(0, rest, template)); + routes.put(frame.span, new Route(0, rest, template, previous)); Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); HTTP_RESOURCE_DECORATOR.withRoute( frame.span, method == null ? null : method.toString(), template, true); @@ -137,11 +142,17 @@ public static final class Route { final int offset; final String remaining; final String template; + final Route previous; Route(int offset, String remaining, String template) { + this(offset, remaining, template, null); + } + + Route(int offset, String remaining, String template, Route previous) { this.offset = offset; this.remaining = remaining; this.template = template; + this.previous = previous; } } } From 6f52ba37b363e8f539ce3cb8979ac87b173e6ee5 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Tue, 3 Nov 2026 12:49:01 +0000 Subject: [PATCH 16/34] Move Akka and Pekko HTTP route state into propagated context --- .../AkkaHttpServerInstrumentationTest.groovy | 62 ++++++++++++++++--- .../scala/AkkaHttpTestWebServer.scala | 6 ++ .../AkkaHttpPathMatcherInstrumentation.java | 26 +------- .../akkahttp/AkkaHttpRouteHelper.java | 42 ++++--------- .../akkahttp/AkkaHttpRouteState.java | 43 +++++++++++++ .../akkahttp/DatadogWrapperHelper.java | 2 +- .../PekkoHttpServerInstrumentationTest.groovy | 62 ++++++++++++++++--- .../scala/PekkoHttpTestWebServer.scala | 6 ++ .../pekkohttp/DatadogWrapperHelper.java | 2 +- .../PekkoHttpPathMatcherInstrumentation.java | 26 +------- .../pekkohttp/PekkoHttpRouteHelper.java | 42 ++++--------- .../pekkohttp/PekkoHttpRouteState.java | 43 +++++++++++++ 12 files changed, 238 insertions(+), 124 deletions(-) create mode 100644 dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java create mode 100644 dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy index 90054031741..63f5b6dd8c8 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy @@ -280,7 +280,13 @@ class AkkaHttpServerInstrumentationSyncV1ForkedTest extends AkkaHttpServerInstru class AkkaHttpServerInstrumentationSyncOtelSemanticsForkedTest extends AkkaHttpServerInstrumentationSyncV0ForkedTest implements TestingOtelHttpSemantics { } -class AkkaHttpRouteOtelSemanticsForkedTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0, TestingOtelHttpSemantics { +abstract class AkkaHttpRouteOtelSemanticsTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0, TestingOtelHttpSemantics { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig(TraceInstrumentationConfig.LEGACY_CONTEXT_MANAGER_ENABLED, "true") + } + @Override HttpServer server() { new AkkaHttpTestWebServer(AkkaHttpTestWebServer.BindAndHandle()) @@ -300,14 +306,56 @@ class AkkaHttpRouteOtelSemanticsForkedTest extends WithHttpServer + String route = span.getTag('url.path').toString().startsWith('/route-async/') ? + '/route-async/{param}/detail/{param}' : '/route-alternative/{param}/accepted' + span.getTag('http.route') == route && span.resourceName.toString() == "GET $route" + } + } +} + +class AkkaHttpRouteOtelSemanticsForkedTest extends AkkaHttpRouteOtelSemanticsTest { +} + +class AkkaHttpRouteOtelSemanticsContextSwapForkedTest extends AkkaHttpRouteOtelSemanticsTest { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig(TraceInstrumentationConfig.LEGACY_CONTEXT_MANAGER_ENABLED, "false") + } } class AkkaHttpServerInstrumentationAsyncTest extends AkkaHttpServerInstrumentationTest { diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/scala/AkkaHttpTestWebServer.scala b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/scala/AkkaHttpTestWebServer.scala index 274c9845673..41472cd0e44 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/scala/AkkaHttpTestWebServer.scala +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/scala/AkkaHttpTestWebServer.scala @@ -241,6 +241,12 @@ object AkkaHttpTestWebServer { complete(HttpResponse(status = ERROR.getStatus, entity = ERROR.getBody)) } ~ path(EXCEPTION.relativePath) { throw new Exception(EXCEPTION.getBody) + } ~ pathPrefix("route-async" / IntNumber) { id => + onSuccess(Future { id }) { _ => + path("detail" / IntNumber) { detail => + complete(s"$id:$detail") + } + } } ~ pathPrefix("route-alternative") { pathPrefix(IntNumber) { _ => reject } ~ path(IntNumber / "accepted") { id => complete(s"$id") } diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java index 1cd43014118..13bdd4728ef 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java @@ -2,7 +2,6 @@ import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.extendsClass; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static java.util.Collections.singletonMap; import static net.bytebuddy.matcher.ElementMatchers.isBridge; import static net.bytebuddy.matcher.ElementMatchers.not; import static net.bytebuddy.matcher.ElementMatchers.returns; @@ -15,9 +14,6 @@ import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.api.Config; -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import java.util.Map; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.matcher.ElementMatcher; @@ -45,22 +41,6 @@ public ElementMatcher hierarchyMatcher() { return extendsClass(named(hierarchyMarkerType())); } - @Override - public Map contextStore() { - return singletonMap( - "datadog.trace.bootstrap.instrumentation.api.AgentSpan", - packageName + ".AkkaHttpRouteHelper$Route"); - } - - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".AkkaHttpRouteHelper", - packageName + ".AkkaHttpRouteHelper$Frame", - packageName + ".AkkaHttpRouteHelper$Route" - }; - } - @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( @@ -83,11 +63,7 @@ public static void exit( @Advice.Enter AkkaHttpRouteHelper.Frame frame, @Advice.Return PathMatcher.Matching matching, @Advice.Thrown Throwable throwable) { - AkkaHttpRouteHelper.exit( - frame, - matching, - throwable, - InstrumentationContext.get(AgentSpan.class, AkkaHttpRouteHelper.Route.class)); + AkkaHttpRouteHelper.exit(frame, matching, throwable); } } } diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java index 6d3c66ea52d..0453c446762 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java @@ -6,9 +6,9 @@ import akka.http.scaladsl.model.Uri; import akka.http.scaladsl.server.PathMatcher; import datadog.trace.api.KnownTags; -import datadog.trace.bootstrap.ContextStore; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.URIUtils; +import datadog.trace.instrumentation.akkahttp.AkkaHttpRouteState.Route; import java.util.ArrayList; import java.util.List; import scala.Product; @@ -25,10 +25,14 @@ public static Frame enter(Uri.Path path) { Frame parent = CURRENT.get(); if (parent != null) { // nested matchers share the root span already checked by the outermost frame - Frame frame = new Frame(path, parent.span, parent); + Frame frame = new Frame(path, parent.span, parent.state, parent); CURRENT.set(frame); return frame; } + AkkaHttpRouteState state = AkkaHttpRouteState.current(); + if (state == null) { + return null; + } AgentSpan active = activeSpan(); if (active == null) { return null; @@ -37,16 +41,12 @@ public static Frame enter(Uri.Path path) { if (root == null || root.getTag(KnownTags.URL_PATH_NAME) == null) { return null; } - Frame frame = new Frame(path, root, null); + Frame frame = new Frame(path, root, state, null); CURRENT.set(frame); return frame; } - public static void exit( - Frame frame, - PathMatcher.Matching matching, - Throwable throwable, - ContextStore routes) { + public static void exit(Frame frame, PathMatcher.Matching matching, Throwable throwable) { if (frame == null) { return; } @@ -79,7 +79,7 @@ public static void exit( Route previous = null; Object requestPath = frame.span.getTag(KnownTags.URL_PATH_NAME); if (requestPath == null || !requestPath.toString().equals(URIUtils.decode(path))) { - previous = routes.get(frame.span); + previous = frame.state.route; // A rejected route can retry an earlier unmatched path in another alternative. while (previous != null && !previous.remaining.equals(path)) { previous = previous.previous; @@ -89,7 +89,7 @@ public static void exit( } template = previous.template + template; } - routes.put(frame.span, new Route(0, rest, template, previous)); + frame.state.route = new Route(0, rest, template, previous); Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); HTTP_RESOURCE_DECORATOR.withRoute( frame.span, method == null ? null : method.toString(), template, true); @@ -98,13 +98,15 @@ public static void exit( public static final class Frame { final Uri.Path uriPath; final AgentSpan span; + final AkkaHttpRouteState state; final Frame parent; List parts; private String path; - Frame(Uri.Path uriPath, AgentSpan span, Frame parent) { + Frame(Uri.Path uriPath, AgentSpan span, AkkaHttpRouteState state, Frame parent) { this.uriPath = uriPath; this.span = span; + this.state = state; this.parent = parent; } @@ -137,22 +139,4 @@ String template(int consumed, Object extractions) { return template.append(path, cursor, consumed).toString(); } } - - public static final class Route { - final int offset; - final String remaining; - final String template; - final Route previous; - - Route(int offset, String remaining, String template) { - this(offset, remaining, template, null); - } - - Route(int offset, String remaining, String template, Route previous) { - this.offset = offset; - this.remaining = remaining; - this.template = template; - this.previous = previous; - } - } } diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java new file mode 100644 index 00000000000..cc995c886fe --- /dev/null +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java @@ -0,0 +1,43 @@ +package datadog.trace.instrumentation.akkahttp; + +import static datadog.context.ContextKey.named; + +import datadog.context.Context; +import datadog.context.ContextKey; +import datadog.trace.api.Config; + +/** Request-local route state, independent of the HTTP routing DSL. */ +public final class AkkaHttpRouteState { + private static final boolean OTEL_SEMANTICS = Config.get().isTraceOtelSemanticsEnabled(); + private static final ContextKey KEY = named("akka-http-route"); + + Route route; + + private AkkaHttpRouteState() {} + + public static Context init(Context context) { + return OTEL_SEMANTICS ? context.with(KEY, new AkkaHttpRouteState()) : context; + } + + static AkkaHttpRouteState current() { + return Context.current().get(KEY); + } + + public static final class Route { + final int offset; + final String remaining; + final String template; + final Route previous; + + Route(int offset, String remaining, String template) { + this(offset, remaining, template, null); + } + + Route(int offset, String remaining, String template, Route previous) { + this.offset = offset; + this.remaining = remaining; + this.template = template; + this.previous = previous; + } + } +} diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/DatadogWrapperHelper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/DatadogWrapperHelper.java index cf0a5d68258..1829657b59b 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/DatadogWrapperHelper.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/DatadogWrapperHelper.java @@ -33,7 +33,7 @@ private static Context startSpan(final HttpRequest request) { DECORATE.afterStart(span); DECORATE.onRequest(span, request, request, parentContext); - return context; + return AkkaHttpRouteState.init(context); } static final class SwappedContextScope implements ContextScope { diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy index efedbe5961e..c1a5cccf4e4 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy @@ -110,7 +110,13 @@ class PekkoHttpServerInstrumentationSyncV0ForkedTest extends PekkoHttpServerInst class PekkoHttpServerInstrumentationSyncV1ForkedTest extends PekkoHttpServerInstrumentationSyncTest implements TestingGenericHttpNamingConventions.ServerV1 { } -class PekkoHttpRouteOtelSemanticsForkedTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0, TestingOtelHttpSemantics { +abstract class PekkoHttpRouteOtelSemanticsTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0, TestingOtelHttpSemantics { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig(TraceInstrumentationConfig.LEGACY_CONTEXT_MANAGER_ENABLED, "true") + } + @Override HttpServer server() { new PekkoHttpTestWebServer(PekkoHttpTestWebServer.BindAndHandle()) @@ -130,14 +136,56 @@ class PekkoHttpRouteOtelSemanticsForkedTest extends WithHttpServer + String route = span.getTag('url.path').toString().startsWith('/route-async/') ? + '/route-async/{param}/detail/{param}' : '/route-alternative/{param}/accepted' + span.getTag('http.route') == route && span.resourceName.toString() == "GET $route" + } + } +} + +class PekkoHttpRouteOtelSemanticsForkedTest extends PekkoHttpRouteOtelSemanticsTest { +} + +class PekkoHttpRouteOtelSemanticsContextSwapForkedTest extends PekkoHttpRouteOtelSemanticsTest { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig(TraceInstrumentationConfig.LEGACY_CONTEXT_MANAGER_ENABLED, "false") + } } class PekkoHttpServerInstrumentationAsyncTest extends PekkoHttpServerInstrumentationTest { diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/scala/PekkoHttpTestWebServer.scala b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/scala/PekkoHttpTestWebServer.scala index da00fb413c4..301ce37d8be 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/scala/PekkoHttpTestWebServer.scala +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/scala/PekkoHttpTestWebServer.scala @@ -207,6 +207,12 @@ object PekkoHttpTestWebServer { complete(HttpResponse(status = ERROR.getStatus, entity = ERROR.getBody)) } ~ path(EXCEPTION.relativePath) { throw new Exception(EXCEPTION.getBody) + } ~ pathPrefix("route-async" / IntNumber) { id => + onSuccess(Future { id }) { _ => + path("detail" / IntNumber) { detail => + complete(s"$id:$detail") + } + } } ~ pathPrefix("route-alternative") { pathPrefix(IntNumber) { _ => reject } ~ path(IntNumber / "accepted") { id => complete(s"$id") } diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/DatadogWrapperHelper.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/DatadogWrapperHelper.java index 7766d5018e6..af62ebe08a5 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/DatadogWrapperHelper.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/DatadogWrapperHelper.java @@ -33,7 +33,7 @@ private static Context startSpan(final HttpRequest request) { DECORATE.afterStart(span); DECORATE.onRequest(span, request, request, parentContext); - return context; + return PekkoHttpRouteState.init(context); } static final class SwappedContextScope implements ContextScope { diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java index d9a9c37fbde..28df92ed87b 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java @@ -2,7 +2,6 @@ import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.extendsClass; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static java.util.Collections.singletonMap; import static net.bytebuddy.matcher.ElementMatchers.isBridge; import static net.bytebuddy.matcher.ElementMatchers.not; import static net.bytebuddy.matcher.ElementMatchers.returns; @@ -13,9 +12,6 @@ import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.api.Config; -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import java.util.Map; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.matcher.ElementMatcher; @@ -45,22 +41,6 @@ public ElementMatcher hierarchyMatcher() { return extendsClass(named(hierarchyMarkerType())); } - @Override - public Map contextStore() { - return singletonMap( - "datadog.trace.bootstrap.instrumentation.api.AgentSpan", - packageName + ".PekkoHttpRouteHelper$Route"); - } - - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".PekkoHttpRouteHelper", - packageName + ".PekkoHttpRouteHelper$Frame", - packageName + ".PekkoHttpRouteHelper$Route" - }; - } - @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( @@ -83,11 +63,7 @@ public static void exit( @Advice.Enter PekkoHttpRouteHelper.Frame frame, @Advice.Return PathMatcher.Matching matching, @Advice.Thrown Throwable throwable) { - PekkoHttpRouteHelper.exit( - frame, - matching, - throwable, - InstrumentationContext.get(AgentSpan.class, PekkoHttpRouteHelper.Route.class)); + PekkoHttpRouteHelper.exit(frame, matching, throwable); } } } diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java index 2aa47a023d6..8226a328998 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java @@ -4,9 +4,9 @@ import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR; import datadog.trace.api.KnownTags; -import datadog.trace.bootstrap.ContextStore; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.URIUtils; +import datadog.trace.instrumentation.pekkohttp.PekkoHttpRouteState.Route; import java.util.ArrayList; import java.util.List; import org.apache.pekko.http.scaladsl.model.Uri; @@ -25,10 +25,14 @@ public static Frame enter(Uri.Path path) { Frame parent = CURRENT.get(); if (parent != null) { // nested matchers share the root span already checked by the outermost frame - Frame frame = new Frame(path, parent.span, parent); + Frame frame = new Frame(path, parent.span, parent.state, parent); CURRENT.set(frame); return frame; } + PekkoHttpRouteState state = PekkoHttpRouteState.current(); + if (state == null) { + return null; + } AgentSpan active = activeSpan(); if (active == null) { return null; @@ -37,16 +41,12 @@ public static Frame enter(Uri.Path path) { if (root == null || root.getTag(KnownTags.URL_PATH_NAME) == null) { return null; } - Frame frame = new Frame(path, root, null); + Frame frame = new Frame(path, root, state, null); CURRENT.set(frame); return frame; } - public static void exit( - Frame frame, - PathMatcher.Matching matching, - Throwable throwable, - ContextStore routes) { + public static void exit(Frame frame, PathMatcher.Matching matching, Throwable throwable) { if (frame == null) { return; } @@ -79,7 +79,7 @@ public static void exit( Route previous = null; Object requestPath = frame.span.getTag(KnownTags.URL_PATH_NAME); if (requestPath == null || !requestPath.toString().equals(URIUtils.decode(path))) { - previous = routes.get(frame.span); + previous = frame.state.route; // A rejected route can retry an earlier unmatched path in another alternative. while (previous != null && !previous.remaining.equals(path)) { previous = previous.previous; @@ -89,7 +89,7 @@ public static void exit( } template = previous.template + template; } - routes.put(frame.span, new Route(0, rest, template, previous)); + frame.state.route = new Route(0, rest, template, previous); Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); HTTP_RESOURCE_DECORATOR.withRoute( frame.span, method == null ? null : method.toString(), template, true); @@ -98,13 +98,15 @@ public static void exit( public static final class Frame { final Uri.Path uriPath; final AgentSpan span; + final PekkoHttpRouteState state; final Frame parent; List parts; private String path; - Frame(Uri.Path uriPath, AgentSpan span, Frame parent) { + Frame(Uri.Path uriPath, AgentSpan span, PekkoHttpRouteState state, Frame parent) { this.uriPath = uriPath; this.span = span; + this.state = state; this.parent = parent; } @@ -137,22 +139,4 @@ String template(int consumed, Object extractions) { return template.append(path, cursor, consumed).toString(); } } - - public static final class Route { - final int offset; - final String remaining; - final String template; - final Route previous; - - Route(int offset, String remaining, String template) { - this(offset, remaining, template, null); - } - - Route(int offset, String remaining, String template, Route previous) { - this.offset = offset; - this.remaining = remaining; - this.template = template; - this.previous = previous; - } - } } diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java new file mode 100644 index 00000000000..0e23609b35b --- /dev/null +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java @@ -0,0 +1,43 @@ +package datadog.trace.instrumentation.pekkohttp; + +import static datadog.context.ContextKey.named; + +import datadog.context.Context; +import datadog.context.ContextKey; +import datadog.trace.api.Config; + +/** Request-local route state, independent of the HTTP routing DSL. */ +public final class PekkoHttpRouteState { + private static final boolean OTEL_SEMANTICS = Config.get().isTraceOtelSemanticsEnabled(); + private static final ContextKey KEY = named("pekko-http-route"); + + Route route; + + private PekkoHttpRouteState() {} + + public static Context init(Context context) { + return OTEL_SEMANTICS ? context.with(KEY, new PekkoHttpRouteState()) : context; + } + + static PekkoHttpRouteState current() { + return Context.current().get(KEY); + } + + public static final class Route { + final int offset; + final String remaining; + final String template; + final Route previous; + + Route(int offset, String remaining, String template) { + this(offset, remaining, template, null); + } + + Route(int offset, String remaining, String template, Route previous) { + this.offset = offset; + this.remaining = remaining; + this.template = template; + this.previous = previous; + } + } +} From 5a4acd77a51486b9c423740c1a5b0d7b75454054 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Tue, 3 Nov 2026 14:19:15 +0000 Subject: [PATCH 17/34] Drop Akka and Pekko HTTP routes from rejected requests A path prefix that matched before the request was rejected left its partial template on the span, so 404s under a valid prefix were reported as real endpoints such as "GET /users/{param}". Follow the OpenTelemetry Akka HTTP rule: the route only stands if the last outermost path matcher tried also matched. Otherwise, when the response is written, remove http.route and restore the method-only resource that OpenTelemetry semantics give unrouted requests. --- .../AkkaHttpServerInstrumentationTest.groovy | 23 +++++++++++++++++ .../akkahttp/AkkaHttpRouteHelper.java | 4 ++- .../akkahttp/AkkaHttpRouteState.java | 25 +++++++++++++++++++ .../akkahttp/DatadogWrapperHelper.java | 2 ++ .../PekkoHttpServerInstrumentationTest.groovy | 23 +++++++++++++++++ .../pekkohttp/DatadogWrapperHelper.java | 2 ++ .../pekkohttp/PekkoHttpRouteHelper.java | 4 ++- .../pekkohttp/PekkoHttpRouteState.java | 25 +++++++++++++++++++ 8 files changed, 106 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy index 63f5b6dd8c8..b03e9af4e10 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy @@ -316,6 +316,29 @@ abstract class AkkaHttpRouteOtelSemanticsTest extends WithHttpServer matching, Throwable if (frame == null) { return; } + boolean isMatch = throwable == null && matching instanceof PathMatcher.Matched; if (frame.parent == null) { CURRENT.remove(); + frame.state.lastMatched = isMatch; } else { CURRENT.set(frame.parent); } - if (throwable != null || !(matching instanceof PathMatcher.Matched)) { + if (!isMatch) { return; } PathMatcher.Matched matched = (PathMatcher.Matched) matching; diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java index cc995c886fe..603f20ae328 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java @@ -1,10 +1,14 @@ package datadog.trace.instrumentation.akkahttp; import static datadog.context.ContextKey.named; +import static datadog.trace.bootstrap.instrumentation.api.OtelHttpSemantics.spanNameMethod; +import static datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities.HTTP_FRAMEWORK_ROUTE; import datadog.context.Context; import datadog.context.ContextKey; import datadog.trace.api.Config; +import datadog.trace.api.KnownTags; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; /** Request-local route state, independent of the HTTP routing DSL. */ public final class AkkaHttpRouteState { @@ -13,6 +17,9 @@ public final class AkkaHttpRouteState { Route route; + /** Whether the last outermost path matcher tried matched; OTel drops the route otherwise. */ + boolean lastMatched; + private AkkaHttpRouteState() {} public static Context init(Context context) { @@ -23,6 +30,24 @@ static AkkaHttpRouteState current() { return Context.current().get(KEY); } + /** Drops the route of a request rejected after its last path matcher failed. */ + public static void dropRejectedRoute(Context context, AgentSpan span) { + AkkaHttpRouteState state = context.get(KEY); + if (state == null || state.route == null || state.lastMatched) { + return; + } + state.route = null; + AgentSpan root = span.getLocalRootSpan(); + if (root == null) { + root = span; + } + root.setTag(KnownTags.HTTP_ROUTE_ID, (String) null); + Object method = root.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); + // restore the method-only resource that OTel semantics give unrouted requests + root.setResourceName( + spanNameMethod(method == null ? null : method.toString()), HTTP_FRAMEWORK_ROUTE); + } + public static final class Route { final int offset; final String remaining; diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/DatadogWrapperHelper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/DatadogWrapperHelper.java index 1829657b59b..a537a49cb81 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/DatadogWrapperHelper.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/DatadogWrapperHelper.java @@ -64,6 +64,7 @@ public void close() { public static void finishSpan(final Context context, final HttpResponse response) { final AgentSpan span = fromContext(context); + AkkaHttpRouteState.dropRejectedRoute(context, span); DECORATE.onResponse(span, response); DECORATE.beforeFinish(context); @@ -72,6 +73,7 @@ public static void finishSpan(final Context context, final HttpResponse response public static void finishSpan(final Context context, final Throwable t) { final AgentSpan span = fromContext(context); + AkkaHttpRouteState.dropRejectedRoute(context, span); DECORATE.onError(span, t); span.setHttpStatusCode(500); DECORATE.beforeFinish(context); diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy index c1a5cccf4e4..5458c3eb46e 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy @@ -146,6 +146,29 @@ abstract class PekkoHttpRouteOtelSemanticsTest extends WithHttpServer matching, Throwable if (frame == null) { return; } + boolean isMatch = throwable == null && matching instanceof PathMatcher.Matched; if (frame.parent == null) { CURRENT.remove(); + frame.state.lastMatched = isMatch; } else { CURRENT.set(frame.parent); } - if (throwable != null || !(matching instanceof PathMatcher.Matched)) { + if (!isMatch) { return; } PathMatcher.Matched matched = (PathMatcher.Matched) matching; diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java index 0e23609b35b..45d18aa6818 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java @@ -1,10 +1,14 @@ package datadog.trace.instrumentation.pekkohttp; import static datadog.context.ContextKey.named; +import static datadog.trace.bootstrap.instrumentation.api.OtelHttpSemantics.spanNameMethod; +import static datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities.HTTP_FRAMEWORK_ROUTE; import datadog.context.Context; import datadog.context.ContextKey; import datadog.trace.api.Config; +import datadog.trace.api.KnownTags; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; /** Request-local route state, independent of the HTTP routing DSL. */ public final class PekkoHttpRouteState { @@ -13,6 +17,9 @@ public final class PekkoHttpRouteState { Route route; + /** Whether the last outermost path matcher tried matched; OTel drops the route otherwise. */ + boolean lastMatched; + private PekkoHttpRouteState() {} public static Context init(Context context) { @@ -23,6 +30,24 @@ static PekkoHttpRouteState current() { return Context.current().get(KEY); } + /** Drops the route of a request rejected after its last path matcher failed. */ + public static void dropRejectedRoute(Context context, AgentSpan span) { + PekkoHttpRouteState state = context.get(KEY); + if (state == null || state.route == null || state.lastMatched) { + return; + } + state.route = null; + AgentSpan root = span.getLocalRootSpan(); + if (root == null) { + root = span; + } + root.setTag(KnownTags.HTTP_ROUTE_ID, (String) null); + Object method = root.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); + // restore the method-only resource that OTel semantics give unrouted requests + root.setResourceName( + spanNameMethod(method == null ? null : method.toString()), HTTP_FRAMEWORK_ROUTE); + } + public static final class Route { final int offset; final String remaining; From 9a9fc10b3bab90792b63ac7795b48d813aa80c6c Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 3 Nov 2026 15:11:56 -0800 Subject: [PATCH 18/34] Preserve transport SSRF checks for suppressed Netty client spans --- .github/CODEOWNERS | 1 + .../decorator/HttpClientDecorator.java | 21 +++++ .../HttpClientRequestTracingHandler.java | 6 +- .../SuppressedClientSecurityForkedTest.java | 91 +++++++++++++++++++ 4 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSecurityForkedTest.java diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1decc476fda..0ba9c3950b8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -247,6 +247,7 @@ /dd-java-agent/instrumentation/rs/**/MessageBodyWriterInstrumentation.java @DataDog/apm-java @DataDog/asm-java # netty-4.1 blocking glue +/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/**/SuppressedClientSecurityForkedTest.java @DataDog/apm-java @DataDog/apm-idm-java @DataDog/asm-java /dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/**/NettyMultipartHelper.java @DataDog/apm-java @DataDog/asm-java /dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/**/HttpPostRequestDecoderInstrumentation.java @DataDog/apm-java @DataDog/asm-java /dd-java-agent/instrumentation/netty/**/server/BlockingResponseHandler.java @DataDog/apm-java @DataDog/asm-java diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java index 40e3a3d2970..d2cd4349f7f 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java @@ -131,6 +131,27 @@ public final void onRequest(final AgentSpan span, final REQUEST request) { } } + /** Checks the transport destination without decorating a span owned by another HTTP client. */ + public final void onRequestSecurity(final AgentSpan span, final REQUEST request) { + if (request == null) { + return; + } + try { + if (APPSEC_RASP_ENABLED) { + URI destination = url(request); + if (destination != null) { + onHttpClientRequest(span, destination.toString()); + } + } + } catch (BlockingException e) { + throw e; + } catch (Throwable t) { + log.debug("Failed to check HTTP transport destination", t); + } finally { + ssrfIastCheck(request); + } + } + protected void doOnRequest(final AgentSpan span, final REQUEST request) { if (request != null) { AgentTracer.get() diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java index 60799663f07..d6a4ba86115 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java @@ -73,15 +73,15 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann if (ctx.channel().parent() == null) { ctx.channel().attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(parent); } + boolean isSecure = SSL_HANDLER != null && ctx.pipeline().get(SSL_HANDLER) != null; + NettyHttpClientDecorator decorate = isSecure ? DECORATE_SECURE : DECORATE; if (isNestedClientSpanSuppressed(parentContext, parent)) { + decorate.onRequestSecurity(parent, request); DECORATE.injectContext(parentContext, request.headers(), SETTER); ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(parentContext); ctx.write(msg, prm); return; } - boolean isSecure = SSL_HANDLER != null && ctx.pipeline().get(SSL_HANDLER) != null; - NettyHttpClientDecorator decorate = isSecure ? DECORATE_SECURE : DECORATE; - final AgentSpan span = startSpan(NETTY_CLIENT.toString(), NETTY_CLIENT_REQUEST); final Context context = parentContext.with(span); try (final ContextScope scope = activateSpan(span)) { diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSecurityForkedTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSecurityForkedTest.java new file mode 100644 index 00000000000..13a2b4b86a5 --- /dev/null +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSecurityForkedTest.java @@ -0,0 +1,91 @@ +package datadog.trace.instrumentation.netty41.client; + +import static datadog.trace.agent.test.assertions.SpanMatcher.span; +import static datadog.trace.agent.test.assertions.TraceMatcher.trace; +import static datadog.trace.api.gateway.Events.EVENTS; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator.suppressNestedClientSpans; +import static io.netty.handler.codec.http.HttpMethod.GET; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import datadog.appsec.api.blocking.BlockingContentType; +import datadog.appsec.api.blocking.BlockingException; +import datadog.context.ContextScope; +import datadog.trace.agent.test.AbstractInstrumentationTest; +import datadog.trace.api.appsec.HttpClientRequest; +import datadog.trace.api.gateway.Flow; +import datadog.trace.api.gateway.RequestContext; +import datadog.trace.api.gateway.RequestContextSlot; +import datadog.trace.api.gateway.Subscription; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.TagContext; +import datadog.trace.test.junit.utils.config.WithConfig; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.http.DefaultHttpRequest; +import io.netty.handler.codec.http.HttpRequest; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import org.tabletest.junit.TableTest; + +@WithConfig(key = "trace.otel.semantics.enabled", value = "true") +@WithConfig(key = "appsec.enabled", value = "true") +@WithConfig(key = "appsec.rasp.enabled", value = "true") +class SuppressedClientSecurityForkedTest extends AbstractInstrumentationTest { + @TableTest({ + "Scenario | Blocked", + "allowed | false ", + "blocked | true " + }) + void checksTransportDestinationWithoutChangingOwnerMetadata(boolean blocked) { + AtomicReference checkedUrl = new AtomicReference<>(); + Subscription subscription = + AgentTracer.get() + .getSubscriptionService(RequestContextSlot.APPSEC) + .registerCallback( + EVENTS.httpClientRequest(), + new BiFunction>() { + @Override + public Flow apply(RequestContext context, HttpClientRequest request) { + checkedUrl.set(request.getUrl()); + return blocked + ? new Flow.ResultFlow(null) { + @Override + public Action getAction() { + return new Action.RequestBlockingAction(403, BlockingContentType.AUTO); + } + } + : Flow.ResultFlow.empty(); + } + }); + AgentSpan owner = + AgentTracer.get() + .startSpan( + "test", "webclient", new TagContext().withRequestContextDataAppSec(new Object())); + owner.setTag("url.full", "http://original.example/original"); + EmbeddedChannel channel = new EmbeddedChannel(HttpClientRequestTracingHandler.INSTANCE); + HttpRequest request = new DefaultHttpRequest(HTTP_1_1, GET, "/rewritten"); + request.headers().set("Host", "transport.example"); + try (ContextScope scope = activateSpan(owner); + ContextScope suppression = suppressNestedClientSpans(owner)) { + if (blocked) { + assertThrows(BlockingException.class, () -> channel.writeOutbound(request)); + assertNull(channel.readOutbound()); + } else { + channel.writeOutbound(request); + assertSame(request, channel.readOutbound()); + } + assertEquals("http://transport.example/rewritten", checkedUrl.get()); + assertEquals("http://original.example/original", owner.getTag("url.full")); + } finally { + subscription.cancel(); + channel.finishAndReleaseAll(); + owner.finish(); + } + assertTraces(trace(span().root().operationName("webclient"))); + } +} From c72412357505cb569f9cfff5fe8b490a57e0e5b4 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 3 Nov 2026 15:12:51 -0800 Subject: [PATCH 19/34] Match encoded Akka and Pekko route paths consistently --- .../groovy/AkkaHttpServerInstrumentationTest.groovy | 11 +++++++++++ .../src/baseTest/scala/AkkaHttpTestWebServer.scala | 2 ++ .../instrumentation/akkahttp/AkkaHttpRouteHelper.java | 5 ++++- .../groovy/PekkoHttpServerInstrumentationTest.groovy | 11 +++++++++++ .../src/baseTest/scala/PekkoHttpTestWebServer.scala | 2 ++ .../pekkohttp/PekkoHttpRouteHelper.java | 5 ++++- 6 files changed, 34 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy index b03e9af4e10..fc3060c13ae 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy @@ -308,6 +308,9 @@ abstract class AkkaHttpRouteOtelSemanticsTest extends WithHttpServer + complete("item") } ~ pathPrefix("route-async" / IntNumber) { id => onSuccess(Future { id }) { _ => path("detail" / IntNumber) { detail => diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java index fb07ff40855..e3192894132 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java @@ -5,6 +5,7 @@ import akka.http.scaladsl.model.Uri; import akka.http.scaladsl.server.PathMatcher; +import datadog.trace.api.Config; import datadog.trace.api.KnownTags; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.URIUtils; @@ -17,6 +18,7 @@ * Builds templates from matched literals and extracted parameters, retaining no parameter values. */ public final class AkkaHttpRouteHelper { + private static final boolean RAW_RESOURCE = Config.get().isHttpServerRawResource(); private static final ThreadLocal CURRENT = new ThreadLocal<>(); private AkkaHttpRouteHelper() {} @@ -80,7 +82,8 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable } Route previous = null; Object requestPath = frame.span.getTag(KnownTags.URL_PATH_NAME); - if (requestPath == null || !requestPath.toString().equals(URIUtils.decode(path))) { + if (requestPath == null + || !requestPath.toString().equals(RAW_RESOURCE ? path : URIUtils.decode(path))) { previous = frame.state.route; // A rejected route can retry an earlier unmatched path in another alternative. while (previous != null && !previous.remaining.equals(path)) { diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy index 5458c3eb46e..dd8d5fff837 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy @@ -138,6 +138,9 @@ abstract class PekkoHttpRouteOtelSemanticsTest extends WithHttpServer + complete("item") } ~ pathPrefix("route-async" / IntNumber) { id => onSuccess(Future { id }) { _ => path("detail" / IntNumber) { detail => diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java index 7bb37ad0aa1..3ab3b58ee96 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java @@ -3,6 +3,7 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR; +import datadog.trace.api.Config; import datadog.trace.api.KnownTags; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.URIUtils; @@ -17,6 +18,7 @@ * Builds templates from matched literals and extracted parameters, retaining no parameter values. */ public final class PekkoHttpRouteHelper { + private static final boolean RAW_RESOURCE = Config.get().isHttpServerRawResource(); private static final ThreadLocal CURRENT = new ThreadLocal<>(); private PekkoHttpRouteHelper() {} @@ -80,7 +82,8 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable } Route previous = null; Object requestPath = frame.span.getTag(KnownTags.URL_PATH_NAME); - if (requestPath == null || !requestPath.toString().equals(URIUtils.decode(path))) { + if (requestPath == null + || !requestPath.toString().equals(RAW_RESOURCE ? path : URIUtils.decode(path))) { previous = frame.state.route; // A rejected route can retry an earlier unmatched path in another alternative. while (previous != null && !previous.remaining.equals(path)) { From 57321162d35ce10b4b040d08ff6aa554495716de Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 3 Nov 2026 15:14:42 -0800 Subject: [PATCH 20/34] Restore upstream context after suppressed HTTP response headers --- .github/CODEOWNERS | 2 +- .../decorator/HttpClientDecorator.java | 17 +- .../HttpClientResponseTracingHandler.java | 25 +- .../SuppressedClientSecurityForkedTest.java | 91 ------- .../SuppressedClientSpanForkedTest.java | 250 ++++++++++++++++++ .../client/WebClientTracingFilter.java | 4 +- 6 files changed, 293 insertions(+), 96 deletions(-) delete mode 100644 dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSecurityForkedTest.java create mode 100644 dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSpanForkedTest.java diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0ba9c3950b8..b5daeda821f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -247,7 +247,7 @@ /dd-java-agent/instrumentation/rs/**/MessageBodyWriterInstrumentation.java @DataDog/apm-java @DataDog/asm-java # netty-4.1 blocking glue -/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/**/SuppressedClientSecurityForkedTest.java @DataDog/apm-java @DataDog/apm-idm-java @DataDog/asm-java +/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/**/SuppressedClientSpanForkedTest.java @DataDog/apm-java @DataDog/apm-idm-java @DataDog/asm-java /dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/**/NettyMultipartHelper.java @DataDog/apm-java @DataDog/asm-java /dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/**/HttpPostRequestDecoderInstrumentation.java @DataDog/apm-java @DataDog/asm-java /dd-java-agent/instrumentation/netty/**/server/BlockingResponseHandler.java @DataDog/apm-java @DataDog/asm-java diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java index d2cd4349f7f..9b6ad3364ea 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java @@ -45,6 +45,8 @@ public abstract class HttpClientDecorator extends UriBasedCli private static final ContextKey CLIENT_SPAN_KEY = ContextKey.named("dd-http-client-span"); + private static final ContextKey CLIENT_PARENT_CONTEXT_KEY = + ContextKey.named("dd-http-client-parent-context"); private static final String DATADOG_META_LANG_HEADER_NAME = "Datadog-Meta-Lang"; private static final String DD_CLIENT_LIBRARY_LANGUAGE_HEADER_NAME = "DD-Client-Library-Language"; @@ -61,8 +63,19 @@ public abstract class HttpClientDecorator extends UriBasedCli private static final boolean APPSEC_RASP_ENABLED = Config.get().isAppSecRaspEnabled(); /** Suppresses duplicate transport spans when OpenTelemetry semantics are enabled. */ - public static ContextScope suppressNestedClientSpans(final AgentSpan span) { - return OTEL_SEMANTICS ? Context.current().with(CLIENT_SPAN_KEY, span).attach() : null; + public static ContextScope suppressNestedClientSpans( + final AgentSpan span, final Context parentContext) { + return OTEL_SEMANTICS + ? Context.current() + .with(CLIENT_SPAN_KEY, span) + .with(CLIENT_PARENT_CONTEXT_KEY, parentContext) + .attach() + : null; + } + + /** Returns the context to restore after the owning client's response headers arrive. */ + public static Context nestedClientParentContext(final Context context) { + return context.get(CLIENT_PARENT_CONTEXT_KEY); } /** Returns whether {@code span} owns the HTTP request represented by {@code context}. */ diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java index c08e0a13787..b9bbaa78489 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java @@ -3,6 +3,7 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.noopSpan; import static datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator.isNestedClientSpanSuppressed; +import static datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator.nestedClientParentContext; import static datadog.trace.instrumentation.netty41.AttributeKeys.CLIENT_PARENT_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.client.NettyHttpClientDecorator.DECORATE; @@ -34,6 +35,12 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { // The outer client span owns this request, so run the callback under it. try (final ContextScope scope = activateSpan(span)) { ctx.fireChannelRead(msg); + } finally { + if (msg instanceof HttpResponse + && (((HttpResponse) msg).status().code() >= 200 + || ((HttpResponse) msg).status().code() == 101)) { + restoreParentContext(ctx, storedContext, parent); + } } return; } @@ -79,6 +86,8 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E // The outer client span owns this request, so run the callback under it. try (final ContextScope scope = activateSpan(span)) { super.exceptionCaught(ctx, cause); + } finally { + restoreParentContext(ctx, storedContext, parent); } return; } @@ -88,7 +97,7 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(parent)); } - if (span != null) { + if (span != null && span != parent) { // If an exception is passed to this point, it likely means it was unhandled and the // client span won't be finished with a proper response, so we should finish the span here. try (final ContextScope scope = activateSpan(span)) { @@ -114,6 +123,8 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { // The outer client span owns this request, so run the callback under it. try (final ContextScope scope = activateSpan(span)) { super.channelInactive(ctx); + } finally { + restoreParentContext(ctx, storedContext, parent); } return; } @@ -134,4 +145,16 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { super.channelInactive(ctx); } } + + private static void restoreParentContext( + ChannelHandlerContext ctx, Context context, AgentSpan previousParent) { + Context parentContext = nestedClientParentContext(context); + if (parentContext != null + && ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).compareAndSet(context, parentContext)) { + AgentSpan parent = AgentSpan.fromContext(parentContext); + ctx.channel() + .attr(CLIENT_PARENT_ATTRIBUTE_KEY) + .compareAndSet(previousParent, parent == null ? noopSpan() : parent); + } + } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSecurityForkedTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSecurityForkedTest.java deleted file mode 100644 index 13a2b4b86a5..00000000000 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSecurityForkedTest.java +++ /dev/null @@ -1,91 +0,0 @@ -package datadog.trace.instrumentation.netty41.client; - -import static datadog.trace.agent.test.assertions.SpanMatcher.span; -import static datadog.trace.agent.test.assertions.TraceMatcher.trace; -import static datadog.trace.api.gateway.Events.EVENTS; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; -import static datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator.suppressNestedClientSpans; -import static io.netty.handler.codec.http.HttpMethod.GET; -import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import datadog.appsec.api.blocking.BlockingContentType; -import datadog.appsec.api.blocking.BlockingException; -import datadog.context.ContextScope; -import datadog.trace.agent.test.AbstractInstrumentationTest; -import datadog.trace.api.appsec.HttpClientRequest; -import datadog.trace.api.gateway.Flow; -import datadog.trace.api.gateway.RequestContext; -import datadog.trace.api.gateway.RequestContextSlot; -import datadog.trace.api.gateway.Subscription; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import datadog.trace.bootstrap.instrumentation.api.TagContext; -import datadog.trace.test.junit.utils.config.WithConfig; -import io.netty.channel.embedded.EmbeddedChannel; -import io.netty.handler.codec.http.DefaultHttpRequest; -import io.netty.handler.codec.http.HttpRequest; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.BiFunction; -import org.tabletest.junit.TableTest; - -@WithConfig(key = "trace.otel.semantics.enabled", value = "true") -@WithConfig(key = "appsec.enabled", value = "true") -@WithConfig(key = "appsec.rasp.enabled", value = "true") -class SuppressedClientSecurityForkedTest extends AbstractInstrumentationTest { - @TableTest({ - "Scenario | Blocked", - "allowed | false ", - "blocked | true " - }) - void checksTransportDestinationWithoutChangingOwnerMetadata(boolean blocked) { - AtomicReference checkedUrl = new AtomicReference<>(); - Subscription subscription = - AgentTracer.get() - .getSubscriptionService(RequestContextSlot.APPSEC) - .registerCallback( - EVENTS.httpClientRequest(), - new BiFunction>() { - @Override - public Flow apply(RequestContext context, HttpClientRequest request) { - checkedUrl.set(request.getUrl()); - return blocked - ? new Flow.ResultFlow(null) { - @Override - public Action getAction() { - return new Action.RequestBlockingAction(403, BlockingContentType.AUTO); - } - } - : Flow.ResultFlow.empty(); - } - }); - AgentSpan owner = - AgentTracer.get() - .startSpan( - "test", "webclient", new TagContext().withRequestContextDataAppSec(new Object())); - owner.setTag("url.full", "http://original.example/original"); - EmbeddedChannel channel = new EmbeddedChannel(HttpClientRequestTracingHandler.INSTANCE); - HttpRequest request = new DefaultHttpRequest(HTTP_1_1, GET, "/rewritten"); - request.headers().set("Host", "transport.example"); - try (ContextScope scope = activateSpan(owner); - ContextScope suppression = suppressNestedClientSpans(owner)) { - if (blocked) { - assertThrows(BlockingException.class, () -> channel.writeOutbound(request)); - assertNull(channel.readOutbound()); - } else { - channel.writeOutbound(request); - assertSame(request, channel.readOutbound()); - } - assertEquals("http://transport.example/rewritten", checkedUrl.get()); - assertEquals("http://original.example/original", owner.getTag("url.full")); - } finally { - subscription.cancel(); - channel.finishAndReleaseAll(); - owner.finish(); - } - assertTraces(trace(span().root().operationName("webclient"))); - } -} diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSpanForkedTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSpanForkedTest.java new file mode 100644 index 00000000000..6ef0bf7a065 --- /dev/null +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSpanForkedTest.java @@ -0,0 +1,250 @@ +package datadog.trace.instrumentation.netty41.client; + +import static datadog.trace.agent.test.assertions.SpanMatcher.span; +import static datadog.trace.agent.test.assertions.TraceMatcher.trace; +import static datadog.trace.api.gateway.Events.EVENTS; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; +import static datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator.suppressNestedClientSpans; +import static datadog.trace.instrumentation.netty41.AttributeKeys.CLIENT_PARENT_ATTRIBUTE_KEY; +import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; +import static io.netty.handler.codec.http.HttpMethod.GET; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import datadog.appsec.api.blocking.BlockingContentType; +import datadog.appsec.api.blocking.BlockingException; +import datadog.context.Context; +import datadog.context.ContextScope; +import datadog.trace.agent.test.AbstractInstrumentationTest; +import datadog.trace.api.appsec.HttpClientRequest; +import datadog.trace.api.gateway.Flow; +import datadog.trace.api.gateway.RequestContext; +import datadog.trace.api.gateway.RequestContextSlot; +import datadog.trace.api.gateway.Subscription; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.TagContext; +import datadog.trace.core.DDSpan; +import datadog.trace.test.junit.utils.config.WithConfig; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.http.DefaultHttpContent; +import io.netty.handler.codec.http.DefaultHttpRequest; +import io.netty.handler.codec.http.DefaultHttpResponse; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; +import org.junit.jupiter.api.Test; +import org.tabletest.junit.TableTest; + +@WithConfig(key = "trace.otel.semantics.enabled", value = "true") +@WithConfig(key = "appsec.enabled", value = "true") +@WithConfig(key = "appsec.rasp.enabled", value = "true") +class SuppressedClientSpanForkedTest extends AbstractInstrumentationTest { + @Test + void restoresUpstreamContextAfterAnErrorBeforeHeaders() { + AgentSpan parent = AgentTracer.startSpan("test", "parent"); + AgentSpan owner = AgentTracer.startSpan("test", "webclient", parent.spanContext()); + Context parentContext = Context.current().with(parent); + List observed = new ArrayList<>(); + EmbeddedChannel channel = + new EmbeddedChannel( + new HttpClientTracingHandler(), + new ChannelInboundHandlerAdapter() { + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable error) { + observed.add(activeSpan()); + owner.finish(); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) { + observed.add(activeSpan()); + } + }); + try { + try (ContextScope scope = activateSpan(owner); + ContextScope suppression = suppressNestedClientSpans(owner, parentContext)) { + channel.writeOutbound(new DefaultHttpRequest(HTTP_1_1, GET, "/test")); + } + channel.pipeline().fireExceptionCaught(new IOException("connection failure")); + assertSame(parentContext, channel.attr(CONTEXT_ATTRIBUTE_KEY).get()); + channel.close(); + assertSame(owner, observed.get(0)); + assertSame(parent, observed.get(1)); + assertFalse(((DDSpan) parent).isFinished()); + } finally { + channel.finishAndReleaseAll(); + owner.finish(); + parent.finish(); + } + } + + @Test + void preservesANewRequestStartedFromTheResponseCallback() { + AgentSpan owner = AgentTracer.startSpan("test", "first-request"); + AgentSpan nextOwner = AgentTracer.startSpan("test", "next-request"); + EmbeddedChannel channel = + new EmbeddedChannel( + new HttpClientTracingHandler(), + new ChannelInboundHandlerAdapter() { + @Override + public void channelRead(ChannelHandlerContext ctx, Object message) { + owner.finish(); + try (ContextScope scope = activateSpan(nextOwner); + ContextScope suppression = + suppressNestedClientSpans(nextOwner, Context.root())) { + ctx.channel().writeAndFlush(new DefaultHttpRequest(HTTP_1_1, GET, "/next")); + } + } + }); + try { + try (ContextScope scope = activateSpan(owner); + ContextScope suppression = suppressNestedClientSpans(owner, Context.root())) { + channel.writeOutbound(new DefaultHttpRequest(HTTP_1_1, GET, "/first")); + } + channel.writeInbound(new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.OK)); + assertSame(nextOwner, AgentSpan.fromContext(channel.attr(CONTEXT_ATTRIBUTE_KEY).get())); + } finally { + channel.finishAndReleaseAll(); + owner.finish(); + nextOwner.finish(); + } + } + + @TableTest({ + "Scenario | Response status | Stream channel", + "response headers | 200 | false ", + "protocol upgrade | 101 | false ", + "multiplexed stream | 200 | true " + }) + void restoresUpstreamContextForBodyErrorsAndClose(int responseStatus, boolean streamChannel) { + AgentSpan parent = AgentTracer.startSpan("test", "parent"); + AgentSpan owner = AgentTracer.startSpan("test", "webclient", parent.spanContext()); + Context parentContext = Context.current().with(parent); + List observed = new ArrayList<>(); + EmbeddedChannel channel = + new EmbeddedChannel( + new HttpClientTracingHandler(), + new ChannelInboundHandlerAdapter() { + @Override + public void channelRead(ChannelHandlerContext ctx, Object message) { + observed.add(activeSpan()); + if (message instanceof HttpResponse + && ((HttpResponse) message).status().code() == responseStatus) { + owner.finish(); + } + ctx.fireChannelRead(message); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable error) { + observed.add(activeSpan()); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) { + observed.add(activeSpan()); + } + }); + try { + try (ContextScope scope = activateSpan(owner); + ContextScope suppression = suppressNestedClientSpans(owner, parentContext)) { + channel.writeOutbound(new DefaultHttpRequest(HTTP_1_1, GET, "/test")); + channel.readOutbound(); + } + if (streamChannel) { + // Stream channels inherit the request context without a client-parent attribute. + channel.attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(null); + } + channel.writeInbound(new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.CONTINUE)); + assertSame(owner, AgentSpan.fromContext(channel.attr(CONTEXT_ATTRIBUTE_KEY).get())); + channel.writeInbound( + new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.valueOf(responseStatus))); + assertSame(parentContext, channel.attr(CONTEXT_ATTRIBUTE_KEY).get()); + channel.writeInbound(new DefaultHttpContent(Unpooled.buffer().writeByte(1))); + channel.pipeline().fireExceptionCaught(new IOException("body failure")); + channel.close(); + assertEquals(5, observed.size()); + assertSame(owner, observed.get(0)); + assertSame(owner, observed.get(1)); + for (int i = 2; i < observed.size(); i++) { + assertSame(parent, observed.get(i)); + } + assertFalse(((DDSpan) parent).isFinished()); + } finally { + channel.finishAndReleaseAll(); + owner.finish(); + parent.finish(); + } + assertTraces( + trace( + span().root().operationName("parent"), + span().childOfPrevious().operationName("webclient"))); + } + + @TableTest({ + "Scenario | Blocked", + "allowed | false ", + "blocked | true " + }) + void checksTransportDestinationWithoutChangingOwnerMetadata(boolean blocked) { + AtomicReference checkedUrl = new AtomicReference<>(); + Subscription subscription = + AgentTracer.get() + .getSubscriptionService(RequestContextSlot.APPSEC) + .registerCallback( + EVENTS.httpClientRequest(), + new BiFunction>() { + @Override + public Flow apply(RequestContext context, HttpClientRequest request) { + checkedUrl.set(request.getUrl()); + return blocked + ? new Flow.ResultFlow(null) { + @Override + public Action getAction() { + return new Action.RequestBlockingAction(403, BlockingContentType.AUTO); + } + } + : Flow.ResultFlow.empty(); + } + }); + AgentSpan owner = + AgentTracer.get() + .startSpan( + "test", "webclient", new TagContext().withRequestContextDataAppSec(new Object())); + owner.setTag("url.full", "http://original.example/original"); + EmbeddedChannel channel = new EmbeddedChannel(HttpClientRequestTracingHandler.INSTANCE); + HttpRequest request = new DefaultHttpRequest(HTTP_1_1, GET, "/rewritten"); + request.headers().set("Host", "transport.example"); + try (ContextScope scope = activateSpan(owner); + ContextScope suppression = suppressNestedClientSpans(owner, Context.root())) { + if (blocked) { + assertThrows(BlockingException.class, () -> channel.writeOutbound(request)); + assertNull(channel.readOutbound()); + } else { + channel.writeOutbound(request); + assertSame(request, channel.readOutbound()); + } + assertEquals("http://transport.example/rewritten", checkedUrl.get()); + assertEquals("http://original.example/original", owner.getTag("url.full")); + } finally { + subscription.cancel(); + channel.finishAndReleaseAll(); + owner.finish(); + } + assertTraces(trace(span().root().operationName("webclient"))); + } +} diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java index b4b4e4961ac..1fbe0bafc7f 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java @@ -7,6 +7,7 @@ import static datadog.trace.instrumentation.springwebflux.client.SpringWebfluxHttpClientDecorator.DECORATE; import static datadog.trace.instrumentation.springwebflux.client.SpringWebfluxHttpClientDecorator.HTTP_REQUEST; +import datadog.context.Context; import datadog.context.ContextScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import java.util.List; @@ -55,8 +56,9 @@ public void subscribe(final CoreSubscriber subscriber) { DECORATE.afterStart(span); DECORATE.onRequest(span, request); AgentSpan parent = activeSpan(); + Context parentContext = Context.current(); try (final ContextScope scope = activateSpan(span); - final ContextScope suppressScope = suppressNestedClientSpans(span)) { + final ContextScope suppressScope = suppressNestedClientSpans(span, parentContext)) { TraceWebClientSubscriber tracingSubscriber = new TraceWebClientSubscriber(subscriber, span, parent); next.exchange(request).doOnCancel(tracingSubscriber::onCancel).subscribe(tracingSubscriber); From 8c5dec8525ce81b6121fd3e36a1b95f787f4e319 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 3 Nov 2026 15:34:44 -0800 Subject: [PATCH 21/34] Discard Akka and Pekko routes rejected by later directives --- .../bytebuddy/matcher/ignored_class_name.trie | 1 + .../AkkaHttpServerInstrumentationTest.groovy | 18 ++++++++++++++++++ .../baseTest/scala/AkkaHttpTestWebServer.scala | 10 ++++++++++ .../AkkaHttpPathMatcherInstrumentation.java | 17 +++++++++++++++-- .../akkahttp/AkkaHttpRouteHelper.java | 2 +- .../akkahttp/AkkaHttpRouteState.java | 15 +++++++++++---- .../PekkoHttpServerInstrumentationTest.groovy | 18 ++++++++++++++++++ .../scala/PekkoHttpTestWebServer.scala | 10 ++++++++++ .../PekkoHttpPathMatcherInstrumentation.java | 18 ++++++++++++++++-- .../pekkohttp/PekkoHttpRouteHelper.java | 2 +- .../pekkohttp/PekkoHttpRouteState.java | 15 +++++++++++---- 11 files changed, 112 insertions(+), 14 deletions(-) diff --git a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie index e1bb3d199ae..0c142f9cb58 100644 --- a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie +++ b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie @@ -165,6 +165,7 @@ # Path matchers supply route templates when OTel HTTP semantics are enabled 0 akka.http.scaladsl.server.PathMatcher* 0 akka.http.scaladsl.server.ImplicitPathMatcherConstruction* +0 akka.http.scaladsl.server.RouteResult$Rejected 0 akka.http.scaladsl.server.directives.ParameterDirectives$class 0 akka.http.scaladsl.server.directives.ParameterDirectives 0 akka.http.scaladsl.server.directives.BasicDirectives$class diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy index fc3060c13ae..42e1fc1abbb 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy @@ -308,6 +308,7 @@ abstract class AkkaHttpRouteOtelSemanticsTest extends WithHttpServer + onSuccess(Future { 1 }) { _ => + parameter("required") { _ => complete("accepted") } + } + } ~ complete("fallback") + } ~ pathPrefix("header-rejection") { + path(Segment) { _ => + headerValueByName("authorization") { _ => complete("accepted") } + } ~ complete("fallback") } ~ path("items" / Segment) { _ => complete("item") } ~ pathPrefix("route-async" / IntNumber) { id => diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java index 13bdd4728ef..2a760139b3d 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java @@ -3,6 +3,8 @@ import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.extendsClass; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static net.bytebuddy.matcher.ElementMatchers.isBridge; +import static net.bytebuddy.matcher.ElementMatchers.isConstructor; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; import static net.bytebuddy.matcher.ElementMatchers.not; import static net.bytebuddy.matcher.ElementMatchers.returns; import static net.bytebuddy.matcher.ElementMatchers.takesArgument; @@ -18,7 +20,7 @@ import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.matcher.ElementMatcher; -/** Captures route templates from successful Akka HTTP path matchers. */ +/** Tracks matched Akka HTTP route templates and directive rejection results. */ @AutoService(InstrumenterModule.class) public final class AkkaHttpPathMatcherInstrumentation extends InstrumenterModule.Tracing implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { @@ -38,11 +40,15 @@ public String hierarchyMarkerType() { @Override public ElementMatcher hierarchyMatcher() { - return extendsClass(named(hierarchyMarkerType())); + return extendsClass(named(hierarchyMarkerType())) + .or(named("akka.http.scaladsl.server.RouteResult$Rejected")); } @Override public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isConstructor().and(isDeclaredBy(named("akka.http.scaladsl.server.RouteResult$Rejected"))), + getClass().getName() + "$RejectedAdvice"); transformer.applyAdvice( named("apply") .and(takesArguments(1)) @@ -52,6 +58,13 @@ public void methodAdvice(MethodTransformer transformer) { getClass().getName() + "$MatchAdvice"); } + public static final class RejectedAdvice { + @Advice.OnMethodExit(suppress = Throwable.class) + public static void exit() { + AkkaHttpRouteState.onRejected(); + } + } + public static final class MatchAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) public static AkkaHttpRouteHelper.Frame enter(@Advice.Argument(0) Uri.Path path) { diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java index e3192894132..ec57234bcef 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java @@ -55,7 +55,7 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable boolean isMatch = throwable == null && matching instanceof PathMatcher.Matched; if (frame.parent == null) { CURRENT.remove(); - frame.state.lastMatched = isMatch; + frame.state.routeMatched = isMatch; } else { CURRENT.set(frame.parent); } diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java index 603f20ae328..933a7a3e187 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java @@ -17,8 +17,8 @@ public final class AkkaHttpRouteState { Route route; - /** Whether the last outermost path matcher tried matched; OTel drops the route otherwise. */ - boolean lastMatched; + /** Whether the latest route candidate has matched without subsequently being rejected. */ + boolean routeMatched; private AkkaHttpRouteState() {} @@ -30,10 +30,17 @@ static AkkaHttpRouteState current() { return Context.current().get(KEY); } - /** Drops the route of a request rejected after its last path matcher failed. */ + public static void onRejected() { + AkkaHttpRouteState state = current(); + if (state != null) { + state.routeMatched = false; + } + } + + /** Drops a route candidate rejected by path matching or a later directive. */ public static void dropRejectedRoute(Context context, AgentSpan span) { AkkaHttpRouteState state = context.get(KEY); - if (state == null || state.route == null || state.lastMatched) { + if (state == null || state.route == null || state.routeMatched) { return; } state.route = null; diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy index dd8d5fff837..ca67f32d1dc 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy @@ -138,6 +138,7 @@ abstract class PekkoHttpRouteOtelSemanticsTest extends WithHttpServer + onSuccess(Future { 1 }) { _ => + parameter("required") { _ => complete("accepted") } + } + } ~ complete("fallback") + } ~ pathPrefix("header-rejection") { + path(Segment) { _ => + headerValueByName("authorization") { _ => complete("accepted") } + } ~ complete("fallback") } ~ path("items" / Segment) { _ => complete("item") } ~ pathPrefix("route-async" / IntNumber) { id => diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java index 28df92ed87b..57730802fdb 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java @@ -3,6 +3,8 @@ import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.extendsClass; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static net.bytebuddy.matcher.ElementMatchers.isBridge; +import static net.bytebuddy.matcher.ElementMatchers.isConstructor; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; import static net.bytebuddy.matcher.ElementMatchers.not; import static net.bytebuddy.matcher.ElementMatchers.returns; import static net.bytebuddy.matcher.ElementMatchers.takesArgument; @@ -18,7 +20,7 @@ import org.apache.pekko.http.scaladsl.model.Uri; import org.apache.pekko.http.scaladsl.server.PathMatcher; -/** Captures route templates from successful Pekko HTTP path matchers. */ +/** Tracks matched Pekko HTTP route templates and directive rejection results. */ @AutoService(InstrumenterModule.class) public final class PekkoHttpPathMatcherInstrumentation extends InstrumenterModule.Tracing implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { @@ -38,11 +40,16 @@ public String hierarchyMarkerType() { @Override public ElementMatcher hierarchyMatcher() { - return extendsClass(named(hierarchyMarkerType())); + return extendsClass(named(hierarchyMarkerType())) + .or(named("org.apache.pekko.http.scaladsl.server.RouteResult$Rejected")); } @Override public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isConstructor() + .and(isDeclaredBy(named("org.apache.pekko.http.scaladsl.server.RouteResult$Rejected"))), + getClass().getName() + "$RejectedAdvice"); transformer.applyAdvice( named("apply") .and(takesArguments(1)) @@ -52,6 +59,13 @@ public void methodAdvice(MethodTransformer transformer) { getClass().getName() + "$MatchAdvice"); } + public static final class RejectedAdvice { + @Advice.OnMethodExit(suppress = Throwable.class) + public static void exit() { + PekkoHttpRouteState.onRejected(); + } + } + public static final class MatchAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) public static PekkoHttpRouteHelper.Frame enter(@Advice.Argument(0) Uri.Path path) { diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java index 3ab3b58ee96..663828df94c 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java @@ -55,7 +55,7 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable boolean isMatch = throwable == null && matching instanceof PathMatcher.Matched; if (frame.parent == null) { CURRENT.remove(); - frame.state.lastMatched = isMatch; + frame.state.routeMatched = isMatch; } else { CURRENT.set(frame.parent); } diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java index 45d18aa6818..896bfcf155d 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java @@ -17,8 +17,8 @@ public final class PekkoHttpRouteState { Route route; - /** Whether the last outermost path matcher tried matched; OTel drops the route otherwise. */ - boolean lastMatched; + /** Whether the latest route candidate has matched without subsequently being rejected. */ + boolean routeMatched; private PekkoHttpRouteState() {} @@ -30,10 +30,17 @@ static PekkoHttpRouteState current() { return Context.current().get(KEY); } - /** Drops the route of a request rejected after its last path matcher failed. */ + public static void onRejected() { + PekkoHttpRouteState state = current(); + if (state != null) { + state.routeMatched = false; + } + } + + /** Drops a route candidate rejected by path matching or a later directive. */ public static void dropRejectedRoute(Context context, AgentSpan span) { PekkoHttpRouteState state = context.get(KEY); - if (state == null || state.route == null || state.lastMatched) { + if (state == null || state.route == null || state.routeMatched) { return; } state.route = null; From f78e11c19048d2358ce82f6ebd76121a8a49c935 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 3 Nov 2026 15:39:05 -0800 Subject: [PATCH 22/34] Limit WebClient span suppression to its transport exchange --- .../client/WebClientTracingFilter.java | 116 +++++++++++++++++- .../SpringWebfluxHttpClientBasicTest.groovy | 62 ++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java index 1fbe0bafc7f..c7a4e89a95d 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java @@ -8,9 +8,12 @@ import static datadog.trace.instrumentation.springwebflux.client.SpringWebfluxHttpClientDecorator.HTTP_REQUEST; import datadog.context.Context; +import datadog.context.ContextKey; import datadog.context.ContextScope; +import datadog.trace.api.Config; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import java.util.List; +import org.reactivestreams.Subscription; import org.springframework.web.reactive.function.client.ClientRequest; import org.springframework.web.reactive.function.client.ClientResponse; import org.springframework.web.reactive.function.client.ExchangeFilterFunction; @@ -23,7 +26,14 @@ * https://github.com/spring-cloud/spring-cloud-sleuth/blob/master/spring-cloud-sleuth-core/src/main/java/org/springframework/cloud/sleuth/instrument/web/client/TraceWebClientBeanPostProcessor.java */ public class WebClientTracingFilter implements ExchangeFilterFunction { + private static final boolean OTEL_SEMANTICS = Config.get().isTraceOtelSemanticsEnabled(); + private static final ContextKey WEBCLIENT_SPAN = + ContextKey.named("spring-webclient-span"); + private static final ContextKey WEBCLIENT_PARENT = + ContextKey.named("spring-webclient-parent"); private static final WebClientTracingFilter INSTANCE = new WebClientTracingFilter(); + private static final ExchangeFilterFunction TRANSPORT_FILTER = + (request, next) -> new MonoWebClientTransport(request, next); /** * It's not expected that this method would be called concurrently. This should only be created @@ -34,6 +44,10 @@ public static void addFilter(final List exchangeFilterFu // Since the builder we instrument the can be reused, we need to only add the filter once exchangeFilterFunctions.removeIf(filterFunction -> filterFunction == INSTANCE); exchangeFilterFunctions.add(0, INSTANCE); + if (OTEL_SEMANTICS) { + exchangeFilterFunctions.removeIf(filterFunction -> filterFunction == TRANSPORT_FILTER); + exchangeFilterFunctions.add(TRANSPORT_FILTER); + } } @Override @@ -58,11 +72,111 @@ public void subscribe(final CoreSubscriber subscriber) { AgentSpan parent = activeSpan(); Context parentContext = Context.current(); try (final ContextScope scope = activateSpan(span); - final ContextScope suppressScope = suppressNestedClientSpans(span, parentContext)) { + final ContextScope ownerScope = + OTEL_SEMANTICS + ? Context.current() + .with(WEBCLIENT_SPAN, span) + .with(WEBCLIENT_PARENT, parentContext) + .attach() + : null) { TraceWebClientSubscriber tracingSubscriber = new TraceWebClientSubscriber(subscriber, span, parent); next.exchange(request).doOnCancel(tracingSubscriber::onCancel).subscribe(tracingSubscriber); } } } + + /** Applies transport suppression after the application's filters have run. */ + public static final class MonoWebClientTransport extends Mono { + private final ClientRequest request; + private final ExchangeFunction next; + + public MonoWebClientTransport(ClientRequest request, ExchangeFunction next) { + this.request = request; + this.next = next; + } + + @Override + public void subscribe(CoreSubscriber subscriber) { + Context context = Context.current(); + AgentSpan owner = context.get(WEBCLIENT_SPAN); + if (owner == null || owner != activeSpan()) { + next.exchange(request).subscribe(subscriber); + return; + } + try (ContextScope suppression = + suppressNestedClientSpans(owner, context.get(WEBCLIENT_PARENT))) { + next.exchange(request) + .subscribe(new TransportSubscriber(subscriber, context, Context.current())); + } + } + } + + /** + * Keeps connector demand in its transport context and filter callbacks in their owner context. + */ + public static final class TransportSubscriber + implements CoreSubscriber, Subscription { + private final CoreSubscriber actual; + private final Context ownerContext; + private final Context transportContext; + private Subscription subscription; + + public TransportSubscriber( + CoreSubscriber actual, + Context ownerContext, + Context transportContext) { + this.actual = actual; + this.ownerContext = ownerContext; + this.transportContext = transportContext; + } + + @Override + public reactor.util.context.Context currentContext() { + return actual.currentContext(); + } + + @Override + public void onSubscribe(Subscription subscription) { + this.subscription = subscription; + try (ContextScope scope = ownerContext.attach()) { + actual.onSubscribe(this); + } + } + + @Override + public void request(long count) { + try (ContextScope scope = transportContext.attach()) { + subscription.request(count); + } + } + + @Override + public void cancel() { + try (ContextScope scope = transportContext.attach()) { + subscription.cancel(); + } + } + + @Override + public void onNext(ClientResponse response) { + try (ContextScope scope = ownerContext.attach()) { + actual.onNext(response); + } + } + + @Override + public void onError(Throwable error) { + try (ContextScope scope = ownerContext.attach()) { + actual.onError(error); + } + } + + @Override + public void onComplete() { + try (ContextScope scope = ownerContext.attach()) { + actual.onComplete(); + } + } + } } diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBasicTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBasicTest.groovy index a6cd953b133..bb7d11c0474 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBasicTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBasicTest.groovy @@ -1,7 +1,17 @@ package dd.trace.instrumentation.springwebflux.client import datadog.trace.agent.test.naming.TestingOtelHttpSemantics +import datadog.trace.instrumentation.netty41.client.HttpClientTracingHandler +import io.netty.channel.embedded.EmbeddedChannel +import io.netty.handler.codec.http.DefaultHttpRequest +import io.netty.handler.codec.http.DefaultHttpResponse +import io.netty.handler.codec.http.HttpMethod +import io.netty.handler.codec.http.HttpResponseStatus +import io.netty.handler.codec.http.HttpVersion +import org.springframework.web.reactive.function.client.ExchangeFilterFunction import org.springframework.web.reactive.function.client.WebClient +import reactor.core.publisher.Mono +import reactor.core.scheduler.Schedulers import spock.lang.Timeout @Timeout(5) @@ -19,4 +29,56 @@ class SpringWebfluxHttpClientBasicTest extends SpringWebfluxHttpClientBase { @Timeout(5) class SpringWebfluxHttpClientOtelSemanticsForkedTest extends SpringWebfluxHttpClientBasicTest implements TestingOtelHttpSemantics { + def "an independent Netty request in a user filter retains its client span"() { + setup: + def independentRequest = { + def channel = new EmbeddedChannel(new HttpClientTracingHandler()) + try { + channel.writeOutbound(new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, 'http://filter.example/token')) + channel.readOutbound() + channel.writeInbound(new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK)) + } finally { + channel.finishAndReleaseAll() + } + } + def webClient = WebClient.builder().filter({ request, next -> + def exchange = Mono.defer { + if (beforeResponse) { + independentRequest() + } + next.exchange(request).doOnNext { + if (!beforeResponse) { + independentRequest() + } + } + } + async ? exchange.subscribeOn(Schedulers.parallel()) : exchange + } as ExchangeFilterFunction).build() + + when: + def response = webClient.get().uri(server.address.resolve('/success')).exchange().block() + consumeResponse(response, 'GET') + TEST_WRITER.waitForTraces(1) + + then: + def spans = TEST_WRITER.flatten() + def webClientSpan = spans.find { it.getTag('component')?.toString() == 'spring-webflux-client' } + def transportSpans = spans.findAll { it.getTag('component')?.toString() == 'netty-client' } + transportSpans.size() == 1 + transportSpans[0].getTag('url.full').toString() == 'http://filter.example/token' + transportSpans[0].parentId == webClientSpan.spanId + + where: + beforeResponse | async + true | false + true | true + false | false + false | true + } +} + +class SpringWebfluxHttpClientFilterBuilderReuseOtelSemanticsForkedTest extends SpringWebfluxHttpClientFilterBuilderReuseTest implements TestingOtelHttpSemantics { +} + +class SpringWebfluxHttpClientFiltersRemovedOtelSemanticsForkedTest extends SpringWebfluxHttpClientFiltersRemovedTest implements TestingOtelHttpSemantics { } From 628617dbbf39fab2af73a0f64843d779d95e1fae Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 3 Nov 2026 16:58:41 -0800 Subject: [PATCH 23/34] fixup: Preserve transport SSRF checks --- .../decorator/HttpClientDecorator.java | 6 +- .../SuppressedClientSpanForkedTest.java | 100 ++++++------------ 2 files changed, 38 insertions(+), 68 deletions(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java index 9b6ad3364ea..1c82f273f1a 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java @@ -151,9 +151,9 @@ public final void onRequestSecurity(final AgentSpan span, final REQUEST request) } try { if (APPSEC_RASP_ENABLED) { - URI destination = url(request); - if (destination != null) { - onHttpClientRequest(span, destination.toString()); + final URI url = url(request); + if (url != null) { + onHttpClientRequest(span, url.toString()); } } } catch (BlockingException e) { diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSpanForkedTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSpanForkedTest.java index 6ef0bf7a065..a4c6afdb22a 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSpanForkedTest.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSpanForkedTest.java @@ -10,11 +10,16 @@ import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; import static io.netty.handler.codec.http.HttpMethod.GET; import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import datadog.appsec.api.blocking.BlockingContentType; import datadog.appsec.api.blocking.BlockingException; @@ -44,7 +49,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import org.junit.jupiter.api.Test; import org.tabletest.junit.TableTest; @@ -53,45 +57,6 @@ @WithConfig(key = "appsec.enabled", value = "true") @WithConfig(key = "appsec.rasp.enabled", value = "true") class SuppressedClientSpanForkedTest extends AbstractInstrumentationTest { - @Test - void restoresUpstreamContextAfterAnErrorBeforeHeaders() { - AgentSpan parent = AgentTracer.startSpan("test", "parent"); - AgentSpan owner = AgentTracer.startSpan("test", "webclient", parent.spanContext()); - Context parentContext = Context.current().with(parent); - List observed = new ArrayList<>(); - EmbeddedChannel channel = - new EmbeddedChannel( - new HttpClientTracingHandler(), - new ChannelInboundHandlerAdapter() { - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable error) { - observed.add(activeSpan()); - owner.finish(); - } - - @Override - public void channelInactive(ChannelHandlerContext ctx) { - observed.add(activeSpan()); - } - }); - try { - try (ContextScope scope = activateSpan(owner); - ContextScope suppression = suppressNestedClientSpans(owner, parentContext)) { - channel.writeOutbound(new DefaultHttpRequest(HTTP_1_1, GET, "/test")); - } - channel.pipeline().fireExceptionCaught(new IOException("connection failure")); - assertSame(parentContext, channel.attr(CONTEXT_ATTRIBUTE_KEY).get()); - channel.close(); - assertSame(owner, observed.get(0)); - assertSame(parent, observed.get(1)); - assertFalse(((DDSpan) parent).isFinished()); - } finally { - channel.finishAndReleaseAll(); - owner.finish(); - parent.finish(); - } - } - @Test void preservesANewRequestStartedFromTheResponseCallback() { AgentSpan owner = AgentTracer.startSpan("test", "first-request"); @@ -126,15 +91,16 @@ public void channelRead(ChannelHandlerContext ctx, Object message) { @TableTest({ "Scenario | Response status | Stream channel", + "before headers | | false ", "response headers | 200 | false ", "protocol upgrade | 101 | false ", "multiplexed stream | 200 | true " }) - void restoresUpstreamContextForBodyErrorsAndClose(int responseStatus, boolean streamChannel) { + void restoresUpstreamContextAfterErrorsAndClose(Integer responseStatus, boolean streamChannel) { AgentSpan parent = AgentTracer.startSpan("test", "parent"); AgentSpan owner = AgentTracer.startSpan("test", "webclient", parent.spanContext()); Context parentContext = Context.current().with(parent); - List observed = new ArrayList<>(); + List observed = new ArrayList<>(); EmbeddedChannel channel = new EmbeddedChannel( new HttpClientTracingHandler(), @@ -152,6 +118,7 @@ public void channelRead(ChannelHandlerContext ctx, Object message) { @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable error) { observed.add(activeSpan()); + owner.finish(); } @Override @@ -169,20 +136,22 @@ public void channelInactive(ChannelHandlerContext ctx) { // Stream channels inherit the request context without a client-parent attribute. channel.attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(null); } - channel.writeInbound(new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.CONTINUE)); - assertSame(owner, AgentSpan.fromContext(channel.attr(CONTEXT_ATTRIBUTE_KEY).get())); - channel.writeInbound( - new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.valueOf(responseStatus))); + if (responseStatus != null) { + channel.writeInbound(new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.CONTINUE)); + assertSame(owner, AgentSpan.fromContext(channel.attr(CONTEXT_ATTRIBUTE_KEY).get())); + channel.writeInbound( + new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.valueOf(responseStatus))); + assertSame(parentContext, channel.attr(CONTEXT_ATTRIBUTE_KEY).get()); + channel.writeInbound(new DefaultHttpContent(Unpooled.buffer().writeByte(1))); + } + channel.pipeline().fireExceptionCaught(new IOException("request failure")); assertSame(parentContext, channel.attr(CONTEXT_ATTRIBUTE_KEY).get()); - channel.writeInbound(new DefaultHttpContent(Unpooled.buffer().writeByte(1))); - channel.pipeline().fireExceptionCaught(new IOException("body failure")); channel.close(); - assertEquals(5, observed.size()); - assertSame(owner, observed.get(0)); - assertSame(owner, observed.get(1)); - for (int i = 2; i < observed.size(); i++) { - assertSame(parent, observed.get(i)); - } + assertIterableEquals( + responseStatus == null + ? asList(owner, parent) + : asList(owner, owner, parent, parent, parent), + observed); assertFalse(((DDSpan) parent).isFinished()); } finally { channel.finishAndReleaseAll(); @@ -201,7 +170,15 @@ public void channelInactive(ChannelHandlerContext ctx) { "blocked | true " }) void checksTransportDestinationWithoutChangingOwnerMetadata(boolean blocked) { - AtomicReference checkedUrl = new AtomicReference<>(); + List checkedUrls = new ArrayList<>(); + @SuppressWarnings("unchecked") + Flow flow = mock(Flow.class); + when(flow.getAction()) + .thenReturn( + blocked + ? new Flow.Action.RequestBlockingAction(403, BlockingContentType.AUTO) + : Flow.Action.Noop.INSTANCE); + // A lambda would expose bootstrap types in this test class's synthetic method signature. Subscription subscription = AgentTracer.get() .getSubscriptionService(RequestContextSlot.APPSEC) @@ -210,15 +187,8 @@ void checksTransportDestinationWithoutChangingOwnerMetadata(boolean blocked) { new BiFunction>() { @Override public Flow apply(RequestContext context, HttpClientRequest request) { - checkedUrl.set(request.getUrl()); - return blocked - ? new Flow.ResultFlow(null) { - @Override - public Action getAction() { - return new Action.RequestBlockingAction(403, BlockingContentType.AUTO); - } - } - : Flow.ResultFlow.empty(); + checkedUrls.add(request.getUrl()); + return flow; } }); AgentSpan owner = @@ -238,7 +208,7 @@ public Action getAction() { channel.writeOutbound(request); assertSame(request, channel.readOutbound()); } - assertEquals("http://transport.example/rewritten", checkedUrl.get()); + assertEquals(singletonList("http://transport.example/rewritten"), checkedUrls); assertEquals("http://original.example/original", owner.getTag("url.full")); } finally { subscription.cancel(); From 89f581857f595e14af8029cf671ff508922aae4d Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 3 Nov 2026 17:19:51 -0800 Subject: [PATCH 24/34] Revert changes to WebFlux => netty span suppression since it's not strictly required by OpenTelemetry semantic conventions. This will simplify the code changes, and we can enforce a suppresion policy at a later time. --- .github/CODEOWNERS | 1 - .../decorator/HttpClientDecorator.java | 53 ----- .../HttpClientRequestTracingHandler.java | 79 +++---- .../HttpClientResponseTracingHandler.java | 47 +--- .../SuppressedClientSpanForkedTest.java | 220 ------------------ .../client/TraceWebClientSubscriber.java | 33 +-- .../client/WebClientTracingFilter.java | 120 +--------- .../client/SpringWebfluxHttpClientBase.groovy | 49 ++-- .../SpringWebfluxHttpClientBasicTest.groovy | 62 ----- ...bfluxHttpClientDoAfterTerminateTest.groovy | 8 +- ...luxHttpClientDoOnSuccessOrErrorTest.groovy | 8 +- ...ingWebfluxHttpClientDoOnSuccessTest.groovy | 8 +- ...gWebfluxHttpClientDoOnTerminateTest.groovy | 8 +- .../bootTest/groovy/SpringWebfluxTest.groovy | 65 +++--- .../client/SpringWebfluxHttpClientBase.groovy | 11 +- .../SpringWebfluxHttpClientBasicTest.groovy | 5 - ...bfluxHttpClientDoAfterTerminateTest.groovy | 8 +- ...luxHttpClientDoOnSuccessOrErrorTest.groovy | 8 +- ...ingWebfluxHttpClientDoOnSuccessTest.groovy | 8 +- ...gWebfluxHttpClientDoOnTerminateTest.groovy | 8 +- 20 files changed, 161 insertions(+), 648 deletions(-) delete mode 100644 dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSpanForkedTest.java diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b5daeda821f..1decc476fda 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -247,7 +247,6 @@ /dd-java-agent/instrumentation/rs/**/MessageBodyWriterInstrumentation.java @DataDog/apm-java @DataDog/asm-java # netty-4.1 blocking glue -/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/**/SuppressedClientSpanForkedTest.java @DataDog/apm-java @DataDog/apm-idm-java @DataDog/asm-java /dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/**/NettyMultipartHelper.java @DataDog/apm-java @DataDog/asm-java /dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/**/HttpPostRequestDecoderInstrumentation.java @DataDog/apm-java @DataDog/asm-java /dd-java-agent/instrumentation/netty/**/server/BlockingResponseHandler.java @DataDog/apm-java @DataDog/asm-java diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java index 1c82f273f1a..3bee2a6afbf 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java @@ -6,9 +6,6 @@ import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR; import datadog.appsec.api.blocking.BlockingException; -import datadog.context.Context; -import datadog.context.ContextKey; -import datadog.context.ContextScope; import datadog.trace.api.Config; import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.KnownTags; @@ -43,11 +40,6 @@ public abstract class HttpClientDecorator extends UriBasedCli private static final Logger log = LoggerFactory.getLogger(HttpClientDecorator.class); - private static final ContextKey CLIENT_SPAN_KEY = - ContextKey.named("dd-http-client-span"); - private static final ContextKey CLIENT_PARENT_CONTEXT_KEY = - ContextKey.named("dd-http-client-parent-context"); - private static final String DATADOG_META_LANG_HEADER_NAME = "Datadog-Meta-Lang"; private static final String DD_CLIENT_LIBRARY_LANGUAGE_HEADER_NAME = "DD-Client-Library-Language"; @@ -62,30 +54,6 @@ public abstract class HttpClientDecorator extends UriBasedCli private static final boolean APPSEC_RASP_ENABLED = Config.get().isAppSecRaspEnabled(); - /** Suppresses duplicate transport spans when OpenTelemetry semantics are enabled. */ - public static ContextScope suppressNestedClientSpans( - final AgentSpan span, final Context parentContext) { - return OTEL_SEMANTICS - ? Context.current() - .with(CLIENT_SPAN_KEY, span) - .with(CLIENT_PARENT_CONTEXT_KEY, parentContext) - .attach() - : null; - } - - /** Returns the context to restore after the owning client's response headers arrive. */ - public static Context nestedClientParentContext(final Context context) { - return context.get(CLIENT_PARENT_CONTEXT_KEY); - } - - /** Returns whether {@code span} owns the HTTP request represented by {@code context}. */ - public static boolean isNestedClientSpanSuppressed(final Context context, final AgentSpan span) { - return OTEL_SEMANTICS - && context != null - && span != null - && context.get(CLIENT_SPAN_KEY) == span; - } - protected abstract String method(REQUEST request); protected abstract URI url(REQUEST request) throws URISyntaxException; @@ -144,27 +112,6 @@ public final void onRequest(final AgentSpan span, final REQUEST request) { } } - /** Checks the transport destination without decorating a span owned by another HTTP client. */ - public final void onRequestSecurity(final AgentSpan span, final REQUEST request) { - if (request == null) { - return; - } - try { - if (APPSEC_RASP_ENABLED) { - final URI url = url(request); - if (url != null) { - onHttpClientRequest(span, url.toString()); - } - } - } catch (BlockingException e) { - throw e; - } catch (Throwable t) { - log.debug("Failed to check HTTP transport destination", t); - } finally { - ssrfIastCheck(request); - } - } - protected void doOnRequest(final AgentSpan span, final REQUEST request) { if (request != null) { AgentTracer.get() diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java index d6a4ba86115..47134f4a0cf 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java @@ -3,7 +3,6 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; -import static datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator.isNestedClientSpanSuppressed; import static datadog.trace.instrumentation.netty41.AttributeKeys.CLIENT_PARENT_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONNECT_PARENT_CONTINUATION_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; @@ -60,54 +59,50 @@ public void write(final ChannelHandlerContext ctx, final Object msg, final Chann parentScope = continuation.resume(); } - try { - final HttpRequest request = (HttpRequest) msg; - boolean awsClientCall = request.headers().contains("amz-sdk-invocation-id"); - if (!AWS_LEGACY_TRACING && awsClientCall) { - // avoid creating an extra HTTP client span beneath the AWS client call - ctx.write(msg, prm); - return; - } - final AgentSpan parent = activeSpan(); - final Context parentContext = Context.current(); - if (ctx.channel().parent() == null) { - ctx.channel().attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(parent); - } - boolean isSecure = SSL_HANDLER != null && ctx.pipeline().get(SSL_HANDLER) != null; - NettyHttpClientDecorator decorate = isSecure ? DECORATE_SECURE : DECORATE; - if (isNestedClientSpanSuppressed(parentContext, parent)) { - decorate.onRequestSecurity(parent, request); - DECORATE.injectContext(parentContext, request.headers(), SETTER); - ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(parentContext); + final HttpRequest request = (HttpRequest) msg; + boolean awsClientCall = request.headers().contains("amz-sdk-invocation-id"); + if (!AWS_LEGACY_TRACING && awsClientCall) { + // avoid creating an extra HTTP client span beneath the AWS client call + try { ctx.write(msg, prm); return; + } finally { + if (null != parentScope) { + parentScope.close(); + } } - final AgentSpan span = startSpan(NETTY_CLIENT.toString(), NETTY_CLIENT_REQUEST); - final Context context = parentContext.with(span); - try (final ContextScope scope = activateSpan(span)) { - decorate.afterStart(span); - decorate.onRequest(span, request); + } + if (ctx.channel().parent() == null) { + ctx.channel().attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(activeSpan()); + } + boolean isSecure = SSL_HANDLER != null && ctx.pipeline().get(SSL_HANDLER) != null; + NettyHttpClientDecorator decorate = isSecure ? DECORATE_SECURE : DECORATE; - SocketAddress socketAddress = ctx.channel().remoteAddress(); - if (socketAddress instanceof InetSocketAddress) { - decorate.onPeerConnection(span, (InetSocketAddress) socketAddress); - } + final AgentSpan span = startSpan(NETTY_CLIENT.toString(), NETTY_CLIENT_REQUEST); + final Context context = Context.current().with(span); + try (final ContextScope scope = activateSpan(span)) { + decorate.afterStart(span); + decorate.onRequest(span, request); - // AWS calls are often signed, so we can't add headers without breaking the signature. - if (!awsClientCall) { - DECORATE.injectContext(Context.current(), request.headers(), SETTER); - } + SocketAddress socketAddress = ctx.channel().remoteAddress(); + if (socketAddress instanceof InetSocketAddress) { + decorate.onPeerConnection(span, (InetSocketAddress) socketAddress); + } - ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(context); + // AWS calls are often signed, so we can't add headers without breaking the signature. + if (!awsClientCall) { + DECORATE.injectContext(Context.current(), request.headers(), SETTER); + } - try { - ctx.write(msg, prm); - } catch (final Throwable throwable) { - decorate.onError(span, throwable); - decorate.beforeFinish(span); - span.finish(); - throw throwable; - } + ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(context); + + try { + ctx.write(msg, prm); + } catch (final Throwable throwable) { + decorate.onError(span, throwable); + decorate.beforeFinish(span); + span.finish(); + throw throwable; } } finally { if (null != parentScope) { diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java index b9bbaa78489..b80f967a3fb 100644 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java +++ b/dd-java-agent/instrumentation/netty/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientResponseTracingHandler.java @@ -2,8 +2,6 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.noopSpan; -import static datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator.isNestedClientSpanSuppressed; -import static datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator.nestedClientParentContext; import static datadog.trace.instrumentation.netty41.AttributeKeys.CLIENT_PARENT_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; import static datadog.trace.instrumentation.netty41.client.NettyHttpClientDecorator.DECORATE; @@ -31,19 +29,6 @@ public void channelRead(final ChannelHandlerContext ctx, final Object msg) { final AgentSpan parent = parentAttr.get(); final Context storedContext = ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).get(); final AgentSpan span = AgentSpan.fromContext(storedContext); - if (isNestedClientSpanSuppressed(storedContext, span)) { - // The outer client span owns this request, so run the callback under it. - try (final ContextScope scope = activateSpan(span)) { - ctx.fireChannelRead(msg); - } finally { - if (msg instanceof HttpResponse - && (((HttpResponse) msg).status().code() >= 200 - || ((HttpResponse) msg).status().code() == 101)) { - restoreParentContext(ctx, storedContext, parent); - } - } - return; - } // Set parent context back to maintain the same functionality as getAndSet(parent) if (storedContext != null) { @@ -82,22 +67,13 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E final AgentSpan parent = parentAttr.get(); final Context storedContext = ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).get(); final AgentSpan span = AgentSpan.fromContext(storedContext); - if (isNestedClientSpanSuppressed(storedContext, span)) { - // The outer client span owns this request, so run the callback under it. - try (final ContextScope scope = activateSpan(span)) { - super.exceptionCaught(ctx, cause); - } finally { - restoreParentContext(ctx, storedContext, parent); - } - return; - } // Set parent context back to maintain the same functionality as getAndSet(parent) if (storedContext != null) { ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).set(storedContext.with(parent)); } - if (span != null && span != parent) { + if (span != null) { // If an exception is passed to this point, it likely means it was unhandled and the // client span won't be finished with a proper response, so we should finish the span here. try (final ContextScope scope = activateSpan(span)) { @@ -119,15 +95,6 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { final AgentSpan parent = parentAttr.get(); final Context storedContext = ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).get(); final AgentSpan span = AgentSpan.fromContext(storedContext); - if (isNestedClientSpanSuppressed(storedContext, span)) { - // The outer client span owns this request, so run the callback under it. - try (final ContextScope scope = activateSpan(span)) { - super.channelInactive(ctx); - } finally { - restoreParentContext(ctx, storedContext, parent); - } - return; - } // Set parent context back to maintain the same functionality as getAndSet(parent) if (storedContext != null) { @@ -145,16 +112,4 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception { super.channelInactive(ctx); } } - - private static void restoreParentContext( - ChannelHandlerContext ctx, Context context, AgentSpan previousParent) { - Context parentContext = nestedClientParentContext(context); - if (parentContext != null - && ctx.channel().attr(CONTEXT_ATTRIBUTE_KEY).compareAndSet(context, parentContext)) { - AgentSpan parent = AgentSpan.fromContext(parentContext); - ctx.channel() - .attr(CLIENT_PARENT_ATTRIBUTE_KEY) - .compareAndSet(previousParent, parent == null ? noopSpan() : parent); - } - } } diff --git a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSpanForkedTest.java b/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSpanForkedTest.java deleted file mode 100644 index a4c6afdb22a..00000000000 --- a/dd-java-agent/instrumentation/netty/netty-4.1/src/test/java/datadog/trace/instrumentation/netty41/client/SuppressedClientSpanForkedTest.java +++ /dev/null @@ -1,220 +0,0 @@ -package datadog.trace.instrumentation.netty41.client; - -import static datadog.trace.agent.test.assertions.SpanMatcher.span; -import static datadog.trace.agent.test.assertions.TraceMatcher.trace; -import static datadog.trace.api.gateway.Events.EVENTS; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; -import static datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator.suppressNestedClientSpans; -import static datadog.trace.instrumentation.netty41.AttributeKeys.CLIENT_PARENT_ATTRIBUTE_KEY; -import static datadog.trace.instrumentation.netty41.AttributeKeys.CONTEXT_ATTRIBUTE_KEY; -import static io.netty.handler.codec.http.HttpMethod.GET; -import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; -import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertIterableEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import datadog.appsec.api.blocking.BlockingContentType; -import datadog.appsec.api.blocking.BlockingException; -import datadog.context.Context; -import datadog.context.ContextScope; -import datadog.trace.agent.test.AbstractInstrumentationTest; -import datadog.trace.api.appsec.HttpClientRequest; -import datadog.trace.api.gateway.Flow; -import datadog.trace.api.gateway.RequestContext; -import datadog.trace.api.gateway.RequestContextSlot; -import datadog.trace.api.gateway.Subscription; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import datadog.trace.bootstrap.instrumentation.api.TagContext; -import datadog.trace.core.DDSpan; -import datadog.trace.test.junit.utils.config.WithConfig; -import io.netty.buffer.Unpooled; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.channel.embedded.EmbeddedChannel; -import io.netty.handler.codec.http.DefaultHttpContent; -import io.netty.handler.codec.http.DefaultHttpRequest; -import io.netty.handler.codec.http.DefaultHttpResponse; -import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.codec.http.HttpResponseStatus; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.function.BiFunction; -import org.junit.jupiter.api.Test; -import org.tabletest.junit.TableTest; - -@WithConfig(key = "trace.otel.semantics.enabled", value = "true") -@WithConfig(key = "appsec.enabled", value = "true") -@WithConfig(key = "appsec.rasp.enabled", value = "true") -class SuppressedClientSpanForkedTest extends AbstractInstrumentationTest { - @Test - void preservesANewRequestStartedFromTheResponseCallback() { - AgentSpan owner = AgentTracer.startSpan("test", "first-request"); - AgentSpan nextOwner = AgentTracer.startSpan("test", "next-request"); - EmbeddedChannel channel = - new EmbeddedChannel( - new HttpClientTracingHandler(), - new ChannelInboundHandlerAdapter() { - @Override - public void channelRead(ChannelHandlerContext ctx, Object message) { - owner.finish(); - try (ContextScope scope = activateSpan(nextOwner); - ContextScope suppression = - suppressNestedClientSpans(nextOwner, Context.root())) { - ctx.channel().writeAndFlush(new DefaultHttpRequest(HTTP_1_1, GET, "/next")); - } - } - }); - try { - try (ContextScope scope = activateSpan(owner); - ContextScope suppression = suppressNestedClientSpans(owner, Context.root())) { - channel.writeOutbound(new DefaultHttpRequest(HTTP_1_1, GET, "/first")); - } - channel.writeInbound(new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.OK)); - assertSame(nextOwner, AgentSpan.fromContext(channel.attr(CONTEXT_ATTRIBUTE_KEY).get())); - } finally { - channel.finishAndReleaseAll(); - owner.finish(); - nextOwner.finish(); - } - } - - @TableTest({ - "Scenario | Response status | Stream channel", - "before headers | | false ", - "response headers | 200 | false ", - "protocol upgrade | 101 | false ", - "multiplexed stream | 200 | true " - }) - void restoresUpstreamContextAfterErrorsAndClose(Integer responseStatus, boolean streamChannel) { - AgentSpan parent = AgentTracer.startSpan("test", "parent"); - AgentSpan owner = AgentTracer.startSpan("test", "webclient", parent.spanContext()); - Context parentContext = Context.current().with(parent); - List observed = new ArrayList<>(); - EmbeddedChannel channel = - new EmbeddedChannel( - new HttpClientTracingHandler(), - new ChannelInboundHandlerAdapter() { - @Override - public void channelRead(ChannelHandlerContext ctx, Object message) { - observed.add(activeSpan()); - if (message instanceof HttpResponse - && ((HttpResponse) message).status().code() == responseStatus) { - owner.finish(); - } - ctx.fireChannelRead(message); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable error) { - observed.add(activeSpan()); - owner.finish(); - } - - @Override - public void channelInactive(ChannelHandlerContext ctx) { - observed.add(activeSpan()); - } - }); - try { - try (ContextScope scope = activateSpan(owner); - ContextScope suppression = suppressNestedClientSpans(owner, parentContext)) { - channel.writeOutbound(new DefaultHttpRequest(HTTP_1_1, GET, "/test")); - channel.readOutbound(); - } - if (streamChannel) { - // Stream channels inherit the request context without a client-parent attribute. - channel.attr(CLIENT_PARENT_ATTRIBUTE_KEY).set(null); - } - if (responseStatus != null) { - channel.writeInbound(new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.CONTINUE)); - assertSame(owner, AgentSpan.fromContext(channel.attr(CONTEXT_ATTRIBUTE_KEY).get())); - channel.writeInbound( - new DefaultHttpResponse(HTTP_1_1, HttpResponseStatus.valueOf(responseStatus))); - assertSame(parentContext, channel.attr(CONTEXT_ATTRIBUTE_KEY).get()); - channel.writeInbound(new DefaultHttpContent(Unpooled.buffer().writeByte(1))); - } - channel.pipeline().fireExceptionCaught(new IOException("request failure")); - assertSame(parentContext, channel.attr(CONTEXT_ATTRIBUTE_KEY).get()); - channel.close(); - assertIterableEquals( - responseStatus == null - ? asList(owner, parent) - : asList(owner, owner, parent, parent, parent), - observed); - assertFalse(((DDSpan) parent).isFinished()); - } finally { - channel.finishAndReleaseAll(); - owner.finish(); - parent.finish(); - } - assertTraces( - trace( - span().root().operationName("parent"), - span().childOfPrevious().operationName("webclient"))); - } - - @TableTest({ - "Scenario | Blocked", - "allowed | false ", - "blocked | true " - }) - void checksTransportDestinationWithoutChangingOwnerMetadata(boolean blocked) { - List checkedUrls = new ArrayList<>(); - @SuppressWarnings("unchecked") - Flow flow = mock(Flow.class); - when(flow.getAction()) - .thenReturn( - blocked - ? new Flow.Action.RequestBlockingAction(403, BlockingContentType.AUTO) - : Flow.Action.Noop.INSTANCE); - // A lambda would expose bootstrap types in this test class's synthetic method signature. - Subscription subscription = - AgentTracer.get() - .getSubscriptionService(RequestContextSlot.APPSEC) - .registerCallback( - EVENTS.httpClientRequest(), - new BiFunction>() { - @Override - public Flow apply(RequestContext context, HttpClientRequest request) { - checkedUrls.add(request.getUrl()); - return flow; - } - }); - AgentSpan owner = - AgentTracer.get() - .startSpan( - "test", "webclient", new TagContext().withRequestContextDataAppSec(new Object())); - owner.setTag("url.full", "http://original.example/original"); - EmbeddedChannel channel = new EmbeddedChannel(HttpClientRequestTracingHandler.INSTANCE); - HttpRequest request = new DefaultHttpRequest(HTTP_1_1, GET, "/rewritten"); - request.headers().set("Host", "transport.example"); - try (ContextScope scope = activateSpan(owner); - ContextScope suppression = suppressNestedClientSpans(owner, Context.root())) { - if (blocked) { - assertThrows(BlockingException.class, () -> channel.writeOutbound(request)); - assertNull(channel.readOutbound()); - } else { - channel.writeOutbound(request); - assertSame(request, channel.readOutbound()); - } - assertEquals(singletonList("http://transport.example/rewritten"), checkedUrls); - assertEquals("http://original.example/original", owner.getTag("url.full")); - } finally { - subscription.cancel(); - channel.finishAndReleaseAll(); - owner.finish(); - } - assertTraces(trace(span().root().operationName("webclient"))); - } -} diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/TraceWebClientSubscriber.java b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/TraceWebClientSubscriber.java index 44c9668e1bd..c0d75bf4999 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/TraceWebClientSubscriber.java +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/TraceWebClientSubscriber.java @@ -5,7 +5,6 @@ import static datadog.trace.instrumentation.springwebflux.client.SpringWebfluxHttpClientDecorator.DECORATE; import datadog.context.ContextScope; -import datadog.trace.api.Config; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import org.reactivestreams.Subscription; import org.springframework.web.reactive.function.client.ClientResponse; @@ -13,7 +12,6 @@ import reactor.util.context.Context; public final class TraceWebClientSubscriber implements CoreSubscriber { - private static final boolean OTEL_SEMANTICS = Config.get().isTraceOtelSemanticsEnabled(); final CoreSubscriber actual; @@ -41,30 +39,23 @@ public void onSubscribe(final Subscription subscription) { @Override public void onNext(final ClientResponse response) { - // Without a transport span, callbacks may wait for this span to finish. - if (OTEL_SEMANTICS) { - finishSpan(response); - } try (final ContextScope scope = activateSpan(parent)) { actual.onNext(response); } finally { - if (!OTEL_SEMANTICS) { - finishSpan(response); - } + DECORATE.onResponse(span, response); + DECORATE.beforeFinish(span); + span.finish(); } } @Override public void onError(final Throwable t) { - if (OTEL_SEMANTICS) { - finishSpan(t); - } try (final ContextScope scope = activateSpan(parent)) { actual.onError(t); } finally { - if (!OTEL_SEMANTICS) { - finishSpan(t); - } + DECORATE.onError(span, t); + DECORATE.beforeFinish(span); + span.finish(); } } @@ -85,16 +76,4 @@ public void onCancel() { public Context currentContext() { return context; } - - private void finishSpan(final ClientResponse response) { - DECORATE.onResponse(span, response); - DECORATE.beforeFinish(span); - span.finish(); - } - - private void finishSpan(final Throwable error) { - DECORATE.onError(span, error); - DECORATE.beforeFinish(span); - span.finish(); - } } diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java index c7a4e89a95d..ad84ba4be92 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/main/java/datadog/trace/instrumentation/springwebflux/client/WebClientTracingFilter.java @@ -3,17 +3,12 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; -import static datadog.trace.bootstrap.instrumentation.decorator.HttpClientDecorator.suppressNestedClientSpans; import static datadog.trace.instrumentation.springwebflux.client.SpringWebfluxHttpClientDecorator.DECORATE; import static datadog.trace.instrumentation.springwebflux.client.SpringWebfluxHttpClientDecorator.HTTP_REQUEST; -import datadog.context.Context; -import datadog.context.ContextKey; import datadog.context.ContextScope; -import datadog.trace.api.Config; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import java.util.List; -import org.reactivestreams.Subscription; import org.springframework.web.reactive.function.client.ClientRequest; import org.springframework.web.reactive.function.client.ClientResponse; import org.springframework.web.reactive.function.client.ExchangeFilterFunction; @@ -26,14 +21,7 @@ * https://github.com/spring-cloud/spring-cloud-sleuth/blob/master/spring-cloud-sleuth-core/src/main/java/org/springframework/cloud/sleuth/instrument/web/client/TraceWebClientBeanPostProcessor.java */ public class WebClientTracingFilter implements ExchangeFilterFunction { - private static final boolean OTEL_SEMANTICS = Config.get().isTraceOtelSemanticsEnabled(); - private static final ContextKey WEBCLIENT_SPAN = - ContextKey.named("spring-webclient-span"); - private static final ContextKey WEBCLIENT_PARENT = - ContextKey.named("spring-webclient-parent"); private static final WebClientTracingFilter INSTANCE = new WebClientTracingFilter(); - private static final ExchangeFilterFunction TRANSPORT_FILTER = - (request, next) -> new MonoWebClientTransport(request, next); /** * It's not expected that this method would be called concurrently. This should only be created @@ -44,10 +32,6 @@ public static void addFilter(final List exchangeFilterFu // Since the builder we instrument the can be reused, we need to only add the filter once exchangeFilterFunctions.removeIf(filterFunction -> filterFunction == INSTANCE); exchangeFilterFunctions.add(0, INSTANCE); - if (OTEL_SEMANTICS) { - exchangeFilterFunctions.removeIf(filterFunction -> filterFunction == TRANSPORT_FILTER); - exchangeFilterFunctions.add(TRANSPORT_FILTER); - } } @Override @@ -70,113 +54,11 @@ public void subscribe(final CoreSubscriber subscriber) { DECORATE.afterStart(span); DECORATE.onRequest(span, request); AgentSpan parent = activeSpan(); - Context parentContext = Context.current(); - try (final ContextScope scope = activateSpan(span); - final ContextScope ownerScope = - OTEL_SEMANTICS - ? Context.current() - .with(WEBCLIENT_SPAN, span) - .with(WEBCLIENT_PARENT, parentContext) - .attach() - : null) { + try (final ContextScope scope = activateSpan(span)) { TraceWebClientSubscriber tracingSubscriber = new TraceWebClientSubscriber(subscriber, span, parent); next.exchange(request).doOnCancel(tracingSubscriber::onCancel).subscribe(tracingSubscriber); } } } - - /** Applies transport suppression after the application's filters have run. */ - public static final class MonoWebClientTransport extends Mono { - private final ClientRequest request; - private final ExchangeFunction next; - - public MonoWebClientTransport(ClientRequest request, ExchangeFunction next) { - this.request = request; - this.next = next; - } - - @Override - public void subscribe(CoreSubscriber subscriber) { - Context context = Context.current(); - AgentSpan owner = context.get(WEBCLIENT_SPAN); - if (owner == null || owner != activeSpan()) { - next.exchange(request).subscribe(subscriber); - return; - } - try (ContextScope suppression = - suppressNestedClientSpans(owner, context.get(WEBCLIENT_PARENT))) { - next.exchange(request) - .subscribe(new TransportSubscriber(subscriber, context, Context.current())); - } - } - } - - /** - * Keeps connector demand in its transport context and filter callbacks in their owner context. - */ - public static final class TransportSubscriber - implements CoreSubscriber, Subscription { - private final CoreSubscriber actual; - private final Context ownerContext; - private final Context transportContext; - private Subscription subscription; - - public TransportSubscriber( - CoreSubscriber actual, - Context ownerContext, - Context transportContext) { - this.actual = actual; - this.ownerContext = ownerContext; - this.transportContext = transportContext; - } - - @Override - public reactor.util.context.Context currentContext() { - return actual.currentContext(); - } - - @Override - public void onSubscribe(Subscription subscription) { - this.subscription = subscription; - try (ContextScope scope = ownerContext.attach()) { - actual.onSubscribe(this); - } - } - - @Override - public void request(long count) { - try (ContextScope scope = transportContext.attach()) { - subscription.request(count); - } - } - - @Override - public void cancel() { - try (ContextScope scope = transportContext.attach()) { - subscription.cancel(); - } - } - - @Override - public void onNext(ClientResponse response) { - try (ContextScope scope = ownerContext.attach()) { - actual.onNext(response); - } - } - - @Override - public void onError(Throwable error) { - try (ContextScope scope = ownerContext.attach()) { - actual.onError(error); - } - } - - @Override - public void onComplete() { - try (ContextScope scope = ownerContext.attach()) { - actual.onComplete(); - } - } - } } diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBase.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBase.groovy index 11c86b54dbf..672f19c70ae 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBase.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBase.groovy @@ -6,6 +6,8 @@ import datadog.trace.agent.test.naming.TestingGenericHttpNamingConventions import datadog.trace.api.Config import datadog.trace.api.DDSpanTypes import datadog.trace.api.DDTags +import datadog.trace.api.KnownTags +import datadog.trace.bootstrap.instrumentation.api.OtelHttpSemantics import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.bootstrap.instrumentation.api.URIUtils import datadog.trace.instrumentation.netty41.client.NettyHttpClientDecorator @@ -40,16 +42,19 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes } .block() - finishRequest(response, method, hasParent, callback) + if (hasParent) { + blockUntilChildSpansFinished(callback ? 3 : 2) + } + + check() + + consumeResponse(response) } - protected static int consumeResponse(ClientResponse response, String method) { + protected static int consumeResponse(ClientResponse response) { int status = response.statusCode().value() Channel channel = responseChannel(response) - // Only skip HEAD body consumption in OpenTelemetry mode. - if (!Config.get().isTraceOtelSemanticsEnabled() || method != "HEAD") { - response.bodyToMono(Void).block() - } + response.bodyToMono(Void).block() // Body completion can occur inside channelRead, before channelReadComplete is fired. channel.eventLoop().submit({} as Runnable).sync() return status @@ -67,16 +72,6 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes return field.get(target) } - protected int finishRequest(ClientResponse response, String method, boolean hasParent, Closure callback) { - if (hasParent) { - blockUntilChildSpansFinished(size(callback ? 2 : 1)) - } - - check() - - consumeResponse(response, method) - } - @Override CharSequence component() { return SpringWebfluxHttpClientDecorator.DECORATE.component() @@ -98,9 +93,10 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes boolean ignorePeer = false, Map extraTags = null) { + def otelSemantics = Config.get().isTraceOtelSemanticsEnabled() def leafParentId = trace.spanAssertCount.get() super.clientSpan(trace, parentSpan, method, renameService, tagQueryString, uri, status, error, exception, ignorePeer, extraTags) - if (!exception && !Config.get().isTraceOtelSemanticsEnabled()) { + if (!exception) { def expectedQuery = tagQueryString ? uri.query : null def expectedUrl = URIUtils.buildURL(uri.scheme, uri.host, uri.port, uri.path) if (expectedQuery != null && !expectedQuery.empty) { @@ -112,9 +108,9 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes serviceName("localhost") } operationName NettyHttpClientDecorator.DECORATE.operationName() - resourceName "$method $uri.path" + resourceName otelSemantics ? OtelHttpSemantics.spanNameMethod(method) : "$method $uri.path" spanType DDSpanTypes.HTTP_CLIENT - errored error + errored otelSemantics && status != null && status >= 400 ? true : error measured true tags { "$Tags.COMPONENT" NettyHttpClientDecorator.DECORATE.component() @@ -127,17 +123,26 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes if (status) { "$Tags.HTTP_STATUS" status } - if (tagQueryString) { + if (otelSemantics) { + "$KnownTags.HTTP_QUERY_STRING_NAME" null + "$DDTags.HTTP_FRAGMENT" null + } else if (tagQueryString) { "$DDTags.HTTP_QUERY" expectedQuery "$DDTags.HTTP_FRAGMENT" { it == null || it == uri.fragment } // Optional } if (exception) { errorTags(exception.class, exception.message) + } else if (otelSemantics && status != null && status >= 400) { + "$DDTags.ERROR_TYPE" status.toString() } if ({ isDataStreamsEnabled() }) { "$DDTags.PATHWAY_HASH" { String } } - defaultTags() + if (otelSemantics) { + defaultTagsNoPeerService() + } else { + defaultTags() + } if (extraTags) { it.addTags(extraTags) } @@ -148,7 +153,7 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes @Override int size(int size) { - return Config.get().isTraceOtelSemanticsEnabled() ? size : size + 1 + return size + 1 } boolean testRedirects() { diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBasicTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBasicTest.groovy index bb7d11c0474..a6cd953b133 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBasicTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientBasicTest.groovy @@ -1,17 +1,7 @@ package dd.trace.instrumentation.springwebflux.client import datadog.trace.agent.test.naming.TestingOtelHttpSemantics -import datadog.trace.instrumentation.netty41.client.HttpClientTracingHandler -import io.netty.channel.embedded.EmbeddedChannel -import io.netty.handler.codec.http.DefaultHttpRequest -import io.netty.handler.codec.http.DefaultHttpResponse -import io.netty.handler.codec.http.HttpMethod -import io.netty.handler.codec.http.HttpResponseStatus -import io.netty.handler.codec.http.HttpVersion -import org.springframework.web.reactive.function.client.ExchangeFilterFunction import org.springframework.web.reactive.function.client.WebClient -import reactor.core.publisher.Mono -import reactor.core.scheduler.Schedulers import spock.lang.Timeout @Timeout(5) @@ -29,56 +19,4 @@ class SpringWebfluxHttpClientBasicTest extends SpringWebfluxHttpClientBase { @Timeout(5) class SpringWebfluxHttpClientOtelSemanticsForkedTest extends SpringWebfluxHttpClientBasicTest implements TestingOtelHttpSemantics { - def "an independent Netty request in a user filter retains its client span"() { - setup: - def independentRequest = { - def channel = new EmbeddedChannel(new HttpClientTracingHandler()) - try { - channel.writeOutbound(new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, 'http://filter.example/token')) - channel.readOutbound() - channel.writeInbound(new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK)) - } finally { - channel.finishAndReleaseAll() - } - } - def webClient = WebClient.builder().filter({ request, next -> - def exchange = Mono.defer { - if (beforeResponse) { - independentRequest() - } - next.exchange(request).doOnNext { - if (!beforeResponse) { - independentRequest() - } - } - } - async ? exchange.subscribeOn(Schedulers.parallel()) : exchange - } as ExchangeFilterFunction).build() - - when: - def response = webClient.get().uri(server.address.resolve('/success')).exchange().block() - consumeResponse(response, 'GET') - TEST_WRITER.waitForTraces(1) - - then: - def spans = TEST_WRITER.flatten() - def webClientSpan = spans.find { it.getTag('component')?.toString() == 'spring-webflux-client' } - def transportSpans = spans.findAll { it.getTag('component')?.toString() == 'netty-client' } - transportSpans.size() == 1 - transportSpans[0].getTag('url.full').toString() == 'http://filter.example/token' - transportSpans[0].parentId == webClientSpan.spanId - - where: - beforeResponse | async - true | false - true | true - false | false - false | true - } -} - -class SpringWebfluxHttpClientFilterBuilderReuseOtelSemanticsForkedTest extends SpringWebfluxHttpClientFilterBuilderReuseTest implements TestingOtelHttpSemantics { -} - -class SpringWebfluxHttpClientFiltersRemovedOtelSemanticsForkedTest extends SpringWebfluxHttpClientFiltersRemovedTest implements TestingOtelHttpSemantics { } diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy index 3cefc17f0a2..8355d53d18d 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy @@ -23,7 +23,13 @@ class SpringWebfluxHttpClientDoAfterTerminateTest extends SpringWebfluxHttpClien } .block() - finishRequest(response, method, hasParent, callback) + if (hasParent) { + blockUntilChildSpansFinished(callback ? 3 : 2) + } + + check() + + consumeResponse(response) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy index 01ebace29ee..366e87d6df1 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy @@ -23,7 +23,13 @@ abstract class SpringWebfluxHttpClientDoOnSuccessOrErrorTest extends SpringWebfl } .block() - finishRequest(response, method, hasParent, callback) + if (hasParent) { + blockUntilChildSpansFinished(callback ? 3 : 2) + } + + check() + + consumeResponse(response) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy index 5cffc4e6b26..d6fbb96bc75 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy @@ -23,7 +23,13 @@ class SpringWebfluxHttpClientDoOnSuccessTest extends SpringWebfluxHttpClientBase } .block() - finishRequest(response, method, hasParent, callback) + if (hasParent) { + blockUntilChildSpansFinished(callback ? 3 : 2) + } + + check() + + consumeResponse(response) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy index 2706301f93f..b6b05def34d 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-5.0/src/test/groovy/dd/trace/instrumentation/springwebflux/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy @@ -23,7 +23,13 @@ class SpringWebfluxHttpClientDoOnTerminateTest extends SpringWebfluxHttpClientBa } .block() - finishRequest(response, method, hasParent, callback) + if (hasParent) { + blockUntilChildSpansFinished(callback ? 3 : 2) + } + + check() + + consumeResponse(response) } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/bootTest/groovy/SpringWebfluxTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/bootTest/groovy/SpringWebfluxTest.groovy index 2d820b8a4da..9b49386871c 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/bootTest/groovy/SpringWebfluxTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/bootTest/groovy/SpringWebfluxTest.groovy @@ -77,8 +77,9 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { def traceParent sortSpansByStart() - trace(clientSpanCount()) { - traceParent = clientSpans(it, "GET", URI.create(url)) + trace(2) { + clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) } trace(2) { span { @@ -158,8 +159,9 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { sortSpansByStart() def traceParent - trace(clientSpanCount()) { - traceParent = clientSpans(it, "GET", URI.create(url)) + trace(2) { + clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) } trace(3) { span { @@ -251,8 +253,9 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { sortSpansByStart() def traceParent - trace(clientSpanCount()) { - traceParent = clientSpans(it, "GET", URI.create(url)) + trace(2) { + clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) } trace(3) { span { @@ -298,8 +301,9 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { sortSpansByStart() def traceParent - trace(clientSpanCount()) { - traceParent = clientSpans(it, "GET", URI.create(url), 404, true) + trace(2) { + clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), 404, true) + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), 404, true) } trace(2) { span { @@ -353,8 +357,9 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { sortSpansByStart() def traceParent - trace(clientSpanCount()) { - traceParent = clientSpans(it, "POST", URI.create(url), 202) + trace(2) { + clientSpan(it, null, "http.request", "spring-webflux-client", "POST", URI.create(url), 202) + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "POST", URI.create(url), 202) } trace(3) { span { @@ -417,8 +422,9 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { sortSpansByStart() def traceParent - trace(clientSpanCount()) { - traceParent = clientSpans(it, "GET", URI.create(url), 500) + trace(2) { + clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), 500) + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), 500) } trace(2) { span { @@ -507,9 +513,10 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(4, SORT_TRACES_BY_START) { def traceParent1, traceParent2 - trace(clientSpanCount()) { + trace(2) { sortSpansByStart() - traceParent1 = clientSpans(it, "GET", URI.create(url), 307) + clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), 307) + traceParent1 = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), 307) } trace(2) { @@ -552,9 +559,10 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { } } } - trace(clientSpanCount()) { + trace(2) { sortSpansByStart() - traceParent2 = clientSpans(it, "GET", URI.create(finalUrl)) + clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(finalUrl)) + traceParent2 = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(finalUrl)) } trace(2) { sortSpansByStart() @@ -612,8 +620,9 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { sortSpansByStart() responses.eachWithIndex { def response, int i -> def traceParent - trace(clientSpanCount()) { - traceParent = clientSpans(it, "GET", URI.create(url)) + trace(2) { + clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url)) + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url)) } trace(2) { span { @@ -686,12 +695,10 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { assertTraces(2) { def traceParent sortSpansByStart() - trace(clientSpanCount()) { - traceParent = clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), null, false, false, + trace(2) { + clientSpan(it, null, "http.request", "spring-webflux-client", "GET", URI.create(url), null, false, false, ["message": "The subscription was cancelled", "event": "cancelled"]) - if (!Config.get().isTraceOtelSemanticsEnabled()) { - traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), null) - } + traceParent = clientSpan(it, span(0), "netty.client.request", "netty-client", "GET", URI.create(url), null) } trace(2) { span { @@ -812,18 +819,6 @@ class SpringWebfluxHttp11Test extends InstrumentationSpecification { } } - int clientSpanCount() { - Config.get().isTraceOtelSemanticsEnabled() ? 1 : 2 - } - - def clientSpans(TraceAssert trace, String method, URI uri, Integer status = 200, boolean error = false) { - def parent = clientSpan(trace, null, "http.request", "spring-webflux-client", method, uri, status, error) - if (!Config.get().isTraceOtelSemanticsEnabled()) { - parent = clientSpan(trace, parent, "netty.client.request", "netty-client", method, uri, status, error) - } - return parent - } - def clientSpan( TraceAssert trace, Object parentSpan, diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBase.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBase.groovy index b7185c2dcd3..b419398ca6a 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBase.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBase.groovy @@ -3,7 +3,6 @@ package dd.trace.instrumentation.springwebflux6.client import datadog.trace.agent.test.asserts.TraceAssert import datadog.trace.agent.test.base.HttpClientTest import datadog.trace.agent.test.naming.TestingGenericHttpNamingConventions -import datadog.trace.api.Config import datadog.trace.api.DDSpanTypes import datadog.trace.api.DDTags import datadog.trace.api.naming.SpanNaming @@ -44,12 +43,8 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes } .block() - finishRequest(response, hasParent, callback) - } - - protected int finishRequest(ClientResponse response, boolean hasParent, Closure callback) { if (hasParent) { - blockUntilChildSpansFinished(size(callback ? 2 : 1)) + blockUntilChildSpansFinished(callback ? 3 : 2) } check() @@ -79,7 +74,7 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes Map extraTags = null) { def leafParentId = trace.spanAssertCount.get() super.clientSpan(trace, parentSpan, method, renameService, tagQueryString, uri, status, error, exception, ignorePeer, extraTags) - if (!exception && !Config.get().isTraceOtelSemanticsEnabled()) { + if (!exception) { def expectedQuery = tagQueryString ? uri.query : null def expectedUrl = URIUtils.buildURL(uri.scheme, uri.host, uri.port, uri.path) if (expectedQuery != null && !expectedQuery.empty) { @@ -127,7 +122,7 @@ abstract class SpringWebfluxHttpClientBase extends HttpClientTest implements Tes @Override int size(int size) { - return Config.get().isTraceOtelSemanticsEnabled() ? size : size + 1 + return size + 1 } boolean testRedirects() { diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBasicTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBasicTest.groovy index 4ff945bade9..b6510d4f7bc 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBasicTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientBasicTest.groovy @@ -1,6 +1,5 @@ package dd.trace.instrumentation.springwebflux6.client -import datadog.trace.agent.test.naming.TestingOtelHttpSemantics import org.springframework.web.reactive.function.client.WebClient import spock.lang.Timeout @@ -16,7 +15,3 @@ class SpringWebfluxHttpClientBasicTest extends SpringWebfluxHttpClientBase { void check() { } } - -@Timeout(5) -class SpringWebfluxHttpClientOtelSemanticsForkedTest extends SpringWebfluxHttpClientBasicTest implements TestingOtelHttpSemantics { -} diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy index 3ef6d2eb8bc..36adaa1b9c4 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoAfterTerminateTest.groovy @@ -29,7 +29,13 @@ class SpringWebfluxHttpClientDoAfterTerminateTest extends SpringWebfluxHttpClien } .block() - finishRequest(response, hasParent, callback) + if (hasParent) { + blockUntilChildSpansFinished(callback ? 3 : 2) + } + + check() + + response.statusCode().value() } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy index 582ef9aed8c..625d91084ec 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessOrErrorTest.groovy @@ -29,7 +29,13 @@ abstract class SpringWebfluxHttpClientDoOnSuccessOrErrorTest extends SpringWebfl } .block() - finishRequest(response, hasParent, callback) + if (hasParent) { + blockUntilChildSpansFinished(callback ? 3 : 2) + } + + check() + + response.statusCode().value() } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy index 3b51ff35fca..be489e8a6fe 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnSuccessTest.groovy @@ -29,7 +29,13 @@ class SpringWebfluxHttpClientDoOnSuccessTest extends SpringWebfluxHttpClientBase } .block() - finishRequest(response, hasParent, callback) + if (hasParent) { + blockUntilChildSpansFinished(callback ? 3 : 2) + } + + check() + + response.statusCode().value() } @Override diff --git a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy index 6e035bc0240..c4dc5bd9d2f 100644 --- a/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy +++ b/dd-java-agent/instrumentation/spring/spring-webflux/spring-webflux-6.0/src/test/groovy/dd/trace/instrumentation/springwebflux6/client/SpringWebfluxHttpClientDoOnTerminateTest.groovy @@ -29,7 +29,13 @@ class SpringWebfluxHttpClientDoOnTerminateTest extends SpringWebfluxHttpClientBa } .block() - finishRequest(response, hasParent, callback) + if (hasParent) { + blockUntilChildSpansFinished(callback ? 3 : 2) + } + + check() + + response.statusCode().value() } @Override From cf0fce04f32f7d16c28e46e828dcfd8e9dcb94d4 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Thu, 5 Nov 2026 10:56:22 +0100 Subject: [PATCH 25/34] Allow opting into Akka and Pekko HTTP route tags independently of OTel semantics --- .../AkkaHttpServerInstrumentationTest.groovy | 79 ++++++++++++++++++- .../akkahttp/AkkaHttpRouteHelperTest.java | 37 +++++++++ .../scala/AkkaHttpTestWebServer.scala | 3 + .../AkkaHttpPathMatcherInstrumentation.java | 10 ++- .../akkahttp/AkkaHttpRouteHelper.java | 32 +++++--- .../akkahttp/AkkaHttpRouteState.java | 43 +++++++--- .../akkahttp/DatadogWrapperHelper.java | 6 +- .../PekkoHttpServerInstrumentationTest.groovy | 79 ++++++++++++++++++- .../pekkohttp/PekkoHttpRouteHelperTest.java | 37 +++++++++ .../scala/PekkoHttpTestWebServer.scala | 3 + .../pekkohttp/DatadogWrapperHelper.java | 6 +- .../PekkoHttpPathMatcherInstrumentation.java | 10 ++- .../pekkohttp/PekkoHttpRouteHelper.java | 32 +++++--- .../pekkohttp/PekkoHttpRouteState.java | 43 +++++++--- .../main/java/datadog/trace/api/Config.java | 19 ++--- .../datadog/trace/api/InstrumenterConfig.java | 12 +++ .../java/datadog/trace/api/ConfigTest.java | 27 +++++++ .../trace/api/InstrumenterConfigTest.java | 15 ++++ metadata/supported-configurations.json | 16 ++++ 19 files changed, 432 insertions(+), 77 deletions(-) create mode 100644 dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelperTest.java create mode 100644 dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelperTest.java diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy index 42e1fc1abbb..528fb8e6886 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy @@ -357,7 +357,7 @@ abstract class AkkaHttpRouteOtelSemanticsTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0 { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig(TraceInstrumentationConfig.LEGACY_CONTEXT_MANAGER_ENABLED, "true") + } + + @Override + HttpServer server() { + new AkkaHttpTestWebServer(AkkaHttpTestWebServer.BindAndHandle()) + } + + boolean routeEnabled() { + false + } + boolean otelEnabled() { + false + } + + def "route opt in respects naming semantics"() { + when: + def response = client.newCall(new Request.Builder().url(address.resolve('/items/value').toString()).build()).execute() + assert response.code() == 200 + response.close() + TEST_WRITER.waitForTraces(1) + + then: + def span = TEST_WRITER.flatten().find { it.getTag('component')?.toString() == 'akka-http-server' } + span.getTag('http.route') == (routeEnabled() ? '/items/{param}' : null) + span.resourceName.toString() == (otelEnabled() ? 'GET' : 'GET /items/value') + } + + def "rejected route leaves the original resource"() { + when: + def response = client.newCall(new Request.Builder().url(address.resolve(path).toString()).build()).execute() + assert response.code() == 200 + assert response.body().string() == 'fallback' + response.close() + TEST_WRITER.waitForTraces(1) + + then: + def span = TEST_WRITER.flatten().find { it.getTag('component')?.toString() == 'akka-http-server' } + span.getTag('http.route') == null + span.resourceName.toString() == (otelEnabled() ? 'GET' : "GET $path") + + where: + path << ['/header-rejection/value', '/zero-consumption-rejection/value'] + } +} + +class AkkaHttpRouteEnabledForkedTest extends AkkaHttpRouteConfigurationTest { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig('integration.akka-http-route.enabled', 'true') + } + + @Override + boolean routeEnabled() { + true + } +} + +class AkkaHttpRouteDisabledOtelSemanticsForkedTest extends AkkaHttpRouteConfigurationTest implements TestingOtelHttpSemantics { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig('integration.akka-http-route.enabled', 'false') + } + + @Override + boolean otelEnabled() { + true + } +} diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelperTest.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelperTest.java new file mode 100644 index 00000000000..2e5f95329a3 --- /dev/null +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelperTest.java @@ -0,0 +1,37 @@ +package datadog.trace.instrumentation.akkahttp; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +import java.lang.reflect.Field; +import org.junit.jupiter.api.Test; + +class AkkaHttpRouteHelperTest { + @Test + @SuppressWarnings("unchecked") + void throwingNestedMatcherRestoresAndRemovesSynchronousFrames() throws Exception { + Field field = AkkaHttpRouteHelper.class.getDeclaredField("CURRENT"); + field.setAccessible(true); + ThreadLocal current = + (ThreadLocal) field.get(null); + AkkaHttpRouteState state = new AkkaHttpRouteState("/items/value"); + state.routeMatched = true; + AkkaHttpRouteHelper.Frame outer = new AkkaHttpRouteHelper.Frame(null, null, state, null); + current.set(outer); + try { + AkkaHttpRouteHelper.Frame inner = AkkaHttpRouteHelper.enter(null); + assertSame(outer, inner.parent); + assertSame(inner, current.get()); + RuntimeException failure = new RuntimeException("matcher failure"); + AkkaHttpRouteHelper.exit(inner, null, failure); + assertSame(outer, current.get()); + AkkaHttpRouteHelper.exit(outer, null, failure); + assertNull(current.get()); + assertFalse(state.routeMatched); + assertNull(AkkaHttpRouteHelper.enter(null)); + } finally { + current.remove(); + } + } +} diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/scala/AkkaHttpTestWebServer.scala b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/scala/AkkaHttpTestWebServer.scala index dde27b7d83d..9ac627171c3 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/scala/AkkaHttpTestWebServer.scala +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/scala/AkkaHttpTestWebServer.scala @@ -251,6 +251,9 @@ object AkkaHttpTestWebServer { path(Segment) { _ => headerValueByName("authorization") { _ => complete("accepted") } } ~ complete("fallback") + } ~ pathPrefix("zero-consumption-rejection") { + path(Segment) { _ => reject } ~ + rawPathPrefix(Segment.?) { _ => complete("fallback") } } ~ path("items" / Segment) { _ => complete("item") } ~ pathPrefix("route-async" / IntNumber) { id => diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java index 2a760139b3d..12a318ed4ec 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java @@ -15,7 +15,7 @@ import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.api.Config; +import datadog.trace.api.InstrumenterConfig; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.matcher.ElementMatcher; @@ -25,12 +25,14 @@ public final class AkkaHttpPathMatcherInstrumentation extends InstrumenterModule.Tracing implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { public AkkaHttpPathMatcherInstrumentation() { - super("akka-http", "akka-http-server"); + // Keep this opt-in independent of an explicit parent HTTP integration enablement. + super("akka-http-route"); } @Override - public boolean isEnabled() { - return super.isEnabled() && Config.get().isTraceOtelSemanticsEnabled(); + protected boolean defaultEnabled() { + // OTel enables route capture by default; the dedicated integration switch can override it. + return super.defaultEnabled() && InstrumenterConfig.get().isTraceOtelSemanticsEnabled(); } @Override diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java index ec57234bcef..6e9925cf7e4 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java @@ -14,11 +14,10 @@ import java.util.List; import scala.Product; -/** - * Builds templates from matched literals and extracted parameters, retaining no parameter values. - */ +/** Builds route templates from matched literals and parameter placeholders. */ public final class AkkaHttpRouteHelper { - private static final boolean RAW_RESOURCE = Config.get().isHttpServerRawResource(); + private static final boolean OTEL_SEMANTICS_ENABLED = Config.get().isTraceOtelSemanticsEnabled(); + // Matcher calls nest synchronously; request state carries routes across asynchronous directives. private static final ThreadLocal CURRENT = new ThreadLocal<>(); private AkkaHttpRouteHelper() {} @@ -32,7 +31,7 @@ public static Frame enter(Uri.Path path) { return frame; } AkkaHttpRouteState state = AkkaHttpRouteState.current(); - if (state == null) { + if (state == null || state.requestPath == null) { return null; } AgentSpan active = activeSpan(); @@ -40,7 +39,7 @@ public static Frame enter(Uri.Path path) { return null; } AgentSpan root = active.getLocalRootSpan(); - if (root == null || root.getTag(KnownTags.URL_PATH_NAME) == null) { + if (root == null) { return null; } Frame frame = new Frame(path, root, state, null); @@ -55,7 +54,9 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable boolean isMatch = throwable == null && matching instanceof PathMatcher.Matched; if (frame.parent == null) { CURRENT.remove(); - frame.state.routeMatched = isMatch; + if (!isMatch) { + frame.state.routeMatched = false; + } } else { CURRENT.set(frame.parent); } @@ -67,6 +68,7 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable String path = frame.path(); int consumed = path.length() - rest.length(); if (consumed <= 0) { + // An empty matcher must not revive a candidate rejected by an earlier alternative. return; } String template = frame.template(consumed, matched.extractions()); @@ -81,9 +83,7 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable return; } Route previous = null; - Object requestPath = frame.span.getTag(KnownTags.URL_PATH_NAME); - if (requestPath == null - || !requestPath.toString().equals(RAW_RESOURCE ? path : URIUtils.decode(path))) { + if (!frame.state.requestPath.equals(path)) { previous = frame.state.route; // A rejected route can retry an earlier unmatched path in another alternative. while (previous != null && !previous.remaining.equals(path)) { @@ -95,9 +95,15 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable template = previous.template + template; } frame.state.route = new Route(0, rest, template, previous); - Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); - HTTP_RESOURCE_DECORATOR.withRoute( - frame.span, method == null ? null : method.toString(), template, true); + frame.state.routeMatched = true; + if (OTEL_SEMANTICS_ENABLED) { + Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); + HTTP_RESOURCE_DECORATOR.withRoute( + frame.span, method == null ? null : method.toString(), template, true); + } else { + // withRoute also applies legacy route-based naming; this opt-in only adds the tag. + frame.span.setTag(KnownTags.HTTP_ROUTE_ID, URIUtils.decode(template)); + } } public static final class Frame { diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java index 933a7a3e187..187bba605ae 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java @@ -3,27 +3,41 @@ import static datadog.context.ContextKey.named; import static datadog.trace.bootstrap.instrumentation.api.OtelHttpSemantics.spanNameMethod; import static datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities.HTTP_FRAMEWORK_ROUTE; +import static java.util.Collections.singletonList; +import akka.http.scaladsl.model.Uri; import datadog.context.Context; import datadog.context.ContextKey; import datadog.trace.api.Config; +import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.KnownTags; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; /** Request-local route state, independent of the HTTP routing DSL. */ public final class AkkaHttpRouteState { - private static final boolean OTEL_SEMANTICS = Config.get().isTraceOtelSemanticsEnabled(); + private static final boolean OTEL_SEMANTICS_ENABLED = Config.get().isTraceOtelSemanticsEnabled(); private static final ContextKey KEY = named("akka-http-route"); + private static final boolean ROUTE_CAPTURE_ENABLED = + InstrumenterConfig.get() + .isIntegrationEnabled( + singletonList("akka-http-route"), + InstrumenterConfig.get().isIntegrationsEnabled() && OTEL_SEMANTICS_ENABLED); + + String requestPath; Route route; /** Whether the latest route candidate has matched without subsequently being rejected. */ boolean routeMatched; - private AkkaHttpRouteState() {} + AkkaHttpRouteState(String requestPath) { + this.requestPath = requestPath; + } - public static Context init(Context context) { - return OTEL_SEMANTICS ? context.with(KEY, new AkkaHttpRouteState()) : context; + public static Context init(Context context, Uri.Path requestPath) { + return ROUTE_CAPTURE_ENABLED + ? context.with(KEY, new AkkaHttpRouteState(requestPath.toString())) + : context; } static AkkaHttpRouteState current() { @@ -37,22 +51,29 @@ public static void onRejected() { } } - /** Drops a route candidate rejected by path matching or a later directive. */ - public static void dropRejectedRoute(Context context, AgentSpan span) { + /** Drops rejected candidates and releases route history after request completion. */ + public static void onComplete(Context context, AgentSpan span) { AkkaHttpRouteState state = context.get(KEY); - if (state == null || state.route == null || state.routeMatched) { + if (state == null) { return; } + state.requestPath = null; + boolean rejected = state.route != null && !state.routeMatched; state.route = null; + if (!rejected) { + return; + } AgentSpan root = span.getLocalRootSpan(); if (root == null) { root = span; } root.setTag(KnownTags.HTTP_ROUTE_ID, (String) null); - Object method = root.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); - // restore the method-only resource that OTel semantics give unrouted requests - root.setResourceName( - spanNameMethod(method == null ? null : method.toString()), HTTP_FRAMEWORK_ROUTE); + if (OTEL_SEMANTICS_ENABLED) { + Object method = root.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); + // Restore the method-only resource that OTel semantics give unrouted requests. + root.setResourceName( + spanNameMethod(method == null ? null : method.toString()), HTTP_FRAMEWORK_ROUTE); + } } public static final class Route { diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/DatadogWrapperHelper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/DatadogWrapperHelper.java index a537a49cb81..b9d9b052274 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/DatadogWrapperHelper.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/DatadogWrapperHelper.java @@ -33,7 +33,7 @@ private static Context startSpan(final HttpRequest request) { DECORATE.afterStart(span); DECORATE.onRequest(span, request, request, parentContext); - return AkkaHttpRouteState.init(context); + return AkkaHttpRouteState.init(context, request.uri().path()); } static final class SwappedContextScope implements ContextScope { @@ -64,7 +64,7 @@ public void close() { public static void finishSpan(final Context context, final HttpResponse response) { final AgentSpan span = fromContext(context); - AkkaHttpRouteState.dropRejectedRoute(context, span); + AkkaHttpRouteState.onComplete(context, span); DECORATE.onResponse(span, response); DECORATE.beforeFinish(context); @@ -73,7 +73,7 @@ public static void finishSpan(final Context context, final HttpResponse response public static void finishSpan(final Context context, final Throwable t) { final AgentSpan span = fromContext(context); - AkkaHttpRouteState.dropRejectedRoute(context, span); + AkkaHttpRouteState.onComplete(context, span); DECORATE.onError(span, t); span.setHttpStatusCode(500); DECORATE.beforeFinish(context); diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy index ca67f32d1dc..472da749d31 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy @@ -187,7 +187,7 @@ abstract class PekkoHttpRouteOtelSemanticsTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0 { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig(TraceInstrumentationConfig.LEGACY_CONTEXT_MANAGER_ENABLED, "true") + } + + @Override + HttpServer server() { + new PekkoHttpTestWebServer(PekkoHttpTestWebServer.BindAndHandle()) + } + + boolean routeEnabled() { + false + } + boolean otelEnabled() { + false + } + + def "route opt in respects naming semantics"() { + when: + def response = client.newCall(new Request.Builder().url(address.resolve('/items/value').toString()).build()).execute() + assert response.code() == 200 + response.close() + TEST_WRITER.waitForTraces(1) + + then: + def span = TEST_WRITER.flatten().find { it.getTag('component')?.toString() == 'pekko-http-server' } + span.getTag('http.route') == (routeEnabled() ? '/items/{param}' : null) + span.resourceName.toString() == (otelEnabled() ? 'GET' : 'GET /items/value') + } + + def "rejected route leaves the original resource"() { + when: + def response = client.newCall(new Request.Builder().url(address.resolve(path).toString()).build()).execute() + assert response.code() == 200 + assert response.body().string() == 'fallback' + response.close() + TEST_WRITER.waitForTraces(1) + + then: + def span = TEST_WRITER.flatten().find { it.getTag('component')?.toString() == 'pekko-http-server' } + span.getTag('http.route') == null + span.resourceName.toString() == (otelEnabled() ? 'GET' : "GET $path") + + where: + path << ['/header-rejection/value', '/zero-consumption-rejection/value'] + } +} + +class PekkoHttpRouteEnabledForkedTest extends PekkoHttpRouteConfigurationTest { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig('integration.pekko-http-route.enabled', 'true') + } + + @Override + boolean routeEnabled() { + true + } +} + +class PekkoHttpRouteDisabledOtelSemanticsForkedTest extends PekkoHttpRouteConfigurationTest implements TestingOtelHttpSemantics { + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig('integration.pekko-http-route.enabled', 'false') + } + + @Override + boolean otelEnabled() { + true + } +} diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelperTest.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelperTest.java new file mode 100644 index 00000000000..890c561435c --- /dev/null +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelperTest.java @@ -0,0 +1,37 @@ +package datadog.trace.instrumentation.pekkohttp; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +import java.lang.reflect.Field; +import org.junit.jupiter.api.Test; + +class PekkoHttpRouteHelperTest { + @Test + @SuppressWarnings("unchecked") + void throwingNestedMatcherRestoresAndRemovesSynchronousFrames() throws Exception { + Field field = PekkoHttpRouteHelper.class.getDeclaredField("CURRENT"); + field.setAccessible(true); + ThreadLocal current = + (ThreadLocal) field.get(null); + PekkoHttpRouteState state = new PekkoHttpRouteState("/items/value"); + state.routeMatched = true; + PekkoHttpRouteHelper.Frame outer = new PekkoHttpRouteHelper.Frame(null, null, state, null); + current.set(outer); + try { + PekkoHttpRouteHelper.Frame inner = PekkoHttpRouteHelper.enter(null); + assertSame(outer, inner.parent); + assertSame(inner, current.get()); + RuntimeException failure = new RuntimeException("matcher failure"); + PekkoHttpRouteHelper.exit(inner, null, failure); + assertSame(outer, current.get()); + PekkoHttpRouteHelper.exit(outer, null, failure); + assertNull(current.get()); + assertFalse(state.routeMatched); + assertNull(PekkoHttpRouteHelper.enter(null)); + } finally { + current.remove(); + } + } +} diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/scala/PekkoHttpTestWebServer.scala b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/scala/PekkoHttpTestWebServer.scala index a5346ae78e5..6e411221b26 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/scala/PekkoHttpTestWebServer.scala +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/scala/PekkoHttpTestWebServer.scala @@ -217,6 +217,9 @@ object PekkoHttpTestWebServer { path(Segment) { _ => headerValueByName("authorization") { _ => complete("accepted") } } ~ complete("fallback") + } ~ pathPrefix("zero-consumption-rejection") { + path(Segment) { _ => reject } ~ + rawPathPrefix(Segment.?) { _ => complete("fallback") } } ~ path("items" / Segment) { _ => complete("item") } ~ pathPrefix("route-async" / IntNumber) { id => diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/DatadogWrapperHelper.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/DatadogWrapperHelper.java index c21694d2e0b..93ae8d984f6 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/DatadogWrapperHelper.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/DatadogWrapperHelper.java @@ -33,7 +33,7 @@ private static Context startSpan(final HttpRequest request) { DECORATE.afterStart(span); DECORATE.onRequest(span, request, request, parentContext); - return PekkoHttpRouteState.init(context); + return PekkoHttpRouteState.init(context, request.uri().path()); } static final class SwappedContextScope implements ContextScope { @@ -64,7 +64,7 @@ public void close() { public static void finishSpan(final Context context, final HttpResponse response) { final AgentSpan span = fromContext(context); - PekkoHttpRouteState.dropRejectedRoute(context, span); + PekkoHttpRouteState.onComplete(context, span); try { DECORATE.onResponse(span, response); DECORATE.beforeFinish(context); @@ -75,7 +75,7 @@ public static void finishSpan(final Context context, final HttpResponse response public static void finishSpan(final Context context, final Throwable t) { final AgentSpan span = fromContext(context); - PekkoHttpRouteState.dropRejectedRoute(context, span); + PekkoHttpRouteState.onComplete(context, span); try { DECORATE.onError(span, t); span.setHttpStatusCode(500); diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java index 57730802fdb..1f7f28bf8d1 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java @@ -13,7 +13,7 @@ import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.api.Config; +import datadog.trace.api.InstrumenterConfig; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.matcher.ElementMatcher; @@ -25,12 +25,14 @@ public final class PekkoHttpPathMatcherInstrumentation extends InstrumenterModule.Tracing implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { public PekkoHttpPathMatcherInstrumentation() { - super("pekko-http", "pekko-http-server"); + // Keep this opt-in independent of an explicit parent HTTP integration enablement. + super("pekko-http-route"); } @Override - public boolean isEnabled() { - return super.isEnabled() && Config.get().isTraceOtelSemanticsEnabled(); + protected boolean defaultEnabled() { + // OTel enables route capture by default; the dedicated integration switch can override it. + return super.defaultEnabled() && InstrumenterConfig.get().isTraceOtelSemanticsEnabled(); } @Override diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java index 663828df94c..e2c4a93ad99 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java @@ -14,11 +14,10 @@ import org.apache.pekko.http.scaladsl.server.PathMatcher; import scala.Product; -/** - * Builds templates from matched literals and extracted parameters, retaining no parameter values. - */ +/** Builds route templates from matched literals and parameter placeholders. */ public final class PekkoHttpRouteHelper { - private static final boolean RAW_RESOURCE = Config.get().isHttpServerRawResource(); + private static final boolean OTEL_SEMANTICS_ENABLED = Config.get().isTraceOtelSemanticsEnabled(); + // Matcher calls nest synchronously; request state carries routes across asynchronous directives. private static final ThreadLocal CURRENT = new ThreadLocal<>(); private PekkoHttpRouteHelper() {} @@ -32,7 +31,7 @@ public static Frame enter(Uri.Path path) { return frame; } PekkoHttpRouteState state = PekkoHttpRouteState.current(); - if (state == null) { + if (state == null || state.requestPath == null) { return null; } AgentSpan active = activeSpan(); @@ -40,7 +39,7 @@ public static Frame enter(Uri.Path path) { return null; } AgentSpan root = active.getLocalRootSpan(); - if (root == null || root.getTag(KnownTags.URL_PATH_NAME) == null) { + if (root == null) { return null; } Frame frame = new Frame(path, root, state, null); @@ -55,7 +54,9 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable boolean isMatch = throwable == null && matching instanceof PathMatcher.Matched; if (frame.parent == null) { CURRENT.remove(); - frame.state.routeMatched = isMatch; + if (!isMatch) { + frame.state.routeMatched = false; + } } else { CURRENT.set(frame.parent); } @@ -67,6 +68,7 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable String path = frame.path(); int consumed = path.length() - rest.length(); if (consumed <= 0) { + // An empty matcher must not revive a candidate rejected by an earlier alternative. return; } String template = frame.template(consumed, matched.extractions()); @@ -81,9 +83,7 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable return; } Route previous = null; - Object requestPath = frame.span.getTag(KnownTags.URL_PATH_NAME); - if (requestPath == null - || !requestPath.toString().equals(RAW_RESOURCE ? path : URIUtils.decode(path))) { + if (!frame.state.requestPath.equals(path)) { previous = frame.state.route; // A rejected route can retry an earlier unmatched path in another alternative. while (previous != null && !previous.remaining.equals(path)) { @@ -95,9 +95,15 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable template = previous.template + template; } frame.state.route = new Route(0, rest, template, previous); - Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); - HTTP_RESOURCE_DECORATOR.withRoute( - frame.span, method == null ? null : method.toString(), template, true); + frame.state.routeMatched = true; + if (OTEL_SEMANTICS_ENABLED) { + Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); + HTTP_RESOURCE_DECORATOR.withRoute( + frame.span, method == null ? null : method.toString(), template, true); + } else { + // withRoute also applies legacy route-based naming; this opt-in only adds the tag. + frame.span.setTag(KnownTags.HTTP_ROUTE_ID, URIUtils.decode(template)); + } } public static final class Frame { diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java index 896bfcf155d..8e01347da7b 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java @@ -3,27 +3,41 @@ import static datadog.context.ContextKey.named; import static datadog.trace.bootstrap.instrumentation.api.OtelHttpSemantics.spanNameMethod; import static datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities.HTTP_FRAMEWORK_ROUTE; +import static java.util.Collections.singletonList; import datadog.context.Context; import datadog.context.ContextKey; import datadog.trace.api.Config; +import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.KnownTags; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import org.apache.pekko.http.scaladsl.model.Uri; /** Request-local route state, independent of the HTTP routing DSL. */ public final class PekkoHttpRouteState { - private static final boolean OTEL_SEMANTICS = Config.get().isTraceOtelSemanticsEnabled(); + private static final boolean OTEL_SEMANTICS_ENABLED = Config.get().isTraceOtelSemanticsEnabled(); private static final ContextKey KEY = named("pekko-http-route"); + private static final boolean ROUTE_CAPTURE_ENABLED = + InstrumenterConfig.get() + .isIntegrationEnabled( + singletonList("pekko-http-route"), + InstrumenterConfig.get().isIntegrationsEnabled() && OTEL_SEMANTICS_ENABLED); + + String requestPath; Route route; /** Whether the latest route candidate has matched without subsequently being rejected. */ boolean routeMatched; - private PekkoHttpRouteState() {} + PekkoHttpRouteState(String requestPath) { + this.requestPath = requestPath; + } - public static Context init(Context context) { - return OTEL_SEMANTICS ? context.with(KEY, new PekkoHttpRouteState()) : context; + public static Context init(Context context, Uri.Path requestPath) { + return ROUTE_CAPTURE_ENABLED + ? context.with(KEY, new PekkoHttpRouteState(requestPath.toString())) + : context; } static PekkoHttpRouteState current() { @@ -37,22 +51,29 @@ public static void onRejected() { } } - /** Drops a route candidate rejected by path matching or a later directive. */ - public static void dropRejectedRoute(Context context, AgentSpan span) { + /** Drops rejected candidates and releases route history after request completion. */ + public static void onComplete(Context context, AgentSpan span) { PekkoHttpRouteState state = context.get(KEY); - if (state == null || state.route == null || state.routeMatched) { + if (state == null) { return; } + state.requestPath = null; + boolean rejected = state.route != null && !state.routeMatched; state.route = null; + if (!rejected) { + return; + } AgentSpan root = span.getLocalRootSpan(); if (root == null) { root = span; } root.setTag(KnownTags.HTTP_ROUTE_ID, (String) null); - Object method = root.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); - // restore the method-only resource that OTel semantics give unrouted requests - root.setResourceName( - spanNameMethod(method == null ? null : method.toString()), HTTP_FRAMEWORK_ROUTE); + if (OTEL_SEMANTICS_ENABLED) { + Object method = root.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); + // Restore the method-only resource that OTel semantics give unrouted requests. + root.setResourceName( + spanNameMethod(method == null ? null : method.toString()), HTTP_FRAMEWORK_ROUTE); + } } public static final class Route { diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index ee09c1e9e2f..0df9a2579a8 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -187,7 +187,6 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_LONG_RUNNING_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_LONG_RUNNING_FLUSH_INTERVAL; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_LONG_RUNNING_INITIAL_FLUSH_INTERVAL; -import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_POST_PROCESSING_TIMEOUT; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_PROPAGATION_BEHAVIOR_EXTRACT; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_PROPAGATION_EXTRACT_FIRST; @@ -1039,7 +1038,6 @@ public static String getHostName() { private final OtlpConfig.Temporality otlpMetricsTemporalityPreference; private final boolean otelTracesSpanMetricsEnabled; - private final boolean traceOtelSemanticsEnabled; private final String traceOtelExporter; private final String otlpTracesEndpoint; @@ -1553,11 +1551,8 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins integrationSynapseLegacyOperationName = configProvider.getBoolean(INTEGRATION_SYNAPSE_LEGACY_OPERATION_NAME, false); - traceOtelSemanticsEnabled = - configProvider.getBoolean( - TRACE_OTEL_SEMANTICS_ENABLED, DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED); String configuredTraceOtelExporter = configProvider.getString(TRACE_OTEL_EXPORTER); - if (traceOtelSemanticsEnabled) { + if (isTraceOtelSemanticsEnabled()) { if (configuredTraceOtelExporter != null && !"otlp".equalsIgnoreCase(configuredTraceOtelExporter)) { logOverriddenSettingWarning(TRACE_OTEL_EXPORTER, TRACE_OTEL_SEMANTICS_ENABLED, "otlp"); @@ -1571,7 +1566,7 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins String configuredWriterType = configProvider.getString( WRITER_TYPE, otlpTracesExporter ? OTLP_WRITER_TYPE : DEFAULT_AGENT_WRITER_TYPE); - if (traceOtelSemanticsEnabled) { + if (isTraceOtelSemanticsEnabled()) { if (!OTLP_WRITER_TYPE.equals(configuredWriterType)) { logOverriddenSettingWarning(WRITER_TYPE, TRACE_OTEL_SEMANTICS_ENABLED, OTLP_WRITER_TYPE); } @@ -1793,7 +1788,7 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins isInjectDatadogAttributeEnabled(DEFAULT_INJECT_DATADOG_ATTRIBUTE, "sqs"); int configuredSpanAttributeSchemaVersion = schemaVersionFromConfig(); - if (traceOtelSemanticsEnabled) { + if (isTraceOtelSemanticsEnabled()) { if (configuredSpanAttributeSchemaVersion != SpanNaming.SCHEMA_MIN_VERSION) { logOtelSemanticsOverride(TRACE_SPAN_ATTRIBUTE_SCHEMA, "v" + SpanNaming.SCHEMA_MIN_VERSION); } @@ -1809,7 +1804,7 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins // in v1+ defaults are always calculated regardless this feature flag boolean configuredPeerServiceDefaultsEnabled = configProvider.getBoolean(TRACE_PEER_SERVICE_DEFAULTS_ENABLED, false); - if (traceOtelSemanticsEnabled) { + if (isTraceOtelSemanticsEnabled()) { if (configuredPeerServiceDefaultsEnabled) { logOtelSemanticsOverride(TRACE_PEER_SERVICE_DEFAULTS_ENABLED, false); } @@ -1849,7 +1844,7 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins httpClientErrorStatuses = configProvider.getIntegerRange( TRACE_HTTP_CLIENT_ERROR_STATUSES, - traceOtelSemanticsEnabled + isTraceOtelSemanticsEnabled() ? DEFAULT_OTEL_HTTP_CLIENT_ERROR_STATUSES : DEFAULT_HTTP_CLIENT_ERROR_STATUSES, HTTP_CLIENT_ERROR_STATUSES); @@ -5953,7 +5948,7 @@ public boolean isOtelTracesSpanMetricsEnabled() { } public boolean isTraceOtelSemanticsEnabled() { - return traceOtelSemanticsEnabled; + return instrumenterConfig.isTraceOtelSemanticsEnabled(); } public boolean isTraceOtelEnabled() { @@ -7183,8 +7178,6 @@ public String toString() { + otlpMetricsTemporalityPreference + ", otelTracesSpanMetricsEnabled=" + otelTracesSpanMetricsEnabled - + ", traceOtelSemanticsEnabled=" - + traceOtelSemanticsEnabled + ", traceStatsInterval=" + traceStatsInterval + ", traceOtelExporter=" diff --git a/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java b/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java index 21aa2a87173..9911ad6a901 100644 --- a/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java +++ b/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java @@ -28,6 +28,7 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_METHODS; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_NATIVE_METHODS; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_OTEL_ENABLED; +import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_USM_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_WEBSOCKET_MESSAGES_ENABLED; import static datadog.trace.api.FeatureFlaggingSource.AGENTLESS; @@ -47,6 +48,7 @@ import static datadog.trace.api.config.GeneralConfig.INTERNAL_EXIT_ON_FAILURE; import static datadog.trace.api.config.GeneralConfig.TELEMETRY_ENABLED; import static datadog.trace.api.config.GeneralConfig.TRACE_DEBUG; +import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED; import static datadog.trace.api.config.GeneralConfig.TRACE_TRIAGE; import static datadog.trace.api.config.GeneralConfig.TRIAGE_REPORT_TRIGGER; import static datadog.trace.api.config.IastConfig.IAST_ENABLED; @@ -173,6 +175,7 @@ public class InstrumenterConfig { private final boolean codeOriginInterfaceSupport; private final boolean traceEnabled; private final boolean traceOtelEnabled; + private final boolean traceOtelSemanticsEnabled; private final boolean metricsOtelEnabled; private final boolean logsOtelEnabled; private final ProfilingEnablement profilingEnabled; @@ -303,6 +306,9 @@ private InstrumenterConfig() { featureFlaggingConfigurationSource = AGENTLESS; } traceOtelEnabled = configProvider.getBoolean(TRACE_OTEL_ENABLED, DEFAULT_TRACE_OTEL_ENABLED); + traceOtelSemanticsEnabled = + configProvider.getBoolean( + TRACE_OTEL_SEMANTICS_ENABLED, DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED); metricsOtelEnabled = configProvider.getBoolean(METRICS_OTEL_ENABLED, DEFAULT_METRICS_OTEL_ENABLED); logsOtelEnabled = configProvider.getBoolean(LOGS_OTEL_ENABLED, DEFAULT_LOGS_OTEL_ENABLED); @@ -547,6 +553,10 @@ public boolean isTraceOtelEnabled() { return traceOtelEnabled; } + public boolean isTraceOtelSemanticsEnabled() { + return traceOtelSemanticsEnabled; + } + public boolean isMetricsOtelEnabled() { return metricsOtelEnabled; } @@ -859,6 +869,8 @@ public String toString() { + traceEnabled + ", traceOtelEnabled=" + traceOtelEnabled + + ", traceOtelSemanticsEnabled=" + + traceOtelSemanticsEnabled + ", metricsOtelEnabled=" + metricsOtelEnabled + ", logsOtelEnabled=" diff --git a/internal-api/src/test/java/datadog/trace/api/ConfigTest.java b/internal-api/src/test/java/datadog/trace/api/ConfigTest.java index 59230a4097e..38275eaf2b6 100644 --- a/internal-api/src/test/java/datadog/trace/api/ConfigTest.java +++ b/internal-api/src/test/java/datadog/trace/api/ConfigTest.java @@ -172,6 +172,7 @@ import static datadog.trace.api.config.TracerConfig.WRITER_TYPE; import static datadog.trace.api.env.FixedCapturedEnvironment.useFixedEnv; import static datadog.trace.bootstrap.config.provider.ConfigProvider.withPropertiesOverride; +import static datadog.trace.test.junit.utils.config.WithConfigExtension.injectSysConfig; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; @@ -724,6 +725,32 @@ void otelSemanticsForcesItsRequiredConfiguration() { assertEquals(false, calculated.get(TRACE_PEER_SERVICE_DEFAULTS_ENABLED).value); } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void otelSemanticsUsesTheInstanceConfiguration(boolean enabled) { + injectSysConfig(TRACE_OTEL_SEMANTICS_ENABLED, Boolean.toString(!enabled)); + // Keep the singleton snapshot while allowing the instance properties to supply the value. + System.clearProperty(PREFIX + TRACE_OTEL_SEMANTICS_ENABLED); + Properties properties = new Properties(); + properties.setProperty(TRACE_OTEL_SEMANTICS_ENABLED, Boolean.toString(enabled)); + properties.setProperty(TRACE_OTEL_EXPORTER, "agent"); + properties.setProperty(WRITER_TYPE, "LoggingWriter"); + properties.setProperty(TRACE_SPAN_ATTRIBUTE_SCHEMA, "v1"); + properties.setProperty(TRACE_PEER_SERVICE_DEFAULTS_ENABLED, "true"); + + Config config = Config.get(properties); + + assertEquals(!enabled, InstrumenterConfig.get().isTraceOtelSemanticsEnabled()); + assertEquals(enabled, config.isTraceOtelSemanticsEnabled()); + assertEquals(enabled, config.isTraceOtlpExporterEnabled()); + assertEquals(enabled ? "OtlpWriter" : "LoggingWriter", config.getWriterType()); + assertEquals(enabled ? 0 : 1, config.getSpanAttributeSchemaVersion()); + assertEquals(!enabled, config.isPeerServiceDefaultsEnabled()); + assertEquals( + enabled ? toBitSet(400, 599) : DEFAULT_HTTP_CLIENT_ERROR_STATUSES, + config.getHttpClientErrorStatuses()); + } + @Test void otelCheckSyntaxForOtlpHeaders() { Properties prop = new Properties(); diff --git a/internal-api/src/test/java/datadog/trace/api/InstrumenterConfigTest.java b/internal-api/src/test/java/datadog/trace/api/InstrumenterConfigTest.java index a2540030bf3..be353f87c30 100644 --- a/internal-api/src/test/java/datadog/trace/api/InstrumenterConfigTest.java +++ b/internal-api/src/test/java/datadog/trace/api/InstrumenterConfigTest.java @@ -2,6 +2,7 @@ import static datadog.environment.JavaVirtualMachine.isJavaVersionAtLeast; import static datadog.trace.api.config.AppSecConfig.APPSEC_ENABLED; +import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED; import static datadog.trace.api.config.IastConfig.IAST_ENABLED; import static datadog.trace.api.config.TraceInstrumentationConfig.RESOLVER_CACHE_CONFIG; import static datadog.trace.api.config.TraceInstrumentationConfig.RUNTIME_CONTEXT_MAP_PER_STORE; @@ -31,6 +32,20 @@ class InstrumenterConfigTest { private StrictnessPolicy strictness; + @TableTest({ + "Scenario | Configured | Expected", + "default | | false ", + "enabled | true | true ", + "disabled | false | false " + }) + void traceOtelSemantics(String configured, boolean expected) { + if (configured != null) { + injectSysConfig(TRACE_OTEL_SEMANTICS_ENABLED, configured); + } + assertEquals(expected, InstrumenterConfig.get().isTraceOtelSemanticsEnabled()); + assertEquals(expected, Config.get().isTraceOtelSemanticsEnabled()); + } + @BeforeEach void setup() { strictness = ConfigHelper.get().configInversionStrictFlag(); diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 4b71411c15d..4db17d95baf 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -4572,6 +4572,14 @@ "aliases": ["DD_TRACE_INTEGRATION_AKKA_HTTP_ENABLED", "DD_INTEGRATION_AKKA_HTTP_ENABLED"] } ], + "DD_TRACE_AKKA_HTTP_ROUTE_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": ["DD_TRACE_INTEGRATION_AKKA_HTTP_ROUTE_ENABLED", "DD_INTEGRATION_AKKA_HTTP_ROUTE_ENABLED"] + } + ], "DD_TRACE_AKKA_HTTP_SERVER_ANALYTICS_ENABLED": [ { "version": "A", @@ -9100,6 +9108,14 @@ "aliases": ["DD_TRACE_INTEGRATION_PEKKO_HTTP_ENABLED", "DD_INTEGRATION_PEKKO_HTTP_ENABLED"] } ], + "DD_TRACE_PEKKO_HTTP_ROUTE_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": ["DD_TRACE_INTEGRATION_PEKKO_HTTP_ROUTE_ENABLED", "DD_INTEGRATION_PEKKO_HTTP_ROUTE_ENABLED"] + } + ], "DD_TRACE_PEKKO_HTTP_SERVER_ANALYTICS_ENABLED": [ { "version": "A", From bb5f38f5d679ee518870260072cee892e0c2505c Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Thu, 5 Nov 2026 11:06:27 +0100 Subject: [PATCH 26/34] update golden stuff --- metadata/agent-jar-checks.properties | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metadata/agent-jar-checks.properties b/metadata/agent-jar-checks.properties index a5f8380fd4c..39977ab5205 100644 --- a/metadata/agent-jar-checks.properties +++ b/metadata/agent-jar-checks.properties @@ -12,6 +12,7 @@ classes.minimum.guard = 17000 expected.integrations = IastInstrumentation,\ aerospike,\ akka-http,\ + akka-http-route,\ akka-http2,\ akka_actor_mailbox,\ akka_actor_receive,\ @@ -138,6 +139,7 @@ expected.integrations = IastInstrumentation,\ opentracing,\ org-json,\ pekko-http,\ + pekko-http-route,\ pekko-http2,\ pekko_actor_mailbox,\ pekko_actor_receive,\ @@ -145,9 +147,9 @@ expected.integrations = IastInstrumentation,\ play,\ play-ws,\ quartz,\ + r2dbc,\ ratpack,\ ratpack-request-body,\ - r2dbc,\ reactive-streams,\ reactor-core,\ reactor-netty,\ From 9df879da6a780e9c84acdac50f94574b714b7c54 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Thu, 5 Nov 2026 11:24:35 +0100 Subject: [PATCH 27/34] improvements --- .../AkkaHttpServerInstrumentationTest.groovy | 19 ++++++++++++- .../akkahttp/AkkaHttpRouteHelper.java | 20 +++++++------- .../akkahttp/AkkaHttpRouteState.java | 12 ++++----- .../PekkoHttpServerInstrumentationTest.groovy | 19 ++++++++++++- .../pekkohttp/PekkoHttpRouteHelper.java | 20 +++++++------- .../pekkohttp/PekkoHttpRouteState.java | 12 ++++----- .../main/java/datadog/trace/api/Config.java | 19 ++++++++----- .../java/datadog/trace/api/ConfigTest.java | 27 ------------------- 8 files changed, 83 insertions(+), 65 deletions(-) diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy index 528fb8e6886..84bad173cfc 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy @@ -532,6 +532,10 @@ abstract class AkkaHttpRouteConfigurationTest extends WithHttpServer CURRENT = new ThreadLocal<>(); @@ -94,16 +96,16 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable } template = previous.template + template; } + if (frame.state.route == null && ROUTE_BASED_NAMING_ENABLED) { + frame.state.originalResourceName = frame.span.getResourceName(); + } frame.state.route = new Route(0, rest, template, previous); frame.state.routeMatched = true; - if (OTEL_SEMANTICS_ENABLED) { - Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); - HTTP_RESOURCE_DECORATOR.withRoute( - frame.span, method == null ? null : method.toString(), template, true); - } else { - // withRoute also applies legacy route-based naming; this opt-in only adds the tag. - frame.span.setTag(KnownTags.HTTP_ROUTE_ID, URIUtils.decode(template)); - } + Object method = + frame.span.getTag( + OTEL_SEMANTICS_ENABLED ? KnownTags.HTTP_METHOD_OTEL_NAME : Tags.HTTP_METHOD); + HTTP_RESOURCE_DECORATOR.withRoute( + frame.span, method == null ? null : method.toString(), template, true); } public static final class Frame { diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java index 187bba605ae..6c88d86375f 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java @@ -1,7 +1,6 @@ package datadog.trace.instrumentation.akkahttp; import static datadog.context.ContextKey.named; -import static datadog.trace.bootstrap.instrumentation.api.OtelHttpSemantics.spanNameMethod; import static datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities.HTTP_FRAMEWORK_ROUTE; import static java.util.Collections.singletonList; @@ -26,6 +25,7 @@ public final class AkkaHttpRouteState { String requestPath; Route route; + CharSequence originalResourceName; /** Whether the latest route candidate has matched without subsequently being rejected. */ boolean routeMatched; @@ -60,6 +60,8 @@ public static void onComplete(Context context, AgentSpan span) { state.requestPath = null; boolean rejected = state.route != null && !state.routeMatched; state.route = null; + CharSequence originalResourceName = state.originalResourceName; + state.originalResourceName = null; if (!rejected) { return; } @@ -68,11 +70,9 @@ public static void onComplete(Context context, AgentSpan span) { root = span; } root.setTag(KnownTags.HTTP_ROUTE_ID, (String) null); - if (OTEL_SEMANTICS_ENABLED) { - Object method = root.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); - // Restore the method-only resource that OTel semantics give unrouted requests. - root.setResourceName( - spanNameMethod(method == null ? null : method.toString()), HTTP_FRAMEWORK_ROUTE); + if (originalResourceName != null) { + // Undo tentative route naming without overriding higher-priority application names. + root.setResourceName(originalResourceName, HTTP_FRAMEWORK_ROUTE); } } diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy index 472da749d31..ad3bc984409 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy @@ -312,6 +312,10 @@ abstract class PekkoHttpRouteConfigurationTest extends WithHttpServer CURRENT = new ThreadLocal<>(); @@ -94,16 +96,16 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable } template = previous.template + template; } + if (frame.state.route == null && ROUTE_BASED_NAMING_ENABLED) { + frame.state.originalResourceName = frame.span.getResourceName(); + } frame.state.route = new Route(0, rest, template, previous); frame.state.routeMatched = true; - if (OTEL_SEMANTICS_ENABLED) { - Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); - HTTP_RESOURCE_DECORATOR.withRoute( - frame.span, method == null ? null : method.toString(), template, true); - } else { - // withRoute also applies legacy route-based naming; this opt-in only adds the tag. - frame.span.setTag(KnownTags.HTTP_ROUTE_ID, URIUtils.decode(template)); - } + Object method = + frame.span.getTag( + OTEL_SEMANTICS_ENABLED ? KnownTags.HTTP_METHOD_OTEL_NAME : Tags.HTTP_METHOD); + HTTP_RESOURCE_DECORATOR.withRoute( + frame.span, method == null ? null : method.toString(), template, true); } public static final class Frame { diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java index 8e01347da7b..d030356fdd0 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java @@ -1,7 +1,6 @@ package datadog.trace.instrumentation.pekkohttp; import static datadog.context.ContextKey.named; -import static datadog.trace.bootstrap.instrumentation.api.OtelHttpSemantics.spanNameMethod; import static datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities.HTTP_FRAMEWORK_ROUTE; import static java.util.Collections.singletonList; @@ -26,6 +25,7 @@ public final class PekkoHttpRouteState { String requestPath; Route route; + CharSequence originalResourceName; /** Whether the latest route candidate has matched without subsequently being rejected. */ boolean routeMatched; @@ -60,6 +60,8 @@ public static void onComplete(Context context, AgentSpan span) { state.requestPath = null; boolean rejected = state.route != null && !state.routeMatched; state.route = null; + CharSequence originalResourceName = state.originalResourceName; + state.originalResourceName = null; if (!rejected) { return; } @@ -68,11 +70,9 @@ public static void onComplete(Context context, AgentSpan span) { root = span; } root.setTag(KnownTags.HTTP_ROUTE_ID, (String) null); - if (OTEL_SEMANTICS_ENABLED) { - Object method = root.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); - // Restore the method-only resource that OTel semantics give unrouted requests. - root.setResourceName( - spanNameMethod(method == null ? null : method.toString()), HTTP_FRAMEWORK_ROUTE); + if (originalResourceName != null) { + // Undo tentative route naming without overriding higher-priority application names. + root.setResourceName(originalResourceName, HTTP_FRAMEWORK_ROUTE); } } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 0df9a2579a8..ee09c1e9e2f 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -187,6 +187,7 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_LONG_RUNNING_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_LONG_RUNNING_FLUSH_INTERVAL; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_LONG_RUNNING_INITIAL_FLUSH_INTERVAL; +import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_POST_PROCESSING_TIMEOUT; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_PROPAGATION_BEHAVIOR_EXTRACT; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_PROPAGATION_EXTRACT_FIRST; @@ -1038,6 +1039,7 @@ public static String getHostName() { private final OtlpConfig.Temporality otlpMetricsTemporalityPreference; private final boolean otelTracesSpanMetricsEnabled; + private final boolean traceOtelSemanticsEnabled; private final String traceOtelExporter; private final String otlpTracesEndpoint; @@ -1551,8 +1553,11 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins integrationSynapseLegacyOperationName = configProvider.getBoolean(INTEGRATION_SYNAPSE_LEGACY_OPERATION_NAME, false); + traceOtelSemanticsEnabled = + configProvider.getBoolean( + TRACE_OTEL_SEMANTICS_ENABLED, DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED); String configuredTraceOtelExporter = configProvider.getString(TRACE_OTEL_EXPORTER); - if (isTraceOtelSemanticsEnabled()) { + if (traceOtelSemanticsEnabled) { if (configuredTraceOtelExporter != null && !"otlp".equalsIgnoreCase(configuredTraceOtelExporter)) { logOverriddenSettingWarning(TRACE_OTEL_EXPORTER, TRACE_OTEL_SEMANTICS_ENABLED, "otlp"); @@ -1566,7 +1571,7 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins String configuredWriterType = configProvider.getString( WRITER_TYPE, otlpTracesExporter ? OTLP_WRITER_TYPE : DEFAULT_AGENT_WRITER_TYPE); - if (isTraceOtelSemanticsEnabled()) { + if (traceOtelSemanticsEnabled) { if (!OTLP_WRITER_TYPE.equals(configuredWriterType)) { logOverriddenSettingWarning(WRITER_TYPE, TRACE_OTEL_SEMANTICS_ENABLED, OTLP_WRITER_TYPE); } @@ -1788,7 +1793,7 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins isInjectDatadogAttributeEnabled(DEFAULT_INJECT_DATADOG_ATTRIBUTE, "sqs"); int configuredSpanAttributeSchemaVersion = schemaVersionFromConfig(); - if (isTraceOtelSemanticsEnabled()) { + if (traceOtelSemanticsEnabled) { if (configuredSpanAttributeSchemaVersion != SpanNaming.SCHEMA_MIN_VERSION) { logOtelSemanticsOverride(TRACE_SPAN_ATTRIBUTE_SCHEMA, "v" + SpanNaming.SCHEMA_MIN_VERSION); } @@ -1804,7 +1809,7 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins // in v1+ defaults are always calculated regardless this feature flag boolean configuredPeerServiceDefaultsEnabled = configProvider.getBoolean(TRACE_PEER_SERVICE_DEFAULTS_ENABLED, false); - if (isTraceOtelSemanticsEnabled()) { + if (traceOtelSemanticsEnabled) { if (configuredPeerServiceDefaultsEnabled) { logOtelSemanticsOverride(TRACE_PEER_SERVICE_DEFAULTS_ENABLED, false); } @@ -1844,7 +1849,7 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins httpClientErrorStatuses = configProvider.getIntegerRange( TRACE_HTTP_CLIENT_ERROR_STATUSES, - isTraceOtelSemanticsEnabled() + traceOtelSemanticsEnabled ? DEFAULT_OTEL_HTTP_CLIENT_ERROR_STATUSES : DEFAULT_HTTP_CLIENT_ERROR_STATUSES, HTTP_CLIENT_ERROR_STATUSES); @@ -5948,7 +5953,7 @@ public boolean isOtelTracesSpanMetricsEnabled() { } public boolean isTraceOtelSemanticsEnabled() { - return instrumenterConfig.isTraceOtelSemanticsEnabled(); + return traceOtelSemanticsEnabled; } public boolean isTraceOtelEnabled() { @@ -7178,6 +7183,8 @@ public String toString() { + otlpMetricsTemporalityPreference + ", otelTracesSpanMetricsEnabled=" + otelTracesSpanMetricsEnabled + + ", traceOtelSemanticsEnabled=" + + traceOtelSemanticsEnabled + ", traceStatsInterval=" + traceStatsInterval + ", traceOtelExporter=" diff --git a/internal-api/src/test/java/datadog/trace/api/ConfigTest.java b/internal-api/src/test/java/datadog/trace/api/ConfigTest.java index 38275eaf2b6..59230a4097e 100644 --- a/internal-api/src/test/java/datadog/trace/api/ConfigTest.java +++ b/internal-api/src/test/java/datadog/trace/api/ConfigTest.java @@ -172,7 +172,6 @@ import static datadog.trace.api.config.TracerConfig.WRITER_TYPE; import static datadog.trace.api.env.FixedCapturedEnvironment.useFixedEnv; import static datadog.trace.bootstrap.config.provider.ConfigProvider.withPropertiesOverride; -import static datadog.trace.test.junit.utils.config.WithConfigExtension.injectSysConfig; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; @@ -725,32 +724,6 @@ void otelSemanticsForcesItsRequiredConfiguration() { assertEquals(false, calculated.get(TRACE_PEER_SERVICE_DEFAULTS_ENABLED).value); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void otelSemanticsUsesTheInstanceConfiguration(boolean enabled) { - injectSysConfig(TRACE_OTEL_SEMANTICS_ENABLED, Boolean.toString(!enabled)); - // Keep the singleton snapshot while allowing the instance properties to supply the value. - System.clearProperty(PREFIX + TRACE_OTEL_SEMANTICS_ENABLED); - Properties properties = new Properties(); - properties.setProperty(TRACE_OTEL_SEMANTICS_ENABLED, Boolean.toString(enabled)); - properties.setProperty(TRACE_OTEL_EXPORTER, "agent"); - properties.setProperty(WRITER_TYPE, "LoggingWriter"); - properties.setProperty(TRACE_SPAN_ATTRIBUTE_SCHEMA, "v1"); - properties.setProperty(TRACE_PEER_SERVICE_DEFAULTS_ENABLED, "true"); - - Config config = Config.get(properties); - - assertEquals(!enabled, InstrumenterConfig.get().isTraceOtelSemanticsEnabled()); - assertEquals(enabled, config.isTraceOtelSemanticsEnabled()); - assertEquals(enabled, config.isTraceOtlpExporterEnabled()); - assertEquals(enabled ? "OtlpWriter" : "LoggingWriter", config.getWriterType()); - assertEquals(enabled ? 0 : 1, config.getSpanAttributeSchemaVersion()); - assertEquals(!enabled, config.isPeerServiceDefaultsEnabled()); - assertEquals( - enabled ? toBitSet(400, 599) : DEFAULT_HTTP_CLIENT_ERROR_STATUSES, - config.getHttpClientErrorStatuses()); - } - @Test void otelCheckSyntaxForOtlpHeaders() { Properties prop = new Properties(); From 190329bd04d31e0559f8c6e7c2469efb340a1dd1 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Thu, 5 Nov 2026 11:44:02 +0100 Subject: [PATCH 28/34] defer completion --- .../AkkaHttpServerInstrumentationTest.groovy | 25 ++++++++----- .../scala/AkkaHttpTestWebServer.scala | 2 ++ .../akkahttp/AkkaHttpRouteHelper.java | 36 +++++++++++-------- .../akkahttp/AkkaHttpRouteState.java | 14 ++++++-- .../PekkoHttpServerInstrumentationTest.groovy | 25 ++++++++----- .../scala/PekkoHttpTestWebServer.scala | 2 ++ .../pekkohttp/PekkoHttpRouteHelper.java | 36 +++++++++++-------- .../pekkohttp/PekkoHttpRouteState.java | 14 ++++++-- 8 files changed, 106 insertions(+), 48 deletions(-) diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy index 84bad173cfc..866b987f4d9 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy @@ -308,6 +308,8 @@ abstract class AkkaHttpRouteOtelSemanticsTest extends WithHttpServer reject } ~ rawPathPrefix(Segment.?) { _ => complete("fallback") } + } ~ pathPrefix("prefix-empty-fallback") { + path("detail") { complete("detail") } ~ pathEnd { complete("fallback") } } ~ path("items" / Segment) { _ => complete("item") } ~ pathPrefix("route-async" / IntNumber) { id => diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java index aeea2210c99..0443e9dc795 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java @@ -8,7 +8,7 @@ import datadog.trace.api.Config; import datadog.trace.api.KnownTags; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.bootstrap.instrumentation.api.URIUtils; import datadog.trace.instrumentation.akkahttp.AkkaHttpRouteState.Route; import java.util.ArrayList; import java.util.List; @@ -17,8 +17,6 @@ /** Builds route templates from matched literals and parameter placeholders. */ public final class AkkaHttpRouteHelper { private static final boolean OTEL_SEMANTICS_ENABLED = Config.get().isTraceOtelSemanticsEnabled(); - private static final boolean ROUTE_BASED_NAMING_ENABLED = - OTEL_SEMANTICS_ENABLED || Config.get().isHttpServerRouteBasedNaming(); // Matcher calls nest synchronously; request state carries routes across asynchronous directives. private static final ThreadLocal CURRENT = new ThreadLocal<>(); @@ -69,11 +67,10 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable String rest = matched.pathRest().toString(); String path = frame.path(); int consumed = path.length() - rest.length(); - if (consumed <= 0) { - // An empty matcher must not revive a candidate rejected by an earlier alternative. + if (consumed < 0 || (consumed == 0 && frame.parent != null)) { return; } - String template = frame.template(consumed, matched.extractions()); + String template = consumed == 0 ? "" : frame.template(consumed, matched.extractions()); if (frame.parent != null) { String parentPath = frame.parent.path(); if (parentPath.endsWith(path)) { @@ -96,16 +93,27 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable } template = previous.template + template; } - if (frame.state.route == null && ROUTE_BASED_NAMING_ENABLED) { - frame.state.originalResourceName = frame.span.getResourceName(); + if (consumed == 0) { + // Keep only the enclosing prefix, not a rejected candidate that consumed more of the path. + if (previous == null) { + return; + } + frame.state.route = previous; + } else { + if (frame.state.route == null && OTEL_SEMANTICS_ENABLED) { + frame.state.originalResourceName = frame.span.getResourceName(); + } + frame.state.route = new Route(0, rest, template, previous); } - frame.state.route = new Route(0, rest, template, previous); frame.state.routeMatched = true; - Object method = - frame.span.getTag( - OTEL_SEMANTICS_ENABLED ? KnownTags.HTTP_METHOD_OTEL_NAME : Tags.HTTP_METHOD); - HTTP_RESOURCE_DECORATOR.withRoute( - frame.span, method == null ? null : method.toString(), template, true); + if (OTEL_SEMANTICS_ENABLED) { + Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); + HTTP_RESOURCE_DECORATOR.withRoute( + frame.span, method == null ? null : method.toString(), template, true); + } else { + // Commit legacy route naming at completion so rejected candidates cannot block 404 naming. + frame.span.setTag(KnownTags.HTTP_ROUTE_ID, URIUtils.decode(template)); + } } public static final class Frame { diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java index 6c88d86375f..3e242c11e22 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java @@ -2,6 +2,7 @@ import static datadog.context.ContextKey.named; import static datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities.HTTP_FRAMEWORK_ROUTE; +import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR; import static java.util.Collections.singletonList; import akka.http.scaladsl.model.Uri; @@ -11,6 +12,7 @@ import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.KnownTags; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; /** Request-local route state, independent of the HTTP routing DSL. */ public final class AkkaHttpRouteState { @@ -58,17 +60,25 @@ public static void onComplete(Context context, AgentSpan span) { return; } state.requestPath = null; - boolean rejected = state.route != null && !state.routeMatched; + Route route = state.route; state.route = null; CharSequence originalResourceName = state.originalResourceName; state.originalResourceName = null; - if (!rejected) { + if (route == null) { return; } AgentSpan root = span.getLocalRootSpan(); if (root == null) { root = span; } + if (state.routeMatched) { + if (!OTEL_SEMANTICS_ENABLED) { + Object method = root.getTag(Tags.HTTP_METHOD); + HTTP_RESOURCE_DECORATOR.withRoute( + root, method == null ? null : method.toString(), route.template, true); + } + return; + } root.setTag(KnownTags.HTTP_ROUTE_ID, (String) null); if (originalResourceName != null) { // Undo tentative route naming without overriding higher-priority application names. diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy index ad3bc984409..468b1113954 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy @@ -138,6 +138,8 @@ abstract class PekkoHttpRouteOtelSemanticsTest extends WithHttpServer reject } ~ rawPathPrefix(Segment.?) { _ => complete("fallback") } + } ~ pathPrefix("prefix-empty-fallback") { + path("detail") { complete("detail") } ~ pathEnd { complete("fallback") } } ~ path("items" / Segment) { _ => complete("item") } ~ pathPrefix("route-async" / IntNumber) { id => diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java index 476d36cfa78..e8c02a6e635 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java @@ -6,7 +6,7 @@ import datadog.trace.api.Config; import datadog.trace.api.KnownTags; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.bootstrap.instrumentation.api.URIUtils; import datadog.trace.instrumentation.pekkohttp.PekkoHttpRouteState.Route; import java.util.ArrayList; import java.util.List; @@ -17,8 +17,6 @@ /** Builds route templates from matched literals and parameter placeholders. */ public final class PekkoHttpRouteHelper { private static final boolean OTEL_SEMANTICS_ENABLED = Config.get().isTraceOtelSemanticsEnabled(); - private static final boolean ROUTE_BASED_NAMING_ENABLED = - OTEL_SEMANTICS_ENABLED || Config.get().isHttpServerRouteBasedNaming(); // Matcher calls nest synchronously; request state carries routes across asynchronous directives. private static final ThreadLocal CURRENT = new ThreadLocal<>(); @@ -69,11 +67,10 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable String rest = matched.pathRest().toString(); String path = frame.path(); int consumed = path.length() - rest.length(); - if (consumed <= 0) { - // An empty matcher must not revive a candidate rejected by an earlier alternative. + if (consumed < 0 || (consumed == 0 && frame.parent != null)) { return; } - String template = frame.template(consumed, matched.extractions()); + String template = consumed == 0 ? "" : frame.template(consumed, matched.extractions()); if (frame.parent != null) { String parentPath = frame.parent.path(); if (parentPath.endsWith(path)) { @@ -96,16 +93,27 @@ public static void exit(Frame frame, PathMatcher.Matching matching, Throwable } template = previous.template + template; } - if (frame.state.route == null && ROUTE_BASED_NAMING_ENABLED) { - frame.state.originalResourceName = frame.span.getResourceName(); + if (consumed == 0) { + // Keep only the enclosing prefix, not a rejected candidate that consumed more of the path. + if (previous == null) { + return; + } + frame.state.route = previous; + } else { + if (frame.state.route == null && OTEL_SEMANTICS_ENABLED) { + frame.state.originalResourceName = frame.span.getResourceName(); + } + frame.state.route = new Route(0, rest, template, previous); } - frame.state.route = new Route(0, rest, template, previous); frame.state.routeMatched = true; - Object method = - frame.span.getTag( - OTEL_SEMANTICS_ENABLED ? KnownTags.HTTP_METHOD_OTEL_NAME : Tags.HTTP_METHOD); - HTTP_RESOURCE_DECORATOR.withRoute( - frame.span, method == null ? null : method.toString(), template, true); + if (OTEL_SEMANTICS_ENABLED) { + Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); + HTTP_RESOURCE_DECORATOR.withRoute( + frame.span, method == null ? null : method.toString(), template, true); + } else { + // Commit legacy route naming at completion so rejected candidates cannot block 404 naming. + frame.span.setTag(KnownTags.HTTP_ROUTE_ID, URIUtils.decode(template)); + } } public static final class Frame { diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java index d030356fdd0..35e6eed25fa 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java @@ -2,6 +2,7 @@ import static datadog.context.ContextKey.named; import static datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities.HTTP_FRAMEWORK_ROUTE; +import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR; import static java.util.Collections.singletonList; import datadog.context.Context; @@ -10,6 +11,7 @@ import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.KnownTags; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; import org.apache.pekko.http.scaladsl.model.Uri; /** Request-local route state, independent of the HTTP routing DSL. */ @@ -58,17 +60,25 @@ public static void onComplete(Context context, AgentSpan span) { return; } state.requestPath = null; - boolean rejected = state.route != null && !state.routeMatched; + Route route = state.route; state.route = null; CharSequence originalResourceName = state.originalResourceName; state.originalResourceName = null; - if (!rejected) { + if (route == null) { return; } AgentSpan root = span.getLocalRootSpan(); if (root == null) { root = span; } + if (state.routeMatched) { + if (!OTEL_SEMANTICS_ENABLED) { + Object method = root.getTag(Tags.HTTP_METHOD); + HTTP_RESOURCE_DECORATOR.withRoute( + root, method == null ? null : method.toString(), route.template, true); + } + return; + } root.setTag(KnownTags.HTTP_ROUTE_ID, (String) null); if (originalResourceName != null) { // Undo tentative route naming without overriding higher-priority application names. From 909a0c3f4147ecd5a169eb53b17bf95541d70721 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Thu, 5 Nov 2026 11:36:31 +0000 Subject: [PATCH 29/34] Remove redundant context manager settings from route tests --- .../groovy/AkkaHttpServerInstrumentationTest.groovy | 6 ------ .../groovy/PekkoHttpServerInstrumentationTest.groovy | 6 ------ 2 files changed, 12 deletions(-) diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy index 42e1fc1abbb..95323f10b1b 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy @@ -281,12 +281,6 @@ class AkkaHttpServerInstrumentationSyncOtelSemanticsForkedTest extends AkkaHttpS } abstract class AkkaHttpRouteOtelSemanticsTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0, TestingOtelHttpSemantics { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig(TraceInstrumentationConfig.LEGACY_CONTEXT_MANAGER_ENABLED, "true") - } - @Override HttpServer server() { new AkkaHttpTestWebServer(AkkaHttpTestWebServer.BindAndHandle()) diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy index ca67f32d1dc..a6b578ac02f 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy @@ -111,12 +111,6 @@ class PekkoHttpServerInstrumentationSyncV1ForkedTest extends PekkoHttpServerInst } abstract class PekkoHttpRouteOtelSemanticsTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0, TestingOtelHttpSemantics { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig(TraceInstrumentationConfig.LEGACY_CONTEXT_MANAGER_ENABLED, "true") - } - @Override HttpServer server() { new PekkoHttpTestWebServer(PekkoHttpTestWebServer.BindAndHandle()) From 6585d3affa9ba1f544ba9027b296e7cafd4aacb5 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Thu, 5 Nov 2026 16:05:09 +0000 Subject: [PATCH 30/34] Preserve Akka and Pekko routes across handled rejections Track directive outcomes with branch-local route state and capture literal matcher metadata at construction. Group route instrumenters under one module per framework and remove matcher frames and rejection-constructor tracking. Keep {param} templates without suffix-specific instrumentation or trailing wildcards. --- .../bytebuddy/matcher/ignored_class_name.trie | 4 +- .../akka-http/akka-http-10.0/build.gradle | 2 + .../AkkaHttpServerInstrumentationTest.groovy | 56 +++++- .../akkahttp/AkkaHttpRouteHelperTest.java | 90 +++++++--- .../akkahttp/AkkaHttpRouteWrapperTest.java | 77 ++++++++ .../scala/AkkaHttpTestWebServer.scala | 51 +++++- .../AkkaHttpDirectiveInstrumentation.java | 51 ++++++ .../AkkaHttpPathMatcherInstrumentation.java | 64 +++---- .../akkahttp/AkkaHttpRouteHelper.java | 165 +++--------------- .../akkahttp/AkkaHttpRouteModule.java | 38 ++++ .../akkahttp/AkkaHttpRouteState.java | 128 ++++++++------ .../akkahttp/AkkaHttpRouteWrapper.java | 73 ++++++++ .../PekkoHttpServerInstrumentationTest.groovy | 56 +++++- .../pekkohttp/PekkoHttpRouteHelperTest.java | 90 +++++++--- .../pekkohttp/PekkoHttpRouteWrapperTest.java | 77 ++++++++ .../scala/PekkoHttpTestWebServer.scala | 49 +++++- .../PekkoHttpDirectiveInstrumentation.java | 51 ++++++ .../PekkoHttpPathMatcherInstrumentation.java | 65 +++---- .../pekkohttp/PekkoHttpRouteHelper.java | 165 +++--------------- .../pekkohttp/PekkoHttpRouteModule.java | 39 +++++ .../pekkohttp/PekkoHttpRouteState.java | 126 +++++++------ .../pekkohttp/PekkoHttpRouteWrapper.java | 73 ++++++++ 22 files changed, 1055 insertions(+), 535 deletions(-) create mode 100644 dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteWrapperTest.java create mode 100644 dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpDirectiveInstrumentation.java create mode 100644 dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteModule.java create mode 100644 dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteWrapper.java create mode 100644 dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteWrapperTest.java create mode 100644 dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpDirectiveInstrumentation.java create mode 100644 dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteModule.java create mode 100644 dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteWrapper.java diff --git a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie index 0c142f9cb58..9ebe433dcce 100644 --- a/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie +++ b/dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie @@ -162,10 +162,10 @@ 0 akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 0 akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport$class 0 akka.http.scaladsl.unmarshalling.* -# Path matchers supply route templates when OTel HTTP semantics are enabled +# HTTP routing supplies route templates when OTel HTTP semantics are enabled +0 akka.http.scaladsl.server.Directive* 0 akka.http.scaladsl.server.PathMatcher* 0 akka.http.scaladsl.server.ImplicitPathMatcherConstruction* -0 akka.http.scaladsl.server.RouteResult$Rejected 0 akka.http.scaladsl.server.directives.ParameterDirectives$class 0 akka.http.scaladsl.server.directives.ParameterDirectives 0 akka.http.scaladsl.server.directives.BasicDirectives$class diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/build.gradle b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/build.gradle index 57739ecbbd5..dc342eeeb1a 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/build.gradle +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/build.gradle @@ -12,7 +12,9 @@ addForkedTestTask('baseTest') addTestSuiteForDir('version101Test', 'baseTest') addTestSuiteExtendingForDir('version101ForkedTest', 'version101Test', 'baseTest') addTestSuiteForDir('version102Scala213Test', 'latestDepTest') +addForkedTestTask('version102Scala213Test') addTestSuite('latestDepTest') +addForkedTestTask('latestDepTest') addTestSuite('lagomTest') addTestSuite('iastTest') addTestSuiteForDir 'version101IastTest', 'iastTest' diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy index 721c1d53155..a9e81a0750f 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy @@ -301,6 +301,16 @@ abstract class AkkaHttpRouteOtelSemanticsTest extends WithHttpServer + def body = method == 'POST' ? okhttp3.RequestBody.create(okhttp3.MediaType.parse('text/plain'), '') : null + def response = client.newCall(new Request.Builder().url(address.resolve(path).toString()).method(method, body).build()).execute() + assert response.code() == 200 + response.close() + TEST_WRITER.waitForTraces(1) + def span = TEST_WRITER.flatten().find { it.getTag('component')?.toString() == 'akka-http-server' } + assert span.getTag('http.route') == route + assert span.resourceName.toString() == "$method $route" + TEST_WRITER.clear() + } } def "concurrent requests retain their own routes across asynchronous matching and rejection"() { @@ -565,7 +592,6 @@ abstract class AkkaHttpRouteConfigurationTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0, TestingOtelHttpSemantics { + @Override + HttpServer server() { + new AkkaHttpTestWebServer(AkkaHttpTestWebServer.BindRejectedRoute()) + } + + def "a single rejected route leaves no server route"() { + when: + def response = client.newCall(new Request.Builder().url(address.resolve('/single/value').toString()).build()).execute() + assert response.code() == 404 + response.close() + TEST_WRITER.waitForTraces(1) + + then: + def span = TEST_WRITER.flatten().find { it.getTag('component')?.toString() == 'akka-http-server' } + span.getTag('http.route') == null + span.resourceName.toString() == 'GET' + } +} diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelperTest.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelperTest.java index 2e5f95329a3..821f7562fc3 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelperTest.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelperTest.java @@ -1,37 +1,75 @@ package datadog.trace.instrumentation.akkahttp; -import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import java.lang.reflect.Field; +import akka.http.scaladsl.model.Uri; +import akka.http.scaladsl.server.PathMatcher; +import datadog.context.Context; +import datadog.context.ContextScope; import org.junit.jupiter.api.Test; class AkkaHttpRouteHelperTest { + private static Uri.Path path(String value) { + return Uri.apply(value).path(); + } + + @Test + void failedCompositeRollsBackItsSuccessfulLeaves() { + AkkaHttpRouteState state = new AkkaHttpRouteState(path("/items/value")); + state.matched(path("/items/value"), path("items/value"), "/"); + state.matched(path("items/value"), path("/value"), "items"); + state.matched(path("/value"), path("value"), "/"); + state.unmatched(path("/items/value")); + assertNull(state.template()); + assertEquals(path("/items/value"), state.remaining); + } + + @Test + void rejectedAlternativeCannotChangeItsParent() { + AkkaHttpRouteState parent = new AkkaHttpRouteState(path("/items/value")); + parent.matched(path("/items/value"), path("/value"), "/items"); + AkkaHttpRouteState child = parent.fork(); + child.matched(path("/value"), path(""), "/{param}"); + assertEquals("/items", parent.template()); + assertEquals("/items/{param}", child.template()); + AkkaHttpRouteState fallback = parent.fork(); + parent.select(fallback); + assertEquals("/items", parent.template()); + } + + @Test + void completedAlternativePublishesItsOwnRoute() { + AkkaHttpRouteState parent = new AkkaHttpRouteState(path("/items/value")); + AkkaHttpRouteState child = parent.fork(); + child.matched(path("/items/value"), path(""), "/items/{param}"); + parent.select(child); + assertEquals("/items/{param}", parent.template()); + } + + @Test + void consumingMatchReplacesLookaheadWithoutChangingItsParent() { + AkkaHttpRouteState parent = new AkkaHttpRouteState(path("/api/items/value")); + parent.matched(path("/api/items/value"), path("/items/value"), "/api"); + parent.matched(path("/items/value"), path("/value"), "/{param}"); + AkkaHttpRouteState child = parent.fork(); + child.matched(path("/items/value"), path("items/value"), "/"); + child.matched(path("items/value"), path("/value"), "items"); + child.matched(path("/value"), path(""), "/{param}"); + assertEquals("/api/items/{param}", child.template()); + assertEquals("/api/{param}", parent.template()); + } + @Test - @SuppressWarnings("unchecked") - void throwingNestedMatcherRestoresAndRemovesSynchronousFrames() throws Exception { - Field field = AkkaHttpRouteHelper.class.getDeclaredField("CURRENT"); - field.setAccessible(true); - ThreadLocal current = - (ThreadLocal) field.get(null); - AkkaHttpRouteState state = new AkkaHttpRouteState("/items/value"); - state.routeMatched = true; - AkkaHttpRouteHelper.Frame outer = new AkkaHttpRouteHelper.Frame(null, null, state, null); - current.set(outer); - try { - AkkaHttpRouteHelper.Frame inner = AkkaHttpRouteHelper.enter(null); - assertSame(outer, inner.parent); - assertSame(inner, current.get()); - RuntimeException failure = new RuntimeException("matcher failure"); - AkkaHttpRouteHelper.exit(inner, null, failure); - assertSame(outer, current.get()); - AkkaHttpRouteHelper.exit(outer, null, failure); - assertNull(current.get()); - assertFalse(state.routeMatched); - assertNull(AkkaHttpRouteHelper.enter(null)); - } finally { - current.remove(); + void parentCompositeDoesNotDuplicateLeafContributions() { + AkkaHttpRouteState state = new AkkaHttpRouteState(path("/items")); + state.matched(path("/items"), path("items"), "/"); + state.matched(path("items"), path(""), "items"); + PathMatcher.Matched result = new PathMatcher.Matched<>(path(""), null, null); + // No matcher metadata should be accessed for a remainder already recorded by the leaves. + try (ContextScope ignored = state.storeIn(Context.root()).attach()) { + AkkaHttpRouteHelper.matched(null, path("/items"), result, null); } + assertEquals("/items", state.template()); } } diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteWrapperTest.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteWrapperTest.java new file mode 100644 index 00000000000..1df580ca065 --- /dev/null +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteWrapperTest.java @@ -0,0 +1,77 @@ +package datadog.trace.instrumentation.akkahttp; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import akka.http.scaladsl.model.Uri; +import akka.http.scaladsl.server.RequestContext; +import akka.http.scaladsl.server.RequestContextImpl; +import akka.http.scaladsl.server.RouteResult; +import datadog.context.Context; +import datadog.context.ContextScope; +import org.junit.jupiter.api.Test; +import scala.concurrent.ExecutionContextExecutor; +import scala.concurrent.Future; +import scala.concurrent.Future$; +import scala.concurrent.Promise; +import scala.concurrent.Promise$; +import scala.runtime.AbstractFunction1; + +class AkkaHttpRouteWrapperTest { + private static final ExecutionContextExecutor DIRECT = + new ExecutionContextExecutor() { + @Override + public ExecutionContextExecutor prepare() { + return this; + } + + @Override + public void execute(Runnable task) { + task.run(); + } + + @Override + public void reportFailure(Throwable failure) { + throw new AssertionError(failure); + } + }; + + @Test + void delayedRejectedAlternativeCannotEraseACompletedSibling() { + Uri.Path path = Uri.apply("/items/value").path(); + RequestContext request = new RequestContextImpl(null, path, DIRECT, null, null, null, null); + AkkaHttpRouteState parent = new AkkaHttpRouteState(path); + Promise rejected = Promise$.MODULE$.apply(); + try (ContextScope ignored = parent.storeIn(Context.root()).attach()) { + Future losing = + AkkaHttpRouteWrapper.wrap( + new AbstractFunction1>() { + @Override + public Future apply(RequestContext ctx) { + AkkaHttpRouteState.current() + .matched(path, Uri.apply("").path(), "/loser/{param}"); + return rejected.future(); + } + }) + .apply(request); + AkkaHttpRouteWrapper.wrap( + new AbstractFunction1>() { + @Override + public Future apply(RequestContext ctx) { + AkkaHttpRouteState.current() + .matched(path, Uri.apply("").path(), "/items/{param}"); + return Future$.MODULE$.successful( + new RouteResult.Complete( + (akka.http.scaladsl.model.HttpResponse) + akka.http.javadsl.model.HttpResponse.create())); + } + }) + .apply(request); + assertEquals("/items/{param}", parent.template()); + rejected.success(new RouteResult.Rejected(scala.collection.immutable.List$.MODULE$.empty())); + assertTrue(losing.value().isDefined()); + assertEquals("/items/{param}", parent.template()); + assertEquals(parent, AkkaHttpRouteState.current()); + } + } +} diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/scala/AkkaHttpTestWebServer.scala b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/scala/AkkaHttpTestWebServer.scala index c4ab1d96ac3..7536ad855b0 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/scala/AkkaHttpTestWebServer.scala +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/scala/AkkaHttpTestWebServer.scala @@ -84,6 +84,16 @@ object AkkaHttpTestWebServer { ): Future[ServerBinding] } + val BindRejectedRoute: Binder = new Binder { + override def name: String = "single-rejected-route" + override def bind( + port: Int + )(implicit system: ActorSystem, materializer: Materializer): Future[ServerBinding] = { + import materializer.executionContext + Http().bindAndHandle(path("single" / Segment) { _ => reject }, "localhost", port) + } + } + val BindAndHandle: Binder = new Binder { override def name: String = "bind-and-handle" @@ -205,10 +215,22 @@ object AkkaHttpTestWebServer { }) .result() + private val opaqueSegment = new PathMatcher[Unit] { + override def apply(path: Uri.Path): PathMatcher.Matching[Unit] = path match { + case Uri.Path.Segment(_, tail) => PathMatcher.Matched(tail, ()) + case _ => PathMatcher.Unmatched + } + } + def route(implicit ec: ExecutionContext): Route = withController { handleRejections(blockingRejectionHandler) { respondWithDefaultHeader(defaultHeader) { - get { + pathPrefix("crud") { + pathEnd { concat(get { complete("list") }, post { complete("created") }) } ~ + path(Segment) { _ => + concat(get { complete("detail") }, delete { complete("deleted") }) + } + } ~ get { path(SUCCESS.relativePath) { complete( HttpResponse(status = SUCCESS.getStatus, entity = SUCCESS.getBody) @@ -256,6 +278,33 @@ object AkkaHttpTestWebServer { rawPathPrefix(Segment.?) { _ => complete("fallback") } } ~ pathPrefix("prefix-empty-fallback") { path("detail") { complete("detail") } ~ pathEnd { complete("fallback") } + } ~ path("handled-rejection" / Segment) { _ => + handleRejections( + RejectionHandler + .newBuilder() + .handleAll[MissingQueryParamRejection] { _ => complete("handled") } + .result() + ) { + parameter("required") { _ => complete("accepted") } + } + } ~ pathPrefix("delayed-rejection") { + path(Segment) { _ => onSuccess(Future { Thread.sleep(10); 1 }) { _ => reject } } ~ + path(Segment) { _ => complete("accepted") } + } ~ (path("directive-or" / Segment).tfilter(_ => false) | + (path("directive-or" / "literal") & provide("literal"))) { _ => + complete("accepted") + } ~ pathPrefix("directive-or-empty") { + (path(Segment).tfilter(_ => false) | provide("fallback")) { _ => complete("fallback") } + } ~ pathPrefix("lookahead") { + pathPrefixTest("users") { + path("users" / Segment) { _ => complete("accepted") } + } + } ~ pathPrefix("lookahead-dynamic") { + pathPrefixTest(Segment) { _ => + path("users" / Segment) { _ => complete("accepted") } + } + } ~ path("opaque" / opaqueSegment) { + complete("accepted") } ~ path("items" / Segment) { _ => complete("item") } ~ pathPrefix("route-async" / IntNumber) { id => diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpDirectiveInstrumentation.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpDirectiveInstrumentation.java new file mode 100644 index 00000000000..77adf2798e3 --- /dev/null +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpDirectiveInstrumentation.java @@ -0,0 +1,51 @@ +package datadog.trace.instrumentation.akkahttp; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.extendsClass; +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isBridge; +import static net.bytebuddy.matcher.ElementMatchers.not; +import static net.bytebuddy.matcher.ElementMatchers.returns; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import akka.http.scaladsl.server.RequestContext; +import akka.http.scaladsl.server.RouteResult; +import datadog.trace.agent.tooling.Instrumenter; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; +import scala.Function1; +import scala.concurrent.Future; + +/** Isolates directive execution until its route result is known. */ +public final class AkkaHttpDirectiveInstrumentation + implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { + @Override + public String hierarchyMarkerType() { + return "akka.http.scaladsl.server.Directive"; + } + + @Override + public ElementMatcher hierarchyMatcher() { + return extendsClass(named(hierarchyMarkerType())); + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + named("tapply") + .and(takesArguments(1)) + .and(takesArgument(0, named("scala.Function1"))) + .and(returns(named("scala.Function1"))) + .and(not(isBridge())), + getClass().getName() + "$RouteAdvice"); + } + + public static final class RouteAdvice { + @Advice.OnMethodExit(suppress = Throwable.class) + public static void exit( + @Advice.Return(readOnly = false) Function1> route) { + route = AkkaHttpRouteWrapper.wrap(route); + } + } +} diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java index 6ca7ab5568f..4c9554e3a5e 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpPathMatcherInstrumentation.java @@ -2,10 +2,7 @@ import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.extendsClass; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED; -import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED; import static net.bytebuddy.matcher.ElementMatchers.isBridge; -import static net.bytebuddy.matcher.ElementMatchers.isConstructor; import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; import static net.bytebuddy.matcher.ElementMatchers.not; import static net.bytebuddy.matcher.ElementMatchers.returns; @@ -14,32 +11,15 @@ import akka.http.scaladsl.model.Uri; import akka.http.scaladsl.server.PathMatcher; -import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.bootstrap.config.provider.ConfigProvider; +import datadog.trace.bootstrap.InstrumentationContext; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.matcher.ElementMatcher; -/** Tracks matched Akka HTTP route templates and directive rejection results. */ -@AutoService(InstrumenterModule.class) -public final class AkkaHttpPathMatcherInstrumentation extends InstrumenterModule.Tracing +/** Records literal metadata and matched route components. */ +public final class AkkaHttpPathMatcherInstrumentation implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { - public AkkaHttpPathMatcherInstrumentation() { - // Keep this opt-in independent of an explicit parent HTTP integration enablement. - super("akka-http-route"); - } - - @Override - protected boolean defaultEnabled() { - // OTel enables route capture by default; the dedicated integration switch can override it. - // Avoid loading the global Config while setting up instrumentation. - return super.defaultEnabled() - && ConfigProvider.getInstance() - .getBoolean(TRACE_OTEL_SEMANTICS_ENABLED, DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED); - } - @Override public String hierarchyMarkerType() { return "akka.http.scaladsl.server.PathMatcher"; @@ -48,14 +28,11 @@ public String hierarchyMarkerType() { @Override public ElementMatcher hierarchyMatcher() { return extendsClass(named(hierarchyMarkerType())) - .or(named("akka.http.scaladsl.server.RouteResult$Rejected")); + .or(named("akka.http.scaladsl.server.PathMatcher$")); } @Override public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isConstructor().and(isDeclaredBy(named("akka.http.scaladsl.server.RouteResult$Rejected"))), - getClass().getName() + "$RejectedAdvice"); transformer.applyAdvice( named("apply") .and(takesArguments(1)) @@ -63,27 +40,30 @@ public void methodAdvice(MethodTransformer transformer) { .and(returns(named("akka.http.scaladsl.server.PathMatcher$Matching"))) .and(not(isBridge())), getClass().getName() + "$MatchAdvice"); - } - - public static final class RejectedAdvice { - @Advice.OnMethodExit(suppress = Throwable.class) - public static void exit() { - AkkaHttpRouteState.onRejected(); - } + transformer.applyAdvice( + named("apply") + .and(takesArgument(0, named("akka.http.scaladsl.model.Uri$Path"))) + .and(returns(named("akka.http.scaladsl.server.PathMatcher"))) + .and(isDeclaredBy(named("akka.http.scaladsl.server.PathMatcher$"))), + getClass().getName() + "$LiteralAdvice"); } public static final class MatchAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static AkkaHttpRouteHelper.Frame enter(@Advice.Argument(0) Uri.Path path) { - return AkkaHttpRouteHelper.enter(path); + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void exit( + @Advice.This PathMatcher matcher, + @Advice.Argument(0) Uri.Path path, + @Advice.Return PathMatcher.Matching matching) { + AkkaHttpRouteHelper.matched( + matcher, path, matching, InstrumentationContext.get(PathMatcher.class, String.class)); } + } - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static final class LiteralAdvice { + @Advice.OnMethodExit(suppress = Throwable.class) public static void exit( - @Advice.Enter AkkaHttpRouteHelper.Frame frame, - @Advice.Return PathMatcher.Matching matching, - @Advice.Thrown Throwable throwable) { - AkkaHttpRouteHelper.exit(frame, matching, throwable); + @Advice.Argument(0) Uri.Path path, @Advice.Return PathMatcher matcher) { + InstrumentationContext.get(PathMatcher.class, String.class).put(matcher, path.toString()); } } } diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java index 7dd9eb20a1e..9ea64263d4d 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteHelper.java @@ -1,163 +1,44 @@ package datadog.trace.instrumentation.akkahttp; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; -import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR; - import akka.http.scaladsl.model.Uri; import akka.http.scaladsl.server.PathMatcher; -import datadog.trace.api.Config; -import datadog.trace.api.KnownTags; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.URIUtils; -import datadog.trace.instrumentation.akkahttp.AkkaHttpRouteState.Route; -import java.util.ArrayList; -import java.util.List; -import scala.Product; +import akka.http.scaladsl.server.PathMatchers; +import datadog.trace.bootstrap.ContextStore; -/** Builds route templates from matched literals and parameter placeholders. */ +/** Records successful leaf matches without allocating frames for matcher invocations. */ public final class AkkaHttpRouteHelper { - private static final boolean OTEL_SEMANTICS = Config.get().isTraceOtelSemanticsEnabled(); - // Matcher calls nest synchronously; request state carries routes across asynchronous directives. - private static final ThreadLocal CURRENT = new ThreadLocal<>(); - private AkkaHttpRouteHelper() {} - public static Frame enter(Uri.Path path) { - Frame parent = CURRENT.get(); - if (parent != null) { - // nested matchers share the root span already checked by the outermost frame - Frame frame = new Frame(path, parent.span, parent.state, parent); - CURRENT.set(frame); - return frame; - } + public static void matched( + PathMatcher matcher, + Uri.Path before, + PathMatcher.Matching result, + ContextStore prefixes) { AkkaHttpRouteState state = AkkaHttpRouteState.current(); - if (state == null || state.requestPath == null) { - return null; - } - AgentSpan active = activeSpan(); - if (active == null) { - return null; - } - AgentSpan root = active.getLocalRootSpan(); - if (root == null) { - return null; - } - Frame frame = new Frame(path, root, state, null); - CURRENT.set(frame); - return frame; - } - - public static void exit(Frame frame, PathMatcher.Matching matching, Throwable throwable) { - if (frame == null) { + if (state == null || state.remaining == null) { return; } - boolean isMatch = throwable == null && matching instanceof PathMatcher.Matched; - if (frame.parent == null) { - CURRENT.remove(); - if (!isMatch) { - frame.state.routeMatched = false; - } - } else { - CURRENT.set(frame.parent); - } - if (!isMatch) { + if (!(result instanceof PathMatcher.Matched)) { + state.unmatched(before); return; } - PathMatcher.Matched matched = (PathMatcher.Matched) matching; - String rest = matched.pathRest().toString(); - String path = frame.path(); - int consumed = path.length() - rest.length(); - if (consumed < 0 || (consumed == 0 && frame.parent != null)) { + PathMatcher.Matched matched = (PathMatcher.Matched) result; + Uri.Path after = matched.pathRest(); + if (before.equals(after)) { return; } - String template = consumed == 0 ? "" : frame.template(consumed, matched.extractions()); - if (frame.parent != null) { - String parentPath = frame.parent.path(); - if (parentPath.endsWith(path)) { - if (frame.parent.parts == null) { - frame.parent.parts = new ArrayList<>(); - } - frame.parent.parts.add(new Route(parentPath.length() - path.length(), rest, template)); - } + // Most composed matchers finish at the remainder already recorded by their leaves. + if (after.equals(state.remaining)) { return; } - Route previous = null; - if (!frame.state.requestPath.equals(path)) { - previous = frame.state.route; - // A rejected route can retry an earlier unmatched path in another alternative. - while (previous != null && !previous.remaining.equals(path)) { - previous = previous.previous; - } - if (previous == null) { - return; - } - template = previous.template + template; - } - if (consumed == 0) { - // Keep only the enclosing prefix, not a rejected candidate that consumed more of the path. - if (previous == null) { - return; - } - frame.state.route = previous; - } else { - if (frame.state.route == null && OTEL_SEMANTICS) { - frame.state.originalResourceName = frame.span.getResourceName(); - } - frame.state.route = new Route(0, rest, template, previous); - } - frame.state.routeMatched = true; - if (OTEL_SEMANTICS) { - Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); - HTTP_RESOURCE_DECORATOR.withRoute( - frame.span, method == null ? null : method.toString(), template, true); - } else { - // Commit legacy route naming at completion so rejected candidates cannot block 404 naming. - frame.span.setTag(KnownTags.HTTP_ROUTE_ID, URIUtils.decode(template)); - } - } - - public static final class Frame { - final Uri.Path uriPath; - final AgentSpan span; - final AkkaHttpRouteState state; - final Frame parent; - List parts; - private String path; - - Frame(Uri.Path uriPath, AgentSpan span, AkkaHttpRouteState state, Frame parent) { - this.uriPath = uriPath; - this.span = span; - this.state = state; - this.parent = parent; - } - - /** Renders the path on demand, as most match attempts fail and never need it. */ - String path() { - if (path == null) { - path = uriPath.toString(); - } - return path; - } - - String template(int consumed, Object extractions) { - String path = path(); - if (parts == null) { - if (extractions instanceof Product && ((Product) extractions).productArity() > 0) { - return path.charAt(0) == '/' ? "/{param}" : "{param}"; - } - return path.substring(0, consumed); - } - StringBuilder template = new StringBuilder(); - int cursor = 0; - for (Route part : parts) { - int end = path.length() - part.remaining.length(); - if (part.offset < cursor || end > consumed) { - continue; - } - template.append(path, cursor, part.offset).append(part.template); - cursor = end; + String literal = prefixes.get(matcher); + if (literal == null) { + if (matcher.getClass() == PathMatchers.Slash$.class) { + literal = "/"; + } else { + literal = before.startsWithSlash() ? "/{param}" : "{param}"; } - return template.append(path, cursor, consumed).toString(); } + state.matched(before, after, literal); } } diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteModule.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteModule.java new file mode 100644 index 00000000000..3c50f3f79bf --- /dev/null +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteModule.java @@ -0,0 +1,38 @@ +package datadog.trace.instrumentation.akkahttp; + +import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED; +import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED; +import static java.util.Arrays.asList; +import static java.util.Collections.singletonMap; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.config.provider.ConfigProvider; +import java.util.List; +import java.util.Map; + +/** Captures matched routes and selects their completed directive branches. */ +@AutoService(InstrumenterModule.class) +public final class AkkaHttpRouteModule extends InstrumenterModule.Tracing { + public AkkaHttpRouteModule() { + super("akka-http-route"); + } + + @Override + protected boolean defaultEnabled() { + return super.defaultEnabled() + && ConfigProvider.getInstance() + .getBoolean(TRACE_OTEL_SEMANTICS_ENABLED, DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED); + } + + @Override + public Map contextStore() { + return singletonMap("akka.http.scaladsl.server.PathMatcher", "java.lang.String"); + } + + @Override + public List typeInstrumentations() { + return asList(new AkkaHttpPathMatcherInstrumentation(), new AkkaHttpDirectiveInstrumentation()); + } +} diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java index eff062ebc16..b235413f68f 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java @@ -1,7 +1,6 @@ package datadog.trace.instrumentation.akkahttp; import static datadog.context.ContextKey.named; -import static datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities.HTTP_FRAMEWORK_ROUTE; import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR; import static java.util.Collections.singletonList; @@ -14,92 +13,119 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.Tags; -/** Request-local route state, independent of the HTTP routing DSL. */ +/** + * Route contributions owned by one directive invocation, with immutable history shared by child + * directives. + */ public final class AkkaHttpRouteState { private static final boolean OTEL_SEMANTICS = Config.get().isTraceOtelSemanticsEnabled(); private static final ContextKey KEY = named("akka-http-route"); - private static final boolean ROUTE_CAPTURE_ENABLED = InstrumenterConfig.get() .isIntegrationEnabled( singletonList("akka-http-route"), InstrumenterConfig.get().isIntegrationsEnabled() && OTEL_SEMANTICS); - String requestPath; - Route route; - CharSequence originalResourceName; - - /** Whether the latest route candidate has matched without subsequently being rejected. */ - boolean routeMatched; + Uri.Path remaining; + Part part; - AkkaHttpRouteState(String requestPath) { - this.requestPath = requestPath; + AkkaHttpRouteState(Uri.Path remaining) { + this.remaining = remaining; } public static Context init(Context context, Uri.Path requestPath) { - return ROUTE_CAPTURE_ENABLED - ? context.with(KEY, new AkkaHttpRouteState(requestPath.toString())) - : context; + return ROUTE_CAPTURE_ENABLED ? context.with(KEY, new AkkaHttpRouteState(requestPath)) : context; } static AkkaHttpRouteState current() { return Context.current().get(KEY); } - public static void onRejected() { - AkkaHttpRouteState state = current(); - if (state != null) { - state.routeMatched = false; + Context storeIn(Context context) { + return context.with(KEY, this); + } + + AkkaHttpRouteState fork() { + AkkaHttpRouteState child = new AkkaHttpRouteState(remaining); + child.part = part; + return child; + } + + void select(AkkaHttpRouteState child) { + part = child.part; + remaining = child.remaining; + } + + void matched(Uri.Path before, Uri.Path after, String literal) { + if (!before.equals(remaining)) { + // Lookahead matchers can inspect input that a later directive consumes again. + unmatched(before); + if (!before.equals(remaining)) { + return; + } + } + part = new Part(before, literal, part); + remaining = after; + } + + void unmatched(Uri.Path before) { + // A composed matcher can fail after some of its leaves have consumed input. + for (Part cursor = part; cursor != null; cursor = cursor.previous) { + if (cursor.before.equals(before)) { + part = cursor.previous; + remaining = before; + return; + } + } + } + + String template() { + if (part == null) { + return null; + } + int length = 0; + for (Part cursor = part; cursor != null; cursor = cursor.previous) { + length += cursor.literal.length(); + } + char[] chars = new char[length]; + for (Part cursor = part; cursor != null; cursor = cursor.previous) { + String literal = cursor.literal; + length -= literal.length(); + literal.getChars(0, literal.length(), chars, length); } + return new String(chars); } - /** Drops rejected candidates and releases route history after request completion. */ + /** Publishes only the selected route, before normal HTTP response naming. */ public static void onComplete(Context context, AgentSpan span) { AkkaHttpRouteState state = context.get(KEY); if (state == null) { return; } - state.requestPath = null; - Route route = state.route; - state.route = null; - CharSequence originalResourceName = state.originalResourceName; - state.originalResourceName = null; - if (route == null) { + String template = state.template(); + state.part = null; + state.remaining = null; + if (template == null) { return; } AgentSpan root = span.getLocalRootSpan(); if (root == null) { root = span; } - if (state.routeMatched) { - if (!OTEL_SEMANTICS) { - Object method = root.getTag(Tags.HTTP_METHOD); - HTTP_RESOURCE_DECORATOR.withRoute( - root, method == null ? null : method.toString(), route.template, true); - } - return; - } - root.setTag(KnownTags.HTTP_ROUTE_ID, (String) null); - if (originalResourceName != null) { - // Undo tentative route naming without overriding higher-priority application names. - root.setResourceName(originalResourceName, HTTP_FRAMEWORK_ROUTE); - } + Object method = + root.getTag(OTEL_SEMANTICS ? KnownTags.HTTP_METHOD_OTEL_NAME : Tags.HTTP_METHOD); + HTTP_RESOURCE_DECORATOR.withRoute( + root, method == null ? null : method.toString(), template, true); } - public static final class Route { - final int offset; - final String remaining; - final String template; - final Route previous; - - Route(int offset, String remaining, String template) { - this(offset, remaining, template, null); - } + static final class Part { + final Uri.Path before; + final String literal; + final Part previous; - Route(int offset, String remaining, String template, Route previous) { - this.offset = offset; - this.remaining = remaining; - this.template = template; + Part(Uri.Path before, String literal, Part previous) { + this.before = before; + this.literal = literal; this.previous = previous; } } diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteWrapper.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteWrapper.java new file mode 100644 index 00000000000..2beee66639e --- /dev/null +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteWrapper.java @@ -0,0 +1,73 @@ +package datadog.trace.instrumentation.akkahttp; + +import akka.http.scaladsl.server.RequestContext; +import akka.http.scaladsl.server.RouteResult; +import datadog.context.Context; +import datadog.context.ContextScope; +import scala.Function1; +import scala.Option; +import scala.concurrent.Future; +import scala.runtime.AbstractFunction1; +import scala.runtime.AbstractPartialFunction; +import scala.runtime.BoxedUnit; +import scala.util.Try; + +/** Isolates a directive invocation until its route result is known. */ +public final class AkkaHttpRouteWrapper + extends AbstractFunction1> { + private final Function1> delegate; + + private AkkaHttpRouteWrapper(Function1> delegate) { + this.delegate = delegate; + } + + public static Function1> wrap( + Function1> route) { + return route instanceof AkkaHttpRouteWrapper ? route : new AkkaHttpRouteWrapper(route); + } + + @Override + public Future apply(RequestContext request) { + AkkaHttpRouteState parent = AkkaHttpRouteState.current(); + if (parent == null) { + return delegate.apply(request); + } + AkkaHttpRouteState child = parent.fork(); + try (ContextScope ignored = child.storeIn(Context.current()).attach()) { + Future future; + try { + future = delegate.apply(request); + } catch (Throwable failure) { + parent.select(child); + throw failure; + } + Option> value = future.value(); + if (value.isDefined()) { + select(parent, child, value.get()); + return future; + } + return future.andThen( + new AbstractPartialFunction, BoxedUnit>() { + @Override + public boolean isDefinedAt(Try result) { + return true; + } + + @Override + public BoxedUnit apply(Try result) { + select(parent, child, result); + return BoxedUnit.UNIT; + } + }, + request.executionContext()); + } + } + + private static void select( + AkkaHttpRouteState parent, AkkaHttpRouteState child, Try result) { + // Exceptions may be converted to a response by an enclosing exception handler. + if (result.isFailure() || result.get() instanceof RouteResult.Complete) { + parent.select(child); + } + } +} diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy index e6f9663910a..a17d22cb036 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy @@ -131,6 +131,16 @@ abstract class PekkoHttpRouteOtelSemanticsTest extends WithHttpServer + def body = method == 'POST' ? okhttp3.RequestBody.create(okhttp3.MediaType.parse('text/plain'), '') : null + def response = client.newCall(new Request.Builder().url(address.resolve(path).toString()).method(method, body).build()).execute() + assert response.code() == 200 + response.close() + TEST_WRITER.waitForTraces(1) + def span = TEST_WRITER.flatten().find { it.getTag('component')?.toString() == 'pekko-http-server' } + assert span.getTag('http.route') == route + assert span.resourceName.toString() == "$method $route" + TEST_WRITER.clear() + } } def "concurrent requests retain their own routes across asynchronous matching and rejection"() { @@ -345,7 +372,6 @@ abstract class PekkoHttpRouteConfigurationTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0, TestingOtelHttpSemantics { + @Override + HttpServer server() { + new PekkoHttpTestWebServer(PekkoHttpTestWebServer.BindRejectedRoute()) + } + + def "a single rejected route leaves no server route"() { + when: + def response = client.newCall(new Request.Builder().url(address.resolve('/single/value').toString()).build()).execute() + assert response.code() == 404 + response.close() + TEST_WRITER.waitForTraces(1) + + then: + def span = TEST_WRITER.flatten().find { it.getTag('component')?.toString() == 'pekko-http-server' } + span.getTag('http.route') == null + span.resourceName.toString() == 'GET' + } +} diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelperTest.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelperTest.java index 890c561435c..56752ca7975 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelperTest.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelperTest.java @@ -1,37 +1,75 @@ package datadog.trace.instrumentation.pekkohttp; -import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import java.lang.reflect.Field; +import datadog.context.Context; +import datadog.context.ContextScope; +import org.apache.pekko.http.scaladsl.model.Uri; +import org.apache.pekko.http.scaladsl.server.PathMatcher; import org.junit.jupiter.api.Test; class PekkoHttpRouteHelperTest { + private static Uri.Path path(String value) { + return Uri.apply(value).path(); + } + + @Test + void failedCompositeRollsBackItsSuccessfulLeaves() { + PekkoHttpRouteState state = new PekkoHttpRouteState(path("/items/value")); + state.matched(path("/items/value"), path("items/value"), "/"); + state.matched(path("items/value"), path("/value"), "items"); + state.matched(path("/value"), path("value"), "/"); + state.unmatched(path("/items/value")); + assertNull(state.template()); + assertEquals(path("/items/value"), state.remaining); + } + + @Test + void rejectedAlternativeCannotChangeItsParent() { + PekkoHttpRouteState parent = new PekkoHttpRouteState(path("/items/value")); + parent.matched(path("/items/value"), path("/value"), "/items"); + PekkoHttpRouteState child = parent.fork(); + child.matched(path("/value"), path(""), "/{param}"); + assertEquals("/items", parent.template()); + assertEquals("/items/{param}", child.template()); + PekkoHttpRouteState fallback = parent.fork(); + parent.select(fallback); + assertEquals("/items", parent.template()); + } + + @Test + void completedAlternativePublishesItsOwnRoute() { + PekkoHttpRouteState parent = new PekkoHttpRouteState(path("/items/value")); + PekkoHttpRouteState child = parent.fork(); + child.matched(path("/items/value"), path(""), "/items/{param}"); + parent.select(child); + assertEquals("/items/{param}", parent.template()); + } + + @Test + void consumingMatchReplacesLookaheadWithoutChangingItsParent() { + PekkoHttpRouteState parent = new PekkoHttpRouteState(path("/api/items/value")); + parent.matched(path("/api/items/value"), path("/items/value"), "/api"); + parent.matched(path("/items/value"), path("/value"), "/{param}"); + PekkoHttpRouteState child = parent.fork(); + child.matched(path("/items/value"), path("items/value"), "/"); + child.matched(path("items/value"), path("/value"), "items"); + child.matched(path("/value"), path(""), "/{param}"); + assertEquals("/api/items/{param}", child.template()); + assertEquals("/api/{param}", parent.template()); + } + @Test - @SuppressWarnings("unchecked") - void throwingNestedMatcherRestoresAndRemovesSynchronousFrames() throws Exception { - Field field = PekkoHttpRouteHelper.class.getDeclaredField("CURRENT"); - field.setAccessible(true); - ThreadLocal current = - (ThreadLocal) field.get(null); - PekkoHttpRouteState state = new PekkoHttpRouteState("/items/value"); - state.routeMatched = true; - PekkoHttpRouteHelper.Frame outer = new PekkoHttpRouteHelper.Frame(null, null, state, null); - current.set(outer); - try { - PekkoHttpRouteHelper.Frame inner = PekkoHttpRouteHelper.enter(null); - assertSame(outer, inner.parent); - assertSame(inner, current.get()); - RuntimeException failure = new RuntimeException("matcher failure"); - PekkoHttpRouteHelper.exit(inner, null, failure); - assertSame(outer, current.get()); - PekkoHttpRouteHelper.exit(outer, null, failure); - assertNull(current.get()); - assertFalse(state.routeMatched); - assertNull(PekkoHttpRouteHelper.enter(null)); - } finally { - current.remove(); + void parentCompositeDoesNotDuplicateLeafContributions() { + PekkoHttpRouteState state = new PekkoHttpRouteState(path("/items")); + state.matched(path("/items"), path("items"), "/"); + state.matched(path("items"), path(""), "items"); + PathMatcher.Matched result = new PathMatcher.Matched<>(path(""), null, null); + // No matcher metadata should be accessed for a remainder already recorded by the leaves. + try (ContextScope ignored = state.storeIn(Context.root()).attach()) { + PekkoHttpRouteHelper.matched(null, path("/items"), result, null); } + assertEquals("/items", state.template()); } } diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteWrapperTest.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteWrapperTest.java new file mode 100644 index 00000000000..1449a3897f6 --- /dev/null +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteWrapperTest.java @@ -0,0 +1,77 @@ +package datadog.trace.instrumentation.pekkohttp; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.context.Context; +import datadog.context.ContextScope; +import org.apache.pekko.http.scaladsl.model.Uri; +import org.apache.pekko.http.scaladsl.server.RequestContext; +import org.apache.pekko.http.scaladsl.server.RequestContextImpl; +import org.apache.pekko.http.scaladsl.server.RouteResult; +import org.junit.jupiter.api.Test; +import scala.concurrent.ExecutionContextExecutor; +import scala.concurrent.Future; +import scala.concurrent.Future$; +import scala.concurrent.Promise; +import scala.concurrent.Promise$; +import scala.runtime.AbstractFunction1; + +class PekkoHttpRouteWrapperTest { + private static final ExecutionContextExecutor DIRECT = + new ExecutionContextExecutor() { + @Override + public ExecutionContextExecutor prepare() { + return this; + } + + @Override + public void execute(Runnable task) { + task.run(); + } + + @Override + public void reportFailure(Throwable failure) { + throw new AssertionError(failure); + } + }; + + @Test + void delayedRejectedAlternativeCannotEraseACompletedSibling() { + Uri.Path path = Uri.apply("/items/value").path(); + RequestContext request = new RequestContextImpl(null, path, DIRECT, null, null, null, null); + PekkoHttpRouteState parent = new PekkoHttpRouteState(path); + Promise rejected = Promise$.MODULE$.apply(); + try (ContextScope ignored = parent.storeIn(Context.root()).attach()) { + Future losing = + PekkoHttpRouteWrapper.wrap( + new AbstractFunction1>() { + @Override + public Future apply(RequestContext ctx) { + PekkoHttpRouteState.current() + .matched(path, Uri.apply("").path(), "/loser/{param}"); + return rejected.future(); + } + }) + .apply(request); + PekkoHttpRouteWrapper.wrap( + new AbstractFunction1>() { + @Override + public Future apply(RequestContext ctx) { + PekkoHttpRouteState.current() + .matched(path, Uri.apply("").path(), "/items/{param}"); + return Future$.MODULE$.successful( + new RouteResult.Complete( + (org.apache.pekko.http.scaladsl.model.HttpResponse) + org.apache.pekko.http.javadsl.model.HttpResponse.create())); + } + }) + .apply(request); + assertEquals("/items/{param}", parent.template()); + rejected.success(new RouteResult.Rejected(scala.collection.immutable.List$.MODULE$.empty())); + assertTrue(losing.value().isDefined()); + assertEquals("/items/{param}", parent.template()); + assertEquals(parent, PekkoHttpRouteState.current()); + } + } +} diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/scala/PekkoHttpTestWebServer.scala b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/scala/PekkoHttpTestWebServer.scala index 19780f0b324..fc8975c6a15 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/scala/PekkoHttpTestWebServer.scala +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/scala/PekkoHttpTestWebServer.scala @@ -69,6 +69,16 @@ object PekkoHttpTestWebServer { ): Future[ServerBinding] } + val BindRejectedRoute: Binder = new Binder { + override def name: String = "single-rejected-route" + override def bind( + port: Int + )(implicit system: ActorSystem, materializer: Materializer): Future[ServerBinding] = { + import materializer.executionContext + Http().bindAndHandle(path("single" / Segment) { _ => reject }, "localhost", port) + } + } + val BindAndHandle: Binder = new Binder { override def name: String = "bind-and-handle" @@ -177,9 +187,19 @@ object PekkoHttpTestWebServer { private val defaultHeader = RawHeader(HttpServerTest.getIG_RESPONSE_HEADER, HttpServerTest.getIG_RESPONSE_HEADER_VALUE) + private val opaqueSegment = new PathMatcher[Unit] { + override def apply(path: Uri.Path): PathMatcher.Matching[Unit] = path match { + case Uri.Path.Segment(_, tail) => PathMatcher.Matched(tail, ()) + case _ => PathMatcher.Unmatched + } + } + def route(implicit ec: ExecutionContext): Route = withController { respondWithDefaultHeader(defaultHeader) { - get { + pathPrefix("crud") { + pathEnd { concat(get { complete("list") }, post { complete("created") }) } ~ + path(Segment) { _ => concat(get { complete("detail") }, delete { complete("deleted") }) } + } ~ get { path(SUCCESS.relativePath) { complete( HttpResponse(status = SUCCESS.getStatus, entity = SUCCESS.getBody) @@ -222,6 +242,33 @@ object PekkoHttpTestWebServer { rawPathPrefix(Segment.?) { _ => complete("fallback") } } ~ pathPrefix("prefix-empty-fallback") { path("detail") { complete("detail") } ~ pathEnd { complete("fallback") } + } ~ path("handled-rejection" / Segment) { _ => + handleRejections( + RejectionHandler + .newBuilder() + .handleAll[MissingQueryParamRejection] { _ => complete("handled") } + .result() + ) { + parameter("required") { _ => complete("accepted") } + } + } ~ pathPrefix("delayed-rejection") { + path(Segment) { _ => onSuccess(Future { Thread.sleep(10); 1 }) { _ => reject } } ~ + path(Segment) { _ => complete("accepted") } + } ~ (path("directive-or" / Segment).tfilter(_ => false) | + (path("directive-or" / "literal") & provide("literal"))) { _ => + complete("accepted") + } ~ pathPrefix("directive-or-empty") { + (path(Segment).tfilter(_ => false) | provide("fallback")) { _ => complete("fallback") } + } ~ pathPrefix("lookahead") { + pathPrefixTest("users") { + path("users" / Segment) { _ => complete("accepted") } + } + } ~ pathPrefix("lookahead-dynamic") { + pathPrefixTest(Segment) { _ => + path("users" / Segment) { _ => complete("accepted") } + } + } ~ path("opaque" / opaqueSegment) { + complete("accepted") } ~ path("items" / Segment) { _ => complete("item") } ~ pathPrefix("route-async" / IntNumber) { id => diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpDirectiveInstrumentation.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpDirectiveInstrumentation.java new file mode 100644 index 00000000000..65925ef21fb --- /dev/null +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpDirectiveInstrumentation.java @@ -0,0 +1,51 @@ +package datadog.trace.instrumentation.pekkohttp; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.extendsClass; +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isBridge; +import static net.bytebuddy.matcher.ElementMatchers.not; +import static net.bytebuddy.matcher.ElementMatchers.returns; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import datadog.trace.agent.tooling.Instrumenter; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; +import org.apache.pekko.http.scaladsl.server.RequestContext; +import org.apache.pekko.http.scaladsl.server.RouteResult; +import scala.Function1; +import scala.concurrent.Future; + +/** Isolates directive execution until its route result is known. */ +public final class PekkoHttpDirectiveInstrumentation + implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { + @Override + public String hierarchyMarkerType() { + return "org.apache.pekko.http.scaladsl.server.Directive"; + } + + @Override + public ElementMatcher hierarchyMatcher() { + return extendsClass(named(hierarchyMarkerType())); + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + named("tapply") + .and(takesArguments(1)) + .and(takesArgument(0, named("scala.Function1"))) + .and(returns(named("scala.Function1"))) + .and(not(isBridge())), + getClass().getName() + "$RouteAdvice"); + } + + public static final class RouteAdvice { + @Advice.OnMethodExit(suppress = Throwable.class) + public static void exit( + @Advice.Return(readOnly = false) Function1> route) { + route = PekkoHttpRouteWrapper.wrap(route); + } + } +} diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java index 715a219a7e6..0ea8fd990b7 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpPathMatcherInstrumentation.java @@ -2,44 +2,24 @@ import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.extendsClass; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED; -import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED; import static net.bytebuddy.matcher.ElementMatchers.isBridge; -import static net.bytebuddy.matcher.ElementMatchers.isConstructor; import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; import static net.bytebuddy.matcher.ElementMatchers.not; import static net.bytebuddy.matcher.ElementMatchers.returns; import static net.bytebuddy.matcher.ElementMatchers.takesArgument; import static net.bytebuddy.matcher.ElementMatchers.takesArguments; -import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.bootstrap.config.provider.ConfigProvider; +import datadog.trace.bootstrap.InstrumentationContext; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.matcher.ElementMatcher; import org.apache.pekko.http.scaladsl.model.Uri; import org.apache.pekko.http.scaladsl.server.PathMatcher; -/** Tracks matched Pekko HTTP route templates and directive rejection results. */ -@AutoService(InstrumenterModule.class) -public final class PekkoHttpPathMatcherInstrumentation extends InstrumenterModule.Tracing +/** Records literal metadata and matched route components. */ +public final class PekkoHttpPathMatcherInstrumentation implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { - public PekkoHttpPathMatcherInstrumentation() { - // Keep this opt-in independent of an explicit parent HTTP integration enablement. - super("pekko-http-route"); - } - - @Override - protected boolean defaultEnabled() { - // OTel enables route capture by default; the dedicated integration switch can override it. - // Avoid loading the global Config while setting up instrumentation. - return super.defaultEnabled() - && ConfigProvider.getInstance() - .getBoolean(TRACE_OTEL_SEMANTICS_ENABLED, DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED); - } - @Override public String hierarchyMarkerType() { return "org.apache.pekko.http.scaladsl.server.PathMatcher"; @@ -48,15 +28,11 @@ public String hierarchyMarkerType() { @Override public ElementMatcher hierarchyMatcher() { return extendsClass(named(hierarchyMarkerType())) - .or(named("org.apache.pekko.http.scaladsl.server.RouteResult$Rejected")); + .or(named("org.apache.pekko.http.scaladsl.server.PathMatcher$")); } @Override public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isConstructor() - .and(isDeclaredBy(named("org.apache.pekko.http.scaladsl.server.RouteResult$Rejected"))), - getClass().getName() + "$RejectedAdvice"); transformer.applyAdvice( named("apply") .and(takesArguments(1)) @@ -64,27 +40,30 @@ public void methodAdvice(MethodTransformer transformer) { .and(returns(named("org.apache.pekko.http.scaladsl.server.PathMatcher$Matching"))) .and(not(isBridge())), getClass().getName() + "$MatchAdvice"); - } - - public static final class RejectedAdvice { - @Advice.OnMethodExit(suppress = Throwable.class) - public static void exit() { - PekkoHttpRouteState.onRejected(); - } + transformer.applyAdvice( + named("apply") + .and(takesArgument(0, named("org.apache.pekko.http.scaladsl.model.Uri$Path"))) + .and(returns(named("org.apache.pekko.http.scaladsl.server.PathMatcher"))) + .and(isDeclaredBy(named("org.apache.pekko.http.scaladsl.server.PathMatcher$"))), + getClass().getName() + "$LiteralAdvice"); } public static final class MatchAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static PekkoHttpRouteHelper.Frame enter(@Advice.Argument(0) Uri.Path path) { - return PekkoHttpRouteHelper.enter(path); + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void exit( + @Advice.This PathMatcher matcher, + @Advice.Argument(0) Uri.Path path, + @Advice.Return PathMatcher.Matching matching) { + PekkoHttpRouteHelper.matched( + matcher, path, matching, InstrumentationContext.get(PathMatcher.class, String.class)); } + } - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static final class LiteralAdvice { + @Advice.OnMethodExit(suppress = Throwable.class) public static void exit( - @Advice.Enter PekkoHttpRouteHelper.Frame frame, - @Advice.Return PathMatcher.Matching matching, - @Advice.Thrown Throwable throwable) { - PekkoHttpRouteHelper.exit(frame, matching, throwable); + @Advice.Argument(0) Uri.Path path, @Advice.Return PathMatcher matcher) { + InstrumentationContext.get(PathMatcher.class, String.class).put(matcher, path.toString()); } } } diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java index a2e78bbd9d8..979f3b6c815 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteHelper.java @@ -1,163 +1,44 @@ package datadog.trace.instrumentation.pekkohttp; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; -import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR; - -import datadog.trace.api.Config; -import datadog.trace.api.KnownTags; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.URIUtils; -import datadog.trace.instrumentation.pekkohttp.PekkoHttpRouteState.Route; -import java.util.ArrayList; -import java.util.List; +import datadog.trace.bootstrap.ContextStore; import org.apache.pekko.http.scaladsl.model.Uri; import org.apache.pekko.http.scaladsl.server.PathMatcher; -import scala.Product; +import org.apache.pekko.http.scaladsl.server.PathMatchers; -/** Builds route templates from matched literals and parameter placeholders. */ +/** Records successful leaf matches without allocating frames for matcher invocations. */ public final class PekkoHttpRouteHelper { - private static final boolean OTEL_SEMANTICS = Config.get().isTraceOtelSemanticsEnabled(); - // Matcher calls nest synchronously; request state carries routes across asynchronous directives. - private static final ThreadLocal CURRENT = new ThreadLocal<>(); - private PekkoHttpRouteHelper() {} - public static Frame enter(Uri.Path path) { - Frame parent = CURRENT.get(); - if (parent != null) { - // nested matchers share the root span already checked by the outermost frame - Frame frame = new Frame(path, parent.span, parent.state, parent); - CURRENT.set(frame); - return frame; - } + public static void matched( + PathMatcher matcher, + Uri.Path before, + PathMatcher.Matching result, + ContextStore prefixes) { PekkoHttpRouteState state = PekkoHttpRouteState.current(); - if (state == null || state.requestPath == null) { - return null; - } - AgentSpan active = activeSpan(); - if (active == null) { - return null; - } - AgentSpan root = active.getLocalRootSpan(); - if (root == null) { - return null; - } - Frame frame = new Frame(path, root, state, null); - CURRENT.set(frame); - return frame; - } - - public static void exit(Frame frame, PathMatcher.Matching matching, Throwable throwable) { - if (frame == null) { + if (state == null || state.remaining == null) { return; } - boolean isMatch = throwable == null && matching instanceof PathMatcher.Matched; - if (frame.parent == null) { - CURRENT.remove(); - if (!isMatch) { - frame.state.routeMatched = false; - } - } else { - CURRENT.set(frame.parent); - } - if (!isMatch) { + if (!(result instanceof PathMatcher.Matched)) { + state.unmatched(before); return; } - PathMatcher.Matched matched = (PathMatcher.Matched) matching; - String rest = matched.pathRest().toString(); - String path = frame.path(); - int consumed = path.length() - rest.length(); - if (consumed < 0 || (consumed == 0 && frame.parent != null)) { + PathMatcher.Matched matched = (PathMatcher.Matched) result; + Uri.Path after = matched.pathRest(); + if (before.equals(after)) { return; } - String template = consumed == 0 ? "" : frame.template(consumed, matched.extractions()); - if (frame.parent != null) { - String parentPath = frame.parent.path(); - if (parentPath.endsWith(path)) { - if (frame.parent.parts == null) { - frame.parent.parts = new ArrayList<>(); - } - frame.parent.parts.add(new Route(parentPath.length() - path.length(), rest, template)); - } + // Most composed matchers finish at the remainder already recorded by their leaves. + if (after.equals(state.remaining)) { return; } - Route previous = null; - if (!frame.state.requestPath.equals(path)) { - previous = frame.state.route; - // A rejected route can retry an earlier unmatched path in another alternative. - while (previous != null && !previous.remaining.equals(path)) { - previous = previous.previous; - } - if (previous == null) { - return; - } - template = previous.template + template; - } - if (consumed == 0) { - // Keep only the enclosing prefix, not a rejected candidate that consumed more of the path. - if (previous == null) { - return; - } - frame.state.route = previous; - } else { - if (frame.state.route == null && OTEL_SEMANTICS) { - frame.state.originalResourceName = frame.span.getResourceName(); - } - frame.state.route = new Route(0, rest, template, previous); - } - frame.state.routeMatched = true; - if (OTEL_SEMANTICS) { - Object method = frame.span.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); - HTTP_RESOURCE_DECORATOR.withRoute( - frame.span, method == null ? null : method.toString(), template, true); - } else { - // Commit legacy route naming at completion so rejected candidates cannot block 404 naming. - frame.span.setTag(KnownTags.HTTP_ROUTE_ID, URIUtils.decode(template)); - } - } - - public static final class Frame { - final Uri.Path uriPath; - final AgentSpan span; - final PekkoHttpRouteState state; - final Frame parent; - List parts; - private String path; - - Frame(Uri.Path uriPath, AgentSpan span, PekkoHttpRouteState state, Frame parent) { - this.uriPath = uriPath; - this.span = span; - this.state = state; - this.parent = parent; - } - - /** Renders the path on demand, as most match attempts fail and never need it. */ - String path() { - if (path == null) { - path = uriPath.toString(); - } - return path; - } - - String template(int consumed, Object extractions) { - String path = path(); - if (parts == null) { - if (extractions instanceof Product && ((Product) extractions).productArity() > 0) { - return path.charAt(0) == '/' ? "/{param}" : "{param}"; - } - return path.substring(0, consumed); - } - StringBuilder template = new StringBuilder(); - int cursor = 0; - for (Route part : parts) { - int end = path.length() - part.remaining.length(); - if (part.offset < cursor || end > consumed) { - continue; - } - template.append(path, cursor, part.offset).append(part.template); - cursor = end; + String literal = prefixes.get(matcher); + if (literal == null) { + if (matcher.getClass() == PathMatchers.Slash$.class) { + literal = "/"; + } else { + literal = before.startsWithSlash() ? "/{param}" : "{param}"; } - return template.append(path, cursor, consumed).toString(); } + state.matched(before, after, literal); } } diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteModule.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteModule.java new file mode 100644 index 00000000000..c9772715c9e --- /dev/null +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteModule.java @@ -0,0 +1,39 @@ +package datadog.trace.instrumentation.pekkohttp; + +import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED; +import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED; +import static java.util.Arrays.asList; +import static java.util.Collections.singletonMap; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.config.provider.ConfigProvider; +import java.util.List; +import java.util.Map; + +/** Captures matched routes and selects their completed directive branches. */ +@AutoService(InstrumenterModule.class) +public final class PekkoHttpRouteModule extends InstrumenterModule.Tracing { + public PekkoHttpRouteModule() { + super("pekko-http-route"); + } + + @Override + protected boolean defaultEnabled() { + return super.defaultEnabled() + && ConfigProvider.getInstance() + .getBoolean(TRACE_OTEL_SEMANTICS_ENABLED, DEFAULT_TRACE_OTEL_SEMANTICS_ENABLED); + } + + @Override + public Map contextStore() { + return singletonMap("org.apache.pekko.http.scaladsl.server.PathMatcher", "java.lang.String"); + } + + @Override + public List typeInstrumentations() { + return asList( + new PekkoHttpPathMatcherInstrumentation(), new PekkoHttpDirectiveInstrumentation()); + } +} diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java index c07770c0408..cb02dbf7d8b 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java @@ -1,7 +1,6 @@ package datadog.trace.instrumentation.pekkohttp; import static datadog.context.ContextKey.named; -import static datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities.HTTP_FRAMEWORK_ROUTE; import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR; import static java.util.Collections.singletonList; @@ -14,31 +13,29 @@ import datadog.trace.bootstrap.instrumentation.api.Tags; import org.apache.pekko.http.scaladsl.model.Uri; -/** Request-local route state, independent of the HTTP routing DSL. */ +/** + * Route contributions owned by one directive invocation, with immutable history shared by child + * directives. + */ public final class PekkoHttpRouteState { private static final boolean OTEL_SEMANTICS = Config.get().isTraceOtelSemanticsEnabled(); private static final ContextKey KEY = named("pekko-http-route"); - private static final boolean ROUTE_CAPTURE_ENABLED = InstrumenterConfig.get() .isIntegrationEnabled( singletonList("pekko-http-route"), InstrumenterConfig.get().isIntegrationsEnabled() && OTEL_SEMANTICS); - String requestPath; - Route route; - CharSequence originalResourceName; - - /** Whether the latest route candidate has matched without subsequently being rejected. */ - boolean routeMatched; + Uri.Path remaining; + Part part; - PekkoHttpRouteState(String requestPath) { - this.requestPath = requestPath; + PekkoHttpRouteState(Uri.Path remaining) { + this.remaining = remaining; } public static Context init(Context context, Uri.Path requestPath) { return ROUTE_CAPTURE_ENABLED - ? context.with(KEY, new PekkoHttpRouteState(requestPath.toString())) + ? context.with(KEY, new PekkoHttpRouteState(requestPath)) : context; } @@ -46,60 +43,91 @@ static PekkoHttpRouteState current() { return Context.current().get(KEY); } - public static void onRejected() { - PekkoHttpRouteState state = current(); - if (state != null) { - state.routeMatched = false; + Context storeIn(Context context) { + return context.with(KEY, this); + } + + PekkoHttpRouteState fork() { + PekkoHttpRouteState child = new PekkoHttpRouteState(remaining); + child.part = part; + return child; + } + + void select(PekkoHttpRouteState child) { + part = child.part; + remaining = child.remaining; + } + + void matched(Uri.Path before, Uri.Path after, String literal) { + if (!before.equals(remaining)) { + // Lookahead matchers can inspect input that a later directive consumes again. + unmatched(before); + if (!before.equals(remaining)) { + return; + } + } + part = new Part(before, literal, part); + remaining = after; + } + + void unmatched(Uri.Path before) { + // A composed matcher can fail after some of its leaves have consumed input. + for (Part cursor = part; cursor != null; cursor = cursor.previous) { + if (cursor.before.equals(before)) { + part = cursor.previous; + remaining = before; + return; + } + } + } + + String template() { + if (part == null) { + return null; + } + int length = 0; + for (Part cursor = part; cursor != null; cursor = cursor.previous) { + length += cursor.literal.length(); + } + char[] chars = new char[length]; + for (Part cursor = part; cursor != null; cursor = cursor.previous) { + String literal = cursor.literal; + length -= literal.length(); + literal.getChars(0, literal.length(), chars, length); } + return new String(chars); } - /** Drops rejected candidates and releases route history after request completion. */ + /** Publishes only the selected route, before normal HTTP response naming. */ public static void onComplete(Context context, AgentSpan span) { PekkoHttpRouteState state = context.get(KEY); if (state == null) { return; } - state.requestPath = null; - Route route = state.route; - state.route = null; - CharSequence originalResourceName = state.originalResourceName; - state.originalResourceName = null; - if (route == null) { + String template = state.template(); + state.part = null; + state.remaining = null; + if (template == null) { return; } AgentSpan root = span.getLocalRootSpan(); if (root == null) { root = span; } - if (state.routeMatched) { - if (!OTEL_SEMANTICS) { - Object method = root.getTag(Tags.HTTP_METHOD); - HTTP_RESOURCE_DECORATOR.withRoute( - root, method == null ? null : method.toString(), route.template, true); - } - return; - } - root.setTag(KnownTags.HTTP_ROUTE_ID, (String) null); - if (originalResourceName != null) { - // Undo tentative route naming without overriding higher-priority application names. - root.setResourceName(originalResourceName, HTTP_FRAMEWORK_ROUTE); - } + Object method = + root.getTag(OTEL_SEMANTICS ? KnownTags.HTTP_METHOD_OTEL_NAME : Tags.HTTP_METHOD); + HTTP_RESOURCE_DECORATOR.withRoute( + root, method == null ? null : method.toString(), template, true); } - public static final class Route { - final int offset; - final String remaining; - final String template; - final Route previous; - - Route(int offset, String remaining, String template) { - this(offset, remaining, template, null); - } + static final class Part { + final Uri.Path before; + final String literal; + final Part previous; - Route(int offset, String remaining, String template, Route previous) { - this.offset = offset; - this.remaining = remaining; - this.template = template; + Part(Uri.Path before, String literal, Part previous) { + this.before = before; + this.literal = literal; this.previous = previous; } } diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteWrapper.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteWrapper.java new file mode 100644 index 00000000000..1d584e49f67 --- /dev/null +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteWrapper.java @@ -0,0 +1,73 @@ +package datadog.trace.instrumentation.pekkohttp; + +import datadog.context.Context; +import datadog.context.ContextScope; +import org.apache.pekko.http.scaladsl.server.RequestContext; +import org.apache.pekko.http.scaladsl.server.RouteResult; +import scala.Function1; +import scala.Option; +import scala.concurrent.Future; +import scala.runtime.AbstractFunction1; +import scala.runtime.AbstractPartialFunction; +import scala.runtime.BoxedUnit; +import scala.util.Try; + +/** Isolates a directive invocation until its route result is known. */ +public final class PekkoHttpRouteWrapper + extends AbstractFunction1> { + private final Function1> delegate; + + private PekkoHttpRouteWrapper(Function1> delegate) { + this.delegate = delegate; + } + + public static Function1> wrap( + Function1> route) { + return route instanceof PekkoHttpRouteWrapper ? route : new PekkoHttpRouteWrapper(route); + } + + @Override + public Future apply(RequestContext request) { + PekkoHttpRouteState parent = PekkoHttpRouteState.current(); + if (parent == null) { + return delegate.apply(request); + } + PekkoHttpRouteState child = parent.fork(); + try (ContextScope ignored = child.storeIn(Context.current()).attach()) { + Future future; + try { + future = delegate.apply(request); + } catch (Throwable failure) { + parent.select(child); + throw failure; + } + Option> value = future.value(); + if (value.isDefined()) { + select(parent, child, value.get()); + return future; + } + return future.andThen( + new AbstractPartialFunction, BoxedUnit>() { + @Override + public boolean isDefinedAt(Try result) { + return true; + } + + @Override + public BoxedUnit apply(Try result) { + select(parent, child, result); + return BoxedUnit.UNIT; + } + }, + request.executionContext()); + } + } + + private static void select( + PekkoHttpRouteState parent, PekkoHttpRouteState child, Try result) { + // Exceptions may be converted to a response by an enclosing exception handler. + if (result.isFailure() || result.get() instanceof RouteResult.Complete) { + parent.select(child); + } + } +} From 4a7edf649a3a2571f7ceb52363d427b96680d19e Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Thu, 5 Nov 2026 18:16:46 +0000 Subject: [PATCH 31/34] Apply HTTP routes to application server spans Apply Akka and Pekko routes to their request spans and copy Play routes to the active transport span. The local root can be an inferred gateway, whose route must remain independent of the application's matched route. Keep Play route propagation gated on OpenTelemetry semantics. --- .../AkkaHttpServerInstrumentationTest.groovy | 40 +++++++++++++++++++ .../akkahttp/AkkaHttpRouteState.java | 8 +--- .../PekkoHttpServerInstrumentationTest.groovy | 40 +++++++++++++++++++ .../pekkohttp/PekkoHttpRouteState.java | 8 +--- .../play23/PlayHttpServerDecorator.java | 7 ---- .../instrumentation/play24/PlayAdvice.java | 4 +- .../play24/PlayHttpServerDecorator.java | 22 ++++++---- .../instrumentation/play26/PlayAdvice.java | 4 +- .../play26/PlayHttpServerDecorator.java | 22 ++++++---- .../play26/server/PlayServerTest.groovy | 24 +++++++++++ ...rverRoutesScalaWithErrorHandlerTest.groovy | 36 +++++++++++++++++ 11 files changed, 180 insertions(+), 35 deletions(-) diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy index a9e81a0750f..db1322a5911 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy @@ -654,3 +654,43 @@ class AkkaHttpSingleRejectedRouteForkedTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0, TestingOtelHttpSemantics { + @Override + protected void configurePreAgent() { + super.configurePreAgent() + injectSysConfig('trace.inferred.proxy.services.enabled', 'true') + } + + @Override + HttpServer server() { + new AkkaHttpTestWebServer(AkkaHttpTestWebServer.BindAndHandle()) + } + + def "application routes preserve the inferred gateway route"() { + when: + def response = client.newCall(new Request.Builder() + .url(address.resolve('/items/value').toString()) + .header('x-dd-proxy', 'aws-apigateway') + .header('x-dd-proxy-request-time-ms', Long.toString(System.currentTimeMillis())) + .header('x-dd-proxy-httpmethod', 'GET') + .header('x-dd-proxy-path', '/public/items/value') + .header('x-dd-proxy-resource-path', '/public/{proxy+}') + .build()).execute() + assert response.code() == 200 + response.close() + TEST_WRITER.waitForTraces(1) + + then: + def spans = TEST_WRITER.flatten() + def gateway = spans.find { it.getTag('_dd.inferred_span') != null } + def server = spans.find { it.getTag('component')?.toString() == 'akka-http-server' } + gateway != null + server != null + server.parentId == gateway.spanId + gateway.getTag('http.route') == '/public/{proxy+}' + gateway.resourceName.toString() == 'GET /public/{proxy+}' + server.getTag('http.route') == '/items/{param}' + server.resourceName.toString() == 'GET /items/{param}' + } +} diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java index b235413f68f..2c81fc10030 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/main/java/datadog/trace/instrumentation/akkahttp/AkkaHttpRouteState.java @@ -108,14 +108,10 @@ public static void onComplete(Context context, AgentSpan span) { if (template == null) { return; } - AgentSpan root = span.getLocalRootSpan(); - if (root == null) { - root = span; - } Object method = - root.getTag(OTEL_SEMANTICS ? KnownTags.HTTP_METHOD_OTEL_NAME : Tags.HTTP_METHOD); + span.getTag(OTEL_SEMANTICS ? KnownTags.HTTP_METHOD_OTEL_NAME : Tags.HTTP_METHOD); HTTP_RESOURCE_DECORATOR.withRoute( - root, method == null ? null : method.toString(), template, true); + span, method == null ? null : method.toString(), template, true); } static final class Part { diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy index a17d22cb036..75d227f82bd 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy @@ -434,3 +434,43 @@ class PekkoHttpSingleRejectedRouteForkedTest extends WithHttpServer implements TestingGenericHttpNamingConventions.ServerV0, TestingOtelHttpSemantics { + @Override + protected void configurePreAgent() { + super.configurePreAgent() + injectSysConfig('trace.inferred.proxy.services.enabled', 'true') + } + + @Override + HttpServer server() { + new PekkoHttpTestWebServer(PekkoHttpTestWebServer.BindAndHandle()) + } + + def "application routes preserve the inferred gateway route"() { + when: + def response = client.newCall(new Request.Builder() + .url(address.resolve('/items/value').toString()) + .header('x-dd-proxy', 'aws-apigateway') + .header('x-dd-proxy-request-time-ms', Long.toString(System.currentTimeMillis())) + .header('x-dd-proxy-httpmethod', 'GET') + .header('x-dd-proxy-path', '/public/items/value') + .header('x-dd-proxy-resource-path', '/public/{proxy+}') + .build()).execute() + assert response.code() == 200 + response.close() + TEST_WRITER.waitForTraces(1) + + then: + def spans = TEST_WRITER.flatten() + def gateway = spans.find { it.getTag('_dd.inferred_span') != null } + def server = spans.find { it.getTag('component')?.toString() == 'pekko-http-server' } + gateway != null + server != null + server.parentId == gateway.spanId + gateway.getTag('http.route') == '/public/{proxy+}' + gateway.resourceName.toString() == 'GET /public/{proxy+}' + server.getTag('http.route') == '/items/{param}' + server.resourceName.toString() == 'GET /items/{param}' + } +} diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java index cb02dbf7d8b..0322ebd9bb1 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/main/java/datadog/trace/instrumentation/pekkohttp/PekkoHttpRouteState.java @@ -110,14 +110,10 @@ public static void onComplete(Context context, AgentSpan span) { if (template == null) { return; } - AgentSpan root = span.getLocalRootSpan(); - if (root == null) { - root = span; - } Object method = - root.getTag(OTEL_SEMANTICS ? KnownTags.HTTP_METHOD_OTEL_NAME : Tags.HTTP_METHOD); + span.getTag(OTEL_SEMANTICS ? KnownTags.HTTP_METHOD_OTEL_NAME : Tags.HTTP_METHOD); HTTP_RESOURCE_DECORATOR.withRoute( - root, method == null ? null : method.toString(), template, true); + span, method == null ? null : method.toString(), template, true); } static final class Part { diff --git a/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java b/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java index c447a7fe0e7..3904b0f77c9 100644 --- a/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java @@ -105,13 +105,6 @@ protected void doOnRequest( if (!pathOption.isEmpty()) { final String path = (String) pathOption.get(); HTTP_RESOURCE_DECORATOR.withRoute(span, request.method(), path); - if (OTEL_SEMANTICS) { - final AgentSpan rootSpan = span.getLocalRootSpan(); - if (rootSpan != null && rootSpan != span) { - // Set the transport span's route before application code can trigger sampling. - HTTP_RESOURCE_DECORATOR.withRoute(rootSpan, request.method(), path); - } - } dispatchRoute(span, path); } } diff --git a/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayAdvice.java b/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayAdvice.java index deb84b7b0bf..1f481903c2b 100644 --- a/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayAdvice.java +++ b/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayAdvice.java @@ -30,7 +30,8 @@ public static ContextScope onEnter(@Advice.Argument(value = 0, readOnly = false) return null; } - if (activeSpan() == null) { + final AgentSpan transportSpan = activeSpan(); + if (transportSpan == null) { final Headers headers = req.headers(); final Context parentContext = DECORATE.extract(headers); final Context context = DECORATE.startSpan(headers, parentContext); @@ -51,6 +52,7 @@ public static ContextScope onEnter(@Advice.Argument(value = 0, readOnly = false) // Moved from OnMethodExit // Call onRequest on return after tags are populated. DECORATE.onRequest(span, req, req, rootContext()); + DECORATE.propagateRoute(span, transportSpan); return scope; } diff --git a/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java b/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java index e7948f90164..5c52903fa44 100644 --- a/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java @@ -5,6 +5,7 @@ import datadog.context.Context; import datadog.trace.api.Config; +import datadog.trace.api.KnownTags; import datadog.trace.api.gateway.CallbackProvider; import datadog.trace.api.gateway.RequestContext; import datadog.trace.api.gateway.RequestContextSlot; @@ -34,6 +35,20 @@ public class PlayHttpServerDecorator public static final CharSequence PLAY_ACTION = UTF8BytesString.create("play-action"); public static final PlayHttpServerDecorator DECORATE = new PlayHttpServerDecorator(); + /** + * Copies the matched route to the transport span before application code can trigger sampling. + */ + public void propagateRoute(AgentSpan playSpan, AgentSpan transportSpan) { + if (OTEL_SEMANTICS && transportSpan != null) { + Object route = playSpan.getTag(KnownTags.HTTP_ROUTE_NAME); + if (route instanceof CharSequence) { + Object method = playSpan.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); + HTTP_RESOURCE_DECORATOR.withRoute( + transportSpan, method == null ? null : method.toString(), (CharSequence) route); + } + } + } + @Override protected String[] instrumentationNames() { return new String[] {"play"}; @@ -105,13 +120,6 @@ protected void doOnRequest( if (!pathOption.isEmpty()) { final String path = (String) pathOption.get(); HTTP_RESOURCE_DECORATOR.withRoute(span, request.method(), path); - if (OTEL_SEMANTICS) { - final AgentSpan rootSpan = span.getLocalRootSpan(); - if (rootSpan != null && rootSpan != span) { - // Set the transport span's route before application code can trigger sampling. - HTTP_RESOURCE_DECORATOR.withRoute(rootSpan, request.method(), path); - } - } dispatchRoute(span, path); } } diff --git a/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayAdvice.java b/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayAdvice.java index f1c66f17bbb..9a26bf841d8 100644 --- a/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayAdvice.java +++ b/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayAdvice.java @@ -31,7 +31,8 @@ public static ContextScope onEnter(@Advice.Argument(value = 0, readOnly = false) return null; } - if (activeSpan() == null) { + final AgentSpan transportSpan = activeSpan(); + if (transportSpan == null) { final Headers headers = req.headers(); parentContext = DECORATE.extract(headers); final Context context = DECORATE.startSpan(headers, parentContext); @@ -53,6 +54,7 @@ public static ContextScope onEnter(@Advice.Argument(value = 0, readOnly = false) // Moved from OnMethodExit // Call onRequest on return after tags are populated. DECORATE.onRequest(span, req, req, parentContext); + DECORATE.propagateRoute(span, transportSpan); return scope; } diff --git a/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java b/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java index fc621b75de6..bcd0a1e0418 100644 --- a/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java @@ -5,6 +5,7 @@ import datadog.context.Context; import datadog.trace.api.Config; +import datadog.trace.api.KnownTags; import datadog.trace.api.cache.DDCache; import datadog.trace.api.cache.DDCaches; import datadog.trace.api.gateway.CallbackProvider; @@ -71,6 +72,20 @@ public class PlayHttpServerDecorator TYPED_KEY_GET_UNDERLYING = typedKeyGetUnderlyingCheck; } + /** + * Copies the matched route to the transport span before application code can trigger sampling. + */ + public void propagateRoute(AgentSpan playSpan, AgentSpan transportSpan) { + if (OTEL_SEMANTICS && transportSpan != null) { + Object route = playSpan.getTag(KnownTags.HTTP_ROUTE_NAME); + if (route instanceof CharSequence) { + Object method = playSpan.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); + HTTP_RESOURCE_DECORATOR.withRoute( + transportSpan, method == null ? null : method.toString(), (CharSequence) route); + } + } + } + @Override protected String[] instrumentationNames() { return new String[] {"play"}; @@ -153,13 +168,6 @@ protected void doOnRequest( PATH_CACHE.computeIfAbsent( defOption.get().path(), p -> addMissingSlash(p, request.path())); HTTP_RESOURCE_DECORATOR.withRoute(span, request.method(), path, true); - if (OTEL_SEMANTICS) { - final AgentSpan rootSpan = span.getLocalRootSpan(); - if (rootSpan != null && rootSpan != span) { - // Set the transport span's route before application code can trigger sampling. - HTTP_RESOURCE_DECORATOR.withRoute(rootSpan, request.method(), path, true); - } - } dispatchRoute(span, path); } } diff --git a/dd-java-agent/instrumentation/play/play-2.6/src/test/groovy/datadog/trace/instrumentation/play26/server/PlayServerTest.groovy b/dd-java-agent/instrumentation/play/play-2.6/src/test/groovy/datadog/trace/instrumentation/play26/server/PlayServerTest.groovy index a12670a1285..a1a7762cc39 100644 --- a/dd-java-agent/instrumentation/play/play-2.6/src/test/groovy/datadog/trace/instrumentation/play26/server/PlayServerTest.groovy +++ b/dd-java-agent/instrumentation/play/play-2.6/src/test/groovy/datadog/trace/instrumentation/play26/server/PlayServerTest.groovy @@ -1,6 +1,9 @@ package datadog.trace.instrumentation.play26.server +import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.naming.TestingOtelHttpSemantics +import datadog.trace.bootstrap.instrumentation.api.AgentSpan +import datadog.trace.instrumentation.play26.PlayHttpServerDecorator class PlayServerTest extends AbstractPlayServerTest { // Disabling involving Appsec / Instrumentation Gateway, they are tested in play-appsec-2.x modules @@ -53,3 +56,24 @@ class PlayServerTest extends AbstractPlayServerTest { class PlayServerOtelSemanticsForkedTest extends PlayServerTest implements TestingOtelHttpSemantics { } + +class PlayRoutePropagationDatadogSemanticsForkedTest extends InstrumentationSpecification { + @Override + protected void configurePreAgent() { + super.configurePreAgent() + injectSysConfig('trace.otel.semantics.enabled', 'false') + } + + def "route propagation does not touch spans with Datadog semantics"() { + setup: + def playSpan = Mock(AgentSpan) + def transportSpan = Mock(AgentSpan) + + when: + PlayHttpServerDecorator.DECORATE.propagateRoute(playSpan, transportSpan) + + then: + 0 * playSpan._ + 0 * transportSpan._ + } +} diff --git a/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy b/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy index 9f517dacfe6..e0ea6f6e822 100644 --- a/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy +++ b/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy @@ -12,6 +12,7 @@ import datadog.trace.api.config.TraceInstrumentationConfig import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.instrumentation.play26.PlayHttpServerDecorator import datadog.trace.instrumentation.play26.server.TestHttpErrorHandler +import okhttp3.Request import play.api.BuiltInComponents import play.libs.concurrent.ClassLoaderExecution import play.routing.Router @@ -131,4 +132,39 @@ class PlayAsyncServerRoutesScalaWithErrorHandlerTest extends AbstractPlayServer2 } class PlayAsyncServerRoutesOtelSemanticsForkedTest extends PlayAsyncServerRoutesScalaWithErrorHandlerTest implements TestingOtelHttpSemantics { + @Override + protected void configurePreAgent() { + super.configurePreAgent() + injectSysConfig('trace.inferred.proxy.services.enabled', 'true') + } + + def "application routes preserve the inferred gateway route"() { + when: + def response = client.newCall(new Request.Builder() + .url(address.resolve('/path/123/param').toString()) + .header('x-dd-proxy', 'aws-apigateway') + .header('x-dd-proxy-request-time-ms', Long.toString(System.currentTimeMillis())) + .header('x-dd-proxy-httpmethod', 'GET') + .header('x-dd-proxy-path', '/public/path/123/param') + .header('x-dd-proxy-resource-path', '/public/{proxy+}') + .build()).execute() + assert response.code() == 200 + response.close() + TEST_WRITER.waitForTraces(1) + + then: + def spans = TEST_WRITER.flatten() + def gateway = spans.find { it.getTag('_dd.inferred_span') != null } + def transport = spans.find { it.getTag('component')?.toString() == 'akka-http-server' } + def play = spans.find { it.getTag('component')?.toString() == 'play-action' } + gateway != null + transport != null + play != null + transport.parentId == gateway.spanId + gateway.getTag('http.route') == '/public/{proxy+}' + gateway.resourceName.toString() == 'GET /public/{proxy+}' + transport.getTag('http.route') == '/path/$path<[^/]+>/param' + transport.resourceName.toString() == 'GET /path/$path<[^/]+>/param' + play.getTag('http.route') == '/path/$path<[^/]+>/param' + } } From 42e5bcf831f01ca4123e1b4a888d0f9d610422ac Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Thu, 5 Nov 2026 18:55:29 +0000 Subject: [PATCH 32/34] Select the HTTP server span when propagating Play routes Use the local root for ordinary requests and the application server span already registered beneath an inferred proxy. A tracing filter can be active when Play starts its action span, so selecting that active span leaves the HTTP server without its matched route before sampling. Propagate the route before route callbacks while preserving the gateway's route and keeping the change limited to OpenTelemetry semantics. --- .../decorator/HttpServerDecorator.java | 6 ++ .../instrumentation/play23/PlayAdvice.java | 11 +-- .../play23/PlayHttpServerDecorator.java | 6 ++ .../instrumentation/play24/PlayAdvice.java | 16 ++-- .../play24/PlayHttpServerDecorator.java | 22 ++---- .../instrumentation/play26/PlayAdvice.java | 16 ++-- .../play26/PlayHttpServerDecorator.java | 22 ++---- .../play26/server/PlayServerTest.groovy | 24 ------ .../server/PlayScalaRoutesServerTest.groovy | 72 ++++++++++++++++++ ...rverRoutesScalaWithErrorHandlerTest.groovy | 76 +++++++++++++++++++ .../trace/api/gateway/InferredProxySpan.java | 5 ++ 11 files changed, 193 insertions(+), 83 deletions(-) diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java index 60ebcf8a1f9..9cc7132402b 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java @@ -230,6 +230,12 @@ private void registerServiceEntrySpanInInferredProxy( } } + /** Returns the application HTTP server span, beneath any inferred proxy span. */ + protected static AgentSpan getHttpServerSpan(AgentSpan span) { + InferredProxySpan inferredProxy = InferredProxySpan.fromContext(Context.current()); + return inferredProxy != null ? inferredProxy.getServiceEntrySpan() : span.getLocalRootSpan(); + } + private void resetServiceNameIfUnderInferredProxy(Context parentContext, AgentSpan span) { if (InferredProxySpan.fromContext(parentContext) != null) { span.setServiceName(Config.get().getServiceName()); diff --git a/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayAdvice.java b/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayAdvice.java index f114e48dcbf..334aefa5ead 100644 --- a/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayAdvice.java +++ b/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayAdvice.java @@ -31,8 +31,7 @@ public static ContextScope onEnter(@Advice.Argument(0) final Request req) { span = spanFromContext(context); scope = context.attach(); } else { - // An upstream framework (e.g. akka-http, netty) has already started the span. - // Do not extract the context. + // Keep the existing trace context instead of extracting the request headers again. span = startSpan(PLAY_ACTION.toString(), PLAY_REQUEST); span.setMeasured(true); scope = span.attachWithContext(); @@ -51,7 +50,6 @@ public static void stopTraceOnResponse( @Advice.Return(readOnly = false) final Future responseFuture) { final AgentSpan playControllerSpan = spanFromContext(playControllerScope.context()); - // Call onRequest on return after tags are populated. DECORATE.onRequest(playControllerSpan, req, req, rootContext()); if (throwable == null) { @@ -69,10 +67,9 @@ public static void stopTraceOnResponse( playControllerScope.close(); // span finished in RequestCompleteCallback - final AgentSpan rootSpan = activeSpan(); - // set the resource name on the upstream akka/netty span if there is one - if (rootSpan != null) { - DECORATE.onRequest(rootSpan, req, req, rootContext()); + final AgentSpan parentSpan = activeSpan(); + if (parentSpan != null) { + DECORATE.onRequest(parentSpan, req, req, rootContext()); } } } diff --git a/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java b/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java index 3904b0f77c9..13fcf40bf6a 100644 --- a/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/play/play-2.3/src/main/java/datadog/trace/instrumentation/play23/PlayHttpServerDecorator.java @@ -105,6 +105,12 @@ protected void doOnRequest( if (!pathOption.isEmpty()) { final String path = (String) pathOption.get(); HTTP_RESOURCE_DECORATOR.withRoute(span, request.method(), path); + if (OTEL_SEMANTICS) { + final AgentSpan serverSpan = getHttpServerSpan(span); + if (serverSpan != null && serverSpan != span) { + HTTP_RESOURCE_DECORATOR.withRoute(serverSpan, request.method(), path); + } + } dispatchRoute(span, path); } } diff --git a/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayAdvice.java b/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayAdvice.java index 1f481903c2b..b97b094f212 100644 --- a/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayAdvice.java +++ b/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayAdvice.java @@ -30,16 +30,14 @@ public static ContextScope onEnter(@Advice.Argument(value = 0, readOnly = false) return null; } - final AgentSpan transportSpan = activeSpan(); - if (transportSpan == null) { + if (activeSpan() == null) { final Headers headers = req.headers(); final Context parentContext = DECORATE.extract(headers); final Context context = DECORATE.startSpan(headers, parentContext); span = spanFromContext(context); scope = context.attach(); } else { - // An upstream framework (e.g. akka-http, netty) has already started the span. - // Do not extract the context. + // Keep the existing trace context instead of extracting the request headers again. span = startSpan(PLAY_ACTION.toString(), PLAY_REQUEST); span.setMeasured(true); scope = span.attachWithContext(); @@ -49,10 +47,7 @@ public static ContextScope onEnter(@Advice.Argument(value = 0, readOnly = false) req = RequestHelper.withTag(req, "_dd_HasPlayRequestSpan", "true"); - // Moved from OnMethodExit - // Call onRequest on return after tags are populated. DECORATE.onRequest(span, req, req, rootContext()); - DECORATE.propagateRoute(span, transportSpan); return scope; } @@ -86,10 +81,9 @@ public static void stopTraceOnResponse( playControllerScope.close(); // span finished in RequestCompleteCallback - final AgentSpan rootSpan = activeSpan(); - // set the resource name on the upstream akka/netty span if there is one - if (rootSpan != null) { - DECORATE.onRequest(rootSpan, req, req, rootContext()); + final AgentSpan parentSpan = activeSpan(); + if (parentSpan != null) { + DECORATE.onRequest(parentSpan, req, req, rootContext()); } } diff --git a/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java b/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java index 5c52903fa44..100c56b5527 100644 --- a/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/play/play-2.4/src/main/java/datadog/trace/instrumentation/play24/PlayHttpServerDecorator.java @@ -5,7 +5,6 @@ import datadog.context.Context; import datadog.trace.api.Config; -import datadog.trace.api.KnownTags; import datadog.trace.api.gateway.CallbackProvider; import datadog.trace.api.gateway.RequestContext; import datadog.trace.api.gateway.RequestContextSlot; @@ -35,20 +34,6 @@ public class PlayHttpServerDecorator public static final CharSequence PLAY_ACTION = UTF8BytesString.create("play-action"); public static final PlayHttpServerDecorator DECORATE = new PlayHttpServerDecorator(); - /** - * Copies the matched route to the transport span before application code can trigger sampling. - */ - public void propagateRoute(AgentSpan playSpan, AgentSpan transportSpan) { - if (OTEL_SEMANTICS && transportSpan != null) { - Object route = playSpan.getTag(KnownTags.HTTP_ROUTE_NAME); - if (route instanceof CharSequence) { - Object method = playSpan.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); - HTTP_RESOURCE_DECORATOR.withRoute( - transportSpan, method == null ? null : method.toString(), (CharSequence) route); - } - } - } - @Override protected String[] instrumentationNames() { return new String[] {"play"}; @@ -120,6 +105,13 @@ protected void doOnRequest( if (!pathOption.isEmpty()) { final String path = (String) pathOption.get(); HTTP_RESOURCE_DECORATOR.withRoute(span, request.method(), path); + if (OTEL_SEMANTICS) { + final AgentSpan serverSpan = getHttpServerSpan(span); + if (serverSpan != null && serverSpan != span) { + // Sampling may run inside the action, so set the server route now. + HTTP_RESOURCE_DECORATOR.withRoute(serverSpan, request.method(), path); + } + } dispatchRoute(span, path); } } diff --git a/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayAdvice.java b/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayAdvice.java index 9a26bf841d8..5120acadea3 100644 --- a/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayAdvice.java +++ b/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayAdvice.java @@ -31,16 +31,14 @@ public static ContextScope onEnter(@Advice.Argument(value = 0, readOnly = false) return null; } - final AgentSpan transportSpan = activeSpan(); - if (transportSpan == null) { + if (activeSpan() == null) { final Headers headers = req.headers(); parentContext = DECORATE.extract(headers); final Context context = DECORATE.startSpan(headers, parentContext); span = spanFromContext(context); scope = context.attach(); } else { - // An upstream framework (e.g. akka-http, netty) has already started the span. - // Do not extract the context. + // Keep the existing trace context instead of extracting the request headers again. parentContext = rootContext(); span = startSpan(PLAY_ACTION.toString(), PLAY_REQUEST); scope = span.attachWithContext(); @@ -51,10 +49,7 @@ public static ContextScope onEnter(@Advice.Argument(value = 0, readOnly = false) req = req.addAttr(HasPlayRequestSpan.KEY, HasPlayRequestSpan.INSTANCE); - // Moved from OnMethodExit - // Call onRequest on return after tags are populated. DECORATE.onRequest(span, req, req, parentContext); - DECORATE.propagateRoute(span, transportSpan); return scope; } @@ -85,10 +80,9 @@ public static void stopTraceOnResponse( playControllerScope.close(); // span finished in RequestCompleteCallback - final AgentSpan rootSpan = activeSpan(); - // set the resource name on the upstream akka/netty span if there is one - if (rootSpan != null && playControllerSpan.getResourceName() != null) { - rootSpan.setResourceName( + final AgentSpan parentSpan = activeSpan(); + if (parentSpan != null && playControllerSpan.getResourceName() != null) { + parentSpan.setResourceName( playControllerSpan.getResourceName(), ResourceNamePriorities.HTTP_FRAMEWORK_ROUTE); } } diff --git a/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java b/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java index bcd0a1e0418..791ef666cda 100644 --- a/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java +++ b/dd-java-agent/instrumentation/play/play-2.6/src/main/java/datadog/trace/instrumentation/play26/PlayHttpServerDecorator.java @@ -5,7 +5,6 @@ import datadog.context.Context; import datadog.trace.api.Config; -import datadog.trace.api.KnownTags; import datadog.trace.api.cache.DDCache; import datadog.trace.api.cache.DDCaches; import datadog.trace.api.gateway.CallbackProvider; @@ -72,20 +71,6 @@ public class PlayHttpServerDecorator TYPED_KEY_GET_UNDERLYING = typedKeyGetUnderlyingCheck; } - /** - * Copies the matched route to the transport span before application code can trigger sampling. - */ - public void propagateRoute(AgentSpan playSpan, AgentSpan transportSpan) { - if (OTEL_SEMANTICS && transportSpan != null) { - Object route = playSpan.getTag(KnownTags.HTTP_ROUTE_NAME); - if (route instanceof CharSequence) { - Object method = playSpan.getTag(KnownTags.HTTP_METHOD_OTEL_NAME); - HTTP_RESOURCE_DECORATOR.withRoute( - transportSpan, method == null ? null : method.toString(), (CharSequence) route); - } - } - } - @Override protected String[] instrumentationNames() { return new String[] {"play"}; @@ -168,6 +153,13 @@ protected void doOnRequest( PATH_CACHE.computeIfAbsent( defOption.get().path(), p -> addMissingSlash(p, request.path())); HTTP_RESOURCE_DECORATOR.withRoute(span, request.method(), path, true); + if (OTEL_SEMANTICS) { + final AgentSpan serverSpan = getHttpServerSpan(span); + if (serverSpan != null && serverSpan != span) { + // Sampling may run inside the action, so set the server route now. + HTTP_RESOURCE_DECORATOR.withRoute(serverSpan, request.method(), path, true); + } + } dispatchRoute(span, path); } } diff --git a/dd-java-agent/instrumentation/play/play-2.6/src/test/groovy/datadog/trace/instrumentation/play26/server/PlayServerTest.groovy b/dd-java-agent/instrumentation/play/play-2.6/src/test/groovy/datadog/trace/instrumentation/play26/server/PlayServerTest.groovy index a1a7762cc39..a12670a1285 100644 --- a/dd-java-agent/instrumentation/play/play-2.6/src/test/groovy/datadog/trace/instrumentation/play26/server/PlayServerTest.groovy +++ b/dd-java-agent/instrumentation/play/play-2.6/src/test/groovy/datadog/trace/instrumentation/play26/server/PlayServerTest.groovy @@ -1,9 +1,6 @@ package datadog.trace.instrumentation.play26.server -import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.naming.TestingOtelHttpSemantics -import datadog.trace.bootstrap.instrumentation.api.AgentSpan -import datadog.trace.instrumentation.play26.PlayHttpServerDecorator class PlayServerTest extends AbstractPlayServerTest { // Disabling involving Appsec / Instrumentation Gateway, they are tested in play-appsec-2.x modules @@ -56,24 +53,3 @@ class PlayServerTest extends AbstractPlayServerTest { class PlayServerOtelSemanticsForkedTest extends PlayServerTest implements TestingOtelHttpSemantics { } - -class PlayRoutePropagationDatadogSemanticsForkedTest extends InstrumentationSpecification { - @Override - protected void configurePreAgent() { - super.configurePreAgent() - injectSysConfig('trace.otel.semantics.enabled', 'false') - } - - def "route propagation does not touch spans with Datadog semantics"() { - setup: - def playSpan = Mock(AgentSpan) - def transportSpan = Mock(AgentSpan) - - when: - PlayHttpServerDecorator.DECORATE.propagateRoute(playSpan, transportSpan) - - then: - 0 * playSpan._ - 0 * transportSpan._ - } -} diff --git a/dd-java-agent/instrumentation/play/play-appsec-2.5/src/test/groovy/datadog/trace/instrumentation/play25/server/PlayScalaRoutesServerTest.groovy b/dd-java-agent/instrumentation/play/play-appsec-2.5/src/test/groovy/datadog/trace/instrumentation/play25/server/PlayScalaRoutesServerTest.groovy index abbac40db83..bb24b4fb5d7 100644 --- a/dd-java-agent/instrumentation/play/play-appsec-2.5/src/test/groovy/datadog/trace/instrumentation/play25/server/PlayScalaRoutesServerTest.groovy +++ b/dd-java-agent/instrumentation/play/play-appsec-2.5/src/test/groovy/datadog/trace/instrumentation/play25/server/PlayScalaRoutesServerTest.groovy @@ -1,15 +1,21 @@ package datadog.trace.instrumentation.play25.server +import datadog.context.Context +import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.asserts.TraceAssert import datadog.trace.agent.test.base.HttpServer import datadog.trace.agent.test.base.HttpServerTest import datadog.trace.api.DDSpanTypes import datadog.trace.api.DDTags import datadog.trace.api.config.TraceInstrumentationConfig +import datadog.trace.api.gateway.InferredProxySpan import datadog.trace.bootstrap.instrumentation.api.Tags +import datadog.trace.instrumentation.play24.PlayAdvice +import datadog.trace.instrumentation.play24.PlayHttpServerDecorator import datadog.trace.instrumentation.play25.PlayController import datadog.trace.test.util.Flaky import groovy.transform.CompileStatic +import play.api.mvc.Request import scala.concurrent.ExecutionContext$ import spock.lang.Shared @@ -22,6 +28,7 @@ import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.FORWAR import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.NOT_FOUND import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.PATH_PARAM import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.SUCCESS +import static datadog.trace.bootstrap.instrumentation.api.AgentSpan.fromContext class PlayScalaRoutesServerTest extends PlayServerTest { @Shared @@ -116,3 +123,68 @@ class PlayScalaRoutesServerTest extends PlayServerTest { } } } + +class PlayRouteTracingFilterOtelSemanticsForkedTest extends InstrumentationSpecification { + @Override + protected void configurePreAgent() { + super.configurePreAgent() + injectSysConfig('trace.inferred.proxy.services.enabled', 'true') + injectSysConfig('trace.otel.semantics.enabled', 'true') + } + + def "tracing filters do not receive the HTTP server route before sampling"() { + setup: + def routeTags = scala.collection.immutable.Map$.MODULE$.empty().updated('ROUTE_PATTERN', '/items/{param}') + def request = Stub(Request) { + tags() >> routeTags + method() >> 'GET' + path() >> '/items/value' + host() >> 'localhost' + remoteAddress() >> '127.0.0.1' + rawQueryString() >> '' + } + request.withTag(_, _) >> request + def inferred = InferredProxySpan.fromHeaders([ + 'x-dd-proxy': 'aws-apigateway', + 'x-dd-proxy-request-time-ms': Long.toString(System.currentTimeMillis()), + 'x-dd-proxy-httpmethod': 'GET', + 'x-dd-proxy-path': '/public/items/value', + 'x-dd-proxy-resource-path': '/public/{proxy+}' + ]) + def parent = inferredProxy ? Context.root().with(inferred) : Context.root() + def serverContext = PlayHttpServerDecorator.DECORATE.startSpan(null, parent) + def server = fromContext(serverContext) + server.setResourceName('GET') + def serverScope = serverContext.attach() + def filter = TEST_TRACER.startSpan('test', 'filter1') + filter.setResourceName('filter1') + def filterScope = TEST_TRACER.activateManualSpan(filter) + def playScope + def play + + when: + playScope = PlayAdvice.onEnter(request) + play = fromContext(playScope.context()) + + then: + play.getTag('http.route') == '/items/{param}' + server.getTag('http.route') == '/items/{param}' + server.resourceName.toString() == 'GET /items/{param}' + filter.getTag('http.route') == null + filter.resourceName.toString() == 'filter1' + !inferredProxy || server.localRootSpan.getTag('http.route') == '/public/{proxy+}' + !inferredProxy || server.localRootSpan.resourceName.toString() == 'GET /public/{proxy+}' + + cleanup: + playScope?.close() + play?.finish() + filterScope?.close() + filter?.finish() + serverScope?.close() + server?.finish() + inferred?.finish() + + where: + inferredProxy << [false, true] + } +} diff --git a/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy b/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy index e0ea6f6e822..28b28c348bf 100644 --- a/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy +++ b/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy @@ -1,5 +1,7 @@ package datadog.trace.instrumentation.play27.server.test +import datadog.context.Context +import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.asserts.TraceAssert import datadog.trace.agent.test.base.HttpServer import datadog.trace.agent.test.base.HttpServerTest @@ -9,11 +11,16 @@ import datadog.trace.api.DDSpanTypes import datadog.trace.api.DDTags import datadog.trace.api.KnownTags import datadog.trace.api.config.TraceInstrumentationConfig +import datadog.trace.api.gateway.InferredProxySpan import datadog.trace.bootstrap.instrumentation.api.Tags +import datadog.trace.instrumentation.play26.PlayAdvice import datadog.trace.instrumentation.play26.PlayHttpServerDecorator import datadog.trace.instrumentation.play26.server.TestHttpErrorHandler import okhttp3.Request import play.api.BuiltInComponents +import play.api.libs.typedmap.TypedMap$ +import play.api.mvc.Request as ScalaRequest +import play.api.routing.HandlerDef import play.libs.concurrent.ClassLoaderExecution import play.routing.Router import spock.lang.Shared @@ -28,6 +35,7 @@ import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.FORWAR import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.NOT_FOUND import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.PATH_PARAM import static datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint.SUCCESS +import static datadog.trace.bootstrap.instrumentation.api.AgentSpan.fromContext class PlayAsyncServerRoutesScalaWithErrorHandlerTest extends AbstractPlayServer27WithErrorHandlerTest { @Shared @@ -168,3 +176,71 @@ class PlayAsyncServerRoutesOtelSemanticsForkedTest extends PlayAsyncServerRoutes play.getTag('http.route') == '/path/$path<[^/]+>/param' } } + +class PlayRouteTracingFilterOtelSemanticsForkedTest extends InstrumentationSpecification { + @Override + protected void configurePreAgent() { + super.configurePreAgent() + injectSysConfig('trace.inferred.proxy.services.enabled', 'true') + injectSysConfig('trace.otel.semantics.enabled', 'true') + } + + def "tracing filters do not receive the HTTP server route before sampling"() { + setup: + def handler = Stub(HandlerDef) { + path() >> '/items/{param}' + } + def requestAttributes = TypedMap$.MODULE$.empty().updated(Router.Attrs.HANDLER_DEF.asScala(), handler) + def request = Stub(ScalaRequest) { + attrs() >> requestAttributes + method() >> 'GET' + path() >> '/items/value' + host() >> 'localhost' + remoteAddress() >> '127.0.0.1' + rawQueryString() >> '' + } + request.addAttr(_, _) >> request + def inferred = InferredProxySpan.fromHeaders([ + 'x-dd-proxy': 'aws-apigateway', + 'x-dd-proxy-request-time-ms': Long.toString(System.currentTimeMillis()), + 'x-dd-proxy-httpmethod': 'GET', + 'x-dd-proxy-path': '/public/items/value', + 'x-dd-proxy-resource-path': '/public/{proxy+}' + ]) + def parent = inferredProxy ? Context.root().with(inferred) : Context.root() + def serverContext = PlayHttpServerDecorator.DECORATE.startSpan(null, parent) + def server = fromContext(serverContext) + server.setResourceName('GET') + def serverScope = serverContext.attach() + def filter = TEST_TRACER.startSpan('test', 'filter1') + filter.setResourceName('filter1') + def filterScope = TEST_TRACER.activateManualSpan(filter) + def playScope + def play + + when: + playScope = PlayAdvice.onEnter(request) + play = fromContext(playScope.context()) + + then: + play.getTag('http.route') == '/items/{param}' + server.getTag('http.route') == '/items/{param}' + server.resourceName.toString() == 'GET /items/{param}' + filter.getTag('http.route') == null + filter.resourceName.toString() == 'filter1' + !inferredProxy || server.localRootSpan.getTag('http.route') == '/public/{proxy+}' + !inferredProxy || server.localRootSpan.resourceName.toString() == 'GET /public/{proxy+}' + + cleanup: + playScope?.close() + play?.finish() + filterScope?.close() + filter?.finish() + serverScope?.close() + server?.finish() + inferred?.finish() + + where: + inferredProxy << [false, true] + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/gateway/InferredProxySpan.java b/internal-api/src/main/java/datadog/trace/api/gateway/InferredProxySpan.java index 2c19b0772e3..f6ee480db3e 100644 --- a/internal-api/src/main/java/datadog/trace/api/gateway/InferredProxySpan.java +++ b/internal-api/src/main/java/datadog/trace/api/gateway/InferredProxySpan.java @@ -221,6 +221,11 @@ public void registerServiceEntrySpan(AgentSpan serviceEntrySpan) { this.registeredServiceEntrySpan = serviceEntrySpan; } + /** Returns the registered application server span, or {@code null} if none is registered. */ + public AgentSpan getServiceEntrySpan() { + return registeredServiceEntrySpan; + } + public void finish() { finish(null); } From 9d3bd47f973dfacfcf65b293ae68b9fea2dfb9ca Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Thu, 5 Nov 2026 19:50:42 +0000 Subject: [PATCH 33/34] Make Play gateway route regression deterministic Cover application route propagation with and without inferred proxies and tracing filters in the direct advice test. Remove the HTTP regression that depends on concurrent inferred-span completion, keeping this change limited to route tests. --- ...rverRoutesScalaWithErrorHandlerTest.groovy | 59 +++++-------------- 1 file changed, 14 insertions(+), 45 deletions(-) diff --git a/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy b/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy index 28b28c348bf..59ddf821ee5 100644 --- a/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy +++ b/dd-java-agent/instrumentation/play/play-appsec-2.7/src/test/groovy/datadog/trace/instrumentation/play27/server/test/PlayAsyncServerRoutesScalaWithErrorHandlerTest.groovy @@ -16,7 +16,6 @@ import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.instrumentation.play26.PlayAdvice import datadog.trace.instrumentation.play26.PlayHttpServerDecorator import datadog.trace.instrumentation.play26.server.TestHttpErrorHandler -import okhttp3.Request import play.api.BuiltInComponents import play.api.libs.typedmap.TypedMap$ import play.api.mvc.Request as ScalaRequest @@ -139,43 +138,7 @@ class PlayAsyncServerRoutesScalaWithErrorHandlerTest extends AbstractPlayServer2 } } -class PlayAsyncServerRoutesOtelSemanticsForkedTest extends PlayAsyncServerRoutesScalaWithErrorHandlerTest implements TestingOtelHttpSemantics { - @Override - protected void configurePreAgent() { - super.configurePreAgent() - injectSysConfig('trace.inferred.proxy.services.enabled', 'true') - } - - def "application routes preserve the inferred gateway route"() { - when: - def response = client.newCall(new Request.Builder() - .url(address.resolve('/path/123/param').toString()) - .header('x-dd-proxy', 'aws-apigateway') - .header('x-dd-proxy-request-time-ms', Long.toString(System.currentTimeMillis())) - .header('x-dd-proxy-httpmethod', 'GET') - .header('x-dd-proxy-path', '/public/path/123/param') - .header('x-dd-proxy-resource-path', '/public/{proxy+}') - .build()).execute() - assert response.code() == 200 - response.close() - TEST_WRITER.waitForTraces(1) - - then: - def spans = TEST_WRITER.flatten() - def gateway = spans.find { it.getTag('_dd.inferred_span') != null } - def transport = spans.find { it.getTag('component')?.toString() == 'akka-http-server' } - def play = spans.find { it.getTag('component')?.toString() == 'play-action' } - gateway != null - transport != null - play != null - transport.parentId == gateway.spanId - gateway.getTag('http.route') == '/public/{proxy+}' - gateway.resourceName.toString() == 'GET /public/{proxy+}' - transport.getTag('http.route') == '/path/$path<[^/]+>/param' - transport.resourceName.toString() == 'GET /path/$path<[^/]+>/param' - play.getTag('http.route') == '/path/$path<[^/]+>/param' - } -} +class PlayAsyncServerRoutesOtelSemanticsForkedTest extends PlayAsyncServerRoutesScalaWithErrorHandlerTest implements TestingOtelHttpSemantics {} class PlayRouteTracingFilterOtelSemanticsForkedTest extends InstrumentationSpecification { @Override @@ -185,7 +148,7 @@ class PlayRouteTracingFilterOtelSemanticsForkedTest extends InstrumentationSpeci injectSysConfig('trace.otel.semantics.enabled', 'true') } - def "tracing filters do not receive the HTTP server route before sampling"() { + def "application routes preserve gateway routes and skip tracing filters before sampling"() { setup: def handler = Stub(HandlerDef) { path() >> '/items/{param}' @@ -212,9 +175,9 @@ class PlayRouteTracingFilterOtelSemanticsForkedTest extends InstrumentationSpeci def server = fromContext(serverContext) server.setResourceName('GET') def serverScope = serverContext.attach() - def filter = TEST_TRACER.startSpan('test', 'filter1') - filter.setResourceName('filter1') - def filterScope = TEST_TRACER.activateManualSpan(filter) + def filter = tracingFilter ? TEST_TRACER.startSpan('test', 'filter1') : null + filter?.setResourceName('filter1') + def filterScope = filter ? TEST_TRACER.activateManualSpan(filter) : null def playScope def play @@ -226,8 +189,10 @@ class PlayRouteTracingFilterOtelSemanticsForkedTest extends InstrumentationSpeci play.getTag('http.route') == '/items/{param}' server.getTag('http.route') == '/items/{param}' server.resourceName.toString() == 'GET /items/{param}' - filter.getTag('http.route') == null - filter.resourceName.toString() == 'filter1' + play.parentId == (filter ?: server).spanId + !filter || filter.getTag('http.route') == null + !filter || filter.resourceName.toString() == 'filter1' + !inferredProxy || server.parentId == server.localRootSpan.spanId !inferredProxy || server.localRootSpan.getTag('http.route') == '/public/{proxy+}' !inferredProxy || server.localRootSpan.resourceName.toString() == 'GET /public/{proxy+}' @@ -241,6 +206,10 @@ class PlayRouteTracingFilterOtelSemanticsForkedTest extends InstrumentationSpeci inferred?.finish() where: - inferredProxy << [false, true] + inferredProxy | tracingFilter + false | false + false | true + true | false + true | true } } From ad447e99e655c339b59477240f3a839b3d36479e Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Thu, 5 Nov 2026 20:49:11 +0000 Subject: [PATCH 34/34] Keep route fallbacks on the live parent after async rejection With Scala promise completion priority enabled, a fallback alternative can run under the completion context of a directive that already finished, so its route match was recorded on a discarded state. Retire each child state once its result is known and resolve retired states to their parent. --- .../AkkaHttpServerInstrumentationTest.groovy | 8 ++++++++ .../akkahttp/AkkaHttpRouteState.java | 14 +++++++++++++- .../akkahttp/AkkaHttpRouteWrapper.java | 2 ++ .../PekkoHttpServerInstrumentationTest.groovy | 8 ++++++++ .../pekkohttp/PekkoHttpRouteState.java | 14 +++++++++++++- .../pekkohttp/PekkoHttpRouteWrapper.java | 2 ++ 6 files changed, 46 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy index db1322a5911..c652aa8551f 100644 --- a/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/akka/akka-http/akka-http-10.0/src/baseTest/groovy/AkkaHttpServerInstrumentationTest.groovy @@ -417,6 +417,14 @@ abstract class AkkaHttpRouteOtelSemanticsTest extends WithHttpServer apply(RequestContext request) { future = delegate.apply(request); } catch (Throwable failure) { parent.select(child); + parent.retire(child); throw failure; } Option> value = future.value(); @@ -69,5 +70,6 @@ private static void select( if (result.isFailure() || result.get() instanceof RouteResult.Complete) { parent.select(child); } + parent.retire(child); } } diff --git a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy index 75d227f82bd..0e67ac7dd02 100644 --- a/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/pekko/pekko-http-1.0/src/baseTest/groovy/PekkoHttpServerInstrumentationTest.groovy @@ -247,6 +247,14 @@ abstract class PekkoHttpRouteOtelSemanticsTest extends WithHttpServer apply(RequestContext request) { future = delegate.apply(request); } catch (Throwable failure) { parent.select(child); + parent.retire(child); throw failure; } Option> value = future.value(); @@ -69,5 +70,6 @@ private static void select( if (result.isFailure() || result.get() instanceof RouteResult.Complete) { parent.select(child); } + parent.retire(child); } }