From d3004b038b8648b5d216e736e096214d2e8b9f65 Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Mon, 26 Oct 2026 16:47:00 -0400 Subject: [PATCH 1/3] Write global tags once on the OTLP trace resource Build the OTLP trace resource from each trace's config snapshot (DD_TAGS or its remote config override) instead of static Config, and stop repeating those global tags on every span. Spans keep their own tags plus the span defaults they still carry. The payload is flushed when a trace's global tags differ from the payload's, so each payload has one accurate resource. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../common/writer/OtlpPayloadDispatcher.java | 4 + .../datadog/trace/core/DDSpanContext.java | 4 + .../otlp/common/OtlpResourceAttributes.java | 28 +-- .../core/otlp/common/OtlpResourceJson.java | 23 ++- .../core/otlp/common/OtlpResourceProto.java | 25 ++- .../trace/core/otlp/trace/OtlpSpanTags.java | 67 +++++++ .../core/otlp/trace/OtlpTraceCollector.java | 21 +++ .../trace/core/otlp/trace/OtlpTraceJson.java | 5 +- .../otlp/trace/OtlpTraceJsonCollector.java | 20 ++- .../trace/core/otlp/trace/OtlpTraceProto.java | 5 +- .../otlp/trace/OtlpTraceProtoCollector.java | 23 ++- .../otlp/common/OtlpResourceJsonTest.java | 6 +- .../core/otlp/trace/OtlpSpanTagsTest.java | 167 ++++++++++++++++++ .../main/java/datadog/trace/api/TagMap.java | 22 +++ .../trace/api/TagMapReadThroughTest.java | 14 ++ 15 files changed, 399 insertions(+), 35 deletions(-) create mode 100644 dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpSpanTags.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpSpanTagsTest.java diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/OtlpPayloadDispatcher.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/OtlpPayloadDispatcher.java index ed69f08c37e..08246bb22ad 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/OtlpPayloadDispatcher.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/OtlpPayloadDispatcher.java @@ -26,6 +26,10 @@ final class OtlpPayloadDispatcher implements PayloadDispatcher { @Override public void addTrace(List> trace) { + // a payload carries one resource: start a new one when the trace's global tags differ + if (!collector.sharesResource(trace)) { + flush(); + } collector.addTrace(trace); // flush proactively to keep payload size bounded if (collector.sizeInBytes() >= FLUSH_THRESHOLD_BYTES) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index 24b652e7593..32b4708c901 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -876,6 +876,10 @@ public TraceCollector getTraceCollector() { return traceCollector; } + public TagMap getGlobalTags() { + return traceCollector.getTraceConfig().globalTags; + } + public RequestContext getRequestContext() { return this; } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceAttributes.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceAttributes.java index 6489c7ae5bf..c339f271bc7 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceAttributes.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceAttributes.java @@ -43,9 +43,15 @@ private OtlpResourceAttributes() {} /** * {@code value} is a {@link String}, except {@code datadog.process_tags}: a {@code List}. + * + *

{@code globalTags} (none when null) are the trace's from its config snapshot, so the trace + * resource follows remote config tag changes, or {@code config.getGlobalTags()} for the others. */ static void visitResourceAttributes( - Config config, Map extraAttributes, BiConsumer visitor) { + Config config, + Map globalTags, + Map extraAttributes, + BiConsumer visitor) { String serviceName = config.getServiceName(); String env = config.getEnv(); String version = config.getVersion(); @@ -67,16 +73,16 @@ static void visitResourceAttributes( visitor.accept("telemetry.sdk.version", TRACER_VERSION); visitor.accept("telemetry.sdk.language", "java"); - config - .getGlobalTags() - .forEach( - (key, value) -> { - // ignore global tags replaced by canonical or extra resource attributes - if (!IGNORED_GLOBAL_TAGS.contains(key.toLowerCase(Locale.ROOT)) - && !extraAttributes.containsKey(key)) { - visitor.accept(key, value); - } - }); + if (globalTags != null) { + globalTags.forEach( + (key, value) -> { + // ignore global tags replaced by canonical or extra resource attributes + if (!IGNORED_GLOBAL_TAGS.contains(key.toLowerCase(Locale.ROOT)) + && !extraAttributes.containsKey(key)) { + visitor.accept(key, String.valueOf(value)); + } + }); + } extraAttributes.forEach(visitor); } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceJson.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceJson.java index f6b8cfc6227..0716f83c44b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceJson.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceJson.java @@ -28,21 +28,28 @@ private OtlpResourceJson() {} public static final String RESOURCE_FRAGMENT_WITH_DATADOG_ATTRS = buildResourceFragment(Config.get(), datadogResourceAttributes(Config.get())); - /** - * Resource used by the OTLP trace export. Identical to {@link #RESOURCE_FRAGMENT} but adds the - * {@code _dd.stats_computed} marker when the SDK is computing OTLP span metrics, so a downstream - * Agent does not recompute them from the exported spans. - */ - public static final String TRACE_RESOURCE_FRAGMENT = - buildResourceFragment(Config.get(), traceResourceAttributes(Config.get())); + private static final Map TRACE_RESOURCE_ATTRIBUTES = + traceResourceAttributes(Config.get()); + + public static String buildTraceResourceFragment(Map globalTags) { + return buildResourceFragment(Config.get(), globalTags, TRACE_RESOURCE_ATTRIBUTES); + } static String buildResourceFragment(Config config, Map extraAttributes) { + return buildResourceFragment(config, config.getGlobalTags(), extraAttributes); + } + + static String buildResourceFragment( + Config config, Map globalTags, Map extraAttributes) { try (JsonWriter writer = new JsonWriter()) { writer.beginObject(); writer.name("attributes").beginArray(); visitResourceAttributes( - config, extraAttributes, (key, value) -> writeResourceAttribute(writer, key, value)); + config, + globalTags, + extraAttributes, + (key, value) -> writeResourceAttribute(writer, key, value)); writer.endArray(); writer.endObject(); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceProto.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceProto.java index 3a2d4c0c460..259ff414e6c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceProto.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceProto.java @@ -31,19 +31,32 @@ private OtlpResourceProto() {} public static final byte[] RESOURCE_MESSAGE_WITH_DATADOG_ATTRS = buildResourceMessage(Config.get(), datadogResourceAttributes(Config.get())); + private static final Map TRACE_RESOURCE_ATTRIBUTES = + traceResourceAttributes(Config.get()); + /** - * Resource used by the OTLP trace export. Identical to {@link #RESOURCE_MESSAGE} but adds the - * {@code _dd.stats_computed} marker when the SDK is computing OTLP span metrics, so a downstream - * Agent does not recompute them from the exported spans. + * Builds the resource used by the OTLP trace export, carrying the trace's {@code globalTags}. + * Like {@link #RESOURCE_MESSAGE} but adds the {@code _dd.stats_computed} marker when the SDK is + * computing OTLP span metrics, so a downstream Agent does not recompute them from the exported + * spans. */ - public static final byte[] TRACE_RESOURCE_MESSAGE = - buildResourceMessage(Config.get(), traceResourceAttributes(Config.get())); + public static byte[] buildTraceResourceMessage(Map globalTags) { + return buildResourceMessage(Config.get(), globalTags, TRACE_RESOURCE_ATTRIBUTES); + } static byte[] buildResourceMessage(Config config, Map extraAttributes) { + return buildResourceMessage(config, config.getGlobalTags(), extraAttributes); + } + + static byte[] buildResourceMessage( + Config config, Map globalTags, Map extraAttributes) { GrowableBuffer buf = new GrowableBuffer(512); visitResourceAttributes( - config, extraAttributes, (key, value) -> writeResourceAttribute(buf, key, value)); + config, + globalTags, + extraAttributes, + (key, value) -> writeResourceAttribute(buf, key, value)); OtlpProtoBuffer protobuf = new OtlpProtoBuffer(buf.capacity()); int numBytes = protobuf.recordMessage(buf, 1); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpSpanTags.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpSpanTags.java new file mode 100644 index 00000000000..624b95b96bb --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpSpanTags.java @@ -0,0 +1,67 @@ +package datadog.trace.core.otlp.trace; + +import datadog.trace.api.TagMap; +import datadog.trace.api.function.TriConsumer; +import datadog.trace.core.Metadata; +import java.util.ArrayList; +import java.util.List; + +/** + * Visits the tags an OTLP span carries as attributes: the span's direct tags, plus the span + * defaults ({@code DD_TRACE_SPAN_TAGS}, {@code env}, ...) it doesn't set itself. The global tags + * are left to the resource, which carries them once per payload. + * + *

A span default is only written while the span still carries it: one the span removed, or that + * interception consumed into a span field (such as {@code resource.name}), is skipped. When the + * tracer tags need interception they're copied into each span, so global tags then also appear on + * the span, as before. + * + *

Not thread-safe: each writer owns one. + */ +final class OtlpSpanTags { + private final T target; + private final TriConsumer tagWriter; + + // per-span state + private Integer direction; + private TagMap spanTags; + private TagMap spanDefaultTags; + private final List overriddenDefaults = new ArrayList<>(4); + + OtlpSpanTags(T target, TriConsumer tagWriter) { + this.target = target; + this.tagWriter = tagWriter; + } + + /** Writes the span's attribute tags, naming them for the span's {@code direction}. */ + void write(Metadata metadata, Integer direction) { + this.direction = direction; + this.spanTags = metadata.getTags(); + this.spanDefaultTags = metadata.getSpanDefaultTags(); + try { + spanTags.forEachLocal(this, OtlpSpanTags::writeDirectTag); + if (!spanDefaultTags.isEmpty()) { + spanDefaultTags.forEach(this, OtlpSpanTags::writeDefaultTag); + } + } finally { + this.spanTags = null; + this.spanDefaultTags = null; + overriddenDefaults.clear(); + } + } + + private void writeDirectTag(TagMap.EntryReader entry) { + if (!spanDefaultTags.isEmpty() && spanDefaultTags.containsKey(entry.tag())) { + overriddenDefaults.add(entry.tag()); + } + tagWriter.accept(target, direction, entry); + } + + private void writeDefaultTag(TagMap.EntryReader entry) { + // skip defaults the span set itself, or no longer carries (removed, or consumed by + // interception) + if (!overriddenDefaults.contains(entry.tag()) && spanTags.getEntry(entry.tag()) != null) { + tagWriter.accept(target, direction, entry); + } + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceCollector.java index 127e27af467..83a128d31de 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceCollector.java @@ -2,13 +2,18 @@ import static datadog.trace.core.DDSpanContext.SPAN_SAMPLING_MECHANISM_TAG; +import datadog.trace.api.TagMap; import datadog.trace.core.CoreSpan; +import datadog.trace.core.DDSpan; import datadog.trace.core.otlp.common.OtlpPayload; import java.util.List; /** Collects traces ready for export. */ public abstract class OtlpTraceCollector { + /** The global tags of the payload being collected; null until its first trace. */ + protected TagMap payloadGlobalTags; + /** Adds spans from the given trace to the collector. */ public abstract void addTrace(List> spans); @@ -18,8 +23,24 @@ public abstract class OtlpTraceCollector { /** Returns the number of bytes buffered since the last collection. */ public abstract int sizeInBytes(); + /** + * Whether the given trace can join the payload being collected. A payload carries the global tags + * once, on its resource, so a trace with different global tags (after a remote config tag change) + * needs a payload of its own. + */ + public final boolean sharesResource(List> spans) { + TagMap globalTags = globalTagsOf(spans); + return payloadGlobalTags == null || globalTags == null || globalTags == payloadGlobalTags; + } + protected final boolean shouldExport(CoreSpan span) { return span.samplingPriority() > 0 // trace-level sampling priority || span.getTag(SPAN_SAMPLING_MECHANISM_TAG) != null; // span-level sampling priority } + + /** The trace's global tags, shared by all its spans; null when they can't be read. */ + protected static TagMap globalTagsOf(List> spans) { + CoreSpan first = spans.isEmpty() ? null : spans.get(0); + return first instanceof DDSpan ? ((DDSpan) first).spanContext().getGlobalTags() : null; + } } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java index 68c80100d61..18d18c01b4f 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java @@ -186,12 +186,14 @@ private static void writeSpanTag(JsonWriter writer, UTF8BytesString key, long va public static class MetaWriter implements MetadataConsumer { private final JsonWriter writer; + private final OtlpSpanTags spanTags; private boolean includeProcessTags; private boolean includeSamplingTags; public MetaWriter(JsonWriter writer) { this.writer = writer; + this.spanTags = new OtlpSpanTags<>(writer, OtlpTraceJson::writeSpanTag); } /** Call this to ensure process tags are written out for the next span. */ @@ -258,7 +260,8 @@ public void accept(Metadata metadata) { writeAttribute(writer, STRING_ATTRIBUTE, entry.getKey(), entry.getValue()); } - tags.forEach(writer, direction, OtlpTraceJson::writeSpanTag); + // global tags are left to the resource + spanTags.write(metadata, direction); // reset for next span includeProcessTags = false; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollector.java index cbb39ed615b..8ddf62cc472 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJsonCollector.java @@ -3,10 +3,11 @@ import static datadog.trace.core.otlp.common.OtlpCommonJson.writeScopeAndSchema; import static datadog.trace.core.otlp.common.OtlpPayload.JSON_CONTENT_TYPE; import static datadog.trace.core.otlp.common.OtlpProtoBuffer.MAX_CAPACITY_BYTES; -import static datadog.trace.core.otlp.common.OtlpResourceJson.TRACE_RESOURCE_FRAGMENT; +import static datadog.trace.core.otlp.common.OtlpResourceJson.buildTraceResourceFragment; import static datadog.trace.core.otlp.trace.OtlpTraceJson.writeSpan; import datadog.json.JsonWriter; +import datadog.trace.api.TagMap; import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; import datadog.trace.core.CoreSpan; @@ -47,10 +48,15 @@ public final class OtlpTraceJsonCollector extends OtlpTraceCollector { private DDSpan currentSpan; private List currentSpanLinks = Collections.emptyList(); + // resource for the last global tags seen; they rarely change, so it's rarely rebuilt + private TagMap resourceGlobalTags; + private String resourceFragment; + /** Adds the given trace spans to the collector. */ @Override public void addTrace(List> spans) { if (!payloadStarted) { + payloadGlobalTags = globalTagsOf(spans); start(); payloadStarted = true; } @@ -96,7 +102,7 @@ private void start() { writer.beginObject(); writer.name("resourceSpans").beginArray(); writer.beginObject(); - writer.name("resource").jsonValue(TRACE_RESOURCE_FRAGMENT); + writer.name("resource").jsonValue(resourceFragment()); writer.name("scopeSpans").beginArray(); // for now put all spans under the default scope @@ -106,6 +112,7 @@ private void start() { /** Cleanup elements used to collect trace data. */ private void stop() { payloadStarted = false; + payloadGlobalTags = null; anySpanWritten = false; writer = null; @@ -116,6 +123,15 @@ private void stop() { currentSpanLinks = Collections.emptyList(); } + private String resourceFragment() { + // Update resourceGlobalTags when RC updates the DD_TAGS value (reflected by payloadGlobalTags) + if (resourceFragment == null || resourceGlobalTags != payloadGlobalTags) { + resourceFragment = buildTraceResourceFragment(payloadGlobalTags); + resourceGlobalTags = payloadGlobalTags; + } + return resourceFragment; + } + private void visitScopedSpans(OtelInstrumentationScope scope) { if (currentScope != null) { completeScope(); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java index 865411dbe03..fa8c6b7a548 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java @@ -264,12 +264,14 @@ private static void writeSpanTag(StreamingBuffer buf, UTF8BytesString key, long public static class MetaWriter implements MetadataConsumer { private final StreamingBuffer buf; + private final OtlpSpanTags spanTags; private boolean includeProcessTags; private boolean includeSamplingTags; public MetaWriter(StreamingBuffer buf) { this.buf = buf; + this.spanTags = new OtlpSpanTags<>(buf, OtlpTraceProto::writeSpanTag); } /** Call this to ensure process tags are written out for the next span. */ @@ -337,7 +339,8 @@ public void accept(Metadata metadata) { writeAttribute(buf, STRING_ATTRIBUTE, entry.getKey(), entry.getValue()); } - tags.forEach(buf, direction, OtlpTraceProto::writeSpanTag); + // global tags are left to the resource + spanTags.write(metadata, direction); // reset for next span includeProcessTags = false; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProtoCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProtoCollector.java index ba51d9f9f3d..0c5b70ce363 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProtoCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProtoCollector.java @@ -1,11 +1,12 @@ package datadog.trace.core.otlp.trace; -import static datadog.trace.core.otlp.common.OtlpResourceProto.TRACE_RESOURCE_MESSAGE; +import static datadog.trace.core.otlp.common.OtlpResourceProto.buildTraceResourceMessage; import static datadog.trace.core.otlp.trace.OtlpTraceProto.recordScopedSpansMessage; import static datadog.trace.core.otlp.trace.OtlpTraceProto.recordSpanLinkMessage; import static datadog.trace.core.otlp.trace.OtlpTraceProto.recordSpanMessage; import datadog.communication.serialization.GrowableBuffer; +import datadog.trace.api.TagMap; import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope; import datadog.trace.core.CoreSpan; @@ -27,7 +28,7 @@ * span message and its chunked span-links to the scoped chunks. Once the scope is complete we add * the first part of the scoped spans message and all its chunks (span messages and any span-links) * to the payload. Once all the span data has been chunked we add the enclosing resource span - * message to the start of the payload. + * message, carrying the payload's global tags, to the start of the payload. */ public final class OtlpTraceProtoCollector extends OtlpTraceCollector { @@ -48,10 +49,15 @@ public final class OtlpTraceProtoCollector extends OtlpTraceCollector { private OtelInstrumentationScope currentScope; private DDSpan currentSpan; + // resource for the last global tags seen; they rarely change, so it's rarely rebuilt + private TagMap resourceGlobalTags; + private byte[] resourceMessage; + /** Adds the given trace spans to the collector. */ @Override public void addTrace(List> spans) { if (!payloadStarted) { + payloadGlobalTags = globalTagsOf(spans); start(); payloadStarted = true; } @@ -99,6 +105,7 @@ private void start() { /** Cleanup elements used to collect trace data. */ private void stop() { payloadStarted = false; + payloadGlobalTags = null; buf.reset(); protobuf.reset(); @@ -148,14 +155,22 @@ private OtlpPayload completePayload() { return OtlpPayload.EMPTY; } - // prepend the canned resource chunk - payloadBytes += protobuf.recordMessage(TRACE_RESOURCE_MESSAGE); + // prepend the resource chunk, which carries the payload's global tags + payloadBytes += protobuf.recordMessage(resourceMessage()); // finally prepend the total length of all collected chunks protobuf.recordMessage(buf, 1, payloadBytes); return protobuf.toPayload(); } + private byte[] resourceMessage() { + if (resourceMessage == null || resourceGlobalTags != payloadGlobalTags) { + resourceMessage = buildTraceResourceMessage(payloadGlobalTags); + resourceGlobalTags = payloadGlobalTags; + } + return resourceMessage; + } + // called once we've processed all spans in a specific scope private void completeScope() { if (currentSpan != null) { diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpResourceJsonTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpResourceJsonTest.java index 102027cf668..7626f0357a3 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpResourceJsonTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpResourceJsonTest.java @@ -11,6 +11,7 @@ import static datadog.trace.api.config.TracerConfig.TRACE_REPORT_HOSTNAME; import static datadog.trace.core.otlp.common.OtlpResourceAttributes.datadogResourceAttributes; import static datadog.trace.core.otlp.common.OtlpResourceAttributes.traceResourceAttributes; +import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -246,6 +247,7 @@ void statsComputedVariantCarriesMarker() throws IOException { @Test void cannedFragmentsMatchTheirProtoCounterparts() throws IOException { + Map globalTags = singletonMap("team", "checkout"); assertEquals( parseResourceAttributesFromProto(OtlpResourceProto.RESOURCE_MESSAGE), parseResourceAttributes(OtlpResourceJson.RESOURCE_FRAGMENT)); @@ -253,8 +255,8 @@ void cannedFragmentsMatchTheirProtoCounterparts() throws IOException { parseResourceAttributesFromProto(OtlpResourceProto.RESOURCE_MESSAGE_WITH_DATADOG_ATTRS), parseResourceAttributes(OtlpResourceJson.RESOURCE_FRAGMENT_WITH_DATADOG_ATTRS)); assertEquals( - parseResourceAttributesFromProto(OtlpResourceProto.TRACE_RESOURCE_MESSAGE), - parseResourceAttributes(OtlpResourceJson.TRACE_RESOURCE_FRAGMENT)); + parseResourceAttributesFromProto(OtlpResourceProto.buildTraceResourceMessage(globalTags)), + parseResourceAttributes(OtlpResourceJson.buildTraceResourceFragment(globalTags))); } // ── parsing helpers ─────────────────────────────────────────────────────── diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpSpanTagsTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpSpanTagsTest.java new file mode 100644 index 00000000000..f5905f4e1c1 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/trace/OtlpSpanTagsTest.java @@ -0,0 +1,167 @@ +package datadog.trace.core.otlp.trace; + +import static java.util.Arrays.asList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.json.JsonMapper; +import datadog.trace.api.config.GeneralConfig; +import datadog.trace.api.config.TracerConfig; +import datadog.trace.common.writer.ListWriter; +import datadog.trace.core.CoreSpan; +import datadog.trace.core.CoreTracer; +import datadog.trace.core.DDSpan; +import datadog.trace.core.otlp.common.OtlpPayload; +import datadog.trace.test.junit.utils.config.WithConfig; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * End-to-end check that the global tags are exported once, as OTLP resource attributes taken from + * the trace's config snapshot, while span defaults and the span's own tags stay on each span. + */ +@WithConfig(key = GeneralConfig.TAGS, value = "team:core,region:us1") +@WithConfig(key = TracerConfig.SPAN_TAGS, value = "owner:alice,tier:gold") +class OtlpSpanTagsTest { + + private CoreTracer tracer; + + @BeforeEach + void setUp() { + tracer = CoreTracer.builder().writer(new ListWriter()).build(); + } + + @AfterEach + void tearDown() { + tracer.close(); + } + + @Test + void globalTagsOnlyOnResource() throws IOException { + DDSpan span = (DDSpan) tracer.startSpan("test", "op"); + span.setTag("http.method", "GET"); + span.setTag("region", "eu1"); // a span's own value for a global tag is kept + span.setTag("tier", "silver"); // a span's own value for a span default replaces it + span.finish(); + + Map resourceSpans = collect(span); + + Map resourceAttributes = attributes(resource(resourceSpans)); + assertEquals("core", resourceAttributes.get("team")); + assertEquals("us1", resourceAttributes.get("region")); + + Map spanAttributes = attributes(onlySpan(resourceSpans)); + assertFalse(spanAttributes.containsKey("team")); + assertEquals("eu1", spanAttributes.get("region")); + assertEquals("alice", spanAttributes.get("owner")); + assertEquals("silver", spanAttributes.get("tier")); + assertEquals("GET", spanAttributes.get("http.method")); + } + + @Test + void removedSpanDefaultIsNotWritten() throws IOException { + DDSpan span = (DDSpan) tracer.startSpan("test", "op"); + span.setTag("owner", (String) null); // removes the span default + span.finish(); + + Map spanAttributes = attributes(onlySpan(collect(span))); + assertFalse(spanAttributes.containsKey("owner")); + assertEquals("gold", spanAttributes.get("tier")); + } + + /** A span default that interception consumes into a span field isn't written again as a tag. */ + @Test + @WithConfig(key = TracerConfig.SPAN_TAGS, value = "owner:alice,resource.name:configured") + void consumedSpanDefaultIsNotWrittenTwice() throws IOException { + CoreTracer interceptingTracer = CoreTracer.builder().writer(new ListWriter()).build(); + try { + DDSpan span = (DDSpan) interceptingTracer.startSpan("test", "op"); + span.finish(); + + // attributes() fails on a duplicate resource.name + Map spanAttributes = attributes(onlySpan(collect(span))); + assertEquals("configured", spanAttributes.get("resource.name")); + assertEquals("alice", spanAttributes.get("owner")); + } finally { + interceptingTracer.close(); + } + } + + @Test + void traceWithOtherGlobalTagsNeedsItsOwnPayload() { + CoreTracer otherTracer = CoreTracer.builder().writer(new ListWriter()).build(); + try { + List> trace = finishedTrace(tracer); + List> sameTagsTrace = finishedTrace(tracer); + // another tracer's snapshot holds its own global tags map + List> otherTagsTrace = finishedTrace(otherTracer); + + OtlpTraceJsonCollector collector = new OtlpTraceJsonCollector(); + assertTrue(collector.sharesResource(trace)); // empty payload takes any trace + collector.addTrace(trace); + assertTrue(collector.sharesResource(sameTagsTrace)); + assertFalse(collector.sharesResource(otherTagsTrace)); + + collector.collectTraces(); + assertTrue(collector.sharesResource(otherTagsTrace)); // a new payload takes it + } finally { + otherTracer.close(); + } + } + + private static Map collect(DDSpan span) throws IOException { + OtlpTraceJsonCollector collector = new OtlpTraceJsonCollector(); + collector.addTrace(asList((CoreSpan) span)); + return onlyResourceSpans(collector.collectTraces()); + } + + private static List> finishedTrace(CoreTracer tracer) { + DDSpan span = (DDSpan) tracer.startSpan("test", "op"); + span.finish(); + return asList((CoreSpan) span); + } + + @SuppressWarnings("unchecked") + private static Map onlyResourceSpans(OtlpPayload payload) throws IOException { + byte[] bytes = new byte[payload.getContentLength()]; + payload.getContent().get(bytes); + Map root = JsonMapper.fromJsonToMap(new String(bytes, StandardCharsets.UTF_8)); + List resourceSpans = (List) root.get("resourceSpans"); + assertEquals(1, resourceSpans.size()); + return (Map) resourceSpans.get(0); + } + + @SuppressWarnings("unchecked") + private static Map resource(Map resourceSpans) { + return (Map) resourceSpans.get("resource"); + } + + @SuppressWarnings("unchecked") + private static Map onlySpan(Map resourceSpans) { + List scopeSpans = (List) resourceSpans.get("scopeSpans"); + List spans = (List) ((Map) scopeSpans.get(0)).get("spans"); + assertEquals(1, spans.size()); + return (Map) spans.get(0); + } + + /** Flattens OTLP JSON {@code KeyValue} attributes, failing on duplicate keys. */ + @SuppressWarnings("unchecked") + private static Map attributes(Map owner) { + Map result = new HashMap<>(); + for (Object attribute : (List) owner.get("attributes")) { + Map keyValue = (Map) attribute; + Map value = (Map) keyValue.get("value"); + Object previous = result.put((String) keyValue.get("key"), value.values().iterator().next()); + assertNull(previous, "duplicate attribute " + keyValue.get("key")); + } + return result; + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/TagMap.java b/internal-api/src/main/java/datadog/trace/api/TagMap.java index 296d52399a6..ecaf191de88 100644 --- a/internal-api/src/main/java/datadog/trace/api/TagMap.java +++ b/internal-api/src/main/java/datadog/trace/api/TagMap.java @@ -2292,6 +2292,28 @@ public void forEach(T thisObj, BiConsumer con } } + /** + * Visits only this map's direct entries, skipping everything read through from a parent. For a + * serializer that sources the shared tags from elsewhere, such as an OTLP resource. + */ + public void forEachLocal(T thisObj, BiConsumer consumer) { + Object[] thisBuckets = this.buckets; + + for (int i = 0; i < thisBuckets.length; ++i) { + Object thisBucket = thisBuckets[i]; + + if (thisBucket instanceof Entry) { + Entry thisEntry = (Entry) thisBucket; + + consumer.accept(thisObj, thisEntry); + } else if (thisBucket instanceof BucketGroup) { + BucketGroup thisGroup = (BucketGroup) thisBucket; + + thisGroup.forEachInChain(thisObj, consumer); + } + } + } + private void forEachParent(T thisObj, BiConsumer consumer) { for (TagMap ancestor = this.parent; ancestor != null; ancestor = ancestor.parent) { Object[] parentBuckets = ancestor.buckets; diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapReadThroughTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapReadThroughTest.java index 5001994e2b0..34af93e9c9b 100644 --- a/internal-api/src/test/java/datadog/trace/api/TagMapReadThroughTest.java +++ b/internal-api/src/test/java/datadog/trace/api/TagMapReadThroughTest.java @@ -727,4 +727,18 @@ void fillMapWithoutParentFillsOnlyLocalEntries() { assertEquals("x-val", out.get("x")); assertEquals("y-val", out.get("y")); } + + @Test + void forEachLocalSkipsParentEntries() { + TagMap child = TagMap.createFromParent(frozenParent()); + child.set("b", "child-b"); // shadows parent "b" + child.set("c", "child-c"); + + Map out = new HashMap<>(); + child.forEachLocal(out, (map, entry) -> map.put(entry.tag(), entry.objectValue())); + + assertEquals(2, out.size()); + assertEquals("child-b", out.get("b")); + assertEquals("child-c", out.get("c")); + } } From f547078ae9ddc08ac92c41d61fe8aad1f2b9cb5d Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Tue, 27 Oct 2026 16:49:59 -0400 Subject: [PATCH 2/3] Fall back to startup global tags for unreadable traces A trace whose config snapshot can't be read (not a DDSpan) now resolves to the startup global tags instead of null, so its payload's resource keeps Config's global tags as a floor and a trace with different tags can't join that payload and lose its own. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../core/otlp/trace/OtlpTraceCollector.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceCollector.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceCollector.java index 83a128d31de..083b572641b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceCollector.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceCollector.java @@ -2,6 +2,7 @@ import static datadog.trace.core.DDSpanContext.SPAN_SAMPLING_MECHANISM_TAG; +import datadog.trace.api.Config; import datadog.trace.api.TagMap; import datadog.trace.core.CoreSpan; import datadog.trace.core.DDSpan; @@ -11,6 +12,10 @@ /** Collects traces ready for export. */ public abstract class OtlpTraceCollector { + /** The startup global tags, for a trace whose config snapshot can't be read. */ + private static final TagMap CONFIG_GLOBAL_TAGS = + TagMap.fromMapImmutable(Config.get().getGlobalTags()); + /** The global tags of the payload being collected; null until its first trace. */ protected TagMap payloadGlobalTags; @@ -29,8 +34,7 @@ public abstract class OtlpTraceCollector { * needs a payload of its own. */ public final boolean sharesResource(List> spans) { - TagMap globalTags = globalTagsOf(spans); - return payloadGlobalTags == null || globalTags == null || globalTags == payloadGlobalTags; + return payloadGlobalTags == null || globalTagsOf(spans) == payloadGlobalTags; } protected final boolean shouldExport(CoreSpan span) { @@ -38,9 +42,14 @@ protected final boolean shouldExport(CoreSpan span) { || span.getTag(SPAN_SAMPLING_MECHANISM_TAG) != null; // span-level sampling priority } - /** The trace's global tags, shared by all its spans; null when they can't be read. */ + /** + * The trace's global tags, shared by all its spans; the startup global tags when its config + * snapshot can't be read. + */ protected static TagMap globalTagsOf(List> spans) { CoreSpan first = spans.isEmpty() ? null : spans.get(0); - return first instanceof DDSpan ? ((DDSpan) first).spanContext().getGlobalTags() : null; + return first instanceof DDSpan + ? ((DDSpan) first).spanContext().getGlobalTags() + : CONFIG_GLOBAL_TAGS; } } From eb350ade5079eef38a38418ae49ca69a8c31f31d Mon Sep 17 00:00:00 2001 From: Matthew Li Date: Wed, 28 Oct 2026 09:56:09 -0400 Subject: [PATCH 3/3] Name OTLP resource global tags by the configured semantics The trace's global tags come from its config snapshot as a TagMap, which stores known tags under their Datadog name, so the resource emitted that name even under OTel semantics (db.system came out as db.type). Name them like span attributes instead: the OpenTelemetry name under OTel semantics, else the Datadog name. Other exports keep Config's keys as configured. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../otlp/common/OtlpResourceAttributes.java | 37 ++++++++++++++----- .../otlp/common/OtlpResourceJsonTest.java | 37 +++++++++++++++++++ 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceAttributes.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceAttributes.java index c339f271bc7..9b8f5518f1d 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceAttributes.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/common/OtlpResourceAttributes.java @@ -6,6 +6,7 @@ import datadog.trace.api.Config; import datadog.trace.api.ProcessTags; +import datadog.trace.api.TagMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -45,7 +46,9 @@ private OtlpResourceAttributes() {} * {@code value} is a {@link String}, except {@code datadog.process_tags}: a {@code List}. * *

{@code globalTags} (none when null) are the trace's from its config snapshot, so the trace - * resource follows remote config tag changes, or {@code config.getGlobalTags()} for the others. + * resource follows remote config tag changes, or {@code config.getGlobalTags()} for the others. A + * {@link TagMap} holds known tags under their Datadog name, so its tags are named like span + * attributes: by their OpenTelemetry name under OTel semantics, else their Datadog name. */ static void visitResourceAttributes( Config config, @@ -73,20 +76,36 @@ static void visitResourceAttributes( visitor.accept("telemetry.sdk.version", TRACER_VERSION); visitor.accept("telemetry.sdk.language", "java"); - if (globalTags != null) { + if (globalTags instanceof TagMap) { + boolean otelSemantics = config.isTraceOtelSemanticsEnabled(); + ((TagMap) globalTags) + .forEach( + entry -> + visitGlobalTag( + otelSemantics ? entry.openTelemetryTag() : entry.tag(), + entry.stringValue(), + extraAttributes, + visitor)); + } else if (globalTags != null) { globalTags.forEach( - (key, value) -> { - // ignore global tags replaced by canonical or extra resource attributes - if (!IGNORED_GLOBAL_TAGS.contains(key.toLowerCase(Locale.ROOT)) - && !extraAttributes.containsKey(key)) { - visitor.accept(key, String.valueOf(value)); - } - }); + (key, value) -> visitGlobalTag(key, String.valueOf(value), extraAttributes, visitor)); } extraAttributes.forEach(visitor); } + private static void visitGlobalTag( + String key, + String value, + Map extraAttributes, + BiConsumer visitor) { + // ignore global tags replaced by canonical or extra resource attributes + if (!IGNORED_GLOBAL_TAGS.contains(key.toLowerCase(Locale.ROOT)) + && !extraAttributes.containsKey(key)) { + visitor.accept(key, value); + } + } + private static final String PROCESS_TAGS_KEY = DATADOG_PREFIX + "process_tags"; /** diff --git a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpResourceJsonTest.java b/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpResourceJsonTest.java index 7626f0357a3..dd2a805d858 100644 --- a/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpResourceJsonTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/core/otlp/common/OtlpResourceJsonTest.java @@ -14,11 +14,13 @@ import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.json.JsonMapper; import datadog.trace.api.Config; import datadog.trace.api.ProcessTags; +import datadog.trace.api.TagMap; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -49,6 +51,14 @@ private static Properties props(String... keyValues) { return props; } + private static Map buildMap(String... keyValues) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < keyValues.length; i += 2) { + map.put(keyValues[i], keyValues[i + 1]); + } + return map; + } + private static Map attrs(String... keyValues) { Map map = new LinkedHashMap<>(); for (int i = 0; i < keyValues.length; i += 2) { @@ -245,6 +255,33 @@ void statsComputedVariantCarriesMarker() throws IOException { assertEquals("true", without.get("_dd.sdk.otlp_export")); } + /** Snapshot global tags are named like span attributes, by the configured semantics. */ + @Test + void snapshotGlobalTagsFollowConfiguredSemantics() throws IOException { + // a TagMap stores either spelling under the Datadog name + TagMap globalTags = + TagMap.fromMapImmutable(buildMap("http.request.method", "GET", "team", "core")); + + Map otel = + parseResourceAttributes( + OtlpResourceJson.buildResourceFragment( + Config.get(props(TRACE_OTEL_SEMANTICS_ENABLED, "true")), + globalTags, + Collections.emptyMap())); + assertEquals("GET", otel.get("http.request.method")); + assertEquals("core", otel.get("team")); + + Map datadog = + parseResourceAttributes( + OtlpResourceJson.buildResourceFragment( + Config.get(props(TRACE_OTEL_SEMANTICS_ENABLED, "false")), + globalTags, + Collections.emptyMap())); + assertEquals("GET", datadog.get("http.method")); + assertNull(datadog.get("http.request.method")); + assertEquals("core", datadog.get("team")); + } + @Test void cannedFragmentsMatchTheirProtoCounterparts() throws IOException { Map globalTags = singletonMap("team", "checkout");