From fd48fac2cd61c410795627b785f356b5b52414bb Mon Sep 17 00:00:00 2001 From: Irina Dominte Date: Tue, 2 Jun 2026 16:08:34 +0300 Subject: [PATCH 01/22] Starting working branch for Otel --- .../When_processing_incoming_message.cs | 32 ++++++++++ .../Traces/When_publishing_messages.cs | 63 +++++++++++++++++++ .../Traces/When_sending_messages.cs | 32 ++++++++++ .../Traces/When_sending_replies.cs | 34 ++++++++++ ...IApprovals.ApproveNServiceBus.approved.txt | 14 +++++ .../OpenTelemetry/ActivityFactoryTests.cs | 4 +- .../Pipeline/MainPipelineExecutorTests.cs | 2 +- .../Pipeline/TestableMessageOperations.cs | 2 +- .../RoutingToDispatchConnectorTests.cs | 20 +++--- .../Hosting/HostingComponent.Configuration.cs | 2 +- .../Hosting/HostingComponent.Settings.cs | 2 + .../OpenTelemetry/InstrumentationOptions.cs | 42 +++++++++++++ .../OpenTelemetry/OpenTelemetryExtensions.cs | 13 ++++ .../OpenTelemetry/OpenTelemetryFeature.cs | 15 +++++ .../PromoteMessagePropertiesToTagsBehavior.cs | 51 +++++++++++++++ .../Tracing/ActivityDisplayNames.cs | 5 ++ .../OpenTelemetry/Tracing/ActivityFactory.cs | 11 +++- .../OpenTelemetry/Tracing/IActivityFactory.cs | 1 + .../Tracing/NoOpActivityFactory.cs | 2 + .../Outgoing/RoutingToDispatchConnector.cs | 18 ++++++ .../Pipeline/Outgoing/SendComponent.cs | 2 +- .../Unicast/MessageOperations.cs | 6 +- 22 files changed, 355 insertions(+), 18 deletions(-) create mode 100644 src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs create mode 100644 src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs index 99fecbaf11c..d386a9906d2 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs @@ -80,5 +80,37 @@ public Task Handle(IncomingMessage message, IMessageHandlerContext context) } } + [Test] + public async Task Should_use_receive_address_in_span_name_when_opted_in() + { + await Scenario.Define() + .WithEndpoint(e => e + .When(s => s.SendLocal(new IncomingMessage()))) + .Run(); + + var incomingMessageActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + Assert.That(incomingMessageActivities, Has.Count.EqualTo(1)); + + var incomingActivity = incomingMessageActivities.Single(); + Assert.That(incomingActivity.DisplayName, Does.StartWith("process ")); + Assert.That(incomingActivity.DisplayName, Is.Not.EqualTo("process message")); + } + + public class ReceivingEndpointWithDestinationNaming : EndpointConfigurationBuilder + { + public ReceivingEndpointWithDestinationNaming() => + EndpointSetup(b => b.Tracing().UseMessageDestinationInSpanNames = true); + + [Handler] + public class MessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(IncomingMessage message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + public class IncomingMessage : IMessage; } \ No newline at end of file diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs index 8c402afb504..0595803fbef 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs @@ -179,5 +179,68 @@ public Task Handle(ThisIsAnEvent @event, IMessageHandlerContext context) } } + [Test] + public async Task Should_use_event_type_in_span_name_when_opted_in() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(ctx => ctx.SomeEventSubscribed, s => s.Publish())) + .WithEndpoint(b => b.When((session, ctx) => + { + if (ctx.HasNativePubSubSupport) + { + ctx.SomeEventSubscribed = true; + } + + return Task.CompletedTask; + })) + .Run(); + + var outgoingEventActivities = NServiceBusActivityListener.CompletedActivities.GetPublishEventActivities(); + Assert.That(outgoingEventActivities, Has.Count.EqualTo(1)); + + var publishedMessage = outgoingEventActivities.Single(); + Assert.That(publishedMessage.DisplayName, Is.EqualTo("publish ThisIsAnEvent")); + } + + class PublisherWithDestinationNaming : EndpointConfigurationBuilder + { + public PublisherWithDestinationNaming() => + EndpointSetup(b => + { + b.Tracing().UseMessageDestinationInSpanNames = true; + b.OnEndpointSubscribed((s, context) => + { + if (s.SubscriberEndpoint.Contains(Conventions.EndpointNamingConvention(typeof(SubscriberForPublisherWithDestinationNaming)))) + { + if (s.MessageType == typeof(ThisIsAnEvent).AssemblyQualifiedName) + { + context.SomeEventSubscribed = true; + } + } + }); + }); + } + + class SubscriberForPublisherWithDestinationNaming : EndpointConfigurationBuilder + { + public SubscriberForPublisherWithDestinationNaming() => + EndpointSetup(c => { }, + metadata => + { + metadata.RegisterPublisherFor(); + }); + + [Handler] + public class ThisHandlesSomethingHandler(Context testContext) : IHandleMessages + { + public Task Handle(ThisIsAnEvent @event, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + public class ThisIsAnEvent : IEvent; } \ No newline at end of file diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs index af0751530d7..e14df1d05e6 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs @@ -131,5 +131,37 @@ public Task Handle(OutgoingMessage message, IMessageHandlerContext context) } } + [Test] + public async Task Should_use_destination_in_send_span_name_when_opted_in() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => s.SendLocal(new OutgoingMessage()))) + .Run(); + + var outgoingMessageActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + Assert.That(outgoingMessageActivities, Has.Count.EqualTo(1)); + + var sentMessage = outgoingMessageActivities.Single(); + Assert.That(sentMessage.DisplayName, Does.StartWith("send ")); + Assert.That(sentMessage.DisplayName, Is.Not.EqualTo("send message")); + } + + public class TestEndpointWithDestinationNaming : EndpointConfigurationBuilder + { + public TestEndpointWithDestinationNaming() => + EndpointSetup(b => b.Tracing().UseMessageDestinationInSpanNames = true); + + [Handler] + public class MessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(OutgoingMessage message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + public class OutgoingMessage : IMessage; } \ No newline at end of file diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs index e6299389c51..c6c505d9236 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs @@ -55,6 +55,40 @@ public Task Handle(OutgoingReply message, IMessageHandlerContext context) } } + [Test] + public async Task Should_use_destination_in_reply_span_name_when_opted_in() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => s.SendLocal(new IncomingMessage()))) + .Run(); + + var outgoingMessageActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + Assert.That(outgoingMessageActivities, Has.Count.EqualTo(2), "2 messages are being sent"); + var replyMessage = outgoingMessageActivities[1]; + + Assert.That(replyMessage.DisplayName, Does.StartWith("reply ")); + } + + public class TestEndpointWithDestinationNaming : EndpointConfigurationBuilder + { + public TestEndpointWithDestinationNaming() => + EndpointSetup(b => b.Tracing().UseMessageDestinationInSpanNames = true); + + [Handler] + public class MessageHandler(Context testContext) : IHandleMessages, + IHandleMessages + { + public Task Handle(IncomingMessage message, IMessageHandlerContext context) => context.Reply(new OutgoingReply()); + + public Task Handle(OutgoingReply message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + public class IncomingMessage : IMessage; public class OutgoingReply : IMessage; diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index acbacb5d5ae..e5b50f8da1b 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -632,6 +632,12 @@ namespace NServiceBus StopApplication = 0, Continue = 1, } + public class InstrumentationOptions + { + public InstrumentationOptions() { } + public NServiceBus.MessagePayloadAsTag MessagePayloadAsTag { get; set; } + public bool UseMessageDestinationInSpanNames { get; set; } + } public sealed class KeyedServiceKey { public const string Any = "______________"; @@ -718,6 +724,13 @@ namespace NServiceBus Unsubscribe = 4, Reply = 5, } + public enum MessagePayloadAsTag + { + None = 0, + IncomingMessage = 1, + OutgoingMessage = 2, + All = 3, + } public static class MessageProcessingContextExtensions { public static System.Threading.Tasks.Task Reply(this NServiceBus.IMessageProcessingContext context, object message) { } @@ -772,6 +785,7 @@ namespace NServiceBus { public static void ContinueExistingTraceOnReceive(this NServiceBus.PublishOptions publishOptions) { } public static void StartNewTraceOnReceive(this NServiceBus.SendOptions sendOptions) { } + public static NServiceBus.InstrumentationOptions Tracing(this NServiceBus.EndpointConfiguration config) { } } public static class OutboxConfigExtensions { diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs index 6c22b89bb7b..500740cc6ba 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs @@ -17,7 +17,7 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; [TestFixture] public class ActivityFactoryTests { - readonly ActivityFactory activityFactory = new(); + readonly ActivityFactory activityFactory = new(new InstrumentationOptions()); TestingActivityListener nsbActivityListener; @@ -29,7 +29,7 @@ public class ActivityFactoryTests class NoDiagnosticListeners { - readonly ActivityFactory activityFactory = new(); + readonly ActivityFactory activityFactory = new(new InstrumentationOptions()); [Test] public void Should_return_null_incoming_activity_when_no_listeners() diff --git a/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs b/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs index f6223a25eac..ee6b76d9e3b 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs @@ -130,7 +130,7 @@ static MainPipelineExecutor CreateMainPipelineExecutor(ServiceProvider servicePr new TestableMessageOperations(), new Notification(), receivePipeline, - new ActivityFactory(), + new ActivityFactory(new InstrumentationOptions()), incomingPipelineMetrics, new EnvelopeUnwrapper([], incomingPipelineMetrics)); diff --git a/src/NServiceBus.Core.Tests/Pipeline/TestableMessageOperations.cs b/src/NServiceBus.Core.Tests/Pipeline/TestableMessageOperations.cs index ddb58d3a66e..909b9fb796a 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/TestableMessageOperations.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/TestableMessageOperations.cs @@ -13,7 +13,7 @@ class TestableMessageOperations : MessageOperations public Pipeline SubscribePipeline => (Pipeline)subscribePipeline; public Pipeline UnsubscribePipeline => (Pipeline)unsubscribePipeline; - public TestableMessageOperations() : base(new MessageMapper(), new Pipeline(), new Pipeline(), new Pipeline(), new Pipeline(), new Pipeline(), new ActivityFactory()) + public TestableMessageOperations() : base(new MessageMapper(), new Pipeline(), new Pipeline(), new Pipeline(), new Pipeline(), new Pipeline(), new ActivityFactory(new InstrumentationOptions())) { } diff --git a/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs b/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs index ad4d34e8037..da97e86e33f 100644 --- a/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs @@ -17,7 +17,7 @@ public class RoutingToDispatchConnectorTests [Test] public async Task Should_preserve_message_state_for_one_routing_strategy_for_allocation_reasons() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); IEnumerable operations = null; var testableRoutingContext = new TestableRoutingContext { @@ -59,7 +59,7 @@ await behavior.Invoke(testableRoutingContext, context => [Test] public async Task Should_copy_message_state_for_multiple_routing_strategies() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); List operations = null; var testableRoutingContext = new TestableRoutingContext { @@ -135,7 +135,7 @@ await behavior.Invoke(testableRoutingContext, context => [Test] public async Task Should_preserve_headers_generated_by_custom_routing_strategy() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); Dictionary headers = null; await behavior.Invoke(new TestableRoutingContext { RoutingStrategies = [new HeaderModifyingRoutingStrategy()] }, context => { @@ -153,7 +153,7 @@ public async Task Should_dispatch_immediately_if_user_requested() options.RequireImmediateDispatch(); var dispatched = false; - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var message = new OutgoingMessage("ID", [], Array.Empty()); await behavior.Invoke(new RoutingContext(message, @@ -170,7 +170,7 @@ await behavior.Invoke(new RoutingContext(message, public async Task Should_dispatch_immediately_if_not_sending_from_a_handler() { var dispatched = false; - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var message = new OutgoingMessage("ID", [], Array.Empty()); await behavior.Invoke(new RoutingContext(message, @@ -187,7 +187,7 @@ await behavior.Invoke(new RoutingContext(message, public async Task Should_not_dispatch_by_default() { var dispatched = false; - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var message = new OutgoingMessage("ID", [], Array.Empty()); await behavior.Invoke(new RoutingContext(message, @@ -203,7 +203,7 @@ await behavior.Invoke(new RoutingContext(message, [Test] public async Task Should_promote_message_headers_to_pipeline_activity() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var routingContext = new TestableRoutingContext(); routingContext.Message.Headers[Headers.ContentType] = "test content type"; // one of the headers that will be mapped to tags @@ -257,7 +257,7 @@ class MyMessage : IMessage; [Test] public async Task Should_merge_receive_properties_when_declared_by_transport() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var receiveProperties = new ReceiveProperties(new Dictionary { @@ -290,7 +290,7 @@ await behavior.Invoke(routingContext, context => [Test] public async Task Should_not_override_user_set_dispatch_property() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var receiveProperties = new ReceiveProperties(new Dictionary { @@ -324,7 +324,7 @@ await behavior.Invoke(routingContext, context => [Test] public async Task Should_preserve_user_dispatch_properties_even_with_receive_properties() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var receiveProperties = new ReceiveProperties(new Dictionary { diff --git a/src/NServiceBus.Core/Hosting/HostingComponent.Configuration.cs b/src/NServiceBus.Core/Hosting/HostingComponent.Configuration.cs index 4ed38527c71..6326b1a5006 100644 --- a/src/NServiceBus.Core/Hosting/HostingComponent.Configuration.cs +++ b/src/NServiceBus.Core/Hosting/HostingComponent.Configuration.cs @@ -26,7 +26,7 @@ public static Configuration PrepareConfiguration(Settings settings, List a serviceCollection, settings.ShouldRunInstallers, settings.UserRegistrations, - new ActivityFactory(), + new ActivityFactory(settings.InstrumentationOptions), persistenceConfiguration, installerComponent); diff --git a/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs b/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs index f46a6101f05..8e85864c2be 100644 --- a/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs +++ b/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs @@ -95,6 +95,8 @@ public bool WriteDiagnosticsToLog get; set; } + public InstrumentationOptions InstrumentationOptions => settings.GetOrDefault() ?? new InstrumentationOptions(); + internal void ConfigureHostLogging(object? endpointIdentifier) { EndpointIdentifier = endpointIdentifier; diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs new file mode 100644 index 00000000000..aeb480769e1 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -0,0 +1,42 @@ +#nullable enable + +namespace NServiceBus; + +/// +/// Controls opt-in OpenTelemetry instrumentation behaviors. +/// Accessed via endpointConfiguration.Tracing(). +/// +public class InstrumentationOptions +{ + /// + /// Appends the destination to span names following the OTel messaging convention + /// {messaging.operation.name} {destination}, e.g. "process orders" or "send payments". + /// Disabled by default for backward compatibility. + /// + public bool UseMessageDestinationInSpanNames { get; set; } + + /// + /// Promotes public properties of message instances to span attributes + /// as nservicebus.message.{PropertyName}. + /// Defaults to . May expose sensitive data and incurs reflection cost. + /// + public MessagePayloadAsTag MessagePayloadAsTag { get; set; } +} + +/// +/// Controls which message payloads are promoted to span attributes. +/// +public enum MessagePayloadAsTag +{ + /// No message properties are promoted to span tags. + None, + + /// Public properties of the incoming message instance are promoted to span tags. + IncomingMessage, + + /// Public properties of outgoing message instances are promoted to span tags. + OutgoingMessage, + + /// Public properties of both incoming and outgoing message instances are promoted to span tags. + All +} diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs index bc263ffe423..9c3ad6e63e8 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs @@ -2,11 +2,24 @@ namespace NServiceBus; +using System; + /// /// Gives users control over the depth of an OpenTelemetry trace. /// public static class OpenTelemetryExtensions { + /// + /// Provides access to instrumentation options for OpenTelemetry tracing. + /// + /// The endpoint configuration. + /// The instance for this endpoint. + public static InstrumentationOptions Tracing(this EndpointConfiguration config) + { + ArgumentNullException.ThrowIfNull(config); + return config.Settings.GetOrCreate(); + } + /// /// Start a new OpenTelemetry trace conversation. /// diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs index c485129eee0..e4804d76a0c 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs @@ -22,5 +22,20 @@ protected override void Setup(FeatureConfigurationContext context) new PopulateRecoverabilityTraceMetadataBehavior(), "Populates the recoverability metadata" ); + + var options = context.Settings.GetOrDefault(); + if (options?.MessagePayloadAsTag is MessagePayloadAsTag.IncomingMessage or MessagePayloadAsTag.All) + { + context.Pipeline.Register( + new IncomingMessagePayloadToTagsBehavior(), + "Promotes incoming message properties to span tags"); + } + + if (options?.MessagePayloadAsTag is MessagePayloadAsTag.OutgoingMessage or MessagePayloadAsTag.All) + { + context.Pipeline.Register( + new OutgoingMessagePayloadToTagsBehavior(), + "Promotes outgoing message properties to span tags"); + } } } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs new file mode 100644 index 00000000000..4435190901a --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs @@ -0,0 +1,51 @@ +#nullable enable + +namespace NServiceBus; + +using System; +using System.Diagnostics; +using System.Reflection; +using System.Threading.Tasks; +using Pipeline; + +class IncomingMessagePayloadToTagsBehavior : IBehavior +{ + public Task Invoke(IIncomingLogicalMessageContext context, Func next) + { + var activity = Activity.Current; + if (activity?.IsAllDataRequested == true) + { + PromoteProperties(activity, context.Message.Instance); + } + + return next(context); + } + + // this needs to be changed to be base64 encoding of the entire body not just the properties + + internal static void PromoteProperties(Activity activity, object instance) + { + foreach (var property in instance.GetType().GetProperties(BindingFlags.Public | BindingFlags.Instance)) + { + var value = property.GetValue(instance); + if (value is not null) + { + activity.SetTag($"nservicebus.message.{property.Name}", value.ToString()); + } + } + } +} + +class OutgoingMessagePayloadToTagsBehavior : IBehavior +{ + public Task Invoke(IOutgoingLogicalMessageContext context, Func next) + { + var activity = Activity.Current; + if (activity?.IsAllDataRequested == true) + { + IncomingMessagePayloadToTagsBehavior.PromoteProperties(activity, context.Message.Instance); + } + + return next(context); + } +} diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs index ca6d81aff76..cf56402f4dd 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs @@ -10,4 +10,9 @@ static class ActivityDisplayNames public const string UnsubscribeEvent = "unsubscribe event"; public const string SendMessage = "send message"; public const string ReplyMessage = "reply"; + + // Operation-only prefixes used when UseMessageDestinationInSpanNames is enabled + internal const string ProcessOperation = "process"; + internal const string PublishOperation = "publish"; + internal const string SendOperation = "send"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index 6c0202a51d7..b6548eef2f0 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -8,6 +8,13 @@ namespace NServiceBus; sealed class ActivityFactory : IActivityFactory { + public ActivityFactory(InstrumentationOptions options) + { + Options = options; + } + + public InstrumentationOptions Options { get; } + public Activity? StartIncomingPipelineActivity(MessageContext context) { // CreateActivity is a no-op if there are no listeners but we are doing a fast path check @@ -66,7 +73,9 @@ sealed class ActivityFactory : IActivityFactory ContextPropagation.PropagateContextFromHeaders(activity, context.Headers); - activity.DisplayName = ActivityDisplayNames.ProcessMessage; + activity.DisplayName = Options.UseMessageDestinationInSpanNames + ? $"{ActivityDisplayNames.ProcessOperation} {context.ReceiveAddress}" + : ActivityDisplayNames.ProcessMessage; activity.SetIdFormat(ActivityIdFormat.W3C); activity.AddTag(ActivityTags.NativeMessageId, context.NativeMessageId); diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs index 8a47acb7fdd..daa6553a2eb 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs @@ -8,6 +8,7 @@ namespace NServiceBus; interface IActivityFactory { + InstrumentationOptions Options { get; } Activity? StartIncomingPipelineActivity(MessageContext context); Activity? StartOutgoingPipelineActivity(string activityName, string displayName, IBehaviorContext outgoingContext); Activity? StartHandlerActivity(MessageHandler messageHandler); diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs index a36d268a3bd..fe1f2f82c12 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs @@ -8,6 +8,8 @@ namespace NServiceBus; sealed class NoOpActivityFactory : IActivityFactory { + public InstrumentationOptions Options { get; } = new InstrumentationOptions(); + public Activity? StartIncomingPipelineActivity(MessageContext context) => null; public Activity? StartOutgoingPipelineActivity(string activityName, string displayName, IBehaviorContext outgoingContext) => null; diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs b/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs index bc4be17db8b..55f62023b4b 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs @@ -14,6 +14,13 @@ namespace NServiceBus; class RoutingToDispatchConnector : StageConnector { + readonly IActivityFactory activityFactory; + + public RoutingToDispatchConnector(IActivityFactory activityFactory) + { + this.activityFactory = activityFactory; + } + public override Task Invoke(IRoutingContext context, Func stage) { var dispatchConsistency = DispatchConsistency.Default; @@ -60,6 +67,17 @@ public override Task Invoke(IRoutingContext context, Func 0 + && operations[0].AddressTag is UnicastAddressTag unicastTag + && outgoingMessage.Headers.TryGetValue(Headers.MessageIntent, out var intentStr) + && intentStr is "Send" or "Reply") + { + activity.DisplayName = $"{activity.DisplayName} {unicastTag.Destination}"; + } } if (dispatchConsistency == DispatchConsistency.Default && context.Extensions.TryGet(out var pendingOperations)) diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/SendComponent.cs b/src/NServiceBus.Core/Pipeline/Outgoing/SendComponent.cs index c00d0880656..5899d352981 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/SendComponent.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/SendComponent.cs @@ -27,7 +27,7 @@ public static SendComponent Initialize(PipelineSettings pipelineSettings, Hostin pipelineSettings.Register(new OutgoingPhysicalToRoutingConnector(), "Starts the message dispatch pipeline"); - pipelineSettings.Register(new RoutingToDispatchConnector(), + pipelineSettings.Register(new RoutingToDispatchConnector(hostingConfiguration.ActivityFactory), "Decides if the current message should be batched or immediately be dispatched to the transport"); pipelineSettings.Register(new BatchToDispatchConnector(), "Passes batched messages over to the immediate dispatch part of the pipeline"); pipelineSettings.Register(b => new ImmediateDispatchTerminator(b.GetRequiredService()), "Hands the outgoing messages over to the transport for immediate delivery"); diff --git a/src/NServiceBus.Core/Unicast/MessageOperations.cs b/src/NServiceBus.Core/Unicast/MessageOperations.cs index 34c60f3cda1..7f3af927a2a 100644 --- a/src/NServiceBus.Core/Unicast/MessageOperations.cs +++ b/src/NServiceBus.Core/Unicast/MessageOperations.cs @@ -65,7 +65,11 @@ async Task Publish(IBehaviorContext context, Type messageType, object message, P MergeDispatchProperties(publishContext, options.DispatchProperties); - using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingEventActivityName, ActivityDisplayNames.PublishEvent, publishContext); + var publishDisplayName = activityFactory.Options.UseMessageDestinationInSpanNames + ? $"{ActivityDisplayNames.PublishOperation} {messageType.Name}" + : ActivityDisplayNames.PublishEvent; + + using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingEventActivityName, publishDisplayName, publishContext); await publishPipeline.Invoke(publishContext, activity).ConfigureAwait(false); } From dd3a9dbe1e2fbb6b997cacc42a9460f95f96a16c Mon Sep 17 00:00:00 2001 From: Irina Dominte Date: Tue, 2 Jun 2026 16:59:57 +0300 Subject: [PATCH 02/22] Added base64 encoded body --- .../MessagePayloadToTagsBehaviorTests.cs | 102 ++++++++++++++++++ ...or.cs => MessagePayloadToTagsBehaviors.cs} | 22 ++-- 2 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs rename src/NServiceBus.Core/OpenTelemetry/{PromoteMessagePropertiesToTagsBehavior.cs => MessagePayloadToTagsBehaviors.cs} (57%) diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs new file mode 100644 index 00000000000..4a2cfd6ce2a --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs @@ -0,0 +1,102 @@ +#nullable enable + +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System; +using System.Collections.Immutable; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Helpers; +using NServiceBus.Pipeline; +using NUnit.Framework; +using Testing; +using Unicast.Messages; + +[TestFixture] +public class MessagePayloadToTagsBehaviorTests +{ + TestingActivityListener activityListener; + + [SetUp] + public void SetUp() => activityListener = TestingActivityListener.SetupNServiceBusDiagnosticListener(); + + [TearDown] + public void TearDown() => activityListener.Dispose(); + + [Test] + public async Task Incoming_should_set_base64_encoded_json_body_tag() + { + using var activity = ActivitySources.Main.StartActivity("test"); + + var message = new TestMessage { Name = "Hello", Value = 42 }; + var context = new TestableIncomingLogicalMessageContext + { + Message = new LogicalMessage(new MessageMetadata(typeof(TestMessage)), message) + }; + + await new IncomingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask); + + var tags = activity!.Tags.ToImmutableDictionary(); + Assert.That(tags.ContainsKey("nservicebus.message.body"), Is.True); + + var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(tags["nservicebus.message.body"]!)); + var deserialized = JsonSerializer.Deserialize(decoded); + + using (Assert.EnterMultipleScope()) + { + Assert.That(deserialized!.Name, Is.EqualTo("Hello")); + Assert.That(deserialized.Value, Is.EqualTo(42)); + } + } + + [Test] + public async Task Incoming_should_not_set_tag_when_no_active_activity() + { + var context = new TestableIncomingLogicalMessageContext + { + Message = new LogicalMessage(new MessageMetadata(typeof(TestMessage)), new TestMessage()) + }; + + Assert.DoesNotThrowAsync(() => new IncomingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask)); + } + + [Test] + public async Task Outgoing_should_set_base64_encoded_json_body_tag() + { + using var activity = ActivitySources.Main.StartActivity("test"); + + var message = new TestMessage { Name = "World", Value = 99 }; + var context = new TestableOutgoingLogicalMessageContext(); + context.UpdateMessage(message); + + await new OutgoingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask); + + var tags = activity!.Tags.ToImmutableDictionary(); + Assert.That(tags.ContainsKey("nservicebus.message.body"), Is.True); + + var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(tags["nservicebus.message.body"]!)); + var deserialized = JsonSerializer.Deserialize(decoded); + + using (Assert.EnterMultipleScope()) + { + Assert.That(deserialized!.Name, Is.EqualTo("World")); + Assert.That(deserialized.Value, Is.EqualTo(99)); + } + } + + [Test] + public async Task Outgoing_should_not_set_tag_when_no_active_activity() + { + var context = new TestableOutgoingLogicalMessageContext(); + context.UpdateMessage(new TestMessage()); + + Assert.DoesNotThrowAsync(() => new OutgoingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask)); + } + + class TestMessage + { + public string? Name { get; set; } + public int Value { get; set; } + } +} diff --git a/src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs similarity index 57% rename from src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs rename to src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs index 4435190901a..7868307458f 100644 --- a/src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs @@ -4,7 +4,8 @@ namespace NServiceBus; using System; using System.Diagnostics; -using System.Reflection; +using System.Text; +using System.Text.Json; using System.Threading.Tasks; using Pipeline; @@ -15,24 +16,17 @@ public Task Invoke(IIncomingLogicalMessageContext context, Func Date: Wed, 3 Jun 2026 11:01:48 +0200 Subject: [PATCH 03/22] Apply suggestion from @ramonsmits --- .../OpenTelemetry/MessagePayloadToTagsBehaviors.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs b/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs index 7868307458f..7f4129c4705 100644 --- a/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs +++ b/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs @@ -35,7 +35,7 @@ class OutgoingMessagePayloadToTagsBehavior : IBehavior next) { var activity = Activity.Current; - if (activity?.IsAllDataRequested == true) + if (activity?.IsAllDataRequested) { IncomingMessagePayloadToTagsBehavior.SetMessageBodyTag(activity, context.Message.Instance); } From b083629851bab4a72c9b622fd2aa53f2f9a068e5 Mon Sep 17 00:00:00 2001 From: Irina Dominte Date: Wed, 10 Jun 2026 14:15:16 +0300 Subject: [PATCH 04/22] removed the body serialization --- ...IApprovals.ApproveNServiceBus.approved.txt | 8 -- .../MessagePayloadToTagsBehaviorTests.cs | 102 ------------------ .../OpenTelemetry/InstrumentationOptions.cs | 25 ----- .../MessagePayloadToTagsBehaviors.cs | 45 -------- .../OpenTelemetry/OpenTelemetryFeature.cs | 15 --- 5 files changed, 195 deletions(-) delete mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs delete mode 100644 src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index e5b50f8da1b..a5bdc1d3893 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -635,7 +635,6 @@ namespace NServiceBus public class InstrumentationOptions { public InstrumentationOptions() { } - public NServiceBus.MessagePayloadAsTag MessagePayloadAsTag { get; set; } public bool UseMessageDestinationInSpanNames { get; set; } } public sealed class KeyedServiceKey @@ -724,13 +723,6 @@ namespace NServiceBus Unsubscribe = 4, Reply = 5, } - public enum MessagePayloadAsTag - { - None = 0, - IncomingMessage = 1, - OutgoingMessage = 2, - All = 3, - } public static class MessageProcessingContextExtensions { public static System.Threading.Tasks.Task Reply(this NServiceBus.IMessageProcessingContext context, object message) { } diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs deleted file mode 100644 index 4a2cfd6ce2a..00000000000 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs +++ /dev/null @@ -1,102 +0,0 @@ -#nullable enable - -namespace NServiceBus.Core.Tests.OpenTelemetry; - -using System; -using System.Collections.Immutable; -using System.Text; -using System.Text.Json; -using System.Threading.Tasks; -using Helpers; -using NServiceBus.Pipeline; -using NUnit.Framework; -using Testing; -using Unicast.Messages; - -[TestFixture] -public class MessagePayloadToTagsBehaviorTests -{ - TestingActivityListener activityListener; - - [SetUp] - public void SetUp() => activityListener = TestingActivityListener.SetupNServiceBusDiagnosticListener(); - - [TearDown] - public void TearDown() => activityListener.Dispose(); - - [Test] - public async Task Incoming_should_set_base64_encoded_json_body_tag() - { - using var activity = ActivitySources.Main.StartActivity("test"); - - var message = new TestMessage { Name = "Hello", Value = 42 }; - var context = new TestableIncomingLogicalMessageContext - { - Message = new LogicalMessage(new MessageMetadata(typeof(TestMessage)), message) - }; - - await new IncomingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask); - - var tags = activity!.Tags.ToImmutableDictionary(); - Assert.That(tags.ContainsKey("nservicebus.message.body"), Is.True); - - var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(tags["nservicebus.message.body"]!)); - var deserialized = JsonSerializer.Deserialize(decoded); - - using (Assert.EnterMultipleScope()) - { - Assert.That(deserialized!.Name, Is.EqualTo("Hello")); - Assert.That(deserialized.Value, Is.EqualTo(42)); - } - } - - [Test] - public async Task Incoming_should_not_set_tag_when_no_active_activity() - { - var context = new TestableIncomingLogicalMessageContext - { - Message = new LogicalMessage(new MessageMetadata(typeof(TestMessage)), new TestMessage()) - }; - - Assert.DoesNotThrowAsync(() => new IncomingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask)); - } - - [Test] - public async Task Outgoing_should_set_base64_encoded_json_body_tag() - { - using var activity = ActivitySources.Main.StartActivity("test"); - - var message = new TestMessage { Name = "World", Value = 99 }; - var context = new TestableOutgoingLogicalMessageContext(); - context.UpdateMessage(message); - - await new OutgoingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask); - - var tags = activity!.Tags.ToImmutableDictionary(); - Assert.That(tags.ContainsKey("nservicebus.message.body"), Is.True); - - var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(tags["nservicebus.message.body"]!)); - var deserialized = JsonSerializer.Deserialize(decoded); - - using (Assert.EnterMultipleScope()) - { - Assert.That(deserialized!.Name, Is.EqualTo("World")); - Assert.That(deserialized.Value, Is.EqualTo(99)); - } - } - - [Test] - public async Task Outgoing_should_not_set_tag_when_no_active_activity() - { - var context = new TestableOutgoingLogicalMessageContext(); - context.UpdateMessage(new TestMessage()); - - Assert.DoesNotThrowAsync(() => new OutgoingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask)); - } - - class TestMessage - { - public string? Name { get; set; } - public int Value { get; set; } - } -} diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index aeb480769e1..f920a28b93a 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -14,29 +14,4 @@ public class InstrumentationOptions /// Disabled by default for backward compatibility. /// public bool UseMessageDestinationInSpanNames { get; set; } - - /// - /// Promotes public properties of message instances to span attributes - /// as nservicebus.message.{PropertyName}. - /// Defaults to . May expose sensitive data and incurs reflection cost. - /// - public MessagePayloadAsTag MessagePayloadAsTag { get; set; } -} - -/// -/// Controls which message payloads are promoted to span attributes. -/// -public enum MessagePayloadAsTag -{ - /// No message properties are promoted to span tags. - None, - - /// Public properties of the incoming message instance are promoted to span tags. - IncomingMessage, - - /// Public properties of outgoing message instances are promoted to span tags. - OutgoingMessage, - - /// Public properties of both incoming and outgoing message instances are promoted to span tags. - All } diff --git a/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs b/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs deleted file mode 100644 index 7f4129c4705..00000000000 --- a/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs +++ /dev/null @@ -1,45 +0,0 @@ -#nullable enable - -namespace NServiceBus; - -using System; -using System.Diagnostics; -using System.Text; -using System.Text.Json; -using System.Threading.Tasks; -using Pipeline; - -class IncomingMessagePayloadToTagsBehavior : IBehavior -{ - public Task Invoke(IIncomingLogicalMessageContext context, Func next) - { - var activity = Activity.Current; - if (activity?.IsAllDataRequested == true) - { - SetMessageBodyTag(activity, context.Message.Instance); - } - - return next(context); - } - - internal static void SetMessageBodyTag(Activity activity, object instance) - { - var json = JsonSerializer.Serialize(instance, instance.GetType()); - var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(json)); - activity.SetTag("nservicebus.message.body", base64); - } -} - -class OutgoingMessagePayloadToTagsBehavior : IBehavior -{ - public Task Invoke(IOutgoingLogicalMessageContext context, Func next) - { - var activity = Activity.Current; - if (activity?.IsAllDataRequested) - { - IncomingMessagePayloadToTagsBehavior.SetMessageBodyTag(activity, context.Message.Instance); - } - - return next(context); - } -} diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs index e4804d76a0c..c485129eee0 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs @@ -22,20 +22,5 @@ protected override void Setup(FeatureConfigurationContext context) new PopulateRecoverabilityTraceMetadataBehavior(), "Populates the recoverability metadata" ); - - var options = context.Settings.GetOrDefault(); - if (options?.MessagePayloadAsTag is MessagePayloadAsTag.IncomingMessage or MessagePayloadAsTag.All) - { - context.Pipeline.Register( - new IncomingMessagePayloadToTagsBehavior(), - "Promotes incoming message properties to span tags"); - } - - if (options?.MessagePayloadAsTag is MessagePayloadAsTag.OutgoingMessage or MessagePayloadAsTag.All) - { - context.Pipeline.Register( - new OutgoingMessagePayloadToTagsBehavior(), - "Promotes outgoing message properties to span tags"); - } } } \ No newline at end of file From af64227212fae3bbe38e38272cd1a7182b692125 Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Tue, 16 Jun 2026 13:36:27 +0200 Subject: [PATCH 05/22] access level fixes for acceptance tests --- .../Core/OpenTelemetry/Traces/When_publishing_messages.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs index 0595803fbef..4281ddc170a 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs @@ -203,7 +203,7 @@ await Scenario.Define() Assert.That(publishedMessage.DisplayName, Is.EqualTo("publish ThisIsAnEvent")); } - class PublisherWithDestinationNaming : EndpointConfigurationBuilder + public class PublisherWithDestinationNaming : EndpointConfigurationBuilder { public PublisherWithDestinationNaming() => EndpointSetup(b => @@ -222,7 +222,7 @@ public PublisherWithDestinationNaming() => }); } - class SubscriberForPublisherWithDestinationNaming : EndpointConfigurationBuilder + public class SubscriberForPublisherWithDestinationNaming : EndpointConfigurationBuilder { public SubscriberForPublisherWithDestinationNaming() => EndpointSetup(c => { }, From bba80f7bd2e6ef25bd942bb85032c0bf5b14c88d Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Tue, 16 Jun 2026 15:12:22 +0200 Subject: [PATCH 06/22] Using DistributedContextPropagator instead of hand-written baggage propagator (#7820) * initial migration to the DistributedContextPropagator for W3C compatibility * new propagator with backwards compatiblity tests * fixing the startnewtrace header progagation * refactor ContextPropagation.cs * small tweak * warning fixes * att tests reflect supported baggage and tracestate formats --- .../Metrics/When_envelope_handler_succeeds.cs | 1 + .../When_ambient_trace_in_message_session.cs | 2 +- ...hen_incoming_message_has_baggage_header.cs | 4 +- .../When_outgoing_activity_has_baggage.cs | 2 +- .../ContextPropagationIncompatibilityTests.cs | 84 +++++++++++++++++ .../OpenTelemetry/ContextPropagationTests.cs | 91 +++++++++++++------ .../OpenTelemetry/LegacyContextPropagator.cs | 87 ++++++++++++++++++ .../Tracing/ActivityExtensions.cs | 2 + .../Tracing/ContextPropagation.cs | 74 ++++++--------- 9 files changed, 269 insertions(+), 78 deletions(-) create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_envelope_handler_succeeds.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_envelope_handler_succeeds.cs index 16c7d1da43b..faca6eed9bb 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_envelope_handler_succeeds.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_envelope_handler_succeeds.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.ApplicationInsights.Extensibility; using NServiceBus; using NServiceBus.AcceptanceTesting; using NServiceBus.AcceptanceTests.Core.OpenTelemetry; diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_ambient_trace_in_message_session.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_ambient_trace_in_message_session.cs index 51ae62b9a41..6c3f58c65e5 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_ambient_trace_in_message_session.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_ambient_trace_in_message_session.cs @@ -15,7 +15,7 @@ public async Task Should_attach_to_ambient_trace() using var externalActivitySource = new ActivitySource("external trace source"); using var _ = TestingActivityListener.SetupDiagnosticListener(externalActivitySource.Name); // need to have a registered listener for activities to be created - const string wrapperActivityTraceState = "test trace state"; + const string wrapperActivityTraceState = "tracekey=traceValue"; var context = await Scenario.Define() .WithEndpoint(b => b diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_baggage_header.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_baggage_header.cs index 29acf036898..ffcc39fd5a1 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_baggage_header.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_baggage_header.cs @@ -17,7 +17,7 @@ public async Task Should_propagate_baggage_to_activity() { var sendOptions = new SendOptions(); sendOptions.RouteToThisEndpoint(); - sendOptions.SetHeader(Headers.DiagnosticsBaggage, "key1=value1,key2=value2,key3="); + sendOptions.SetHeader(Headers.DiagnosticsBaggage, "key1=value1,key2=value2,key3=value3"); await session.Send(new SomeMessage(), sendOptions); }) ) @@ -29,7 +29,7 @@ public async Task Should_propagate_baggage_to_activity() VerifyBaggageItem("key1", "value1"); VerifyBaggageItem("key2", "value2"); - VerifyBaggageItem("key3", ""); + VerifyBaggageItem("key3", "value3"); return; void VerifyBaggageItem(string key, string expectedValue) diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs index 93ff05bcfbf..e8bed02db26 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs @@ -33,7 +33,7 @@ public async Task Should_propagate_baggage_to_headers() ) .Run(); - Assert.That(context.BaggageHeader, Is.EqualTo("key3=,key2=value2,key1=value1")); + Assert.That(context.BaggageHeader, Is.EqualTo("key3 = , key2 = value2, key1 = value1")); } public class TestEndpoint : EndpointConfigurationBuilder diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs new file mode 100644 index 00000000000..4aa93147137 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs @@ -0,0 +1,84 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System.Collections.Generic; +using System.Diagnostics; +using Extensibility; +using NUnit.Framework; + +[TestFixture] +public class ContextPropagationIncompatibilityTests +{ + delegate void Writer(Activity activity, Dictionary headers, ContextBag context); + delegate void Reader(Activity activity, IDictionary headers); + + static readonly Writer LegacyWrite = LegacyContextPropagator.PropagateContextToHeaders; + static readonly Reader LegacyRead = LegacyContextPropagator.PropagateContextFromHeaders; + static readonly Writer NewWrite = ContextPropagation.PropagateContextToHeaders; + static readonly Reader NewRead = ContextPropagation.PropagateContextFromHeaders; + + // A value exercising every class of special character: structural baggage delimiters + // (',' ';' '='), the escape char '%', quotes, brackets, slashes, ampersand, Unicode and + // an emoji, plus interior spaces. Deliberately has NO leading/trailing whitespace, so this + // value isolates "what happens to special characters" from the separate edge-whitespace + // issue covered by New_propagation_loses_leading_whitespace_in_a_value. + // This already includes property-like syntax (the ';' and '=' delimiters), so a value such as + // "zone=eu;sensitive" is just a subset and needs no separate case here. + const string AllSpecialCharacters = "a b,c;d=e&f'g\"h\\i(j)k{l}m[n]o%p/q?r:s@t~u|vx é ü 😀 z"; + + static Dictionary Send(string value, Writer write) + { + using var sender = new Activity(ActivityNames.OutgoingMessageActivityName); + sender.SetIdFormat(ActivityIdFormat.W3C); + sender.Start(); + sender.AddBaggage("key", value); + + var headers = new Dictionary(); + write(sender, headers, new ContextBag()); + sender.Stop(); + return headers; + } + + static string Receive(Dictionary headers, Reader read) + { + using var receiver = new Activity(ActivityNames.IncomingMessageActivityName); + receiver.SetIdFormat(ActivityIdFormat.W3C); + receiver.Start(); + read(receiver, headers); + return receiver.GetBaggageItem("key"); + } + + static string Transmit(string value, Writer write, Reader read) => Receive(Send(value, write), read); + + [Test] + public void Legacy_sender_to_new_receiver_preserves_the_value() + { + var received = Transmit(AllSpecialCharacters, LegacyWrite, NewRead); + Assert.That(received, Is.EqualTo(AllSpecialCharacters)); + } + + [Test] + public void New_sender_to_legacy_receiver_prepends_a_leading_space_but_keeps_the_special_characters() + { + var received = Transmit(AllSpecialCharacters, NewWrite, LegacyRead); + + Assert.That(received, Is.EqualTo(" " + AllSpecialCharacters), + "ignoring the leading space, every special character round-trips correctly"); + } + + [Test] + public void New_propagation_loses_leading_whitespace_in_a_value() + { + const string valueWithLeadingSpace = " hasLeadingSpace"; + + var legacyRoundTrip = Transmit(valueWithLeadingSpace, LegacyWrite, LegacyRead); + var newRoundTrip = Transmit(valueWithLeadingSpace, NewWrite, NewRead); + + using (Assert.EnterMultipleScope()) + { + Assert.That(legacyRoundTrip, Is.EqualTo(valueWithLeadingSpace), + "legacy propagation preserves leading whitespace via percent-encoding"); + Assert.That(newRoundTrip, Is.EqualTo("hasLeadingSpace"), + "new propagation strips the leading whitespace from the value"); + } + } +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs index 6de0d1ade7a..48f95c1fe86 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs @@ -1,6 +1,5 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; -using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; @@ -94,7 +93,9 @@ public void Can_propagate_baggage_from_header_to_activity(ContextPropagationTest headers[Headers.DiagnosticsBaggage] = testCase.BaggageHeaderValue; } - var activity = new Activity(ActivityNames.IncomingMessageActivityName); + using var activity = new Activity(ActivityNames.IncomingMessageActivityName); + activity.SetIdFormat(ActivityIdFormat.W3C); + activity.Start(); ContextPropagation.PropagateContextFromHeaders(activity, headers); @@ -114,7 +115,9 @@ public void Can_propagate_baggage_from_activity_to_header(ContextPropagationTest var headers = new Dictionary(); - var activity = new Activity(ActivityNames.OutgoingMessageActivityName); + using var activity = new Activity(ActivityNames.OutgoingMessageActivityName); + activity.SetIdFormat(ActivityIdFormat.W3C); + activity.Start(); foreach (var baggageItem in testCase.ExpectedBaggageItems.Reverse()) { @@ -131,7 +134,7 @@ public void Can_propagate_baggage_from_activity_to_header(ContextPropagationTest { Assert.That(baggageHeaderSet, Is.True, "Should have a baggage header if there is baggage"); - Assert.That(baggageValue, Is.EqualTo(testCase.BaggageHeaderValueWithoutOptionalWhitespace), "baggage header is set but is not correct"); + Assert.That(baggageValue, Is.EqualTo(testCase.BaggageHeaderValue), "baggage header is set but is not correct"); } } else @@ -146,7 +149,9 @@ public void Can_roundtrip_baggage(ContextPropagationTestCase testCase) TestContext.Out.WriteLine($"Baggage header: {testCase.BaggageHeaderValue}"); var outgoingHeaders = new Dictionary(); - var outgoingActivity = new Activity(ActivityNames.OutgoingMessageActivityName); + using var outgoingActivity = new Activity(ActivityNames.OutgoingMessageActivityName); + outgoingActivity.SetIdFormat(ActivityIdFormat.W3C); + outgoingActivity.Start(); foreach (var baggageItem in testCase.ExpectedBaggageItems.Reverse()) { @@ -157,7 +162,9 @@ public void Can_roundtrip_baggage(ContextPropagationTestCase testCase) // Simulate wire transfer var incomingHeaders = outgoingHeaders; - var incomingActivity = new Activity(ActivityNames.IncomingMessageActivityName); + using var incomingActivity = new Activity(ActivityNames.IncomingMessageActivityName); + incomingActivity.SetIdFormat(ActivityIdFormat.W3C); + incomingActivity.Start(); ContextPropagation.PropagateContextFromHeaders(incomingActivity, incomingHeaders); @@ -170,55 +177,87 @@ public void Can_roundtrip_baggage(ContextPropagationTestCase testCase) } } + [Test] + public void Can_not_roundtrip_baggage_value_with_optional_whitespaces() + { + var outgoingHeaders = new Dictionary(); + using var outgoingActivity = new Activity(ActivityNames.OutgoingMessageActivityName); + outgoingActivity.SetIdFormat(ActivityIdFormat.W3C); + outgoingActivity.Start(); + + outgoingActivity.AddBaggage("key1", " value1"); + outgoingActivity.AddBaggage("key2", "value2 "); + + ContextPropagation.PropagateContextToHeaders(outgoingActivity, outgoingHeaders, new ContextBag()); + + // Simulate wire transfer + var incomingHeaders = outgoingHeaders; + using var incomingActivity = new Activity(ActivityNames.IncomingMessageActivityName); + incomingActivity.SetIdFormat(ActivityIdFormat.W3C); + incomingActivity.Start(); + + ContextPropagation.PropagateContextFromHeaders(incomingActivity, incomingHeaders); + + using (Assert.EnterMultipleScope()) + { + foreach (var baggageItem in outgoingActivity.Baggage) + { + var key = baggageItem.Key; + var actualValue = incomingActivity.GetBaggageItem(key); + Assert.That(actualValue, Is.Not.Null, $"Baggage is missing item with key |{key}|"); + Assert.That(actualValue, Is.EqualTo(baggageItem.Value.Trim()), $"Baggage item |{key}| has the wrong value"); + } + } + } + // HINT: Many of these test cases are given as examples in the spec https://www.w3.org/TR/baggage/#example static IEnumerable TestCases => new object[] { new ContextPropagationTestCase("without any baggage"), new ContextPropagationTestCase("with a single key") - .WithBaggage("key1", "value1"), + .WithBaggage("key1", "value1") + .WithHeaderValue("key1 = value1"), new ContextPropagationTestCase("with multiple keys") .WithBaggage("key1", "value1") - .WithBaggage("key2", "value2"), - - new ContextPropagationTestCase("with whitespace") - .WithBaggage("key1 ", " value1") - .WithBaggage(" key2", "value2 ") - .WithBaggage(" key3 ", " value3 "), + .WithBaggage("key2", "value2") + .WithHeaderValue("key1 = value1, key2 = value2"), new ContextPropagationTestCase("with properties that do not have keys") - .WithBaggage("key1", "value1;property1;property2"), + .WithBaggage("key1", "value1;property1;property2") + .WithHeaderValue("key1 = value1%3Bproperty1%3Bproperty2"), new ContextPropagationTestCase("with properties that have keys") - .WithBaggage("key3", "value3; propertyKey=propertyValue"), + .WithBaggage("key3", "value3; propertyKey=propertyValue") + .WithHeaderValue("key3 = value3%3B%20propertyKey=propertyValue"), new ContextPropagationTestCase("with values containing whitespace") - .WithBaggage("serverNode", "DF 28"), + .WithBaggage("serverNode", "DF 28") + .WithHeaderValue("serverNode = DF%2028"), new ContextPropagationTestCase("with values containing unicode") .WithBaggage("userId", "Amélie") + .WithHeaderValue("userId = Am%C3%A9lie") }; - public class ContextPropagationTestCase + public class ContextPropagationTestCase(string caseName) { - string caseName; - Dictionary baggageItems = []; + readonly Dictionary baggageItems = []; - public ContextPropagationTestCase(string caseName) + public ContextPropagationTestCase WithBaggage(string key, string value) { - this.caseName = caseName; + baggageItems.Add(key, value); + return this; } - public ContextPropagationTestCase WithBaggage(string key, string value) + public ContextPropagationTestCase WithHeaderValue(string headerValue) { - baggageItems.Add(key, value); + BaggageHeaderValue = headerValue; return this; } - public string BaggageHeaderValue => string.Join(",", from kvp in baggageItems select $"{kvp.Key}={Uri.EscapeDataString(kvp.Value)}"); - public string BaggageHeaderValueWithoutOptionalWhitespace - => string.Join(",", from kvp in baggageItems select $"{kvp.Key.Trim()}={Uri.EscapeDataString(kvp.Value)}"); + public string BaggageHeaderValue { get; private set; } public IEnumerable> ExpectedBaggageItems => from kvp in baggageItems select new KeyValuePair( kvp.Key.Trim(), diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs new file mode 100644 index 00000000000..dc35002215b --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs @@ -0,0 +1,87 @@ +#nullable enable + +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Extensibility; + +static class LegacyContextPropagator +{ + public static void PropagateContextToHeaders(Activity activity, Dictionary headers, ContextBag contextBag) + { + if (activity is null) + { + return; + } + + if (activity.Id is not null) + { + headers[Headers.DiagnosticsTraceParent] = activity.Id; + } + + if (activity.TraceStateString is not null) + { + headers[Headers.DiagnosticsTraceState] = activity.TraceStateString; + } + + // Check whether the startnewtrace setting was set in the context, if so, add it to the headers now the trace parent was added + if (contextBag.TryGet(Headers.StartNewTrace, out var headerContent)) + { + headers[Headers.StartNewTrace] = headerContent; + } + + var baggage = string.Join(",", activity.Baggage.Select(item => $"{item.Key}={Uri.EscapeDataString(item.Value ?? string.Empty)}")); + if (!string.IsNullOrEmpty(baggage)) + { + headers[Headers.DiagnosticsBaggage] = baggage; + } + } + + public static void PropagateContextFromHeaders(Activity? activity, IDictionary headers) + { + if (activity is null) + { + return; + } + + if (headers.TryGetValue(Headers.DiagnosticsTraceState, out var traceState)) + { + activity.TraceStateString = traceState; + } + + if (headers.TryGetValue(Headers.DiagnosticsBaggage, out var baggageValue)) + { + var baggageSpan = baggageValue.AsSpan(); + // HINT: Iterate in reverse order because Activity baggage is LIFO + while (!baggageSpan.IsEmpty) + { + var lastComma = baggageSpan.LastIndexOf(','); + ReadOnlySpan baggageItem; + + if (lastComma >= 0) + { + baggageItem = baggageSpan[(lastComma + 1)..]; + baggageSpan = baggageSpan[..lastComma]; + } + else + { + baggageItem = baggageSpan; + baggageSpan = []; + } + + var firstEquals = baggageItem.IndexOf('='); + if (firstEquals < 0 || firstEquals >= baggageItem.Length) + { + continue; + } + + var key = baggageItem[..firstEquals].Trim(); + var value = baggageItem[(firstEquals + 1)..]; + activity.AddBaggage(key.ToString(), Uri.UnescapeDataString(value)); + } + } + } +} \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs index 04e287679c0..30b0bf1066f 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs @@ -41,6 +41,8 @@ public static void SetErrorStatus(this Activity activity, Exception ex) activity.SetStatus(ActivityStatusCode.Error, ex.Message); activity.SetTag("otel.status_code", "ERROR"); activity.SetTag("otel.status_description", ex.Message); + + activity.AddEvent(new ActivityEvent("exception", DateTimeOffset.UtcNow, [ new KeyValuePair("exception.escaped", true), diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs index 0f1d8868270..b6ac0314b4c 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs @@ -2,10 +2,8 @@ namespace NServiceBus; -using System; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using Extensibility; static class ContextPropagation @@ -17,26 +15,14 @@ public static void PropagateContextToHeaders(Activity? activity, Dictionary(Headers.StartNewTrace, out var startNewTrace); - // Check whether the startnewtrace setting was set in the context, if so, add it to the headers now the trace parent was added - if (contextBag.TryGet(Headers.StartNewTrace, out var headerContent)) + if (traceParentExists && startNewTraceOnReceive) { - headers[Headers.StartNewTrace] = headerContent; - } - - var baggage = string.Join(",", activity.Baggage.Select(item => $"{item.Key}={Uri.EscapeDataString(item.Value ?? string.Empty)}")); - if (!string.IsNullOrEmpty(baggage)) - { - headers[Headers.DiagnosticsBaggage] = baggage; + headers[Headers.StartNewTrace] = startNewTrace!; } } @@ -47,41 +33,33 @@ public static void PropagateContextFromHeaders(Activity? activity, IDictionary baggageItem; + var baggage = DistributedContextPropagator.Current.ExtractBaggage(headers, Getter); - if (lastComma >= 0) - { - baggageItem = baggageSpan[(lastComma + 1)..]; - baggageSpan = baggageSpan[..lastComma]; - } - else - { - baggageItem = baggageSpan; - baggageSpan = []; - } - - var firstEquals = baggageItem.IndexOf('='); - if (firstEquals < 0 || firstEquals >= baggageItem.Length) - { - continue; - } + if (baggage is null) + { + return; + } - var key = baggageItem[..firstEquals].Trim(); - var value = baggageItem[(firstEquals + 1)..]; - activity.AddBaggage(key.ToString(), Uri.UnescapeDataString(value)); - } + foreach (var baggageItem in baggage) + { + activity.AddBaggage(baggageItem.Key, baggageItem.Value); } } + + static readonly DistributedContextPropagator.PropagatorSetterCallback Setter = static (carrier, key, value) => + ((IDictionary)carrier!)[key] = value; + + static readonly DistributedContextPropagator.PropagatorGetterCallback Getter = + static (carrier, key, out value, out values) => + { + values = null; + value = ((IReadOnlyDictionary)carrier!).GetValueOrDefault(key); + }; } \ No newline at end of file From ca842e67166c25f183fc0da3744d7b1e9e001df5 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 17 Jun 2026 12:47:23 +0200 Subject: [PATCH 07/22] =?UTF-8?q?=F0=9F=90=9B=20Add=20regression=20test=20?= =?UTF-8?q?for=20null=20baggage=20value=20(#6983)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Activity baggage with a null value used to throw ArgumentNullException during context propagation (Uri.EscapeDataString(null)). The switch to DistributedContextPropagator on this branch fixes the root cause; this test pins the behavior so it cannot regress. --- .../OpenTelemetry/ContextPropagationTests.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs index 48f95c1fe86..0141d6c9a15 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs @@ -81,6 +81,22 @@ public void Overwrites_existing_propagation_header() Assert.That(activity.Id, Is.EqualTo(headers[Headers.DiagnosticsTraceParent])); } + [Test] + public void Should_not_throw_when_baggage_value_is_null() + { + // Reproduces https://github.com/Particular/NServiceBus/issues/6983 + // A baggage item with a null value used to make the hand-written propagator call + // Uri.EscapeDataString(null), throwing ArgumentNullException while sending a message. + using var activity = new Activity(ActivityNames.OutgoingMessageActivityName); + activity.SetIdFormat(ActivityIdFormat.W3C); + activity.Start(); + activity.AddBaggage("test", null); + + var headers = new Dictionary(); + + Assert.DoesNotThrow(() => ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag())); + } + [TestCaseSource(nameof(TestCases))] public void Can_propagate_baggage_from_header_to_activity(ContextPropagationTestCase testCase) { From a33484eaa8812a4d148c801bfc1f2f18b4f4091d Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 19 Jun 2026 14:10:30 +0200 Subject: [PATCH 08/22] Make DistributedContextPropagator opt-in (keep OTel propagation backwards compatible until v11) (#7825) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Make DistributedContextPropagator opt-in via AppContext switch The switch to System.Diagnostics.DistributedContextPropagator changes the OpenTelemetry baggage wire format (W3C OWS encoding + whitespace trimming), which is breaking on rolling upgrades. Keep the legacy percent-encoded propagator as the default and gate the new propagator behind the NServiceBus.Core.OpenTelemetry.UseDistributedContextPropagator AppContext switch (default in v11). All temporary code (switch plumbing + legacy propagator) lives in obsolete_v11.cs so v11 cleanup is a single file deletion plus removing the two delegation blocks in ContextPropagation.cs. Follows the existing AppContextSwitches.UseV2DeterministicGuid / PreObsolete pattern. - ContextPropagation: delegate to the legacy propagator unless the switch is on - obsolete_v11.cs: switch + byte-for-byte revert of the pre-10.3 propagator - Tests asserting the new W3C format enable the switch per-fixture - New ContextPropagationDefaultBehaviorTests locks in legacy default behavior - Acceptance baggage assertion reverted to the legacy default wire format Span naming (UseMessageDestinationInSpanNames) is already opt-in and unchanged. * Refactor: Replace `ObsoleteV11` with `LegacyContextPropagation` * Add test to verify null baggage value does not throw in legacy propagator * Added comments about preserve legacy baggage handling behavior when escaping and trimming values * Add test to verify that we are preserving whitespace in legacy propagator baggage values. * add comments to clarify intent behind legacy propagator handling * update ContextPropagationCompatibilityTests to use correct LegacyContextPropagation delegates * rename ContextPropagationTests to LegacyContextPropagationTests and remove outdated baggage handling tests * allow changing the propagator implementation at runtime --------- Co-authored-by: Tomasz Masternak --- .../When_outgoing_activity_has_baggage.cs | 5 +- ...> ContextPropagationCompatibilityTests.cs} | 42 +++++- .../ContextPropagationDefaultBehaviorTests.cs | 72 +++++++++ ...ts.cs => LegacyContextPropagationTests.cs} | 48 +----- .../OpenTelemetry/LegacyContextPropagator.cs | 87 ----------- .../Tracing/ContextPropagation.cs | 19 +++ .../OpenTelemetry/Tracing/obsolete_v11.cs | 139 ++++++++++++++++++ 7 files changed, 279 insertions(+), 133 deletions(-) rename src/NServiceBus.Core.Tests/OpenTelemetry/{ContextPropagationIncompatibilityTests.cs => ContextPropagationCompatibilityTests.cs} (68%) create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs rename src/NServiceBus.Core.Tests/OpenTelemetry/{ContextPropagationTests.cs => LegacyContextPropagationTests.cs} (83%) delete mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs create mode 100644 src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs index e8bed02db26..b6a2310ad25 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs @@ -33,7 +33,10 @@ public async Task Should_propagate_baggage_to_headers() ) .Run(); - Assert.That(context.BaggageHeader, Is.EqualTo("key3 = , key2 = value2, key1 = value1")); + // Default (backwards-compatible) propagation produces the legacy comma-separated, percent-encoded format. + // The W3C OWS format ("key3 = , key2 = value2, key1 = value1") is produced only when the + // NServiceBus.Core.OpenTelemetry.UseDistributedContextPropagator AppContext switch is enabled (default in v11). + Assert.That(context.BaggageHeader, Is.EqualTo("key3=,key2=value2,key1=value1")); } public class TestEndpoint : EndpointConfigurationBuilder diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs similarity index 68% rename from src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs rename to src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs index 4aa93147137..7f4332e0cde 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs @@ -1,18 +1,33 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; +using System; using System.Collections.Generic; using System.Diagnostics; using Extensibility; using NUnit.Framework; [TestFixture] -public class ContextPropagationIncompatibilityTests +public class ContextPropagationCompatibilityTests { + [SetUp] + public void EnableDistributedContextPropagator() + { + AppContext.SetSwitch(LegacyContextPropagation.UseDistributedContextPropagatorSwitchName, true); + LegacyContextPropagation.ResetUseDistributedContextPropagator(); + } + + [TearDown] + public void ResetDistributedContextPropagator() + { + AppContext.SetSwitch(LegacyContextPropagation.UseDistributedContextPropagatorSwitchName, false); + LegacyContextPropagation.ResetUseDistributedContextPropagator(); + } + delegate void Writer(Activity activity, Dictionary headers, ContextBag context); delegate void Reader(Activity activity, IDictionary headers); - static readonly Writer LegacyWrite = LegacyContextPropagator.PropagateContextToHeaders; - static readonly Reader LegacyRead = LegacyContextPropagator.PropagateContextFromHeaders; + static readonly Writer LegacyWrite = LegacyContextPropagation.PropagateContextToHeaders; + static readonly Reader LegacyRead = LegacyContextPropagation.PropagateContextFromHeaders; static readonly Writer NewWrite = ContextPropagation.PropagateContextToHeaders; static readonly Reader NewRead = ContextPropagation.PropagateContextFromHeaders; @@ -22,7 +37,7 @@ public class ContextPropagationIncompatibilityTests // value isolates "what happens to special characters" from the separate edge-whitespace // issue covered by New_propagation_loses_leading_whitespace_in_a_value. // This already includes property-like syntax (the ';' and '=' delimiters), so a value such as - // "zone=eu;sensitive" is just a subset and needs no separate case here. + // "zone=eu;sensitive" is just a subset and needs no separate case here. const string AllSpecialCharacters = "a b,c;d=e&f'g\"h\\i(j)k{l}m[n]o%p/q?r:s@t~u|vx é ü 😀 z"; static Dictionary Send(string value, Writer write) @@ -81,4 +96,21 @@ public void New_propagation_loses_leading_whitespace_in_a_value() "new propagation strips the leading whitespace from the value"); } } -} + + [TestCase(null, "", "")] + [TestCase("", "", "")] + [TestCase(" ", "", " ")] + [TestCase(" x ", "x", " x ")] + [TestCase(" x x ", "x x", " x x ")] + public void ValidateThatLegacyPropagatorPreservesLeadingAndTrailingWhitespaceInBaggageValues(string input, string expectedNew, string expectedLegacy) + { + var outputNew = Transmit(input, NewWrite, NewRead); + var outputLegacy = Transmit(input, LegacyWrite, LegacyRead); + + using (Assert.EnterMultipleScope()) + { + Assert.That(expectedNew, Is.EqualTo(outputNew), "Native propagator isn't trimming all leading and trailing whitespaces"); + Assert.That(expectedLegacy, Is.EqualTo(outputLegacy), "Legacy propagator isn't preserving leading and trailing whitespace for backwards compatibility"); + } + } +} \ No newline at end of file diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs new file mode 100644 index 00000000000..fbe65df4b51 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs @@ -0,0 +1,72 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using Extensibility; +using NUnit.Framework; + +[TestFixture] +public class ContextPropagationDefaultBehaviorTests +{ + // Without the opt-in switch, the endpoint default must remain the backwards-compatible + // legacy propagator (percent-encoded, comma-separated, whitespace preserved). + [SetUp] + public void EnsureDefault() + { + AppContext.SetSwitch(LegacyContextPropagation.UseDistributedContextPropagatorSwitchName, false); + LegacyContextPropagation.ResetUseDistributedContextPropagator(); + } + + [Test] + public void Default_uses_legacy_percent_encoded_baggage_format() + { + using var activity = new Activity(ActivityNames.OutgoingMessageActivityName); + activity.SetIdFormat(ActivityIdFormat.W3C); + activity.Start(); + activity.AddBaggage("serverNode", "DF 28"); + + var headers = new Dictionary(); + ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag()); + + Assert.That(headers[Headers.DiagnosticsBaggage], Is.EqualTo("serverNode=DF%2028")); + } + + [Test] + public void Default_does_not_throw_when_baggage_value_is_null() + { + // Reproduces https://github.com/Particular/NServiceBus/issues/6983 on the legacy propagator. + // A null baggage value must not make the legacy propagator call Uri.EscapeDataString(null). + // Calls LegacyContextPropagation directly so the assertion is independent of the AppContext switch. + using var activity = new Activity(ActivityNames.OutgoingMessageActivityName); + activity.SetIdFormat(ActivityIdFormat.W3C); + activity.Start(); + activity.AddBaggage("test", null); + + var headers = new Dictionary(); + + Assert.DoesNotThrow(() => LegacyContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag())); + Assert.That(headers[Headers.DiagnosticsBaggage], Is.EqualTo("test=")); + } + + [Test] + public void Default_round_trip_preserves_value_whitespace() + { + using var outgoing = new Activity(ActivityNames.OutgoingMessageActivityName); + outgoing.SetIdFormat(ActivityIdFormat.W3C); + outgoing.Start(); + outgoing.AddBaggage("key1", " leading-and-trailing "); + + var headers = new Dictionary(); + ContextPropagation.PropagateContextToHeaders(outgoing, headers, new ContextBag()); + + using var incoming = new Activity(ActivityNames.IncomingMessageActivityName); + incoming.SetIdFormat(ActivityIdFormat.W3C); + incoming.Start(); + ContextPropagation.PropagateContextFromHeaders(incoming, headers); + + // Legacy propagation preserves leading/trailing whitespace via percent-encoding; + // the DistributedContextPropagator (opt-in) would trim it. + Assert.That(incoming.GetBaggageItem("key1"), Is.EqualTo(" leading-and-trailing ")); + } +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs similarity index 83% rename from src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs rename to src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs index 0141d6c9a15..4c62f70cd71 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs @@ -1,5 +1,6 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; +using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; @@ -8,7 +9,7 @@ using NUnit.Framework; [TestFixture] -public class ContextPropagationTests +public class LegacyContextPropagationTests { [Test] public void Propagate_activity_id_to_header() @@ -193,39 +194,6 @@ public void Can_roundtrip_baggage(ContextPropagationTestCase testCase) } } - [Test] - public void Can_not_roundtrip_baggage_value_with_optional_whitespaces() - { - var outgoingHeaders = new Dictionary(); - using var outgoingActivity = new Activity(ActivityNames.OutgoingMessageActivityName); - outgoingActivity.SetIdFormat(ActivityIdFormat.W3C); - outgoingActivity.Start(); - - outgoingActivity.AddBaggage("key1", " value1"); - outgoingActivity.AddBaggage("key2", "value2 "); - - ContextPropagation.PropagateContextToHeaders(outgoingActivity, outgoingHeaders, new ContextBag()); - - // Simulate wire transfer - var incomingHeaders = outgoingHeaders; - using var incomingActivity = new Activity(ActivityNames.IncomingMessageActivityName); - incomingActivity.SetIdFormat(ActivityIdFormat.W3C); - incomingActivity.Start(); - - ContextPropagation.PropagateContextFromHeaders(incomingActivity, incomingHeaders); - - using (Assert.EnterMultipleScope()) - { - foreach (var baggageItem in outgoingActivity.Baggage) - { - var key = baggageItem.Key; - var actualValue = incomingActivity.GetBaggageItem(key); - Assert.That(actualValue, Is.Not.Null, $"Baggage is missing item with key |{key}|"); - Assert.That(actualValue, Is.EqualTo(baggageItem.Value.Trim()), $"Baggage item |{key}| has the wrong value"); - } - } - } - // HINT: Many of these test cases are given as examples in the spec https://www.w3.org/TR/baggage/#example static IEnumerable TestCases => new object[] { @@ -233,28 +201,28 @@ public void Can_not_roundtrip_baggage_value_with_optional_whitespaces() new ContextPropagationTestCase("with a single key") .WithBaggage("key1", "value1") - .WithHeaderValue("key1 = value1"), + .WithHeaderValue("key1=value1"), new ContextPropagationTestCase("with multiple keys") .WithBaggage("key1", "value1") .WithBaggage("key2", "value2") - .WithHeaderValue("key1 = value1, key2 = value2"), + .WithHeaderValue("key1=value1,key2=value2"), new ContextPropagationTestCase("with properties that do not have keys") .WithBaggage("key1", "value1;property1;property2") - .WithHeaderValue("key1 = value1%3Bproperty1%3Bproperty2"), + .WithHeaderValue("key1=value1%3Bproperty1%3Bproperty2"), new ContextPropagationTestCase("with properties that have keys") .WithBaggage("key3", "value3; propertyKey=propertyValue") - .WithHeaderValue("key3 = value3%3B%20propertyKey=propertyValue"), + .WithHeaderValue("key3=value3%3B%20propertyKey%3DpropertyValue"), new ContextPropagationTestCase("with values containing whitespace") .WithBaggage("serverNode", "DF 28") - .WithHeaderValue("serverNode = DF%2028"), + .WithHeaderValue("serverNode=DF%2028"), new ContextPropagationTestCase("with values containing unicode") .WithBaggage("userId", "Amélie") - .WithHeaderValue("userId = Am%C3%A9lie") + .WithHeaderValue("userId=Am%C3%A9lie") }; public class ContextPropagationTestCase(string caseName) diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs deleted file mode 100644 index dc35002215b..00000000000 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs +++ /dev/null @@ -1,87 +0,0 @@ -#nullable enable - -namespace NServiceBus.Core.Tests.OpenTelemetry; - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using Extensibility; - -static class LegacyContextPropagator -{ - public static void PropagateContextToHeaders(Activity activity, Dictionary headers, ContextBag contextBag) - { - if (activity is null) - { - return; - } - - if (activity.Id is not null) - { - headers[Headers.DiagnosticsTraceParent] = activity.Id; - } - - if (activity.TraceStateString is not null) - { - headers[Headers.DiagnosticsTraceState] = activity.TraceStateString; - } - - // Check whether the startnewtrace setting was set in the context, if so, add it to the headers now the trace parent was added - if (contextBag.TryGet(Headers.StartNewTrace, out var headerContent)) - { - headers[Headers.StartNewTrace] = headerContent; - } - - var baggage = string.Join(",", activity.Baggage.Select(item => $"{item.Key}={Uri.EscapeDataString(item.Value ?? string.Empty)}")); - if (!string.IsNullOrEmpty(baggage)) - { - headers[Headers.DiagnosticsBaggage] = baggage; - } - } - - public static void PropagateContextFromHeaders(Activity? activity, IDictionary headers) - { - if (activity is null) - { - return; - } - - if (headers.TryGetValue(Headers.DiagnosticsTraceState, out var traceState)) - { - activity.TraceStateString = traceState; - } - - if (headers.TryGetValue(Headers.DiagnosticsBaggage, out var baggageValue)) - { - var baggageSpan = baggageValue.AsSpan(); - // HINT: Iterate in reverse order because Activity baggage is LIFO - while (!baggageSpan.IsEmpty) - { - var lastComma = baggageSpan.LastIndexOf(','); - ReadOnlySpan baggageItem; - - if (lastComma >= 0) - { - baggageItem = baggageSpan[(lastComma + 1)..]; - baggageSpan = baggageSpan[..lastComma]; - } - else - { - baggageItem = baggageSpan; - baggageSpan = []; - } - - var firstEquals = baggageItem.IndexOf('='); - if (firstEquals < 0 || firstEquals >= baggageItem.Length) - { - continue; - } - - var key = baggageItem[..firstEquals].Trim(); - var value = baggageItem[(firstEquals + 1)..]; - activity.AddBaggage(key.ToString(), Uri.UnescapeDataString(value)); - } - } - } -} \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs index b6ac0314b4c..8df14752012 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs @@ -10,6 +10,16 @@ static class ContextPropagation { public static void PropagateContextToHeaders(Activity? activity, Dictionary headers, ContextBag contextBag) { + // TODO: investigate if we need to improve the switch check for better performance + // Removed in v11, see obsolete_v11.cs + if (!LegacyContextPropagation.UseDistributedContextPropagator) + { + LegacyContextPropagation.PropagateContextToHeaders(activity, headers, contextBag); + return; + } + + // The following part was intentionally not extracted to a separate class to prevent + // accidental leftovers when because that the legacy propagator will be removed in v11 if (activity is null) { return; @@ -28,6 +38,15 @@ public static void PropagateContextToHeaders(Activity? activity, Dictionary headers) { + // Removed in v11, see obsolete_v11.cs + if (!LegacyContextPropagation.UseDistributedContextPropagator) + { + LegacyContextPropagation.PropagateContextFromHeaders(activity, headers); + return; + } + + // The following part was intentionally not extracted to a separate class to prevent + // accidental leftovers when because that the legacy propagator will be removed in v11 if (activity is null) { return; diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs new file mode 100644 index 00000000000..b1dec663683 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs @@ -0,0 +1,139 @@ +#nullable enable + +namespace NServiceBus; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Extensibility; +using Particular.Obsoletes; + +// ============================================================================= +// EVERYTHING IN THIS FILE IS TEMPORARY AND WILL BE REMOVED IN v11. +// +// In v10.3, switching to System.Diagnostics.DistributedContextPropagator for +// OpenTelemetry trace-context/baggage propagation changes the baggage wire +// format (W3C OWS encoding + whitespace trimming) and is therefore breaking on +// rolling upgrades. To stay backwards compatible it is opt-in via an AppContext +// switch and the legacy propagator below remains the default. +// +// In v11 the new propagator becomes the default: delete this entire file and +// remove the two `if (!ObsoleteV11.UseDistributedContextPropagator)` delegation +// blocks in ContextPropagation.cs. +// ============================================================================= +static class LegacyContextPropagation +{ + enum SwitchState : byte + { + Unchecked = 0, + Enabled = 1, + Disabled = 2 + } + + static SwitchState cachedUseDistributedContextPropagator; + + [PreObsolete("https://github.com/Particular/NServiceBus/issues/7825", + Note = "In v11, DistributedContextPropagator-based context propagation becomes the default and this switch will be removed together with the legacy propagator in obsolete_v11.cs.", + ReplacementTypeOrMember = "ContextPropagation")] + public const string UseDistributedContextPropagatorSwitchName = "NServiceBus.Core.OpenTelemetry.UseDistributedContextPropagator"; + + [PreObsolete("https://github.com/Particular/NServiceBus/issues/7825", + Note = "In v11, DistributedContextPropagator-based context propagation becomes the default and this switch will be removed together with the legacy propagator in obsolete_v11.cs.", + ReplacementTypeOrMember = "ContextPropagation")] + public static bool UseDistributedContextPropagator + { + get + { + var state = cachedUseDistributedContextPropagator; + if (state != SwitchState.Unchecked) + { + return state == SwitchState.Enabled; + } + + state = AppContext.TryGetSwitch(UseDistributedContextPropagatorSwitchName, out var isEnabled) && isEnabled + ? SwitchState.Enabled + : SwitchState.Disabled; + cachedUseDistributedContextPropagator = state; + + return state == SwitchState.Enabled; + } + } + + internal static void ResetUseDistributedContextPropagator() => cachedUseDistributedContextPropagator = SwitchState.Unchecked; + + public static void PropagateContextToHeaders(Activity? activity, Dictionary headers, ContextBag contextBag) + { + if (activity is null) + { + return; + } + + if (activity.Id is not null) + { + headers[Headers.DiagnosticsTraceParent] = activity.Id; + } + + if (activity.TraceStateString is not null) + { + headers[Headers.DiagnosticsTraceState] = activity.TraceStateString; + } + + // Check whether the startnewtrace setting was set in the context, if so, add it to the headers now the trace parent was added + if (contextBag.TryGet(Headers.StartNewTrace, out var headerContent)) + { + headers[Headers.StartNewTrace] = headerContent; + } + + var baggage = string.Join(",", activity.Baggage.Select(item => $"{item.Key}={Uri.EscapeDataString(item.Value ?? string.Empty)}")); + if (!string.IsNullOrEmpty(baggage)) + { + headers[Headers.DiagnosticsBaggage] = baggage; + } + } + + public static void PropagateContextFromHeaders(Activity? activity, IDictionary headers) + { + if (activity is null) + { + return; + } + + if (headers.TryGetValue(Headers.DiagnosticsTraceState, out var traceState)) + { + activity.TraceStateString = traceState; + } + + if (headers.TryGetValue(Headers.DiagnosticsBaggage, out var baggageValue)) + { + var baggageSpan = baggageValue.AsSpan(); + // HINT: Iterate in reverse order because Activity baggage is LIFO + while (!baggageSpan.IsEmpty) + { + var lastComma = baggageSpan.LastIndexOf(','); + ReadOnlySpan baggageItem; + + if (lastComma >= 0) + { + baggageItem = baggageSpan[(lastComma + 1)..]; + baggageSpan = baggageSpan[..lastComma]; + } + else + { + baggageItem = baggageSpan; + baggageSpan = []; + } + + var firstEquals = baggageItem.IndexOf('='); + if (firstEquals < 0 || firstEquals >= baggageItem.Length) + { + continue; + } + + var key = baggageItem[..firstEquals].Trim(); + var value = baggageItem[(firstEquals + 1)..]; + activity.AddBaggage(key.ToString(), Uri.UnescapeDataString(value)); + } + } + } +} \ No newline at end of file From 68274cf0706a08a2e33b50c4b58bc15c2bd6c919 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 8 Jul 2026 14:14:58 +0200 Subject: [PATCH 09/22] Emit handler spans from a dedicated NServiceBus.Core.Handler ActivitySource (#7844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Handler (InvokeHandler) spans can now be emitted from a dedicated "NServiceBus.Core.Handler" ActivitySource so they can be filtered or sampled independently of the pipeline spans (#7284). Opting in without subscribing to the new source suppresses handler spans entirely, making Activity.Current inside handlers/behaviors the "process message" span — the flattened trace requested in the issue. Existing OpenTelemetry configurations only subscribe to "NServiceBus.Core" and would silently lose handler spans, so the new source is opt-in via the NServiceBus.Core.OpenTelemetry.UseHandlerActivitySource AppContext switch until v11, following the DistributedContextPropagator pattern: - ActivitySources: add permanent Handler source - ActivityFactory.StartHandlerActivity: pick source based on the switch - obsolete_v11.cs: cached switch with PreObsolete markers; in v11 delete the class and make ActivitySources.Handler unconditional - Unit tests cover default source, opt-in source, preserved span data, and no-listener suppression; acceptance test locks in the default --- ...g_message_with_default_activity_sources.cs | 48 ++++++++++ .../HandlerActivitySourceTests.cs | 95 +++++++++++++++++++ .../OpenTelemetry/Tracing/ActivityFactory.cs | 8 +- .../OpenTelemetry/Tracing/ActivitySources.cs | 4 + .../OpenTelemetry/Tracing/obsolete_v11.cs | 50 ++++++++++ 5 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_message_with_default_activity_sources.cs create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_message_with_default_activity_sources.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_message_with_default_activity_sources.cs new file mode 100644 index 00000000000..a233aa178b1 --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_message_with_default_activity_sources.cs @@ -0,0 +1,48 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Traces; + +using System.Linq; +using System.Threading.Tasks; +using EndpointTemplates; +using NServiceBus.AcceptanceTesting; +using NUnit.Framework; + +public class When_processing_message_with_default_activity_sources : OpenTelemetryAcceptanceTest +{ + // Until v11, handler spans are emitted from the "NServiceBus.Core" ActivitySource by default + // for backwards compatibility. The dedicated "NServiceBus.Core.Handler" source is opt-in via + // the NServiceBus.Core.OpenTelemetry.UseHandlerActivitySource AppContext switch (default in v11). + [Test] + public async Task Should_emit_handler_span_from_main_source() + { + await Scenario.Define() + .WithEndpoint(b => + b.When(session => session.SendLocal(new SomeMessage())) + ) + .Run(); + + var invokedHandlerActivities = NServiceBusActivityListener.CompletedActivities.GetInvokedHandlerActivities(); + + Assert.That(invokedHandlerActivities, Has.Count.EqualTo(1)); + Assert.That(invokedHandlerActivities.Single().Source.Name, Is.EqualTo("NServiceBus.Core"), + "without the opt-in switch, handler spans must keep coming from the main source so existing OpenTelemetry configurations keep seeing them"); + } + + public class Context : ScenarioContext; + + public class ReceivingEndpoint : EndpointConfigurationBuilder + { + public ReceivingEndpoint() => EndpointSetup(); + + [Handler] + public class MessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(SomeMessage message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + + public class SomeMessage : IMessage; +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs new file mode 100644 index 00000000000..8af728a9c8a --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs @@ -0,0 +1,95 @@ +#nullable enable + +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Linq; +using Helpers; +using NServiceBus.Pipeline; +using NUnit.Framework; + +[TestFixture] +public class HandlerActivitySourceTests +{ + readonly ActivityFactory activityFactory = new(new InstrumentationOptions()); + + TestingActivityListener mainListener; + + [SetUp] + public void SetUp() => mainListener = TestingActivityListener.SetupNServiceBusDiagnosticListener(); + + [TearDown] + public void TearDown() + { + mainListener.Dispose(); + AppContext.SetSwitch(HandlerActivitySourceSwitch.UseHandlerActivitySourceSwitchName, false); + HandlerActivitySourceSwitch.ResetUseHandlerActivitySource(); + } + + static void OptIn() + { + AppContext.SetSwitch(HandlerActivitySourceSwitch.UseHandlerActivitySourceSwitchName, true); + HandlerActivitySourceSwitch.ResetUseHandlerActivitySource(); + } + + [Test] + public void Default_emits_handler_activity_from_main_source() + { + using var ambientActivity = new Activity("ambient activity"); + ambientActivity.Start(); + + var activity = activityFactory.StartHandlerActivity(new MessageHandler { HandlerType = typeof(HandlerActivitySourceTests) }); + + Assert.That(activity, Is.Not.Null); + Assert.That(activity!.Source.Name, Is.EqualTo("NServiceBus.Core")); + } + + [Test] + public void Opt_in_emits_handler_activity_from_handler_source() + { + OptIn(); + using var handlerListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core.Handler"); + + using var ambientActivity = new Activity("ambient activity"); + ambientActivity.Start(); + + var activity = activityFactory.StartHandlerActivity(new MessageHandler { HandlerType = typeof(HandlerActivitySourceTests) }); + + Assert.That(activity, Is.Not.Null); + Assert.That(activity!.Source.Name, Is.EqualTo("NServiceBus.Core.Handler")); + } + + [Test] + public void Opt_in_preserves_display_name_and_handler_type_tag() + { + OptIn(); + using var handlerListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core.Handler"); + + using var ambientActivity = new Activity("ambient activity"); + ambientActivity.Start(); + + Type handlerType = typeof(HandlerActivitySourceTests); + var activity = activityFactory.StartHandlerActivity(new MessageHandler { HandlerType = handlerType }); + + Assert.That(activity, Is.Not.Null); + Assert.That(activity!.DisplayName, Is.EqualTo(handlerType.Name)); + var tags = activity.Tags.ToImmutableDictionary(); + Assert.That(tags[ActivityTags.HandlerType], Is.EqualTo(handlerType.FullName)); + } + + [Test] + public void Opt_in_without_handler_source_listener_does_not_create_handler_activity() + { + OptIn(); + + using var ambientActivity = new Activity("ambient activity"); + ambientActivity.Start(); + + var activity = activityFactory.StartHandlerActivity(new MessageHandler { HandlerType = typeof(HandlerActivitySourceTests) }); + + Assert.That(activity, Is.Null, "handler activity must not be created when the dedicated source has no listeners"); + Assert.That(Activity.Current, Is.SameAs(ambientActivity), "user tags must land on the parent (process message) activity"); + } +} diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index b6548eef2f0..d5dcf4ec07a 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -111,7 +111,13 @@ public ActivityFactory(InstrumentationOptions options) return null; } - var activity = ActivitySources.Main.StartActivity(ActivityNames.InvokeHandlerActivityName); + // Until v11 the dedicated handler source is opt-in; existing configurations only + // subscribe to the main source and must keep receiving handler spans from it. + var source = HandlerActivitySourceSwitch.UseHandlerActivitySource + ? ActivitySources.Handler + : ActivitySources.Main; + + var activity = source.StartActivity(ActivityNames.InvokeHandlerActivityName); if (activity is null) { diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs index 8f6e26858e1..939305dd0eb 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs @@ -9,4 +9,8 @@ static class ActivitySources public static readonly ActivitySource Main = new("NServiceBus.Core", "0.1.0"); + + public static readonly ActivitySource Handler = + new("NServiceBus.Core.Handler", + "0.1.0"); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs index b1dec663683..499be45157c 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs @@ -136,4 +136,54 @@ public static void PropagateContextFromHeaders(Activity? activity, IDictionary cachedUseHandlerActivitySource = SwitchState.Unchecked; } \ No newline at end of file From beeadd0241d8ed0807557009baddc0267f76d407 Mon Sep 17 00:00:00 2001 From: "Irina Dominte(Scurtu)" Date: Thu, 16 Jul 2026 14:11:47 +0300 Subject: [PATCH 10/22] Optout on dispatching events (#7846) * Optout on dispatching events * Aproval tests * Update src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt Co-authored-by: Tomasz Masternak * Renamed Suppress to Emmit * Wrap `AddEvent` call in null-check for `TryGetRecordingIncomingPipelineActivity`. --------- Co-authored-by: Tomasz Masternak Co-authored-by: Tomasz Masternak --- .../APIApprovals.ApproveNServiceBus.approved.txt | 1 + ...ransportReceiveToPhysicalMessageConnectorTests.cs | 2 +- .../OpenTelemetry/InstrumentationOptions.cs | 8 ++++++++ .../TransportReceiveToPhysicalMessageConnector.cs | 12 ++++++++---- src/NServiceBus.Core/Receiving/ReceiveComponent.cs | 2 +- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index a5bdc1d3893..67152cc72d5 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -635,6 +635,7 @@ namespace NServiceBus public class InstrumentationOptions { public InstrumentationOptions() { } + public bool EmitMessageDispatchingEvents { get; set; } public bool UseMessageDestinationInSpanNames { get; set; } } public sealed class KeyedServiceKey diff --git a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs index 96e3f57b8e8..2431d74b3ab 100644 --- a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs @@ -192,7 +192,7 @@ public void SetUp() fakeOutbox = new FakeOutboxStorage(); fakeBatchPipeline = new FakeBatchPipeline(); - behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc")); + behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc"), new InstrumentationOptions()); } Task Invoke(ITransportReceiveContext context, Func next = null) => behavior.Invoke(context, next ?? (_ => Task.CompletedTask)); diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index f920a28b93a..a725e1f0f1c 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -14,4 +14,12 @@ public class InstrumentationOptions /// Disabled by default for backward compatibility. /// public bool UseMessageDestinationInSpanNames { get; set; } + + /// + /// Controls whether the "Start dispatching" and "Finished dispatching" activity events + /// are added to the incoming message span when outgoing messages are dispatched. + /// Enabled by default for backward compatibility. Disable to avoid the ingestion cost + /// of these events when they add no diagnostic value. + /// + public bool EmitMessageDispatchingEvents { get; set; } = true; } diff --git a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs index 0504c24f16e..96f31beecc9 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs @@ -15,7 +15,8 @@ namespace NServiceBus; class TransportReceiveToPhysicalMessageConnector( IOutboxStorage outboxStorage, - IncomingPipelineMetrics incomingPipelineMetrics) + IncomingPipelineMetrics incomingPipelineMetrics, + InstrumentationOptions instrumentationOptions) : IStageForkConnector { public async Task Invoke(ITransportReceiveContext context, Func next) @@ -65,10 +66,13 @@ public async Task Invoke(ITransportReceiveContext context, Func { var storage = b.GetService() ?? new NoOpOutboxStorage(); - return new TransportReceiveToPhysicalMessageConnector(storage, b.GetRequiredService()); + return new TransportReceiveToPhysicalMessageConnector(storage, b.GetRequiredService(), hostingConfiguration.ActivityFactory.Options); }, "Allows to abort processing the message"); pipelineSettings.Register("LoadHandlersConnector", b => new LoadHandlersConnector(b.GetRequiredService(), hostingConfiguration.ActivityFactory), "Gets all the handlers to invoke from the MessageHandler registry based on the message type."); From 338ea5f79bd0bd9d431e0bf9ae1edf314dc08d72 Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Thu, 16 Jul 2026 13:28:00 +0200 Subject: [PATCH 11/22] Gauge meter for active message processings (#7841) * gauge meter counting active handler invocations * update MeterAPI approval file to reflect version 0.3.0 changes and new active handlers metric * Improvements * switching to active messages gauge * remove obsolete MessageMetadataRegistry dependency from IncomingPipelineMetrics and capture enclosed message types on the gauge meter * remove unused Unicast.Messages imports from test files * Apply suggestion from @tmasternak * move `ActiveMessageScope` definition to correct location in `IncomingPipelineMetrics` file --------- Co-authored-by: Irina Dominte --- ...hen_messages_are_processed_concurrently.cs | 72 +++++++++++++++++++ .../MeterTests.Verify_MeterAPI.approved.txt | 4 +- .../Envelopes/EnvelopeUnwrapperTests.cs | 2 - .../Incoming/InvokeHandlerTerminatorTest.cs | 2 +- ...tReceiveToPhysicalMessageConnectorTests.cs | 2 +- .../OpenTelemetry/Metrics/MeterTags.cs | 1 + .../Incoming/IncomingPipelineMetrics.cs | 36 ++++++++-- .../Pipeline/MainPipelineExecutor.cs | 3 + 8 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_messages_are_processed_concurrently.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_messages_are_processed_concurrently.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_messages_are_processed_concurrently.cs new file mode 100644 index 00000000000..da6b3382f0e --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_messages_are_processed_concurrently.cs @@ -0,0 +1,72 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Metrics; + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using EndpointTemplates; +using NServiceBus; +using AcceptanceTesting; +using NUnit.Framework; +using Conventions = AcceptanceTesting.Customization.Conventions; + +public class When_messages_are_processed_concurrently : OpenTelemetryAcceptanceTest +{ + const string ActiveMessagesMetric = "nservicebus.messaging.active_messages"; + const int numberOfMessages = 5; + + [Test] + public async Task Should_report_active_messages_gauge_that_balances_once_idle() + { + using var metricsListener = TestingMetricListener.SetupNServiceBusMetricsListener(); + + _ = await Scenario.Define() + .WithEndpoint(b => b.CustomConfig(c => + { + c.MakeInstanceUniquelyAddressable("instanceId"); + c.LimitMessageProcessingConcurrencyTo(10); + }).When(async (session, ctx) => + { + for (var x = 0; x < numberOfMessages; x++) + { + await session.SendLocal(new OutgoingMessage()); + } + })) + .Run(); + + Assert.That(metricsListener.ReportedMeters.TryGetValue(ActiveMessagesMetric, out var net), Is.True, + $"'{ActiveMessagesMetric}' gauge should be reported"); + Assert.That(net, Is.EqualTo(0), + "increments and decrements should balance once all messages have been processed"); + + metricsListener.AssertTags(ActiveMessagesMetric, + new Dictionary + { + ["nservicebus.queue"] = Conventions.EndpointNamingConvention(typeof(EndpointWithMetrics)), + ["nservicebus.discriminator"] = "instanceId", + ["nservicebus.enclosed_message_types"] = typeof(OutgoingMessage).AssemblyQualifiedName + }); + } + + public class Context : ScenarioContext + { + public int OutgoingMessagesReceived; + } + + public class EndpointWithMetrics : EndpointConfigurationBuilder + { + public EndpointWithMetrics() => EndpointSetup(); + + [Handler] + public class MessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(OutgoingMessage message, IMessageHandlerContext context) + { + var messagesHandled = Interlocked.Increment(ref testContext.OutgoingMessagesReceived); + testContext.MarkAsCompleted(messagesHandled == numberOfMessages); + return Task.CompletedTask; + } + } + } + + public class OutgoingMessage : IMessage; +} diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt index a572d43098f..d97f8c61a26 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt @@ -1,11 +1,12 @@ { "Note": "Changes to metrics API should result in an update to NServiceBusMeter version.", "MetricsSourceName": "NServiceBus.Core.Pipeline.Incoming", - "MetricsSourceVersion": "0.2.0", + "MetricsSourceVersion": "0.3.0", "Tags": [ "error.type", "execution.result", "nservicebus.discriminator", + "nservicebus.enclosed_message_types", "nservicebus.envelope.unwrapper_type", "nservicebus.message_handler_type", "nservicebus.message_handler_types", @@ -14,6 +15,7 @@ ], "Metrics": [ "nservicebus.envelope.unwrapped => Counter", + "nservicebus.messaging.active_messages => UpDownCounter", "nservicebus.messaging.critical_time => Histogram, Unit: s", "nservicebus.messaging.failures => Counter", "nservicebus.messaging.fetches => Counter", diff --git a/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs b/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs index 1f2460a8b9c..1824dc35f2c 100644 --- a/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs +++ b/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs @@ -4,12 +4,10 @@ namespace NServiceBus.Core.Tests.Envelopes; using System; using System.Buffers; using System.Collections.Generic; -using System.Text; using Extensibility; using NUnit.Framework; using Transport; - public class EnvelopeUnwrapperTests { string nativeId; diff --git a/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs b/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs index f36af8a855a..47276e798e4 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs @@ -11,7 +11,7 @@ [TestFixture] public class InvokeHandlerTerminatorTest { - InvokeHandlerTerminator terminator = new(new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc")); + readonly InvokeHandlerTerminator terminator = new(new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc")); [Test] public async Task When_saga_found_and_handler_is_saga_should_invoke_handler() diff --git a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs index 2431d74b3ab..52d0fb8ad11 100644 --- a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs @@ -1,4 +1,4 @@ -namespace NServiceBus.Core.Tests.Reliability.Outbox; +namespace NServiceBus.Core.Tests.Reliability.Outbox; using System; using System.Collections.Generic; diff --git a/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs b/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs index 9eb8795c87f..d99b135197e 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs @@ -7,6 +7,7 @@ static class MeterTags public const string EndpointDiscriminator = "nservicebus.discriminator"; public const string QueueName = "nservicebus.queue"; public const string MessageType = "nservicebus.message_type"; + public const string EnclosedMessageTypes = "nservicebus.enclosed_message_types"; public const string MessageHandlerTypes = "nservicebus.message_handler_types"; public const string MessageHandlerType = "nservicebus.message_handler_type"; public const string ExecutionResult = "execution.result"; diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs index b842e912a3c..6a063b2abee 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs @@ -21,10 +21,11 @@ class IncomingPipelineMetrics const string RecoverabilityDelayed = "nservicebus.recoverability.delayed"; const string RecoverabilityError = "nservicebus.recoverability.error"; const string EnvelopeUnwrapping = "nservicebus.envelope.unwrapped"; + const string ActiveMessages = "nservicebus.messaging.active_messages"; public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, string discriminator) { - var meter = meterFactory.Create("NServiceBus.Core.Pipeline.Incoming", "0.2.0"); + var meter = meterFactory.Create("NServiceBus.Core.Pipeline.Incoming", "0.3.0"); totalProcessedSuccessfully = meter.CreateCounter(TotalProcessedSuccessfully, description: "Total number of messages processed successfully by the endpoint."); totalFetched = meter.CreateCounter(TotalFetched, @@ -45,6 +46,8 @@ public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, str description: "Total number of messages sent to the error queue."); totalEnvelopeUnwrapping = meter.CreateCounter(EnvelopeUnwrapping, description: "Total number of unwrapping attempts by the endpoint."); + activeMessages = meter.CreateUpDownCounter(ActiveMessages, + description: "Number of messages currently being processed by the endpoint."); queueNameBase = queueName; endpointDiscriminator = discriminator; @@ -53,7 +56,7 @@ public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, str public void AddDefaultIncomingPipelineMetricTags(IncomingPipelineMetricTags incomingPipelineMetricsTags) { incomingPipelineMetricsTags.Add(MeterTags.QueueName, queueNameBase); - incomingPipelineMetricsTags.Add(MeterTags.EndpointDiscriminator, endpointDiscriminator ?? ""); + incomingPipelineMetricsTags.Add(MeterTags.EndpointDiscriminator, endpointDiscriminator); } public void RecordProcessingTime(ITransportReceiveContext context, TimeSpan elapsed) @@ -243,6 +246,24 @@ public void RecordSendToErrorQueue(IRecoverabilityContext recoverabilityContext) totalSentToErrorQueue.Add(1, meterTags); } + public ActiveMessageScope TrackMessageProcessing(IncomingPipelineMetricTags incomingPipelineMetricTags, IncomingMessage message) + { + if (!activeMessages.Enabled) + { + return default; + } + + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [MeterTags.QueueName, MeterTags.EndpointDiscriminator]); + if (message.Headers.TryGetValue(Headers.EnclosedMessageTypes, out var enclosedMessageTypes)) + { + tags.Add(new KeyValuePair(MeterTags.EnclosedMessageTypes, enclosedMessageTypes)); + } + + activeMessages.Add(1, tags); + return new ActiveMessageScope(activeMessages, tags); + } + public void EnvelopeUnwrappingSucceeded(MessageContext messageContext, IEnvelopeHandler type) => RecordEnvelopeUnwrapping(messageContext, type, true, null); public void EnvelopeUnwrappingFailed(MessageContext messageContext, IEnvelopeHandler type, Exception? exception) => RecordEnvelopeUnwrapping(messageContext, type, false, exception); void RecordEnvelopeUnwrapping(MessageContext messageContext, IEnvelopeHandler type, bool succeeded, Exception? exception) @@ -266,6 +287,11 @@ void RecordEnvelopeUnwrapping(MessageContext messageContext, IEnvelopeHandler ty totalEnvelopeUnwrapping.Add(succeeded ? 0 : 1, meterTags); } + public readonly struct ActiveMessageScope(UpDownCounter? counter, TagList tags) : IDisposable + { + public void Dispose() => counter?.Add(-1, tags); + } + readonly Counter totalProcessedSuccessfully; readonly Counter totalFetched; readonly Counter totalFailures; @@ -276,6 +302,8 @@ void RecordEnvelopeUnwrapping(MessageContext messageContext, IEnvelopeHandler ty readonly Counter totalDelayedRetries; readonly Counter totalSentToErrorQueue; readonly Counter totalEnvelopeUnwrapping; - string queueNameBase; - string endpointDiscriminator; + readonly UpDownCounter activeMessages; + + readonly string queueNameBase; + readonly string endpointDiscriminator; } \ No newline at end of file diff --git a/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs b/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs index 07560a36c13..7bab6ffb264 100644 --- a/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs +++ b/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs @@ -35,6 +35,9 @@ public async Task Invoke(MessageContext messageContext, CancellationToken cancel using var incomingMessageHandle = envelopeUnwrapper.UnwrapEnvelope(messageContext); IncomingMessage message = incomingMessageHandle; + //This needs to happen after envelope unwrapping to ensure the proper value of the EnclosedMessageTypes header + using var activeMessageScope = incomingPipelineMetrics.TrackMessageProcessing(incomingPipelineMetricsTags, message); + var transportReceiveContext = new TransportReceiveContext( childScope.ServiceProvider, messageOperations, From 7d8cc9342a8f329e65b735543b1539935f54fde3 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 16 Jul 2026 14:06:23 +0200 Subject: [PATCH 12/22] Endpoint-level trace connector defaults for sends and publishes (#7867) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Add TraceConnector enum and endpoint-level trace connector defaults to InstrumentationOptions * ✨ Honor endpoint-level trace connector defaults in send/publish behaviors with per-message overrides * ✨ Add acceptance tests for endpoint-level trace connector defaults and per-message overrides * ✨ Approve public API additions for trace connector configuration * Replace `TraceConnector` with `TraceMode` for clearer terminology and updated tracing behavior. * Update approval file after replacing `TraceConnector` with `TraceMode` in public API --------- Co-authored-by: Tomasz Masternak --- .../Traces/When_publishing_messages.cs | 120 ++++++++++++++++++ .../Traces/When_sending_messages.cs | 80 ++++++++++++ ...IApprovals.ApproveNServiceBus.approved.txt | 9 ++ .../InstrumentationOptionsTests.cs | 19 +++ .../OpenTelemetryExtensionsTests.cs | 63 +++++++++ .../OpenTelemetryPublishBehaviorTests.cs | 55 ++++++++ .../OpenTelemetrySendBehaviorTests.cs | 55 ++++++++ .../OpenTelemetry/InstrumentationOptions.cs | 16 +++ .../OpenTelemetry/OpenTelemetryExtensions.cs | 36 +++++- .../OpenTelemetry/OpenTelemetryFeature.cs | 6 +- .../OpenTelemetryPublishBehavior.cs | 17 +-- .../OpenTelemetrySendBehavior.cs | 17 +-- .../OpenTelemetry/TraceMode.cs | 19 +++ 13 files changed, 484 insertions(+), 28 deletions(-) create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryPublishBehaviorTests.cs create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetrySendBehaviorTests.cs create mode 100644 src/NServiceBus.Core/OpenTelemetry/TraceMode.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs index 4281ddc170a..7aed06a30a8 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs @@ -242,5 +242,125 @@ public Task Handle(ThisIsAnEvent @event, IMessageHandlerContext context) } } + [Test] + public async Task Should_create_child_on_receive_when_endpoint_defaults_to_child_span() + { + var context = await Scenario.Define() + .WithEndpoint(b => b + .When(ctx => ctx.SomeEventSubscribed, s => s.Publish(new ThisIsAnEvent()))) + .WithEndpoint(b => b.When((session, ctx) => + { + if (ctx.HasNativePubSubSupport) + { + ctx.SomeEventSubscribed = true; + } + + return Task.CompletedTask; + })) + .Run(); + + var publishMessageActivities = NServiceBusActivityListener.CompletedActivities.GetPublishEventActivities(); + var receiveMessageActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + using (Assert.EnterMultipleScope()) + { + Assert.That(publishMessageActivities, Has.Count.EqualTo(1), "1 message is published as part of this test"); + Assert.That(receiveMessageActivities, Has.Count.EqualTo(1), "1 message is received as part of this test"); + } + + var publishRequest = publishMessageActivities[0]; + var receiveRequest = receiveMessageActivities[0]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(receiveRequest.RootId, Is.EqualTo(publishRequest.RootId), "publish and receive operations are part the same root activity"); + Assert.That(receiveRequest.ParentId, Is.Not.Null, "incoming message does have a parent"); + } + + Assert.That(receiveRequest.Links, Is.Empty, "receive does not have links"); + } + + [Test] + public async Task Should_create_new_linked_trace_on_receive_when_option_overrides_endpoint_connector() + { + var context = await Scenario.Define() + .WithEndpoint(b => b + .When(ctx => ctx.SomeEventSubscribed, s => + { + var publishOptions = new PublishOptions(); + publishOptions.StartNewTraceOnReceive(); + return s.Publish(new ThisIsAnEvent(), publishOptions); + })) + .WithEndpoint(b => b.When((session, ctx) => + { + if (ctx.HasNativePubSubSupport) + { + ctx.SomeEventSubscribed = true; + } + + return Task.CompletedTask; + })) + .Run(); + + var publishMessageActivities = NServiceBusActivityListener.CompletedActivities.GetPublishEventActivities(); + var receiveMessageActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + using (Assert.EnterMultipleScope()) + { + Assert.That(publishMessageActivities, Has.Count.EqualTo(1), "1 message is published as part of this test"); + Assert.That(receiveMessageActivities, Has.Count.EqualTo(1), "1 message is received as part of this test"); + } + + var publishRequest = publishMessageActivities[0]; + var receiveRequest = receiveMessageActivities[0]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(receiveRequest.RootId, Is.Not.EqualTo(publishRequest.RootId), "publish and receive operations are part of different root activities"); + Assert.That(receiveRequest.ParentId, Is.Null, "incoming message does not have a parent, it's a root"); + } + + ActivityLink link = receiveRequest.Links.FirstOrDefault(); + Assert.That(link, Is.Not.EqualTo(default(ActivityLink)), "Receive has a link"); + Assert.That(link.Context.TraceId, Is.EqualTo(publishRequest.TraceId), "receive is linked to publish operation"); + } + + public class PublisherWithChildSpanConnector : EndpointConfigurationBuilder + { + public PublisherWithChildSpanConnector() => + EndpointSetup(b => + { + b.Tracing().PublishTraceMode = TraceMode.ContinueExisting; + b.OnEndpointSubscribed((s, context) => + { + if (s.SubscriberEndpoint.Contains(Conventions.EndpointNamingConvention(typeof(SubscriberForPublisherWithChildSpanConnector)))) + { + if (s.MessageType == typeof(ThisIsAnEvent).AssemblyQualifiedName) + { + context.SomeEventSubscribed = true; + } + } + }); + }); + } + + public class SubscriberForPublisherWithChildSpanConnector : EndpointConfigurationBuilder + { + public SubscriberForPublisherWithChildSpanConnector() => + EndpointSetup(c => { }, + metadata => + { + metadata.RegisterPublisherFor(); + }); + + [Handler] + public class ThisHandlesSomethingHandler(Context testContext) : IHandleMessages + { + public Task Handle(ThisIsAnEvent @event, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + public class ThisIsAnEvent : IEvent; } \ No newline at end of file diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs index e14df1d05e6..c08e660e883 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs @@ -163,5 +163,85 @@ public Task Handle(OutgoingMessage message, IMessageHandlerContext context) } } + [Test] + public async Task Should_create_new_linked_trace_on_receive_when_endpoint_defaults_to_span_link() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => s.SendLocal(new OutgoingMessage()))) + .Run(); + + var sendMessageActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + var receiveMessageActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + using (Assert.EnterMultipleScope()) + { + Assert.That(sendMessageActivities, Has.Count.EqualTo(1), "1 message is sent as part of this test"); + Assert.That(receiveMessageActivities, Has.Count.EqualTo(1), "1 message is received as part of this test"); + } + + var sendRequest = sendMessageActivities[0]; + var receiveRequest = receiveMessageActivities[0]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(receiveRequest.RootId, Is.Not.EqualTo(sendRequest.RootId), "send and receive operations are part of different root activities"); + Assert.That(receiveRequest.ParentId, Is.Null, "incoming message does not have a parent, it's a root"); + } + + ActivityLink link = receiveRequest.Links.FirstOrDefault(); + Assert.That(link, Is.Not.EqualTo(default(ActivityLink)), "Receive has a link"); + Assert.That(link.Context.TraceId, Is.EqualTo(sendRequest.TraceId), "receive is linked to send operation"); + } + + [Test] + public async Task Should_create_child_on_receive_when_option_overrides_endpoint_connector() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => + { + var sendOptions = new SendOptions(); + sendOptions.RouteToThisEndpoint(); + sendOptions.ContinueExistingTraceOnReceive(); + return s.Send(new OutgoingMessage(), sendOptions); + })) + .Run(); + + var sendMessageActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + var receiveMessageActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + using (Assert.EnterMultipleScope()) + { + Assert.That(sendMessageActivities, Has.Count.EqualTo(1), "1 message is sent as part of this test"); + Assert.That(receiveMessageActivities, Has.Count.EqualTo(1), "1 message is received as part of this test"); + } + + var sendRequest = sendMessageActivities[0]; + var receiveRequest = receiveMessageActivities[0]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(receiveRequest.RootId, Is.EqualTo(sendRequest.RootId), "send and receive operations are part of the same root activity"); + Assert.That(receiveRequest.ParentId, Is.Not.Null, "incoming message does have a parent"); + } + + Assert.That(receiveRequest.Links, Is.Empty, "receive does not have links"); + } + + public class TestEndpointWithSpanLinkConnector : EndpointConfigurationBuilder + { + public TestEndpointWithSpanLinkConnector() => + EndpointSetup(b => b.Tracing().SendTraceMode = TraceMode.StartNew); + + [Handler] + public class MessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(OutgoingMessage message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + public class OutgoingMessage : IMessage; } \ No newline at end of file diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index 67152cc72d5..30981403a3d 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -636,6 +636,8 @@ namespace NServiceBus { public InstrumentationOptions() { } public bool EmitMessageDispatchingEvents { get; set; } + public NServiceBus.TraceMode PublishTraceMode { get; set; } + public NServiceBus.TraceMode SendTraceMode { get; set; } public bool UseMessageDestinationInSpanNames { get; set; } } public sealed class KeyedServiceKey @@ -777,6 +779,8 @@ namespace NServiceBus public static class OpenTelemetryExtensions { public static void ContinueExistingTraceOnReceive(this NServiceBus.PublishOptions publishOptions) { } + public static void ContinueExistingTraceOnReceive(this NServiceBus.SendOptions sendOptions) { } + public static void StartNewTraceOnReceive(this NServiceBus.PublishOptions publishOptions) { } public static void StartNewTraceOnReceive(this NServiceBus.SendOptions sendOptions) { } public static NServiceBus.InstrumentationOptions Tracing(this NServiceBus.EndpointConfiguration config) { } } @@ -1174,6 +1178,11 @@ namespace NServiceBus public ToSagaExpression(NServiceBus.IConfigureHowToFindSagaWithMessage sagaMessageFindingConfiguration, System.Linq.Expressions.Expression> messageProperty) { } public void ToSaga(System.Linq.Expressions.Expression> sagaEntityProperty) { } } + public enum TraceMode + { + ContinueExisting = 0, + StartNew = 1, + } public static class TransportConfig { extension(NServiceBus.EndpointConfiguration endpointConfiguration) diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs new file mode 100644 index 00000000000..ded2e6a41c2 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs @@ -0,0 +1,19 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using NUnit.Framework; + +[TestFixture] +public class InstrumentationOptionsTests +{ + [Test] + public void Should_default_trace_connectors_to_current_behavior() + { + var options = new InstrumentationOptions(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(options.SendTraceMode, Is.EqualTo(TraceMode.ContinueExisting), "sends continue the trace by default"); + Assert.That(options.PublishTraceMode, Is.EqualTo(TraceMode.StartNew), "publishes start a new linked trace by default"); + } + } +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs new file mode 100644 index 00000000000..14752802842 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs @@ -0,0 +1,63 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using NUnit.Framework; + +[TestFixture] +public class OpenTelemetryExtensionsTests +{ + [Test] + public void StartNewTraceOnReceive_should_set_span_link_override_on_send_options() + { + var options = new SendOptions(); + + options.StartNewTraceOnReceive(); + + Assert.That(options.Context.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode connector), Is.True); + Assert.That(connector, Is.EqualTo(TraceMode.StartNew)); + } + + [Test] + public void ContinueExistingTraceOnReceive_should_set_child_span_override_on_send_options() + { + var options = new SendOptions(); + + options.ContinueExistingTraceOnReceive(); + + Assert.That(options.Context.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode connector), Is.True); + Assert.That(connector, Is.EqualTo(TraceMode.ContinueExisting)); + } + + [Test] + public void StartNewTraceOnReceive_should_set_span_link_override_on_publish_options() + { + var options = new PublishOptions(); + + options.StartNewTraceOnReceive(); + + Assert.That(options.Context.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode connector), Is.True); + Assert.That(connector, Is.EqualTo(TraceMode.StartNew)); + } + + [Test] + public void ContinueExistingTraceOnReceive_should_set_child_span_override_on_publish_options() + { + var options = new PublishOptions(); + + options.ContinueExistingTraceOnReceive(); + + Assert.That(options.Context.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode connector), Is.True); + Assert.That(connector, Is.EqualTo(TraceMode.ContinueExisting)); + } + + [Test] + public void Last_override_call_wins() + { + var options = new PublishOptions(); + + options.ContinueExistingTraceOnReceive(); + options.StartNewTraceOnReceive(); + + Assert.That(options.Context.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode connector), Is.True); + Assert.That(connector, Is.EqualTo(TraceMode.StartNew)); + } +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryPublishBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryPublishBehaviorTests.cs new file mode 100644 index 00000000000..3cc9e44ee06 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryPublishBehaviorTests.cs @@ -0,0 +1,55 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System.Threading.Tasks; +using NUnit.Framework; +using Testing; + +[TestFixture] +public class OpenTelemetryPublishBehaviorTests +{ + [Test] + public async Task Should_start_new_trace_on_receive_by_default() + { + var behavior = new OpenTelemetryPublishBehavior(new InstrumentationOptions()); + var context = new TestableOutgoingPublishContext(); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.TrueString)); + } + + [Test] + public async Task Should_continue_trace_on_receive_when_endpoint_connector_is_child_span() + { + var behavior = new OpenTelemetryPublishBehavior(new InstrumentationOptions { PublishTraceMode = TraceMode.ContinueExisting }); + var context = new TestableOutgoingPublishContext(); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.FalseString)); + } + + [Test] + public async Task Should_prefer_child_span_option_over_endpoint_connector() + { + var behavior = new OpenTelemetryPublishBehavior(new InstrumentationOptions { PublishTraceMode = TraceMode.StartNew }); + var context = new TestableOutgoingPublishContext(); + context.Extensions.Set(OpenTelemetryExtensions.TraceConnectorOverrideKey, TraceMode.ContinueExisting); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.FalseString)); + } + + [Test] + public async Task Should_prefer_span_link_option_over_endpoint_connector() + { + var behavior = new OpenTelemetryPublishBehavior(new InstrumentationOptions { PublishTraceMode = TraceMode.ContinueExisting }); + var context = new TestableOutgoingPublishContext(); + context.Extensions.Set(OpenTelemetryExtensions.TraceConnectorOverrideKey, TraceMode.StartNew); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.TrueString)); + } +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetrySendBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetrySendBehaviorTests.cs new file mode 100644 index 00000000000..8e26d213544 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetrySendBehaviorTests.cs @@ -0,0 +1,55 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System.Threading.Tasks; +using NUnit.Framework; +using Testing; + +[TestFixture] +public class OpenTelemetrySendBehaviorTests +{ + [Test] + public async Task Should_continue_trace_on_receive_by_default() + { + var behavior = new OpenTelemetrySendBehavior(new InstrumentationOptions()); + var context = new TestableOutgoingSendContext(); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.FalseString)); + } + + [Test] + public async Task Should_start_new_trace_on_receive_when_endpoint_connector_is_span_link() + { + var behavior = new OpenTelemetrySendBehavior(new InstrumentationOptions { SendTraceMode = TraceMode.StartNew }); + var context = new TestableOutgoingSendContext(); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.TrueString)); + } + + [Test] + public async Task Should_prefer_span_link_option_over_endpoint_connector() + { + var behavior = new OpenTelemetrySendBehavior(new InstrumentationOptions { SendTraceMode = TraceMode.ContinueExisting }); + var context = new TestableOutgoingSendContext(); + context.Extensions.Set(OpenTelemetryExtensions.TraceConnectorOverrideKey, TraceMode.StartNew); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.TrueString)); + } + + [Test] + public async Task Should_prefer_child_span_option_over_endpoint_connector() + { + var behavior = new OpenTelemetrySendBehavior(new InstrumentationOptions { SendTraceMode = TraceMode.StartNew }); + var context = new TestableOutgoingSendContext(); + context.Extensions.Set(OpenTelemetryExtensions.TraceConnectorOverrideKey, TraceMode.ContinueExisting); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.FalseString)); + } +} diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index a725e1f0f1c..dcdd85a1c93 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -22,4 +22,20 @@ public class InstrumentationOptions /// of these events when they add no diagnostic value. /// public bool EmitMessageDispatchingEvents { get; set; } = true; + + /// + /// Controls how the receive-side processing span relates to the send span for messages sent by this endpoint. + /// Defaults to : receivers continue the trace. + /// Can be overridden per message via + /// or . + /// + public TraceMode SendTraceMode { get; set; } = TraceMode.ContinueExisting; + + /// + /// Controls how the receive-side processing span relates to the publish span for events published by this endpoint. + /// Defaults to : receivers start a new trace linked back to the publish span. + /// Can be overridden per message via + /// or . + /// + public TraceMode PublishTraceMode { get; set; } = TraceMode.StartNew; } diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs index 9c3ad6e63e8..6db97f31694 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs @@ -21,20 +21,48 @@ public static InstrumentationOptions Tracing(this EndpointConfiguration config) } /// - /// Start a new OpenTelemetry trace conversation. + /// Start a new OpenTelemetry trace on receive of this message, linked back to the send span. + /// Overrides for this message. /// /// The option being extended. public static void StartNewTraceOnReceive(this SendOptions sendOptions) { - sendOptions.Context.Set(OpenTelemetrySendBehavior.StartNewTraceOnReceive, true); + ArgumentNullException.ThrowIfNull(sendOptions); + sendOptions.Context.Set(TraceConnectorOverrideKey, TraceMode.StartNew); } /// - /// Start a new OpenTelemetry trace conversation. + /// Continue the existing OpenTelemetry trace on receive of this message. + /// Overrides for this message. + /// + /// The option being extended. + public static void ContinueExistingTraceOnReceive(this SendOptions sendOptions) + { + ArgumentNullException.ThrowIfNull(sendOptions); + sendOptions.Context.Set(TraceConnectorOverrideKey, TraceMode.ContinueExisting); + } + + /// + /// Start a new OpenTelemetry trace on receive of this event, linked back to the publish span. + /// Overrides for this message. + /// + /// The option being extended. + public static void StartNewTraceOnReceive(this PublishOptions publishOptions) + { + ArgumentNullException.ThrowIfNull(publishOptions); + publishOptions.Context.Set(TraceConnectorOverrideKey, TraceMode.StartNew); + } + + /// + /// Continue the existing OpenTelemetry trace on receive of this event. + /// Overrides for this message. /// /// The option being extended. public static void ContinueExistingTraceOnReceive(this PublishOptions publishOptions) { - publishOptions.Context.Set(OpenTelemetryPublishBehavior.ContinueTraceOnReceive, true); + ArgumentNullException.ThrowIfNull(publishOptions); + publishOptions.Context.Set(TraceConnectorOverrideKey, TraceMode.ContinueExisting); } + + internal const string TraceConnectorOverrideKey = "NServiceBus.OpenTelemetry.TraceConnectorOverride"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs index c485129eee0..b108b409c92 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs @@ -8,13 +8,15 @@ sealed class OpenTelemetryFeature : Feature { protected override void Setup(FeatureConfigurationContext context) { + var instrumentationOptions = context.Settings.GetOrDefault() ?? new InstrumentationOptions(); + context.Pipeline.Register( - new OpenTelemetryPublishBehavior(), + new OpenTelemetryPublishBehavior(instrumentationOptions), "Manages the depth of the trace for publishes" ); context.Pipeline.Register( - new OpenTelemetrySendBehavior(), + new OpenTelemetrySendBehavior(instrumentationOptions), "Manages the depth of the trace for sends" ); diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryPublishBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryPublishBehavior.cs index 55617620d7f..367647a0148 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryPublishBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryPublishBehavior.cs @@ -6,22 +6,17 @@ namespace NServiceBus; using System.Threading.Tasks; using Pipeline; -class OpenTelemetryPublishBehavior : IBehavior +class OpenTelemetryPublishBehavior(InstrumentationOptions instrumentationOptions) : IBehavior { public Task Invoke(IOutgoingPublishContext context, Func next) { - // publishes always start a new trace on receive - context.Headers[Headers.StartNewTrace] = bool.TrueString; + // the per-message override wins over the endpoint-level default + var connector = context.Extensions.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode requestedConnector) + ? requestedConnector + : instrumentationOptions.PublishTraceMode; - // unless the user explicitly requests to continue the trace - bool continueTraceWasSet = context.Extensions.TryGet(ContinueTraceOnReceive, out var continueTraceRequested); - if (continueTraceWasSet && continueTraceRequested) - { - context.Headers[Headers.StartNewTrace] = bool.FalseString; - } + context.Headers[Headers.StartNewTrace] = connector == TraceMode.StartNew ? bool.TrueString : bool.FalseString; return next(context); } - - public const string ContinueTraceOnReceive = "ContinueTraceRequested"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs index b8ca6de993c..75338de54ea 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs @@ -6,22 +6,17 @@ namespace NServiceBus; using System.Threading.Tasks; using Pipeline; -class OpenTelemetrySendBehavior : IBehavior +class OpenTelemetrySendBehavior(InstrumentationOptions instrumentationOptions) : IBehavior { public Task Invoke(IOutgoingSendContext context, Func next) { - // sends never start a new trace on receive - context.Headers[Headers.StartNewTrace] = bool.FalseString; + // the per-message override wins over the endpoint-level default + var connector = context.Extensions.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode requestedConnector) + ? requestedConnector + : instrumentationOptions.SendTraceMode; - // unless the user explicitly requests to start a new trace - bool breakTraceWasSet = context.Extensions.TryGet(StartNewTraceOnReceive, out var breakTraceWasRequested); - if (breakTraceWasSet && breakTraceWasRequested) - { - context.Headers[Headers.StartNewTrace] = bool.TrueString; - } + context.Headers[Headers.StartNewTrace] = connector == TraceMode.StartNew ? bool.TrueString : bool.FalseString; return next(context); } - - public const string StartNewTraceOnReceive = "BreakTraceRequested"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/TraceMode.cs b/src/NServiceBus.Core/OpenTelemetry/TraceMode.cs new file mode 100644 index 00000000000..590258b3b71 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/TraceMode.cs @@ -0,0 +1,19 @@ +#nullable enable + +namespace NServiceBus; + +/// +/// Controls how the receive-side processing span relates to the outgoing send or publish span. +/// +public enum TraceMode +{ + /// + /// The receiving endpoint continues the trace: the processing span becomes a child of the outgoing span. + /// + ContinueExisting, + + /// + /// The receiving endpoint starts a new trace: the processing span becomes the root of a new trace with a link back to the outgoing span. + /// + StartNew +} From b7eb4692fbcae16c52c5f3b26f0950561d65dd5b Mon Sep 17 00:00:00 2001 From: "Irina Dominte(Scurtu)" Date: Thu, 23 Jul 2026 15:13:49 +0300 Subject: [PATCH 13/22] Add meter for total number of messages deduplicated via Outbox (#7864) * first stab at metric counter * Text key fix * Apply suggestion from @tmasternak * update metric name for deduplication tracking * rename meterFactory to fakeMeterFactory in unit tests * subscribe test meter listern to all available instruments to handle instruments with duplicated names caused by multiple creations of IncomingPipelineMetrics objects --------- Co-authored-by: Tomasz Masternak Co-authored-by: Tomasz Masternak --- .../MeterTests.Verify_MeterAPI.approved.txt | 1 + .../Helpers/TestingMetricListener.cs | 7 ------ ...tReceiveToPhysicalMessageConnectorTests.cs | 23 ++++++++++++++++++- .../Incoming/IncomingPipelineMetrics.cs | 23 ++++++++++++++++++- ...nsportReceiveToPhysicalMessageConnector.cs | 1 + 5 files changed, 46 insertions(+), 9 deletions(-) diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt index d97f8c61a26..8cfd1afa89c 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt @@ -22,6 +22,7 @@ "nservicebus.messaging.handler_time => Histogram, Unit: s", "nservicebus.messaging.processing_time => Histogram, Unit: s", "nservicebus.messaging.successes => Counter", + "nservicebus.outbox.duplicates => Counter", "nservicebus.recoverability.delayed => Counter", "nservicebus.recoverability.error => Counter", "nservicebus.recoverability.immediate => Counter" diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/Helpers/TestingMetricListener.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/Helpers/TestingMetricListener.cs index cbae1435e61..f288d4f4476 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/Helpers/TestingMetricListener.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/Helpers/TestingMetricListener.cs @@ -10,7 +10,6 @@ namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Metrics; class TestingMetricListener : IDisposable { readonly MeterListener meterListener; - readonly ConcurrentDictionary subscribedInstruments = new(StringComparer.Ordinal); public readonly List metrics = []; public string version = ""; public string metricsSourceName = ""; @@ -26,12 +25,6 @@ class TestingMetricListener : IDisposable return; } - var instrumentKey = $"{instrument.Meter.Name}|{instrument.Name}|{instrument.GetType().FullName}"; - if (!subscribedInstruments.TryAdd(instrumentKey, 0)) - { - return; - } - TestContext.Out.WriteLine($"Subscribing to {instrument.Meter.Name}\\{instrument.Name}"); listener.EnableMeasurementEvents(instrument); metrics.Add(instrument); diff --git a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs index 52d0fb8ad11..8f66cf3bf60 100644 --- a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs @@ -1,4 +1,4 @@ -namespace NServiceBus.Core.Tests.Reliability.Outbox; +namespace NServiceBus.Core.Tests.Reliability.Outbox; using System; using System.Collections.Generic; @@ -6,6 +6,7 @@ namespace NServiceBus.Core.Tests.Reliability.Outbox; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; +using AcceptanceTests.Core.OpenTelemetry.Metrics; using NServiceBus.Outbox; using NServiceBus.Pipeline; using NServiceBus.Routing; @@ -130,6 +131,20 @@ public async Task Should_add_outbox_span_tag_when_deduplicating() Assert.That(pipelineActivity.TagObjects.ToImmutableDictionary()["nservicebus.outbox.deduplicate-message"], Is.EqualTo(true)); } + [Test] + public async Task Should_report_deduplicated_message_metric_when_deduplicating() + { + using var metricsListener = TestingMetricListener.SetupNServiceBusMetricsListener(); + + string messageId = Guid.NewGuid().ToString(); + fakeOutbox.ExistingMessage = new OutboxMessage(messageId, Array.Empty()); + var context = CreateContext(fakeBatchPipeline, messageId); + + await Invoke(context); + + metricsListener.AssertMetric("nservicebus.outbox.duplicates", 1); + } + [Test] public async Task Should_add_batch_dispatch_events_when_sending_batched_messages() { @@ -182,6 +197,7 @@ static TestableTransportReceiveContext CreateContext(FakeBatchPipeline pipeline, }; context.Extensions.Set(new FakePipelineCache(pipeline)); + context.Extensions.Set(new IncomingPipelineMetricTags()); return context; } @@ -191,16 +207,21 @@ public void SetUp() { fakeOutbox = new FakeOutboxStorage(); fakeBatchPipeline = new FakeBatchPipeline(); + fakeMeterFactory = new TestMeterFactory(); behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc"), new InstrumentationOptions()); } + [TearDown] + public void TearDown() => fakeMeterFactory.Dispose(); + Task Invoke(ITransportReceiveContext context, Func next = null) => behavior.Invoke(context, next ?? (_ => Task.CompletedTask)); TransportReceiveToPhysicalMessageConnector behavior; FakeBatchPipeline fakeBatchPipeline; FakeOutboxStorage fakeOutbox; + TestMeterFactory fakeMeterFactory; class MyEvent; diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs index 6a063b2abee..8fd252bf161 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs @@ -1,4 +1,4 @@ -#nullable enable +#nullable enable namespace NServiceBus; @@ -22,6 +22,7 @@ class IncomingPipelineMetrics const string RecoverabilityError = "nservicebus.recoverability.error"; const string EnvelopeUnwrapping = "nservicebus.envelope.unwrapped"; const string ActiveMessages = "nservicebus.messaging.active_messages"; + const string TotalDeduplicated = "nservicebus.outbox.duplicates"; public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, string discriminator) { @@ -32,6 +33,8 @@ public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, str description: "Total number of messages fetched from the queue by the endpoint."); totalFailures = meter.CreateCounter(TotalFailures, description: "Total number of messages processed unsuccessfully by the endpoint."); + totalDeduplicated = meter.CreateCounter(TotalDeduplicated, + description: "Total number of duplicate messages detected by the Outbox."); messageHandlerTime = meter.CreateHistogram(MessageHandlerTime, "s", "The time in seconds for the execution of the business code."); criticalTime = meter.CreateHistogram(CriticalTime, "s", @@ -148,6 +151,23 @@ public void RecordFetchedMessage(IncomingPipelineMetricTags incomingPipelineMetr totalFetched.Add(1, tags); } + public void RecordDeduplicatedMessage(ITransportReceiveContext context) + { + if (!totalDeduplicated.Enabled) + { + return; + } + + var incomingPipelineMetricTags = context.Extensions.Get(); + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.EndpointDiscriminator, + MeterTags.QueueName, + MeterTags.MessageType]); + + totalDeduplicated.Add(1, tags); + } + public void RecordSuccessfulMessageHandlerTime(IInvokeHandlerContext invokeHandlerContext, TimeSpan elapsed) { if (!messageHandlerTime.Enabled) @@ -295,6 +315,7 @@ public readonly struct ActiveMessageScope(UpDownCounter? counter, TagList readonly Counter totalProcessedSuccessfully; readonly Counter totalFetched; readonly Counter totalFailures; + readonly Counter totalDeduplicated; readonly Histogram messageHandlerTime; readonly Histogram criticalTime; readonly Histogram processingTime; diff --git a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs index 96f31beecc9..6e6a3d05cf1 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs @@ -59,6 +59,7 @@ public async Task Invoke(ITransportReceiveContext context, Func Date: Wed, 29 Jul 2026 10:52:46 +0200 Subject: [PATCH 14/22] Allow changing trace continuation behavior for delayed messages (#7845) * Add tracing for recoverability pipeline using ActivityFactory * Refactor recoverability pipeline to use `Activity` fallback for tagging recoverability actions * Replace `new NoOpActivityFactory()` with `NoOpActivityFactory.Instance` in tests and implementation to enforce singleton usage. * Normalize recoverability action tags and update Activity display names for better trace clarity in recoverability pipeline. * Normalize recoverability action tags and update Activity display names to improve trace clarity in the recoverability pipeline. * Fixed approval file * Update ActivityTagsTests.Verify_ActivityTags.approved.txt * Ensure that tracemode operations for delayed messages (user operations, sagas, and recoverability) are happening before RoutingContext stage, and that StartNewTrace header is properly set before reaching that state. * Replace RecoverabilityTraceMode with the existing TraceMode enum; only set recoverability trace metadata for DelayedRetry * Remove custom recoverability action spans; keep this PR focused on delayed-message trace mode The dedicated "Recoverability" span/activity work (ActivityFactory.StartRecoverabilityActivity and its wiring/tagging) has been split out to the recoverability-action-spans branch. This PR now only concerns how trace mode is decided for delayed sends, saga timeouts, and delayed retries via InstrumentationOptions. * Simplify `InstrumentationOptions` by removing `MoveToErrorTraceMode`; refactor `ActivityFactory` constructor for improved initialization consistency * Add acceptance tests for delayed-message trace mode; fix MoveToError regression Covers the three delayed-message origins this PR configures: explicit delayed sends (SendOperationTraceMode), saga timeouts (SagaTimeoutTraceMode), and recoverability delayed retries (Recoverability.DelayedRetryTraceMode). Each area verifies the backward-compatible default (start a new linked trace) and that the new option flips it to continuing the existing trace. The delayed-send tests also cover per-message override precedence: an explicit StartNewTraceOnReceive always wins, and a ContinueExistingTraceOnReceive request cannot defeat the delayed-send backward-compatible default. Also fixes When_retrying_messages.Should_correlate_delayed_retry_with_send, which asserted the opposite (same-trace) behavior and never actually ran due to Requires.DelayedDelivery() gating - switched to the in-memory transport's native delayed delivery support so it exercises the real code path. Restores PopulateRecoverabilityTraceMetadataBehavior's MoveToError handling (dropped when DelayedRetry-only gating was introduced), which had silently broken When_incoming_message_moved_to_error_queue.Should_add_start_new_trace_header. MoveToError itself stays hardcoded to always start a new trace, matching pre-existing behavior; it was not part of this PR's configurable scope after MoveToErrorTraceMode was removed from InstrumentationOptions. * Fix trailing whitespace flagged by IDE0055 * test tweaks * Apply suggestion from @tmasternak * Update delayed message trace mode configuration in acceptance tests Replaces `TestEndpointContinuingTrace` with a unified `TestEndpoint` that dynamically configures `TraceMode` for delayed sends and saga timeouts. Simplifies `RetryingEndpoint` initialization. Removes redundant handler setup in trace tests. * Apply suggestions from code review Co-authored-by: Tomasz Masternak * fixing att: fix endpoint setups for delayed retries and fix access modifiers in otel tests saga * Simplify delayed message trace tests by consolidating scenarios and removing redundant per-message overrides * better test names --------- Co-authored-by: Irina Dominte Co-authored-by: Tomasz Masternak Co-authored-by: Tomasz Masternak --- .../Traces/When_retrying_messages.cs | 64 +++++++-- .../Traces/When_saga_requests_a_timeout.cs | 115 ++++++++++++++++ .../Traces/When_sending_a_delayed_message.cs | 124 ++++++++++++++++++ ...IApprovals.ApproveNServiceBus.approved.txt | 13 ++ .../ContextPropagationCompatibilityTests.cs | 5 +- .../ContextPropagationDefaultBehaviorTests.cs | 7 +- .../LegacyContextPropagationTests.cs | 43 +----- ...ecoverabilityTraceMetadataBehaviorTests.cs | 64 ++++++++- .../OpenTelemetry/InstrumentationOptions.cs | 43 ++++++ .../OpenTelemetry/OpenTelemetryFeature.cs | 2 +- .../OpenTelemetrySendBehavior.cs | 31 ++++- .../OpenTelemetry/Tracing/ActivityFactory.cs | 9 +- .../Tracing/ContextPropagation.cs | 13 +- ...lateRecoverabilityTraceMetadataBehavior.cs | 18 ++- .../OpenTelemetry/Tracing/obsolete_v11.cs | 9 +- ...ttachSenderRelatedInfoOnMessageBehavior.cs | 2 - .../Outgoing/RoutingToDispatchConnector.cs | 2 +- .../MessageDrivenSubscribeTerminator.cs | 2 +- .../MessageDrivenUnsubscribeTerminator.cs | 2 +- .../MigrationSubscribeTerminator.cs | 2 +- .../MigrationUnsubscribeTerminator.cs | 2 +- 21 files changed, 471 insertions(+), 101 deletions(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_saga_requests_a_timeout.cs create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_a_delayed_message.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_retrying_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_retrying_messages.cs index b0f02c3c4b5..76f76a5f3e9 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_retrying_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_retrying_messages.cs @@ -1,10 +1,11 @@ namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Traces; using System; +using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using EndpointTemplates; -using NServiceBus.AcceptanceTesting; +using AcceptanceTesting; using NUnit.Framework; public class When_retrying_messages : OpenTelemetryAcceptanceTest @@ -37,10 +38,8 @@ await Scenario.Define() } [Test] - public async Task Should_correlate_delayed_retry_with_send() + public async Task Should_start_new_trace_on_receive_by_default() { - Requires.DelayedDelivery(); - await Scenario.Define() .WithEndpoint(e => e .CustomConfig(c => c.Recoverability().Delayed(i => i.NumberOfRetries(1).TimeIncrease(TimeSpan.FromMilliseconds(1)))) @@ -48,21 +47,58 @@ await Scenario.Define() .When(s => s.SendLocal(new FailingMessage()))) .Run(); - var receiveActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); - var sendActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + var (sendRequest, firstAttempt, retryAttempt) = GetDelayedRetryActivities(); using (Assert.EnterMultipleScope()) { - Assert.That(sendActivities, Has.Count.EqualTo(1)); - Assert.That(receiveActivities, Has.Count.EqualTo(2), "the message should be processed twice due to one immediate retry"); + Assert.That(firstAttempt.TraceId, Is.EqualTo(sendRequest.TraceId), "the first attempt is part of the original send's trace"); + Assert.That(firstAttempt.ParentId, Is.EqualTo(sendRequest.Id)); + + Assert.That(retryAttempt.TraceId, Is.Not.EqualTo(sendRequest.TraceId), "a delayed retry should start a new trace on receive by default (backward compatible)"); + Assert.That(retryAttempt.ParentId, Is.Null, "the retry attempt should be a new root"); } + + var link = retryAttempt.Links.FirstOrDefault(); + Assert.That(link, Is.Not.Default, "the retry attempt should be linked back to the original send operation"); + Assert.That(link.Context.TraceId, Is.EqualTo(sendRequest.TraceId)); + } + + [Test] + public async Task Should_continue_existing_trace_on_receive_when_configured() + { + await Scenario.Define() + .WithEndpoint(e => e + .CustomConfig(c => + { + c.Recoverability().Delayed(i => i.NumberOfRetries(1).TimeIncrease(TimeSpan.FromMilliseconds(1))); + c.Tracing().Recoverability.DelayedRetryTraceMode = TraceMode.ContinueExisting; + }) + .DoNotFailOnErrorMessages() + .When(s => s.SendLocal(new FailingMessage()))) + .Run(); + + var (sendRequest, _, retryAttempt) = GetDelayedRetryActivities(); + using (Assert.EnterMultipleScope()) { - Assert.That(receiveActivities[0].ParentId, Is.EqualTo(sendActivities[0].Id), "should not change parent span"); - Assert.That(receiveActivities[1].ParentId, Is.EqualTo(sendActivities[0].Id), "should not change parent span"); + Assert.That(retryAttempt.TraceId, Is.EqualTo(sendRequest.TraceId), "a delayed retry should continue the existing trace when DelayedRetryTraceMode is set to ContinueExisting"); + Assert.That(retryAttempt.ParentId, Is.EqualTo(sendRequest.Id)); + Assert.That(retryAttempt.Links, Is.Empty); + } + } - Assert.That(sendActivities.Concat(receiveActivities).All(a => a.TraceId == sendActivities[0].TraceId), Is.True, "all activities should be part of the same trace"); + (Activity SendRequest, Activity FirstAttempt, Activity RetryAttempt) GetDelayedRetryActivities() + { + var receiveActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + var sendActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(sendActivities, Has.Count.EqualTo(1)); + Assert.That(receiveActivities, Has.Count.EqualTo(2), "the message should be processed twice due to one delayed retry"); } + + return (sendActivities[0], receiveActivities[0], receiveActivities[1]); } public class Context : ScenarioContext @@ -72,7 +108,11 @@ public class Context : ScenarioContext public class RetryingEndpoint : EndpointConfigurationBuilder { - public RetryingEndpoint() => EndpointSetup(); + public RetryingEndpoint() => + EndpointSetup(new DefaultServer + { + TransportConfiguration = new ConfigureEndpointAcceptanceTestingTransport(false, true) + }, (c, _) => { }, _ => { }); [Handler] public class Handler(Context testContext) : IHandleMessages diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_saga_requests_a_timeout.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_saga_requests_a_timeout.cs new file mode 100644 index 00000000000..d0dfc71b143 --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_saga_requests_a_timeout.cs @@ -0,0 +1,115 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Traces; + +using System; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using AcceptanceTesting; +using EndpointTemplates; +using NUnit.Framework; + +public class When_saga_requests_a_timeout : OpenTelemetryAcceptanceTest +{ + [Test] + public async Task Should_start_new_trace_on_receive_by_default() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => s.SendLocal(new StartSagaMessage { SomeId = Guid.NewGuid().ToString() }))) + .Run(); + + var (timeoutSend, timeoutReceive) = GetTimeoutActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(timeoutReceive.TraceId, Is.Not.EqualTo(timeoutSend.TraceId), "a saga timeout should start a new trace on receive by default (backward compatible)"); + Assert.That(timeoutReceive.ParentId, Is.Null, "timeout receive should be a new root"); + } + + var link = timeoutReceive.Links.FirstOrDefault(); + Assert.That(link, Is.Not.Default, "timeout receive should be linked back to the timeout send operation"); + Assert.That(link.Context.TraceId, Is.EqualTo(timeoutSend.TraceId)); + } + + [Test] + public async Task Should_continue_existing_trace_on_receive_when_configured() + { + await Scenario.Define() + .WithEndpoint(b => b.CustomConfig(c => + { + c.Tracing().DelayedDelivery.SagaTimeoutTraceMode = TraceMode.ContinueExisting; + }) + .When(s => s.SendLocal(new StartSagaMessage { SomeId = Guid.NewGuid().ToString() }))) + .Run(); + + var (timeoutSend, timeoutReceive) = GetTimeoutActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(timeoutReceive.TraceId, Is.EqualTo(timeoutSend.TraceId), "a saga timeout should continue the existing trace when SagaTimeoutTraceMode is set to ContinueExisting"); + Assert.That(timeoutReceive.ParentId, Is.EqualTo(timeoutSend.Id)); + Assert.That(timeoutReceive.Links, Is.Empty); + } + } + + (Activity TimeoutSend, Activity TimeoutReceive) GetTimeoutActivities() + { + var sendActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + var receiveActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(sendActivities, Has.Count.EqualTo(2), "start-saga send and timeout send"); + Assert.That(receiveActivities, Has.Count.EqualTo(2), "start-saga receive and timeout receive"); + } + + return (sendActivities[1], receiveActivities[1]); + } + + public class Context : ScenarioContext + { + public bool SagaMarkedComplete { get; set; } + } + + public class SagaEndpoint : EndpointConfigurationBuilder + { + public SagaEndpoint() => + EndpointSetup(new DefaultServer + { + TransportConfiguration = new ConfigureEndpointAcceptanceTestingTransport(false, true) + }, (c, _) => { }, _ => { }); + + [Saga] + public class TimeoutSaga(Context testContext) : Saga, IAmStartedByMessages, IHandleTimeouts + { + protected override void ConfigureHowToFindSaga(SagaPropertyMapper mapper) => + mapper.MapSaga(s => s.SomeId).ToMessage(m => m.SomeId); + + public Task Handle(StartSagaMessage message, IMessageHandlerContext context) + { + Data.SomeId = message.SomeId; + return RequestTimeout(context, DateTimeOffset.UtcNow.AddMilliseconds(2)); + } + + public Task Timeout(SagaTimeout state, IMessageHandlerContext context) + { + MarkAsComplete(); + testContext.SagaMarkedComplete = true; + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + + public class TimeoutSagaData : ContainSagaData + { + public virtual string SomeId { get; set; } + } + + public class StartSagaMessage : IMessage + { + public string SomeId { get; set; } + } + + public class SagaTimeout; +} diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_a_delayed_message.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_a_delayed_message.cs new file mode 100644 index 00000000000..1e0e72e17cc --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_a_delayed_message.cs @@ -0,0 +1,124 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Traces; + +using System; +using System.Linq; +using System.Threading.Tasks; +using AcceptanceTesting; +using EndpointTemplates; +using NUnit.Framework; + +public class When_sending_a_delayed_message : OpenTelemetryAcceptanceTest +{ + [Test] + public async Task Should_start_new_trace_on_receive_by_default() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => s.Send(new DelayedMessage(), DelayedSend()))) + .Run(); + + var (send, receive) = GetActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(receive.TraceId, Is.Not.EqualTo(send.TraceId), "a delayed send should start a new trace on receive by default (backward compatible)"); + Assert.That(receive.ParentId, Is.Null, "receive should be a new root"); + } + + var link = receive.Links.FirstOrDefault(); + Assert.That(link, Is.Not.Default, "receive should be linked back to the send operation"); + Assert.That(link.Context.TraceId, Is.EqualTo(send.TraceId)); + } + + [Test] + public async Task Should_continue_existing_trace_on_receive_when_configured() + { + await Scenario.Define() + .WithEndpoint(b => b.CustomConfig(c => + { + c.Tracing().DelayedDelivery.SendOperationTraceMode = TraceMode.ContinueExisting; + }) + .When(s => s.Send(new DelayedMessage(), DelayedSend()))) + .Run(); + + var (send, receive) = GetActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(receive.TraceId, Is.EqualTo(send.TraceId), "a delayed send should continue the existing trace when SendOperationTraceMode is set to ContinueExisting"); + Assert.That(receive.ParentId, Is.EqualTo(send.Id)); + Assert.That(receive.Links, Is.Empty); + } + } + + [Test] + public async Task Should_start_new_trace_by_default_no_matter_per_message_option() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => + { + var sendOptions = DelayedSend(); + sendOptions.ContinueExistingTraceOnReceive(); + return s.Send(new DelayedMessage(), sendOptions); + })) + .Run(); + + var (send, receive) = GetActivities(); + + Assert.That(receive.TraceId, Is.Not.EqualTo(send.TraceId), + "a per-message request to continue the existing trace must not defeat the backward-compatible default for delayed sends"); + } + + static SendOptions DelayedSend() + { + var sendOptions = new SendOptions(); + sendOptions.RouteToThisEndpoint(); + sendOptions.DelayDeliveryWith(TimeSpan.FromMilliseconds(1)); + return sendOptions; + } + + (System.Diagnostics.Activity Send, System.Diagnostics.Activity Receive) GetActivities() + { + var sendActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + var receiveActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(sendActivities, Has.Count.EqualTo(1), "1 message is sent as part of this test"); + Assert.That(receiveActivities, Has.Count.EqualTo(1), "1 message is received as part of this test"); + } + + return (sendActivities[0], receiveActivities[0]); + } + + public class Context : ScenarioContext + { + public bool DelayedMessageReceived { get; set; } + } + + public class TestEndpoint : EndpointConfigurationBuilder + { + public TestEndpoint() + { + var template = new DefaultServer + { + TransportConfiguration = new ConfigureEndpointAcceptanceTestingTransport(false, true) + }; + EndpointSetup(template, (c, _) => { }, metadata => { }); + } + + [Handler] + public class DelayedMessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(DelayedMessage message, IMessageHandlerContext context) + { + testContext.DelayedMessageReceived = true; + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + + public class DelayedMessage : IMessage; +} diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index 30981403a3d..59a401d4526 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -214,6 +214,12 @@ namespace NServiceBus public int MaxNumberOfRetries { get; } public System.TimeSpan TimeIncrease { get; } } + public class DelayedDeliveryInstrumentationOptions + { + public DelayedDeliveryInstrumentationOptions() { } + public NServiceBus.TraceMode SagaTimeoutTraceMode { get; set; } + public NServiceBus.TraceMode SendOperationTraceMode { get; set; } + } public static class DelayedDeliveryOptionExtensions { public static void DelayDeliveryWith(this NServiceBus.SendOptions options, System.TimeSpan delay) { } @@ -635,8 +641,10 @@ namespace NServiceBus public class InstrumentationOptions { public InstrumentationOptions() { } + public NServiceBus.DelayedDeliveryInstrumentationOptions DelayedDelivery { get; } public bool EmitMessageDispatchingEvents { get; set; } public NServiceBus.TraceMode PublishTraceMode { get; set; } + public NServiceBus.RecoverabilityInstrumentationOptions Recoverability { get; } public NServiceBus.TraceMode SendTraceMode { get; set; } public bool UseMessageDestinationInSpanNames { get; set; } } @@ -900,6 +908,11 @@ namespace NServiceBus { public static NServiceBus.RecoverabilitySettings Recoverability(this NServiceBus.EndpointConfiguration configuration) { } } + public class RecoverabilityInstrumentationOptions + { + public RecoverabilityInstrumentationOptions() { } + public NServiceBus.TraceMode DelayedRetryTraceMode { get; set; } + } public class RecoverabilitySettings : NServiceBus.Configuration.AdvancedExtensibility.ExposeSettings { public NServiceBus.RecoverabilitySettings AddUnrecoverableException(System.Type exceptionType) { } diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs index 7f4332e0cde..3dee5e145f4 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs @@ -3,7 +3,6 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; using System; using System.Collections.Generic; using System.Diagnostics; -using Extensibility; using NUnit.Framework; [TestFixture] @@ -23,7 +22,7 @@ public void ResetDistributedContextPropagator() LegacyContextPropagation.ResetUseDistributedContextPropagator(); } - delegate void Writer(Activity activity, Dictionary headers, ContextBag context); + delegate void Writer(Activity activity, Dictionary headers); delegate void Reader(Activity activity, IDictionary headers); static readonly Writer LegacyWrite = LegacyContextPropagation.PropagateContextToHeaders; @@ -48,7 +47,7 @@ static Dictionary Send(string value, Writer write) sender.AddBaggage("key", value); var headers = new Dictionary(); - write(sender, headers, new ContextBag()); + write(sender, headers); sender.Stop(); return headers; } diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs index fbe65df4b51..0984daf14fd 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs @@ -3,7 +3,6 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; using System; using System.Collections.Generic; using System.Diagnostics; -using Extensibility; using NUnit.Framework; [TestFixture] @@ -27,7 +26,7 @@ public void Default_uses_legacy_percent_encoded_baggage_format() activity.AddBaggage("serverNode", "DF 28"); var headers = new Dictionary(); - ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(activity, headers); Assert.That(headers[Headers.DiagnosticsBaggage], Is.EqualTo("serverNode=DF%2028")); } @@ -45,7 +44,7 @@ public void Default_does_not_throw_when_baggage_value_is_null() var headers = new Dictionary(); - Assert.DoesNotThrow(() => LegacyContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag())); + Assert.DoesNotThrow(() => LegacyContextPropagation.PropagateContextToHeaders(activity, headers)); Assert.That(headers[Headers.DiagnosticsBaggage], Is.EqualTo("test=")); } @@ -58,7 +57,7 @@ public void Default_round_trip_preserves_value_whitespace() outgoing.AddBaggage("key1", " leading-and-trailing "); var headers = new Dictionary(); - ContextPropagation.PropagateContextToHeaders(outgoing, headers, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(outgoing, headers); using var incoming = new Activity(ActivityNames.IncomingMessageActivityName); incoming.SetIdFormat(ActivityIdFormat.W3C); diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs index 4c62f70cd71..4e90b6b3e1b 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using Extensibility; using NUnit.Framework; [TestFixture] @@ -20,7 +19,7 @@ public void Propagate_activity_id_to_header() var headers = new Dictionary(); - ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(activity, headers); Assert.That(activity.Id, Is.EqualTo(headers[Headers.DiagnosticsTraceParent])); } @@ -30,41 +29,11 @@ public void Should_not_set_header_without_activity() { var headers = new Dictionary(); - ContextPropagation.PropagateContextToHeaders(null, headers, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(null, headers); Assert.That(headers, Is.Empty); } - [Test] - public void Should_set_start_new_trace_header_when_adding_trace_parent_header() - { - using var activity = new Activity("test"); - activity.SetIdFormat(ActivityIdFormat.W3C); - activity.Start(); - - var headers = new Dictionary(); - var contextBag = new ContextBag(); - contextBag.Set(Headers.StartNewTrace, bool.TrueString); - ContextPropagation.PropagateContextToHeaders(activity, headers, contextBag); - - using (Assert.EnterMultipleScope()) - { - Assert.That(headers.ContainsKey(Headers.StartNewTrace), Is.True, bool.TrueString); - Assert.That(bool.TrueString, Is.EqualTo(headers[Headers.StartNewTrace])); - } - } - - [Test] - public void Should_not_set_start_new_trace_header_when_no_trace_parent_header_is_added() - { - var headers = new Dictionary(); - var contextBag = new ContextBag(); - contextBag.Set(Headers.StartNewTrace, bool.TrueString); - ContextPropagation.PropagateContextToHeaders(null, headers, contextBag); - - Assert.That(headers.ContainsKey(Headers.StartNewTrace), Is.False); - } - [Test] public void Overwrites_existing_propagation_header() { @@ -77,7 +46,7 @@ public void Overwrites_existing_propagation_header() { Headers.DiagnosticsTraceParent, "some existing id" } }; - ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(activity, headers); Assert.That(activity.Id, Is.EqualTo(headers[Headers.DiagnosticsTraceParent])); } @@ -95,7 +64,7 @@ public void Should_not_throw_when_baggage_value_is_null() var headers = new Dictionary(); - Assert.DoesNotThrow(() => ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag())); + Assert.DoesNotThrow(() => ContextPropagation.PropagateContextToHeaders(activity, headers)); } [TestCaseSource(nameof(TestCases))] @@ -141,7 +110,7 @@ public void Can_propagate_baggage_from_activity_to_header(ContextPropagationTest activity.AddBaggage(baggageItem.Key, baggageItem.Value); } - ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(activity, headers); var baggageHeaderSet = headers.TryGetValue(Headers.DiagnosticsBaggage, out var baggageValue); @@ -175,7 +144,7 @@ public void Can_roundtrip_baggage(ContextPropagationTestCase testCase) outgoingActivity.AddBaggage(baggageItem.Key, baggageItem.Value); } - ContextPropagation.PropagateContextToHeaders(outgoingActivity, outgoingHeaders, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(outgoingActivity, outgoingHeaders); // Simulate wire transfer var incomingHeaders = outgoingHeaders; diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/PopulateRecoverabilityTraceMetadataBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/PopulateRecoverabilityTraceMetadataBehaviorTests.cs index cee304d848d..cb8e2d5823b 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/PopulateRecoverabilityTraceMetadataBehaviorTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/PopulateRecoverabilityTraceMetadataBehaviorTests.cs @@ -12,7 +12,7 @@ public class PopulateRecoverabilityTraceMetadataBehaviorTests [Test] public async Task Should_not_write_metadata_when_trace_not_present() { - var behavior = new PopulateRecoverabilityTraceMetadataBehavior(); + var behavior = new PopulateRecoverabilityTraceMetadataBehavior(new InstrumentationOptions()); var context = new TestableRecoverabilityContext(); await behavior.Invoke(context, _ => Task.CompletedTask); @@ -25,10 +25,10 @@ public async Task Should_not_write_metadata_when_trace_not_present() } [Test] - [TestCaseSource(nameof(Actions))] + [TestCaseSource(nameof(ActionsThatWriteMetadata))] public async Task Should_write_metadata_when_trace_present(RecoverabilityAction recoverabilityAction) { - var behavior = new PopulateRecoverabilityTraceMetadataBehavior(); + var behavior = new PopulateRecoverabilityTraceMetadataBehavior(new InstrumentationOptions()); var context = new TestableRecoverabilityContext { @@ -45,9 +45,63 @@ public async Task Should_write_metadata_when_trace_present(RecoverabilityAction } } - static IEnumerable Actions() + [Test] + public async Task Should_not_write_metadata_for_immediate_retry() + { + var behavior = new PopulateRecoverabilityTraceMetadataBehavior(new InstrumentationOptions()); + + var context = new TestableRecoverabilityContext + { + Headers = { { Headers.DiagnosticsTraceParent, "traceparent" } }, + RecoverabilityAction = new ImmediateRetry() + }; + + await behavior.Invoke(context, _ => Task.CompletedTask); + + using (Assert.EnterMultipleScope()) + { + Assert.That(context.Headers, Does.Not.ContainKey(Headers.StartNewTrace)); + Assert.That(context.Metadata, Does.Not.ContainKey(Headers.StartNewTrace)); + } + } + + [Test] + public async Task Should_always_start_new_trace_for_move_to_error() + { + var behavior = new PopulateRecoverabilityTraceMetadataBehavior(new InstrumentationOptions()); + + var context = new TestableRecoverabilityContext + { + Headers = { { Headers.DiagnosticsTraceParent, "traceparent" } }, + RecoverabilityAction = new MoveToError("errorqueue") + }; + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Metadata[Headers.StartNewTrace], Is.EqualTo(bool.TrueString)); + } + + [Test] + public async Task Should_honor_delayed_retry_trace_mode() + { + var behavior = new PopulateRecoverabilityTraceMetadataBehavior(new InstrumentationOptions + { + Recoverability = { DelayedRetryTraceMode = TraceMode.ContinueExisting } + }); + + var context = new TestableRecoverabilityContext + { + Headers = { { Headers.DiagnosticsTraceParent, "traceparent" } }, + RecoverabilityAction = new DelayedRetry(TimeSpan.FromSeconds(10)) + }; + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Metadata[Headers.StartNewTrace], Is.EqualTo(bool.FalseString)); + } + + static IEnumerable ActionsThatWriteMetadata() { - yield return new ImmediateRetry(); yield return new DelayedRetry(TimeSpan.FromSeconds(10)); yield return new MoveToError("errorqueue"); } diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index dcdd85a1c93..78f80d8ead1 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -15,6 +15,19 @@ public class InstrumentationOptions /// public bool UseMessageDestinationInSpanNames { get; set; } + /// + /// Controls instrumentation of the recoverability pipeline (retries and error handling). + /// + public RecoverabilityInstrumentationOptions Recoverability { get; } = new(); + + /// + /// Controls instrumentation of explicitly delayed messages (SendOptions.DelayDeliveryWith + /// / DoNotDeliverBefore and saga timeouts. + /// Recoverability-driven delayed retries are controlled separately via + /// . + /// + public DelayedDeliveryInstrumentationOptions DelayedDelivery { get; } = new(); + /// /// Controls whether the "Start dispatching" and "Finished dispatching" activity events /// are added to the incoming message span when outgoing messages are dispatched. @@ -39,3 +52,33 @@ public class InstrumentationOptions /// public TraceMode PublishTraceMode { get; set; } = TraceMode.StartNew; } + +/// +/// Controls instrumentation of the recoverability pipeline (retries and error handling). +/// +public class RecoverabilityInstrumentationOptions +{ + /// + /// Controls how the span for a delayed retry relates to the failed + /// attempt's trace. + /// + public TraceMode DelayedRetryTraceMode { get; set; } = TraceMode.StartNew; +} + +/// +/// Controls instrumentation of delayed messages. +/// +public class DelayedDeliveryInstrumentationOptions +{ + /// + /// Controls how a delayed Send relates to the sender's trace, when requested directly + /// by application code via SendOptions.DelayDeliveryWith/DoNotDeliverBefore. + /// + public TraceMode SendOperationTraceMode { get; set; } = TraceMode.StartNew; + + /// + /// Controls how a saga timeout (Saga.RequestTimeout) relates to the trace of the + /// message that requested it. + /// + public TraceMode SagaTimeoutTraceMode { get; set; } = TraceMode.StartNew; +} diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs index b108b409c92..e207ee6ce57 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs @@ -21,7 +21,7 @@ protected override void Setup(FeatureConfigurationContext context) ); context.Pipeline.Register( - new PopulateRecoverabilityTraceMetadataBehavior(), + new PopulateRecoverabilityTraceMetadataBehavior(instrumentationOptions), "Populates the recoverability metadata" ); } diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs index 75338de54ea..ef86db89c94 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs @@ -5,17 +5,44 @@ namespace NServiceBus; using System; using System.Threading.Tasks; using Pipeline; +using Transport; class OpenTelemetrySendBehavior(InstrumentationOptions instrumentationOptions) : IBehavior { public Task Invoke(IOutgoingSendContext context, Func next) { // the per-message override wins over the endpoint-level default - var connector = context.Extensions.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode requestedConnector) + var operationTraceMode = context.Extensions.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode requestedConnector) ? requestedConnector : instrumentationOptions.SendTraceMode; - context.Headers[Headers.StartNewTrace] = connector == TraceMode.StartNew ? bool.TrueString : bool.FalseString; + if (operationTraceMode == TraceMode.StartNew) + { + context.Headers[Headers.StartNewTrace] = bool.TrueString; + } + else + { + // This is needed to ensure the trace continuation behavior is backwards compatible. + // If the message is delayed, we always start a new trace unless different behavior is explicitly configured. + var isDelayed = context.Extensions.TryGet(out var dispatchProperties) && + (dispatchProperties.DelayDeliveryWith != null || dispatchProperties.DoNotDeliverBefore != null); + + bool startNewTrace; + if (isDelayed) + { + var isSagaTimeout = context.Headers.ContainsKey(Headers.IsSagaTimeoutMessage); + var mode = isSagaTimeout + ? instrumentationOptions.DelayedDelivery.SagaTimeoutTraceMode + : instrumentationOptions.DelayedDelivery.SendOperationTraceMode; + startNewTrace = mode == TraceMode.StartNew; + } + else + { + startNewTrace = false; + } + + context.Headers[Headers.StartNewTrace] = startNewTrace ? bool.TrueString : bool.FalseString; + } return next(context); } diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index d5dcf4ec07a..bca611216d9 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -6,14 +6,9 @@ namespace NServiceBus; using Pipeline; using Transport; -sealed class ActivityFactory : IActivityFactory +sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory { - public ActivityFactory(InstrumentationOptions options) - { - Options = options; - } - - public InstrumentationOptions Options { get; } + public InstrumentationOptions Options { get; } = options; public Activity? StartIncomingPipelineActivity(MessageContext context) { diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs index 8df14752012..dc0e568dd87 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs @@ -4,17 +4,16 @@ namespace NServiceBus; using System.Collections.Generic; using System.Diagnostics; -using Extensibility; static class ContextPropagation { - public static void PropagateContextToHeaders(Activity? activity, Dictionary headers, ContextBag contextBag) + public static void PropagateContextToHeaders(Activity? activity, Dictionary headers) { // TODO: investigate if we need to improve the switch check for better performance // Removed in v11, see obsolete_v11.cs if (!LegacyContextPropagation.UseDistributedContextPropagator) { - LegacyContextPropagation.PropagateContextToHeaders(activity, headers, contextBag); + LegacyContextPropagation.PropagateContextToHeaders(activity, headers); return; } @@ -26,14 +25,6 @@ public static void PropagateContextToHeaders(Activity? activity, Dictionary(Headers.StartNewTrace, out var startNewTrace); - - if (traceParentExists && startNewTraceOnReceive) - { - headers[Headers.StartNewTrace] = startNewTrace!; - } } public static void PropagateContextFromHeaders(Activity? activity, IDictionary headers) diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/PopulateRecoverabilityTraceMetadataBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/PopulateRecoverabilityTraceMetadataBehavior.cs index 0bb256ecef2..0a1a5c50881 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/PopulateRecoverabilityTraceMetadataBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/PopulateRecoverabilityTraceMetadataBehavior.cs @@ -4,7 +4,7 @@ namespace NServiceBus; using System.Threading.Tasks; using Pipeline; -class PopulateRecoverabilityTraceMetadataBehavior : IBehavior +class PopulateRecoverabilityTraceMetadataBehavior(InstrumentationOptions instrumentationOptions) : IBehavior { public Task Invoke(IRecoverabilityContext context, Func next) { @@ -13,9 +13,19 @@ public Task Invoke(IRecoverabilityContext context, Func cachedUseDistributedContextPropagator = SwitchState.Unchecked; - public static void PropagateContextToHeaders(Activity? activity, Dictionary headers, ContextBag contextBag) + public static void PropagateContextToHeaders(Activity? activity, Dictionary headers) { if (activity is null) { @@ -79,12 +78,6 @@ public static void PropagateContextToHeaders(Activity? activity, Dictionary(Headers.StartNewTrace, out var headerContent)) - { - headers[Headers.StartNewTrace] = headerContent; - } - var baggage = string.Join(",", activity.Baggage.Select(item => $"{item.Key}={Uri.EscapeDataString(item.Value ?? string.Empty)}")); if (!string.IsNullOrEmpty(baggage)) { diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/AttachSenderRelatedInfoOnMessageBehavior.cs b/src/NServiceBus.Core/Pipeline/Outgoing/AttachSenderRelatedInfoOnMessageBehavior.cs index 3472a9355f5..7ba822fbc6e 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/AttachSenderRelatedInfoOnMessageBehavior.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/AttachSenderRelatedInfoOnMessageBehavior.cs @@ -35,12 +35,10 @@ public Task Invoke(IRoutingContext context, Func next) { var timeDelay = dispatchProperties.DelayDeliveryWith.Delay; message.Headers[Headers.DeliverAt] = DateTimeOffsetHelper.ToWireFormattedString(utcNow.Add(timeDelay)); - context.Extensions.Set(Headers.StartNewTrace, bool.TrueString); } else if (dispatchProperties.DoNotDeliverBefore != null) { message.Headers[Headers.DeliverAt] = DateTimeOffsetHelper.ToWireFormattedString(dispatchProperties.DoNotDeliverBefore.At); - context.Extensions.Set(Headers.StartNewTrace, bool.TrueString); } } } diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs b/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs index 55f62023b4b..3243dd7b150 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs @@ -33,7 +33,7 @@ public override Task Invoke(IRoutingContext context, Func Date: Wed, 29 Jul 2026 13:57:16 +0300 Subject: [PATCH 15/22] Added error.type (#7885) --- ...ctivityTagsTests.Verify_ActivityTags.approved.txt | 3 ++- .../OpenTelemetry/Tracing/ActivityExtensions.cs | 12 ++---------- .../OpenTelemetry/Tracing/ActivityTags.cs | 1 + 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt index 27d7bc2fc19..69a83e1f04c 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt @@ -38,6 +38,7 @@ "HandlerType => nservicebus.handler.handler_type", "HandlerSagaId => nservicebus.handler.saga_id", "EventTypes => nservicebus.event_types", - "CancelledTask => nservicebus.cancelled" + "CancelledTask => nservicebus.cancelled", + "ErrorType => error.type" ] } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs index 30b0bf1066f..4d7586fae2a 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs @@ -3,7 +3,6 @@ namespace NServiceBus; using System; -using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; @@ -41,15 +40,8 @@ public static void SetErrorStatus(this Activity activity, Exception ex) activity.SetStatus(ActivityStatusCode.Error, ex.Message); activity.SetTag("otel.status_code", "ERROR"); activity.SetTag("otel.status_description", ex.Message); - - - activity.AddEvent(new ActivityEvent("exception", DateTimeOffset.UtcNow, - [ - new KeyValuePair("exception.escaped", true), - new KeyValuePair("exception.type", ex.GetType()), - new KeyValuePair("exception.message", ex.Message), - new KeyValuePair("exception.stacktrace", ex.ToString()) - ])); + activity.SetTag(ActivityTags.ErrorType, ex.GetType().FullName); + activity.AddException(ex, new TagList { { "exception.escaped", true } }); if (ex is TaskCanceledException) { diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs index c7597a63f2e..fbbb2e0caad 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs @@ -41,4 +41,5 @@ static class ActivityTags public const string HandlerSagaId = "nservicebus.handler.saga_id"; public const string EventTypes = "nservicebus.event_types"; public const string CancelledTask = "nservicebus.cancelled"; + public const string ErrorType = "error.type"; } \ No newline at end of file From 7e6873bd7c990d51357207a0d47a8e865974b5cd Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Fri, 31 Jul 2026 08:44:17 +0200 Subject: [PATCH 16/22] Spans for recoverability actions (#7890) * Add tracing for recoverability pipeline using ActivityFactory * Refactor recoverability pipeline to use `Activity` fallback for tagging recoverability actions * Replace `new NoOpActivityFactory()` with `NoOpActivityFactory.Instance` in tests and implementation to enforce singleton usage. * Normalize recoverability action tags and update Activity display names for better trace clarity in recoverability pipeline. * Normalize recoverability action tags and update Activity display names to improve trace clarity in the recoverability pipeline. * Fixed approval file * Update ActivityTagsTests.Verify_ActivityTags.approved.txt * recoverabilty span should not be parenting the follow-up processing attempts. Span tags and display names update moved directly after recoverability aciton resolution * refactor: centralize recoverability span updates and add display name adjustments * fix tests * fix: correct typo in DelayedRetryOperation constant * Enable the Recoverability ActivitySource in acceptance tests; add dedicated recoverability span tests TestingActivityListener now accepts multiple source names so OpenTelemetryAcceptanceTest can subscribe to both "NServiceBus.Core" and "NServiceBus.Core.Recoverability" - previously only Main was subscribed, so recoverability spans were silently never created in any acceptance test. Adds When_recoverability_action_occurs.cs covering all four recoverability outcomes (immediate retry, delayed retry, move to error, discard): span creation, display name, and the nservicebus.recoverability_action tag, plus the UseMessageDestinationInSpanNames toggle for immediate retry. * Simplify When_recoverability_action_occurs: one endpoint, no ErrorSpy Collapsed all five scenarios onto a single RecoverabilityEndpoint, moving the per-test differences (Recoverability policy, UseMessageDestinationInSpanNames) into each test's CustomConfig callback instead of separate endpoint classes. Replaced the ErrorSpy endpoint (used only to signal test completion for the move-to-error case) with a Done() predicate that waits directly on the recoverability span being captured - the actual signal the test cares about, rather than an indirect one requiring a second endpoint and message hop. * Combine immediate/delayed/move-to-error/discard into one recoverability test One endpoint config now drives a single message through the default policy's full immediate-retry -> delayed-retry -> move-to-error cascade, plus a second message routed straight to Discard via a CustomPolicy branch that otherwise delegates to DefaultRecoverabilityPolicy.Invoke for the first message. Asserts by matching on the nservicebus.recoverability_action tag rather than activity position/count, since the exact number of immediate-retry attempts before falling through isn't a contract worth pinning down. Kept the destination-naming test separate since it needs a different Tracing() configuration that can't share the same CustomConfig block. * bug fixes and att for recoverability spans * Fix ActivitySource versioning and make sure the versiosn are checked in tests * small ActivityFactory refactor * fix: add missing newline at end of ActivityTagsTests.cs --------- Co-authored-by: Ramon Smits Co-authored-by: Irina Dominte --- .../OpenTelemetryAcceptanceTest.cs | 2 +- .../Traces/TestingActivityListener.cs | 9 +- .../When_recoverability_action_occurs.cs | 112 ++++++++++++++++++ ...TagsTests.Verify_ActivityTags.approved.txt | 18 ++- .../OpenTelemetry/ActivityTagsTests.cs | 12 +- .../RecoverabilityExecutorTests.cs | 7 +- .../RoutingToDispatchConnectorTests.cs | 20 ++-- .../Unicast/LoadHandlersConnectorTests.cs | 8 +- .../Tracing/ActivityDisplayNames.cs | 5 + .../OpenTelemetry/Tracing/ActivityFactory.cs | 110 ++++++++++++++--- .../OpenTelemetry/Tracing/ActivityNames.cs | 1 + .../OpenTelemetry/Tracing/ActivitySources.cs | 4 + .../OpenTelemetry/Tracing/ActivityTags.cs | 1 + .../OpenTelemetry/Tracing/IActivityFactory.cs | 2 + .../Tracing/NoOpActivityFactory.cs | 8 +- .../Receiving/ReceiveComponent.cs | 4 +- .../Recoverability/RecoverabilityComponent.cs | 7 +- .../RecoverabilityPipelineExecutor.cs | 70 +++++------ .../RecoverabilityRoutingConnector.cs | 2 +- 19 files changed, 314 insertions(+), 88 deletions(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/OpenTelemetryAcceptanceTest.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/OpenTelemetryAcceptanceTest.cs index 99b8b43ab54..a2748753077 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/OpenTelemetryAcceptanceTest.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/OpenTelemetryAcceptanceTest.cs @@ -9,7 +9,7 @@ public abstract class OpenTelemetryAcceptanceTest : NServiceBusAcceptanceTest protected TestingActivityListener NServiceBusActivityListener { get; private set; } [SetUp] - public void Setup() => NServiceBusActivityListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core"); + public void Setup() => NServiceBusActivityListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core", "NServiceBus.Core.Recoverability"); [TearDown] public void Cleanup() diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/TestingActivityListener.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/TestingActivityListener.cs index a5fdb5ce06a..38dc6113d83 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/TestingActivityListener.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/TestingActivityListener.cs @@ -11,19 +11,19 @@ public class TestingActivityListener : IDisposable { readonly ActivityListener activityListener; - public static TestingActivityListener SetupDiagnosticListener(string sourceName) + public static TestingActivityListener SetupDiagnosticListener(params string[] sourceNames) { - var testingListener = new TestingActivityListener(sourceName); + var testingListener = new TestingActivityListener(sourceNames); ActivitySource.AddActivityListener(testingListener.activityListener); return testingListener; } - TestingActivityListener(string sourceName = null) + TestingActivityListener(params string[] sourceNames) { activityListener = new ActivityListener { - ShouldListenTo = source => string.IsNullOrEmpty(sourceName) || source.Name == sourceName, + ShouldListenTo = source => sourceNames.Length == 0 || sourceNames.Contains(source.Name), Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllData, SampleUsingParentId = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllData }; @@ -62,4 +62,5 @@ public static List GetReceiveMessageActivities(this ConcurrentQueue GetSendMessageActivities(this ConcurrentQueue activities) => activities.Where(a => a.OperationName == "NServiceBus.Diagnostics.SendMessage").ToList(); public static List GetPublishEventActivities(this ConcurrentQueue activities) => activities.Where(a => a.OperationName == "NServiceBus.Diagnostics.PublishMessage").ToList(); public static List GetInvokedHandlerActivities(this ConcurrentQueue activities) => activities.Where(a => a.OperationName == "NServiceBus.Diagnostics.InvokeHandler").ToList(); + public static List GetRecoverabilityActivities(this ConcurrentQueue activities) => activities.Where(a => a.OperationName == "NServiceBus.Diagnostics.Recoverability").ToList(); } diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs new file mode 100644 index 00000000000..ec458ed6ba7 --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs @@ -0,0 +1,112 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Traces; + +using System; +using System.Linq; +using System.Threading.Tasks; +using AcceptanceTesting; +using AcceptanceTesting.Customization; +using EndpointTemplates; +using NUnit.Framework; + +public class When_recoverability_action_occurs : OpenTelemetryAcceptanceTest +{ + [Test] + public async Task Should_create_spans_for_all_recoverability_actions() + { + await Scenario.Define() + .WithEndpoint(b => b + .CustomConfig(c => c.Recoverability() + .Immediate(i => i.NumberOfRetries(1)) + .Delayed(i => i.NumberOfRetries(1).TimeIncrease(TimeSpan.FromMilliseconds(1))) + .CustomPolicy((cfg, errorContext) => + errorContext.Headers[Headers.EnclosedMessageTypes].Contains(nameof(DiscardMessage)) + ? RecoverabilityAction.Discard("test discard reason") + : DefaultRecoverabilityPolicy.Invoke(cfg, errorContext))) + .DoNotFailOnErrorMessages() + .When(async s => + { + await s.SendLocal(new FailingMessage()); + await s.SendLocal(new DiscardMessage()); + })) + .Done(_ => ActionTags().Contains("move_to_error") && ActionTags().Contains("discard")) + .Run(); + + var activities = NServiceBusActivityListener.CompletedActivities.GetRecoverabilityActivities(); + + var immediateRetry = activities.FirstOrDefault(a => (string)a.GetTagItem(ActivityTagName) == "immediate_retry"); + var delayedRetry = activities.FirstOrDefault(a => (string)a.GetTagItem(ActivityTagName) == "delayed_retry"); + var moveToError = activities.Single(a => (string)a.GetTagItem(ActivityTagName) == "move_to_error"); + var discard = activities.Single(a => (string)a.GetTagItem(ActivityTagName) == "discard"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(immediateRetry, Is.Not.Null, "expected at least one immediate retry span"); + Assert.That(immediateRetry.DisplayName, Is.EqualTo("immediate retry")); + + Assert.That(delayedRetry, Is.Not.Null, "expected at least one delayed retry span"); + Assert.That(delayedRetry.DisplayName, Is.EqualTo("delayed retry")); + + Assert.That(moveToError.DisplayName, Does.StartWith("move to ")); + Assert.That(discard.DisplayName, Is.EqualTo("discard")); + } + } + + [Test] + public async Task Should_include_destination_in_display_name_when_opted_in() + { + await Scenario.Define() + .WithEndpoint(b => b + .CustomConfig(c => + { + c.Recoverability().Immediate(i => i.NumberOfRetries(1)).Delayed(i => i.NumberOfRetries(0)); + c.Tracing().UseMessageDestinationInSpanNames = true; + }) + .DoNotFailOnErrorMessages() + .When(s => s.SendLocal(new FailingMessage()))) + .Done(_ => ActionTags().Contains("move_to_error")) + .Run(); + + var immediateRetry = NServiceBusActivityListener.CompletedActivities.GetRecoverabilityActivities() + .First(a => (string)a.GetTagItem(ActivityTagName) == "immediate_retry"); + + var endpointName = Conventions.EndpointNamingConvention(typeof(RecoverabilityEndpoint)); + Assert.That(immediateRetry.DisplayName, Is.EqualTo($"immediate retry {endpointName}")); + } + + string[] ActionTags() => + NServiceBusActivityListener.CompletedActivities.GetRecoverabilityActivities() + .Select(a => (string)a.GetTagItem(ActivityTagName)) + .ToArray(); + + const string ActivityTagName = "nservicebus.recoverability_action"; + + public class Context : ScenarioContext; + + public class RecoverabilityEndpoint : EndpointConfigurationBuilder + { + public RecoverabilityEndpoint() + { + var template = new DefaultServer + { + TransportConfiguration = new ConfigureEndpointAcceptanceTestingTransport(false, true) + }; + EndpointSetup(template, (c, _) => { }, metadata => { }); + } + + [Handler] + public class FailingMessageHandler : IHandleMessages + { + public Task Handle(FailingMessage message, IMessageHandlerContext context) => throw new SimulatedException("always fails"); + } + + [Handler] + public class DiscardMessageHandler : IHandleMessages + { + public Task Handle(DiscardMessage message, IMessageHandlerContext context) => throw new SimulatedException("always fails"); + } + } + + public class FailingMessage : IMessage; + + public class DiscardMessage : IMessage; +} diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt index 69a83e1f04c..c012daa39d0 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt @@ -1,6 +1,5 @@ { "Note": "Changes to activity tags should result in ActivitySource version updates", - "ActivitySourceVersion": "0.1.0", "Tags": [ "SagaId => nservicebus.saga.saga_id", "MessageId => nservicebus.message_id", @@ -39,6 +38,21 @@ "HandlerSagaId => nservicebus.handler.saga_id", "EventTypes => nservicebus.event_types", "CancelledTask => nservicebus.cancelled", - "ErrorType => error.type" + "ErrorType => error.type", + "RecoverabilityAction => nservicebus.recoverability_action" + ], + "ActivitySourceVersions": [ + { + "Name": "Main", + "Version": "0.1.0" + }, + { + "Name": "Handler", + "Version": "0.1.0" + }, + { + "Name": "Recoverability", + "Version": "0.1.0" + } ] } \ No newline at end of file diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityTagsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityTagsTests.cs index 5503c58bbef..a8adbd49f86 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityTagsTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityTagsTests.cs @@ -14,14 +14,18 @@ public void Verify_ActivityTags() var activityTags = typeof(ActivityTags) .GetFields(BindingFlags.Public | BindingFlags.Static) .Where(fi => fi.IsLiteral && !fi.IsInitOnly) - .Select(x => $"{x.Name} => {x.GetRawConstantValue()}") - .ToList(); + .Select(x => $"{x.Name} => {x.GetRawConstantValue()}"); Approver.Verify(new { Note = "Changes to activity tags should result in ActivitySource version updates", - ActivitySourceVersion = ActivitySources.Main.Version, - Tags = activityTags + Tags = activityTags, + ActivitySourceVersions = new[] + { + new { Name = nameof(ActivitySources.Main), ActivitySources.Main.Version }, + new { Name = nameof(ActivitySources.Handler), ActivitySources.Handler.Version }, + new { Name = nameof(ActivitySources.Recoverability), ActivitySources.Recoverability.Version } + } }); } } \ No newline at end of file diff --git a/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs b/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs index 1b3a92cf0a9..af8c3d265cb 100644 --- a/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs +++ b/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs @@ -67,10 +67,13 @@ static RecoverabilityPipelineExecutor CreateRecoverabilityExecutor(Testa new ServiceCollection().BuildServiceProvider(), // TODO: Does not get disposed new ThrowingPipelineCache(), new TestableMessageOperations(), - null, (_, _) => RecoverabilityAction.Discard("test"), + null, + (_, _) => RecoverabilityAction.Discard("test"), recoverabilityPipeline, new FaultMetadataExtractor([], _ => { }), - null); + null, + NoOpActivityFactory.Instance + ); return executor; } diff --git a/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs b/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs index da97e86e33f..0d8293f2340 100644 --- a/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs @@ -17,7 +17,7 @@ public class RoutingToDispatchConnectorTests [Test] public async Task Should_preserve_message_state_for_one_routing_strategy_for_allocation_reasons() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); IEnumerable operations = null; var testableRoutingContext = new TestableRoutingContext { @@ -59,7 +59,7 @@ await behavior.Invoke(testableRoutingContext, context => [Test] public async Task Should_copy_message_state_for_multiple_routing_strategies() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); List operations = null; var testableRoutingContext = new TestableRoutingContext { @@ -135,7 +135,7 @@ await behavior.Invoke(testableRoutingContext, context => [Test] public async Task Should_preserve_headers_generated_by_custom_routing_strategy() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); Dictionary headers = null; await behavior.Invoke(new TestableRoutingContext { RoutingStrategies = [new HeaderModifyingRoutingStrategy()] }, context => { @@ -153,7 +153,7 @@ public async Task Should_dispatch_immediately_if_user_requested() options.RequireImmediateDispatch(); var dispatched = false; - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var message = new OutgoingMessage("ID", [], Array.Empty()); await behavior.Invoke(new RoutingContext(message, @@ -170,7 +170,7 @@ await behavior.Invoke(new RoutingContext(message, public async Task Should_dispatch_immediately_if_not_sending_from_a_handler() { var dispatched = false; - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var message = new OutgoingMessage("ID", [], Array.Empty()); await behavior.Invoke(new RoutingContext(message, @@ -187,7 +187,7 @@ await behavior.Invoke(new RoutingContext(message, public async Task Should_not_dispatch_by_default() { var dispatched = false; - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var message = new OutgoingMessage("ID", [], Array.Empty()); await behavior.Invoke(new RoutingContext(message, @@ -203,7 +203,7 @@ await behavior.Invoke(new RoutingContext(message, [Test] public async Task Should_promote_message_headers_to_pipeline_activity() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var routingContext = new TestableRoutingContext(); routingContext.Message.Headers[Headers.ContentType] = "test content type"; // one of the headers that will be mapped to tags @@ -257,7 +257,7 @@ class MyMessage : IMessage; [Test] public async Task Should_merge_receive_properties_when_declared_by_transport() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var receiveProperties = new ReceiveProperties(new Dictionary { @@ -290,7 +290,7 @@ await behavior.Invoke(routingContext, context => [Test] public async Task Should_not_override_user_set_dispatch_property() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var receiveProperties = new ReceiveProperties(new Dictionary { @@ -324,7 +324,7 @@ await behavior.Invoke(routingContext, context => [Test] public async Task Should_preserve_user_dispatch_properties_even_with_receive_properties() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var receiveProperties = new ReceiveProperties(new Dictionary { diff --git a/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs b/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs index 42c5a2cd0d2..60b8a81cca0 100644 --- a/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs @@ -16,7 +16,7 @@ public class LoadHandlersConnectorTests [Test] public void Should_throw_when_there_are_no_registered_message_handlers() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), new NoOpActivityFactory()); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance); var context = new TestableIncomingLogicalMessageContext(); @@ -29,7 +29,7 @@ public void Should_throw_when_there_are_no_registered_message_handlers() [Test] public void Should_throw_if_ambient_transaction_is_different_from_scope_used_by_transport() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), new NoOpActivityFactory()); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance); var context = new TestableIncomingLogicalMessageContext(); @@ -49,7 +49,7 @@ public void Should_throw_if_ambient_transaction_is_different_from_scope_used_by_ [Test] public void Should_throw_if_ambient_transaction_suppressed_when_transport_uses_a_scope() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), new NoOpActivityFactory()); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance); var context = new TestableIncomingLogicalMessageContext(); @@ -77,7 +77,7 @@ public void Should_not_throw_if_ambient_scope_is_same_as_transport_scope() context.Services.AddSingleton(); context.Extensions.Set(new NoOpOutboxTransaction()); - var behavior = new LoadHandlersConnector(messageHandlerRegistry, new NoOpActivityFactory()); + var behavior = new LoadHandlersConnector(messageHandlerRegistry, NoOpActivityFactory.Instance); using (new TransactionScope(TransactionScopeAsyncFlowOption.Enabled)) { diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs index cf56402f4dd..50b874bfac3 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs @@ -10,9 +10,14 @@ static class ActivityDisplayNames public const string UnsubscribeEvent = "unsubscribe event"; public const string SendMessage = "send message"; public const string ReplyMessage = "reply"; + public const string Recoverability = "recover"; // Operation-only prefixes used when UseMessageDestinationInSpanNames is enabled internal const string ProcessOperation = "process"; internal const string PublishOperation = "publish"; internal const string SendOperation = "send"; + internal const string ImmediateRetryOperation = "immediate retry"; + internal const string DelayedRetryOperation = "delayed retry"; + internal const string MoveToErrorOperation = "move to"; + internal const string DiscardOperation = "discard"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index bca611216d9..ec6bce50648 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -2,7 +2,9 @@ namespace NServiceBus; +using System.Collections.Generic; using System.Diagnostics; +using Extensibility; using Pipeline; using Transport; @@ -10,20 +12,20 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory { public InstrumentationOptions Options { get; } = options; - public Activity? StartIncomingPipelineActivity(MessageContext context) + static Activity? CreateActivityFromIncomingMessage(ActivitySource activitySource, string activityName, Dictionary headers, string nativeMessageId, ContextBag extensions) { // CreateActivity is a no-op if there are no listeners but we are doing a fast path check // here nonetheless to avoid having to parse headers, access the extension bag, etc. - if (!ActivitySources.Main.HasListeners()) + if (!activitySource.HasListeners()) { return null; } Activity? activity; - var incomingTraceParentExists = context.Headers.TryGetValue(Headers.DiagnosticsTraceParent, out var sendSpanId); + var incomingTraceParentExists = headers.TryGetValue(Headers.DiagnosticsTraceParent, out var sendSpanId); var activityContextCreatedFromIncomingTraceParent = ActivityContext.TryParse(sendSpanId, null, out var sendSpanContext); - if (context.Extensions.TryGet(out var transportActivity)) // attach to transport span but link receive pipeline span to send pipeline span + if (extensions.TryGet(out var transportActivity)) // attach to transport span but link receive pipeline span to send pipeline span { ActivityLink[]? links = null; if (incomingTraceParentExists && sendSpanId != transportActivity.Id) @@ -34,31 +36,31 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory } } - activity = ActivitySources.Main.CreateActivity(name: ActivityNames.IncomingMessageActivityName, + activity = activitySource.CreateActivity(name: activityName, ActivityKind.Consumer, transportActivity.Context, links: links, idFormat: ActivityIdFormat.W3C); } else if (incomingTraceParentExists && activityContextCreatedFromIncomingTraceParent) // otherwise directly create child from logical send { - var isStartNewTraceHeaderAvailable = context.Headers.TryGetValue(Headers.StartNewTrace, out var shouldStartNewTrace); + var isStartNewTraceHeaderAvailable = headers.TryGetValue(Headers.StartNewTrace, out var shouldStartNewTrace); if (isStartNewTraceHeaderAvailable && shouldStartNewTrace?.Equals(bool.TrueString) is true) { // create a new trace or root activity - ActivityLink[] links = [new ActivityLink(sendSpanContext)]; + ActivityLink[] links = [new(sendSpanContext)]; //null the current activity so that the new one is created as root https://github.com/dotnet/runtime/issues/65528#issuecomment-2613486896 Activity.Current = null; - activity = ActivitySources.Main.StartActivity(name: ActivityNames.IncomingMessageActivityName, ActivityKind.Consumer, parentContext: default, tags: null, links: links); + activity = activitySource.StartActivity(name: activityName, ActivityKind.Consumer, parentContext: default, tags: null, links: links); } else { // no new trace was requested, so start a child trace ActivityContext.TryParse(sendSpanId, null, true, out var remoteParentActivityContext); - activity = ActivitySources.Main.CreateActivity(name: ActivityNames.IncomingMessageActivityName, ActivityKind.Consumer, remoteParentActivityContext); + activity = activitySource.CreateActivity(name: activityName, ActivityKind.Consumer, remoteParentActivityContext); } } - else // otherwise start new trace + else // otherwise start a new trace { // This will set Activity.Current as parent if available - activity = ActivitySources.Main.CreateActivity(name: ActivityNames.IncomingMessageActivityName, ActivityKind.Consumer); + activity = activitySource.CreateActivity(name: activityName, ActivityKind.Consumer); } if (activity is null) @@ -66,15 +68,33 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory return activity; } - ContextPropagation.PropagateContextFromHeaders(activity, context.Headers); + ContextPropagation.PropagateContextFromHeaders(activity, headers); + + activity.SetIdFormat(ActivityIdFormat.W3C); + activity.AddTag(ActivityTags.NativeMessageId, nativeMessageId); + + ActivityDecorator.PromoteHeadersToTags(activity, headers); + + return activity; + } + + public Activity? StartIncomingPipelineActivity(MessageContext context) + { + var activity = CreateActivityFromIncomingMessage( + ActivitySources.Main, + ActivityNames.IncomingMessageActivityName, + context.Headers, + context.NativeMessageId, + context.Extensions); + + if (activity is null) + { + return activity; + } activity.DisplayName = Options.UseMessageDestinationInSpanNames ? $"{ActivityDisplayNames.ProcessOperation} {context.ReceiveAddress}" : ActivityDisplayNames.ProcessMessage; - activity.SetIdFormat(ActivityIdFormat.W3C); - activity.AddTag(ActivityTags.NativeMessageId, context.NativeMessageId); - - ActivityDecorator.PromoteHeadersToTags(activity, context.Headers); activity.Start(); @@ -123,4 +143,62 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory activity.AddTag(ActivityTags.HandlerType, messageHandler.HandlerType.FullName); return activity; } + + public Activity? StartRecoverabilityActivity(ErrorContext context) + { + var activity = CreateActivityFromIncomingMessage( + ActivitySources.Recoverability, + ActivityNames.RecoverabilityActivityName, + context.Headers, + context.NativeMessageId, + context.Extensions); + + if (activity is null) + { + return activity; + } + + activity.DisplayName = ActivityDisplayNames.Recoverability; + + activity.Start(); + + return activity; + } + + public void UpdateActivityFromRecoverabilityAction(Activity activity, RecoverabilityAction recoverabilityAction, string receiveAddress) + { + if (recoverabilityAction is ImmediateRetry) + { + activity.AddTag(ActivityTags.RecoverabilityAction, "immediate_retry"); + activity.DisplayName = ActivityDisplayNames.ImmediateRetryOperation; + + if (Options.UseMessageDestinationInSpanNames) + { + activity.DisplayName += $" {receiveAddress}"; + } + } + else if (recoverabilityAction is DelayedRetry) + { + activity.AddTag(ActivityTags.RecoverabilityAction, "delayed_retry"); + activity.DisplayName = ActivityDisplayNames.DelayedRetryOperation; + + if (Options.UseMessageDestinationInSpanNames) + { + activity.DisplayName += $" {receiveAddress}"; + } + } + else if (recoverabilityAction is MoveToError moveToError) + { + activity.AddTag(ActivityTags.RecoverabilityAction, "move_to_error"); + + activity.DisplayName = Options.UseMessageDestinationInSpanNames + ? $"{ActivityDisplayNames.MoveToErrorOperation} {moveToError.ErrorQueue}" + : $"{ActivityDisplayNames.MoveToErrorOperation} error"; + } + else if (recoverabilityAction is Discard) + { + activity.AddTag(ActivityTags.RecoverabilityAction, "discard"); + activity.DisplayName = ActivityDisplayNames.DiscardOperation; + } + } } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityNames.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityNames.cs index 0399bb931bd..5472c817385 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityNames.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityNames.cs @@ -10,4 +10,5 @@ static class ActivityNames public const string SubscribeActivityName = "NServiceBus.Diagnostics.Subscribe"; public const string UnsubscribeActivityName = "NServiceBus.Diagnostics.Unsubscribe"; public const string InvokeHandlerActivityName = "NServiceBus.Diagnostics.InvokeHandler"; + public const string RecoverabilityActivityName = "NServiceBus.Diagnostics.Recoverability"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs index 939305dd0eb..b3d86aeaf63 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs @@ -13,4 +13,8 @@ static class ActivitySources public static readonly ActivitySource Handler = new("NServiceBus.Core.Handler", "0.1.0"); + + public static readonly ActivitySource Recoverability = + new("NServiceBus.Core.Recoverability", + "0.1.0"); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs index fbbb2e0caad..ff03de38003 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs @@ -42,4 +42,5 @@ static class ActivityTags public const string EventTypes = "nservicebus.event_types"; public const string CancelledTask = "nservicebus.cancelled"; public const string ErrorType = "error.type"; + public const string RecoverabilityAction = "nservicebus.recoverability_action"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs index daa6553a2eb..b7f536a7133 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs @@ -12,4 +12,6 @@ interface IActivityFactory Activity? StartIncomingPipelineActivity(MessageContext context); Activity? StartOutgoingPipelineActivity(string activityName, string displayName, IBehaviorContext outgoingContext); Activity? StartHandlerActivity(MessageHandler messageHandler); + Activity? StartRecoverabilityActivity(ErrorContext context); + void UpdateActivityFromRecoverabilityAction(Activity activity, RecoverabilityAction recoverabilityAction, string receiveAddress); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs index fe1f2f82c12..6d9c92706a9 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs @@ -8,11 +8,17 @@ namespace NServiceBus; sealed class NoOpActivityFactory : IActivityFactory { - public InstrumentationOptions Options { get; } = new InstrumentationOptions(); + NoOpActivityFactory() { } + public static readonly NoOpActivityFactory Instance = new(); + public InstrumentationOptions Options { get; } = new(); public Activity? StartIncomingPipelineActivity(MessageContext context) => null; public Activity? StartOutgoingPipelineActivity(string activityName, string displayName, IBehaviorContext outgoingContext) => null; public Activity? StartHandlerActivity(MessageHandler messageHandler) => null; + public Activity? StartRecoverabilityActivity(ErrorContext context) => null; + public void UpdateActivityFromRecoverabilityAction(Activity activity, RecoverabilityAction recoverabilityAction, string receiveAddress) + { + } } \ No newline at end of file diff --git a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs index 1ad0c78d989..4a0a5a8c1c6 100644 --- a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs +++ b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs @@ -180,7 +180,9 @@ public async Task Initialize( builder, pipelineCache, pipelineComponent, - messageOperations); + messageOperations, + activityFactory + ); await mainPump.Initialize( configuration.PushRuntimeSettings, diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityComponent.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityComponent.cs index a3be93fa090..972bcba47d2 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityComponent.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityComponent.cs @@ -74,7 +74,8 @@ public IRecoverabilityPipelineExecutor CreateRecoverabilityPipelineExecutor( IServiceProvider serviceProvider, IPipelineCache pipelineCache, PipelineComponent pipeline, - MessageOperations messageOperations) + MessageOperations messageOperations, + IActivityFactory activityFactory) { ArgumentNullException.ThrowIfNull(recoverabilityConfig); ArgumentNullException.ThrowIfNull(faultMetadataExtractor); @@ -103,7 +104,9 @@ public IRecoverabilityPipelineExecutor CreateRecoverabilityPipelineExecutor( }, recoverabilityPipeline, faultMetadataExtractor, - (this, policy)); + (this, policy), + activityFactory + ); } public IRecoverabilityPipelineExecutor CreateSatelliteRecoverabilityExecutor( diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs index 70f9c4ffb63..88373e89027 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs @@ -9,60 +9,50 @@ namespace NServiceBus; using NServiceBus.Pipeline; using Transport; -class RecoverabilityPipelineExecutor : IRecoverabilityPipelineExecutor +class RecoverabilityPipelineExecutor( + IServiceProvider serviceProvider, + IPipelineCache pipelineCache, + MessageOperations messageOperations, + RecoverabilityConfig recoverabilityConfig, + Func recoverabilityPolicy, + IPipeline recoverabilityPipeline, + FaultMetadataExtractor faultMetadataExtractor, + TState state, + IActivityFactory activityFactory) : IRecoverabilityPipelineExecutor { - public RecoverabilityPipelineExecutor( - IServiceProvider serviceProvider, - IPipelineCache pipelineCache, - MessageOperations messageOperations, - RecoverabilityConfig recoverabilityConfig, - Func recoverabilityPolicy, - IPipeline recoverabilityPipeline, - FaultMetadataExtractor faultMetadataExtractor, - TState state) - { - this.state = state; - this.serviceProvider = serviceProvider; - this.pipelineCache = pipelineCache; - this.messageOperations = messageOperations; - this.recoverabilityConfig = recoverabilityConfig; - this.recoverabilityPolicy = recoverabilityPolicy; - this.recoverabilityPipeline = recoverabilityPipeline; - this.faultMetadataExtractor = faultMetadataExtractor; - } - public async Task Invoke(ErrorContext errorContext, CancellationToken cancellationToken = default) { var childScope = serviceProvider.CreateAsyncScope(); await using (childScope.ConfigureAwait(false)) { - var recoverabilityAction = recoverabilityPolicy(errorContext, state); + RecoverabilityAction? recoverabilityAction; + + using (var activity = activityFactory.StartRecoverabilityActivity(errorContext)) + { + recoverabilityAction = recoverabilityPolicy(errorContext, state); + + if (activity is not null) + { + activityFactory.UpdateActivityFromRecoverabilityAction(activity, recoverabilityAction, errorContext.ReceiveAddress); + } + } var metadata = faultMetadataExtractor.Extract(errorContext); var recoverabilityContext = new RecoverabilityContext( - childScope.ServiceProvider, - messageOperations, - pipelineCache, - errorContext, - recoverabilityConfig, - metadata, - recoverabilityAction, - errorContext.Extensions, - cancellationToken); + childScope.ServiceProvider, + messageOperations, + pipelineCache, + errorContext, + recoverabilityConfig, + metadata, + recoverabilityAction, + errorContext.Extensions, + cancellationToken); await recoverabilityPipeline.Invoke(recoverabilityContext).ConfigureAwait(false); return recoverabilityContext.RecoverabilityAction.ErrorHandleResult; } } - - readonly IServiceProvider serviceProvider; - readonly IPipelineCache pipelineCache; - readonly MessageOperations messageOperations; - readonly RecoverabilityConfig recoverabilityConfig; - readonly Func recoverabilityPolicy; - readonly IPipeline recoverabilityPipeline; - readonly FaultMetadataExtractor faultMetadataExtractor; - readonly TState state; } \ No newline at end of file diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityRoutingConnector.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityRoutingConnector.cs index b81ad021265..62f10fb27c2 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityRoutingConnector.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityRoutingConnector.cs @@ -56,4 +56,4 @@ public override async Task Invoke(IRecoverabilityContext context, Func Date: Wed, 5 Aug 2026 13:29:20 +0200 Subject: [PATCH 17/22] Add option for exception details capturing via logging (#7899) * Introduce log based exception recording mode * in the dup exception capturing mode the exception deatils are captured only on the inner most span. * Rename ExceptionRecordingMode.Dup to SpanAndLogs for clarity "Dup" didn't convey what the mode actually does; SpanAndLogs states it directly. * Relocate legacy exception tags into obsolete_v11 with removal notes otel.status_code/otel.status_description predate Activity.SetStatus and exception.escaped is Deprecated by the OTel semantic conventions. Neither belongs long-term; group them in LegacyExceptionTags alongside the file's other v11-scheduled removals and document exactly what to delete then. * Default ExceptionRecordingMode from OTEL_SEMCONV_EXCEPTION_SIGNAL_OPT_IN unless explicitly configured Lets users opt into the exceptions-as-logs model ahead of NServiceBus adopting it as the default, per the (unratified) OTel migration convention. Tracks whether the user explicitly touched ExceptionRecordingMode via a field-backed setter so an explicit setting always wins over the environment variable. Scaffolding lives in obsolete_v12.cs since the whole mechanism is meant to be removed once the migration settles. * Remove RecoverabilityAction.LogMessage, build log messages via switch in RecoverabilityPipelineExecutor Keeps the message-formatting logic out of the public RecoverabilityAction API surface; RecoverabilityPipelineExecutor is the only consumer, so it can own the mapping from action type to log message directly. * Migrate recoverability logging from static ILog to DI-resolved MEL ILogger Starts the move away from NServiceBus.Logging's static LogManager pattern in favor of Microsoft.Extensions.Logging, resolved from the DI container that's already flowing through the pipeline. Extracts all of it into a dedicated RecoverabilityActionLogger: each recoverability action type gets its own ILogger (ImmediateRetry, DelayedRetry, MoveToError, Discard, falling back to ILogger for anything else), so hosts can filter/level recoverability logs per action type. Message templates are literal per LoggerMessage-generated method to satisfy CA2254 rather than building a variable string, which also gives cheap structured logging for free. RecoverabilityPipelineExecutor now just delegates to RecoverabilityActionLogger.LogRecoverabilityAction in one line instead of owning the switch/DI-resolution logic itself. * Give OTEL_SEMCONV_EXCEPTION_SIGNAL_OPT_IN precedence over explicit ExceptionRecordingMode configuration Lets operators force the exception-signal behavior via environment variable without a code change/redeploy, overriding whatever the endpoint explicitly configured. Drops ExceptionRecordingModeSetByUser, which is no longer needed now that precedence doesn't depend on whether the user touched the setting. * fixing att test after minor changes to the recoverability action log messages --- .../Traces/When_processing_fails.cs | 7 ++ .../When_immediate_retries_are_enabled.cs | 2 +- .../When_message_fails_retries.cs | 2 +- ...IApprovals.ApproveNServiceBus.approved.txt | 6 ++ .../OpenTelemetryExtensionsTests.cs | 76 +++++++++++++++++++ .../OpenTelemetry/TracingExtensionsTests.cs | 23 +++++- .../RecoverabilityExecutorTests.cs | 3 +- .../OpenTelemetry/ExceptionRecordingMode.cs | 20 +++++ .../OpenTelemetry/InstrumentationOptions.cs | 8 +- .../OpenTelemetry/OpenTelemetryFeature.cs | 2 + .../Tracing/ActivityExtensions.cs | 16 ---- .../OpenTelemetry/Tracing/ActivityFactory.cs | 33 ++++++++ .../OpenTelemetry/Tracing/IActivityFactory.cs | 3 + .../Tracing/NoOpActivityFactory.cs | 6 ++ .../Tracing/RecordedExceptions.cs | 19 +++++ .../Tracing/TracingExtensions.cs | 8 +- .../OpenTelemetry/Tracing/obsolete_v11.cs | 29 +++++++ .../OpenTelemetry/Tracing/obsolete_v12.cs | 39 ++++++++++ .../Incoming/LoadHandlersConnector.cs | 5 +- .../Pipeline/MainPipelineExecutor.cs | 2 +- .../Recoverability/DelayedRetry.cs | 5 -- .../Recoverability/Discard.cs | 9 +-- .../Recoverability/ImmediateRetry.cs | 6 +- .../Recoverability/MoveToError.cs | 5 -- .../Recoverability/RecoverabilityAction.cs | 2 +- .../RecoverabilityActionLogger.cs | 65 ++++++++++++++++ .../RecoverabilityPipelineExecutor.cs | 10 ++- .../Transports/MessageContext.cs | 1 + .../Unicast/MessageOperations.cs | 10 +-- 29 files changed, 360 insertions(+), 62 deletions(-) create mode 100644 src/NServiceBus.Core/OpenTelemetry/ExceptionRecordingMode.cs create mode 100644 src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs create mode 100644 src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs create mode 100644 src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails.cs index a752b82ef29..52ab5ba660e 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails.cs @@ -44,6 +44,13 @@ public async Task Should_mark_span_as_failed() handlerActivityTags.VerifyTag("otel.status_code", "ERROR"); handlerActivityTags.VerifyTag("otel.status_description", ErrorMessage); + using (Assert.EnterMultipleScope()) + { + Assert.That(failedHandlerActivity.Events, Has.Exactly(1).Items, + "the innermost span (the handler invocation) should record the exception details"); + Assert.That(failedPipelineActivity.Events, Is.Empty, + "the outer span should not duplicate the exception details already recorded on the inner span"); + } } public class Context : ScenarioContext; diff --git a/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs b/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs index 51002130852..7f01e523cd3 100644 --- a/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs +++ b/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs @@ -23,7 +23,7 @@ public async Task Should_do_the_configured_number_of_retries() Assert.That(context.ForwardedToErrorQueue, Is.True); Assert.That(context.NumberOfTimesInvoked, Is.EqualTo(numberOfRetries + 1), "Message should be retried 5 times immediately"); Assert.That(context.Logs.Count(l => l.Message - .StartsWith($"Immediate Retry is going to retry message '{context.MessageId}' because of an exception:")), Is.EqualTo(numberOfRetries)); + .StartsWith($"Immediate Retry is going to retry message '{context.MessageId}' because of an exception.")), Is.EqualTo(numberOfRetries)); } } diff --git a/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs b/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs index 8177397224a..ba014c87f9b 100644 --- a/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs +++ b/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs @@ -28,7 +28,7 @@ public void Should_forward_message_to_error_queue() } Assert.That(testContext.Logs.Count(l => l.Message - .StartsWith($"Moving message '{testContext.PhysicalMessageId}' to the error queue 'error' because processing failed due to an exception:")), Is.EqualTo(1)); + .StartsWith($"Moving message '{testContext.PhysicalMessageId}' to the error queue 'error' because processing failed due to an exception.")), Is.EqualTo(1)); } public class RetryEndpoint : EndpointConfigurationBuilder diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index 59a401d4526..0c7e19fe1f1 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -331,6 +331,11 @@ namespace NServiceBus public bool TryGetExplicitlyConfiguredErrorQueueAddress([System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out string? errorQueue) { } } } + public enum ExceptionRecordingMode + { + Logs = 0, + SpanAndLogs = 1, + } public class FailedConfig { public FailedConfig(string errorQueue, System.Collections.Generic.HashSet unrecoverableExceptionTypes) { } @@ -643,6 +648,7 @@ namespace NServiceBus public InstrumentationOptions() { } public NServiceBus.DelayedDeliveryInstrumentationOptions DelayedDelivery { get; } public bool EmitMessageDispatchingEvents { get; set; } + public NServiceBus.ExceptionRecordingMode ExceptionRecordingMode { get; set; } public NServiceBus.TraceMode PublishTraceMode { get; set; } public NServiceBus.RecoverabilityInstrumentationOptions Recoverability { get; } public NServiceBus.TraceMode SendTraceMode { get; set; } diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs index 14752802842..7428d6e660f 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs @@ -1,6 +1,8 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; +using System.Collections.Generic; using NUnit.Framework; +using Settings; [TestFixture] public class OpenTelemetryExtensionsTests @@ -60,4 +62,78 @@ public void Last_override_call_wins() Assert.That(options.Context.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode connector), Is.True); Assert.That(connector, Is.EqualTo(TraceMode.StartNew)); } + + [Test] + public void Defaults_to_span_and_logs_when_opt_in_environment_variable_is_not_set() + { + var settingsHolder = new SettingsHolder(); + settingsHolder.Set(new FakeEnvironment { ValueToReturn = [] }); + + InstrumentationOptions.SetExceptionRecordingModeDefault(settingsHolder); + + Assert.That(settingsHolder.Get().ExceptionRecordingMode, Is.EqualTo(ExceptionRecordingMode.SpanAndLogs)); + } + + [Test] + public void Uses_logs_only_when_opt_in_environment_variable_is_logs() + { + var settingsHolder = new SettingsHolder(); + settingsHolder.Set(new FakeEnvironment + { + ValueToReturn = new Dictionary { { InstrumentationOptions.ExceptionSignalOptInEnvironmentVariableKey, "logs" } } + }); + + InstrumentationOptions.SetExceptionRecordingModeDefault(settingsHolder); + + Assert.That(settingsHolder.Get().ExceptionRecordingMode, Is.EqualTo(ExceptionRecordingMode.Logs)); + } + + [Test] + public void Uses_span_and_logs_when_opt_in_environment_variable_is_logs_dup() + { + var settingsHolder = new SettingsHolder(); + settingsHolder.Set(new FakeEnvironment + { + ValueToReturn = new Dictionary { { InstrumentationOptions.ExceptionSignalOptInEnvironmentVariableKey, "logs/dup" } } + }); + + InstrumentationOptions.SetExceptionRecordingModeDefault(settingsHolder); + + Assert.That(settingsHolder.Get().ExceptionRecordingMode, Is.EqualTo(ExceptionRecordingMode.SpanAndLogs)); + } + + [Test] + public void Environment_variable_takes_precedence_over_explicit_configuration() + { + var settingsHolder = new SettingsHolder(); + settingsHolder.Set(new FakeEnvironment + { + ValueToReturn = new Dictionary { { InstrumentationOptions.ExceptionSignalOptInEnvironmentVariableKey, "logs" } } + }); + + // explicitly configured to something other than what the environment variable resolves to + settingsHolder.Set(new InstrumentationOptions { ExceptionRecordingMode = ExceptionRecordingMode.SpanAndLogs }); + InstrumentationOptions.SetExceptionRecordingModeDefault(settingsHolder); + + Assert.That(settingsHolder.Get().ExceptionRecordingMode, Is.EqualTo(ExceptionRecordingMode.Logs)); + } + + [Test] + public void Explicit_configuration_is_preserved_when_environment_variable_is_not_set() + { + var settingsHolder = new SettingsHolder(); + settingsHolder.Set(new FakeEnvironment { ValueToReturn = [] }); + + settingsHolder.Set(new InstrumentationOptions { ExceptionRecordingMode = ExceptionRecordingMode.Logs }); + InstrumentationOptions.SetExceptionRecordingModeDefault(settingsHolder); + + Assert.That(settingsHolder.Get().ExceptionRecordingMode, Is.EqualTo(ExceptionRecordingMode.Logs)); + } + + class FakeEnvironment : SystemEnvironment + { + public Dictionary ValueToReturn { get; set; } + + public override string GetEnvironmentVariable(string variable) => ValueToReturn.GetValueOrDefault(variable); + } } diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/TracingExtensionsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/TracingExtensionsTests.cs index 64c1947dbf5..4b6473de6f5 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/TracingExtensionsTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/TracingExtensionsTests.cs @@ -21,7 +21,7 @@ public async Task Invoke_should_invoke_pipeline_when_activity_null() return Task.CompletedTask; }); - await pipeline.Invoke(new FakeRootContext(), null); + await pipeline.Invoke(new FakeRootContext(), null, new ActivityFactory(new InstrumentationOptions())); Assert.That(invokedPipeline, Is.True); } @@ -33,7 +33,7 @@ public async Task Invoke_should_set_success_status_when_no_exception() using var activity = new Activity("test activity"); activity.Start(); - await pipeline.Invoke(new FakeRootContext(), activity); + await pipeline.Invoke(new FakeRootContext(), activity, new ActivityFactory(new InstrumentationOptions())); Assert.That(activity.Status, Is.EqualTo(ActivityStatusCode.Ok)); } @@ -46,7 +46,7 @@ public void Invoke_should_set_error_status_and_tags_when_exception() using var activity = new Activity("test activity"); activity.Start(); - Assert.ThrowsAsync(() => pipeline.Invoke(new FakeRootContext(), activity)); + Assert.ThrowsAsync(() => pipeline.Invoke(new FakeRootContext(), activity, new ActivityFactory(new InstrumentationOptions()))); Assert.That(activity.Status, Is.EqualTo(ActivityStatusCode.Error)); @@ -61,6 +61,23 @@ public void Invoke_should_set_error_status_and_tags_when_exception() Assert.That(errorEvent.Name, Is.EqualTo("exception")); } + [Test] + public void Invoke_should_set_error_status_without_exception_event_when_log_mode() + { + var exception = new Exception("test exception"); + var pipeline = new FakePipeline(() => throw exception); + using var activity = new Activity("test activity"); + activity.Start(); + + Assert.ThrowsAsync(() => pipeline.Invoke(new FakeRootContext(), activity, new ActivityFactory(new InstrumentationOptions { ExceptionRecordingMode = ExceptionRecordingMode.Logs }))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(activity.Status, Is.EqualTo(ActivityStatusCode.Error)); + Assert.That(activity.Events, Is.Empty, "no exception event should be added when recording via the log instead"); + } + } + class FakePipeline : IPipeline { readonly Func pipelineAction; diff --git a/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs b/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs index af8c3d265cb..139a3ff7185 100644 --- a/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs +++ b/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs @@ -1,7 +1,6 @@ namespace NServiceBus.Core.Tests.Recoverability; using System; -using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Pipeline; @@ -64,7 +63,7 @@ public async Task Should_use_error_context_extensions_as_extensions_root() static RecoverabilityPipelineExecutor CreateRecoverabilityExecutor(TestableMessageOperations.Pipeline recoverabilityPipeline) { var executor = new RecoverabilityPipelineExecutor( - new ServiceCollection().BuildServiceProvider(), // TODO: Does not get disposed + new ServiceCollection().AddLogging().BuildServiceProvider(), // TODO: Does not get disposed new ThrowingPipelineCache(), new TestableMessageOperations(), null, diff --git a/src/NServiceBus.Core/OpenTelemetry/ExceptionRecordingMode.cs b/src/NServiceBus.Core/OpenTelemetry/ExceptionRecordingMode.cs new file mode 100644 index 00000000000..32c9da69655 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/ExceptionRecordingMode.cs @@ -0,0 +1,20 @@ +#nullable enable + +namespace NServiceBus; + +/// +/// Controls how exception details are recorded when an operation represented by an activity fails. +/// +public enum ExceptionRecordingMode +{ + /// + /// Records the exception details via NServiceBus's logging infrastructure instead of adding an event to the + /// activity. + /// + Logs, + + /// + /// Records the exception details via NServiceBus's logging infrastructure and as an event the activity. + /// + SpanAndLogs +} diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index 78f80d8ead1..7028795c9aa 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -6,7 +6,7 @@ namespace NServiceBus; /// Controls opt-in OpenTelemetry instrumentation behaviors. /// Accessed via endpointConfiguration.Tracing(). /// -public class InstrumentationOptions +public partial class InstrumentationOptions { /// /// Appends the destination to span names following the OTel messaging convention @@ -51,6 +51,12 @@ public class InstrumentationOptions /// or . /// public TraceMode PublishTraceMode { get; set; } = TraceMode.StartNew; + + /// + /// Controls how exception details are recorded when an operation fails. + /// Defaults to : exceptions are recorded as an event on the activity. + /// + public ExceptionRecordingMode ExceptionRecordingMode { get; set; } = ExceptionRecordingMode.SpanAndLogs; } /// diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs index e207ee6ce57..7ae9288c9a7 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs @@ -6,6 +6,8 @@ namespace NServiceBus; sealed class OpenTelemetryFeature : Feature { + public OpenTelemetryFeature() => Defaults(InstrumentationOptions.SetExceptionRecordingModeDefault); + protected override void Setup(FeatureConfigurationContext context) { var instrumentationOptions = context.Settings.GetOrDefault() ?? new InstrumentationOptions(); diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs index 4d7586fae2a..3f784962a9b 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs @@ -2,10 +2,8 @@ namespace NServiceBus; -using System; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; -using System.Threading.Tasks; using Extensibility; static class ActivityExtensions @@ -34,18 +32,4 @@ static bool TryGetRecordingPipelineActivity(this ContextBag pipelineContext, str public static void SetOutgoingPipelineActivity(this ContextBag pipelineContext, Activity activity) => pipelineContext.Set(OutgoingActivityKey, activity); public static void SetIncomingPipelineActivity(this ContextBag pipelineContext, Activity activity) => pipelineContext.Set(IncomingActivityKey, activity); - - public static void SetErrorStatus(this Activity activity, Exception ex) - { - activity.SetStatus(ActivityStatusCode.Error, ex.Message); - activity.SetTag("otel.status_code", "ERROR"); - activity.SetTag("otel.status_description", ex.Message); - activity.SetTag(ActivityTags.ErrorType, ex.GetType().FullName); - activity.AddException(ex, new TagList { { "exception.escaped", true } }); - - if (ex is TaskCanceledException) - { - activity.SetTag(ActivityTags.CancelledTask, true); - } - } } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index ec6bce50648..9c407ce77b5 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -2,9 +2,12 @@ namespace NServiceBus; +using System; using System.Collections.Generic; using System.Diagnostics; +using System.Threading.Tasks; using Extensibility; +using Logging; using Pipeline; using Transport; @@ -201,4 +204,34 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi activity.DisplayName = ActivityDisplayNames.DiscardOperation; } } + + public void RecordError(Activity activity, Exception exception, ContextBag context) + { + activity.SetStatus(ActivityStatusCode.Error, exception.Message); + activity.SetTag(ActivityTags.ErrorType, exception.GetType().FullName); + + LegacyExceptionTags.SetLegacyStatusTags(activity, exception); + + var recordedExceptions = context.GetOrCreate(); + if (!recordedExceptions.HasBeenRecorded(exception)) + { + if (Options.ExceptionRecordingMode == ExceptionRecordingMode.Logs) + { + Logger.Error($"An exception occurred while executing '{activity.DisplayName}'.", exception); + } + else + { + activity.AddException(exception, LegacyExceptionTags.EscapedTagList); + } + + recordedExceptions.MarkAsRecorded(exception); + } + + if (exception is TaskCanceledException) + { + activity.SetTag(ActivityTags.CancelledTask, true); + } + } + + static readonly ILog Logger = LogManager.GetLogger(); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs index b7f536a7133..bccdf87faf6 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs @@ -2,7 +2,9 @@ namespace NServiceBus; +using System; using System.Diagnostics; +using Extensibility; using Pipeline; using Transport; @@ -14,4 +16,5 @@ interface IActivityFactory Activity? StartHandlerActivity(MessageHandler messageHandler); Activity? StartRecoverabilityActivity(ErrorContext context); void UpdateActivityFromRecoverabilityAction(Activity activity, RecoverabilityAction recoverabilityAction, string receiveAddress); + void RecordError(Activity activity, Exception exception, ContextBag context); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs index 6d9c92706a9..6e2e183c637 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs @@ -2,7 +2,9 @@ namespace NServiceBus; +using System; using System.Diagnostics; +using Extensibility; using Pipeline; using Transport; @@ -21,4 +23,8 @@ sealed class NoOpActivityFactory : IActivityFactory public void UpdateActivityFromRecoverabilityAction(Activity activity, RecoverabilityAction recoverabilityAction, string receiveAddress) { } + + public void RecordError(Activity activity, Exception exception, ContextBag context) + { + } } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs new file mode 100644 index 00000000000..4ecfafb8152 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs @@ -0,0 +1,19 @@ +#nullable enable + +namespace NServiceBus; + +using System; +using System.Collections.Generic; + +sealed class RecordedExceptions +{ + // Reference equality is intentional: this tracks specific exception instances as they + // propagate, not exceptions that merely look alike. +#pragma warning disable PS0025 + readonly HashSet recorded = new(ReferenceEqualityComparer.Instance); +#pragma warning restore PS0025 + + public bool HasBeenRecorded(Exception exception) => recorded.Contains(exception); + + public void MarkAsRecorded(Exception exception) => recorded.Add(exception); +} diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/TracingExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/TracingExtensions.cs index ad8557b98a4..afdad2c908a 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/TracingExtensions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/TracingExtensions.cs @@ -9,11 +9,11 @@ namespace NServiceBus; static class TracingExtensions { - public static Task Invoke(this IPipeline pipeline, TContext context, Activity? activity) where TContext : IBehaviorContext + public static Task Invoke(this IPipeline pipeline, TContext context, Activity? activity, IActivityFactory activityFactory) where TContext : IBehaviorContext { - return activity is null ? pipeline.Invoke(context) : TracePipelineStatus(pipeline, context, activity); + return activity is null ? pipeline.Invoke(context) : TracePipelineStatus(pipeline, context, activity, activityFactory); - static async Task TracePipelineStatus(IPipeline pipeline, TContext context, Activity activity) + static async Task TracePipelineStatus(IPipeline pipeline, TContext context, Activity activity, IActivityFactory activityFactory) { #pragma warning disable PS0019 // When catching System.Exception, cancellation needs to be properly accounted for try @@ -23,7 +23,7 @@ static async Task TracePipelineStatus(IPipeline pipeline, TContext con } catch (Exception ex) { - activity.SetErrorStatus(ex); + activityFactory.RecordError(activity, ex, context.Extensions); throw; } #pragma warning restore PS0019 // When catching System.Exception, cancellation needs to be properly accounted for diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs index 74dac3aadbe..0e8dd5f4d4f 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs @@ -179,4 +179,33 @@ public static bool UseHandlerActivitySource } internal static void ResetUseHandlerActivitySource() => cachedUseHandlerActivitySource = SwitchState.Unchecked; +} + + +// This class bridges two independent legacy exception-tagging behaviors, both +// scheduled for removal in v11: +// +// - SetLegacyStatusTags sets the "otel.status_code"/"otel.status_description" +// tags, which predate native support for Activity.SetStatus/Activity.Status +// and are now redundant with it. Kept only for consumers still reading the +// tags directly instead of Activity.Status. +// - EscapedTagList carries "exception.escaped", an attribute the OTel semantic +// conventions have marked Deprecated: +// https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/ +// It's added to the exception event for backward compatibility with +// existing consumers of that attribute. +// +// In v11, delete this entire class, remove the +// `LegacyExceptionTags.SetLegacyStatusTags(activity, exception);` call in +// ActivityFactory.RecordError, and stop passing EscapedTagList to +// activity.AddException in the same method. +static class LegacyExceptionTags +{ + public static void SetLegacyStatusTags(Activity activity, Exception exception) + { + activity.SetTag("otel.status_code", "ERROR"); + activity.SetTag("otel.status_description", exception.Message); + } + + public static TagList EscapedTagList { get; } = new() { { "exception.escaped", true } }; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs new file mode 100644 index 00000000000..13e193f8957 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs @@ -0,0 +1,39 @@ +#nullable enable + +namespace NServiceBus; + +using Settings; + +// This scaffolds a temporary, environment-variable-driven override for +// ExceptionRecordingMode so users can opt in ahead of time to the "logs" +// model described by the OTEL_SEMCONV_EXCEPTION_SIGNAL_OPT_IN convention: +// https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/ +// The environment variable is the highest-priority signal for this setting: +// when it's set to a recognized value it always wins, even over an +// explicitly configured ExceptionRecordingMode, so operators can force the +// exception-signal behavior without a code change/redeploy. +// +// In v12, once the exceptions-as-logs migration has settled, delete this +// entire file (SetExceptionRecordingModeDefault and +// ExceptionSignalOptInEnvironmentVariableKey), and remove the +// `Defaults(InstrumentationOptions.SetExceptionRecordingModeDefault);` call in +// OpenTelemetryFeature's constructor. +public partial class InstrumentationOptions +{ + internal static void SetExceptionRecordingModeDefault(SettingsHolder settings) + { + var options = settings.GetOrCreate(); + + var environment = settings.Get(); + var variableValue = environment.GetEnvironmentVariable(ExceptionSignalOptInEnvironmentVariableKey); + + options.ExceptionRecordingMode = variableValue switch + { + "logs" => ExceptionRecordingMode.Logs, + "logs/dup" => ExceptionRecordingMode.SpanAndLogs, + _ => options.ExceptionRecordingMode + }; + } + + internal static readonly string ExceptionSignalOptInEnvironmentVariableKey = "OTEL_SEMCONV_EXCEPTION_SIGNAL_OPT_IN"; +} diff --git a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs index 1fce445695a..e11af72114e 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs @@ -64,7 +64,10 @@ public override async Task Invoke(IIncomingLogicalMessageContext context, Func GetRoutingContexts(IRecover { var exception = context.Exception; - Logger.Warn($"Delayed Retry will reschedule message '{context.MessageId}' after a delay of {Delay} because of an exception:", exception); - var outgoingMessage = new OutgoingMessage(context.MessageId, new Dictionary(context.Headers), context.Body); var currentDelayedRetriesAttempt = context.DelayedDeliveriesPerformed + 1; @@ -73,6 +70,4 @@ public override IReadOnlyCollection GetRoutingContexts(IRecover }); return [routingContext]; } - - static readonly ILog Logger = LogManager.GetLogger(); } \ No newline at end of file diff --git a/src/NServiceBus.Core/Recoverability/Discard.cs b/src/NServiceBus.Core/Recoverability/Discard.cs index 7ecc8e0b6ea..86862add7dc 100644 --- a/src/NServiceBus.Core/Recoverability/Discard.cs +++ b/src/NServiceBus.Core/Recoverability/Discard.cs @@ -3,7 +3,6 @@ namespace NServiceBus; using System.Collections.Generic; -using Logging; using Pipeline; using Transport; @@ -28,11 +27,5 @@ public class Discard : RecoverabilityAction public override ErrorHandleResult ErrorHandleResult => ErrorHandleResult.Handled; /// - public override IReadOnlyCollection GetRoutingContexts(IRecoverabilityActionContext context) - { - Logger.Info($"Discarding message with id '{context.MessageId}'. Reason: {Reason}", context.Exception); - return []; - } - - static readonly ILog Logger = LogManager.GetLogger(); + public override IReadOnlyCollection GetRoutingContexts(IRecoverabilityActionContext context) => []; } \ No newline at end of file diff --git a/src/NServiceBus.Core/Recoverability/ImmediateRetry.cs b/src/NServiceBus.Core/Recoverability/ImmediateRetry.cs index 2b43ca652eb..71b7fb9cdc9 100644 --- a/src/NServiceBus.Core/Recoverability/ImmediateRetry.cs +++ b/src/NServiceBus.Core/Recoverability/ImmediateRetry.cs @@ -4,8 +4,7 @@ namespace NServiceBus; using System; using System.Collections.Generic; -using NServiceBus.Logging; -using NServiceBus.Transport; +using Transport; using Pipeline; /// @@ -30,7 +29,6 @@ public override IReadOnlyCollection GetRoutingContexts(IRecover { var exception = context.Exception; - Logger.Info($"Immediate Retry is going to retry message '{context.MessageId}' because of an exception:", exception); if (context is IRecoverabilityActionContextNotifications notifications) { notifications.Add(new MessageToBeRetried( @@ -46,6 +44,4 @@ public override IReadOnlyCollection GetRoutingContexts(IRecover } return []; } - - static readonly ILog Logger = LogManager.GetLogger(); } \ No newline at end of file diff --git a/src/NServiceBus.Core/Recoverability/MoveToError.cs b/src/NServiceBus.Core/Recoverability/MoveToError.cs index 8c805833676..cbb148a7a46 100644 --- a/src/NServiceBus.Core/Recoverability/MoveToError.cs +++ b/src/NServiceBus.Core/Recoverability/MoveToError.cs @@ -3,7 +3,6 @@ namespace NServiceBus; using System.Collections.Generic; -using Logging; using Pipeline; using Recoverability; using Routing; @@ -35,8 +34,6 @@ public override IReadOnlyCollection GetRoutingContexts(IRecover var metadata = context.Metadata; var exception = context.Exception; - Logger.Error($"Moving message '{context.MessageId}' to the error queue '{ErrorQueue}' because processing failed due to an exception:", exception); - if (context is IRecoverabilityActionContextNotifications notifications) { notifications.Add(new MessageFaulted(ErrorQueue, context.NativeMessageId, context.MessageId, context.Headers, context.Body, context.ReceiveProperties, exception)); @@ -56,6 +53,4 @@ public override IReadOnlyCollection GetRoutingContexts(IRecover context.CreateRoutingContext(outgoingMessage, new UnicastRoutingStrategy(ErrorQueue)) ]; } - - static readonly ILog Logger = LogManager.GetLogger(); } \ No newline at end of file diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityAction.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityAction.cs index 9d6992336f3..cbdb330cde1 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityAction.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityAction.cs @@ -74,5 +74,5 @@ public static Discard Discard(string reason) /// public abstract ErrorHandleResult ErrorHandleResult { get; } - static readonly ImmediateRetry CachedImmediateRetry = new ImmediateRetry(); + static readonly ImmediateRetry CachedImmediateRetry = new(); } \ No newline at end of file diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs new file mode 100644 index 00000000000..40878f9730c --- /dev/null +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs @@ -0,0 +1,65 @@ +#nullable enable + +namespace NServiceBus; + +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Transport; + +sealed class RecoverabilityActionLogger(IServiceProvider serviceProvider) +{ + public void LogRecoverabilityAction(RecoverabilityAction recoverabilityAction, ErrorContext errorContext, ExceptionRecordingMode exceptionRecordingMode) + { + // ExceptionRecordingMode.SpanAndLogs option preserves exception details behavior from the older version of NSB + // In this mode the exception details are captured both on the span and here in the logs. + // The other option (Logs) emits details in logs. Not here but rather in the proper OTel scope i.e., in the span + // in which the exception was thrown. + var exceptionToLog = exceptionRecordingMode == ExceptionRecordingMode.SpanAndLogs + ? errorContext.Exception + : null; + + switch (recoverabilityAction) + { + case ImmediateRetry: + immediateRetryLogger.ImmediateRetryLogged(exceptionToLog, errorContext.MessageId); + break; + case DelayedRetry delayedRetry: + delayedRetryLogger.DelayedRetryLogged(exceptionToLog, errorContext.MessageId, delayedRetry.Delay); + break; + case MoveToError moveToError: + moveToErrorLogger.MoveToErrorLogged(exceptionToLog, errorContext.MessageId, moveToError.ErrorQueue); + break; + case Discard discard: + discardLogger.DiscardLogged(exceptionToLog, errorContext.MessageId, discard.Reason); + break; + default: + unknownActionLogger.UnknownRecoverabilityActionLogged(exceptionToLog, recoverabilityAction.GetType().Name, errorContext.MessageId); + break; + } + } + + readonly ILogger immediateRetryLogger = serviceProvider.GetRequiredService>(); + readonly ILogger delayedRetryLogger = serviceProvider.GetRequiredService>(); + readonly ILogger moveToErrorLogger = serviceProvider.GetRequiredService>(); + readonly ILogger discardLogger = serviceProvider.GetRequiredService>(); + readonly ILogger unknownActionLogger = serviceProvider.GetRequiredService>(); +} + +static partial class RecoverabilityActionLoggerMessages +{ + [LoggerMessage(Level = LogLevel.Information, Message = "Immediate Retry is going to retry message '{MessageId}' because of an exception.")] + public static partial void ImmediateRetryLogged(this ILogger logger, Exception? exception, string messageId); + + [LoggerMessage(Level = LogLevel.Information, Message = "Delayed Retry will reschedule message '{MessageId}' after a delay of {Delay} because of an exception.")] + public static partial void DelayedRetryLogged(this ILogger logger, Exception? exception, string messageId, TimeSpan delay); + + [LoggerMessage(Level = LogLevel.Information, Message = "Moving message '{MessageId}' to the error queue '{ErrorQueue}' because processing failed due to an exception.")] + public static partial void MoveToErrorLogged(this ILogger logger, Exception? exception, string messageId, string errorQueue); + + [LoggerMessage(Level = LogLevel.Information, Message = "Discarding message with id '{MessageId}'. Reason: {Reason}.")] + public static partial void DiscardLogged(this ILogger logger, Exception? exception, string messageId, string reason); + + [LoggerMessage(Level = LogLevel.Information, Message = "Recoverability action '{ActionType}' invoked for message '{MessageId}'.")] + public static partial void UnknownRecoverabilityActionLogged(this ILogger logger, Exception? exception, string actionType, string messageId); +} diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs index 88373e89027..e6c0e8ba9e1 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs @@ -1,4 +1,4 @@ -#nullable enable +#nullable enable namespace NServiceBus; @@ -6,7 +6,7 @@ namespace NServiceBus; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; -using NServiceBus.Pipeline; +using Pipeline; using Transport; class RecoverabilityPipelineExecutor( @@ -35,6 +35,8 @@ public async Task Invoke(ErrorContext errorContext, Cancellat { activityFactory.UpdateActivityFromRecoverabilityAction(activity, recoverabilityAction, errorContext.ReceiveAddress); } + + recoverabilityActionLogger.LogRecoverabilityAction(recoverabilityAction, errorContext, activityFactory.Options.ExceptionRecordingMode); } var metadata = faultMetadataExtractor.Extract(errorContext); @@ -55,4 +57,6 @@ public async Task Invoke(ErrorContext errorContext, Cancellat return recoverabilityContext.RecoverabilityAction.ErrorHandleResult; } } -} \ No newline at end of file + + readonly RecoverabilityActionLogger recoverabilityActionLogger = new(serviceProvider); +} diff --git a/src/NServiceBus.Core/Transports/MessageContext.cs b/src/NServiceBus.Core/Transports/MessageContext.cs index 7d191154923..a48e6ec3b15 100644 --- a/src/NServiceBus.Core/Transports/MessageContext.cs +++ b/src/NServiceBus.Core/Transports/MessageContext.cs @@ -51,6 +51,7 @@ public MessageContext(string nativeMessageId, Dictionary headers TransportTransaction = transportTransaction; context.GetOrCreate(); + context.GetOrCreate(); } /// diff --git a/src/NServiceBus.Core/Unicast/MessageOperations.cs b/src/NServiceBus.Core/Unicast/MessageOperations.cs index 7f3af927a2a..cfceeb4c684 100644 --- a/src/NServiceBus.Core/Unicast/MessageOperations.cs +++ b/src/NServiceBus.Core/Unicast/MessageOperations.cs @@ -71,7 +71,7 @@ async Task Publish(IBehaviorContext context, Type messageType, object message, P using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingEventActivityName, publishDisplayName, publishContext); - await publishPipeline.Invoke(publishContext, activity).ConfigureAwait(false); + await publishPipeline.Invoke(publishContext, activity, activityFactory).ConfigureAwait(false); } public Task Subscribe(IBehaviorContext context, Type eventType, SubscribeOptions options) @@ -90,7 +90,7 @@ public async Task Subscribe(IBehaviorContext context, Type[] eventTypes, Subscri using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.SubscribeActivityName, ActivityDisplayNames.SubscribeEvent, context); - await subscribePipeline.Invoke(subscribeContext, activity).ConfigureAwait(false); + await subscribePipeline.Invoke(subscribeContext, activity, activityFactory).ConfigureAwait(false); } public async Task Unsubscribe(IBehaviorContext context, Type eventType, UnsubscribeOptions options) @@ -104,7 +104,7 @@ public async Task Unsubscribe(IBehaviorContext context, Type eventType, Unsubscr using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.UnsubscribeActivityName, ActivityDisplayNames.UnsubscribeEvent, context); - await unsubscribePipeline.Invoke(unsubscribeContext, activity).ConfigureAwait(false); + await unsubscribePipeline.Invoke(unsubscribeContext, activity, activityFactory).ConfigureAwait(false); } public Task Send(IBehaviorContext context, Action messageConstructor, SendOptions options) @@ -138,7 +138,7 @@ async Task SendMessage(IBehaviorContext context, Type messageType, object messag using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingMessageActivityName, ActivityDisplayNames.SendMessage, outgoingContext); - await sendPipeline.Invoke(outgoingContext, activity).ConfigureAwait(false); + await sendPipeline.Invoke(outgoingContext, activity, activityFactory).ConfigureAwait(false); } public Task Reply(IBehaviorContext context, object message, ReplyOptions options) @@ -172,7 +172,7 @@ async Task ReplyMessage(IBehaviorContext context, Type messageType, object messa using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingMessageActivityName, ActivityDisplayNames.ReplyMessage, outgoingContext); - await replyPipeline.Invoke(outgoingContext, activity).ConfigureAwait(false); + await replyPipeline.Invoke(outgoingContext, activity, activityFactory).ConfigureAwait(false); } static void MergeDispatchProperties(ContextBag context, DispatchProperties dispatchProperties) From 3345a883eb33b4e53ec78700efd86e062dd57eb8 Mon Sep 17 00:00:00 2001 From: "Irina Dominte(Scurtu)" Date: Mon, 10 Aug 2026 15:06:18 +0300 Subject: [PATCH 18/22] Additional performance-related instruments (#7898) * Started performance metrics * Removed the Performance metrics from public API * Renamed a metric * nullable enable * remove nullable * added a line * test fix * trailing whitespace * Removed execution result * add enclosed message type to the deserialization insturmentation * Renamed classes and made the tags backwards compatible * Apply suggestions from code review Co-authored-by: Tomasz Masternak * fix: correct tag formatting and indentation in OpenTelemetry tests * undo the type renamings for now * fix: rename IncomingPipelineMetricsTags to IncomingPipelineMetricTags * add MetersOptions to API approvals * fix: correct variable name from messagingMetricsMetricses to messagingMetricsMeters --------- Co-authored-by: Tomasz Masternak Co-authored-by: Tomasz Masternak --- .../Metrics/When_message_processing_fails.cs | 2 +- ...IApprovals.ApproveNServiceBus.approved.txt | 6 + .../MeterTests.Verify_MeterAPI.approved.txt | 13 +- .../Envelopes/EnvelopeUnwrapperTests.cs | 2 +- .../OpenTelemetry/MeterTests.cs | 4 +- .../Incoming/InvokeHandlerTerminatorTest.cs | 2 +- .../SerializeMessageConnectorTests.cs | 3 +- .../IncomingPipelineMetricTagsTests.cs | 3 +- .../Pipeline/MainPipelineExecutorTests.cs | 2 +- ...tReceiveToPhysicalMessageConnectorTests.cs | 2 +- .../Unicast/LoadHandlersConnectorTests.cs | 9 +- src/NServiceBus.Core/EndpointCreator.cs | 2 +- .../OpenTelemetry/InstrumentationOptions.cs | 5 + .../OpenTelemetry/MetersOptions.cs | 17 ++ .../OpenTelemetry/Metrics/MeterTags.cs | 1 + .../Incoming/DeserializeMessageConnector.cs | 24 ++- .../Incoming/IncomingPipelineMetrics.cs | 188 +++++++++++++++++- .../Incoming/LoadHandlersConnector.cs | 4 +- ...nsportReceiveToPhysicalMessageConnector.cs | 4 + .../Outgoing/SerializeMessageConnector.cs | 21 +- .../Pipeline/PipelineComponent.cs | 5 +- .../Receiving/ReceiveComponent.cs | 2 +- .../Sagas/SagaPersistenceBehavior.cs | 17 +- src/NServiceBus.Core/Sagas/Sagas.cs | 2 +- .../Serialization/SerializationFeature.cs | 4 +- 25 files changed, 300 insertions(+), 44 deletions(-) create mode 100644 src/NServiceBus.Core/OpenTelemetry/MetersOptions.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_processing_fails.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_processing_fails.cs index 15a7172bde4..9c1aadacc89 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_processing_fails.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_processing_fails.cs @@ -51,7 +51,7 @@ public async Task Should_report_failing_message_metrics() ["nservicebus.discriminator"] = "disc", ["nservicebus.message_type"] = typeof(FailingMessage).FullName, ["execution.result"] = "failure", - ["error.type"] = typeof(SimulatedException).FullName, + ["error.type"] = typeof(SimulatedException).FullName }); } diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index 0c7e19fe1f1..78f0fc99ed0 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -649,6 +649,7 @@ namespace NServiceBus public NServiceBus.DelayedDeliveryInstrumentationOptions DelayedDelivery { get; } public bool EmitMessageDispatchingEvents { get; set; } public NServiceBus.ExceptionRecordingMode ExceptionRecordingMode { get; set; } + public NServiceBus.MetersOptions Meters { get; } public NServiceBus.TraceMode PublishTraceMode { get; set; } public NServiceBus.RecoverabilityInstrumentationOptions Recoverability { get; } public NServiceBus.TraceMode SendTraceMode { get; set; } @@ -768,6 +769,11 @@ namespace NServiceBus public static System.Threading.Tasks.Task Unsubscribe(this NServiceBus.IMessageSession session, System.Type messageType, System.Threading.CancellationToken cancellationToken = default) { } public static System.Threading.Tasks.Task Unsubscribe(this NServiceBus.IMessageSession session, System.Threading.CancellationToken cancellationToken = default) { } } + public class MetersOptions + { + public MetersOptions() { } + public bool EmitExecutionResultTags { get; set; } + } public class MoveToError : NServiceBus.RecoverabilityAction { protected MoveToError(string errorQueue) { } diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt index 8cfd1afa89c..b632b411cce 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt @@ -1,7 +1,7 @@ { "Note": "Changes to metrics API should result in an update to NServiceBusMeter version.", "MetricsSourceName": "NServiceBus.Core.Pipeline.Incoming", - "MetricsSourceVersion": "0.3.0", + "MetricsSourceVersion": "0.4.0", "Tags": [ "error.type", "execution.result", @@ -11,20 +11,27 @@ "nservicebus.message_handler_type", "nservicebus.message_handler_types", "nservicebus.message_type", - "nservicebus.queue" + "nservicebus.queue", + "nservicebus.saga_type" ], "Metrics": [ "nservicebus.envelope.unwrapped => Counter", "nservicebus.messaging.active_messages => UpDownCounter", "nservicebus.messaging.critical_time => Histogram, Unit: s", + "nservicebus.messaging.deserialize_time => Histogram, Unit: s", "nservicebus.messaging.failures => Counter", "nservicebus.messaging.fetches => Counter", "nservicebus.messaging.handler_time => Histogram, Unit: s", "nservicebus.messaging.processing_time => Histogram, Unit: s", + "nservicebus.messaging.serialize_time => Histogram, Unit: s", "nservicebus.messaging.successes => Counter", "nservicebus.outbox.duplicates => Counter", + "nservicebus.outbox.fetch_time => Histogram, Unit: s", + "nservicebus.outbox.store_time => Histogram, Unit: s", + "nservicebus.persistence.commit_time => Histogram, Unit: s", "nservicebus.recoverability.delayed => Counter", "nservicebus.recoverability.error => Counter", - "nservicebus.recoverability.immediate => Counter" + "nservicebus.recoverability.immediate => Counter", + "nservicebus.sagas.fetch_time => Histogram, Unit: s" ] } \ No newline at end of file diff --git a/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs b/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs index 1824dc35f2c..2b52437cca8 100644 --- a/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs +++ b/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs @@ -29,7 +29,7 @@ public void Setup() originalBody = "payload"u8.ToArray().AsMemory(); messageContext = new MessageContext(nativeId, originalHeaders, originalBody, new TransportTransaction(), "receiveAddress", new ContextBag()); meterFactory = new TestMeterFactory(); - incomingPipelineMetrics = new IncomingPipelineMetrics(meterFactory, "queue", "disc"); + incomingPipelineMetrics = new IncomingPipelineMetrics(meterFactory, "queue", "disc", new MetersOptions()); } [TearDown] diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs index b59ee8b556e..83b1e16e1d3 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs @@ -20,9 +20,9 @@ public void Verify_MeterAPI() .ToList(); using var meterFactory = new TestMeterFactory(); - //The IncomingPipelineMetrics constructor creates the meters, therefore a new instance before collecting the metrics. + //The IncomingPipelineMeter constructor creates the meters, therefore a new instance before collecting the metrics. #pragma warning disable CA1806 - new IncomingPipelineMetrics(meterFactory, "queue", "disc"); + new IncomingPipelineMetrics(meterFactory, "queue", "disc", new MetersOptions()); #pragma warning restore CA1806 using var metricsListener = TestingMetricListener.SetupNServiceBusMetricsListener(); diff --git a/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs b/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs index 47276e798e4..b794eb25413 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs @@ -11,7 +11,7 @@ [TestFixture] public class InvokeHandlerTerminatorTest { - readonly InvokeHandlerTerminator terminator = new(new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc")); + readonly InvokeHandlerTerminator terminator = new(new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); [Test] public async Task When_saga_found_and_handler_is_saga_should_invoke_handler() diff --git a/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs index 12b7ed19bc6..76e0efb2e00 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; +using NServiceBus.Core.Tests.OpenTelemetry; using NServiceBus.Pipeline; using NUnit.Framework; using Serialization; @@ -29,7 +30,7 @@ public async Task Should_set_content_type_header() Message = new OutgoingLogicalMessage(typeof(MyMessage), new MyMessage()) }; - var behavior = new SerializeMessageConnector(new FakeSerializer("myContentType"), registry); + var behavior = new SerializeMessageConnector(new FakeSerializer("myContentType"), registry, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); await behavior.Invoke(context, c => Task.CompletedTask); diff --git a/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs b/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs index 52576e26016..e04e3999d2c 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs @@ -5,6 +5,7 @@ namespace NServiceBus.Core.Tests.Pipeline.Incoming; using System.IO; using System.Threading.Tasks; using MessageInterfaces.MessageMapper.Reflection; +using NServiceBus.Core.Tests.OpenTelemetry; using NServiceBus.Pipeline; using NUnit.Framework; using Serialization; @@ -35,7 +36,7 @@ public void Should_not_fail_when_handling_more_than_one_logical_message() }; var messageMapper = new MessageMapper(); - var behavior = new DeserializeMessageConnector(new MessageDeserializerResolver(new FakeSerializer(), []), new LogicalMessageFactory(registry, messageMapper), registry, messageMapper, false); + var behavior = new DeserializeMessageConnector(new MessageDeserializerResolver(new FakeSerializer(), []), new LogicalMessageFactory(registry, messageMapper), registry, messageMapper, false, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); Assert.DoesNotThrowAsync(async () => await behavior.Invoke(context, c => { diff --git a/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs b/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs index ee6b76d9e3b..8befd4e48c9 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs @@ -123,7 +123,7 @@ static MessageContext CreateMessageContext() => static MainPipelineExecutor CreateMainPipelineExecutor(ServiceProvider serviceProvider, IPipeline receivePipeline) { - var incomingPipelineMetrics = new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc"); + var incomingPipelineMetrics = new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions()); var executor = new MainPipelineExecutor( serviceProvider, new PipelineCache(serviceProvider, new PipelineModifications()), diff --git a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs index 8f66cf3bf60..e33a9ef34a2 100644 --- a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs @@ -209,7 +209,7 @@ public void SetUp() fakeBatchPipeline = new FakeBatchPipeline(); fakeMeterFactory = new TestMeterFactory(); - behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc"), new InstrumentationOptions()); + behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions()), new InstrumentationOptions()); } [TearDown] diff --git a/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs b/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs index 60b8a81cca0..67cb7048465 100644 --- a/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using System.Transactions; using Core.Tests.Fakes; +using Core.Tests.OpenTelemetry; using Microsoft.Extensions.DependencyInjection; using NServiceBus.Transport; using NUnit.Framework; @@ -16,7 +17,7 @@ public class LoadHandlersConnectorTests [Test] public void Should_throw_when_there_are_no_registered_message_handlers() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); var context = new TestableIncomingLogicalMessageContext(); @@ -29,7 +30,7 @@ public void Should_throw_when_there_are_no_registered_message_handlers() [Test] public void Should_throw_if_ambient_transaction_is_different_from_scope_used_by_transport() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); var context = new TestableIncomingLogicalMessageContext(); @@ -49,7 +50,7 @@ public void Should_throw_if_ambient_transaction_is_different_from_scope_used_by_ [Test] public void Should_throw_if_ambient_transaction_suppressed_when_transport_uses_a_scope() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); var context = new TestableIncomingLogicalMessageContext(); @@ -77,7 +78,7 @@ public void Should_not_throw_if_ambient_scope_is_same_as_transport_scope() context.Services.AddSingleton(); context.Extensions.Set(new NoOpOutboxTransaction()); - var behavior = new LoadHandlersConnector(messageHandlerRegistry, NoOpActivityFactory.Instance); + var behavior = new LoadHandlersConnector(messageHandlerRegistry, NoOpActivityFactory.Instance, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); using (new TransactionScope(TransactionScopeAsyncFlowOption.Enabled)) { diff --git a/src/NServiceBus.Core/EndpointCreator.cs b/src/NServiceBus.Core/EndpointCreator.cs index 71c893904df..627887e0ff7 100644 --- a/src/NServiceBus.Core/EndpointCreator.cs +++ b/src/NServiceBus.Core/EndpointCreator.cs @@ -157,7 +157,7 @@ void Configure() pipelineSettings); receiveComponent.AddManifest(hostingConfiguration, settings); - pipelineComponent = PipelineComponent.Initialize(pipelineSettings, hostingConfiguration, receiveConfiguration); + pipelineComponent = PipelineComponent.Initialize(pipelineSettings, hostingConfiguration, receiveConfiguration, hostingConfiguration.ActivityFactory.Options.Meters); // The settings can only be locked after initializing the feature component since it uses the settings to store & share feature state. // As well as all the other components have been initialized diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index 7028795c9aa..4930b5d2ffe 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -20,6 +20,11 @@ public partial class InstrumentationOptions /// public RecoverabilityInstrumentationOptions Recoverability { get; } = new(); + /// + /// Controls meter instruments behaviors. + /// + public MetersOptions Meters { get; } = new(); + /// /// Controls instrumentation of explicitly delayed messages (SendOptions.DelayDeliveryWith /// / DoNotDeliverBefore and saga timeouts. diff --git a/src/NServiceBus.Core/OpenTelemetry/MetersOptions.cs b/src/NServiceBus.Core/OpenTelemetry/MetersOptions.cs new file mode 100644 index 00000000000..0838da8aea8 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/MetersOptions.cs @@ -0,0 +1,17 @@ +#nullable enable + +namespace NServiceBus; + +/// +/// Controls opt-in meter instruments behaviors. +/// Accessed via endpointConfiguration.Tracing().Meters. +/// +public class MetersOptions +{ + /// + /// Emits the legacy execution.result tag with values "success" or "failure" + /// on handler time, processing time, saga fetch time, deserialize time, and serialize time metrics. + /// Enabled by default for backwards compatibility. Disable to reduce tag cardinality. + /// + public bool EmitExecutionResultTags { get; set; } = true; +} diff --git a/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs b/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs index d99b135197e..f3fc9666fdb 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs @@ -13,4 +13,5 @@ static class MeterTags public const string ExecutionResult = "execution.result"; public const string ErrorType = "error.type"; public const string EnvelopeUnwrapperType = "nservicebus.envelope.unwrapper_type"; + public const string SagaType = "nservicebus.saga_type"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs index dc01b394b78..573645d8750 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs @@ -1,10 +1,11 @@ -#nullable enable +#nullable enable namespace NServiceBus; using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Threading.Tasks; using Logging; using MessageInterfaces; @@ -17,14 +18,29 @@ class DeserializeMessageConnector( LogicalMessageFactory logicalMessageFactory, MessageMetadataRegistry messageMetadataRegistry, IMessageMapper mapper, - bool allowContentTypeInference) + bool allowContentTypeInference, + IncomingPipelineMetrics incomingPipelineMetrics) : StageConnector { public override async Task Invoke(IIncomingPhysicalMessageContext context, Func stage) { var incomingMessage = context.Message; - var messages = ExtractWithExceptionHandling(incomingMessage); + LogicalMessage[] messages; + var deserializeStart = Stopwatch.GetTimestamp(); + try + { + messages = ExtractWithExceptionHandling(incomingMessage); + } +#pragma warning disable PS0019 + catch (Exception ex) +#pragma warning restore PS0019 + { + incomingPipelineMetrics.RecordDeserializeTime(context, Stopwatch.GetElapsedTime(deserializeStart), error: ex); + throw; + } + + incomingPipelineMetrics.RecordDeserializeTime(context, Stopwatch.GetElapsedTime(deserializeStart)); bool first = true; foreach (var message in messages) @@ -137,4 +153,4 @@ LogicalMessage[] Extract(IncomingMessage physicalMessage) static readonly ILog log = LogManager.GetLogger(); static ReadOnlySpan ImplSuffix => "__impl".AsSpan(); static ReadOnlySpan EnclosedMessageTypeSeparator => ";".AsSpan(); -} \ No newline at end of file +} diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs index 8fd252bf161..5f9a7cff744 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs @@ -23,10 +23,17 @@ class IncomingPipelineMetrics const string EnvelopeUnwrapping = "nservicebus.envelope.unwrapped"; const string ActiveMessages = "nservicebus.messaging.active_messages"; const string TotalDeduplicated = "nservicebus.outbox.duplicates"; - - public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, string discriminator) + const string SagaFetchTime = "nservicebus.sagas.fetch_time"; + const string MessageDeserializeTime = "nservicebus.messaging.deserialize_time"; + const string MessageSerializeTime = "nservicebus.messaging.serialize_time"; + const string OutboxFetchTime = "nservicebus.outbox.fetch_time"; + const string OutboxStoreTime = "nservicebus.outbox.store_time"; + const string CommitTime = "nservicebus.persistence.commit_time"; + + public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, string discriminator, MetersOptions metersOptions) { - var meter = meterFactory.Create("NServiceBus.Core.Pipeline.Incoming", "0.3.0"); + emitExecutionResultTags = metersOptions.EmitExecutionResultTags; + var meter = meterFactory.Create("NServiceBus.Core.Pipeline.Incoming", "0.4.0"); totalProcessedSuccessfully = meter.CreateCounter(TotalProcessedSuccessfully, description: "Total number of messages processed successfully by the endpoint."); totalFetched = meter.CreateCounter(TotalFetched, @@ -51,15 +58,27 @@ public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, str description: "Total number of unwrapping attempts by the endpoint."); activeMessages = meter.CreateUpDownCounter(ActiveMessages, description: "Number of messages currently being processed by the endpoint."); + sagaFetchTime = meter.CreateHistogram(SagaFetchTime, "s", + "The time in seconds for loading saga data from the persister."); + messageDeserializeTime = meter.CreateHistogram(MessageDeserializeTime, "s", + "The time in seconds for deserializing an incoming message."); + messageSerializeTime = meter.CreateHistogram(MessageSerializeTime, "s", + "The time in seconds for serializing an outgoing message."); + outboxFetchTime = meter.CreateHistogram(OutboxFetchTime, "s", + "The time in seconds for querying the outbox storage for deduplication."); + outboxStoreTime = meter.CreateHistogram(OutboxStoreTime, "s", + "The time in seconds for storing a message in the outbox storage."); + persistenceTime = meter.CreateHistogram(CommitTime, "s", + "The time in seconds for completing the synchronized storage session."); queueNameBase = queueName; endpointDiscriminator = discriminator; } - public void AddDefaultIncomingPipelineMetricTags(IncomingPipelineMetricTags incomingPipelineMetricsTags) + public void AddDefaultIncomingPipelineMetricTags(IncomingPipelineMetricTags incomingPipelineMetricTags) { - incomingPipelineMetricsTags.Add(MeterTags.QueueName, queueNameBase); - incomingPipelineMetricsTags.Add(MeterTags.EndpointDiscriminator, endpointDiscriminator); + incomingPipelineMetricTags.Add(MeterTags.QueueName, queueNameBase); + incomingPipelineMetricTags.Add(MeterTags.EndpointDiscriminator, endpointDiscriminator); } public void RecordProcessingTime(ITransportReceiveContext context, TimeSpan elapsed) @@ -72,13 +91,16 @@ public void RecordProcessingTime(ITransportReceiveContext context, TimeSpan elap var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - tags.Add(new(MeterTags.ExecutionResult, "success")); incomingPipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, MeterTags.MessageHandlerTypes]); + if (emitExecutionResultTags) + { + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); + } processingTime.Record(elapsed.TotalSeconds, tags); } @@ -92,13 +114,17 @@ public void RecordCriticalTimeAndTotalProcessed(ITransportReceiveContext context var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - tags.Add(new(MeterTags.ExecutionResult, "success")); incomingPipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, MeterTags.MessageHandlerTypes]); + if (emitExecutionResultTags) + { + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); + } + if (totalProcessedSuccessfully.Enabled) { totalProcessedSuccessfully.Add(1, tags); @@ -124,12 +150,15 @@ public void RecordMessageProcessingFailure(IncomingPipelineMetricTags incomingPi TagList tags; tags.Add(new(MeterTags.ErrorType, error.GetType().FullName)); - tags.Add(new(MeterTags.ExecutionResult, "failure")); incomingPipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, MeterTags.MessageHandlerTypes]); + if (emitExecutionResultTags) + { + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); + } totalFailures.Add(1, tags); // the processing and critical time are intentionally not recorded in case of failure @@ -184,7 +213,10 @@ public void RecordSuccessfulMessageHandlerTime(IInvokeHandlerContext invokeHandl MeterTags.MessageHandlerType]); // This is what Add(string, object) does so skipping an unnecessary stack frame meterTags.Add(new KeyValuePair(MeterTags.MessageHandlerType, invokeHandlerContext.MessageHandler.HandlerType.FullName)); - meterTags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); + if (emitExecutionResultTags) + { + meterTags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); + } messageHandlerTime.Record(elapsed.TotalSeconds, meterTags); } @@ -204,8 +236,11 @@ public void RecordFailedMessageHandlerTime(IInvokeHandlerContext invokeHandlerCo MeterTags.MessageHandlerType]); // This is what Add(string, object) does so skipping an unnecessary stack frame meterTags.Add(new KeyValuePair(MeterTags.MessageHandlerType, invokeHandlerContext.MessageHandler.HandlerType.FullName)); - meterTags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); meterTags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + if (emitExecutionResultTags) + { + meterTags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); + } messageHandlerTime.Record(elapsed.TotalSeconds, meterTags); } @@ -284,6 +319,130 @@ public ActiveMessageScope TrackMessageProcessing(IncomingPipelineMetricTags inco return new ActiveMessageScope(activeMessages, tags); } + public void RecordSagaFetchTime(IInvokeHandlerContext context, TimeSpan elapsed, string sagaType, Exception? error = null) + { + if (!sagaFetchTime.Enabled) + { + return; + } + + var incomingPipelineMetricTags = context.Extensions.Get(); + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType]); + tags.Add(new KeyValuePair(MeterTags.SagaType, sagaType)); + if (error != null) + { + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + } + if (emitExecutionResultTags) + { + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); + } + sagaFetchTime.Record(elapsed.TotalSeconds, tags); + } + + public void RecordDeserializeTime(IIncomingPhysicalMessageContext context, TimeSpan elapsed, Exception? error = null) + { + if (!messageDeserializeTime.Enabled) + { + return; + } + + var incomingPipelineMetricTags = context.Extensions.Get(); + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator + ]); + if (error != null) + { + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + } + if (context.Message.Headers.TryGetValue(Headers.EnclosedMessageTypes, out var messageTypes)) + { + tags.Add(new KeyValuePair(MeterTags.EnclosedMessageTypes, messageTypes)); + } + if (emitExecutionResultTags) + { + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); + } + + messageDeserializeTime.Record(elapsed.TotalSeconds, tags); + } + + public void RecordSerializeTime(TimeSpan elapsed, string? messageType, Exception? error = null) + { + if (!messageSerializeTime.Enabled) + { + return; + } + + TagList tags; + if (messageType != null) + { + tags.Add(new KeyValuePair(MeterTags.MessageType, messageType)); + } + if (error != null) + { + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + } + if (emitExecutionResultTags) + { + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); + } + messageSerializeTime.Record(elapsed.TotalSeconds, tags); + } + + public void RecordOutboxFetchTime(ITransportReceiveContext context, TimeSpan elapsed) + { + if (!outboxFetchTime.Enabled) + { + return; + } + + var incomingPipelineMetricTags = context.Extensions.Get(); + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator]); + outboxFetchTime.Record(elapsed.TotalSeconds, tags); + } + + public void RecordOutboxStoreTime(ITransportReceiveContext context, TimeSpan elapsed) + { + if (!outboxStoreTime.Enabled) + { + return; + } + + var incomingPipelineMetricTags = context.Extensions.Get(); + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator]); + outboxStoreTime.Record(elapsed.TotalSeconds, tags); + } + + public void RecordPersistenceTime(IIncomingLogicalMessageContext context, TimeSpan elapsed) + { + if (!persistenceTime.Enabled) + { + return; + } + + var incomingPipelineMetricTags = context.Extensions.Get(); + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.MessageHandlerTypes]); + persistenceTime.Record(elapsed.TotalSeconds, tags); + } + public void EnvelopeUnwrappingSucceeded(MessageContext messageContext, IEnvelopeHandler type) => RecordEnvelopeUnwrapping(messageContext, type, true, null); public void EnvelopeUnwrappingFailed(MessageContext messageContext, IEnvelopeHandler type, Exception? exception) => RecordEnvelopeUnwrapping(messageContext, type, false, exception); void RecordEnvelopeUnwrapping(MessageContext messageContext, IEnvelopeHandler type, bool succeeded, Exception? exception) @@ -324,7 +483,14 @@ public readonly struct ActiveMessageScope(UpDownCounter? counter, TagList readonly Counter totalSentToErrorQueue; readonly Counter totalEnvelopeUnwrapping; readonly UpDownCounter activeMessages; + readonly Histogram sagaFetchTime; + readonly Histogram messageDeserializeTime; + readonly Histogram messageSerializeTime; + readonly Histogram outboxFetchTime; + readonly Histogram outboxStoreTime; + readonly Histogram persistenceTime; readonly string queueNameBase; readonly string endpointDiscriminator; + readonly bool emitExecutionResultTags; } \ No newline at end of file diff --git a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs index e11af72114e..26a3b0178bf 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs @@ -16,7 +16,7 @@ namespace NServiceBus; using Pipeline; using Unicast; -class LoadHandlersConnector(MessageHandlerRegistry messageHandlerRegistry, IActivityFactory activityFactory) : StageConnector +class LoadHandlersConnector(MessageHandlerRegistry messageHandlerRegistry, IActivityFactory activityFactory, IncomingPipelineMetrics incomingPipelineMetrics) : StageConnector { public override async Task Invoke(IIncomingLogicalMessageContext context, Func stage) { @@ -80,7 +80,9 @@ public override async Task Invoke(IIncomingLogicalMessageContext context, Func(); await outboxTransaction.Commit(context.CancellationToken).ConfigureAwait(false); diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs index 38e3cc223cc..14bbad12000 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs @@ -3,6 +3,7 @@ namespace NServiceBus; using System; +using System.Diagnostics; using System.IO; using System.Threading.Tasks; using Logging; @@ -12,10 +13,11 @@ namespace NServiceBus; class SerializeMessageConnector : StageConnector { - public SerializeMessageConnector(IMessageSerializer messageSerializer, MessageMetadataRegistry messageMetadataRegistry) + public SerializeMessageConnector(IMessageSerializer messageSerializer, MessageMetadataRegistry messageMetadataRegistry, IncomingPipelineMetrics incomingPipelineMetrics) { this.messageSerializer = messageSerializer; this.messageMetadataRegistry = messageMetadataRegistry; + this.incomingPipelineMetrics = incomingPipelineMetrics; } public override async Task Invoke(IOutgoingLogicalMessageContext context, Func stage) @@ -39,7 +41,19 @@ public override async Task Invoke(IOutgoingLogicalMessageContext context, Func(); -} \ No newline at end of file +} diff --git a/src/NServiceBus.Core/Pipeline/PipelineComponent.cs b/src/NServiceBus.Core/Pipeline/PipelineComponent.cs index fba1beea0e1..03326dc412b 100644 --- a/src/NServiceBus.Core/Pipeline/PipelineComponent.cs +++ b/src/NServiceBus.Core/Pipeline/PipelineComponent.cs @@ -12,14 +12,15 @@ sealed class PipelineComponent PipelineComponent(PipelineModifications modifications) => this.modifications = modifications; public static PipelineComponent Initialize(PipelineSettings settings, - HostingComponent.Configuration hostingConfiguration, ReceiveComponent.Configuration receiveConfiguration) + HostingComponent.Configuration hostingConfiguration, ReceiveComponent.Configuration receiveConfiguration, + MetersOptions metersOptions) { // make the PipelineMetrics available to the Pipeline hostingConfiguration.Services.AddSingleton(sp => { var meterFactory = sp.GetRequiredService(); string discriminator = receiveConfiguration.InstanceSpecificQueueAddress?.Discriminator ?? ""; - return new IncomingPipelineMetrics(meterFactory, receiveConfiguration.LocalQueueAddress.BaseAddress, discriminator); + return new IncomingPipelineMetrics(meterFactory, receiveConfiguration.LocalQueueAddress.BaseAddress, discriminator, metersOptions); }); return new PipelineComponent(settings.modifications); diff --git a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs index 4a0a5a8c1c6..5d859914cef 100644 --- a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs +++ b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs @@ -70,7 +70,7 @@ public static ReceiveComponent Configure( return new TransportReceiveToPhysicalMessageConnector(storage, b.GetRequiredService(), hostingConfiguration.ActivityFactory.Options); }, "Allows to abort processing the message"); - pipelineSettings.Register("LoadHandlersConnector", b => new LoadHandlersConnector(b.GetRequiredService(), hostingConfiguration.ActivityFactory), "Gets all the handlers to invoke from the MessageHandler registry based on the message type."); + pipelineSettings.Register("LoadHandlersConnector", b => new LoadHandlersConnector(b.GetRequiredService(), hostingConfiguration.ActivityFactory, b.GetRequiredService()), "Gets all the handlers to invoke from the MessageHandler registry based on the message type."); pipelineSettings.Register("InvokeHandlers", sp => new InvokeHandlerTerminator(sp.GetRequiredService()), "Calls the IHandleMessages.Handle(T)"); diff --git a/src/NServiceBus.Core/Sagas/SagaPersistenceBehavior.cs b/src/NServiceBus.Core/Sagas/SagaPersistenceBehavior.cs index 577a748e28a..66376382029 100644 --- a/src/NServiceBus.Core/Sagas/SagaPersistenceBehavior.cs +++ b/src/NServiceBus.Core/Sagas/SagaPersistenceBehavior.cs @@ -9,7 +9,7 @@ using Pipeline; using Sagas; -class SagaPersistenceBehavior(ISagaPersister persister, ISagaIdGenerator sagaIdGenerator, SagaMetadataCollection sagaMetadataCollection, IServiceProvider serviceProvider) +class SagaPersistenceBehavior(ISagaPersister persister, ISagaIdGenerator sagaIdGenerator, SagaMetadataCollection sagaMetadataCollection, IServiceProvider serviceProvider, IncomingPipelineMetrics incomingPipelineMetrics) : IBehavior { public async Task Invoke(IInvokeHandlerContext context, Func next) @@ -67,7 +67,20 @@ public async Task Invoke(IInvokeHandlerContext context, Func(); // Register the Saga related behaviors for incoming messages - context.Pipeline.Register("InvokeSaga", b => new SagaPersistenceBehavior(b.GetRequiredService(), sagaIdGenerator, sagaMetaModel, b), "Invokes the saga logic"); + context.Pipeline.Register("InvokeSaga", b => new SagaPersistenceBehavior(b.GetRequiredService(), sagaIdGenerator, sagaMetaModel, b, b.GetRequiredService()), "Invokes the saga logic"); context.Pipeline.Register("AttachSagaDetailsToOutGoingMessage", new AttachSagaDetailsToOutGoingMessageBehavior(), "Makes sure that outgoing messages have saga info attached to them"); } } \ No newline at end of file diff --git a/src/NServiceBus.Core/Serialization/SerializationFeature.cs b/src/NServiceBus.Core/Serialization/SerializationFeature.cs index a332f50bd37..41bec7b6d5a 100644 --- a/src/NServiceBus.Core/Serialization/SerializationFeature.cs +++ b/src/NServiceBus.Core/Serialization/SerializationFeature.cs @@ -48,8 +48,8 @@ protected override void Setup(FeatureConfigurationContext context) var allowMessageTypeInference = settings.IsMessageTypeInferenceEnabled(); var resolver = new MessageDeserializerResolver(mainSerializer, additionalDeserializers); var logicalMessageFactory = new LogicalMessageFactory(messageMetadataRegistry, mapper); - context.Pipeline.Register("DeserializeLogicalMessagesConnector", new DeserializeMessageConnector(resolver, logicalMessageFactory, messageMetadataRegistry, mapper, allowMessageTypeInference), "Deserializes the physical message body into logical messages"); - context.Pipeline.Register("SerializeMessageConnector", new SerializeMessageConnector(mainSerializer, messageMetadataRegistry), "Converts a logical message into a physical message"); + context.Pipeline.Register("DeserializeLogicalMessagesConnector", b => new DeserializeMessageConnector(resolver, logicalMessageFactory, messageMetadataRegistry, mapper, allowMessageTypeInference, b.GetRequiredService()), "Deserializes the physical message body into logical messages"); + context.Pipeline.Register("SerializeMessageConnector", b => new SerializeMessageConnector(mainSerializer, messageMetadataRegistry, b.GetRequiredService()), "Converts a logical message into a physical message"); context.Services.AddSingleton(mapper); context.Services.AddSingleton(mapper); From a1b03374dbc96b96d334dfe46c4543a6b34d6ec8 Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Tue, 11 Aug 2026 14:35:33 +0200 Subject: [PATCH 19/22] Minor tweaks to the OpenTelemetry featue (#7908) * adjust log levels for recoverability actions * split exception handling in recoverability action log messages to ensure backwards compatibility * updated exception log message formats in recoverability tests for consistency --- .../When_immediate_retries_are_enabled.cs | 2 +- .../When_message_fails_retries.cs | 2 +- .../RecoverabilityActionLogger.cs | 76 +++++++++++++++++-- 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs b/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs index 7f01e523cd3..51002130852 100644 --- a/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs +++ b/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs @@ -23,7 +23,7 @@ public async Task Should_do_the_configured_number_of_retries() Assert.That(context.ForwardedToErrorQueue, Is.True); Assert.That(context.NumberOfTimesInvoked, Is.EqualTo(numberOfRetries + 1), "Message should be retried 5 times immediately"); Assert.That(context.Logs.Count(l => l.Message - .StartsWith($"Immediate Retry is going to retry message '{context.MessageId}' because of an exception.")), Is.EqualTo(numberOfRetries)); + .StartsWith($"Immediate Retry is going to retry message '{context.MessageId}' because of an exception:")), Is.EqualTo(numberOfRetries)); } } diff --git a/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs b/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs index ba014c87f9b..8177397224a 100644 --- a/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs +++ b/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs @@ -28,7 +28,7 @@ public void Should_forward_message_to_error_queue() } Assert.That(testContext.Logs.Count(l => l.Message - .StartsWith($"Moving message '{testContext.PhysicalMessageId}' to the error queue 'error' because processing failed due to an exception.")), Is.EqualTo(1)); + .StartsWith($"Moving message '{testContext.PhysicalMessageId}' to the error queue 'error' because processing failed due to an exception:")), Is.EqualTo(1)); } public class RetryEndpoint : EndpointConfigurationBuilder diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs index 40878f9730c..ca722b5550f 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs @@ -48,17 +48,81 @@ public void LogRecoverabilityAction(RecoverabilityAction recoverabilityAction, E static partial class RecoverabilityActionLoggerMessages { + // Exception is only attached when it hasn't already been logged separately by + // ActivityFactory.RecordError (see RecoverabilityActionLogger.LogRecoverabilityAction). + // The trailing punctuation differs in both scenarios, and we need to keep the colon when + // an exception is logged to ensure backwards compatibility + public static void ImmediateRetryLogged(this ILogger logger, Exception? exception, string messageId) + { + if (exception is not null) + { + ImmediateRetryLoggedWithException(logger, exception, messageId); + } + else + { + ImmediateRetryLoggedWithoutException(logger, messageId); + } + } + + [LoggerMessage(Level = LogLevel.Information, Message = "Immediate Retry is going to retry message '{MessageId}' because of an exception:")] + static partial void ImmediateRetryLoggedWithException(ILogger logger, Exception exception, string messageId); + [LoggerMessage(Level = LogLevel.Information, Message = "Immediate Retry is going to retry message '{MessageId}' because of an exception.")] - public static partial void ImmediateRetryLogged(this ILogger logger, Exception? exception, string messageId); + static partial void ImmediateRetryLoggedWithoutException(ILogger logger, string messageId); + + public static void DelayedRetryLogged(this ILogger logger, Exception? exception, string messageId, TimeSpan delay) + { + if (exception is not null) + { + DelayedRetryLoggedWithException(logger, exception, messageId, delay); + } + else + { + DelayedRetryLoggedWithoutException(logger, messageId, delay); + } + } + + [LoggerMessage(Level = LogLevel.Warning, Message = "Delayed Retry will reschedule message '{MessageId}' after a delay of {Delay} because of an exception:")] + static partial void DelayedRetryLoggedWithException(ILogger logger, Exception exception, string messageId, TimeSpan delay); - [LoggerMessage(Level = LogLevel.Information, Message = "Delayed Retry will reschedule message '{MessageId}' after a delay of {Delay} because of an exception.")] - public static partial void DelayedRetryLogged(this ILogger logger, Exception? exception, string messageId, TimeSpan delay); + [LoggerMessage(Level = LogLevel.Warning, Message = "Delayed Retry will reschedule message '{MessageId}' after a delay of {Delay} because of an exception.")] + static partial void DelayedRetryLoggedWithoutException(ILogger logger, string messageId, TimeSpan delay); + + public static void MoveToErrorLogged(this ILogger logger, Exception? exception, string messageId, string errorQueue) + { + if (exception is not null) + { + MoveToErrorLoggedWithException(logger, exception, messageId, errorQueue); + } + else + { + MoveToErrorLoggedWithoutException(logger, messageId, errorQueue); + } + } + + [LoggerMessage(Level = LogLevel.Error, Message = "Moving message '{MessageId}' to the error queue '{ErrorQueue}' because processing failed due to an exception:")] + static partial void MoveToErrorLoggedWithException(ILogger logger, Exception exception, string messageId, string errorQueue); + + [LoggerMessage(Level = LogLevel.Error, Message = "Moving message '{MessageId}' to the error queue '{ErrorQueue}' because processing failed due to an exception.")] + static partial void MoveToErrorLoggedWithoutException(ILogger logger, string messageId, string errorQueue); + + public static void DiscardLogged(this ILogger logger, Exception? exception, string messageId, string reason) + { + if (exception is not null) + { + DiscardLoggedWithException(logger, exception, messageId, reason); + } + else + { + DiscardLoggedWithoutException(logger, messageId, reason); + } + } - [LoggerMessage(Level = LogLevel.Information, Message = "Moving message '{MessageId}' to the error queue '{ErrorQueue}' because processing failed due to an exception.")] - public static partial void MoveToErrorLogged(this ILogger logger, Exception? exception, string messageId, string errorQueue); + [LoggerMessage(Level = LogLevel.Information, Message = "Discarding message with id '{MessageId}'. Reason: {Reason}")] + static partial void DiscardLoggedWithException(ILogger logger, Exception exception, string messageId, string reason); [LoggerMessage(Level = LogLevel.Information, Message = "Discarding message with id '{MessageId}'. Reason: {Reason}.")] - public static partial void DiscardLogged(this ILogger logger, Exception? exception, string messageId, string reason); + static partial void DiscardLoggedWithoutException(ILogger logger, string messageId, string reason); [LoggerMessage(Level = LogLevel.Information, Message = "Recoverability action '{ActionType}' invoked for message '{MessageId}'.")] public static partial void UnknownRecoverabilityActionLogged(this ILogger logger, Exception? exception, string actionType, string messageId); From 0c135cdde93772c45ed054972a0c5e77d7e51ecd Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Thu, 13 Aug 2026 11:33:41 +0200 Subject: [PATCH 20/22] Removed RecordedExceptions tracking in favor of directly using exception Data (#7911) * removed RecordedExceptions tracking in favor of directly using exception.Data * moved ExceptionRecordedFlag where it's used * Update src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs Co-authored-by: Daniel Marbach --------- Co-authored-by: Daniel Marbach --- .../OpenTelemetry/Tracing/ActivityFactory.cs | 7 ++++--- .../Tracing/RecordedExceptions.cs | 19 ------------------- .../Transports/MessageContext.cs | 1 - 3 files changed, 4 insertions(+), 23 deletions(-) delete mode 100644 src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index 9c407ce77b5..c8a958733fd 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -212,8 +212,7 @@ public void RecordError(Activity activity, Exception exception, ContextBag conte LegacyExceptionTags.SetLegacyStatusTags(activity, exception); - var recordedExceptions = context.GetOrCreate(); - if (!recordedExceptions.HasBeenRecorded(exception)) + if (!exception.Data.Contains(ExceptionRecordedFlag)) { if (Options.ExceptionRecordingMode == ExceptionRecordingMode.Logs) { @@ -224,7 +223,7 @@ public void RecordError(Activity activity, Exception exception, ContextBag conte activity.AddException(exception, LegacyExceptionTags.EscapedTagList); } - recordedExceptions.MarkAsRecorded(exception); + exception.Data[ExceptionRecordedFlag] = true; } if (exception is TaskCanceledException) @@ -233,5 +232,7 @@ public void RecordError(Activity activity, Exception exception, ContextBag conte } } + const string ExceptionRecordedFlag = "otel.exception.recorded"; + static readonly ILog Logger = LogManager.GetLogger(); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs deleted file mode 100644 index 4ecfafb8152..00000000000 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs +++ /dev/null @@ -1,19 +0,0 @@ -#nullable enable - -namespace NServiceBus; - -using System; -using System.Collections.Generic; - -sealed class RecordedExceptions -{ - // Reference equality is intentional: this tracks specific exception instances as they - // propagate, not exceptions that merely look alike. -#pragma warning disable PS0025 - readonly HashSet recorded = new(ReferenceEqualityComparer.Instance); -#pragma warning restore PS0025 - - public bool HasBeenRecorded(Exception exception) => recorded.Contains(exception); - - public void MarkAsRecorded(Exception exception) => recorded.Add(exception); -} diff --git a/src/NServiceBus.Core/Transports/MessageContext.cs b/src/NServiceBus.Core/Transports/MessageContext.cs index a48e6ec3b15..7d191154923 100644 --- a/src/NServiceBus.Core/Transports/MessageContext.cs +++ b/src/NServiceBus.Core/Transports/MessageContext.cs @@ -51,7 +51,6 @@ public MessageContext(string nativeMessageId, Dictionary headers TransportTransaction = transportTransaction; context.GetOrCreate(); - context.GetOrCreate(); } /// From 96f8a168ea6effb47af4fbeac16cdcd615e79632 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 26 Aug 2026 14:48:30 +0200 Subject: [PATCH 21/22] Fix edge case where InstrumentionOptions could not be a shared instance (#7914) --- ...essing_fails_with_exception_logs_opt_in.cs | 59 +++++++++++++++++++ .../Hosting/HostingComponent.Settings.cs | 2 +- 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails_with_exception_logs_opt_in.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails_with_exception_logs_opt_in.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails_with_exception_logs_opt_in.cs new file mode 100644 index 00000000000..c5d6b664333 --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails_with_exception_logs_opt_in.cs @@ -0,0 +1,59 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Traces; + +using System.Linq; +using System.Threading.Tasks; +using AcceptanceTesting; +using Configuration.AdvancedExtensibility; +using EndpointTemplates; +using NServiceBus; +using NUnit.Framework; + +// The OTEL_SEMCONV_EXCEPTION_SIGNAL_OPT_IN override is applied while the OpenTelemetryFeature defaults +// run, which is AFTER the endpoint's activity factory has already been built from the instrumentation +// options. The opt-in only takes effect when the activity factory and the settings share a single +// InstrumentationOptions instance. +public class When_processing_fails_with_exception_logs_opt_in : OpenTelemetryAcceptanceTest +{ + [Test] + public async Task Should_record_the_exception_as_a_log_instead_of_a_span_event() + { + var context = await Scenario.Define() + .WithEndpoint(e => e + .DoNotFailOnErrorMessages() + .When(s => s.SendLocal(new FailingMessage()))) + .Run(); + + Assert.That(context.FailedMessages, Has.Count.EqualTo(1), "the message should have failed"); + + var handlerActivity = NServiceBusActivityListener.CompletedActivities.GetInvokedHandlerActivities().Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(handlerActivity.Events, Is.Empty, "the exception should not be recorded as a span event when the endpoint opted in to exceptions as logs"); + Assert.That(context.Logs.Any(l => l.LoggerName == "NServiceBus.ActivityFactory" && l.Level == Logging.LogLevel.Error && l.Message.Contains(ErrorMessage)), Is.True, "the exception should be recorded as an error log instead"); + } + } + + public class Context : ScenarioContext; + + public class FailingEndpoint : EndpointConfigurationBuilder + { + // Does not call endpointConfiguration.Tracing(): the instrumentation options only come into existence while the endpoint is being created. + public FailingEndpoint() => EndpointSetup(c => c.GetSettings().Set($"ACCEPTANCETEST_ENV:{OptInEnvironmentVariable}", "logs")); + + [Handler] + public class FailingMessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(FailingMessage message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + throw new SimulatedException(ErrorMessage); + } + } + } + + public class FailingMessage : IMessage; + + const string OptInEnvironmentVariable = "OTEL_SEMCONV_EXCEPTION_SIGNAL_OPT_IN"; + const string ErrorMessage = "boom!"; +} \ No newline at end of file diff --git a/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs b/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs index 8e85864c2be..d2b8bdb4027 100644 --- a/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs +++ b/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs @@ -95,7 +95,7 @@ public bool WriteDiagnosticsToLog get; set; } - public InstrumentationOptions InstrumentationOptions => settings.GetOrDefault() ?? new InstrumentationOptions(); + public InstrumentationOptions InstrumentationOptions => settings.GetOrCreate(); internal void ConfigureHostLogging(object? endpointIdentifier) { From e3eaa5c763dc84292660ac78776597ae520bfe38 Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Tue, 1 Sep 2026 10:42:00 +0200 Subject: [PATCH 22/22] Add support for instrument-specific metric tags in the incoming pipeline (#7912) * Add support for instrument-specific metric tags in the incoming pipeline * Introduce support for optional instrument names in metric tag methods and improve metric handling logic * Simplify metric tagging logic and remove `IncomingPipelineMetricsTagBagConventionTests`. * Simplify `RecordSerializeTime` by removing unnecessary context dependency and refactoring metric tagging logic. * fix to outgoing pipeline metric tagging * Improve outgoing pipeline metric tagging and refactor `RecordSerializeTime` to handle context-based tags. * Remove unused metric tags in `IncomingPipelineMetrics`. * Refactor metric tagging by replacing `Get` with `MetricTags` extension property. Remove IncomingPipelineMetrics dependency from fakes * Introduce `IMetricsTags` interface as the public api for changing metric tags. * Refactor metric tagging logic in `IncomingPipelineMetrics` to simplify tag application and enforce consistent usage of `TagList`. * Refactor `criticalTime` calculation to ensure `completedAt` is only set when needed * Fix incorrect usage of `messageDeserializeTime` to `messageSerializeTime` in metric tagging logic * Rename `Add` to `AddOrOverride` in `IMetricsTags * Remove unnecessary call to `GetOrCreate` in `MessageContext` constructor. All calls are now GetOrCreate via extension property. * change the base type for MetricTagsExtensions to concrete type from ExtensionBag. Makes accessing tags requrie less nesting * Refactor metric tagging tests to use constants for tag keys and simplify assertions * fixing formatting errors --- .../Metrics/When_customizing_metric_tags.cs | 110 +++++++ .../NServiceBus.AcceptanceTests.csproj | 1 + ...IApprovals.ApproveNServiceBus.approved.txt | 21 +- .../NServiceBus.Core.Tests.csproj | 1 - .../IncomingPipelineMetricTagsTests.cs | 2 +- .../Incoming/DeserializeMessageConnector.cs | 2 +- .../Pipeline/Incoming/IMetricsTags.cs | 23 ++ .../Incoming/IncomingPipelineMetricTags.cs | 84 ++++-- .../Incoming/IncomingPipelineMetrics.cs | 279 ++++++++++-------- .../Incoming/LoadHandlersConnector.cs | 2 +- .../Pipeline/Incoming/MetricTagsExtensions.cs | 29 ++ .../Pipeline/MainPipelineExecutor.cs | 2 +- .../Outgoing/SerializeMessageConnector.cs | 4 +- .../Transports/MessageContext.cs | 2 - .../TestablePipelineContext.cs | 6 +- 15 files changed, 398 insertions(+), 170 deletions(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_customizing_metric_tags.cs create mode 100644 src/NServiceBus.Core/Pipeline/Incoming/IMetricsTags.cs create mode 100644 src/NServiceBus.Core/Pipeline/Incoming/MetricTagsExtensions.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_customizing_metric_tags.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_customizing_metric_tags.cs new file mode 100644 index 00000000000..e964f5faf57 --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_customizing_metric_tags.cs @@ -0,0 +1,110 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Metrics; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using EndpointTemplates; +using NServiceBus; +using AcceptanceTesting; +using NServiceBus.Pipeline; +using NUnit.Framework; +using global::OpenTelemetry; +using global::OpenTelemetry.Metrics; + +public class When_customizing_metric_tags : OpenTelemetryAcceptanceTest +{ + const string TotalFetched = "nservicebus.messaging.fetches"; + const string MessageDeserializeTime = "nservicebus.messaging.deserialize_time"; + const string EndpointDiscriminatorTag = "nservicebus.discriminator"; + const string EnclosedMessageTypesTag = "nservicebus.enclosed_message_types"; + const string TenantTag = "acceptance.tenant_id"; + const string FriendlyMessageTypeName = "Order placed (friendly name)"; + + [Test] + public async Task Should_allow_adding_removing_and_overriding_tags_per_instrument() + { + using var metricsListener = TestingMetricListener.SetupNServiceBusMetricsListener(); + + List exportedMetrics = []; + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("NServiceBus.Core.Pipeline.Incoming") + .AddView(TotalFetched, new MetricStreamConfiguration + { + TagKeys = ["nservicebus.queue", "nservicebus.message_type", TenantTag] + }) + .AddReader(new BaseExportingMetricReader(new CapturingExporter(exportedMetrics))) + .Build(); + + await Scenario.Define() + .WithEndpoint(b => b.CustomConfig(c => c.MakeInstanceUniquelyAddressable("disc")) + .When(async session => + { + var sendOptions = new SendOptions(); + sendOptions.RouteToThisEndpoint(); + sendOptions.SetHeader(TenantTag, "acme-corp"); + await session.Send(new MyMessage(), sendOptions); + })) + .Run(); + + meterProvider.ForceFlush(); + + metricsListener.AssertTags(TotalFetched, new Dictionary { [TenantTag] = "acme-corp" }); + + metricsListener.AssertTagKeyExists(TotalFetched, EndpointDiscriminatorTag); + + var overriddenValue = metricsListener.AssertTagKeyExists(MessageDeserializeTime, EnclosedMessageTypesTag); + Assert.That(overriddenValue, Is.EqualTo(FriendlyMessageTypeName)); + } + + public class Context : ScenarioContext; + + public class EndpointWithCustomTags : EndpointConfigurationBuilder + { + public EndpointWithCustomTags() => + EndpointSetup(c => c.Pipeline.Register( + new CustomizeMetricTagsBehavior(), "Adds a tenant tag from a header and overrides the enclosed message type tag")); + + [Handler] + public class MyHandler(Context testContext) : IHandleMessages + { + public Task Handle(MyMessage message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + + class CustomizeMetricTagsBehavior : Behavior + { + public override Task Invoke(IIncomingPhysicalMessageContext context, Func next) + { + var tags = context.MetricTags; + + if (context.Message.Headers.TryGetValue(TenantTag, out var tenantId)) + { + tags.AddOrOverride(TenantTag, tenantId, TotalFetched); + } + + tags.AddOrOverride(EnclosedMessageTypesTag, FriendlyMessageTypeName, MessageDeserializeTime); + + return next(); + } + } + + class CapturingExporter(List exportedMetrics) : BaseExporter + { + public override ExportResult Export(in Batch batch) + { + foreach (var metric in batch) + { + exportedMetrics.Add(metric); + } + + return ExportResult.Success; + } + } + + public class MyMessage : IMessage; +} diff --git a/src/NServiceBus.AcceptanceTests/NServiceBus.AcceptanceTests.csproj b/src/NServiceBus.AcceptanceTests/NServiceBus.AcceptanceTests.csproj index ea546b1c257..1af349bf801 100644 --- a/src/NServiceBus.AcceptanceTests/NServiceBus.AcceptanceTests.csproj +++ b/src/NServiceBus.AcceptanceTests/NServiceBus.AcceptanceTests.csproj @@ -14,6 +14,7 @@ + diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index 78f0fc99ed0..c7302ebe466 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -554,6 +554,10 @@ namespace NServiceBus System.Threading.Tasks.Task Subscribe(System.Type eventType, NServiceBus.SubscribeOptions subscribeOptions, System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.Task Unsubscribe(System.Type eventType, NServiceBus.UnsubscribeOptions unsubscribeOptions, System.Threading.CancellationToken cancellationToken = default); } + public interface IMetricsTags + { + void AddOrOverride(string tagKey, object value, string instrumentName); + } public interface INeedInitialization { void Customize(NServiceBus.EndpointConfiguration configuration); @@ -618,13 +622,6 @@ namespace NServiceBus public override NServiceBus.Transport.ErrorHandleResult ErrorHandleResult { get; } public override System.Collections.Generic.IReadOnlyCollection GetRoutingContexts(NServiceBus.Pipeline.IRecoverabilityActionContext context) { } } - public sealed class IncomingPipelineMetricTags - { - public IncomingPipelineMetricTags() { } - public void Add(string tagKey, object value) { } - public void ApplyTag(ref System.Diagnostics.TagList tagList, string tagKey) { } - public void ApplyTags(ref System.Diagnostics.TagList tagList, System.ReadOnlySpan tagKeys) { } - } public static class InstallConfigExtensions { public static void AddInstaller<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembers(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.None | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicParameterlessConstructor | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicMethods | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.Interfaces)] TInstaller>(this NServiceBus.EndpointConfiguration config) @@ -774,6 +771,16 @@ namespace NServiceBus public MetersOptions() { } public bool EmitExecutionResultTags { get; set; } } + public static class MetricTagsExtensions + { + extension(NServiceBus.Pipeline.IBehaviorContext context) + { + public NServiceBus.IMetricsTags MetricTags { get; } + } + extension(NServiceBus.Transport.MessageContext context) + { + } + } public class MoveToError : NServiceBus.RecoverabilityAction { protected MoveToError(string errorQueue) { } diff --git a/src/NServiceBus.Core.Tests/NServiceBus.Core.Tests.csproj b/src/NServiceBus.Core.Tests/NServiceBus.Core.Tests.csproj index 52ccab6f296..9d3ef87ddac 100644 --- a/src/NServiceBus.Core.Tests/NServiceBus.Core.Tests.csproj +++ b/src/NServiceBus.Core.Tests/NServiceBus.Core.Tests.csproj @@ -4,7 +4,6 @@ net10.0 true ..\NServiceBusTests.snk - 13.0 diff --git a/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs b/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs index e04e3999d2c..1ce103a841c 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs @@ -40,7 +40,7 @@ public void Should_not_fail_when_handling_more_than_one_logical_message() Assert.DoesNotThrowAsync(async () => await behavior.Invoke(context, c => { - c.Extensions.Get().Add("Same", "Same"); + c.IncomingMetricTags.Add("Same", "Same"); return Task.CompletedTask; })); } diff --git a/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs index 573645d8750..1afa8f9986c 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs @@ -47,7 +47,7 @@ public override async Task Invoke(IIncomingPhysicalMessageContext context, Func< { if (first) // ignore the legacy case in which a single message payload contained multiple messages { - var availableMetricTags = context.Extensions.Get(); + var availableMetricTags = context.IncomingMetricTags; availableMetricTags.Add(MeterTags.MessageType, message.MessageType.FullName!); first = false; } diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IMetricsTags.cs b/src/NServiceBus.Core/Pipeline/Incoming/IMetricsTags.cs new file mode 100644 index 00000000000..d5b90d6446a --- /dev/null +++ b/src/NServiceBus.Core/Pipeline/Incoming/IMetricsTags.cs @@ -0,0 +1,23 @@ +#nullable enable + +namespace NServiceBus; + +/// +/// The tags applied to the metrics reported for the message currently being processed. +/// +public interface IMetricsTags +{ + /// + /// Adds the specified tag and value to , overwriting any value previously added + /// for that instrument and tag key, and taking precedence over the value NServiceBus reports for that tag. + /// + /// + /// Tags are scoped to a single instrument because a tag value is frequently only valid for one measurement (for + /// example, which handler just ran) rather than a fact that holds for every metric reported for the message. For + /// the same reason a later call for the same instrument and tag key replaces an earlier one. + /// + /// The tag to add. + /// The value assigned to the tag. + /// The name of the instrument the tag applies to. + void AddOrOverride(string tagKey, object value, string instrumentName); +} diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetricTags.cs b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetricTags.cs index 534d735e08b..f1f0f4d7a47 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetricTags.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetricTags.cs @@ -9,9 +9,26 @@ namespace NServiceBus; /// /// Captures possible metric tags that can be applied to a metric throughout the incoming processing pipeline. /// -public sealed class IncomingPipelineMetricTags +sealed class IncomingPipelineMetricTags : IMetricsTags { - Dictionary>? tags; + readonly Dictionary> tags = []; + readonly Dictionary>> instrumentTags = []; + + /// + /// + /// Unlike , this overwrites rather than keeping the first value, and the tag + /// isn't visible to other instruments applying tags from this collection. + /// + public void AddOrOverride(string tagKey, object value, string instrumentName) + { + if (!instrumentTags.TryGetValue(instrumentName, out var perInstrumentTags)) + { + perInstrumentTags = []; + instrumentTags.Add(instrumentName, perInstrumentTags); + } + + perInstrumentTags[tagKey] = new(tagKey, value); + } /// /// Adds the specified tag and value to the collection if not already present. @@ -19,48 +36,55 @@ public sealed class IncomingPipelineMetricTags /// The tag to add. /// The value assigned to the tag. public void Add(string tagKey, object value) - { - tags ??= []; - // We are using tryAdd to mitigate multiple logical messages transmitted in a single physical message - tags.TryAdd(tagKey, new(tagKey, value)); - } + => tags.TryAdd(tagKey, new KeyValuePair(tagKey, value)); /// - /// Applies the specified tag to the . + /// Applies the specified tags to the , replacing any tag already in + /// with a matching key - so a caller can populate with its + /// own computed defaults before calling this, and have any matching tag from this collection take precedence. + /// General tags (from ) are only applied when their key is in + /// . When is provided, every tag added for that + /// instrument via is applied unconditionally - regardless of whether its + /// key is in - and takes precedence over a general tag with the same key: naming an + /// instrument when adding a tag is already an explicit statement of intent for that one instrument, so callers + /// don't also need to know about it to pull it in. /// - /// The tagList to apply the specified tag to. - /// The tag to add to the . - public void ApplyTag(ref TagList tagList, string tagKey) + /// The tagList to add the tags to. + /// The collection of tag keys to apply to the . + /// The instrument to apply instrument-specific tags for, if any. + public void ApplyTags(ref TagList tagList, ReadOnlySpan tagKeys, string? instrumentName = null) { - if (tags == null) + foreach (var tagKey in tagKeys) { - return; + if (tags.TryGetValue(tagKey, out var keyValuePair)) + { + SetOrAdd(ref tagList, keyValuePair); + } } - if (tags.TryGetValue(tagKey, out var keyValuePair)) + if (instrumentName != null && instrumentTags.TryGetValue(instrumentName, out var perInstrumentTags)) { - tagList.Add(keyValuePair); + foreach (var (_, keyValuePair) in perInstrumentTags) + { + SetOrAdd(ref tagList, keyValuePair); + } } } - /// - /// Applies the specified tags to the . - /// - /// The tagList to add the tags to. - /// The collection of tag keys to apply to the . - public void ApplyTags(ref TagList tagList, ReadOnlySpan tagKeys) + // A caller may have already added a computed default for this key directly to tagList before calling + // ApplyTag/ApplyTags. Replacing it in place - rather than appending a duplicate - is what lets a tag from this + // collection act as an override regardless of how a consumer of the recorded measurement handles duplicate keys. + static void SetOrAdd(ref TagList tagList, KeyValuePair tag) { - if (tags == null || tagKeys.IsEmpty) + for (var i = 0; i < tagList.Count; i++) { - return; - } - - foreach (var tagKey in tagKeys) - { - if (tags.TryGetValue(tagKey, out var keyValuePair)) + if (tagList[i].Key == tag.Key) { - tagList.Add(keyValuePair); + tagList[i] = tag; + return; } } + + tagList.Add(tag); } -} \ No newline at end of file +} diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs index 5f9a7cff744..45ee4da5c5e 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs @@ -88,19 +88,20 @@ public void RecordProcessingTime(ITransportReceiveContext context, TimeSpan elap return; } - var incomingPipelineMetricTags = context.Extensions.Get(); - TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator, - MeterTags.MessageType, - MeterTags.MessageHandlerTypes]); - if (emitExecutionResultTags) { + // Execution result is to be removed so we don't support overriding it by the user tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); } + + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.MessageHandlerTypes], + processingTime.Name); + processingTime.Record(elapsed.TotalSeconds, tags); } @@ -110,32 +111,34 @@ public void RecordCriticalTimeAndTotalProcessed(ITransportReceiveContext context { return; } - - var incomingPipelineMetricTags = context.Extensions.Get(); - TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator, - MeterTags.MessageType, - MeterTags.MessageHandlerTypes]); - if (emitExecutionResultTags) { + // Execution result is to be removed so we don't support overriding it by the user tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); } + // totalProcessedSuccessfully and criticalTime always share the same tags in this method, so overrides are + // looked up under criticalTime's instrument name. + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.MessageHandlerTypes], + criticalTime.Name); + if (totalProcessedSuccessfully.Enabled) { totalProcessedSuccessfully.Add(1, tags); } - var completedAt = DateTimeOffset.UtcNow; if (criticalTime.Enabled) { if (context.Message.Headers.TryGetDeliverAt(out var startTime) || context.Message.Headers.TryGetTimeSent(out startTime)) { + var completedAt = DateTimeOffset.UtcNow; var criticalTimeElapsed = completedAt - startTime; + criticalTime.Record(criticalTimeElapsed.TotalSeconds, tags); } } @@ -149,16 +152,21 @@ public void RecordMessageProcessingFailure(IncomingPipelineMetricTags incomingPi } TagList tags; - tags.Add(new(MeterTags.ErrorType, error.GetType().FullName)); - incomingPipelineMetricTags.ApplyTags(ref tags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator, - MeterTags.MessageType, - MeterTags.MessageHandlerTypes]); if (emitExecutionResultTags) { + // Execution result is to be removed so we don't support overriding it by the user tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); } + + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.MessageHandlerTypes, + MeterTags.ErrorType], + totalFailures.Name); totalFailures.Add(1, tags); // the processing and critical time are intentionally not recorded in case of failure @@ -175,7 +183,8 @@ public void RecordFetchedMessage(IncomingPipelineMetricTags incomingPipelineMetr incomingPipelineMetricTags.ApplyTags(ref tags, [ MeterTags.EndpointDiscriminator, MeterTags.QueueName, - MeterTags.MessageType]); + MeterTags.MessageType], + totalFetched.Name); totalFetched.Add(1, tags); } @@ -187,118 +196,126 @@ public void RecordDeduplicatedMessage(ITransportReceiveContext context) return; } - var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.EndpointDiscriminator, MeterTags.QueueName, - MeterTags.MessageType]); + MeterTags.MessageType], + totalDeduplicated.Name); totalDeduplicated.Add(1, tags); } - public void RecordSuccessfulMessageHandlerTime(IInvokeHandlerContext invokeHandlerContext, TimeSpan elapsed) + public void RecordSuccessfulMessageHandlerTime(IInvokeHandlerContext context, TimeSpan elapsed) { if (!messageHandlerTime.Enabled) { return; } - var incomingPipelineMetricTags = invokeHandlerContext.Extensions.Get(); - TagList meterTags; - incomingPipelineMetricTags.ApplyTags(ref meterTags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator, - MeterTags.MessageType, - MeterTags.MessageHandlerType]); - // This is what Add(string, object) does so skipping an unnecessary stack frame - meterTags.Add(new KeyValuePair(MeterTags.MessageHandlerType, invokeHandlerContext.MessageHandler.HandlerType.FullName)); + TagList tags; if (emitExecutionResultTags) { - meterTags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); + // Execution result is to be removed so we don't support overriding it by the user + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); } - messageHandlerTime.Record(elapsed.TotalSeconds, meterTags); + + tags.Add(new KeyValuePair(MeterTags.MessageHandlerType, context.MessageHandler.HandlerType.FullName)); + + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.MessageHandlerType], + messageHandlerTime.Name); + messageHandlerTime.Record(elapsed.TotalSeconds, tags); } - public void RecordFailedMessageHandlerTime(IInvokeHandlerContext invokeHandlerContext, TimeSpan elapsed, Exception error) + public void RecordFailedMessageHandlerTime(IInvokeHandlerContext context, TimeSpan elapsed, Exception error) { if (!messageHandlerTime.Enabled) { return; } - var incomingPipelineMetricTags = invokeHandlerContext.Extensions.Get(); - TagList meterTags; - incomingPipelineMetricTags.ApplyTags(ref meterTags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator, - MeterTags.MessageType, - MeterTags.MessageHandlerType]); - // This is what Add(string, object) does so skipping an unnecessary stack frame - meterTags.Add(new KeyValuePair(MeterTags.MessageHandlerType, invokeHandlerContext.MessageHandler.HandlerType.FullName)); - meterTags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + TagList tags; if (emitExecutionResultTags) { - meterTags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); + // Execution result is to be removed so we don't support overriding it by the user + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); } - messageHandlerTime.Record(elapsed.TotalSeconds, meterTags); + + tags.Add(new KeyValuePair(MeterTags.MessageHandlerType, context.MessageHandler.HandlerType.FullName)); + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.MessageHandlerType, + MeterTags.ErrorType], + messageHandlerTime.Name); + messageHandlerTime.Record(elapsed.TotalSeconds, tags); } - public void RecordImmediateRetry(IRecoverabilityContext recoverabilityContext) + public void RecordImmediateRetry(IRecoverabilityContext context) { if (!totalImmediateRetries.Enabled) { return; } - var incomingPipelineMetricTags = recoverabilityContext.Extensions.Get(); - TagList meterTags; - incomingPipelineMetricTags.ApplyTags(ref meterTags, [ + TagList tags; + tags.Add(new KeyValuePair(MeterTags.ErrorType, context.Exception.GetType().FullName)); + + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, - MeterTags.MessageHandlerType]); - // This is what Add(string, object) does so skipping an unnecessary stack frame - meterTags.Add(new KeyValuePair(MeterTags.ErrorType, recoverabilityContext.Exception.GetType().FullName)); - totalImmediateRetries.Add(1, meterTags); + MeterTags.MessageHandlerType, + MeterTags.ErrorType], + totalImmediateRetries.Name); + totalImmediateRetries.Add(1, tags); } - public void RecordDelayedRetry(IRecoverabilityContext recoverabilityContext) + public void RecordDelayedRetry(IRecoverabilityContext context) { if (!totalDelayedRetries.Enabled) { return; } - var incomingPipelineMetricTags = recoverabilityContext.Extensions.Get(); - TagList meterTags; - incomingPipelineMetricTags.ApplyTags(ref meterTags, [ + TagList tags; + tags.Add(new KeyValuePair(MeterTags.ErrorType, context.Exception.GetType().FullName)); + + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, - MeterTags.MessageHandlerType]); - // This is what Add(string, object) does so skipping an unnecessary stack frame - meterTags.Add(new KeyValuePair(MeterTags.ErrorType, recoverabilityContext.Exception.GetType().FullName)); - totalDelayedRetries.Add(1, meterTags); + MeterTags.MessageHandlerType, + MeterTags.ErrorType], + totalDelayedRetries.Name); + totalDelayedRetries.Add(1, tags); } - public void RecordSendToErrorQueue(IRecoverabilityContext recoverabilityContext) + public void RecordSendToErrorQueue(IRecoverabilityContext context) { if (!totalSentToErrorQueue.Enabled) { return; } - var incomingPipelineMetricTags = recoverabilityContext.Extensions.Get(); - TagList meterTags; - incomingPipelineMetricTags.ApplyTags(ref meterTags, [ + TagList tags; + tags.Add(new KeyValuePair(MeterTags.ErrorType, context.Exception.GetType().FullName)); + + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, - MeterTags.MessageHandlerType]); - // This is what Add(string, object) does so skipping an unnecessary stack frame - meterTags.Add(new KeyValuePair(MeterTags.ErrorType, recoverabilityContext.Exception.GetType().FullName)); - totalSentToErrorQueue.Add(1, meterTags); + MeterTags.MessageHandlerType, + MeterTags.ErrorType], + totalSentToErrorQueue.Name); + totalSentToErrorQueue.Add(1, tags); } public ActiveMessageScope TrackMessageProcessing(IncomingPipelineMetricTags incomingPipelineMetricTags, IncomingMessage message) @@ -309,11 +326,15 @@ public ActiveMessageScope TrackMessageProcessing(IncomingPipelineMetricTags inco } TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [MeterTags.QueueName, MeterTags.EndpointDiscriminator]); if (message.Headers.TryGetValue(Headers.EnclosedMessageTypes, out var enclosedMessageTypes)) { tags.Add(new KeyValuePair(MeterTags.EnclosedMessageTypes, enclosedMessageTypes)); } + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.EnclosedMessageTypes], + activeMessages.Name); activeMessages.Add(1, tags); return new ActiveMessageScope(activeMessages, tags); @@ -326,21 +347,25 @@ public void RecordSagaFetchTime(IInvokeHandlerContext context, TimeSpan elapsed, return; } - var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator, - MeterTags.MessageType]); + if (emitExecutionResultTags) + { + // Execution result is to be removed so we don't support overriding it by the user + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); + } tags.Add(new KeyValuePair(MeterTags.SagaType, sagaType)); if (error != null) { tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); } - if (emitExecutionResultTags) - { - tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); - } + + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.SagaType, + MeterTags.ErrorType], + sagaFetchTime.Name); sagaFetchTime.Record(elapsed.TotalSeconds, tags); } @@ -351,30 +376,35 @@ public void RecordDeserializeTime(IIncomingPhysicalMessageContext context, TimeS return; } - var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator - ]); - if (error != null) - { - tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); - } if (context.Message.Headers.TryGetValue(Headers.EnclosedMessageTypes, out var messageTypes)) { tags.Add(new KeyValuePair(MeterTags.EnclosedMessageTypes, messageTypes)); } + if (error != null) + { + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + } if (emitExecutionResultTags) { + // Execution result is to be removed so we don't support overriding it by the user tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); } + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.EnclosedMessageTypes, + MeterTags.ErrorType], + messageDeserializeTime.Name); + messageDeserializeTime.Record(elapsed.TotalSeconds, tags); } - public void RecordSerializeTime(TimeSpan elapsed, string? messageType, Exception? error = null) + public void RecordSerializeTime(IOutgoingLogicalMessageContext context, TimeSpan elapsed, string? messageType, Exception? error = null) { + // No incoming pipeline context is available here (this fires from the outgoing send pipeline, which may + // run with no incoming message at all), so there's no IncomingPipelineMetricTags to route these through. if (!messageSerializeTime.Enabled) { return; @@ -383,16 +413,22 @@ public void RecordSerializeTime(TimeSpan elapsed, string? messageType, Exception TagList tags; if (messageType != null) { - tags.Add(new KeyValuePair(MeterTags.MessageType, messageType)); + tags.Add(new KeyValuePair(MeterTags.MessageType, messageType)); // tag-bag-bypass: see comment above } if (error != null) { - tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); // tag-bag-bypass: see comment above } if (emitExecutionResultTags) { - tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); // tag-bag-bypass: see comment above } + + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.MessageType, + MeterTags.ErrorType], + messageSerializeTime.Name); + messageSerializeTime.Record(elapsed.TotalSeconds, tags); } @@ -403,11 +439,12 @@ public void RecordOutboxFetchTime(ITransportReceiveContext context, TimeSpan ela return; } - var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, - MeterTags.EndpointDiscriminator]); + MeterTags.EndpointDiscriminator], + outboxFetchTime.Name); + outboxFetchTime.Record(elapsed.TotalSeconds, tags); } @@ -418,11 +455,12 @@ public void RecordOutboxStoreTime(ITransportReceiveContext context, TimeSpan ela return; } - var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, - MeterTags.EndpointDiscriminator]); + MeterTags.EndpointDiscriminator], + outboxStoreTime.Name); + outboxStoreTime.Record(elapsed.TotalSeconds, tags); } @@ -433,13 +471,14 @@ public void RecordPersistenceTime(IIncomingLogicalMessageContext context, TimeSp return; } - var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, - MeterTags.MessageHandlerTypes]); + MeterTags.MessageHandlerTypes], + persistenceTime.Name); + persistenceTime.Record(elapsed.TotalSeconds, tags); } @@ -451,19 +490,21 @@ void RecordEnvelopeUnwrapping(MessageContext messageContext, IEnvelopeHandler ty { return; } - - var incomingPipelineMetricTags = messageContext.Extensions.Get(); - TagList meterTags; - incomingPipelineMetricTags.ApplyTags(ref meterTags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator]); - meterTags.Add(new KeyValuePair(MeterTags.EnvelopeUnwrapperType, type.GetType().FullName)); + TagList tags; + tags.Add(new KeyValuePair(MeterTags.EnvelopeUnwrapperType, type.GetType().FullName)); if (exception != null) { - meterTags.Add(new KeyValuePair(MeterTags.ErrorType, exception.GetType().FullName)); + tags.Add(new KeyValuePair(MeterTags.ErrorType, exception.GetType().FullName)); } - totalEnvelopeUnwrapping.Add(succeeded ? 0 : 1, meterTags); + messageContext.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.EnvelopeUnwrapperType, + MeterTags.ErrorType], + totalEnvelopeUnwrapping.Name); + + totalEnvelopeUnwrapping.Add(succeeded ? 0 : 1, tags); } public readonly struct ActiveMessageScope(UpDownCounter? counter, TagList tags) : IDisposable @@ -493,4 +534,4 @@ public readonly struct ActiveMessageScope(UpDownCounter? counter, TagList readonly string queueNameBase; readonly string endpointDiscriminator; readonly bool emitExecutionResultTags; -} \ No newline at end of file +} diff --git a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs index 26a3b0178bf..bb140dcb96c 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs @@ -43,7 +43,7 @@ public override async Task Invoke(IIncomingLogicalMessageContext context, Func(); + var availableMetricTags = context.IncomingMetricTags; availableMetricTags.Add(MeterTags.MessageHandlerTypes, string.Join(';', handlersToInvoke.Select(x => x.HandlerType.FullName))); foreach (var messageHandler in handlersToInvoke) diff --git a/src/NServiceBus.Core/Pipeline/Incoming/MetricTagsExtensions.cs b/src/NServiceBus.Core/Pipeline/Incoming/MetricTagsExtensions.cs new file mode 100644 index 00000000000..46b05278214 --- /dev/null +++ b/src/NServiceBus.Core/Pipeline/Incoming/MetricTagsExtensions.cs @@ -0,0 +1,29 @@ +#nullable enable + +namespace NServiceBus; + +using Pipeline; +using Transport; + +/// +/// Provides access to the metric tags captured for the message currently being processed. +/// +public static class MetricTagsExtensions +{ + /// The context to extend. + extension(IBehaviorContext context) + { + /// + /// The collected for the message currently being processed. Add to this + /// collection to have the tags applied to the metrics emitted for that message. + /// + public IMetricsTags MetricTags => context.Extensions.GetOrCreate(); + + internal IncomingPipelineMetricTags IncomingMetricTags => context.Extensions.GetOrCreate(); + } + + extension(MessageContext context) + { + internal IncomingPipelineMetricTags IncomingMetricTags => context.Extensions.GetOrCreate(); + } +} diff --git a/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs b/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs index f6098dd0664..da6bd6ccd10 100644 --- a/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs +++ b/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs @@ -25,7 +25,7 @@ public async Task Invoke(MessageContext messageContext, CancellationToken cancel var pipelineStartedAt = DateTimeOffset.UtcNow; using var activity = activityFactory.StartIncomingPipelineActivity(messageContext); - var incomingPipelineMetricsTags = messageContext.Extensions.Get(); + var incomingPipelineMetricsTags = messageContext.IncomingMetricTags; incomingPipelineMetrics.AddDefaultIncomingPipelineMetricTags(incomingPipelineMetricsTags); diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs index 14bbad12000..f8fe0d459e8 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs @@ -45,13 +45,13 @@ public override async Task Invoke(IOutgoingLogicalMessageContext context, Func headers Extensions = context; ReceiveAddress = receiveAddress; TransportTransaction = transportTransaction; - - context.GetOrCreate(); } /// diff --git a/src/NServiceBus.Testing.Fakes/TestablePipelineContext.cs b/src/NServiceBus.Testing.Fakes/TestablePipelineContext.cs index ca256ad6b42..4090f548108 100644 --- a/src/NServiceBus.Testing.Fakes/TestablePipelineContext.cs +++ b/src/NServiceBus.Testing.Fakes/TestablePipelineContext.cs @@ -16,11 +16,7 @@ public partial class TestablePipelineContext : IPipelineContext /// /// Creates a new instance. /// - public TestablePipelineContext(IMessageCreator messageCreator = null) - { - this.messageCreator = messageCreator ?? new MessageMapper(); - Extensions.GetOrCreate(); - } + public TestablePipelineContext(IMessageCreator messageCreator = null) => this.messageCreator = messageCreator ?? new MessageMapper(); /// /// A list of all messages sent with a saga timeout header.