From 00ee753776a51a5dcdccdd20cc1750a88d777776 Mon Sep 17 00:00:00 2001 From: Calvin Bayer Date: Mon, 19 Oct 2026 23:11:15 +0200 Subject: [PATCH 1/2] Apply automation exclusions and retain ScalaTest declaration tags --- .../scalatest/DatadogReporter.java | 9 ++--- .../instrumentation/scalatest/RunContext.java | 4 +-- .../ScalatestSkipInstrumentation.java | 8 ++--- .../src/test/groovy/ScalatestTest.groovy | 33 +++++++++++++++++++ .../org/example/TestAutomationExcluded.scala | 18 ++++++++++ docs/test-automation-exclusions.md | 6 ++++ 6 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 dd-java-agent/instrumentation/scalatest-3.0.8/src/test/scala/org/example/TestAutomationExcluded.scala diff --git a/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/DatadogReporter.java b/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/DatadogReporter.java index 8bd83c25569..2882e957733 100644 --- a/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/DatadogReporter.java +++ b/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/DatadogReporter.java @@ -1,6 +1,5 @@ package datadog.trace.instrumentation.scalatest; -import datadog.trace.api.civisibility.CIConstants; import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.config.TestSourceData; import datadog.trace.api.civisibility.events.TestDescriptor; @@ -150,13 +149,8 @@ private static void onTestStart(TestStarting event) { String testName = event.testName(); Object testQualifier = null; String testParameters = null; - Collection categories; TestIdentifier testIdentifier = new TestIdentifier(testSuiteName, testName, null); - if (context.itrUnskippable(testIdentifier)) { - categories = Collections.singletonList(CIConstants.Tags.ITR_UNSKIPPABLE_TAG); - } else { - categories = Collections.emptyList(); - } + Collection categories = context.tags(testIdentifier); Class testClass = ScalatestUtils.getClass(event.suiteClassName()); TestEventsHandler eventHandler = context.getEventHandler(); @@ -234,6 +228,7 @@ private static void onTestIgnore(TestIgnored event) { Class testClass = ScalatestUtils.getClass(event.suiteClassName()); TestIdentifier skippableTest = new TestIdentifier(testSuiteName, testName, null); + categories = context.tags(skippableTest); SkipReason reason = context.getSkipReason(skippableTest); TestEventsHandler eventHandler = context.getEventHandler(); diff --git a/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/RunContext.java b/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/RunContext.java index a0e9e378cc4..35a35f64d18 100644 --- a/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/RunContext.java +++ b/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/RunContext.java @@ -116,7 +116,7 @@ private Tuple2 skip( String testName = testNameAndSkipStatus._1(); TestIdentifier test = new TestIdentifier(suiteId, testName, null); - SkipReason skipReason = eventHandler.skipReason(test); + SkipReason skipReason = eventHandler.skipReason(test, TestSourceData.UNKNOWN, tags(test)); if (skipReason == null || skipReason == SkipReason.ITR && itrUnskippable(test)) { return testNameAndSkipStatus; } @@ -138,7 +138,7 @@ public void populateTags(TestIdentifier test, Map> tags) { } public boolean skip(TestIdentifier test, Map> tags) { - SkipReason skipReason = eventHandler.skipReason(test); + SkipReason skipReason = eventHandler.skipReason(test, TestSourceData.UNKNOWN, tags(test)); if (skipReason == null) { return false; } diff --git a/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/ScalatestSkipInstrumentation.java b/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/ScalatestSkipInstrumentation.java index 63e0a718890..beda555b1ec 100644 --- a/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/ScalatestSkipInstrumentation.java +++ b/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/ScalatestSkipInstrumentation.java @@ -92,16 +92,16 @@ public static void apply( @Advice.Argument(value = 1) scala.collection.immutable.Map> tags, @Advice.Argument(value = 2) String suiteId) { - if (filterResult == null // filter terminated exceptionally - || filterResult._1() // test is filtered - || filterResult._2() // test is ignored - ) { + if (filterResult == null) { return; } TestIdentifier test = new TestIdentifier(suiteId, testName, null); RunContext runContext = InstrumentationContext.get(Filter.class, RunContext.class).get(filter); runContext.populateTags(test, tags); + if (filterResult._1() || filterResult._2()) { + return; + } if (runContext.skip(test, tags)) { filterResult = new Tuple2<>(false, true); diff --git a/dd-java-agent/instrumentation/scalatest-3.0.8/src/test/groovy/ScalatestTest.groovy b/dd-java-agent/instrumentation/scalatest-3.0.8/src/test/groovy/ScalatestTest.groovy index 21290071d51..26bd51465fd 100644 --- a/dd-java-agent/instrumentation/scalatest-3.0.8/src/test/groovy/ScalatestTest.groovy +++ b/dd-java-agent/instrumentation/scalatest-3.0.8/src/test/groovy/ScalatestTest.groovy @@ -4,6 +4,7 @@ import datadog.trace.civisibility.CiVisibilityInstrumentationTest import datadog.trace.civisibility.diff.LineDiff import datadog.trace.instrumentation.scalatest.ScalatestUtils import org.example.TestFailed +import org.example.TestAutomationExcluded import org.example.TestFailedParameterized import org.example.TestFailedSuite import org.example.TestFailedThenSucceed @@ -32,6 +33,38 @@ import sbt.testing.TaskDef class ScalatestTest extends CiVisibilityInstrumentationTest { + def "test automation exclusions preserve a blocking failure and execute once"() { + given: + def id = new TestFQN("org.example.TestAutomationExcluded", "protected_failure") + givenDisabledTests([id]) + givenQuarantinedTests([id]) + givenAttemptToFixTests([id]) + givenFlakyRetryEnabled(true) + givenFlakyTests([id]) + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests([]) + + when: + runTests([TestAutomationExcluded], false) + + then: + TEST_WRITER.toList().flatten().count { it.getTag("test.name") == "protected_failure" } == 1 + def testSpan = TEST_WRITER.toList().flatten().find { it.getTag("test.name") == "protected_failure" } + testSpan != null + testSpan.getTag("test.test_management.no_auto_retry") == true + testSpan.getTag("test.test_management.prevent_failure_suppression") == true + testSpan.getTag("test.test_management.is_test_disabled") == true + testSpan.getTag("test.test_management.is_quarantined") == true + testSpan.getTag("test.test_management.is_attempt_to_fix") == true + testSpan.getTag("test.test_management.attempt_to_fix_passed") == null + def skippedSpan = TEST_WRITER.toList().flatten().find { it.getTag("test.name") == "protected_skip" } + skippedSpan != null + skippedSpan.getTag("test.test_management.no_auto_retry") == true + skippedSpan.getTag("test.test_management.prevent_failure_suppression") == true + skippedSpan.getTag("test.status").toString() == "skip" + } + + def "test #testcaseName"() { runTests(tests, success) diff --git a/dd-java-agent/instrumentation/scalatest-3.0.8/src/test/scala/org/example/TestAutomationExcluded.scala b/dd-java-agent/instrumentation/scalatest-3.0.8/src/test/scala/org/example/TestAutomationExcluded.scala new file mode 100644 index 00000000000..8858a70c0ee --- /dev/null +++ b/dd-java-agent/instrumentation/scalatest-3.0.8/src/test/scala/org/example/TestAutomationExcluded.scala @@ -0,0 +1,18 @@ +package org.example + +import datadog.trace.api.civisibility.TestAutomationExclusions +import org.scalatest.Tag +import org.scalatest.funsuite.AnyFunSuite + +class TestAutomationExcluded extends AnyFunSuite { + private val noRetry = new Tag(TestAutomationExclusions.NO_AUTO_RETRY) + private val preventSuppression = new Tag(TestAutomationExclusions.PREVENT_FAILURE_SUPPRESSION) + + test("protected_failure", noRetry, preventSuppression) { + assert(false) + } + + ignore("protected_skip", noRetry, preventSuppression) { + assert(false) + } +} diff --git a/docs/test-automation-exclusions.md b/docs/test-automation-exclusions.md index 33fb0e8474a..8285366066c 100644 --- a/docs/test-automation-exclusions.md +++ b/docs/test-automation-exclusions.md @@ -61,3 +61,9 @@ Tag the Scenario or Scenario Outline with `@datadog_no_auto_retry` and/or `@datadog_prevent_failure_suppression` (the names from the public `TestAutomationExclusions` constants). Both Karate 1.x and 2.x resolve these scenario tags before Datadog skipping and retry policy selection. + +## ScalaTest + +Declare ScalaTest `Tag` instances from `TestAutomationExclusions.NO_AUTO_RETRY` +and `PREVENT_FAILURE_SUPPRESSION`, then pass them to the test declaration. +Native ignored tests remain ignored and carry their known exclusion tags. From 627c8a57785833cf109d9d358c30c7bd84d9a8e4 Mon Sep 17 00:00:00 2001 From: Calvin Bayer Date: Mon, 19 Oct 2026 23:19:44 +0200 Subject: [PATCH 2/2] Preserve existing ScalaTest traits outside exclusion annotations --- .../instrumentation/scalatest/DatadogReporter.java | 8 ++++++-- .../trace/instrumentation/scalatest/RunContext.java | 12 ++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/DatadogReporter.java b/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/DatadogReporter.java index 2882e957733..c8661232dda 100644 --- a/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/DatadogReporter.java +++ b/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/DatadogReporter.java @@ -1,5 +1,6 @@ package datadog.trace.instrumentation.scalatest; +import datadog.trace.api.civisibility.CIConstants; import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.config.TestSourceData; import datadog.trace.api.civisibility.events.TestDescriptor; @@ -150,7 +151,10 @@ private static void onTestStart(TestStarting event) { Object testQualifier = null; String testParameters = null; TestIdentifier testIdentifier = new TestIdentifier(testSuiteName, testName, null); - Collection categories = context.tags(testIdentifier); + Collection categories = context.exclusionTags(testIdentifier); + if (context.itrUnskippable(testIdentifier)) { + categories.add(CIConstants.Tags.ITR_UNSKIPPABLE_TAG); + } Class testClass = ScalatestUtils.getClass(event.suiteClassName()); TestEventsHandler eventHandler = context.getEventHandler(); @@ -228,7 +232,7 @@ private static void onTestIgnore(TestIgnored event) { Class testClass = ScalatestUtils.getClass(event.suiteClassName()); TestIdentifier skippableTest = new TestIdentifier(testSuiteName, testName, null); - categories = context.tags(skippableTest); + categories = context.exclusionTags(skippableTest); SkipReason reason = context.getSkipReason(skippableTest); TestEventsHandler eventHandler = context.getEventHandler(); diff --git a/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/RunContext.java b/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/RunContext.java index 35a35f64d18..8d5b76f0c53 100644 --- a/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/RunContext.java +++ b/dd-java-agent/instrumentation/scalatest-3.0.8/src/main/java/datadog/trace/instrumentation/scalatest/RunContext.java @@ -81,6 +81,18 @@ public Collection tags(TestIdentifier test) { return tagsByTest.getOrDefault(test, Collections.emptyList()); } + /** Exclusions are the only new traits exposed by this integration. */ + public Collection exclusionTags(TestIdentifier test) { + Collection exclusions = new ArrayList<>(); + for (String tag : tags(test)) { + if ("datadog_no_auto_retry".equals(tag) + || "datadog_prevent_failure_suppression".equals(tag)) { + exclusions.add(tag); + } + } + return exclusions; + } + public void populateTags( String suiteId, Map> tags,