From 5df1ae0e5ce650ec1563c07d9f65649e4a0cecc3 Mon Sep 17 00:00:00 2001 From: Calvin Bayer Date: Mon, 19 Oct 2026 23:08:11 +0200 Subject: [PATCH 1/2] Apply automation exclusions to Karate scenarios in both major versions --- .../karate/KarateTracingHook.java | 4 ++- .../src/test/groovy/KarateTest.groovy | 27 +++++++++++++++++++ .../example/TestAutomationExcludedKarate.java | 11 ++++++++ .../example/test_automation_excluded.feature | 5 ++++ .../karate2/KarateTracingListener.java | 4 ++- .../src/test/groovy/KarateV2Test.groovy | 27 +++++++++++++++++++ .../example/TestAutomationExcludedKarate.java | 17 ++++++++++++ .../example/test_automation_excluded.feature | 5 ++++ 8 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/TestAutomationExcludedKarate.java create mode 100644 dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_automation_excluded.feature create mode 100644 dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestAutomationExcludedKarate.java create mode 100644 dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_automation_excluded.feature diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java b/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java index 4542e816c92..710418789ba 100644 --- a/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java @@ -115,7 +115,9 @@ public boolean beforeScenario(ScenarioRuntime sr) { if (Config.get().isCiVisibilityTestSkippingEnabled() || Config.get().isCiVisibilityTestManagementEnabled()) { TestIdentifier skippableTest = KarateUtils.toTestIdentifier(scenario); - SkipReason skipReason = TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason(skippableTest); + SkipReason skipReason = + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason( + skippableTest, TestSourceData.UNKNOWN, categories); if (skipReason != null && !(skipReason == SkipReason.ITR diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy index 4cafd545c09..3ffb1d4a80c 100644 --- a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy @@ -27,6 +27,33 @@ import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass @DisableTestTrace(reason = "avoid self-tracing") class KarateTest extends CiVisibilityInstrumentationTest { + def "test automation exclusions preserve a blocking failure and execute once"() { + given: + def id = new TestFQN("[org/example/test_automation_excluded] automation exclusions", "protected_failure") + givenDisabledTests([id]) + givenQuarantinedTests([id]) + givenAttemptToFixTests([id]) + givenFlakyRetryEnabled(true) + givenFlakyTests([id]) + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests([]) + + when: + runTests([TestAutomationExcludedKarate], false) + + then: + TEST_WRITER.toList().flatten().count { it.getTag("test.name") == "protected_failure" } == 1 + def testSpan = TEST_WRITER.toList().flatten().find { it.getTag("test.name") == "protected_failure" } + testSpan != null + testSpan.getTag("test.test_management.no_auto_retry") == true + testSpan.getTag("test.test_management.prevent_failure_suppression") == true + testSpan.getTag("test.test_management.is_test_disabled") == true + testSpan.getTag("test.test_management.is_quarantined") == true + testSpan.getTag("test.test_management.is_attempt_to_fix") == true + testSpan.getTag("test.test_management.attempt_to_fix_passed") == null + } + + def "test #testcaseName"() { Assumptions.assumeTrue(assumption) diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/TestAutomationExcludedKarate.java b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/TestAutomationExcludedKarate.java new file mode 100644 index 00000000000..574462995be --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/TestAutomationExcludedKarate.java @@ -0,0 +1,11 @@ +package org.example; + +import com.intuit.karate.junit5.Karate; + +public class TestAutomationExcludedKarate { + + @Karate.Test + public Karate testFailed() { + return Karate.run("classpath:org/example/test_automation_excluded.feature"); + } +} diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_automation_excluded.feature b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_automation_excluded.feature new file mode 100644 index 00000000000..77dc5f12a0d --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_automation_excluded.feature @@ -0,0 +1,5 @@ +Feature: automation exclusions + + @datadog_no_auto_retry @datadog_prevent_failure_suppression + Scenario: protected_failure + * assert false diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java b/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java index 84a5adbd524..ead16307d1f 100644 --- a/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java @@ -124,7 +124,9 @@ private boolean beforeScenario(ScenarioRunEvent event) { if (Config.get().isCiVisibilityTestSkippingEnabled() || Config.get().isCiVisibilityTestManagementEnabled()) { TestIdentifier skippableTest = KarateUtils.toTestIdentifier(scenario); - SkipReason skipReason = TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason(skippableTest); + SkipReason skipReason = + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason( + skippableTest, TestSourceData.UNKNOWN, categories); if (skipReason != null && !(skipReason == SkipReason.ITR diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy index a36885edbf4..c3a0e9b3876 100644 --- a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy @@ -21,6 +21,33 @@ import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass @DisableTestTrace(reason = "avoid self-tracing") class KarateV2Test extends CiVisibilityInstrumentationTest { + def "test automation exclusions preserve a blocking failure and execute once"() { + given: + def id = new TestFQN("[org/example/test_automation_excluded] automation exclusions", "protected_failure") + givenDisabledTests([id]) + givenQuarantinedTests([id]) + givenAttemptToFixTests([id]) + givenFlakyRetryEnabled(true) + givenFlakyTests([id]) + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests([]) + + when: + runTests([TestAutomationExcludedKarate], false) + + then: + TEST_WRITER.toList().flatten().count { it.getTag("test.name") == "protected_failure" } == 1 + def testSpan = TEST_WRITER.toList().flatten().find { it.getTag("test.name") == "protected_failure" } + testSpan != null + testSpan.getTag("test.test_management.no_auto_retry") == true + testSpan.getTag("test.test_management.prevent_failure_suppression") == true + testSpan.getTag("test.test_management.is_test_disabled") == true + testSpan.getTag("test.test_management.is_quarantined") == true + testSpan.getTag("test.test_management.is_attempt_to_fix") == true + testSpan.getTag("test.test_management.attempt_to_fix_passed") == null + } + + def "test #testcaseName"() { runTests(tests, success) diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestAutomationExcludedKarate.java b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestAutomationExcludedKarate.java new file mode 100644 index 00000000000..5a9e8ee0a1d --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestAutomationExcludedKarate.java @@ -0,0 +1,17 @@ +package org.example; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.intuit.karate.Results; +import com.intuit.karate.Runner; +import org.junit.jupiter.api.Test; + +public class TestAutomationExcludedKarate { + + @Test + public void test() { + Results results = + Runner.path("classpath:org/example/test_automation_excluded.feature").parallel(1); + assertEquals(0, results.getFailCount(), results.getErrorMessages()); + } +} diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_automation_excluded.feature b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_automation_excluded.feature new file mode 100644 index 00000000000..77dc5f12a0d --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_automation_excluded.feature @@ -0,0 +1,5 @@ +Feature: automation exclusions + + @datadog_no_auto_retry @datadog_prevent_failure_suppression + Scenario: protected_failure + * assert false From 004ac8967b2d67ed6b62794a3e9c484784568436 Mon Sep 17 00:00:00 2001 From: Calvin Bayer Date: Fri, 30 Oct 2026 12:14:11 +0100 Subject: [PATCH 2/2] Address karate annotation review feedback --- .../karate/karate-1.0/src/test/groovy/KarateTest.groovy | 1 - .../karate/karate-2.0/src/test/groovy/KarateV2Test.groovy | 1 - 2 files changed, 2 deletions(-) diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy index 3ffb1d4a80c..1c28f3a90dd 100644 --- a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy @@ -53,7 +53,6 @@ class KarateTest extends CiVisibilityInstrumentationTest { testSpan.getTag("test.test_management.attempt_to_fix_passed") == null } - def "test #testcaseName"() { Assumptions.assumeTrue(assumption) diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy index c3a0e9b3876..b4f02ca2d69 100644 --- a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy @@ -47,7 +47,6 @@ class KarateV2Test extends CiVisibilityInstrumentationTest { testSpan.getTag("test.test_management.attempt_to_fix_passed") == null } - def "test #testcaseName"() { runTests(tests, success)