diff --git a/dd-java-agent/instrumentation/junit/junit-4/junit-4-cucumber-5.4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java b/dd-java-agent/instrumentation/junit/junit-4/junit-4-cucumber-5.4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java index 18ff1661285..ca005d06ab8 100644 --- a/dd-java-agent/instrumentation/junit/junit-4/junit-4-cucumber-5.4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java +++ b/dd-java-agent/instrumentation/junit/junit-4/junit-4-cucumber-5.4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java @@ -11,6 +11,7 @@ import datadog.trace.api.Config; import datadog.trace.api.civisibility.CIConstants; import datadog.trace.api.civisibility.config.TestIdentifier; +import datadog.trace.api.civisibility.config.TestSourceData; import datadog.trace.api.civisibility.telemetry.tag.SkipReason; import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -85,7 +86,7 @@ public static Boolean run( SkipReason skipReason = TestEventsHandlerHolder.HANDLERS .get(TestFrameworkInstrumentation.CUCUMBER) - .skipReason(test); + .skipReason(test, TestSourceData.UNKNOWN, CucumberUtils.getCategories(pickle)); if (skipReason == null) { return null; } diff --git a/dd-java-agent/instrumentation/junit/junit-4/junit-4-cucumber-5.4/src/test/groovy/CucumberTest.groovy b/dd-java-agent/instrumentation/junit/junit-4/junit-4-cucumber-5.4/src/test/groovy/CucumberTest.groovy index 444d3af841c..1f22034f281 100644 --- a/dd-java-agent/instrumentation/junit/junit-4/junit-4-cucumber-5.4/src/test/groovy/CucumberTest.groovy +++ b/dd-java-agent/instrumentation/junit/junit-4/junit-4-cucumber-5.4/src/test/groovy/CucumberTest.groovy @@ -17,6 +17,33 @@ class CucumberTest extends CiVisibilityInstrumentationTest { def runner = new JUnitCore() + def "test automation exclusions keep a protected scenario blocking and execute once"() { + given: + def path = "org/example/cucumber/calculator/automation_excluded.feature" + def id = new TestFQN("classpath:${path}:Basic Arithmetic", "Addition") + givenDisabledTests([id]) + givenQuarantinedTests([id]) + givenAttemptToFixTests([id]) + givenFlakyRetryEnabled(true) + givenFlakyTests([id]) + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests([]) + + when: + runFeatures([path], false) + + then: + def testSpans = TEST_WRITER.toList().flatten().findAll { it.getTag("test.name") == "Addition" } + testSpans.size() == 1 + def testSpan = testSpans.first() + testSpan.getTag("test.test_management.no_auto_retry") == true + testSpan.getTag("test.test_management.prevent_failure_suppression") == true + testSpan.getTag("test.test_management.is_test_disabled") == true + testSpan.getTag("test.test_management.is_quarantined") == true + testSpan.getTag("test.test_management.is_attempt_to_fix") == true + testSpan.getTag("test.test_management.attempt_to_fix_passed") == null + } + def "test #testcaseName"() { runFeatures(features, success) diff --git a/dd-java-agent/instrumentation/junit/junit-4/junit-4-cucumber-5.4/src/test/resources/org/example/cucumber/calculator/automation_excluded.feature b/dd-java-agent/instrumentation/junit/junit-4/junit-4-cucumber-5.4/src/test/resources/org/example/cucumber/calculator/automation_excluded.feature new file mode 100644 index 00000000000..de4f5ca2be1 --- /dev/null +++ b/dd-java-agent/instrumentation/junit/junit-4/junit-4-cucumber-5.4/src/test/resources/org/example/cucumber/calculator/automation_excluded.feature @@ -0,0 +1,10 @@ +@datadog_no_auto_retry @datadog_prevent_failure_suppression +Feature: Basic Arithmetic + + Background: A Calculator + Given a calculator I just turned on + + Scenario: Addition + # Try to change one of the values below to provoke a failure + When I add 4 and 5 + Then the result is 42 diff --git a/dd-java-agent/instrumentation/junit/junit-5/junit-5-cucumber-5.4/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberSkipInstrumentation.java b/dd-java-agent/instrumentation/junit/junit-5/junit-5-cucumber-5.4/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberSkipInstrumentation.java index 1833f955f40..faaf30d3d5a 100644 --- a/dd-java-agent/instrumentation/junit/junit-5/junit-5-cucumber-5.4/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberSkipInstrumentation.java +++ b/dd-java-agent/instrumentation/junit/junit-5/junit-5-cucumber-5.4/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberSkipInstrumentation.java @@ -11,6 +11,7 @@ import datadog.trace.api.Config; import datadog.trace.api.civisibility.CIConstants; import datadog.trace.api.civisibility.config.TestIdentifier; +import datadog.trace.api.civisibility.config.TestSourceData; import datadog.trace.api.civisibility.events.TestEventsHandler; import datadog.trace.api.civisibility.telemetry.tag.SkipReason; import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation; @@ -109,7 +110,9 @@ public static void shouldBeSkipped( return; } - SkipReason skipReason = testEventsHandler.skipReason(test); + SkipReason skipReason = + testEventsHandler.skipReason( + test, TestSourceData.UNKNOWN, JUnitPlatformUtils.getTags(testDescriptor)); if (skipReason == null) { return; } diff --git a/dd-java-agent/instrumentation/junit/junit-5/junit-5-cucumber-5.4/src/test/groovy/CucumberTest.groovy b/dd-java-agent/instrumentation/junit/junit-5/junit-5-cucumber-5.4/src/test/groovy/CucumberTest.groovy index 4bf70dfff60..5a428db5baa 100644 --- a/dd-java-agent/instrumentation/junit/junit-5/junit-5-cucumber-5.4/src/test/groovy/CucumberTest.groovy +++ b/dd-java-agent/instrumentation/junit/junit-5/junit-5-cucumber-5.4/src/test/groovy/CucumberTest.groovy @@ -23,6 +23,33 @@ import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass @DisableTestTrace(reason = "avoid self-tracing") class CucumberTest extends CiVisibilityInstrumentationTest { + def "test automation exclusions keep a protected scenario blocking and execute once"() { + given: + def path = "org/example/cucumber/calculator/automation_excluded.feature" + def id = new TestFQN("classpath:${path}:Basic Arithmetic", "Addition") + givenDisabledTests([id]) + givenQuarantinedTests([id]) + givenAttemptToFixTests([id]) + givenFlakyRetryEnabled(true) + givenFlakyTests([id]) + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests([]) + + when: + runFeatures([path], false, false) + + then: + def testSpans = TEST_WRITER.toList().flatten().findAll { it.getTag("test.name") == "Addition" } + testSpans.size() == 1 + def testSpan = testSpans.first() + testSpan.getTag("test.test_management.no_auto_retry") == true + testSpan.getTag("test.test_management.prevent_failure_suppression") == true + testSpan.getTag("test.test_management.is_test_disabled") == true + testSpan.getTag("test.test_management.is_quarantined") == true + testSpan.getTag("test.test_management.is_attempt_to_fix") == true + testSpan.getTag("test.test_management.attempt_to_fix_passed") == null + } + def "test #testcaseName"() { runFeatures(features, parallel) diff --git a/dd-java-agent/instrumentation/junit/junit-5/junit-5-cucumber-5.4/src/test/resources/org/example/cucumber/calculator/automation_excluded.feature b/dd-java-agent/instrumentation/junit/junit-5/junit-5-cucumber-5.4/src/test/resources/org/example/cucumber/calculator/automation_excluded.feature new file mode 100644 index 00000000000..c4d162941c2 --- /dev/null +++ b/dd-java-agent/instrumentation/junit/junit-5/junit-5-cucumber-5.4/src/test/resources/org/example/cucumber/calculator/automation_excluded.feature @@ -0,0 +1,9 @@ +@datadog_no_auto_retry @datadog_prevent_failure_suppression +Feature: Basic Arithmetic + + Background: A Calculator + Given a calculator I just turned on + + Scenario: Addition + When I add 4 and 5 + Then the result is 45 diff --git a/docs/test-automation-exclusions.md b/docs/test-automation-exclusions.md index eddef730731..cb2294435df 100644 --- a/docs/test-automation-exclusions.md +++ b/docs/test-automation-exclusions.md @@ -47,3 +47,10 @@ constants. User-supplied retry analyzers retain their existing behavior. Apply the public annotations directly to a Spock feature method. They apply to every data-driven iteration. Spec-level skip aggregation inspects the feature declaration before deciding whether the whole spec can be skipped. + +## Cucumber + +For either the JUnit4 or JUnit Platform Cucumber runner, tag a Scenario or +Scenario Outline with `@datadog_no_auto_retry` and/or +`@datadog_prevent_failure_suppression`. These names correspond to the public +`TestAutomationExclusions` constants. Tags apply to every outline example.