From a128fe12ba9852b1c00b05ebbbec8c2af1055ea5 Mon Sep 17 00:00:00 2001 From: Calvin Bayer Date: Mon, 19 Oct 2026 23:02:59 +0200 Subject: [PATCH 1/7] Apply automation exclusions and preserve explicit TestNG retries --- .../testng/TestNGSkipInstrumentation.java | 7 ++- .../testng/execution/RetryAnalyzer.java | 26 ++++++++++- .../execution/RetryAnnotationTransformer.java | 18 +++++++- .../instrumentation/testng/TestNGTest.groovy | 46 +++++++++++++++++++ .../org/example/TestAutomationExcluded.java | 17 +++++++ .../example/TestExplicitRetryExcluded.java | 28 +++++++++++ docs/test-automation-exclusions.md | 6 +++ 7 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestAutomationExcluded.java create mode 100644 dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestExplicitRetryExcluded.java diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGSkipInstrumentation.java b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGSkipInstrumentation.java index d04d698e5ba..7bb61743d56 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGSkipInstrumentation.java +++ b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGSkipInstrumentation.java @@ -10,6 +10,7 @@ import datadog.trace.api.Config; import datadog.trace.api.civisibility.CIConstants; import datadog.trace.api.civisibility.config.TestIdentifier; +import datadog.trace.api.civisibility.config.TestSourceData; import datadog.trace.api.civisibility.telemetry.tag.SkipReason; import java.lang.reflect.Method; import java.util.List; @@ -58,7 +59,11 @@ public static void invokeMethod( @Advice.Argument(2) final Object[] parameters) { TestIdentifier testIdentifier = TestNGUtils.toTestIdentifier(method, instance, parameters); SkipReason skipReason = - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason(testIdentifier); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason( + testIdentifier, + new TestSourceData( + instance != null ? instance.getClass() : method.getDeclaringClass(), method), + TestNGUtils.getGroups(method)); if (skipReason == null) { return; } diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnalyzer.java b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnalyzer.java index c066a75f1fc..a569edcdf6c 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnalyzer.java +++ b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnalyzer.java @@ -1,5 +1,6 @@ package datadog.trace.instrumentation.testng.execution; +import datadog.trace.api.civisibility.config.TestExclusions; import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.config.TestSourceData; import datadog.trace.api.civisibility.execution.TestExecutionPolicy; @@ -8,12 +9,17 @@ import datadog.trace.instrumentation.testng.TestNGUtils; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.Collection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.testng.IRetryAnalyzer; import org.testng.ITestResult; +import org.testng.annotations.Test; public class RetryAnalyzer implements IRetryAnalyzer { + private static final Logger LOGGER = LoggerFactory.getLogger(RetryAnalyzer.class); private volatile TestExecutionPolicy executionPolicy; + private IRetryAnalyzer frameworkRetryAnalyzer; @SuppressFBWarnings( value = "AT_STALE_THREAD_WRITE_OF_PRIMITIVE", @@ -27,6 +33,21 @@ public void createExecutionPolicy(ITestResult result) { TestIdentifier testIdentifier = TestNGUtils.toTestIdentifier(result); TestSourceData testSourceData = TestNGUtils.toTestSourceData(result); Collection testTags = TestNGUtils.getGroups(result); + if (TestExclusions.noAutoRetry(testSourceData, testTags) + && testSourceData.getTestMethod() != null) { + Test declaration = testSourceData.getTestMethod().getAnnotation(Test.class); + if (declaration != null + && IRetryAnalyzer.class.isAssignableFrom(declaration.retryAnalyzer()) + && !RetryAnalyzer.class.equals(declaration.retryAnalyzer())) { + try { + frameworkRetryAnalyzer = + (IRetryAnalyzer) + declaration.retryAnalyzer().getDeclaredConstructor().newInstance(); + } catch (Exception e) { + LOGGER.debug("Could not instantiate explicit framework retry analyzer", e); + } + } + } executionPolicy = TestEventsHandlerHolder.TEST_EVENTS_HANDLER.executionPolicy( testIdentifier, testSourceData, testTags); @@ -41,7 +62,10 @@ public boolean retry(ITestResult result) { return false; } createExecutionPolicy(result); - return executionPolicy.applicable(); + return frameworkRetryAnalyzer != null + && !result.isSuccess() + && frameworkRetryAnalyzer.retry(result) + || executionPolicy.applicable(); } public void setSuppressFailures(ITestResult result) { diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnnotationTransformer.java b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnnotationTransformer.java index 471716729eb..e6dbae30e99 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnnotationTransformer.java +++ b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnnotationTransformer.java @@ -1,9 +1,15 @@ package datadog.trace.instrumentation.testng.execution; +import datadog.trace.api.civisibility.config.TestExclusions; +import datadog.trace.api.civisibility.config.TestSourceData; +import datadog.trace.instrumentation.testng.TestNGUtils; import java.lang.reflect.Constructor; import java.lang.reflect.Method; +import java.util.Arrays; import org.testng.IAnnotationTransformer; +import org.testng.IRetryAnalyzer; import org.testng.annotations.ITestAnnotation; +import org.testng.annotations.Test; public class RetryAnnotationTransformer implements IAnnotationTransformer { private final IAnnotationTransformer delegate; @@ -15,7 +21,17 @@ public RetryAnnotationTransformer(IAnnotationTransformer delegate) { @Override public void transform( ITestAnnotation annotation, Class testClass, Constructor testConstructor, Method testMethod) { - annotation.setRetryAnalyzer(RetryAnalyzer.class); + // TestNG 6 has no Datadog execution-policy instrumentation. Keep explicit native retries. + Test declaration = testMethod != null ? testMethod.getAnnotation(Test.class) : null; + boolean preserveFrameworkRetry = + TestNGUtils.getTestNGVersion().startsWith("6.") + && declaration != null + && IRetryAnalyzer.class.isAssignableFrom(declaration.retryAnalyzer()) + && TestExclusions.noAutoRetry( + new TestSourceData(testClass, testMethod), Arrays.asList(annotation.getGroups())); + if (!preserveFrameworkRetry) { + annotation.setRetryAnalyzer(RetryAnalyzer.class); + } if (delegate != null) { delegate.transform(annotation, testClass, testConstructor, testMethod); } diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy index 57f534def69..1b15aacd4ea 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy @@ -16,6 +16,52 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { static currentTestNGVersion = TestNGUtils.getTestNGVersion() + def "test automation exclusions preserve a blocking failure and execute once"() { + given: + TestAutomationExcluded.executions = 0 + def id = new TestFQN("org.example.TestAutomationExcluded", "protected_failure") + givenDisabledTests([id]) + givenQuarantinedTests([id]) + givenAttemptToFixTests([id]) + givenFlakyRetryEnabled(true) + givenFlakyTests([id]) + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests([]) + + when: + runTests([TestAutomationExcluded], null, false) + + then: + TestAutomationExcluded.executions == 1 + def testSpan = TEST_WRITER.toList().flatten().find { it.getTag("test.name") == "protected_failure" } + testSpan != null + testSpan.getTag("test.test_management.no_auto_retry") == true + testSpan.getTag("test.test_management.prevent_failure_suppression") == true + testSpan.getTag("test.test_management.is_test_disabled") == true + testSpan.getTag("test.test_management.is_quarantined") == true + testSpan.getTag("test.test_management.is_attempt_to_fix") == true + testSpan.getTag("test.test_management.attempt_to_fix_passed") == null + } + + + def "test automation exclusions preserve explicit framework retries"() { + given: + TestExplicitRetryExcluded.executions = 0 + givenFlakyRetryEnabled(true) + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests([]) + + when: + runTests([TestExplicitRetryExcluded], null, !currentTestNGVersion.startsWith("6.")) + + then: + TestExplicitRetryExcluded.executions == 2 + def testSpans = TEST_WRITER.toList().flatten().findAll { it.getTag("test.name") == "explicit_retry" } + testSpans.size() == 2 + testSpans.every { it.getTag("test.test_management.no_auto_retry") == true } + testSpans.every { it.getTag("test.is_retry") == null } + } + def "test #testcaseName"() { runTests(tests, null, success) diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestAutomationExcluded.java b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestAutomationExcluded.java new file mode 100644 index 00000000000..710e5af424b --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestAutomationExcluded.java @@ -0,0 +1,17 @@ +package org.example; + +import datadog.trace.api.civisibility.NoAutoRetry; +import datadog.trace.api.civisibility.PreventFailureSuppression; +import org.testng.annotations.Test; + +public class TestAutomationExcluded { + public static int executions; + + @Test + @NoAutoRetry + @PreventFailureSuppression + public void protected_failure() { + executions++; + throw new AssertionError("protected failure"); + } +} diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestExplicitRetryExcluded.java b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestExplicitRetryExcluded.java new file mode 100644 index 00000000000..667af018d15 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestExplicitRetryExcluded.java @@ -0,0 +1,28 @@ +package org.example; + +import datadog.trace.api.civisibility.NoAutoRetry; +import datadog.trace.api.civisibility.PreventFailureSuppression; +import org.testng.IRetryAnalyzer; +import org.testng.ITestResult; +import org.testng.annotations.Test; + +public class TestExplicitRetryExcluded { + public static int executions; + + @Test(retryAnalyzer = RetryOnce.class) + @NoAutoRetry + @PreventFailureSuppression + public void explicit_retry() { + if (++executions == 1) { + throw new AssertionError("first framework attempt"); + } + } + + public static class RetryOnce implements IRetryAnalyzer { + private int retries; + + public boolean retry(ITestResult result) { + return retries++ == 0; + } + } +} diff --git a/docs/test-automation-exclusions.md b/docs/test-automation-exclusions.md index d0785528f32..634c571a6ef 100644 --- a/docs/test-automation-exclusions.md +++ b/docs/test-automation-exclusions.md @@ -35,3 +35,9 @@ Use the same method annotations with `org.junit.jupiter.api.Test`, `ParameterizedTest`, repeated tests and test templates. Every generated invocation resolves the declaration on its source method. Native JUnit tags can also reference `TestAutomationExclusions` constants. + +## TestNG + +Apply the public annotations to `org.testng.annotations.Test` methods, including +data providers. TestNG groups can also reference `TestAutomationExclusions` +constants. User-supplied retry analyzers retain their existing behavior. From 32f540bab26a326a1b75ec51ce7be1ce3394c9d8 Mon Sep 17 00:00:00 2001 From: Calvin Bayer Date: Mon, 19 Oct 2026 23:15:30 +0200 Subject: [PATCH 2/7] Handle missing TestNG version metadata when preserving native retries --- .../testng/execution/RetryAnnotationTransformer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnnotationTransformer.java b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnnotationTransformer.java index e6dbae30e99..4aaffeffeb7 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnnotationTransformer.java +++ b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnnotationTransformer.java @@ -23,8 +23,10 @@ public void transform( ITestAnnotation annotation, Class testClass, Constructor testConstructor, Method testMethod) { // TestNG 6 has no Datadog execution-policy instrumentation. Keep explicit native retries. Test declaration = testMethod != null ? testMethod.getAnnotation(Test.class) : null; + String frameworkVersion = TestNGUtils.getTestNGVersion(); boolean preserveFrameworkRetry = - TestNGUtils.getTestNGVersion().startsWith("6.") + frameworkVersion != null + && frameworkVersion.startsWith("6.") && declaration != null && IRetryAnalyzer.class.isAssignableFrom(declaration.retryAnalyzer()) && TestExclusions.noAutoRetry( From a3bf9447ea6fc5232124580102a81453deb3ebae Mon Sep 17 00:00:00 2001 From: Calvin Bayer Date: Tue, 20 Oct 2026 00:39:05 +0200 Subject: [PATCH 3/7] Preserve TestNG native retry analyzer resolution and construction --- .../testng/TestNGInstrumentation.java | 4 +- .../testng/TestNGMethodInstrumentation.java | 43 ++++++++++ .../instrumentation/testng/TestNGUtils.java | 62 +++++++++++++- .../testng/execution/RetryAnalyzer.java | 50 +++++------ .../execution/RetryAnnotationTransformer.java | 59 +++++++++---- .../instrumentation/testng/TestNGTest.groovy | 84 ++++++++++++++++++- .../org/example/TestClassRetryExcluded.java | 16 ++++ .../org/example/TestFactoryRetryExcluded.java | 34 ++++++++ .../TestNativeQuarantinedExcluded.java | 15 ++++ .../example/TestTransformedNativeRetry.java | 14 ++++ 10 files changed, 330 insertions(+), 51 deletions(-) create mode 100644 dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGMethodInstrumentation.java create mode 100644 dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestClassRetryExcluded.java create mode 100644 dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestFactoryRetryExcluded.java create mode 100644 dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestNativeQuarantinedExcluded.java create mode 100644 dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestTransformedNativeRetry.java diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGInstrumentation.java b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGInstrumentation.java index a37da23e5d7..4d07a738c23 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGInstrumentation.java +++ b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGInstrumentation.java @@ -71,9 +71,7 @@ public static void addTracingListener(@Advice.This final TestNG testNG) { testNG.addListener((ITestNGListener) suiteListener); if (Config.get().isCiVisibilityExecutionPoliciesEnabled()) { - final RetryAnnotationTransformer transformer = - new RetryAnnotationTransformer(testNG.getAnnotationTransformer()); - testNG.addListener(transformer); + testNG.addListener(new RetryAnnotationTransformer(testNG.getAnnotationTransformer())); } } diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGMethodInstrumentation.java b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGMethodInstrumentation.java new file mode 100644 index 00000000000..a9d62eea483 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGMethodInstrumentation.java @@ -0,0 +1,43 @@ +package datadog.trace.instrumentation.testng; + +import static net.bytebuddy.matcher.ElementMatchers.isConstructor; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.api.Config; +import net.bytebuddy.asm.Advice; +import org.testng.ITestNGMethod; +import org.testng.internal.annotations.IAnnotationFinder; + +@AutoService(InstrumenterModule.class) +public class TestNGMethodInstrumentation extends InstrumenterModule.CiVisibility + implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { + public TestNGMethodInstrumentation() { + super("testng", "test-retry"); + } + + @Override + public boolean isEnabled() { + return super.isEnabled() && Config.get().isCiVisibilityExecutionPoliciesEnabled(); + } + + @Override + public String instrumentedType() { + return "org.testng.internal.TestNGMethod"; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice(isConstructor(), getClass().getName() + "$ConfigureRetryAdvice"); + } + + public static class ConfigureRetryAdvice { + @Advice.OnMethodExit + public static void configure( + @Advice.This ITestNGMethod method, + @Advice.FieldValue("m_annotationFinder") IAnnotationFinder annotationFinder) { + TestNGUtils.configureRetryAnalyzer(method, annotationFinder); + } + } +} diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGUtils.java b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGUtils.java index c67da143832..3d4c5d97978 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGUtils.java +++ b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGUtils.java @@ -2,9 +2,12 @@ import datadog.json.JsonWriter; import datadog.trace.api.civisibility.config.LibraryCapability; +import datadog.trace.api.civisibility.config.TestExclusions; import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.config.TestSourceData; import datadog.trace.api.civisibility.events.TestSuiteDescriptor; +import datadog.trace.instrumentation.testng.execution.RetryAnalyzer; +import datadog.trace.instrumentation.testng.execution.RetryAnnotationTransformer; import datadog.trace.util.ComparableVersion; import java.io.InputStream; import java.lang.invoke.MethodHandle; @@ -26,9 +29,11 @@ import org.testng.ITestNGMethod; import org.testng.ITestResult; import org.testng.TestNG; +import org.testng.annotations.ITestAnnotation; import org.testng.annotations.Test; import org.testng.internal.ConstructorOrMethod; import org.testng.internal.ITestResultNotifier; +import org.testng.internal.annotations.IAnnotationFinder; import org.testng.xml.XmlTest; public abstract class TestNGUtils { @@ -48,6 +53,49 @@ public abstract class TestNGUtils { private static final MethodHandle TEST_METHOD_GET_RETRY_ANALYZER_LEGACY = METHOD_HANDLES.method(ITestNGMethod.class, "getRetryAnalyzer"); + private static final MethodHandle TEST_METHOD_GET_RETRY_ANALYZER_CLASS = + METHOD_HANDLES.method(ITestNGMethod.class, "getRetryAnalyzerClass"); + private static final MethodHandle TEST_METHOD_SET_RETRY_ANALYZER_CLASS = + METHOD_HANDLES.method(ITestNGMethod.class, "setRetryAnalyzerClass", Class.class); + private static final MethodHandle TEST_METHOD_SET_RETRY_ANALYZER_LEGACY = + METHOD_HANDLES.method(ITestNGMethod.class, "setRetryAnalyzer", IRetryAnalyzer.class); + + /** Apply Datadog policy after TestNG resolves annotations and native analyzer construction. */ + public static void configureRetryAnalyzer( + ITestNGMethod method, IAnnotationFinder annotationFinder) { + TestSourceData source = + new TestSourceData(method.getRealClass(), method.getConstructorOrMethod().getMethod()); + ITestAnnotation annotation = + (ITestAnnotation) + annotationFinder.findAnnotation(source.getTestMethod(), ITestAnnotation.class); + if (annotation == null) { + annotation = + (ITestAnnotation) + annotationFinder.findAnnotation( + source.getTestMethod().getDeclaringClass(), ITestAnnotation.class); + } + if (TestExclusions.noAutoRetry(source, Arrays.asList(method.getGroups())) + || RetryAnnotationTransformer.hasFrameworkOverride(annotation)) { + if (TEST_METHOD_GET_RETRY_ANALYZER_CLASS != null) { + Class nativeAnalyzer = + METHOD_HANDLES.invoke(TEST_METHOD_GET_RETRY_ANALYZER_CLASS, method); + if (nativeAnalyzer != null + && !nativeAnalyzer + .getName() + .equals("org.testng.internal.annotations.DisabledRetryAnalyzer")) { + return; + } + } else if (METHOD_HANDLES.invoke(TEST_METHOD_GET_RETRY_ANALYZER_LEGACY, method) != null) { + return; + } + } + if (TEST_METHOD_SET_RETRY_ANALYZER_CLASS != null) { + METHOD_HANDLES.invoke(TEST_METHOD_SET_RETRY_ANALYZER_CLASS, method, RetryAnalyzer.class); + } else { + METHOD_HANDLES.invoke(TEST_METHOD_SET_RETRY_ANALYZER_LEGACY, method, new RetryAnalyzer()); + } + } + private static final ComparableVersion testNGv75 = new ComparableVersion("7.5"); private static final ComparableVersion testNGv70 = new ComparableVersion("7.0"); @@ -233,11 +281,17 @@ public static IRetryAnalyzer getRetryAnalyzer(ITestResult result) { return null; } IRetryAnalyzer analyzer = METHOD_HANDLES.invoke(TEST_METHOD_GET_RETRY_ANALYZER, method, result); - if (analyzer != null) { - return analyzer; - } else { - return METHOD_HANDLES.invoke(TEST_METHOD_GET_RETRY_ANALYZER_LEGACY, method); + if (analyzer == null) { + analyzer = METHOD_HANDLES.invoke(TEST_METHOD_GET_RETRY_ANALYZER_LEGACY, method); + } + if (analyzer != null + && !(analyzer instanceof RetryAnalyzer) + && TestExclusions.noAutoRetry(toTestSourceData(result), getGroups(result))) { + // TestNG invokes its own analyzer. Keep a separate Datadog policy for reporting and + // quarantine. + return RetryAnalyzer.forFrameworkMethod(method, toTestIdentifier(result)); } + return analyzer; } @Nonnull diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnalyzer.java b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnalyzer.java index a569edcdf6c..aaa5e5a678f 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnalyzer.java +++ b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnalyzer.java @@ -1,6 +1,5 @@ package datadog.trace.instrumentation.testng.execution; -import datadog.trace.api.civisibility.config.TestExclusions; import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.config.TestSourceData; import datadog.trace.api.civisibility.execution.TestExecutionPolicy; @@ -9,17 +8,36 @@ import datadog.trace.instrumentation.testng.TestNGUtils; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.Collection; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.WeakHashMap; import org.testng.IRetryAnalyzer; +import org.testng.ITestNGMethod; import org.testng.ITestResult; -import org.testng.annotations.Test; public class RetryAnalyzer implements IRetryAnalyzer { - private static final Logger LOGGER = LoggerFactory.getLogger(RetryAnalyzer.class); + private static final Map> FRAMEWORK_TRACKERS = + Collections.synchronizedMap( + new WeakHashMap>()); private volatile TestExecutionPolicy executionPolicy; - private IRetryAnalyzer frameworkRetryAnalyzer; + + public static RetryAnalyzer forFrameworkMethod(ITestNGMethod method, TestIdentifier test) { + synchronized (FRAMEWORK_TRACKERS) { + Map trackers = FRAMEWORK_TRACKERS.get(method); + if (trackers == null) { + trackers = new HashMap<>(); + FRAMEWORK_TRACKERS.put(method, trackers); + } + RetryAnalyzer tracker = trackers.get(test); + if (tracker == null) { + tracker = new RetryAnalyzer(); + trackers.put(test, tracker); + } + return tracker; + } + } @SuppressFBWarnings( value = "AT_STALE_THREAD_WRITE_OF_PRIMITIVE", @@ -33,21 +51,6 @@ public void createExecutionPolicy(ITestResult result) { TestIdentifier testIdentifier = TestNGUtils.toTestIdentifier(result); TestSourceData testSourceData = TestNGUtils.toTestSourceData(result); Collection testTags = TestNGUtils.getGroups(result); - if (TestExclusions.noAutoRetry(testSourceData, testTags) - && testSourceData.getTestMethod() != null) { - Test declaration = testSourceData.getTestMethod().getAnnotation(Test.class); - if (declaration != null - && IRetryAnalyzer.class.isAssignableFrom(declaration.retryAnalyzer()) - && !RetryAnalyzer.class.equals(declaration.retryAnalyzer())) { - try { - frameworkRetryAnalyzer = - (IRetryAnalyzer) - declaration.retryAnalyzer().getDeclaredConstructor().newInstance(); - } catch (Exception e) { - LOGGER.debug("Could not instantiate explicit framework retry analyzer", e); - } - } - } executionPolicy = TestEventsHandlerHolder.TEST_EVENTS_HANDLER.executionPolicy( testIdentifier, testSourceData, testTags); @@ -62,10 +65,7 @@ public boolean retry(ITestResult result) { return false; } createExecutionPolicy(result); - return frameworkRetryAnalyzer != null - && !result.isSuccess() - && frameworkRetryAnalyzer.retry(result) - || executionPolicy.applicable(); + return executionPolicy.applicable(); } public void setSuppressFailures(ITestResult result) { diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnnotationTransformer.java b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnnotationTransformer.java index 4aaffeffeb7..264dbba148a 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnnotationTransformer.java +++ b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/execution/RetryAnnotationTransformer.java @@ -1,41 +1,64 @@ package datadog.trace.instrumentation.testng.execution; -import datadog.trace.api.civisibility.config.TestExclusions; -import datadog.trace.api.civisibility.config.TestSourceData; -import datadog.trace.instrumentation.testng.TestNGUtils; +import datadog.trace.util.MethodHandles; +import java.lang.invoke.MethodHandle; import java.lang.reflect.Constructor; import java.lang.reflect.Method; -import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.WeakHashMap; import org.testng.IAnnotationTransformer; import org.testng.IRetryAnalyzer; +import org.testng.TestNG; import org.testng.annotations.ITestAnnotation; -import org.testng.annotations.Test; public class RetryAnnotationTransformer implements IAnnotationTransformer { + private static final MethodHandles METHOD_HANDLES = + new MethodHandles(TestNG.class.getClassLoader()); + private static final MethodHandle GET_RETRY_ANALYZER_CLASS = + METHOD_HANDLES.method(ITestAnnotation.class, "getRetryAnalyzerClass"); + private static final MethodHandle GET_RETRY_ANALYZER = + METHOD_HANDLES.method(ITestAnnotation.class, "getRetryAnalyzer"); + private static final Map FRAMEWORK_OVERRIDES = + Collections.synchronizedMap(new WeakHashMap()); private final IAnnotationTransformer delegate; public RetryAnnotationTransformer(IAnnotationTransformer delegate) { this.delegate = delegate; } + public static boolean hasFrameworkOverride(ITestAnnotation annotation) { + if (annotation == null) { + return false; + } + Boolean overridden = FRAMEWORK_OVERRIDES.get(annotation); + // A user-installed transformer can replace ours after TestNG construction. + return overridden == null || overridden; + } + + private static Class retryAnalyzerClass(ITestAnnotation annotation) { + if (GET_RETRY_ANALYZER_CLASS != null) { + return METHOD_HANDLES.invoke(GET_RETRY_ANALYZER_CLASS, annotation); + } + IRetryAnalyzer analyzer = METHOD_HANDLES.invoke(GET_RETRY_ANALYZER, annotation); + return analyzer != null ? analyzer.getClass() : null; + } + @Override public void transform( ITestAnnotation annotation, Class testClass, Constructor testConstructor, Method testMethod) { - // TestNG 6 has no Datadog execution-policy instrumentation. Keep explicit native retries. - Test declaration = testMethod != null ? testMethod.getAnnotation(Test.class) : null; - String frameworkVersion = TestNGUtils.getTestNGVersion(); - boolean preserveFrameworkRetry = - frameworkVersion != null - && frameworkVersion.startsWith("6.") - && declaration != null - && IRetryAnalyzer.class.isAssignableFrom(declaration.retryAnalyzer()) - && TestExclusions.noAutoRetry( - new TestSourceData(testClass, testMethod), Arrays.asList(annotation.getGroups())); - if (!preserveFrameworkRetry) { - annotation.setRetryAnalyzer(RetryAnalyzer.class); - } + FRAMEWORK_OVERRIDES.put(annotation, false); + Class nativeAnalyzer = retryAnalyzerClass(annotation); + annotation.setRetryAnalyzer(RetryAnalyzer.class); if (delegate != null) { delegate.transform(annotation, testClass, testConstructor, testMethod); } + if (retryAnalyzerClass(annotation) != RetryAnalyzer.class) { + // Preserve the existing precedence of explicit user transformer overrides. + FRAMEWORK_OVERRIDES.put(annotation, true); + } else if (nativeAnalyzer != null) { + // Let TestNG resolve class inheritance and effective groups before selecting DD policy. + annotation.setRetryAnalyzer(nativeAnalyzer); + } } } diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy index 1b15aacd4ea..f658ffa370b 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy @@ -7,6 +7,7 @@ import datadog.trace.civisibility.diff.LineDiff import org.example.* import org.junit.jupiter.api.Assumptions import org.testng.TestNG +import org.testng.IAnnotationTransformer import org.testng.xml.SuiteXmlParser import org.testng.xml.XmlSuite @@ -62,6 +63,86 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { testSpans.every { it.getTag("test.is_retry") == null } } + def "test automation exclusions preserve class-level retry analyzers"() { + given: + TestClassRetryExcluded.executions = 0 + givenFlakyRetryEnabled(true) + givenKnownTests([]) + + when: + runTests([TestClassRetryExcluded], null, !currentTestNGVersion.startsWith("6.")) + + then: + TestClassRetryExcluded.executions == 2 + def spans = TEST_WRITER.toList().flatten().findAll { it.getTag("test.name") == "class_retry" } + spans.size() == 2 + spans.every { it.getTag("test.test_management.no_auto_retry") == true } + } + + def "test automation exclusions preserve object-factory retry analyzers"() { + given: + TestFactoryRetryExcluded.executions = 0 + givenFlakyRetryEnabled(true) + givenKnownTests([]) + def factoryType = Class.forName("org.testng.ITestObjectFactory") + Assumptions.assumeTrue(factoryType.methods.any { it.name == "newInstance" && it.parameterTypes[0] == Class }, + "Requires TestNG object factory API with class-based construction") + def factory = [newInstance: { Object type, Object[] arguments -> + if (type == TestFactoryRetryExcluded.ConfiguredRetry) { + return new TestFactoryRetryExcluded.ConfiguredRetry("configured") + } + type instanceof Class ? type.getDeclaredConstructor().newInstance() : type.newInstance(arguments) + }].asType(factoryType) + + when: + runTests([TestFactoryRetryExcluded], null, true, { TestNG testNG -> testNG.setObjectFactory(factory) }) + + then: + TestFactoryRetryExcluded.executions == 2 + def spans = TEST_WRITER.toList().flatten().findAll { it.getTag("test.name") == "factory_retry" } + spans.size() == 2 + spans.every { it.getTag("test.test_management.no_auto_retry") == true } + } + + def "test automation exclusions preserve quarantine with native retries"() { + given: + TestNativeQuarantinedExcluded.executions = 0 + def id = new TestFQN("org.example.TestNativeQuarantinedExcluded", "native_quarantined") + givenQuarantinedTests([id]) + givenAttemptToFixTests([id]) + givenFlakyRetryEnabled(true) + givenKnownTests([]) + + when: + runTests([TestNativeQuarantinedExcluded], null, TestNGUtils.isExceptionSuppressionSupported(currentTestNGVersion)) + + then: + TestNativeQuarantinedExcluded.executions == 2 + def spans = TEST_WRITER.toList().flatten().findAll { it.getTag("test.name") == "native_quarantined" } + spans.size() == 2 + spans.every { it.getTag("test.test_management.no_auto_retry") == true } + spans.every { it.getTag("test.test_management.is_quarantined") == true } + spans.every { it.getTag("test.test_management.attempt_to_fix_passed") == null } + } + + def "test automation exclusions preserve unexcluded transformer retry overrides"() { + given: + TestTransformedNativeRetry.executions = 0 + givenFlakyRetryEnabled(false) + def transformer = [transform: { annotation, testClass, constructor, method -> + if (method?.name == "transformed_native_retry") { + annotation.setRetryAnalyzer(TestExplicitRetryExcluded.RetryOnce) + } + }] as IAnnotationTransformer + + when: + runTests([TestTransformedNativeRetry], null, !currentTestNGVersion.startsWith("6."), + { TestNG testNG -> testNG.addListener(transformer) }) + + then: + TestTransformedNativeRetry.executions == 2 + } + def "test #testcaseName"() { runTests(tests, null, success) @@ -327,13 +408,14 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { "test-capabilities-ordering-efd-suppression" | TestNGUtils.capabilities("7.5") | (TestNGUtils.isTestOrderingSupported(currentTestNGVersion) && TestNGUtils.isEFDSupported(currentTestNGVersion) && TestNGUtils.isExceptionSuppressionSupported(currentTestNGVersion)) } - protected void runTests(List testClasses, String parallelMode = null, boolean expectSuccess = true) { + protected void runTests(List testClasses, String parallelMode = null, boolean expectSuccess = true, Closure configure = null) { TestEventsHandlerHolder.start() def testNG = new TestNG() testNG.setOutputDirectory(testOutputDir) Class[] testClassesArray = testClasses.toArray(new Class[0]) testNG.setTestClasses(testClassesArray) + configure?.call(testNG) if (parallelMode != null) { testNG.setParallel(parallelMode) } diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestClassRetryExcluded.java b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestClassRetryExcluded.java new file mode 100644 index 00000000000..f22fe7d33f3 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestClassRetryExcluded.java @@ -0,0 +1,16 @@ +package org.example; + +import datadog.trace.api.civisibility.NoAutoRetry; +import org.testng.annotations.Test; + +@Test(retryAnalyzer = TestExplicitRetryExcluded.RetryOnce.class) +public class TestClassRetryExcluded { + public static int executions; + + @NoAutoRetry + public void class_retry() { + if (++executions == 1) { + throw new AssertionError("first framework attempt"); + } + } +} diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestFactoryRetryExcluded.java b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestFactoryRetryExcluded.java new file mode 100644 index 00000000000..187241bf1d7 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestFactoryRetryExcluded.java @@ -0,0 +1,34 @@ +package org.example; + +import datadog.trace.api.civisibility.NoAutoRetry; +import org.testng.IRetryAnalyzer; +import org.testng.ITestResult; +import org.testng.annotations.Test; + +public class TestFactoryRetryExcluded { + public static int executions; + + @Test(retryAnalyzer = ConfiguredRetry.class) + @NoAutoRetry + public void factory_retry() { + if (++executions == 1) { + throw new AssertionError("first framework attempt"); + } + } + + public static class ConfiguredRetry implements IRetryAnalyzer { + private final String configuration; + private int retries; + + public ConfiguredRetry(String configuration) { + this.configuration = configuration; + } + + public boolean retry(ITestResult result) { + if (!"configured".equals(configuration)) { + throw new AssertionError("missing object factory configuration"); + } + return retries++ == 0; + } + } +} diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestNativeQuarantinedExcluded.java b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestNativeQuarantinedExcluded.java new file mode 100644 index 00000000000..6be2c6f5e37 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestNativeQuarantinedExcluded.java @@ -0,0 +1,15 @@ +package org.example; + +import datadog.trace.api.civisibility.NoAutoRetry; +import org.testng.annotations.Test; + +public class TestNativeQuarantinedExcluded { + public static int executions; + + @Test(retryAnalyzer = TestExplicitRetryExcluded.RetryOnce.class) + @NoAutoRetry + public void native_quarantined() { + executions++; + throw new AssertionError("native failure remains quarantined"); + } +} diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestTransformedNativeRetry.java b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestTransformedNativeRetry.java new file mode 100644 index 00000000000..c6b0f6acb8e --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestTransformedNativeRetry.java @@ -0,0 +1,14 @@ +package org.example; + +import org.testng.annotations.Test; + +public class TestTransformedNativeRetry { + public static int executions; + + @Test + public void transformed_native_retry() { + if (++executions == 1) { + throw new AssertionError("first framework attempt"); + } + } +} From 4f95e9c4652dd01400562ff5aec40aa6a13cc718 Mon Sep 17 00:00:00 2001 From: Calvin Bayer Date: Tue, 20 Oct 2026 00:53:46 +0200 Subject: [PATCH 4/7] Use effective TestNG groups for exclusion skip decisions --- .../testng/TestNGSkipInstrumentation.java | 11 +++++- .../instrumentation/testng/TestNGTest.groovy | 39 +++++++++++++++++++ .../example/TestInheritedGroupsExcluded.java | 16 ++++++++ .../TestTransformedGroupsExcluded.java | 12 ++++++ 4 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestInheritedGroupsExcluded.java create mode 100644 dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestTransformedGroupsExcluded.java diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGSkipInstrumentation.java b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGSkipInstrumentation.java index 7bb61743d56..243260a4bf5 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGSkipInstrumentation.java +++ b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGSkipInstrumentation.java @@ -15,6 +15,8 @@ import java.lang.reflect.Method; import java.util.List; import net.bytebuddy.asm.Advice; +import org.testng.ITestResult; +import org.testng.Reporter; import org.testng.SkipException; import org.testng.annotations.DataProvider; @@ -58,18 +60,23 @@ public static void invokeMethod( @Advice.Argument(1) final Object instance, @Advice.Argument(2) final Object[] parameters) { TestIdentifier testIdentifier = TestNGUtils.toTestIdentifier(method, instance, parameters); + ITestResult invocation = Reporter.getCurrentTestResult(); + List groups = + invocation != null + && method.equals(TestNGUtils.toTestSourceData(invocation).getTestMethod()) + ? TestNGUtils.getGroups(invocation) + : TestNGUtils.getGroups(method); SkipReason skipReason = TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason( testIdentifier, new TestSourceData( instance != null ? instance.getClass() : method.getDeclaringClass(), method), - TestNGUtils.getGroups(method)); + groups); if (skipReason == null) { return; } if (skipReason == SkipReason.ITR) { - List groups = TestNGUtils.getGroups(method); if (groups.contains(CIConstants.Tags.ITR_UNSKIPPABLE_TAG)) { return; } diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy index f658ffa370b..2cf012e30fd 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy @@ -143,6 +143,45 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { TestTransformedNativeRetry.executions == 2 } + def "test automation exclusions use effective inherited groups"() { + given: + TestInheritedGroupsExcluded.executions = 0 + def id = new TestFQN(TestInheritedGroupsExcluded.name, "inherited_groups") + givenDisabledTests([id]) + + when: + runTests([TestInheritedGroupsExcluded]) + + then: + TestInheritedGroupsExcluded.executions == 1 + def testSpan = TEST_WRITER.toList().flatten().find { it.getTag("test.name") == "inherited_groups" } + testSpan.getTag("test.status").toString() == "pass" + testSpan.getTag("test.test_management.prevent_failure_suppression") == true + testSpan.getTag("test.test_management.is_test_disabled") == true + } + + def "test automation exclusions use effective transformed groups"() { + given: + TestTransformedGroupsExcluded.executions = 0 + def id = new TestFQN(TestTransformedGroupsExcluded.name, "transformed_groups") + givenDisabledTests([id]) + def transformer = [transform: { annotation, testClass, constructor, method -> + if (method?.name == "transformed_groups") { + annotation.setGroups(["datadog_prevent_failure_suppression"] as String[]) + } + }] as IAnnotationTransformer + + when: + runTests([TestTransformedGroupsExcluded], null, true, { TestNG testNG -> testNG.addListener(transformer) }) + + then: + TestTransformedGroupsExcluded.executions == 1 + def testSpan = TEST_WRITER.toList().flatten().find { it.getTag("test.name") == "transformed_groups" } + testSpan.getTag("test.status").toString() == "pass" + testSpan.getTag("test.test_management.prevent_failure_suppression") == true + testSpan.getTag("test.test_management.is_test_disabled") == true + } + def "test #testcaseName"() { runTests(tests, null, success) diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestInheritedGroupsExcluded.java b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestInheritedGroupsExcluded.java new file mode 100644 index 00000000000..dbdf84774a0 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestInheritedGroupsExcluded.java @@ -0,0 +1,16 @@ +package org.example; + +import datadog.trace.api.civisibility.TestAutomationExclusions; +import org.testng.annotations.Test; + +@Test(groups = TestAutomationExclusions.PREVENT_FAILURE_SUPPRESSION) +public class TestInheritedGroupsExcluded extends InheritedGroupsBase { + public static int executions; +} + +class InheritedGroupsBase { + @Test + public void inherited_groups() { + TestInheritedGroupsExcluded.executions++; + } +} diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestTransformedGroupsExcluded.java b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestTransformedGroupsExcluded.java new file mode 100644 index 00000000000..7e64f66c08a --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestTransformedGroupsExcluded.java @@ -0,0 +1,12 @@ +package org.example; + +import org.testng.annotations.Test; + +public class TestTransformedGroupsExcluded { + public static int executions; + + @Test + public void transformed_groups() { + executions++; + } +} From ae9acf22581a4dc393897e63669af6e0257eb56a Mon Sep 17 00:00:00 2001 From: Calvin Bayer Date: Tue, 20 Oct 2026 10:05:51 +0200 Subject: [PATCH 5/7] Load TestNG object factory fixtures with an explicit classloader --- .../datadog/trace/instrumentation/testng/TestNGTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy index 2cf012e30fd..4ec74e24c9c 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy @@ -84,7 +84,7 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { TestFactoryRetryExcluded.executions = 0 givenFlakyRetryEnabled(true) givenKnownTests([]) - def factoryType = Class.forName("org.testng.ITestObjectFactory") + def factoryType = Class.forName("org.testng.ITestObjectFactory", false, TestNG.classLoader) Assumptions.assumeTrue(factoryType.methods.any { it.name == "newInstance" && it.parameterTypes[0] == Class }, "Requires TestNG object factory API with class-based construction") def factory = [newInstance: { Object type, Object[] arguments -> From 9c86fe354e037047ba19425b5fb140fc05ff17dd Mon Sep 17 00:00:00 2001 From: Calvin Bayer Date: Tue, 20 Oct 2026 10:05:52 +0200 Subject: [PATCH 6/7] Include TestNG exclusion policy dependencies in explicit helper lists --- .../testng6/TestNGClassListenerInstrumentation.java | 6 +++++- .../testng7/TestNGClassListenerInstrumentation.java | 6 +++++- .../testng7/TestNGExecutionInstrumentation.java | 1 + .../testng7/order/TestNGOrderInstrumentation.java | 2 ++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/testng/testng-6.4/src/main/java/datadog/trace/instrumentation/testng6/TestNGClassListenerInstrumentation.java b/dd-java-agent/instrumentation/testng/testng-6.4/src/main/java/datadog/trace/instrumentation/testng6/TestNGClassListenerInstrumentation.java index 3a0ca5f0da3..058073f1d6d 100644 --- a/dd-java-agent/instrumentation/testng/testng-6.4/src/main/java/datadog/trace/instrumentation/testng6/TestNGClassListenerInstrumentation.java +++ b/dd-java-agent/instrumentation/testng/testng-6.4/src/main/java/datadog/trace/instrumentation/testng6/TestNGClassListenerInstrumentation.java @@ -59,7 +59,11 @@ public void methodAdvice(MethodTransformer transformer) { @Override public String[] helperClassNames() { return new String[] { - commonPackageName + ".TestNGUtils", commonPackageName + ".TestNGClassListener" + commonPackageName + ".TestEventsHandlerHolder", + commonPackageName + ".execution.RetryAnalyzer", + commonPackageName + ".execution.RetryAnnotationTransformer", + commonPackageName + ".TestNGUtils", + commonPackageName + ".TestNGClassListener" }; } diff --git a/dd-java-agent/instrumentation/testng/testng-7.0/src/main/java/datadog/trace/instrumentation/testng7/TestNGClassListenerInstrumentation.java b/dd-java-agent/instrumentation/testng/testng-7.0/src/main/java/datadog/trace/instrumentation/testng7/TestNGClassListenerInstrumentation.java index 0989ed49195..20abdc9bace 100644 --- a/dd-java-agent/instrumentation/testng/testng-7.0/src/main/java/datadog/trace/instrumentation/testng7/TestNGClassListenerInstrumentation.java +++ b/dd-java-agent/instrumentation/testng/testng-7.0/src/main/java/datadog/trace/instrumentation/testng7/TestNGClassListenerInstrumentation.java @@ -54,7 +54,11 @@ public void methodAdvice(MethodTransformer transformer) { @Override public String[] helperClassNames() { return new String[] { - commonPackageName + ".TestNGUtils", commonPackageName + ".TestNGClassListener" + commonPackageName + ".TestEventsHandlerHolder", + commonPackageName + ".execution.RetryAnalyzer", + commonPackageName + ".execution.RetryAnnotationTransformer", + commonPackageName + ".TestNGUtils", + commonPackageName + ".TestNGClassListener" }; } diff --git a/dd-java-agent/instrumentation/testng/testng-7.0/src/main/java/datadog/trace/instrumentation/testng7/TestNGExecutionInstrumentation.java b/dd-java-agent/instrumentation/testng/testng-7.0/src/main/java/datadog/trace/instrumentation/testng7/TestNGExecutionInstrumentation.java index fd04267d368..6a69c93c8c3 100644 --- a/dd-java-agent/instrumentation/testng/testng-7.0/src/main/java/datadog/trace/instrumentation/testng7/TestNGExecutionInstrumentation.java +++ b/dd-java-agent/instrumentation/testng/testng-7.0/src/main/java/datadog/trace/instrumentation/testng7/TestNGExecutionInstrumentation.java @@ -57,6 +57,7 @@ public String[] helperClassNames() { commonPackageName + ".TestEventsHandlerHolder", commonPackageName + ".TestNGClassListener", commonPackageName + ".execution.RetryAnalyzer", + commonPackageName + ".execution.RetryAnnotationTransformer", commonPackageName + ".TracingListener", }; } diff --git a/dd-java-agent/instrumentation/testng/testng-7.0/src/main/java/datadog/trace/instrumentation/testng7/order/TestNGOrderInstrumentation.java b/dd-java-agent/instrumentation/testng/testng-7.0/src/main/java/datadog/trace/instrumentation/testng7/order/TestNGOrderInstrumentation.java index 9aa927ca17e..79d8d726843 100644 --- a/dd-java-agent/instrumentation/testng/testng-7.0/src/main/java/datadog/trace/instrumentation/testng7/order/TestNGOrderInstrumentation.java +++ b/dd-java-agent/instrumentation/testng/testng-7.0/src/main/java/datadog/trace/instrumentation/testng7/order/TestNGOrderInstrumentation.java @@ -58,6 +58,8 @@ public String[] helperClassNames() { parentPackageName + ".TestNGClassListener", parentPackageName + ".TestNGUtils", parentPackageName + ".TestEventsHandlerHolder", + parentPackageName + ".execution.RetryAnalyzer", + parentPackageName + ".execution.RetryAnnotationTransformer", packageName + ".FailFastOrderInterceptor", }; } From e434620159d441f238c6f2839961ccd842a75153 Mon Sep 17 00:00:00 2001 From: Calvin Bayer Date: Tue, 20 Oct 2026 10:47:39 +0200 Subject: [PATCH 7/7] Preserve explicitly disabled retries in TestNG transformers --- .../instrumentation/testng/TestNGUtils.java | 6 ++-- .../instrumentation/testng/TestNGTest.groovy | 32 +++++++++++++++++++ .../example/TestTransformedDisabledRetry.java | 13 ++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestTransformedDisabledRetry.java diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGUtils.java b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGUtils.java index 3d4c5d97978..68d16812a5d 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGUtils.java +++ b/dd-java-agent/instrumentation/testng/testng-common/src/main/java/datadog/trace/instrumentation/testng/TestNGUtils.java @@ -74,8 +74,10 @@ public static void configureRetryAnalyzer( annotationFinder.findAnnotation( source.getTestMethod().getDeclaringClass(), ITestAnnotation.class); } - if (TestExclusions.noAutoRetry(source, Arrays.asList(method.getGroups())) - || RetryAnnotationTransformer.hasFrameworkOverride(annotation)) { + if (RetryAnnotationTransformer.hasFrameworkOverride(annotation)) { + return; + } + if (TestExclusions.noAutoRetry(source, Arrays.asList(method.getGroups()))) { if (TEST_METHOD_GET_RETRY_ANALYZER_CLASS != null) { Class nativeAnalyzer = METHOD_HANDLES.invoke(TEST_METHOD_GET_RETRY_ANALYZER_CLASS, method); diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy index 4ec74e24c9c..7c750b2fa0e 100644 --- a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy @@ -125,6 +125,38 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { spans.every { it.getTag("test.test_management.attempt_to_fix_passed") == null } } + def "test automation exclusions preserve explicit disabled transformer overrides with #retryFeature"() { + given: + Assumptions.assumeTrue(TestNG.getResource("/org/testng/internal/annotations/DisabledRetryAnalyzer.class") != null, + "Requires class-based disabled retry analyzer") + Assumptions.assumeTrue(retryFeature != "EFD" || TestNGUtils.isEFDSupported(currentTestNGVersion), + "Requires EFD support") + TestTransformedDisabledRetry.executions = 0 + givenFlakyRetryEnabled(retryFeature == "ATR") + givenEarlyFlakinessDetectionEnabled(retryFeature == "EFD") + givenKnownTests([]) + def disabledAnalyzer = Class.forName("org.testng.internal.annotations.DisabledRetryAnalyzer", false, TestNG.classLoader) + def transformer = [transform: { annotation, testClass, constructor, method -> + if (method?.name == "transformed_disabled_retry") { + annotation.setRetryAnalyzer(disabledAnalyzer) + } + }] as IAnnotationTransformer + + when: + runTests([TestTransformedDisabledRetry], null, false, + { TestNG testNG -> testNG.addListener(transformer) }) + + then: + TestTransformedDisabledRetry.executions == 1 + def spans = TEST_WRITER.toList().flatten().findAll { it.getTag("test.name") == "transformed_disabled_retry" } + spans.size() == 1 + spans[0].getTag("test.is_retry") == null + spans[0].getTag("test.retry_reason") == null + + where: + retryFeature << ["ATR", "EFD"] + } + def "test automation exclusions preserve unexcluded transformer retry overrides"() { given: TestTransformedNativeRetry.executions = 0 diff --git a/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestTransformedDisabledRetry.java b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestTransformedDisabledRetry.java new file mode 100644 index 00000000000..64596464fad --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-common/src/testFixtures/java/org/example/TestTransformedDisabledRetry.java @@ -0,0 +1,13 @@ +package org.example; + +import org.testng.annotations.Test; + +public class TestTransformedDisabledRetry { + public static int executions; + + @Test + public void transformed_disabled_retry() { + ++executions; + throw new AssertionError("explicitly disabled framework retries"); + } +}