Support test automation exclusion annotations in JUnit4

.github/CODEOWNERS

@@ -446,3 +446,5 @@
 **/build.gradle.kts                                @DataDog/apm-lang-platform-java
 **/gradle.lockfile                                 @DataDog/apm-lang-platform-java
 /dd-java-agent/instrumentation/**/gradle.lockfile  @DataDog/apm-idm-java
+
+/docs/test-automation-exclusions.md @DataDog/ci-app-libraries

buildSrc/src/main/kotlin/dd-trace-java.instrumentation.testing-framework-tests.gradle.kts

@@ -6,6 +6,7 @@ logger.info("Avoid executing classes used to test testing frameworks instrumenta
 
 tasks.withType<Test>().configureEach {
   exclude("**/TestAssumption*", "**/TestSuiteSetUpAssumption*")
+  exclude("**/TestAutomationExcluded*", "**/TestExplicitRetryExcluded*")
   exclude("**/TestContinueOnStepFailure*")
   exclude("**/TestDisableTestTrace*")
   exclude("**/TestError*")

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java

@@ -45,6 +45,11 @@ TestSuiteImpl testSuiteStart(
   @Nullable
   SkipReason skipReason(TestIdentifier test);
 
+  default SkipReason skipReason(
+      TestIdentifier test, TestSourceData source, Collection<String> tags) {
+    return skipReason(test);
+  }
+
   @Nonnull
   TestExecutionPolicy executionPolicy(
       TestIdentifier test, TestSourceData testSource, Collection<String> testTags);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java

@@ -119,6 +119,12 @@ public boolean isAttemptToFix(TestIdentifier test) {
   }
 
   @Nullable
+  @Override
+  public SkipReason skipReason(
+      TestIdentifier test, TestSourceData source, Collection<String> tags) {
+    return executionStrategy.skipReason(test, source, tags);
+  }
+
   @Override
   public SkipReason skipReason(TestIdentifier test) {
     return executionStrategy.skipReason(test);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java

@@ -104,6 +104,12 @@ public boolean isAttemptToFix(TestIdentifier test) {
   }
 
   @Nullable
+  @Override
+  public SkipReason skipReason(
+      TestIdentifier test, TestSourceData source, Collection<String> tags) {
+    return executionStrategy.skipReason(test, source, tags);
+  }
+
   @Override
   public SkipReason skipReason(TestIdentifier test) {
     return executionStrategy.skipReason(test);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java

@@ -5,6 +5,7 @@
 import datadog.trace.api.civisibility.CIConstants;
 import datadog.trace.api.civisibility.DDTest;
 import datadog.trace.api.civisibility.DDTestSuite;
+import datadog.trace.api.civisibility.config.TestExclusions;
 import datadog.trace.api.civisibility.config.TestIdentifier;
 import datadog.trace.api.civisibility.config.TestSourceData;
 import datadog.trace.api.civisibility.events.TestEventsHandler;
@@ -24,6 +25,7 @@
 import datadog.trace.civisibility.domain.TestFrameworkSession;
 import datadog.trace.civisibility.domain.TestImpl;
 import datadog.trace.civisibility.domain.TestSuiteImpl;
+import datadog.trace.civisibility.execution.AttemptToFix;
 import java.util.Collection;
 import java.util.function.Supplier;
 import javax.annotation.Nonnull;
@@ -186,6 +188,13 @@ public void onTestStart(
     TestImpl test =
         testSuite.testStart(testName, testParameters, testSourceData.getTestMethod(), startTime);
 
+    if (TestExclusions.noAutoRetry(testSourceData, categories)) {
+      test.setTag("test.test_management.no_auto_retry", true);
+    }
+    if (TestExclusions.preventFailureSuppression(testSourceData, categories)) {
+      test.setTag("test.test_management.prevent_failure_suppression", true);
+    }
+
     TestIdentifier thisTest = test.getIdentifier();
     if (testModule.isNew(thisTest)) {
       test.setTag(Tags.TEST_IS_NEW, true);
@@ -234,7 +243,7 @@ public void onTestStart(
           test.setTag(Tags.TEST_ITR_UNSKIPPABLE, true);
           metricCollector.add(CiVisibilityCountMetric.ITR_UNSKIPPABLE, 1, EventType.TEST);
 
-          if (testModule.skipReason(thisTest) == SkipReason.ITR) {
+          if (testModule.skipReason(thisTest, testSourceData, categories) == SkipReason.ITR) {
             test.setTag(Tags.TEST_ITR_FORCED_RUN, true);
             metricCollector.add(CiVisibilityCountMetric.ITR_FORCED_RUN, 1, EventType.TEST);
           }
@@ -303,7 +312,7 @@ public void onTestFinish(
           test.setTag(Tags.TEST_HAS_FAILED_ALL_RETRIES, true);
         }
 
-        if (testModule.isAttemptToFix(test.getIdentifier())) {
+        if (testExecutionTracker instanceof AttemptToFix) {
           test.setTag(
               Tags.TEST_TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED,
               outcome.aggregation() == ExecutionAggregation.ONLY_PASSED);
@@ -356,6 +365,12 @@ public int executionPriority(@Nullable TestIdentifier test, @Nonnull TestSourceD
   }
 
   @Nullable
+  @Override
+  public SkipReason skipReason(
+      TestIdentifier test, TestSourceData source, Collection<String> tags) {
+    return getOrCreateTestModule().skipReason(test, source, tags);
+  }
+
   @Override
   public SkipReason skipReason(TestIdentifier test) {
     return getOrCreateTestModule().skipReason(test);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java

@@ -2,6 +2,7 @@
 
 import datadog.trace.api.Config;
 import datadog.trace.api.civisibility.CIConstants;
+import datadog.trace.api.civisibility.config.TestExclusions;
 import datadog.trace.api.civisibility.config.TestIdentifier;
 import datadog.trace.api.civisibility.config.TestMetadata;
 import datadog.trace.api.civisibility.config.TestSourceData;
@@ -21,6 +22,7 @@
 import datadog.trace.civisibility.source.SourcePathResolver;
 import java.lang.reflect.Method;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 import javax.annotation.Nonnull;
@@ -91,16 +93,21 @@ public boolean isAttemptToFix(TestIdentifier test) {
 
   @Nullable
   public SkipReason skipReason(TestIdentifier test) {
+    return skipReason(test, TestSourceData.UNKNOWN, Collections.emptyList());
+  }
+
+  public SkipReason skipReason(
+      TestIdentifier test, TestSourceData source, Collection<String> tags) {
     if (test == null) {
       return null;
     }
 
     // test should not be skipped if it is an attempt to fix, independent of TIA or Disabled
-    if (isAttemptToFix(test)) {
+    if (isAttemptToFix(test) && !TestExclusions.noAutoRetry(source, tags)) {
       return null;
     }
 
-    if (isDisabled(test)) {
+    if (isDisabled(test) && !TestExclusions.preventFailureSuppression(source, tags)) {
       return SkipReason.DISABLED;
     }
 
@@ -126,34 +133,35 @@ public TestExecutionPolicy executionPolicy(
       return Regular.INSTANCE;
     }
 
-    if (isAttemptToFix(test)) {
+    boolean noAutoRetry = TestExclusions.noAutoRetry(testSource, testTags);
+    boolean quarantined =
+        isQuarantined(test) && !TestExclusions.preventFailureSuppression(testSource, testTags);
+    if (isAttemptToFix(test) && !noAutoRetry) {
       return new AttemptToFix(
           executionSettings.getTestManagementSettings().getAttemptToFixRetries());
     }
 
-    if (isEFDApplicable(test, testSource, testTags)) {
+    if (!noAutoRetry && isEFDApplicable(test, testSource, testTags)) {
       // check-then-act with "earlyFlakeDetectionsUsed" is not atomic here,
       // but we don't care if we go "a bit" over the limit, it does not have to be precise
       earlyFlakeDetectionsUsed.incrementAndGet();
       return new EarlyFlakeDetection(
           executionSettings.getEarlyFlakeDetectionSettings().getExecutionsByDuration(),
-          isQuarantined(test));
+          quarantined);
     }
 
-    if (isAutoRetryApplicable(test)) {
+    if (!noAutoRetry && isAutoRetryApplicable(test)) {
       // check-then-act with "autoRetriesUsed" is not atomic here,
       // but we don't care if we go "a bit" over the limit, it does not have to be precise
       if (executionSettings.getDynamicAutoTestRetrySettings().isEnabled()) {
         return new DynamicAutoTestRetry(
-            executionSettings.getDynamicAutoTestRetrySettings(),
-            isQuarantined(test),
-            autoRetriesUsed);
+            executionSettings.getDynamicAutoTestRetrySettings(), quarantined, autoRetriesUsed);
       }
       return new AutoTestRetry(
-          config.getCiVisibilityFlakyRetryCount(), isQuarantined(test), autoRetriesUsed);
+          config.getCiVisibilityFlakyRetryCount(), quarantined, autoRetriesUsed);
     }
 
-    if (isQuarantined(test)) {
+    if (quarantined) {
       return new Quarantine();
     }
 

dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/events/TestEventsHandlerImplTest.java

@@ -2,22 +2,38 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import datadog.trace.api.civisibility.DDTest;
 import datadog.trace.api.civisibility.DDTestSuite;
 import datadog.trace.api.civisibility.config.TestIdentifier;
+import datadog.trace.api.civisibility.config.TestSourceData;
+import datadog.trace.api.civisibility.domain.TestContext;
+import datadog.trace.api.civisibility.execution.TestStatus;
 import datadog.trace.api.civisibility.telemetry.NoOpMetricCollector;
 import datadog.trace.api.civisibility.telemetry.tag.SkipReason;
+import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
 import datadog.trace.bootstrap.ContextStore;
+import datadog.trace.bootstrap.instrumentation.api.Tags;
 import datadog.trace.civisibility.domain.TestFrameworkModule;
 import datadog.trace.civisibility.domain.TestFrameworkSession;
+import datadog.trace.civisibility.domain.TestImpl;
+import datadog.trace.civisibility.domain.TestSuiteImpl;
+import datadog.trace.civisibility.execution.Regular;
 import datadog.trace.civisibility.utils.StrongMapContextStore;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 
 class TestEventsHandlerImplTest {
 
@@ -82,6 +98,95 @@ void createsSessionImmediatelyWhenRequested() {
     verify(session).end(null);
   }
 
+  @Test
+  void excludedAttemptToFixAssignmentDoesNotProduceCompletionResult() {
+    TestFrameworkSession session = mock(TestFrameworkSession.class);
+    TestFrameworkModule module = mock(TestFrameworkModule.class);
+    TestSuiteImpl suite = mock(TestSuiteImpl.class);
+    TestImpl test = mock(TestImpl.class);
+    TestIdentifier id = new TestIdentifier("suite", "test", null);
+    when(session.testModuleStart("module", null)).thenReturn(module);
+    when(module.testSuiteStart("suite", null, null, false, TestFrameworkInstrumentation.JUNIT4))
+        .thenReturn(suite);
+    when(suite.testStart("test", null, null, null)).thenReturn(test);
+    when(test.getIdentifier()).thenReturn(id);
+    when(test.getContext()).thenReturn(mock(TestContext.class));
+    when(test.getStatus()).thenReturn(TestStatus.fail);
+    when(module.isAttemptToFix(id)).thenReturn(true);
+    TestEventsHandlerImpl<Object, Object> handler = handler(() -> session);
+    Object suiteKey = new Object();
+    Object testKey = new Object();
+    handler.onTestSuiteStart(
+        suiteKey,
+        "suite",
+        null,
+        null,
+        null,
+        null,
+        false,
+        TestFrameworkInstrumentation.JUNIT4,
+        null);
+    handler.onTestStart(
+        suiteKey,
+        testKey,
+        "test",
+        null,
+        null,
+        null,
+        Collections.emptyList(),
+        TestSourceData.UNKNOWN,
+        null,
+        Regular.INSTANCE);
+    handler.onTestFinish(testKey, null, Regular.INSTANCE);
+    verify(test, never()).setTag(eq(Tags.TEST_TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED), any());
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"datadog_prevent_failure_suppression", "datadog_no_auto_retry"})
+  void forcedRunReportingUsesExclusionDeclarations(String exclusion) {
+    TestFrameworkSession session = mock(TestFrameworkSession.class);
+    TestFrameworkModule module = mock(TestFrameworkModule.class);
+    TestSuiteImpl suite = mock(TestSuiteImpl.class);
+    TestImpl test = mock(TestImpl.class);
+    TestIdentifier id = new TestIdentifier("suite", "test", null);
+    TestSourceData source = new TestSourceData(getClass(), null);
+    Collection<String> categories = Arrays.asList("datadog_itr_unskippable", exclusion);
+    when(session.testModuleStart("module", null)).thenReturn(module);
+    when(module.testSuiteStart("suite", null, null, false, TestFrameworkInstrumentation.JUNIT4))
+        .thenReturn(suite);
+    when(suite.testStart("test", null, null, null)).thenReturn(test);
+    when(test.getIdentifier()).thenReturn(id);
+    when(test.getContext()).thenReturn(mock(TestContext.class));
+    when(module.skipReason(id))
+        .thenReturn(
+            exclusion.equals("datadog_prevent_failure_suppression") ? SkipReason.DISABLED : null);
+    when(module.skipReason(id, source, categories)).thenReturn(SkipReason.ITR);
+    TestEventsHandlerImpl<Object, Object> handler = handler(() -> session);
+    Object suiteKey = new Object();
+    handler.onTestSuiteStart(
+        suiteKey,
+        "suite",
+        null,
+        null,
+        null,
+        null,
+        false,
+        TestFrameworkInstrumentation.JUNIT4,
+        null);
+    handler.onTestStart(
+        suiteKey,
+        new Object(),
+        "test",
+        null,
+        null,
+        null,
+        categories,
+        source,
+        null,
+        Regular.INSTANCE);
+    verify(test).setTag(Tags.TEST_ITR_FORCED_RUN, true);
+  }
+
   private static TestEventsHandlerImpl<Object, Object> handler(
       Supplier<TestFrameworkSession> testSessionSupplier) {
     return handler(testSessionSupplier, false);