diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.groovy deleted file mode 100644 index 240131577d1..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.groovy +++ /dev/null @@ -1,135 +0,0 @@ -package datadog.smoketest.springboot - -import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer -import static java.util.concurrent.TimeUnit.SECONDS - -import datadog.smoketest.AbstractSmokeTest -import datadog.trace.agent.test.server.http.TestHttpServer -import datadog.trace.agent.test.server.http.TestHttpServer.HandlerApi.RequestApi -import datadog.trace.agent.test.utils.OkHttpUtils -import datadog.trace.agent.test.utils.PortUtils -import groovy.json.JsonOutput -import groovy.json.JsonSlurper -import okhttp3.MediaType -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.RequestBody -import spock.lang.AutoCleanup -import spock.lang.Shared -import spock.util.concurrent.PollingConditions - -/** The SDK with the Datadog Java agent flushes its pending events when the application exits. */ -abstract class AbstractOpenFeatureShutdownSmokeTest extends AbstractSmokeTest { - - @Shared - protected int httpPort = PortUtils.randomOpenPort() - - protected OkHttpClient client = OkHttpUtils.client() - - @Shared - protected final ufcConfig = new JsonSlurper().parse(fetchResource("ffe-system-test-data/ufc-config.json")) as Map - - /** Serves the flag configuration as the Datadog managed CDN does. */ - @Shared - @AutoCleanup - protected TestHttpServer cdn = httpServer { - handlers { - prefix("/api/v2/feature-flagging/config/rules-based/server") { - response.status(200).send(JsonOutput.toJson([ - data: [type: 'universal-flag-configuration', attributes: ufcConfig] - ])) - } - } - } - - /** @return whether the application runs with tracing enabled. */ - abstract boolean tracing() - - def setupSpec() { - PortUtils.waitForPortToOpen(httpPort, 240, SECONDS, testedProcess) - } - - @Override - ProcessBuilder createProcessBuilder() { - final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") - final command = [javaPath()] - command.addAll(defaultJavaProperties) - command.add("-Ddd.trace.enabled=${tracing()}".toString()) - command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) - final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) - builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL', "http://localhost:${cdn.address.port}".toString()) - return builder - } - - @Override - boolean testTelemetry() { - return false - } - - @Override - Closure decodedTracesCallback() { - return {} - } - - @Override - Closure decodedEvpProxyMessageCallback() { - return { String path, RequestApi request -> - if (!path.contains('api/v2/flagevaluation')) { - return null - } - return [ - origin: request.getHeader('DD-EVP-ORIGIN'), - originVersion: request.getHeader('DD-EVP-ORIGIN-VERSION'), - body: new JsonSlurper().parse(request.body) - ] - } - } - - void 'test pending flag evaluations are flushed at application exit'() { - when: - // The first evaluation is flushed within the 10 seconds flush interval, which then restarts: - // the next evaluation stays pending until the application exits. - evaluate('test-case-boolean-false-assignment.json') - new PollingConditions(timeout: 20).eventually { - assert evaluatedFlags().contains('boolean-false-assignment') - } - evaluate('test-case-boolean-one-of-matches.json') - - then: - !evaluatedFlags().contains('boolean-one-of-matches') - - when: - testedProcess.destroy() - - then: - testedProcess.waitFor(30, SECONDS) - evaluatedFlags().contains('boolean-one-of-matches') - evaluationRequests().every { it.origin == 'dd-trace-java' && it.originVersion } - } - - protected void evaluate(final String testCaseFile) { - final testCase = (new JsonSlurper().parse(fetchResource("ffe-system-test-data/evaluation-cases/${testCaseFile}")) as List>).first() - final request = new Request.Builder() - .url("http://localhost:${httpPort}/openfeature/evaluate") - .post(RequestBody.create(MediaType.parse('application/json'), JsonOutput.toJson(testCase))) - .build() - final response = client.newCall(request).execute() - try { - assert response.code() == 200 - } finally { - response.close() - } - } - - protected List evaluatedFlags() { - return evaluationRequests()*.body*.flagEvaluations.flatten()*.flag*.key as List - } - - protected List> evaluationRequests() { - return evpProxyMessages*.getV2() as List> - } - - protected static URL fetchResource(final String name) { - return Thread.currentThread().getContextClassLoader().getResource(name) - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.groovy deleted file mode 100644 index 147c6fa2fde..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.groovy +++ /dev/null @@ -1,213 +0,0 @@ -package datadog.smoketest.springboot - -import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer - -import datadog.smoketest.AbstractServerSmokeTest -import datadog.trace.agent.test.server.http.TestHttpServer -import datadog.trace.agent.test.server.http.TestHttpServer.HandlerApi.RequestApi -import groovy.json.JsonOutput -import groovy.json.JsonSlurper -import java.nio.file.Files -import java.nio.file.Paths -import okhttp3.MediaType -import okhttp3.Request -import okhttp3.RequestBody -import spock.lang.AutoCleanup -import spock.lang.IgnoreIf -import spock.lang.Shared -import spock.lang.Stepwise -import spock.lang.Unroll -import spock.util.concurrent.PollingConditions - -/** Due to the exposure cache it's important to run the tests in the specified order */ -@Stepwise -abstract class AbstractOpenFeatureSmokeTest extends AbstractServerSmokeTest { - - @Shared - protected final ufcConfig = new JsonSlurper().parse(fetchResource("ffe-system-test-data/ufc-config.json")) as Map - - @Shared - protected final ufcPayload = JsonOutput.toJson(ufcConfig) - - @Shared - protected final loggedAllocations = buildLoggedAllocations(ufcConfig) - - /** Serves the flag configuration as the Datadog managed CDN does. */ - @Shared - @AutoCleanup - protected TestHttpServer cdn = httpServer { - handlers { - prefix("/api/v2/feature-flagging/config/rules-based/server") { - response.status(200).send(JsonOutput.toJson([ - data: [type: 'universal-flag-configuration', attributes: ufcConfig] - ])) - } - } - } - - /** @return whether exposures are delivered through the Datadog Agent event platform proxy. */ - abstract boolean deliversExposures() - - /** Publishes the flag configuration before each evaluation. */ - void publishConfiguration() {} - - @Override - Closure decodedTracesCallback() { - return {} - } - - @Override - Closure decodedEvpProxyMessageCallback() { - return { String path, RequestApi request -> - if (!path.contains('api/v2/exposures')) { - return null - } - return new JsonSlurper().parse(request.body) - } - } - - @IgnoreIf({ !instance.deliversExposures() }) - void 'test open feature exposures'() { - setup: - publishConfiguration() - final testCases = parseTestCases() - assert !testCases.isEmpty() - - when: - final results = testCases.collect { - testCase -> - final response = evaluate(testCase) - final responseBody = new JsonSlurper().parse(response.body().byteStream()) - return [testCase: testCase, response: response, body: responseBody] - } - final expectedExposures = uniqueExpectedExposures(results) - - then: - results.every { - it.response.code() == 200 - } - !expectedExposures.isEmpty() - new PollingConditions(timeout: 10).eventually { - final requests = evpProxyMessages*.getV2() as List> - final events = requests*.exposures.flatten() - assert events.size() == expectedExposures.size() - expectedExposures.each { - expected -> - assert events.find { - event -> - event.flag.key == expected.flag && - event.allocation.key == expected.allocation && - event.variant.key == expected.variant && - event.subject.id == expected.targetingKey - } != null : "Unable to find exposure ${expected}" - } - } - } - - @Unroll("test open feature evaluation - #testCase.fileName[#testCase.index] - flag=#testCase.flag") - void 'test open feature evaluation'() { - setup: - publishConfiguration() - - when: - final response = evaluate(testCase) - - then: - response.code() == 200 - final responseBody = new JsonSlurper().parse(response.body().byteStream()) - responseBody.value == testCase.result.value - responseBody.reason == testCase.result.reason - if (testCase.result.containsKey('errorCode')) { - assert responseBody.errorCode == testCase.result.errorCode - } - if (testCase.result.containsKey('variant')) { - assert responseBody.variant == testCase.result.variant - } - if (testCase.result.flagMetadata?.allocationKey) { - assert responseBody.flagMetadata?.allocationKey == testCase.result.flagMetadata?.allocationKey - } - - where: - testCase << parseTestCases() - } - - protected okhttp3.Response evaluate(final Map testCase) { - final request = new Request.Builder() - .url("http://localhost:${httpPort}/openfeature/evaluate") - .post(RequestBody.create(MediaType.parse('application/json'), JsonOutput.toJson(testCase))) - .build() - return client.newCall(request).execute() - } - - protected static URL fetchResource(final String name) { - return Thread.currentThread().getContextClassLoader().getResource(name) - } - - protected static List> parseTestCases() { - final folder = fetchResource('ffe-system-test-data/evaluation-cases') - final uri = folder.toURI() - final testsPath = Paths.get(uri) - final files = Files.list(testsPath) - .filter(path -> path.toString().endsWith('.json')) - .sorted(Comparator.comparing(path -> path.fileName.toString())) - final result = [] - final slurper = new JsonSlurper() - files.each { - path -> - final testCases = slurper.parse(path.toFile()) as List> - testCases.eachWithIndex { - testCase, index -> - testCase.fileName = path.fileName.toString() - testCase.index = index - } - result.addAll(testCases) - } - assert !result.isEmpty() - return result - } - - protected List> uniqueExpectedExposures(final List> results) { - final expected = [] - final seen = [] as Set - results.each { result -> - final testCase = result.testCase as Map - final body = result.body as Map - final flag = testCase.flag as String - final allocation = body.flagMetadata?.allocationKey as String - final variant = body.variant as String - if (!variant || !allocation || !allocationLogs(flag, allocation)) { - return - } - - final exposure = [ - flag: flag, - allocation: allocation, - variant: variant, - targetingKey: testCase.targetingKey - ] - final key = "${exposure.flag}\u0000${exposure.targetingKey}\u0000${exposure.allocation}\u0000${exposure.variant}" - if (seen.add(key)) { - expected.add(exposure) - } - } - return expected - } - - protected boolean allocationLogs(final String flag, final String allocation) { - return loggedAllocations["${flag}\u0000${allocation}"] == true - } - - protected static Map buildLoggedAllocations(final Map config) { - final logged = [:] - (config.flags as Map).each { flag, definition -> - if (definition.allocations instanceof List) { - definition.allocations.each { allocation -> - if (allocation instanceof Map) { - logged["${flag}\u0000${allocation.key}"] = allocation.doLog == true - } - } - } - } - return logged - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.groovy deleted file mode 100644 index bf7932f7ea8..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.groovy +++ /dev/null @@ -1,24 +0,0 @@ -package datadog.smoketest.springboot - -/** - * The SDK with the Datadog Java agent and the default agentless configuration source: the SDK - * polls the flag configuration from the CDN and delivers exposures through the agent. - */ -class OpenFeatureAgentlessSmokeTest extends AbstractOpenFeatureSmokeTest { - - @Override - ProcessBuilder createProcessBuilder() { - final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") - final command = [javaPath()] - command.addAll(defaultJavaProperties) - command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) - final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) - builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL', "http://localhost:${cdn.address.port}".toString()) - return builder - } - - @Override - boolean deliversExposures() { - return true - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.groovy deleted file mode 100644 index 3108fe0bb8a..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.groovy +++ /dev/null @@ -1,95 +0,0 @@ -package datadog.smoketest.springboot - -import datadog.smoketest.AbstractServerSmokeTest -import okhttp3.Request -import okhttp3.RequestBody - -/** - * A malformed flag configuration that Remote Configuration delivers before the SDK subscribes is - * acknowledged by the agent, then reported as not applied once the SDK rejects it. - */ -class OpenFeatureMalformedRemoteConfigSmokeTest extends AbstractServerSmokeTest { - private static final String CONFIG_PATH = 'datadog/2/FFE_FLAGS/1/config' - private static final String MALFORMED_CONFIG = '{"flags":[]}' - private static final int APPLY_STATE_ACKNOWLEDGED = 2 - private static final int APPLY_STATE_ERROR = 3 - - @Override - ProcessBuilder createProcessBuilder() { - setRemoteConfig(CONFIG_PATH, MALFORMED_CONFIG) - - final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") - final command = [javaPath()] - command.addAll(defaultJavaProperties) - command.add('-Ddd.remote_config.enabled=true') - command.add("-Ddd.remote_config.url=http://localhost:${server.address.port}/v0.7/config".toString()) - command.addAll([ - '-jar', - springBootShadowJar, - "--server.port=${httpPort}".toString(), - '--openfeature.lazy-provider=true' - ]) - final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) - builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE', 'remote_config') - return builder - } - - /** The provider never gets a valid configuration, so the OpenFeature SDK logs its failed initialization. */ - @Override - boolean isErrorLog(final String line) { - return super.isErrorLog(line) && !line.contains(' d.o.sdk.') && !line.contains(' dev.openfeature.sdk.') - } - - void 'test malformed configuration received before the SDK is reported as not applied'() { - setup: - // The base resets the Remote Configuration response before each test. - setRemoteConfig(CONFIG_PATH, MALFORMED_CONFIG) - final served = rcClientMessages.size() - - when: 'the agent receives the configuration while no SDK has subscribed' - final acknowledged = waitForRequestAfter(served) { applyState(it) == APPLY_STATE_ACKNOWLEDGED } - final response = client.newCall(new Request.Builder() - .url("http://localhost:${httpPort}/openfeature/provider") - .post(RequestBody.create(null, new byte[0])) - .build()).execute() - - then: 'the SDK rejects it and the agent reports it as not applied' - response.code() == 202 - final rejected = waitForRequestAfter(acknowledged) { applyState(it) == APPLY_STATE_ERROR } - (configState(rejected).apply_error as String).contains('Malformed flag configuration') - - cleanup: - response?.close() - } - - /** - * Waits for an agent Remote Configuration request matching the predicate. - * - * @param from the index of the first request to consider. - * @param predicate the predicate to match. - * @return the index of the first matching request. - */ - private int waitForRequestAfter(final int from, final Closure predicate) { - int index = -1 - defaultPoll.eventually { - index = rcClientMessages.findIndexOf(from, predicate) - assert index >= 0 - } - return index - } - - private Map configState(final int index) { - final request = rcClientMessages[index] - final states = request.client?.state?.config_states as List> - return states?.find { - it.product == 'FFE_FLAGS' - } - } - - private static Integer applyState(final Map request) { - final states = request.client?.state?.config_states as List> - return states?.find { - it.product == 'FFE_FLAGS' - }?.apply_state as Integer - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy deleted file mode 100644 index d0808be6aea..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy +++ /dev/null @@ -1,101 +0,0 @@ -package datadog.smoketest.springboot - -import datadog.remoteconfig.Capabilities -import datadog.remoteconfig.Product -import groovy.json.JsonOutput -import groovy.json.JsonSlurper -import okhttp3.MediaType -import okhttp3.Request -import okhttp3.RequestBody - -/** The SDK with the Datadog Java agent, receiving the flag configuration from Remote Configuration. */ -class OpenFeatureProviderSmokeTest extends AbstractOpenFeatureSmokeTest { - - @Override - ProcessBuilder createProcessBuilder() { - setRemoteConfig("datadog/2/FFE_FLAGS/1/config", ufcPayload) - - final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") - final command = [javaPath()] - command.addAll(defaultJavaProperties) - command.add('-Ddd.trace.debug=true') - command.add('-Ddd.remote_config.enabled=true') - command.add("-Ddd.remote_config.url=http://localhost:${server.address.port}/v0.7/config".toString()) - command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) - final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) - builder.environment().put('DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED', 'true') - builder.environment().put('DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED', 'true') - builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE', 'remote_config') - return builder - } - - @Override - boolean deliversExposures() { - return true - } - - @Override - void publishConfiguration() { - setRemoteConfig("datadog/2/FFE_FLAGS/1/config", ufcPayload) - } - - void 'test first remote config poll asks agent for feature flags'() { - when: - final firstRcRequest = waitForRcClientRequest { req -> - return true - } - - then: - firstRcRequest == rcClientMessages.first() - // An already-running Agent gives a newly-started tracer one new-client cache bypass. If - // FFE_FLAGS is missing here and only appears on a later poll, the Agent can miss the fast path. - decodeProducts(firstRcRequest).find { it == Product.FFE_FLAGS } != null - final capabilities = decodeCapabilities(firstRcRequest) - hasCapability(capabilities, Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES) - } - - void 'test open feature evaluation enriches the request span'() { - setup: - publishConfiguration() - final request = new Request.Builder() - .url("http://localhost:${httpPort}/openfeature/evaluate") - .post(RequestBody.create(MediaType.parse('application/json'), JsonOutput.toJson([ - flag: 'flag-that-does-not-exist', - variationType: 'STRING', - defaultValue: 'fallback', - targetingKey: 'span-enrichment-smoke-test' - ]))) - .build() - - when: - final response = client.newCall(request).execute() - - then: - response.code() == 200 - new JsonSlurper().parse(response.body().byteStream()).value == 'fallback' - waitForSpan(defaultPoll) { span -> - span.parentId == 0 && - span.meta['ffe_runtime_defaults'] == '{"flag-that-does-not-exist":"fallback"}' - } - - cleanup: - response.close() - } - - private static Set decodeProducts(final Map request) { - return request.client.products.collect { Product.valueOf(it) } - } - - private static long decodeCapabilities(final Map request) { - final clientCapabilities = request.client.capabilities as byte[] - long capabilities = 0l - for (int i = 0; i < clientCapabilities.length; i++) { - capabilities |= (clientCapabilities[i] & 0xFFL) << ((clientCapabilities.length - i - 1) * 8) - } - return capabilities - } - - private static boolean hasCapability(final long capabilities, final long test) { - return (capabilities & test) > 0 - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.groovy deleted file mode 100644 index 70192aa9331..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.groovy +++ /dev/null @@ -1,10 +0,0 @@ -package datadog.smoketest.springboot - -/** The tracing shutdown hook flushes the SDK pending events. */ -class OpenFeatureShutdownSmokeTest extends AbstractOpenFeatureShutdownSmokeTest { - - @Override - boolean tracing() { - return true - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.groovy deleted file mode 100644 index 776ae905efa..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.groovy +++ /dev/null @@ -1,40 +0,0 @@ -package datadog.smoketest.springboot - -/** - * The SDK without the Datadog Java agent: the SDK polls the flag configuration from the CDN. Without - * an API key, events are not delivered. - */ -class OpenFeatureStandaloneSmokeTest extends AbstractOpenFeatureSmokeTest { - - @Override - protected String skipScopeContinuationCheckReason() { - 'Exercises the standalone OpenFeature SDK without the agent required by the diagnostic companion' - } - - @Override - ProcessBuilder createProcessBuilder() { - final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") - final command = [javaPath()] - command.addAll(nativeJavaProperties) - command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) - final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) - builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL', "http://localhost:${cdn.address.port}".toString()) - return builder - } - - @Override - boolean deliversExposures() { - return false - } - - @Override - boolean testTelemetry() { - return false - } - - /** Disables direct intake delivery, which would otherwise reach the real Datadog intake. */ - @Override - String apiKey() { - return '' - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.groovy deleted file mode 100644 index c300962d260..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.groovy +++ /dev/null @@ -1,13 +0,0 @@ -package datadog.smoketest.springboot - -/** - * Without tracing, the Feature Flagging system registers its own shutdown hook to flush the SDK - * pending events. - */ -class OpenFeatureTraceDisabledShutdownSmokeTest extends AbstractOpenFeatureShutdownSmokeTest { - - @Override - boolean tracing() { - return false - } -} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.java new file mode 100644 index 00000000000..1ccc67e21f7 --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.java @@ -0,0 +1,81 @@ +package datadog.smoketest.springboot; + +import static datadog.smoketest.springboot.OpenFeatureFixtures.agentlessApp; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evaluate; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evaluationCase; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evpProxy; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.smoketest.SmokeServerApp; +import datadog.smoketest.backend.EvpProxy; +import datadog.smoketest.springboot.OpenFeatureFixtures.Evaluation; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** The SDK with the Datadog Java agent flushes its pending events when the application exits. */ +abstract class AbstractOpenFeatureShutdownSmokeTest { + private static final String FLAG_EVALUATION_PATH = "api/v2/flagevaluation"; + + static SmokeServerApp shutdownApp(String name, boolean tracing) { + return agentlessApp(name).jvmArgs("-Ddd.trace.enabled=" + tracing).skipTelemetryCheck().build(); + } + + /** + * Gets the application under test. + * + * @return The application under test. + */ + abstract SmokeServerApp app(); + + @Test + void flushesPendingFlagEvaluationsAtExit() throws IOException { + // The first evaluation is flushed within the 10 seconds flush interval, which then restarts: + // the next evaluation stays pending until the application exits. + evaluateCase("test-case-boolean-false-assignment.json"); + evpProxy(app()) + .waitForRequest( + request -> + FLAG_EVALUATION_PATH.equals(request.path()) + && evaluatedFlags(request).contains("boolean-false-assignment"), + 20); + evaluateCase("test-case-boolean-one-of-matches.json"); + assertFalse(evaluatedFlags().contains("boolean-one-of-matches"), "the evaluation is pending"); + + app().terminate(30, SECONDS); + + assertTrue(evaluatedFlags().contains("boolean-one-of-matches"), "the evaluation is flushed"); + for (EvpProxy.Request request : evpProxy(app()).requests(FLAG_EVALUATION_PATH)) { + assertEquals("dd-trace-java", request.header("DD-EVP-ORIGIN"), "origin"); + String originVersion = request.header("DD-EVP-ORIGIN-VERSION"); + assertTrue(originVersion != null && !originVersion.isEmpty(), "origin version"); + } + } + + private void evaluateCase(String evaluationCaseFile) throws IOException { + Evaluation evaluation = evaluate(app(), evaluationCase(evaluationCaseFile).request); + assertEquals(200, evaluation.code, evaluation.body); + } + + private List evaluatedFlags() { + List flags = new ArrayList<>(); + for (EvpProxy.Request request : evpProxy(app()).requests(FLAG_EVALUATION_PATH)) { + flags.addAll(evaluatedFlags(request)); + } + return flags; + } + + private static List evaluatedFlags(EvpProxy.Request request) { + List flags = new ArrayList<>(); + for (Object evaluation : (List) request.json().get("flagEvaluations")) { + Map flag = (Map) ((Map) evaluation).get("flag"); + flags.add((String) flag.get("key")); + } + return flags; + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.java new file mode 100644 index 00000000000..3a2c81cd01f --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.java @@ -0,0 +1,213 @@ +package datadog.smoketest.springboot; + +import static datadog.smoketest.springboot.OpenFeatureFixtures.UFC_CONFIG; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evaluate; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evaluationCases; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evpProxy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import datadog.smoketest.SmokeServerApp; +import datadog.smoketest.backend.EvpProxy; +import datadog.smoketest.springboot.OpenFeatureFixtures.Evaluation; +import datadog.smoketest.springboot.OpenFeatureFixtures.EvaluationCase; +import datadog.trace.test.util.PollingConditions; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Evaluates the system tests cases through the application, and checks the exposures it reports. + * The exposure cache reports each exposure once, so the exposures are checked before any other + * evaluation. + */ +@TestMethodOrder(OrderAnnotation.class) +abstract class AbstractOpenFeatureSmokeTest { + private static final Set LOGGED_ALLOCATIONS = loggedAllocations(); + + /** + * Gets the application under test. + * + * @return The application under test. + */ + abstract SmokeServerApp app(); + + /** + * Tells whether exposures are delivered through the Datadog Agent event platform proxy. + * + * @return Whether exposures are delivered through the Datadog Agent event platform proxy. + */ + abstract boolean deliversExposures(); + + /** Publishes the flag configuration before each evaluation. */ + void publishConfiguration() {} + + @Test + @Order(1) + void reportsExposures() throws IOException { + assumeTrue(deliversExposures(), "exposures are not delivered"); + publishConfiguration(); + Set expected = new LinkedHashSet<>(); + for (EvaluationCase evaluationCase : evaluationCases()) { + Evaluation evaluation = evaluate(app(), evaluationCase.request); + assertEquals(200, evaluation.code, evaluation.body); + Exposure exposure = expectedExposure(evaluationCase, evaluation.json()); + if (exposure != null) { + expected.add(exposure); + } + } + assertFalse(expected.isEmpty(), "some evaluations are exposed"); + + EvpProxy evpProxy = evpProxy(app()); + new PollingConditions(10) + .eventually( + () -> { + List reported = reportedExposures(evpProxy); + assertEquals(expected.size(), reported.size(), "reported exposures: " + reported); + for (Exposure exposure : expected) { + assertTrue(reported.contains(exposure), "Unable to find exposure " + exposure); + } + }); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("datadog.smoketest.springboot.OpenFeatureFixtures#evaluationCases") + @Order(2) + void evaluatesFlags(EvaluationCase evaluationCase) throws IOException { + publishConfiguration(); + + Evaluation evaluation = evaluate(app(), evaluationCase.request); + + assertEquals(200, evaluation.code, evaluation.body); + Map actual = evaluation.json(); + Map expected = evaluationCase.result(); + assertTrue( + Json.equal(expected.get("value"), actual.get("value")), + "value: expected " + expected.get("value") + " but was " + actual.get("value")); + assertEquals(expected.get("reason"), actual.get("reason"), "reason"); + if (expected.containsKey("errorCode")) { + assertEquals(expected.get("errorCode"), actual.get("errorCode"), "error code"); + } + if (expected.containsKey("variant")) { + assertEquals(expected.get("variant"), actual.get("variant"), "variant"); + } + Object allocationKey = allocationKey(expected); + if (allocationKey != null && !"".equals(allocationKey)) { + assertEquals(allocationKey, allocationKey(actual), "allocation key"); + } + } + + private static Exposure expectedExposure( + EvaluationCase evaluationCase, Map response) { + String flag = evaluationCase.flag(); + Object allocation = allocationKey(response); + Object variant = response.get("variant"); + if (variant == null + || "".equals(variant) + || allocation == null + || "".equals(allocation) + || !LOGGED_ALLOCATIONS.contains(flag + '\u0000' + allocation)) { + return null; + } + return new Exposure(flag, allocation, variant, evaluationCase.targetingKey()); + } + + private static List reportedExposures(EvpProxy evpProxy) { + List exposures = new ArrayList<>(); + for (EvpProxy.Request request : evpProxy.requests("api/v2/exposures")) { + for (Object event : (List) request.json().get("exposures")) { + Map exposure = Json.object(event); + exposures.add( + new Exposure( + key(exposure.get("flag")), + key(exposure.get("allocation")), + key(exposure.get("variant")), + Json.object(exposure.get("subject")).get("id"))); + } + } + return exposures; + } + + private static Object key(Object reference) { + return Json.object(reference).get("key"); + } + + private static Object allocationKey(Map evaluation) { + Object metadata = evaluation.get("flagMetadata"); + return metadata == null ? null : Json.object(metadata).get("allocationKey"); + } + + private static Set loggedAllocations() { + Set logged = new HashSet<>(); + Map flags = Json.object(Json.object(Json.parse(UFC_CONFIG)).get("flags")); + for (Map.Entry flag : flags.entrySet()) { + Object allocations = Json.object(flag.getValue()).get("allocations"); + if (allocations instanceof List) { + for (Object allocation : (List) allocations) { + if (allocation instanceof Map + && Boolean.TRUE.equals(Json.object(allocation).get("doLog"))) { + logged.add(flag.getKey() + '\u0000' + Json.object(allocation).get("key")); + } + } + } + } + return logged; + } + + /** An exposure, identified by its flag, allocation, variant and subject. */ + private static final class Exposure { + private final Object flag; + private final Object allocation; + private final Object variant; + private final Object subject; + + Exposure(Object flag, Object allocation, Object variant, Object subject) { + this.flag = flag; + this.allocation = allocation; + this.variant = variant; + this.subject = subject; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Exposure)) { + return false; + } + Exposure that = (Exposure) other; + return Objects.equals(this.flag, that.flag) + && Objects.equals(this.allocation, that.allocation) + && Objects.equals(this.variant, that.variant) + && Objects.equals(this.subject, that.subject); + } + + @Override + public int hashCode() { + return Objects.hash(this.flag, this.allocation, this.variant, this.subject); + } + + @Override + public String toString() { + return "flag=" + + this.flag + + " allocation=" + + this.allocation + + " variant=" + + this.variant + + " subject=" + + this.subject; + } + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/Json.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/Json.java new file mode 100644 index 00000000000..48209e01162 --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/Json.java @@ -0,0 +1,159 @@ +package datadog.smoketest.springboot; + +import com.squareup.moshi.JsonReader; +import com.squareup.moshi.JsonWriter; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import okio.Buffer; + +/** + * Reads and writes JSON, keeping numbers as written: the evaluation cases tell integers from + * decimals, which the application maps to different types. + */ +final class Json { + private Json() {} + + /** + * Parses a JSON document. + * + * @param json The JSON document. + * @return The parsed value: maps, lists, strings, {@link BigDecimal} numbers, booleans or {@code + * null}. + */ + static Object parse(String json) { + try (JsonReader reader = JsonReader.of(new Buffer().writeUtf8(json))) { + return read(reader); + } catch (IOException e) { + throw new UncheckedIOException("Failed to parse JSON: " + json, e); + } + } + + /** + * Writes a value as a JSON document. + * + * @param value The value, as returned by {@link #parse(String)}. + * @return The JSON document. + */ + static String write(Object value) { + Buffer buffer = new Buffer(); + try (JsonWriter writer = JsonWriter.of(buffer)) { + writer.setSerializeNulls(true); + write(writer, value); + } catch (IOException e) { + throw new UncheckedIOException("Failed to write JSON", e); + } + return buffer.readUtf8(); + } + + /** + * Compares two JSON values, numbers by value. + * + * @param expected The expected value. + * @param actual The actual value. + * @return Whether both values are equal. + */ + static boolean equal(Object expected, Object actual) { + if (expected instanceof BigDecimal && actual instanceof BigDecimal) { + return ((BigDecimal) expected).compareTo((BigDecimal) actual) == 0; + } + if (expected instanceof Map && actual instanceof Map) { + Map expectedMap = (Map) expected; + Map actualMap = (Map) actual; + if (!expectedMap.keySet().equals(actualMap.keySet())) { + return false; + } + for (Map.Entry entry : expectedMap.entrySet()) { + if (!equal(entry.getValue(), actualMap.get(entry.getKey()))) { + return false; + } + } + return true; + } + if (expected instanceof List && actual instanceof List) { + List expectedList = (List) expected; + List actualList = (List) actual; + if (expectedList.size() != actualList.size()) { + return false; + } + Iterator actualValues = actualList.iterator(); + for (Object expectedValue : expectedList) { + if (!equal(expectedValue, actualValues.next())) { + return false; + } + } + return true; + } + return Objects.equals(expected, actual); + } + + @SuppressWarnings("unchecked") + static Map object(Object value) { + return (Map) value; + } + + @SuppressWarnings("unchecked") + static List array(Object value) { + return (List) value; + } + + private static Object read(JsonReader reader) throws IOException { + switch (reader.peek()) { + case BEGIN_OBJECT: + Map object = new LinkedHashMap<>(); + reader.beginObject(); + while (reader.hasNext()) { + object.put(reader.nextName(), read(reader)); + } + reader.endObject(); + return object; + case BEGIN_ARRAY: + List array = new ArrayList<>(); + reader.beginArray(); + while (reader.hasNext()) { + array.add(read(reader)); + } + reader.endArray(); + return array; + case NUMBER: + return new BigDecimal(reader.nextString()); + case BOOLEAN: + return reader.nextBoolean(); + case NULL: + return reader.nextNull(); + default: + return reader.nextString(); + } + } + + private static void write(JsonWriter writer, Object value) throws IOException { + if (value instanceof Map) { + writer.beginObject(); + for (Map.Entry entry : ((Map) value).entrySet()) { + writer.name(String.valueOf(entry.getKey())); + write(writer, entry.getValue()); + } + writer.endObject(); + } else if (value instanceof List) { + writer.beginArray(); + for (Object item : (List) value) { + write(writer, item); + } + writer.endArray(); + } else if (value instanceof Number) { + writer.value((Number) value); + } else if (value instanceof Boolean) { + writer.value((Boolean) value); + } else if (value == null) { + writer.nullValue(); + } else { + writer.value(value.toString()); + } + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.java new file mode 100644 index 00000000000..5a8151b3e4d --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.java @@ -0,0 +1,25 @@ +package datadog.smoketest.springboot; + +import static datadog.smoketest.springboot.OpenFeatureFixtures.agentlessApp; + +import datadog.smoketest.SmokeServerApp; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * The SDK with the Datadog Java agent and the default agentless configuration source: the SDK polls + * the flag configuration from the CDN and delivers exposures through the agent. + */ +class OpenFeatureAgentlessSmokeTest extends AbstractOpenFeatureSmokeTest { + @RegisterExtension + static final SmokeServerApp APP = agentlessApp("openfeature-agentless").build(); + + @Override + SmokeServerApp app() { + return APP; + } + + @Override + boolean deliversExposures() { + return true; + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureFixtures.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureFixtures.java new file mode 100644 index 00000000000..6a82f05ebad --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureFixtures.java @@ -0,0 +1,235 @@ +package datadog.smoketest.springboot; + +import static datadog.trace.agent.test.server.http.JavaTestHttpServer.httpServer; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Comparator.comparing; +import static java.util.stream.Collectors.toList; + +import datadog.smoketest.SmokeServerApp; +import datadog.smoketest.backend.AgentBackend; +import datadog.smoketest.backend.EvpProxy; +import datadog.smoketest.backend.MockAgentBackend; +import datadog.trace.agent.test.server.http.JavaTestHttpServer; +import datadog.trace.agent.test.utils.OkHttpUtils; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/** + * The application, flag configuration and evaluation cases shared by the OpenFeature smoke tests. + */ +final class OpenFeatureFixtures { + static final String APPLICATION_JAR = + System.getProperty("datadog.smoketest.springboot.shadowJar.path"); + static final String AGENTLESS_BASE_URL = + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"; + static final String FLAG_CONFIGURATION_PATH = "datadog/2/FFE_FLAGS/1/config"; + + /** The flag configuration, from the system tests data. */ + static final String UFC_CONFIG = read("ufc-config.json"); + + /** The flag configuration, as the Datadog managed CDN serves it. */ + private static final byte[] CDN_RESPONSE = + ("{\"data\":{\"type\":\"universal-flag-configuration\",\"attributes\":" + UFC_CONFIG + "}}") + .getBytes(UTF_8); + + /** Serves the flag configuration as the Datadog managed CDN does. */ + static final JavaTestHttpServer CDN = + httpServer( + server -> + server.handlers( + handlers -> + handlers.prefix( + "/api/v2/feature-flagging/config/rules-based/server", + api -> + api.getResponse() + .status(200) + .sendWithType("application/json", CDN_RESPONSE)))); + + private static final MediaType JSON = MediaType.parse("application/json"); + private static final OkHttpClient CLIENT = OkHttpUtils.client(); + + private OpenFeatureFixtures() {} + + /** + * Creates the agent stand-in. The dd-apm-test-agent neither proxies the flag evaluation events + * nor advertises the event platform proxy headers, so the tests use the in-process mock agent. + * + * @return A new mock agent backend. + */ + @SuppressWarnings("deprecation") + static MockAgentBackend agent() { + return AgentBackend.mockAgent(); + } + + /** + * Starts building the application with the agentless configuration source, polling the CDN. + * + * @param name The application name. + * @return The application builder. + */ + static SmokeServerApp.Builder agentlessApp(String name) { + return SmokeServerApp.named(name) + .jar(APPLICATION_JAR) + .args("--server.port=${app.httpPort}") + .env(AGENTLESS_BASE_URL, CDN.getAddress().toString()) + .backend(agent()); + } + + /** + * Gets the requests the application sent through the agent event platform proxy. + * + * @param app The application, reporting to the {@link #agent()}. + * @return The event platform proxy requests. + */ + static EvpProxy evpProxy(SmokeServerApp app) { + return ((MockAgentBackend) app.backend()).evpProxy(); + } + + /** + * Lists the system tests evaluation cases. + * + * @return The evaluation cases, sorted by file name then position. + */ + static List evaluationCases() { + try (Stream listing = Files.list(resourcePath("evaluation-cases"))) { + List files = + listing + .filter(path -> path.toString().endsWith(".json")) + .sorted(comparing(path -> path.getFileName().toString())) + .collect(toList()); + List cases = new ArrayList<>(); + for (Path file : files) { + cases.addAll(parseCases(file.getFileName().toString(), readFile(file))); + } + if (cases.isEmpty()) { + throw new IllegalStateException("No evaluation case found"); + } + return cases; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Gets the first system tests evaluation case of a file. + * + * @param fileName The evaluation cases file name. + * @return The first evaluation case of the file. + */ + static EvaluationCase evaluationCase(String fileName) { + return parseCases(fileName, read("evaluation-cases/" + fileName)).get(0); + } + + /** + * Evaluates a flag through the application. + * + * @param app The application. + * @param request The evaluation request. + * @return The evaluation response. + * @throws IOException If the request failed. + */ + static Evaluation evaluate(SmokeServerApp app, Map request) throws IOException { + Request httpRequest = + new Request.Builder() + .url(app.url() + "/openfeature/evaluate") + .post(RequestBody.create(JSON, Json.write(request))) + .build(); + try (Response response = CLIENT.newCall(httpRequest).execute()) { + return new Evaluation( + response.code(), response.body() == null ? "" : response.body().string()); + } + } + + private static List parseCases(String fileName, String json) { + List entries = Json.array(Json.parse(json)); + List cases = new ArrayList<>(); + for (int index = 0; index < entries.size(); index++) { + cases.add(new EvaluationCase(fileName, index, Json.object(entries.get(index)))); + } + return cases; + } + + private static Path resourcePath(String name) { + URL url = + Thread.currentThread().getContextClassLoader().getResource("ffe-system-test-data/" + name); + if (url == null) { + throw new IllegalStateException("Missing system tests data: " + name); + } + try { + return Paths.get(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalStateException(e); + } + } + + private static String read(String name) { + try { + return readFile(resourcePath(name)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static String readFile(Path file) throws IOException { + return new String(Files.readAllBytes(file), UTF_8); + } + + /** A system tests evaluation case: the evaluation request, along with its expected result. */ + static final class EvaluationCase { + private final String fileName; + private final int index; + final Map request; + + EvaluationCase(String fileName, int index, Map request) { + this.fileName = fileName; + this.index = index; + this.request = request; + } + + String flag() { + return (String) this.request.get("flag"); + } + + String targetingKey() { + return (String) this.request.get("targetingKey"); + } + + Map result() { + return Json.object(this.request.get("result")); + } + + @Override + public String toString() { + return this.fileName + "[" + this.index + "] flag=" + flag(); + } + } + + /** The application response to an evaluation request. */ + static final class Evaluation { + final int code; + final String body; + + Evaluation(int code, String body) { + this.code = code; + this.body = body; + } + + Map json() { + return Json.object(Json.parse(this.body)); + } + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.java new file mode 100644 index 00000000000..b7ba60092f0 --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.java @@ -0,0 +1,93 @@ +package datadog.smoketest.springboot; + +import static datadog.smoketest.springboot.OpenFeatureFixtures.APPLICATION_JAR; +import static datadog.smoketest.springboot.OpenFeatureFixtures.FLAG_CONFIGURATION_PATH; +import static datadog.smoketest.springboot.OpenFeatureFixtures.agent; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.smoketest.SmokeServerApp; +import datadog.smoketest.backend.MockAgentBackend; +import datadog.smoketest.backend.RemoteConfig; +import datadog.trace.agent.test.utils.OkHttpUtils; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * A malformed flag configuration that Remote Configuration delivers before the SDK subscribes is + * acknowledged by the agent, then reported as not applied once the SDK rejects it. + */ +class OpenFeatureMalformedRemoteConfigSmokeTest { + private static final String MALFORMED_CONFIG = "{\"flags\":[]}"; + private static final int APPLY_STATE_ACKNOWLEDGED = 2; + private static final int APPLY_STATE_ERROR = 3; + private static final MockAgentBackend AGENT = agent(); + + @RegisterExtension + static final SmokeServerApp APP = + SmokeServerApp.named("openfeature-malformed-remote-config") + .jar(APPLICATION_JAR) + .placeholder("agent.port", () -> Integer.toString(AGENT.port())) + .jvmArgs( + "-Ddd.remote_config.enabled=true", + "-Ddd.remote_config.url=http://localhost:${agent.port}/v0.7/config") + .args("--server.port=${app.httpPort}", "--openfeature.lazy-provider=true") + .env("DD_FEATURE_FLAGS_CONFIGURATION_SOURCE", "remote_config") + .backend(AGENT) + // The provider never gets a valid configuration, so the OpenFeature SDK logs its failed + // initialization. + .allowedErrorLogs(" d.o.sdk.", " dev.openfeature.sdk.") + .build(); + + @Test + void reportsAMalformedConfigurationReceivedBeforeTheSdkAsNotApplied() throws IOException { + RemoteConfig remoteConfig = AGENT.remoteConfig(); + remoteConfig.setConfig(FLAG_CONFIGURATION_PATH, MALFORMED_CONFIG); + + // The agent receives the configuration while no SDK has subscribed. + remoteConfig.waitForRequest(request -> applyState(request) == APPLY_STATE_ACKNOWLEDGED); + Request registration = + new Request.Builder() + .url(APP.url() + "/openfeature/provider") + .post(RequestBody.create(null, new byte[0])) + .build(); + try (Response response = OkHttpUtils.client().newCall(registration).execute()) { + assertEquals(202, response.code(), "provider registration"); + } + + // The SDK rejects it and the agent reports it as not applied. + Map rejected = + remoteConfig.waitForRequest(request -> applyState(request) == APPLY_STATE_ERROR); + Object applyError = flagConfigurationState(rejected).get("apply_error"); + assertTrue( + String.valueOf(applyError).contains("Malformed flag configuration"), + "apply error: " + applyError); + } + + private static int applyState(Map request) { + Map state = flagConfigurationState(request); + Object applyState = state == null ? null : state.get("apply_state"); + return applyState instanceof Number ? ((Number) applyState).intValue() : -1; + } + + private static Map flagConfigurationState(Map request) { + Object client = request.get("client"); + Object state = client instanceof Map ? ((Map) client).get("state") : null; + Object configStates = state instanceof Map ? ((Map) state).get("config_states") : null; + if (configStates instanceof List) { + for (Object configState : (List) configStates) { + if (configState instanceof Map + && "FFE_FLAGS".equals(((Map) configState).get("product"))) { + return Json.object(configState); + } + } + } + return null; + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.java new file mode 100644 index 00000000000..527d6a6c539 --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.java @@ -0,0 +1,112 @@ +package datadog.smoketest.springboot; + +import static datadog.remoteconfig.Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES; +import static datadog.smoketest.backend.RemoteConfig.capabilities; +import static datadog.smoketest.backend.RemoteConfig.products; +import static datadog.smoketest.springboot.OpenFeatureFixtures.APPLICATION_JAR; +import static datadog.smoketest.springboot.OpenFeatureFixtures.FLAG_CONFIGURATION_PATH; +import static datadog.smoketest.springboot.OpenFeatureFixtures.UFC_CONFIG; +import static datadog.smoketest.springboot.OpenFeatureFixtures.agent; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evaluate; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.remoteconfig.Product; +import datadog.smoketest.SmokeServerApp; +import datadog.smoketest.backend.MockAgentBackend; +import datadog.smoketest.springboot.OpenFeatureFixtures.Evaluation; +import datadog.trace.test.util.PollingConditions; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * The SDK with the Datadog Java agent, receiving the flag configuration from Remote Configuration. + */ +class OpenFeatureProviderSmokeTest extends AbstractOpenFeatureSmokeTest { + private static final MockAgentBackend AGENT = agent(); + + static { + // The provider initializes with the application, so the configuration is served from start-up. + AGENT.remoteConfig().setConfig(FLAG_CONFIGURATION_PATH, UFC_CONFIG); + } + + @RegisterExtension + static final SmokeServerApp APP = + SmokeServerApp.named("openfeature-provider") + .jar(APPLICATION_JAR) + .placeholder("agent.port", () -> Integer.toString(AGENT.port())) + .jvmArgs( + "-Ddd.trace.debug=true", + "-Ddd.remote_config.enabled=true", + "-Ddd.remote_config.url=http://localhost:${agent.port}/v0.7/config") + .args("--server.port=${app.httpPort}") + .env("DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED", "true") + .env("DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED", "true") + .env("DD_FEATURE_FLAGS_CONFIGURATION_SOURCE", "remote_config") + .backend(AGENT) + .build(); + + private static Map firstPoll; + + /** Captures the first Remote Configuration poll, before the backend resets for the first test. */ + @BeforeAll + static void captureFirstPoll() { + firstPoll = AGENT.remoteConfig().waitForRequest(request -> true); + } + + @Override + SmokeServerApp app() { + return APP; + } + + @Override + boolean deliversExposures() { + return true; + } + + @Override + void publishConfiguration() { + AGENT.remoteConfig().setConfig(FLAG_CONFIGURATION_PATH, UFC_CONFIG); + } + + @Test + void asksTheAgentForFeatureFlagsOnTheFirstPoll() { + // An already-running Agent gives a newly-started tracer one new-client cache bypass. If + // FFE_FLAGS is missing here and only appears on a later poll, the Agent can miss the fast path. + assertTrue(products(firstPoll).contains(Product.FFE_FLAGS.name()), "FFE_FLAGS product"); + assertTrue( + (capabilities(firstPoll) & CAPABILITY_FFE_FLAG_CONFIGURATION_RULES) != 0, + "flag configuration rules capability"); + } + + @Test + void enrichesTheRequestSpan() throws IOException { + publishConfiguration(); + Map request = new HashMap<>(); + request.put("flag", "flag-that-does-not-exist"); + request.put("variationType", "STRING"); + request.put("defaultValue", "fallback"); + request.put("targetingKey", "span-enrichment-smoke-test"); + + Evaluation evaluation = evaluate(APP, request); + + assertEquals(200, evaluation.code, evaluation.body); + assertEquals("fallback", evaluation.json().get("value")); + new PollingConditions(30) + .eventually( + () -> + assertTrue( + APP.traces().getTraces().stream() + .flatMap(trace -> trace.getSpans().stream()) + .anyMatch( + span -> + span.getParentId() == 0 + && "{\"flag-that-does-not-exist\":\"fallback\"}" + .equals(span.getMeta().get("ffe_runtime_defaults"))), + "the request root span carries the runtime default")); + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.java new file mode 100644 index 00000000000..988025dea38 --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.java @@ -0,0 +1,14 @@ +package datadog.smoketest.springboot; + +import datadog.smoketest.SmokeServerApp; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** The tracing shutdown hook flushes the SDK pending events. */ +class OpenFeatureShutdownSmokeTest extends AbstractOpenFeatureShutdownSmokeTest { + @RegisterExtension static final SmokeServerApp APP = shutdownApp("openfeature-shutdown", true); + + @Override + SmokeServerApp app() { + return APP; + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.java new file mode 100644 index 00000000000..738c76170b7 --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.java @@ -0,0 +1,30 @@ +package datadog.smoketest.springboot; + +import static datadog.smoketest.springboot.OpenFeatureFixtures.agentlessApp; + +import datadog.smoketest.SmokeServerApp; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * The SDK without the Datadog Java agent: the SDK polls the flag configuration from the CDN. + * Without an API key, events are not delivered. + */ +class OpenFeatureStandaloneSmokeTest extends AbstractOpenFeatureSmokeTest { + @RegisterExtension + static final SmokeServerApp APP = + agentlessApp("openfeature-standalone") + // Disables direct intake delivery, which would otherwise reach the real Datadog intake. + .env("DD_API_KEY", "") + .noAgent() + .build(); + + @Override + SmokeServerApp app() { + return APP; + } + + @Override + boolean deliversExposures() { + return false; + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.java new file mode 100644 index 00000000000..20e1f5965cf --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.java @@ -0,0 +1,18 @@ +package datadog.smoketest.springboot; + +import datadog.smoketest.SmokeServerApp; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Without tracing, the Feature Flagging system registers its own shutdown hook to flush the SDK + * pending events. + */ +class OpenFeatureTraceDisabledShutdownSmokeTest extends AbstractOpenFeatureShutdownSmokeTest { + @RegisterExtension + static final SmokeServerApp APP = shutdownApp("openfeature-trace-disabled-shutdown", false); + + @Override + SmokeServerApp app() { + return APP; + } +} diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy index 5187f1f8af1..ca6e6946f77 100644 --- a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy @@ -64,12 +64,6 @@ abstract class AbstractSmokeTest extends ProcessManager { @Shared protected TestHttpServer.Headers lastTraceRequestHeaders = null - @Shared - protected CopyOnWriteArrayList> rcClientMessages = new CopyOnWriteArrayList() - - @Shared - private Throwable rcClientDecodingFailure = null - @Shared protected final PollingConditions defaultPoll = new PollingConditions(timeout: 30, initialDelay: 0, delay: 1, factor: 1) @@ -116,14 +110,6 @@ abstract class AbstractSmokeTest extends ProcessManager { response.status(200).send() } prefix("/v0.7/config") { - if (request.getBody() != null) { - try { - final msg = new JsonSlurper().parseText(new String(request.getBody(), StandardCharsets.UTF_8)) as Map - rcClientMessages.add(msg) - } catch (Throwable t) { - rcClientDecodingFailure = t - } - } response.status(200).send(remoteConfigResponse) } prefix("/telemetry/proxy/api/v2/apmtelemetry") { @@ -413,21 +399,6 @@ abstract class AbstractSmokeTest extends ProcessManager { } } - Map waitForRcClientRequest(final Function, Boolean> predicate) { - waitForRcClientRequest(defaultPoll, predicate) - } - - Map waitForRcClientRequest(final PollingConditions poll, final Function, Boolean> predicate) { - def message = null - poll.eventually { - if (rcClientDecodingFailure != null) { - throw rcClientDecodingFailure - } - assert (message = rcClientMessages.find { predicate.apply(it) }) != null - } - return message - } - Tuple2 waitForEvpProxyMessage(final Function, Boolean> predicate) { waitForEvpProxyMessage(defaultPoll, predicate) } diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeServerApp.java b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeServerApp.java index def28a90848..4f4a6989f0e 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeServerApp.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeServerApp.java @@ -8,6 +8,7 @@ import datadog.trace.api.internal.VisibleForTesting; import java.io.IOException; import java.net.URI; +import java.util.concurrent.TimeUnit; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; @@ -35,6 +36,7 @@ public final class SmokeServerApp extends AbstractSmokeApp { private final int httpPort; private final OkHttpClient httpClient = new OkHttpClient(); + private boolean terminated; private SmokeServerApp(Builder builder) { super(builder); @@ -89,6 +91,46 @@ int get(String path) { } } + /** + * Terminates the app as a platform stops it, with a termination signal that runs its shutdown + * hooks, and waits for it to exit. Use it to assert what the app does at exit: later test methods + * can't use the app anymore. The logs the app writes while shutting down are only captured on + * Java 9 and later. + * + * @param timeout The maximum time to wait for the app to exit. + * @param unit The time unit of {@code timeout}. + * @throws AssertionError If the app does not exit within the timeout. + */ + public void terminate(long timeout, TimeUnit unit) { + Process process = process(); + // Complete the test scope diagnostics while the app can still report them. + verifyScopeDiagnostics(); + this.terminated = true; + signalTermination(process); + boolean exited; + try { + exited = process.waitFor(timeout, unit); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("Interrupted while waiting for app '" + name() + "' to exit", e); + } + if (!exited) { + throw new AssertionError( + "App '" + name() + "' did not exit within " + timeout + " " + unit + " once terminated"); + } + } + + private static void signalTermination(Process process) { + // Process.destroy() also closes the app output streams, losing what it logs while shutting + // down, unlike ProcessHandle.destroy() (Java 9+). + try { + Object handle = Process.class.getMethod("toHandle").invoke(process); + Class.forName("java.lang.ProcessHandle").getMethod("destroy").invoke(handle); + } catch (ReflectiveOperationException e) { + process.destroy(); + } + } + @Override protected void onStarted() { waitForPortToOpen(this.httpPort, startupTimeoutSeconds(), SECONDS, process()); @@ -109,7 +151,9 @@ protected void onBeforeEach() { @Override protected void onAfterEach() { - assertAlive("at the end of a test"); + if (!this.terminated) { + assertAlive("at the end of a test"); + } } private void assertAlive(String when) { @@ -117,6 +161,9 @@ private void assertAlive(String when) { if (process == null) { throw new IllegalStateException("App '" + name() + "' was never launched"); } + if (this.terminated) { + throw new IllegalStateException("App '" + name() + "' was terminated by a previous test"); + } if (!process.isAlive()) { throw new IllegalStateException( "App '" + name() + "' exited with value " + process.exitValue() + " " + when); diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/AgentBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/AgentBackend.java index b8172794eb1..75ec0edc17a 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/AgentBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/AgentBackend.java @@ -148,7 +148,7 @@ public void afterAll(ExtensionContext context) { * @deprecated Prefer {@link #testAgent()}, which speaks the real agent protocol. */ @Deprecated - public static AgentBackend mockAgent() { + public static MockAgentBackend mockAgent() { return new MockAgentBackend(); } diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/EvpProxy.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/EvpProxy.java new file mode 100644 index 00000000000..77206af597b --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/EvpProxy.java @@ -0,0 +1,124 @@ +package datadog.smoketest.backend; + +import static datadog.smoketest.backend.Polling.waitForFirst; +import static java.nio.charset.StandardCharsets.UTF_8; + +import datadog.trace.agent.test.server.http.JavaTestHttpServer.Headers; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.function.Supplier; + +/** + * This class is a query facade over the requests the tracer sent through the event platform proxy + * ({@code /evp_proxy/v2/}) of an {@link AgentBackend}. + */ +public final class EvpProxy { + private static final double DEFAULT_TIMEOUT_SECONDS = 30; + + private final Supplier> source; + + EvpProxy(Supplier> source) { + this.source = source; + } + + /** + * Returns the captured requests, in arrival order. + * + * @return The captured requests. + */ + public List requests() { + return this.source.get(); + } + + /** + * Returns the captured requests to an intake path, in arrival order. + * + * @param path The intake path, after the proxy prefix (e.g. {@code api/v2/exposures}). + * @return The captured requests to this path. + */ + public List requests(String path) { + List matching = new ArrayList<>(); + for (Request request : requests()) { + if (path.equals(request.path())) { + matching.add(request); + } + } + return matching; + } + + /** + * Waits up to {@value #DEFAULT_TIMEOUT_SECONDS}s for a captured request satisfying the given + * predicate, and returns it. + * + * @param predicate The predicate a request must satisfy. + * @return The first matching request. + * @throws AssertionError If no matching request arrives before the timeout. + */ + public Request waitForRequest(Predicate predicate) { + return waitForRequest(predicate, DEFAULT_TIMEOUT_SECONDS); + } + + /** + * Waits for a captured request satisfying the given predicate, and returns it. + * + * @param predicate The predicate a request must satisfy. + * @param timeoutSeconds How long to wait, in seconds. + * @return The first matching request. + * @throws AssertionError If no matching request arrives before the timeout. + */ + public Request waitForRequest(Predicate predicate, double timeoutSeconds) { + return waitForFirst(this::requests, predicate, timeoutSeconds, "event platform proxy request"); + } + + /** A request the tracer sent through the event platform proxy. */ + public static final class Request { + private final String path; + private final Headers headers; + private final byte[] body; + + Request(String path, Headers headers, byte[] body) { + this.path = path; + this.headers = headers; + this.body = body; + } + + /** + * Returns the intake path, after the proxy prefix. + * + * @return The intake path (e.g. {@code api/v2/exposures}). + */ + public String path() { + return this.path; + } + + /** + * Returns a request header. + * + * @param name The header name, as sent by the tracer. + * @return The header value, or {@code null} if the request has no such header. + */ + public String header(String name) { + return this.headers.get(name); + } + + /** + * Returns the request body. + * + * @return The request body, as UTF-8 text. + */ + public String body() { + return new String(this.body, UTF_8); + } + + /** + * Decodes the request body as a JSON object. + * + * @return The decoded body. + */ + public Map json() { + return AgentBackendMessages.decodeMessage(this.body); + } + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java index 72b415ad789..38da94e0059 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java @@ -15,10 +15,11 @@ * In-process mock-agent {@link AgentBackend} wrapping the testing {@link JavaTestHttpServer}. It * answers the tracer's {@code /info} probe, decodes the trace payloads the tracer submits (v0.4 / * v0.5 / v1.0 msgpack) via the shared {@link Decoder}, serves Remote Configuration (see {@link - * #remoteConfig()}), and 200s everything else so the app's other agent calls don't error out. + * #remoteConfig()}), captures the event platform proxy requests (see {@link #evpProxy()}), and 200s + * everything else so the app's other agent calls don't error out. * - *

Remaining backend-specific capture surfaces (EVP-proxy / DSM) would hang off this concrete - * type rather than the common {@link AgentBackend} facade. + *

Backend-specific capture surfaces (event platform proxy, DSM) hang off this concrete type + * rather than the common {@link AgentBackend} facade. */ public final class MockAgentBackend extends AgentBackend { /** @@ -27,10 +28,16 @@ public final class MockAgentBackend extends AgentBackend { */ private static final String INFO_BODY = "{\"version\":\"7.77.0\"," - + "\"endpoints\":[\"/v0.4/traces\",\"/v0.5/traces\",\"/v1.0/traces\",\"/telemetry/proxy/\"]," + + "\"endpoints\":[\"/v0.4/traces\",\"/v0.5/traces\",\"/v1.0/traces\",\"/telemetry/proxy/\"," + + "\"/evp_proxy/v2/\"]," + "\"client_drop_p0s\":true," + "\"span_meta_structs\":true," - + "\"long_running_spans\":true}"; + + "\"long_running_spans\":true," + + "\"evp_proxy_allowed_headers\":[\"Content-Type\",\"Accept-Encoding\",\"Content-Encoding\"," + + "\"User-Agent\",\"DD-CI-PROVIDER-NAME\",\"DD-EVP-ORIGIN\",\"DD-EVP-ORIGIN-VERSION\"]}"; + + /** The event platform proxy path prefix. */ + private static final String EVP_PROXY_PREFIX = "/evp_proxy/v2/"; /** JSON mime type. */ private static final String JSON = "application/json"; @@ -41,6 +48,7 @@ public final class MockAgentBackend extends AgentBackend { private final List traces = new CopyOnWriteArrayList<>(); private final List> telemetry = new CopyOnWriteArrayList<>(); private final List> remoteConfigPolls = new CopyOnWriteArrayList<>(); + private final List evpProxyRequests = new CopyOnWriteArrayList<>(); private volatile String remoteConfigResponse = NO_REMOTE_CONFIG; private volatile JavaTestHttpServer server; // Decoding runs on the server thread, where JavaTestHttpServer swallows the failure into a 500. @@ -69,6 +77,7 @@ private void serverSpecs(JavaTestHttpServer server) { h.prefix("/v0.4/traces", api -> collectTraces(api, TraceFormat.V04)); h.prefix("/v0.7/config", this::serveRemoteConfig); h.prefix("/telemetry/proxy", this::collectTelemetry); + h.prefix(EVP_PROXY_PREFIX, this::collectEvpProxyRequest); // Everything else just succeeds. h.all(api -> api.getResponse().status(200).send()); }); @@ -143,6 +152,16 @@ private List> collectedTelemetry() { return new ArrayList<>(this.telemetry); } + private void collectEvpProxyRequest(HandlerApi api) { + HandlerApi.RequestApi request = api.getRequest(); + this.evpProxyRequests.add( + new EvpProxy.Request( + request.getPath().substring(EVP_PROXY_PREFIX.length()), + request.getHeaders(), + request.getBody())); + api.getResponse().status(200).send(); + } + @Override public int port() { return url().getPort(); @@ -172,11 +191,21 @@ public RemoteConfig remoteConfig() { return new RemoteConfig(this::pushRemoteConfig, this::collectedRemoteConfigPolls); } + /** + * Returns the query facade over the requests the tracer sent through the event platform proxy. + * + * @return The {@link EvpProxy} facade for this backend. + */ + public EvpProxy evpProxy() { + return new EvpProxy(() -> new ArrayList<>(this.evpProxyRequests)); + } + @Override public void clear() { this.traces.clear(); this.telemetry.clear(); this.remoteConfigPolls.clear(); + this.evpProxyRequests.clear(); this.remoteConfigResponse = NO_REMOTE_CONFIG; this.traceFailure = null; this.telemetryFailure = null; diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Polling.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Polling.java new file mode 100644 index 00000000000..c6dc0cbc817 --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Polling.java @@ -0,0 +1,43 @@ +package datadog.smoketest.backend; + +import datadog.trace.test.util.PollingConditions; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; +import java.util.function.Supplier; + +/** Waits for the messages captured by an {@link AgentBackend}. */ +final class Polling { + private Polling() {} + + /** + * Waits for a captured message satisfying the given predicate, and returns it. + * + * @param messages The captured messages. + * @param predicate The predicate a message must satisfy. + * @param timeoutSeconds How long to wait, in seconds. + * @param description The kind of message, for the failure message. + * @param The message type. + * @return The first matching message. + * @throws AssertionError If no matching message arrives before the timeout. + */ + static T waitForFirst( + Supplier> messages, + Predicate predicate, + double timeoutSeconds, + String description) { + AtomicReference match = new AtomicReference<>(); + new PollingConditions(timeoutSeconds) + .eventually( + () -> { + for (T message : messages.get()) { + if (predicate.test(message)) { + match.set(message); + return; + } + } + throw new AssertionError("No " + description + " matched yet"); + }); + return match.get(); + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/RemoteConfig.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/RemoteConfig.java index 55a54fb516b..f7888518615 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/RemoteConfig.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/RemoteConfig.java @@ -1,11 +1,11 @@ package datadog.smoketest.backend; -import datadog.trace.test.util.PollingConditions; +import static datadog.smoketest.backend.Polling.waitForFirst; + import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Predicate; import java.util.function.Supplier; @@ -69,19 +69,7 @@ public Map waitForRequest(Predicate> predica */ public Map waitForRequest( Predicate> predicate, double timeoutSeconds) { - AtomicReference> match = new AtomicReference<>(); - new PollingConditions(timeoutSeconds) - .eventually( - () -> { - for (Map request : requests()) { - if (predicate.test(request)) { - match.set(request); - return; - } - } - throw new AssertionError("No remote-config poll request matched yet"); - }); - return match.get(); + return waitForFirst(this::requests, predicate, timeoutSeconds, "remote-config poll request"); } /** diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeServerAppLivenessTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeServerAppLivenessTest.java index 19e4bd2bae9..b5c9517937a 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeServerAppLivenessTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeServerAppLivenessTest.java @@ -1,5 +1,8 @@ package datadog.smoketest; +import static datadog.environment.JavaVirtualMachine.isJavaVersionAtLeast; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -9,8 +12,9 @@ /** * Verifies a {@link SmokeServerApp} whose server dies while a test runs fails that test, including * during the last (or only) method, after which no later callback re-checks liveness and teardown - * accepts the already-exited process. The app lifecycle is driven by hand rather than through - * {@code @RegisterExtension}, since the expected failure would otherwise fail this very test. + * accepts the already-exited process, unless the test terminated the server itself. The app + * lifecycle is driven by hand rather than through {@code @RegisterExtension}, since the expected + * failure would otherwise fail this very test. */ class SmokeServerAppLivenessTest { @@ -37,4 +41,34 @@ void failsWhenTheServerDiesDuringATest() throws Exception { app.afterAll(null); } } + + @Test + void acceptsATestTerminatingTheServer() throws Exception { + SmokeServerApp app = + SmokeServerApp.named("terminated-server") + .mainClass("datadog.smoketest.TestServerApp") + .args("--server.port=${app.httpPort}") + .backend(AgentBackend.mockAgent()) + .noAgent() + .build(); + + app.beforeAll(null); + try { + app.beforeEach(null); + app.terminate(30, SECONDS); + + assertFalse(app.process().isAlive(), "the server exited"); + if (isJavaVersionAtLeast(9)) { + assertTrue( + app.waitForLogLine(line -> line.contains("TestServerApp shutting down")), + "the server shutdown hooks ran, and their logs are captured"); + } + app.afterEach(null); + IllegalStateException failure = + assertThrows(IllegalStateException.class, () -> app.beforeEach(null)); + assertTrue(failure.getMessage().contains("terminated"), failure.getMessage()); + } finally { + app.afterAll(null); + } + } } diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java b/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java index 5d58d280d09..8da8045d268 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java @@ -45,6 +45,8 @@ public static void main(String[] args) throws Exception { os.write(body); } }); + Runtime.getRuntime() + .addShutdownHook(new Thread(() -> System.out.println("TestServerApp shutting down"))); server.start(); System.out.println("TestServerApp listening on " + port); diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java index 0b9d06e9c33..b3ef42ebeae 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java @@ -181,9 +181,35 @@ void infoAdvertisesTraceEndpointsAndCapabilities() throws IOException { String body = response.body().string(); assertTrue(body.contains("/v0.4/traces"), body); assertTrue(body.contains("\"client_drop_p0s\":true"), body); + assertTrue(body.contains("/evp_proxy/v2/"), body); + assertTrue(body.contains("\"DD-EVP-ORIGIN\""), body); } } + @Test + void capturesEventPlatformProxyRequests() throws IOException { + Request request = + new Request.Builder() + .url(backend.url() + "/evp_proxy/v2/api/v2/exposures") + .header("DD-EVP-ORIGIN", "dd-trace-java") + .post(RequestBody.create(JSON, "{\"exposures\":[{\"flag\":{\"key\":\"a\"}}]}")) + .build(); + try (Response response = CLIENT.newCall(request).execute()) { + assertEquals(200, response.code(), "mock agent should accept proxied events"); + } + + EvpProxy evpProxy = backend.evpProxy(); + EvpProxy.Request captured = evpProxy.waitForRequest(r -> "api/v2/exposures".equals(r.path())); + assertEquals("dd-trace-java", captured.header("DD-EVP-ORIGIN")); + assertTrue(captured.body().contains("exposures"), captured.body()); + assertTrue(captured.json().get("exposures") instanceof List, "body decoded as JSON"); + assertEquals(1, evpProxy.requests("api/v2/exposures").size()); + assertTrue(evpProxy.requests("api/v2/flagevaluation").isEmpty(), "filtered by path"); + + backend.clear(); + assertTrue(backend.evpProxy().requests().isEmpty(), "clear() drops proxied events too"); + } + @Test void waitForTraceCountTimesOutWhenTooFew() { // Nothing submitted, so polling for a trace with a short timeout must fail rather than hang. diff --git a/docs/how_to_smoke_test.md b/docs/how_to_smoke_test.md index ffd72574ab1..79bd648975a 100644 --- a/docs/how_to_smoke_test.md +++ b/docs/how_to_smoke_test.md @@ -215,6 +215,8 @@ For a long-running server: For other verbs, headers or bodies, point your own HTTP client at `app.url()`. - The owned backend and the captured logs are cleared before each test method, so a method sees only the traces and output its own requests produced. - The process is asserted alive before and after each method, so a crash fails the method that caused it instead of surfacing later as an unexplained timeout. +- `terminate(timeout, unit)` stops the app with a termination signal, which runs its shutdown hooks, and waits for it to exit. + Use it to assert what the app flushes at exit; later test methods can no longer use the app. ```java @RegisterExtension @@ -511,6 +513,18 @@ Turn it off with `skipTelemetryCheck()` on the app builder, for apps that run wi | `RemoteConfig.products(request)` (static) | Products a poll subscribes to | | `RemoteConfig.capabilities(request)` (static) | Capability bitmask a poll advertises | +## Asserting event platform proxy requests + +The in-process mock agent also captures what the tracer sends through the event platform proxy (`/evp_proxy/v2/`), which the test agent does not record for every intake. +`AgentBackend.mockAgent().evpProxy()` returns the query facade: + +| Method | Purpose | +|-------------------------------------------------------------|-------------------------------------------------------------------------| +| `requests()` / `requests(path)` | Captured requests, all or to an intake path such as `api/v2/exposures` | +| `waitForRequest(predicate)` / `(predicate, timeoutSeconds)` | Block until a request matches, and return it | + +Each request exposes its intake `path()`, a `header(name)`, its `body()`, and the body decoded with `json()`. + ## Asserting application logs | Method | Purpose |