From 361c9cdd0e9b46421f2a5fe3e4d2b6b69f2c378c Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Wed, 28 Oct 2026 13:23:05 +0100 Subject: [PATCH 1/2] test(smoke): Capture event platform proxy requests and terminate server apps The mock agent captures the requests sent through the event platform proxy and advertises it like the Groovy smoke base, and SmokeServerApp.terminate() stops the app with a termination signal so tests can assert what it does at exit. Co-Authored-By: Claude Opus 5.5 --- .../datadog/smoketest/SmokeServerApp.java | 49 ++++++- .../smoketest/backend/AgentBackend.java | 2 +- .../datadog/smoketest/backend/EvpProxy.java | 124 ++++++++++++++++++ .../smoketest/backend/MockAgentBackend.java | 39 +++++- .../datadog/smoketest/backend/Polling.java | 43 ++++++ .../smoketest/backend/RemoteConfig.java | 18 +-- .../smoketest/SmokeServerAppLivenessTest.java | 38 +++++- .../java/datadog/smoketest/TestServerApp.java | 2 + .../backend/MockAgentBackendTest.java | 26 ++++ docs/how_to_smoke_test.md | 14 ++ 10 files changed, 331 insertions(+), 24 deletions(-) create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/backend/EvpProxy.java create mode 100644 dd-smoke-tests/src/main/java/datadog/smoketest/backend/Polling.java diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeServerApp.java b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeServerApp.java index def28a90848..4f4a6989f0e 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeServerApp.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/SmokeServerApp.java @@ -8,6 +8,7 @@ import datadog.trace.api.internal.VisibleForTesting; import java.io.IOException; import java.net.URI; +import java.util.concurrent.TimeUnit; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; @@ -35,6 +36,7 @@ public final class SmokeServerApp extends AbstractSmokeApp { private final int httpPort; private final OkHttpClient httpClient = new OkHttpClient(); + private boolean terminated; private SmokeServerApp(Builder builder) { super(builder); @@ -89,6 +91,46 @@ int get(String path) { } } + /** + * Terminates the app as a platform stops it, with a termination signal that runs its shutdown + * hooks, and waits for it to exit. Use it to assert what the app does at exit: later test methods + * can't use the app anymore. The logs the app writes while shutting down are only captured on + * Java 9 and later. + * + * @param timeout The maximum time to wait for the app to exit. + * @param unit The time unit of {@code timeout}. + * @throws AssertionError If the app does not exit within the timeout. + */ + public void terminate(long timeout, TimeUnit unit) { + Process process = process(); + // Complete the test scope diagnostics while the app can still report them. + verifyScopeDiagnostics(); + this.terminated = true; + signalTermination(process); + boolean exited; + try { + exited = process.waitFor(timeout, unit); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("Interrupted while waiting for app '" + name() + "' to exit", e); + } + if (!exited) { + throw new AssertionError( + "App '" + name() + "' did not exit within " + timeout + " " + unit + " once terminated"); + } + } + + private static void signalTermination(Process process) { + // Process.destroy() also closes the app output streams, losing what it logs while shutting + // down, unlike ProcessHandle.destroy() (Java 9+). + try { + Object handle = Process.class.getMethod("toHandle").invoke(process); + Class.forName("java.lang.ProcessHandle").getMethod("destroy").invoke(handle); + } catch (ReflectiveOperationException e) { + process.destroy(); + } + } + @Override protected void onStarted() { waitForPortToOpen(this.httpPort, startupTimeoutSeconds(), SECONDS, process()); @@ -109,7 +151,9 @@ protected void onBeforeEach() { @Override protected void onAfterEach() { - assertAlive("at the end of a test"); + if (!this.terminated) { + assertAlive("at the end of a test"); + } } private void assertAlive(String when) { @@ -117,6 +161,9 @@ private void assertAlive(String when) { if (process == null) { throw new IllegalStateException("App '" + name() + "' was never launched"); } + if (this.terminated) { + throw new IllegalStateException("App '" + name() + "' was terminated by a previous test"); + } if (!process.isAlive()) { throw new IllegalStateException( "App '" + name() + "' exited with value " + process.exitValue() + " " + when); diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/AgentBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/AgentBackend.java index b8172794eb1..75ec0edc17a 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/AgentBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/AgentBackend.java @@ -148,7 +148,7 @@ public void afterAll(ExtensionContext context) { * @deprecated Prefer {@link #testAgent()}, which speaks the real agent protocol. */ @Deprecated - public static AgentBackend mockAgent() { + public static MockAgentBackend mockAgent() { return new MockAgentBackend(); } diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/EvpProxy.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/EvpProxy.java new file mode 100644 index 00000000000..77206af597b --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/EvpProxy.java @@ -0,0 +1,124 @@ +package datadog.smoketest.backend; + +import static datadog.smoketest.backend.Polling.waitForFirst; +import static java.nio.charset.StandardCharsets.UTF_8; + +import datadog.trace.agent.test.server.http.JavaTestHttpServer.Headers; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.function.Supplier; + +/** + * This class is a query facade over the requests the tracer sent through the event platform proxy + * ({@code /evp_proxy/v2/}) of an {@link AgentBackend}. + */ +public final class EvpProxy { + private static final double DEFAULT_TIMEOUT_SECONDS = 30; + + private final Supplier> source; + + EvpProxy(Supplier> source) { + this.source = source; + } + + /** + * Returns the captured requests, in arrival order. + * + * @return The captured requests. + */ + public List requests() { + return this.source.get(); + } + + /** + * Returns the captured requests to an intake path, in arrival order. + * + * @param path The intake path, after the proxy prefix (e.g. {@code api/v2/exposures}). + * @return The captured requests to this path. + */ + public List requests(String path) { + List matching = new ArrayList<>(); + for (Request request : requests()) { + if (path.equals(request.path())) { + matching.add(request); + } + } + return matching; + } + + /** + * Waits up to {@value #DEFAULT_TIMEOUT_SECONDS}s for a captured request satisfying the given + * predicate, and returns it. + * + * @param predicate The predicate a request must satisfy. + * @return The first matching request. + * @throws AssertionError If no matching request arrives before the timeout. + */ + public Request waitForRequest(Predicate predicate) { + return waitForRequest(predicate, DEFAULT_TIMEOUT_SECONDS); + } + + /** + * Waits for a captured request satisfying the given predicate, and returns it. + * + * @param predicate The predicate a request must satisfy. + * @param timeoutSeconds How long to wait, in seconds. + * @return The first matching request. + * @throws AssertionError If no matching request arrives before the timeout. + */ + public Request waitForRequest(Predicate predicate, double timeoutSeconds) { + return waitForFirst(this::requests, predicate, timeoutSeconds, "event platform proxy request"); + } + + /** A request the tracer sent through the event platform proxy. */ + public static final class Request { + private final String path; + private final Headers headers; + private final byte[] body; + + Request(String path, Headers headers, byte[] body) { + this.path = path; + this.headers = headers; + this.body = body; + } + + /** + * Returns the intake path, after the proxy prefix. + * + * @return The intake path (e.g. {@code api/v2/exposures}). + */ + public String path() { + return this.path; + } + + /** + * Returns a request header. + * + * @param name The header name, as sent by the tracer. + * @return The header value, or {@code null} if the request has no such header. + */ + public String header(String name) { + return this.headers.get(name); + } + + /** + * Returns the request body. + * + * @return The request body, as UTF-8 text. + */ + public String body() { + return new String(this.body, UTF_8); + } + + /** + * Decodes the request body as a JSON object. + * + * @return The decoded body. + */ + public Map json() { + return AgentBackendMessages.decodeMessage(this.body); + } + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java index 72b415ad789..38da94e0059 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/MockAgentBackend.java @@ -15,10 +15,11 @@ * In-process mock-agent {@link AgentBackend} wrapping the testing {@link JavaTestHttpServer}. It * answers the tracer's {@code /info} probe, decodes the trace payloads the tracer submits (v0.4 / * v0.5 / v1.0 msgpack) via the shared {@link Decoder}, serves Remote Configuration (see {@link - * #remoteConfig()}), and 200s everything else so the app's other agent calls don't error out. + * #remoteConfig()}), captures the event platform proxy requests (see {@link #evpProxy()}), and 200s + * everything else so the app's other agent calls don't error out. * - *

Remaining backend-specific capture surfaces (EVP-proxy / DSM) would hang off this concrete - * type rather than the common {@link AgentBackend} facade. + *

Backend-specific capture surfaces (event platform proxy, DSM) hang off this concrete type + * rather than the common {@link AgentBackend} facade. */ public final class MockAgentBackend extends AgentBackend { /** @@ -27,10 +28,16 @@ public final class MockAgentBackend extends AgentBackend { */ private static final String INFO_BODY = "{\"version\":\"7.77.0\"," - + "\"endpoints\":[\"/v0.4/traces\",\"/v0.5/traces\",\"/v1.0/traces\",\"/telemetry/proxy/\"]," + + "\"endpoints\":[\"/v0.4/traces\",\"/v0.5/traces\",\"/v1.0/traces\",\"/telemetry/proxy/\"," + + "\"/evp_proxy/v2/\"]," + "\"client_drop_p0s\":true," + "\"span_meta_structs\":true," - + "\"long_running_spans\":true}"; + + "\"long_running_spans\":true," + + "\"evp_proxy_allowed_headers\":[\"Content-Type\",\"Accept-Encoding\",\"Content-Encoding\"," + + "\"User-Agent\",\"DD-CI-PROVIDER-NAME\",\"DD-EVP-ORIGIN\",\"DD-EVP-ORIGIN-VERSION\"]}"; + + /** The event platform proxy path prefix. */ + private static final String EVP_PROXY_PREFIX = "/evp_proxy/v2/"; /** JSON mime type. */ private static final String JSON = "application/json"; @@ -41,6 +48,7 @@ public final class MockAgentBackend extends AgentBackend { private final List traces = new CopyOnWriteArrayList<>(); private final List> telemetry = new CopyOnWriteArrayList<>(); private final List> remoteConfigPolls = new CopyOnWriteArrayList<>(); + private final List evpProxyRequests = new CopyOnWriteArrayList<>(); private volatile String remoteConfigResponse = NO_REMOTE_CONFIG; private volatile JavaTestHttpServer server; // Decoding runs on the server thread, where JavaTestHttpServer swallows the failure into a 500. @@ -69,6 +77,7 @@ private void serverSpecs(JavaTestHttpServer server) { h.prefix("/v0.4/traces", api -> collectTraces(api, TraceFormat.V04)); h.prefix("/v0.7/config", this::serveRemoteConfig); h.prefix("/telemetry/proxy", this::collectTelemetry); + h.prefix(EVP_PROXY_PREFIX, this::collectEvpProxyRequest); // Everything else just succeeds. h.all(api -> api.getResponse().status(200).send()); }); @@ -143,6 +152,16 @@ private List> collectedTelemetry() { return new ArrayList<>(this.telemetry); } + private void collectEvpProxyRequest(HandlerApi api) { + HandlerApi.RequestApi request = api.getRequest(); + this.evpProxyRequests.add( + new EvpProxy.Request( + request.getPath().substring(EVP_PROXY_PREFIX.length()), + request.getHeaders(), + request.getBody())); + api.getResponse().status(200).send(); + } + @Override public int port() { return url().getPort(); @@ -172,11 +191,21 @@ public RemoteConfig remoteConfig() { return new RemoteConfig(this::pushRemoteConfig, this::collectedRemoteConfigPolls); } + /** + * Returns the query facade over the requests the tracer sent through the event platform proxy. + * + * @return The {@link EvpProxy} facade for this backend. + */ + public EvpProxy evpProxy() { + return new EvpProxy(() -> new ArrayList<>(this.evpProxyRequests)); + } + @Override public void clear() { this.traces.clear(); this.telemetry.clear(); this.remoteConfigPolls.clear(); + this.evpProxyRequests.clear(); this.remoteConfigResponse = NO_REMOTE_CONFIG; this.traceFailure = null; this.telemetryFailure = null; diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Polling.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Polling.java new file mode 100644 index 00000000000..c6dc0cbc817 --- /dev/null +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/Polling.java @@ -0,0 +1,43 @@ +package datadog.smoketest.backend; + +import datadog.trace.test.util.PollingConditions; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; +import java.util.function.Supplier; + +/** Waits for the messages captured by an {@link AgentBackend}. */ +final class Polling { + private Polling() {} + + /** + * Waits for a captured message satisfying the given predicate, and returns it. + * + * @param messages The captured messages. + * @param predicate The predicate a message must satisfy. + * @param timeoutSeconds How long to wait, in seconds. + * @param description The kind of message, for the failure message. + * @param The message type. + * @return The first matching message. + * @throws AssertionError If no matching message arrives before the timeout. + */ + static T waitForFirst( + Supplier> messages, + Predicate predicate, + double timeoutSeconds, + String description) { + AtomicReference match = new AtomicReference<>(); + new PollingConditions(timeoutSeconds) + .eventually( + () -> { + for (T message : messages.get()) { + if (predicate.test(message)) { + match.set(message); + return; + } + } + throw new AssertionError("No " + description + " matched yet"); + }); + return match.get(); + } +} diff --git a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/RemoteConfig.java b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/RemoteConfig.java index 55a54fb516b..f7888518615 100644 --- a/dd-smoke-tests/src/main/java/datadog/smoketest/backend/RemoteConfig.java +++ b/dd-smoke-tests/src/main/java/datadog/smoketest/backend/RemoteConfig.java @@ -1,11 +1,11 @@ package datadog.smoketest.backend; -import datadog.trace.test.util.PollingConditions; +import static datadog.smoketest.backend.Polling.waitForFirst; + import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Predicate; import java.util.function.Supplier; @@ -69,19 +69,7 @@ public Map waitForRequest(Predicate> predica */ public Map waitForRequest( Predicate> predicate, double timeoutSeconds) { - AtomicReference> match = new AtomicReference<>(); - new PollingConditions(timeoutSeconds) - .eventually( - () -> { - for (Map request : requests()) { - if (predicate.test(request)) { - match.set(request); - return; - } - } - throw new AssertionError("No remote-config poll request matched yet"); - }); - return match.get(); + return waitForFirst(this::requests, predicate, timeoutSeconds, "remote-config poll request"); } /** diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeServerAppLivenessTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeServerAppLivenessTest.java index 19e4bd2bae9..b5c9517937a 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeServerAppLivenessTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/SmokeServerAppLivenessTest.java @@ -1,5 +1,8 @@ package datadog.smoketest; +import static datadog.environment.JavaVirtualMachine.isJavaVersionAtLeast; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -9,8 +12,9 @@ /** * Verifies a {@link SmokeServerApp} whose server dies while a test runs fails that test, including * during the last (or only) method, after which no later callback re-checks liveness and teardown - * accepts the already-exited process. The app lifecycle is driven by hand rather than through - * {@code @RegisterExtension}, since the expected failure would otherwise fail this very test. + * accepts the already-exited process, unless the test terminated the server itself. The app + * lifecycle is driven by hand rather than through {@code @RegisterExtension}, since the expected + * failure would otherwise fail this very test. */ class SmokeServerAppLivenessTest { @@ -37,4 +41,34 @@ void failsWhenTheServerDiesDuringATest() throws Exception { app.afterAll(null); } } + + @Test + void acceptsATestTerminatingTheServer() throws Exception { + SmokeServerApp app = + SmokeServerApp.named("terminated-server") + .mainClass("datadog.smoketest.TestServerApp") + .args("--server.port=${app.httpPort}") + .backend(AgentBackend.mockAgent()) + .noAgent() + .build(); + + app.beforeAll(null); + try { + app.beforeEach(null); + app.terminate(30, SECONDS); + + assertFalse(app.process().isAlive(), "the server exited"); + if (isJavaVersionAtLeast(9)) { + assertTrue( + app.waitForLogLine(line -> line.contains("TestServerApp shutting down")), + "the server shutdown hooks ran, and their logs are captured"); + } + app.afterEach(null); + IllegalStateException failure = + assertThrows(IllegalStateException.class, () -> app.beforeEach(null)); + assertTrue(failure.getMessage().contains("terminated"), failure.getMessage()); + } finally { + app.afterAll(null); + } + } } diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java b/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java index 5d58d280d09..8da8045d268 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/TestServerApp.java @@ -45,6 +45,8 @@ public static void main(String[] args) throws Exception { os.write(body); } }); + Runtime.getRuntime() + .addShutdownHook(new Thread(() -> System.out.println("TestServerApp shutting down"))); server.start(); System.out.println("TestServerApp listening on " + port); diff --git a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java index 0b9d06e9c33..b3ef42ebeae 100644 --- a/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java +++ b/dd-smoke-tests/src/test/java/datadog/smoketest/backend/MockAgentBackendTest.java @@ -181,9 +181,35 @@ void infoAdvertisesTraceEndpointsAndCapabilities() throws IOException { String body = response.body().string(); assertTrue(body.contains("/v0.4/traces"), body); assertTrue(body.contains("\"client_drop_p0s\":true"), body); + assertTrue(body.contains("/evp_proxy/v2/"), body); + assertTrue(body.contains("\"DD-EVP-ORIGIN\""), body); } } + @Test + void capturesEventPlatformProxyRequests() throws IOException { + Request request = + new Request.Builder() + .url(backend.url() + "/evp_proxy/v2/api/v2/exposures") + .header("DD-EVP-ORIGIN", "dd-trace-java") + .post(RequestBody.create(JSON, "{\"exposures\":[{\"flag\":{\"key\":\"a\"}}]}")) + .build(); + try (Response response = CLIENT.newCall(request).execute()) { + assertEquals(200, response.code(), "mock agent should accept proxied events"); + } + + EvpProxy evpProxy = backend.evpProxy(); + EvpProxy.Request captured = evpProxy.waitForRequest(r -> "api/v2/exposures".equals(r.path())); + assertEquals("dd-trace-java", captured.header("DD-EVP-ORIGIN")); + assertTrue(captured.body().contains("exposures"), captured.body()); + assertTrue(captured.json().get("exposures") instanceof List, "body decoded as JSON"); + assertEquals(1, evpProxy.requests("api/v2/exposures").size()); + assertTrue(evpProxy.requests("api/v2/flagevaluation").isEmpty(), "filtered by path"); + + backend.clear(); + assertTrue(backend.evpProxy().requests().isEmpty(), "clear() drops proxied events too"); + } + @Test void waitForTraceCountTimesOutWhenTooFew() { // Nothing submitted, so polling for a trace with a short timeout must fail rather than hang. diff --git a/docs/how_to_smoke_test.md b/docs/how_to_smoke_test.md index ffd72574ab1..79bd648975a 100644 --- a/docs/how_to_smoke_test.md +++ b/docs/how_to_smoke_test.md @@ -215,6 +215,8 @@ For a long-running server: For other verbs, headers or bodies, point your own HTTP client at `app.url()`. - The owned backend and the captured logs are cleared before each test method, so a method sees only the traces and output its own requests produced. - The process is asserted alive before and after each method, so a crash fails the method that caused it instead of surfacing later as an unexplained timeout. +- `terminate(timeout, unit)` stops the app with a termination signal, which runs its shutdown hooks, and waits for it to exit. + Use it to assert what the app flushes at exit; later test methods can no longer use the app. ```java @RegisterExtension @@ -511,6 +513,18 @@ Turn it off with `skipTelemetryCheck()` on the app builder, for apps that run wi | `RemoteConfig.products(request)` (static) | Products a poll subscribes to | | `RemoteConfig.capabilities(request)` (static) | Capability bitmask a poll advertises | +## Asserting event platform proxy requests + +The in-process mock agent also captures what the tracer sends through the event platform proxy (`/evp_proxy/v2/`), which the test agent does not record for every intake. +`AgentBackend.mockAgent().evpProxy()` returns the query facade: + +| Method | Purpose | +|-------------------------------------------------------------|-------------------------------------------------------------------------| +| `requests()` / `requests(path)` | Captured requests, all or to an intake path such as `api/v2/exposures` | +| `waitForRequest(predicate)` / `(predicate, timeoutSeconds)` | Block until a request matches, and return it | + +Each request exposes its intake `path()`, a `header(name)`, its `body()`, and the body decoded with `json()`. + ## Asserting application logs | Method | Purpose | From cdaa42ea47869a8c5da58c1c128ceea13ce213ef Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Wed, 28 Oct 2026 13:23:13 +0100 Subject: [PATCH 2/2] test(ffe): Port the OpenFeature smoke tests to JUnit The tests run on the Java smoke test framework, with the in-process mock agent: the test agent neither proxies the flag evaluation events nor advertises the event platform proxy headers. Co-Authored-By: Claude Opus 5.5 --- ...bstractOpenFeatureShutdownSmokeTest.groovy | 135 ---------- .../AbstractOpenFeatureSmokeTest.groovy | 213 ---------------- .../OpenFeatureAgentlessSmokeTest.groovy | 24 -- ...atureMalformedRemoteConfigSmokeTest.groovy | 95 ------- .../OpenFeatureProviderSmokeTest.groovy | 101 -------- .../OpenFeatureShutdownSmokeTest.groovy | 10 - .../OpenFeatureStandaloneSmokeTest.groovy | 40 --- ...atureTraceDisabledShutdownSmokeTest.groovy | 13 - .../AbstractOpenFeatureShutdownSmokeTest.java | 81 ++++++ .../AbstractOpenFeatureSmokeTest.java | 213 ++++++++++++++++ .../datadog/smoketest/springboot/Json.java | 159 ++++++++++++ .../OpenFeatureAgentlessSmokeTest.java | 25 ++ .../springboot/OpenFeatureFixtures.java | 235 ++++++++++++++++++ ...FeatureMalformedRemoteConfigSmokeTest.java | 93 +++++++ .../OpenFeatureProviderSmokeTest.java | 112 +++++++++ .../OpenFeatureShutdownSmokeTest.java | 14 ++ .../OpenFeatureStandaloneSmokeTest.java | 30 +++ ...FeatureTraceDisabledShutdownSmokeTest.java | 18 ++ .../smoketest/AbstractSmokeTest.groovy | 29 --- 19 files changed, 980 insertions(+), 660 deletions(-) delete mode 100644 dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.groovy delete mode 100644 dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.groovy delete mode 100644 dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.groovy delete mode 100644 dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.groovy delete mode 100644 dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy delete mode 100644 dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.groovy delete mode 100644 dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.groovy delete mode 100644 dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.groovy create mode 100644 dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.java create mode 100644 dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.java create mode 100644 dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/Json.java create mode 100644 dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.java create mode 100644 dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureFixtures.java create mode 100644 dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.java create mode 100644 dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.java create mode 100644 dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.java create mode 100644 dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.java create mode 100644 dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.java diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.groovy deleted file mode 100644 index 240131577d1..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.groovy +++ /dev/null @@ -1,135 +0,0 @@ -package datadog.smoketest.springboot - -import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer -import static java.util.concurrent.TimeUnit.SECONDS - -import datadog.smoketest.AbstractSmokeTest -import datadog.trace.agent.test.server.http.TestHttpServer -import datadog.trace.agent.test.server.http.TestHttpServer.HandlerApi.RequestApi -import datadog.trace.agent.test.utils.OkHttpUtils -import datadog.trace.agent.test.utils.PortUtils -import groovy.json.JsonOutput -import groovy.json.JsonSlurper -import okhttp3.MediaType -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.RequestBody -import spock.lang.AutoCleanup -import spock.lang.Shared -import spock.util.concurrent.PollingConditions - -/** The SDK with the Datadog Java agent flushes its pending events when the application exits. */ -abstract class AbstractOpenFeatureShutdownSmokeTest extends AbstractSmokeTest { - - @Shared - protected int httpPort = PortUtils.randomOpenPort() - - protected OkHttpClient client = OkHttpUtils.client() - - @Shared - protected final ufcConfig = new JsonSlurper().parse(fetchResource("ffe-system-test-data/ufc-config.json")) as Map - - /** Serves the flag configuration as the Datadog managed CDN does. */ - @Shared - @AutoCleanup - protected TestHttpServer cdn = httpServer { - handlers { - prefix("/api/v2/feature-flagging/config/rules-based/server") { - response.status(200).send(JsonOutput.toJson([ - data: [type: 'universal-flag-configuration', attributes: ufcConfig] - ])) - } - } - } - - /** @return whether the application runs with tracing enabled. */ - abstract boolean tracing() - - def setupSpec() { - PortUtils.waitForPortToOpen(httpPort, 240, SECONDS, testedProcess) - } - - @Override - ProcessBuilder createProcessBuilder() { - final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") - final command = [javaPath()] - command.addAll(defaultJavaProperties) - command.add("-Ddd.trace.enabled=${tracing()}".toString()) - command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) - final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) - builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL', "http://localhost:${cdn.address.port}".toString()) - return builder - } - - @Override - boolean testTelemetry() { - return false - } - - @Override - Closure decodedTracesCallback() { - return {} - } - - @Override - Closure decodedEvpProxyMessageCallback() { - return { String path, RequestApi request -> - if (!path.contains('api/v2/flagevaluation')) { - return null - } - return [ - origin: request.getHeader('DD-EVP-ORIGIN'), - originVersion: request.getHeader('DD-EVP-ORIGIN-VERSION'), - body: new JsonSlurper().parse(request.body) - ] - } - } - - void 'test pending flag evaluations are flushed at application exit'() { - when: - // The first evaluation is flushed within the 10 seconds flush interval, which then restarts: - // the next evaluation stays pending until the application exits. - evaluate('test-case-boolean-false-assignment.json') - new PollingConditions(timeout: 20).eventually { - assert evaluatedFlags().contains('boolean-false-assignment') - } - evaluate('test-case-boolean-one-of-matches.json') - - then: - !evaluatedFlags().contains('boolean-one-of-matches') - - when: - testedProcess.destroy() - - then: - testedProcess.waitFor(30, SECONDS) - evaluatedFlags().contains('boolean-one-of-matches') - evaluationRequests().every { it.origin == 'dd-trace-java' && it.originVersion } - } - - protected void evaluate(final String testCaseFile) { - final testCase = (new JsonSlurper().parse(fetchResource("ffe-system-test-data/evaluation-cases/${testCaseFile}")) as List>).first() - final request = new Request.Builder() - .url("http://localhost:${httpPort}/openfeature/evaluate") - .post(RequestBody.create(MediaType.parse('application/json'), JsonOutput.toJson(testCase))) - .build() - final response = client.newCall(request).execute() - try { - assert response.code() == 200 - } finally { - response.close() - } - } - - protected List evaluatedFlags() { - return evaluationRequests()*.body*.flagEvaluations.flatten()*.flag*.key as List - } - - protected List> evaluationRequests() { - return evpProxyMessages*.getV2() as List> - } - - protected static URL fetchResource(final String name) { - return Thread.currentThread().getContextClassLoader().getResource(name) - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.groovy deleted file mode 100644 index 147c6fa2fde..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.groovy +++ /dev/null @@ -1,213 +0,0 @@ -package datadog.smoketest.springboot - -import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer - -import datadog.smoketest.AbstractServerSmokeTest -import datadog.trace.agent.test.server.http.TestHttpServer -import datadog.trace.agent.test.server.http.TestHttpServer.HandlerApi.RequestApi -import groovy.json.JsonOutput -import groovy.json.JsonSlurper -import java.nio.file.Files -import java.nio.file.Paths -import okhttp3.MediaType -import okhttp3.Request -import okhttp3.RequestBody -import spock.lang.AutoCleanup -import spock.lang.IgnoreIf -import spock.lang.Shared -import spock.lang.Stepwise -import spock.lang.Unroll -import spock.util.concurrent.PollingConditions - -/** Due to the exposure cache it's important to run the tests in the specified order */ -@Stepwise -abstract class AbstractOpenFeatureSmokeTest extends AbstractServerSmokeTest { - - @Shared - protected final ufcConfig = new JsonSlurper().parse(fetchResource("ffe-system-test-data/ufc-config.json")) as Map - - @Shared - protected final ufcPayload = JsonOutput.toJson(ufcConfig) - - @Shared - protected final loggedAllocations = buildLoggedAllocations(ufcConfig) - - /** Serves the flag configuration as the Datadog managed CDN does. */ - @Shared - @AutoCleanup - protected TestHttpServer cdn = httpServer { - handlers { - prefix("/api/v2/feature-flagging/config/rules-based/server") { - response.status(200).send(JsonOutput.toJson([ - data: [type: 'universal-flag-configuration', attributes: ufcConfig] - ])) - } - } - } - - /** @return whether exposures are delivered through the Datadog Agent event platform proxy. */ - abstract boolean deliversExposures() - - /** Publishes the flag configuration before each evaluation. */ - void publishConfiguration() {} - - @Override - Closure decodedTracesCallback() { - return {} - } - - @Override - Closure decodedEvpProxyMessageCallback() { - return { String path, RequestApi request -> - if (!path.contains('api/v2/exposures')) { - return null - } - return new JsonSlurper().parse(request.body) - } - } - - @IgnoreIf({ !instance.deliversExposures() }) - void 'test open feature exposures'() { - setup: - publishConfiguration() - final testCases = parseTestCases() - assert !testCases.isEmpty() - - when: - final results = testCases.collect { - testCase -> - final response = evaluate(testCase) - final responseBody = new JsonSlurper().parse(response.body().byteStream()) - return [testCase: testCase, response: response, body: responseBody] - } - final expectedExposures = uniqueExpectedExposures(results) - - then: - results.every { - it.response.code() == 200 - } - !expectedExposures.isEmpty() - new PollingConditions(timeout: 10).eventually { - final requests = evpProxyMessages*.getV2() as List> - final events = requests*.exposures.flatten() - assert events.size() == expectedExposures.size() - expectedExposures.each { - expected -> - assert events.find { - event -> - event.flag.key == expected.flag && - event.allocation.key == expected.allocation && - event.variant.key == expected.variant && - event.subject.id == expected.targetingKey - } != null : "Unable to find exposure ${expected}" - } - } - } - - @Unroll("test open feature evaluation - #testCase.fileName[#testCase.index] - flag=#testCase.flag") - void 'test open feature evaluation'() { - setup: - publishConfiguration() - - when: - final response = evaluate(testCase) - - then: - response.code() == 200 - final responseBody = new JsonSlurper().parse(response.body().byteStream()) - responseBody.value == testCase.result.value - responseBody.reason == testCase.result.reason - if (testCase.result.containsKey('errorCode')) { - assert responseBody.errorCode == testCase.result.errorCode - } - if (testCase.result.containsKey('variant')) { - assert responseBody.variant == testCase.result.variant - } - if (testCase.result.flagMetadata?.allocationKey) { - assert responseBody.flagMetadata?.allocationKey == testCase.result.flagMetadata?.allocationKey - } - - where: - testCase << parseTestCases() - } - - protected okhttp3.Response evaluate(final Map testCase) { - final request = new Request.Builder() - .url("http://localhost:${httpPort}/openfeature/evaluate") - .post(RequestBody.create(MediaType.parse('application/json'), JsonOutput.toJson(testCase))) - .build() - return client.newCall(request).execute() - } - - protected static URL fetchResource(final String name) { - return Thread.currentThread().getContextClassLoader().getResource(name) - } - - protected static List> parseTestCases() { - final folder = fetchResource('ffe-system-test-data/evaluation-cases') - final uri = folder.toURI() - final testsPath = Paths.get(uri) - final files = Files.list(testsPath) - .filter(path -> path.toString().endsWith('.json')) - .sorted(Comparator.comparing(path -> path.fileName.toString())) - final result = [] - final slurper = new JsonSlurper() - files.each { - path -> - final testCases = slurper.parse(path.toFile()) as List> - testCases.eachWithIndex { - testCase, index -> - testCase.fileName = path.fileName.toString() - testCase.index = index - } - result.addAll(testCases) - } - assert !result.isEmpty() - return result - } - - protected List> uniqueExpectedExposures(final List> results) { - final expected = [] - final seen = [] as Set - results.each { result -> - final testCase = result.testCase as Map - final body = result.body as Map - final flag = testCase.flag as String - final allocation = body.flagMetadata?.allocationKey as String - final variant = body.variant as String - if (!variant || !allocation || !allocationLogs(flag, allocation)) { - return - } - - final exposure = [ - flag: flag, - allocation: allocation, - variant: variant, - targetingKey: testCase.targetingKey - ] - final key = "${exposure.flag}\u0000${exposure.targetingKey}\u0000${exposure.allocation}\u0000${exposure.variant}" - if (seen.add(key)) { - expected.add(exposure) - } - } - return expected - } - - protected boolean allocationLogs(final String flag, final String allocation) { - return loggedAllocations["${flag}\u0000${allocation}"] == true - } - - protected static Map buildLoggedAllocations(final Map config) { - final logged = [:] - (config.flags as Map).each { flag, definition -> - if (definition.allocations instanceof List) { - definition.allocations.each { allocation -> - if (allocation instanceof Map) { - logged["${flag}\u0000${allocation.key}"] = allocation.doLog == true - } - } - } - } - return logged - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.groovy deleted file mode 100644 index bf7932f7ea8..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.groovy +++ /dev/null @@ -1,24 +0,0 @@ -package datadog.smoketest.springboot - -/** - * The SDK with the Datadog Java agent and the default agentless configuration source: the SDK - * polls the flag configuration from the CDN and delivers exposures through the agent. - */ -class OpenFeatureAgentlessSmokeTest extends AbstractOpenFeatureSmokeTest { - - @Override - ProcessBuilder createProcessBuilder() { - final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") - final command = [javaPath()] - command.addAll(defaultJavaProperties) - command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) - final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) - builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL', "http://localhost:${cdn.address.port}".toString()) - return builder - } - - @Override - boolean deliversExposures() { - return true - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.groovy deleted file mode 100644 index 3108fe0bb8a..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.groovy +++ /dev/null @@ -1,95 +0,0 @@ -package datadog.smoketest.springboot - -import datadog.smoketest.AbstractServerSmokeTest -import okhttp3.Request -import okhttp3.RequestBody - -/** - * A malformed flag configuration that Remote Configuration delivers before the SDK subscribes is - * acknowledged by the agent, then reported as not applied once the SDK rejects it. - */ -class OpenFeatureMalformedRemoteConfigSmokeTest extends AbstractServerSmokeTest { - private static final String CONFIG_PATH = 'datadog/2/FFE_FLAGS/1/config' - private static final String MALFORMED_CONFIG = '{"flags":[]}' - private static final int APPLY_STATE_ACKNOWLEDGED = 2 - private static final int APPLY_STATE_ERROR = 3 - - @Override - ProcessBuilder createProcessBuilder() { - setRemoteConfig(CONFIG_PATH, MALFORMED_CONFIG) - - final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") - final command = [javaPath()] - command.addAll(defaultJavaProperties) - command.add('-Ddd.remote_config.enabled=true') - command.add("-Ddd.remote_config.url=http://localhost:${server.address.port}/v0.7/config".toString()) - command.addAll([ - '-jar', - springBootShadowJar, - "--server.port=${httpPort}".toString(), - '--openfeature.lazy-provider=true' - ]) - final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) - builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE', 'remote_config') - return builder - } - - /** The provider never gets a valid configuration, so the OpenFeature SDK logs its failed initialization. */ - @Override - boolean isErrorLog(final String line) { - return super.isErrorLog(line) && !line.contains(' d.o.sdk.') && !line.contains(' dev.openfeature.sdk.') - } - - void 'test malformed configuration received before the SDK is reported as not applied'() { - setup: - // The base resets the Remote Configuration response before each test. - setRemoteConfig(CONFIG_PATH, MALFORMED_CONFIG) - final served = rcClientMessages.size() - - when: 'the agent receives the configuration while no SDK has subscribed' - final acknowledged = waitForRequestAfter(served) { applyState(it) == APPLY_STATE_ACKNOWLEDGED } - final response = client.newCall(new Request.Builder() - .url("http://localhost:${httpPort}/openfeature/provider") - .post(RequestBody.create(null, new byte[0])) - .build()).execute() - - then: 'the SDK rejects it and the agent reports it as not applied' - response.code() == 202 - final rejected = waitForRequestAfter(acknowledged) { applyState(it) == APPLY_STATE_ERROR } - (configState(rejected).apply_error as String).contains('Malformed flag configuration') - - cleanup: - response?.close() - } - - /** - * Waits for an agent Remote Configuration request matching the predicate. - * - * @param from the index of the first request to consider. - * @param predicate the predicate to match. - * @return the index of the first matching request. - */ - private int waitForRequestAfter(final int from, final Closure predicate) { - int index = -1 - defaultPoll.eventually { - index = rcClientMessages.findIndexOf(from, predicate) - assert index >= 0 - } - return index - } - - private Map configState(final int index) { - final request = rcClientMessages[index] - final states = request.client?.state?.config_states as List> - return states?.find { - it.product == 'FFE_FLAGS' - } - } - - private static Integer applyState(final Map request) { - final states = request.client?.state?.config_states as List> - return states?.find { - it.product == 'FFE_FLAGS' - }?.apply_state as Integer - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy deleted file mode 100644 index d0808be6aea..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy +++ /dev/null @@ -1,101 +0,0 @@ -package datadog.smoketest.springboot - -import datadog.remoteconfig.Capabilities -import datadog.remoteconfig.Product -import groovy.json.JsonOutput -import groovy.json.JsonSlurper -import okhttp3.MediaType -import okhttp3.Request -import okhttp3.RequestBody - -/** The SDK with the Datadog Java agent, receiving the flag configuration from Remote Configuration. */ -class OpenFeatureProviderSmokeTest extends AbstractOpenFeatureSmokeTest { - - @Override - ProcessBuilder createProcessBuilder() { - setRemoteConfig("datadog/2/FFE_FLAGS/1/config", ufcPayload) - - final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") - final command = [javaPath()] - command.addAll(defaultJavaProperties) - command.add('-Ddd.trace.debug=true') - command.add('-Ddd.remote_config.enabled=true') - command.add("-Ddd.remote_config.url=http://localhost:${server.address.port}/v0.7/config".toString()) - command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) - final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) - builder.environment().put('DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED', 'true') - builder.environment().put('DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED', 'true') - builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE', 'remote_config') - return builder - } - - @Override - boolean deliversExposures() { - return true - } - - @Override - void publishConfiguration() { - setRemoteConfig("datadog/2/FFE_FLAGS/1/config", ufcPayload) - } - - void 'test first remote config poll asks agent for feature flags'() { - when: - final firstRcRequest = waitForRcClientRequest { req -> - return true - } - - then: - firstRcRequest == rcClientMessages.first() - // An already-running Agent gives a newly-started tracer one new-client cache bypass. If - // FFE_FLAGS is missing here and only appears on a later poll, the Agent can miss the fast path. - decodeProducts(firstRcRequest).find { it == Product.FFE_FLAGS } != null - final capabilities = decodeCapabilities(firstRcRequest) - hasCapability(capabilities, Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES) - } - - void 'test open feature evaluation enriches the request span'() { - setup: - publishConfiguration() - final request = new Request.Builder() - .url("http://localhost:${httpPort}/openfeature/evaluate") - .post(RequestBody.create(MediaType.parse('application/json'), JsonOutput.toJson([ - flag: 'flag-that-does-not-exist', - variationType: 'STRING', - defaultValue: 'fallback', - targetingKey: 'span-enrichment-smoke-test' - ]))) - .build() - - when: - final response = client.newCall(request).execute() - - then: - response.code() == 200 - new JsonSlurper().parse(response.body().byteStream()).value == 'fallback' - waitForSpan(defaultPoll) { span -> - span.parentId == 0 && - span.meta['ffe_runtime_defaults'] == '{"flag-that-does-not-exist":"fallback"}' - } - - cleanup: - response.close() - } - - private static Set decodeProducts(final Map request) { - return request.client.products.collect { Product.valueOf(it) } - } - - private static long decodeCapabilities(final Map request) { - final clientCapabilities = request.client.capabilities as byte[] - long capabilities = 0l - for (int i = 0; i < clientCapabilities.length; i++) { - capabilities |= (clientCapabilities[i] & 0xFFL) << ((clientCapabilities.length - i - 1) * 8) - } - return capabilities - } - - private static boolean hasCapability(final long capabilities, final long test) { - return (capabilities & test) > 0 - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.groovy deleted file mode 100644 index 70192aa9331..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.groovy +++ /dev/null @@ -1,10 +0,0 @@ -package datadog.smoketest.springboot - -/** The tracing shutdown hook flushes the SDK pending events. */ -class OpenFeatureShutdownSmokeTest extends AbstractOpenFeatureShutdownSmokeTest { - - @Override - boolean tracing() { - return true - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.groovy deleted file mode 100644 index 776ae905efa..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.groovy +++ /dev/null @@ -1,40 +0,0 @@ -package datadog.smoketest.springboot - -/** - * The SDK without the Datadog Java agent: the SDK polls the flag configuration from the CDN. Without - * an API key, events are not delivered. - */ -class OpenFeatureStandaloneSmokeTest extends AbstractOpenFeatureSmokeTest { - - @Override - protected String skipScopeContinuationCheckReason() { - 'Exercises the standalone OpenFeature SDK without the agent required by the diagnostic companion' - } - - @Override - ProcessBuilder createProcessBuilder() { - final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") - final command = [javaPath()] - command.addAll(nativeJavaProperties) - command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) - final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) - builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL', "http://localhost:${cdn.address.port}".toString()) - return builder - } - - @Override - boolean deliversExposures() { - return false - } - - @Override - boolean testTelemetry() { - return false - } - - /** Disables direct intake delivery, which would otherwise reach the real Datadog intake. */ - @Override - String apiKey() { - return '' - } -} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.groovy deleted file mode 100644 index c300962d260..00000000000 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.groovy +++ /dev/null @@ -1,13 +0,0 @@ -package datadog.smoketest.springboot - -/** - * Without tracing, the Feature Flagging system registers its own shutdown hook to flush the SDK - * pending events. - */ -class OpenFeatureTraceDisabledShutdownSmokeTest extends AbstractOpenFeatureShutdownSmokeTest { - - @Override - boolean tracing() { - return false - } -} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.java new file mode 100644 index 00000000000..1ccc67e21f7 --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureShutdownSmokeTest.java @@ -0,0 +1,81 @@ +package datadog.smoketest.springboot; + +import static datadog.smoketest.springboot.OpenFeatureFixtures.agentlessApp; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evaluate; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evaluationCase; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evpProxy; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.smoketest.SmokeServerApp; +import datadog.smoketest.backend.EvpProxy; +import datadog.smoketest.springboot.OpenFeatureFixtures.Evaluation; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** The SDK with the Datadog Java agent flushes its pending events when the application exits. */ +abstract class AbstractOpenFeatureShutdownSmokeTest { + private static final String FLAG_EVALUATION_PATH = "api/v2/flagevaluation"; + + static SmokeServerApp shutdownApp(String name, boolean tracing) { + return agentlessApp(name).jvmArgs("-Ddd.trace.enabled=" + tracing).skipTelemetryCheck().build(); + } + + /** + * Gets the application under test. + * + * @return The application under test. + */ + abstract SmokeServerApp app(); + + @Test + void flushesPendingFlagEvaluationsAtExit() throws IOException { + // The first evaluation is flushed within the 10 seconds flush interval, which then restarts: + // the next evaluation stays pending until the application exits. + evaluateCase("test-case-boolean-false-assignment.json"); + evpProxy(app()) + .waitForRequest( + request -> + FLAG_EVALUATION_PATH.equals(request.path()) + && evaluatedFlags(request).contains("boolean-false-assignment"), + 20); + evaluateCase("test-case-boolean-one-of-matches.json"); + assertFalse(evaluatedFlags().contains("boolean-one-of-matches"), "the evaluation is pending"); + + app().terminate(30, SECONDS); + + assertTrue(evaluatedFlags().contains("boolean-one-of-matches"), "the evaluation is flushed"); + for (EvpProxy.Request request : evpProxy(app()).requests(FLAG_EVALUATION_PATH)) { + assertEquals("dd-trace-java", request.header("DD-EVP-ORIGIN"), "origin"); + String originVersion = request.header("DD-EVP-ORIGIN-VERSION"); + assertTrue(originVersion != null && !originVersion.isEmpty(), "origin version"); + } + } + + private void evaluateCase(String evaluationCaseFile) throws IOException { + Evaluation evaluation = evaluate(app(), evaluationCase(evaluationCaseFile).request); + assertEquals(200, evaluation.code, evaluation.body); + } + + private List evaluatedFlags() { + List flags = new ArrayList<>(); + for (EvpProxy.Request request : evpProxy(app()).requests(FLAG_EVALUATION_PATH)) { + flags.addAll(evaluatedFlags(request)); + } + return flags; + } + + private static List evaluatedFlags(EvpProxy.Request request) { + List flags = new ArrayList<>(); + for (Object evaluation : (List) request.json().get("flagEvaluations")) { + Map flag = (Map) ((Map) evaluation).get("flag"); + flags.add((String) flag.get("key")); + } + return flags; + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.java new file mode 100644 index 00000000000..3a2c81cd01f --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.java @@ -0,0 +1,213 @@ +package datadog.smoketest.springboot; + +import static datadog.smoketest.springboot.OpenFeatureFixtures.UFC_CONFIG; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evaluate; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evaluationCases; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evpProxy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import datadog.smoketest.SmokeServerApp; +import datadog.smoketest.backend.EvpProxy; +import datadog.smoketest.springboot.OpenFeatureFixtures.Evaluation; +import datadog.smoketest.springboot.OpenFeatureFixtures.EvaluationCase; +import datadog.trace.test.util.PollingConditions; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Evaluates the system tests cases through the application, and checks the exposures it reports. + * The exposure cache reports each exposure once, so the exposures are checked before any other + * evaluation. + */ +@TestMethodOrder(OrderAnnotation.class) +abstract class AbstractOpenFeatureSmokeTest { + private static final Set LOGGED_ALLOCATIONS = loggedAllocations(); + + /** + * Gets the application under test. + * + * @return The application under test. + */ + abstract SmokeServerApp app(); + + /** + * Tells whether exposures are delivered through the Datadog Agent event platform proxy. + * + * @return Whether exposures are delivered through the Datadog Agent event platform proxy. + */ + abstract boolean deliversExposures(); + + /** Publishes the flag configuration before each evaluation. */ + void publishConfiguration() {} + + @Test + @Order(1) + void reportsExposures() throws IOException { + assumeTrue(deliversExposures(), "exposures are not delivered"); + publishConfiguration(); + Set expected = new LinkedHashSet<>(); + for (EvaluationCase evaluationCase : evaluationCases()) { + Evaluation evaluation = evaluate(app(), evaluationCase.request); + assertEquals(200, evaluation.code, evaluation.body); + Exposure exposure = expectedExposure(evaluationCase, evaluation.json()); + if (exposure != null) { + expected.add(exposure); + } + } + assertFalse(expected.isEmpty(), "some evaluations are exposed"); + + EvpProxy evpProxy = evpProxy(app()); + new PollingConditions(10) + .eventually( + () -> { + List reported = reportedExposures(evpProxy); + assertEquals(expected.size(), reported.size(), "reported exposures: " + reported); + for (Exposure exposure : expected) { + assertTrue(reported.contains(exposure), "Unable to find exposure " + exposure); + } + }); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("datadog.smoketest.springboot.OpenFeatureFixtures#evaluationCases") + @Order(2) + void evaluatesFlags(EvaluationCase evaluationCase) throws IOException { + publishConfiguration(); + + Evaluation evaluation = evaluate(app(), evaluationCase.request); + + assertEquals(200, evaluation.code, evaluation.body); + Map actual = evaluation.json(); + Map expected = evaluationCase.result(); + assertTrue( + Json.equal(expected.get("value"), actual.get("value")), + "value: expected " + expected.get("value") + " but was " + actual.get("value")); + assertEquals(expected.get("reason"), actual.get("reason"), "reason"); + if (expected.containsKey("errorCode")) { + assertEquals(expected.get("errorCode"), actual.get("errorCode"), "error code"); + } + if (expected.containsKey("variant")) { + assertEquals(expected.get("variant"), actual.get("variant"), "variant"); + } + Object allocationKey = allocationKey(expected); + if (allocationKey != null && !"".equals(allocationKey)) { + assertEquals(allocationKey, allocationKey(actual), "allocation key"); + } + } + + private static Exposure expectedExposure( + EvaluationCase evaluationCase, Map response) { + String flag = evaluationCase.flag(); + Object allocation = allocationKey(response); + Object variant = response.get("variant"); + if (variant == null + || "".equals(variant) + || allocation == null + || "".equals(allocation) + || !LOGGED_ALLOCATIONS.contains(flag + '\u0000' + allocation)) { + return null; + } + return new Exposure(flag, allocation, variant, evaluationCase.targetingKey()); + } + + private static List reportedExposures(EvpProxy evpProxy) { + List exposures = new ArrayList<>(); + for (EvpProxy.Request request : evpProxy.requests("api/v2/exposures")) { + for (Object event : (List) request.json().get("exposures")) { + Map exposure = Json.object(event); + exposures.add( + new Exposure( + key(exposure.get("flag")), + key(exposure.get("allocation")), + key(exposure.get("variant")), + Json.object(exposure.get("subject")).get("id"))); + } + } + return exposures; + } + + private static Object key(Object reference) { + return Json.object(reference).get("key"); + } + + private static Object allocationKey(Map evaluation) { + Object metadata = evaluation.get("flagMetadata"); + return metadata == null ? null : Json.object(metadata).get("allocationKey"); + } + + private static Set loggedAllocations() { + Set logged = new HashSet<>(); + Map flags = Json.object(Json.object(Json.parse(UFC_CONFIG)).get("flags")); + for (Map.Entry flag : flags.entrySet()) { + Object allocations = Json.object(flag.getValue()).get("allocations"); + if (allocations instanceof List) { + for (Object allocation : (List) allocations) { + if (allocation instanceof Map + && Boolean.TRUE.equals(Json.object(allocation).get("doLog"))) { + logged.add(flag.getKey() + '\u0000' + Json.object(allocation).get("key")); + } + } + } + } + return logged; + } + + /** An exposure, identified by its flag, allocation, variant and subject. */ + private static final class Exposure { + private final Object flag; + private final Object allocation; + private final Object variant; + private final Object subject; + + Exposure(Object flag, Object allocation, Object variant, Object subject) { + this.flag = flag; + this.allocation = allocation; + this.variant = variant; + this.subject = subject; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Exposure)) { + return false; + } + Exposure that = (Exposure) other; + return Objects.equals(this.flag, that.flag) + && Objects.equals(this.allocation, that.allocation) + && Objects.equals(this.variant, that.variant) + && Objects.equals(this.subject, that.subject); + } + + @Override + public int hashCode() { + return Objects.hash(this.flag, this.allocation, this.variant, this.subject); + } + + @Override + public String toString() { + return "flag=" + + this.flag + + " allocation=" + + this.allocation + + " variant=" + + this.variant + + " subject=" + + this.subject; + } + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/Json.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/Json.java new file mode 100644 index 00000000000..48209e01162 --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/Json.java @@ -0,0 +1,159 @@ +package datadog.smoketest.springboot; + +import com.squareup.moshi.JsonReader; +import com.squareup.moshi.JsonWriter; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import okio.Buffer; + +/** + * Reads and writes JSON, keeping numbers as written: the evaluation cases tell integers from + * decimals, which the application maps to different types. + */ +final class Json { + private Json() {} + + /** + * Parses a JSON document. + * + * @param json The JSON document. + * @return The parsed value: maps, lists, strings, {@link BigDecimal} numbers, booleans or {@code + * null}. + */ + static Object parse(String json) { + try (JsonReader reader = JsonReader.of(new Buffer().writeUtf8(json))) { + return read(reader); + } catch (IOException e) { + throw new UncheckedIOException("Failed to parse JSON: " + json, e); + } + } + + /** + * Writes a value as a JSON document. + * + * @param value The value, as returned by {@link #parse(String)}. + * @return The JSON document. + */ + static String write(Object value) { + Buffer buffer = new Buffer(); + try (JsonWriter writer = JsonWriter.of(buffer)) { + writer.setSerializeNulls(true); + write(writer, value); + } catch (IOException e) { + throw new UncheckedIOException("Failed to write JSON", e); + } + return buffer.readUtf8(); + } + + /** + * Compares two JSON values, numbers by value. + * + * @param expected The expected value. + * @param actual The actual value. + * @return Whether both values are equal. + */ + static boolean equal(Object expected, Object actual) { + if (expected instanceof BigDecimal && actual instanceof BigDecimal) { + return ((BigDecimal) expected).compareTo((BigDecimal) actual) == 0; + } + if (expected instanceof Map && actual instanceof Map) { + Map expectedMap = (Map) expected; + Map actualMap = (Map) actual; + if (!expectedMap.keySet().equals(actualMap.keySet())) { + return false; + } + for (Map.Entry entry : expectedMap.entrySet()) { + if (!equal(entry.getValue(), actualMap.get(entry.getKey()))) { + return false; + } + } + return true; + } + if (expected instanceof List && actual instanceof List) { + List expectedList = (List) expected; + List actualList = (List) actual; + if (expectedList.size() != actualList.size()) { + return false; + } + Iterator actualValues = actualList.iterator(); + for (Object expectedValue : expectedList) { + if (!equal(expectedValue, actualValues.next())) { + return false; + } + } + return true; + } + return Objects.equals(expected, actual); + } + + @SuppressWarnings("unchecked") + static Map object(Object value) { + return (Map) value; + } + + @SuppressWarnings("unchecked") + static List array(Object value) { + return (List) value; + } + + private static Object read(JsonReader reader) throws IOException { + switch (reader.peek()) { + case BEGIN_OBJECT: + Map object = new LinkedHashMap<>(); + reader.beginObject(); + while (reader.hasNext()) { + object.put(reader.nextName(), read(reader)); + } + reader.endObject(); + return object; + case BEGIN_ARRAY: + List array = new ArrayList<>(); + reader.beginArray(); + while (reader.hasNext()) { + array.add(read(reader)); + } + reader.endArray(); + return array; + case NUMBER: + return new BigDecimal(reader.nextString()); + case BOOLEAN: + return reader.nextBoolean(); + case NULL: + return reader.nextNull(); + default: + return reader.nextString(); + } + } + + private static void write(JsonWriter writer, Object value) throws IOException { + if (value instanceof Map) { + writer.beginObject(); + for (Map.Entry entry : ((Map) value).entrySet()) { + writer.name(String.valueOf(entry.getKey())); + write(writer, entry.getValue()); + } + writer.endObject(); + } else if (value instanceof List) { + writer.beginArray(); + for (Object item : (List) value) { + write(writer, item); + } + writer.endArray(); + } else if (value instanceof Number) { + writer.value((Number) value); + } else if (value instanceof Boolean) { + writer.value((Boolean) value); + } else if (value == null) { + writer.nullValue(); + } else { + writer.value(value.toString()); + } + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.java new file mode 100644 index 00000000000..5a8151b3e4d --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.java @@ -0,0 +1,25 @@ +package datadog.smoketest.springboot; + +import static datadog.smoketest.springboot.OpenFeatureFixtures.agentlessApp; + +import datadog.smoketest.SmokeServerApp; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * The SDK with the Datadog Java agent and the default agentless configuration source: the SDK polls + * the flag configuration from the CDN and delivers exposures through the agent. + */ +class OpenFeatureAgentlessSmokeTest extends AbstractOpenFeatureSmokeTest { + @RegisterExtension + static final SmokeServerApp APP = agentlessApp("openfeature-agentless").build(); + + @Override + SmokeServerApp app() { + return APP; + } + + @Override + boolean deliversExposures() { + return true; + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureFixtures.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureFixtures.java new file mode 100644 index 00000000000..6a82f05ebad --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureFixtures.java @@ -0,0 +1,235 @@ +package datadog.smoketest.springboot; + +import static datadog.trace.agent.test.server.http.JavaTestHttpServer.httpServer; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Comparator.comparing; +import static java.util.stream.Collectors.toList; + +import datadog.smoketest.SmokeServerApp; +import datadog.smoketest.backend.AgentBackend; +import datadog.smoketest.backend.EvpProxy; +import datadog.smoketest.backend.MockAgentBackend; +import datadog.trace.agent.test.server.http.JavaTestHttpServer; +import datadog.trace.agent.test.utils.OkHttpUtils; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/** + * The application, flag configuration and evaluation cases shared by the OpenFeature smoke tests. + */ +final class OpenFeatureFixtures { + static final String APPLICATION_JAR = + System.getProperty("datadog.smoketest.springboot.shadowJar.path"); + static final String AGENTLESS_BASE_URL = + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL"; + static final String FLAG_CONFIGURATION_PATH = "datadog/2/FFE_FLAGS/1/config"; + + /** The flag configuration, from the system tests data. */ + static final String UFC_CONFIG = read("ufc-config.json"); + + /** The flag configuration, as the Datadog managed CDN serves it. */ + private static final byte[] CDN_RESPONSE = + ("{\"data\":{\"type\":\"universal-flag-configuration\",\"attributes\":" + UFC_CONFIG + "}}") + .getBytes(UTF_8); + + /** Serves the flag configuration as the Datadog managed CDN does. */ + static final JavaTestHttpServer CDN = + httpServer( + server -> + server.handlers( + handlers -> + handlers.prefix( + "/api/v2/feature-flagging/config/rules-based/server", + api -> + api.getResponse() + .status(200) + .sendWithType("application/json", CDN_RESPONSE)))); + + private static final MediaType JSON = MediaType.parse("application/json"); + private static final OkHttpClient CLIENT = OkHttpUtils.client(); + + private OpenFeatureFixtures() {} + + /** + * Creates the agent stand-in. The dd-apm-test-agent neither proxies the flag evaluation events + * nor advertises the event platform proxy headers, so the tests use the in-process mock agent. + * + * @return A new mock agent backend. + */ + @SuppressWarnings("deprecation") + static MockAgentBackend agent() { + return AgentBackend.mockAgent(); + } + + /** + * Starts building the application with the agentless configuration source, polling the CDN. + * + * @param name The application name. + * @return The application builder. + */ + static SmokeServerApp.Builder agentlessApp(String name) { + return SmokeServerApp.named(name) + .jar(APPLICATION_JAR) + .args("--server.port=${app.httpPort}") + .env(AGENTLESS_BASE_URL, CDN.getAddress().toString()) + .backend(agent()); + } + + /** + * Gets the requests the application sent through the agent event platform proxy. + * + * @param app The application, reporting to the {@link #agent()}. + * @return The event platform proxy requests. + */ + static EvpProxy evpProxy(SmokeServerApp app) { + return ((MockAgentBackend) app.backend()).evpProxy(); + } + + /** + * Lists the system tests evaluation cases. + * + * @return The evaluation cases, sorted by file name then position. + */ + static List evaluationCases() { + try (Stream listing = Files.list(resourcePath("evaluation-cases"))) { + List files = + listing + .filter(path -> path.toString().endsWith(".json")) + .sorted(comparing(path -> path.getFileName().toString())) + .collect(toList()); + List cases = new ArrayList<>(); + for (Path file : files) { + cases.addAll(parseCases(file.getFileName().toString(), readFile(file))); + } + if (cases.isEmpty()) { + throw new IllegalStateException("No evaluation case found"); + } + return cases; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Gets the first system tests evaluation case of a file. + * + * @param fileName The evaluation cases file name. + * @return The first evaluation case of the file. + */ + static EvaluationCase evaluationCase(String fileName) { + return parseCases(fileName, read("evaluation-cases/" + fileName)).get(0); + } + + /** + * Evaluates a flag through the application. + * + * @param app The application. + * @param request The evaluation request. + * @return The evaluation response. + * @throws IOException If the request failed. + */ + static Evaluation evaluate(SmokeServerApp app, Map request) throws IOException { + Request httpRequest = + new Request.Builder() + .url(app.url() + "/openfeature/evaluate") + .post(RequestBody.create(JSON, Json.write(request))) + .build(); + try (Response response = CLIENT.newCall(httpRequest).execute()) { + return new Evaluation( + response.code(), response.body() == null ? "" : response.body().string()); + } + } + + private static List parseCases(String fileName, String json) { + List entries = Json.array(Json.parse(json)); + List cases = new ArrayList<>(); + for (int index = 0; index < entries.size(); index++) { + cases.add(new EvaluationCase(fileName, index, Json.object(entries.get(index)))); + } + return cases; + } + + private static Path resourcePath(String name) { + URL url = + Thread.currentThread().getContextClassLoader().getResource("ffe-system-test-data/" + name); + if (url == null) { + throw new IllegalStateException("Missing system tests data: " + name); + } + try { + return Paths.get(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalStateException(e); + } + } + + private static String read(String name) { + try { + return readFile(resourcePath(name)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static String readFile(Path file) throws IOException { + return new String(Files.readAllBytes(file), UTF_8); + } + + /** A system tests evaluation case: the evaluation request, along with its expected result. */ + static final class EvaluationCase { + private final String fileName; + private final int index; + final Map request; + + EvaluationCase(String fileName, int index, Map request) { + this.fileName = fileName; + this.index = index; + this.request = request; + } + + String flag() { + return (String) this.request.get("flag"); + } + + String targetingKey() { + return (String) this.request.get("targetingKey"); + } + + Map result() { + return Json.object(this.request.get("result")); + } + + @Override + public String toString() { + return this.fileName + "[" + this.index + "] flag=" + flag(); + } + } + + /** The application response to an evaluation request. */ + static final class Evaluation { + final int code; + final String body; + + Evaluation(int code, String body) { + this.code = code; + this.body = body; + } + + Map json() { + return Json.object(Json.parse(this.body)); + } + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.java new file mode 100644 index 00000000000..b7ba60092f0 --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureMalformedRemoteConfigSmokeTest.java @@ -0,0 +1,93 @@ +package datadog.smoketest.springboot; + +import static datadog.smoketest.springboot.OpenFeatureFixtures.APPLICATION_JAR; +import static datadog.smoketest.springboot.OpenFeatureFixtures.FLAG_CONFIGURATION_PATH; +import static datadog.smoketest.springboot.OpenFeatureFixtures.agent; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.smoketest.SmokeServerApp; +import datadog.smoketest.backend.MockAgentBackend; +import datadog.smoketest.backend.RemoteConfig; +import datadog.trace.agent.test.utils.OkHttpUtils; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * A malformed flag configuration that Remote Configuration delivers before the SDK subscribes is + * acknowledged by the agent, then reported as not applied once the SDK rejects it. + */ +class OpenFeatureMalformedRemoteConfigSmokeTest { + private static final String MALFORMED_CONFIG = "{\"flags\":[]}"; + private static final int APPLY_STATE_ACKNOWLEDGED = 2; + private static final int APPLY_STATE_ERROR = 3; + private static final MockAgentBackend AGENT = agent(); + + @RegisterExtension + static final SmokeServerApp APP = + SmokeServerApp.named("openfeature-malformed-remote-config") + .jar(APPLICATION_JAR) + .placeholder("agent.port", () -> Integer.toString(AGENT.port())) + .jvmArgs( + "-Ddd.remote_config.enabled=true", + "-Ddd.remote_config.url=http://localhost:${agent.port}/v0.7/config") + .args("--server.port=${app.httpPort}", "--openfeature.lazy-provider=true") + .env("DD_FEATURE_FLAGS_CONFIGURATION_SOURCE", "remote_config") + .backend(AGENT) + // The provider never gets a valid configuration, so the OpenFeature SDK logs its failed + // initialization. + .allowedErrorLogs(" d.o.sdk.", " dev.openfeature.sdk.") + .build(); + + @Test + void reportsAMalformedConfigurationReceivedBeforeTheSdkAsNotApplied() throws IOException { + RemoteConfig remoteConfig = AGENT.remoteConfig(); + remoteConfig.setConfig(FLAG_CONFIGURATION_PATH, MALFORMED_CONFIG); + + // The agent receives the configuration while no SDK has subscribed. + remoteConfig.waitForRequest(request -> applyState(request) == APPLY_STATE_ACKNOWLEDGED); + Request registration = + new Request.Builder() + .url(APP.url() + "/openfeature/provider") + .post(RequestBody.create(null, new byte[0])) + .build(); + try (Response response = OkHttpUtils.client().newCall(registration).execute()) { + assertEquals(202, response.code(), "provider registration"); + } + + // The SDK rejects it and the agent reports it as not applied. + Map rejected = + remoteConfig.waitForRequest(request -> applyState(request) == APPLY_STATE_ERROR); + Object applyError = flagConfigurationState(rejected).get("apply_error"); + assertTrue( + String.valueOf(applyError).contains("Malformed flag configuration"), + "apply error: " + applyError); + } + + private static int applyState(Map request) { + Map state = flagConfigurationState(request); + Object applyState = state == null ? null : state.get("apply_state"); + return applyState instanceof Number ? ((Number) applyState).intValue() : -1; + } + + private static Map flagConfigurationState(Map request) { + Object client = request.get("client"); + Object state = client instanceof Map ? ((Map) client).get("state") : null; + Object configStates = state instanceof Map ? ((Map) state).get("config_states") : null; + if (configStates instanceof List) { + for (Object configState : (List) configStates) { + if (configState instanceof Map + && "FFE_FLAGS".equals(((Map) configState).get("product"))) { + return Json.object(configState); + } + } + } + return null; + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.java new file mode 100644 index 00000000000..527d6a6c539 --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.java @@ -0,0 +1,112 @@ +package datadog.smoketest.springboot; + +import static datadog.remoteconfig.Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES; +import static datadog.smoketest.backend.RemoteConfig.capabilities; +import static datadog.smoketest.backend.RemoteConfig.products; +import static datadog.smoketest.springboot.OpenFeatureFixtures.APPLICATION_JAR; +import static datadog.smoketest.springboot.OpenFeatureFixtures.FLAG_CONFIGURATION_PATH; +import static datadog.smoketest.springboot.OpenFeatureFixtures.UFC_CONFIG; +import static datadog.smoketest.springboot.OpenFeatureFixtures.agent; +import static datadog.smoketest.springboot.OpenFeatureFixtures.evaluate; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.remoteconfig.Product; +import datadog.smoketest.SmokeServerApp; +import datadog.smoketest.backend.MockAgentBackend; +import datadog.smoketest.springboot.OpenFeatureFixtures.Evaluation; +import datadog.trace.test.util.PollingConditions; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * The SDK with the Datadog Java agent, receiving the flag configuration from Remote Configuration. + */ +class OpenFeatureProviderSmokeTest extends AbstractOpenFeatureSmokeTest { + private static final MockAgentBackend AGENT = agent(); + + static { + // The provider initializes with the application, so the configuration is served from start-up. + AGENT.remoteConfig().setConfig(FLAG_CONFIGURATION_PATH, UFC_CONFIG); + } + + @RegisterExtension + static final SmokeServerApp APP = + SmokeServerApp.named("openfeature-provider") + .jar(APPLICATION_JAR) + .placeholder("agent.port", () -> Integer.toString(AGENT.port())) + .jvmArgs( + "-Ddd.trace.debug=true", + "-Ddd.remote_config.enabled=true", + "-Ddd.remote_config.url=http://localhost:${agent.port}/v0.7/config") + .args("--server.port=${app.httpPort}") + .env("DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED", "true") + .env("DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED", "true") + .env("DD_FEATURE_FLAGS_CONFIGURATION_SOURCE", "remote_config") + .backend(AGENT) + .build(); + + private static Map firstPoll; + + /** Captures the first Remote Configuration poll, before the backend resets for the first test. */ + @BeforeAll + static void captureFirstPoll() { + firstPoll = AGENT.remoteConfig().waitForRequest(request -> true); + } + + @Override + SmokeServerApp app() { + return APP; + } + + @Override + boolean deliversExposures() { + return true; + } + + @Override + void publishConfiguration() { + AGENT.remoteConfig().setConfig(FLAG_CONFIGURATION_PATH, UFC_CONFIG); + } + + @Test + void asksTheAgentForFeatureFlagsOnTheFirstPoll() { + // An already-running Agent gives a newly-started tracer one new-client cache bypass. If + // FFE_FLAGS is missing here and only appears on a later poll, the Agent can miss the fast path. + assertTrue(products(firstPoll).contains(Product.FFE_FLAGS.name()), "FFE_FLAGS product"); + assertTrue( + (capabilities(firstPoll) & CAPABILITY_FFE_FLAG_CONFIGURATION_RULES) != 0, + "flag configuration rules capability"); + } + + @Test + void enrichesTheRequestSpan() throws IOException { + publishConfiguration(); + Map request = new HashMap<>(); + request.put("flag", "flag-that-does-not-exist"); + request.put("variationType", "STRING"); + request.put("defaultValue", "fallback"); + request.put("targetingKey", "span-enrichment-smoke-test"); + + Evaluation evaluation = evaluate(APP, request); + + assertEquals(200, evaluation.code, evaluation.body); + assertEquals("fallback", evaluation.json().get("value")); + new PollingConditions(30) + .eventually( + () -> + assertTrue( + APP.traces().getTraces().stream() + .flatMap(trace -> trace.getSpans().stream()) + .anyMatch( + span -> + span.getParentId() == 0 + && "{\"flag-that-does-not-exist\":\"fallback\"}" + .equals(span.getMeta().get("ffe_runtime_defaults"))), + "the request root span carries the runtime default")); + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.java new file mode 100644 index 00000000000..988025dea38 --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureShutdownSmokeTest.java @@ -0,0 +1,14 @@ +package datadog.smoketest.springboot; + +import datadog.smoketest.SmokeServerApp; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** The tracing shutdown hook flushes the SDK pending events. */ +class OpenFeatureShutdownSmokeTest extends AbstractOpenFeatureShutdownSmokeTest { + @RegisterExtension static final SmokeServerApp APP = shutdownApp("openfeature-shutdown", true); + + @Override + SmokeServerApp app() { + return APP; + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.java new file mode 100644 index 00000000000..738c76170b7 --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.java @@ -0,0 +1,30 @@ +package datadog.smoketest.springboot; + +import static datadog.smoketest.springboot.OpenFeatureFixtures.agentlessApp; + +import datadog.smoketest.SmokeServerApp; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * The SDK without the Datadog Java agent: the SDK polls the flag configuration from the CDN. + * Without an API key, events are not delivered. + */ +class OpenFeatureStandaloneSmokeTest extends AbstractOpenFeatureSmokeTest { + @RegisterExtension + static final SmokeServerApp APP = + agentlessApp("openfeature-standalone") + // Disables direct intake delivery, which would otherwise reach the real Datadog intake. + .env("DD_API_KEY", "") + .noAgent() + .build(); + + @Override + SmokeServerApp app() { + return APP; + } + + @Override + boolean deliversExposures() { + return false; + } +} diff --git a/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.java b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.java new file mode 100644 index 00000000000..20e1f5965cf --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/java/datadog/smoketest/springboot/OpenFeatureTraceDisabledShutdownSmokeTest.java @@ -0,0 +1,18 @@ +package datadog.smoketest.springboot; + +import datadog.smoketest.SmokeServerApp; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Without tracing, the Feature Flagging system registers its own shutdown hook to flush the SDK + * pending events. + */ +class OpenFeatureTraceDisabledShutdownSmokeTest extends AbstractOpenFeatureShutdownSmokeTest { + @RegisterExtension + static final SmokeServerApp APP = shutdownApp("openfeature-trace-disabled-shutdown", false); + + @Override + SmokeServerApp app() { + return APP; + } +} diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy index 5187f1f8af1..ca6e6946f77 100644 --- a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy @@ -64,12 +64,6 @@ abstract class AbstractSmokeTest extends ProcessManager { @Shared protected TestHttpServer.Headers lastTraceRequestHeaders = null - @Shared - protected CopyOnWriteArrayList> rcClientMessages = new CopyOnWriteArrayList() - - @Shared - private Throwable rcClientDecodingFailure = null - @Shared protected final PollingConditions defaultPoll = new PollingConditions(timeout: 30, initialDelay: 0, delay: 1, factor: 1) @@ -116,14 +110,6 @@ abstract class AbstractSmokeTest extends ProcessManager { response.status(200).send() } prefix("/v0.7/config") { - if (request.getBody() != null) { - try { - final msg = new JsonSlurper().parseText(new String(request.getBody(), StandardCharsets.UTF_8)) as Map - rcClientMessages.add(msg) - } catch (Throwable t) { - rcClientDecodingFailure = t - } - } response.status(200).send(remoteConfigResponse) } prefix("/telemetry/proxy/api/v2/apmtelemetry") { @@ -413,21 +399,6 @@ abstract class AbstractSmokeTest extends ProcessManager { } } - Map waitForRcClientRequest(final Function, Boolean> predicate) { - waitForRcClientRequest(defaultPoll, predicate) - } - - Map waitForRcClientRequest(final PollingConditions poll, final Function, Boolean> predicate) { - def message = null - poll.eventually { - if (rcClientDecodingFailure != null) { - throw rcClientDecodingFailure - } - assert (message = rcClientMessages.find { predicate.apply(it) }) != null - } - return message - } - Tuple2 waitForEvpProxyMessage(final Function, Boolean> predicate) { waitForEvpProxyMessage(defaultPoll, predicate) }