diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 18a202bf6a0..a9b7c4a8bf4 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -302,6 +302,12 @@ jobs: - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION"') run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION + - name: Run FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT"') + run: ./run.sh FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT + - name: Run FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT_DISABLED scenario + if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT_DISABLED"') + run: ./run.sh FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT_DISABLED - name: Run FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS scenario if: steps.build.outcome == 'success' && !cancelled() && contains(inputs.scenarios, '"FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS"') run: ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION_AGENTLESS diff --git a/manifests/c.yml b/manifests/c.yml index 416336889bb..38176847a65 100644 --- a/manifests/c.yml +++ b/manifests/c.yml @@ -4,6 +4,7 @@ manifest: tests/ai_guard/: irrelevant (dd-trace-c does not implement AI Guard) tests/appsec/: irrelevant (dd-trace-c does not implement AppSec or IAST) tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/ffe/test_evaluation_telemetry_opt_out.py: incomplete_test_app (The Perl Mojolicious workload does not provide the /ffe endpoint) tests/integrations/test_dbm.py: incomplete_test_app (The Perl Mojolicious workload does not provide database endpoints) tests/parametric/otel_env_vars/test_otel_exporter_otlp_compression.py::Test_OTEL_EXPORTER_OTLP_COMPRESSION: missing_feature tests/parametric/otel_env_vars/test_otel_exporter_otlp_headers.py::Test_OTEL_EXPORTER_OTLP_HEADERS: missing_feature diff --git a/manifests/cpp.yml b/manifests/cpp.yml index fddd915329b..99b11c7e9bc 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -22,6 +22,7 @@ manifest: tests/debugger/test_debugger_exception_replay.py::Test_Debugger_Exception_Replay::test_exception_replay_stackoverflow: missing_feature (Implemented only for dotnet) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (only parametric tests are run for cpp) tests/ffe/test_flag_eval_evp.py: irrelevant (only parametric tests are run for cpp) tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_sqs.py::Test_SQS_PROPAGATION_VIA_AWS_XRAY_HEADERS: irrelevant (Localstack SQS does not support AWS Xray Header parsing) diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index 47030f0bcc9..4b240565b9a 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -54,6 +54,7 @@ manifest: tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (C++ does not implement per-subject evaluation telemetry opt-out) tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index 30648f6498b..566ddb4e656 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -287,6 +287,7 @@ manifest: tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (C++ does not implement per-subject evaluation telemetry opt-out) tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index cbd41d62906..7c7b5fea0e5 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -822,6 +822,8 @@ manifest: weblog: [poc, uds] - declaration: bug (FFL-2184) excluded_weblog: [poc, uds] + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) + tests/ffe/test_evaluation_telemetry_opt_out.py::Test_FFE_Evaluation_Telemetry_Opt_Out_EVP: missing_feature (Requires a release containing dd-trace-dotnet#9351; FFL-2446) tests/ffe/test_exposure_egress.py: v3.36.0 tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented) tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct_Shutdown: missing_feature (Not yet implemented) diff --git a/manifests/golang.yml b/manifests/golang.yml index d4b4e0e1934..bef4fdfc65f 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1728,6 +1728,7 @@ manifest: weblog: [echo, gin, chi, net-http-orchestrion, net-http, uds-echo] - declaration: bug (FFL-2182) excluded_weblog: [echo, gin, chi, net-http-orchestrion, net-http, uds-echo] + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_exposure_egress.py: v2.6.0-dev # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.5.0 tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: v2.12.0-dev tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct_Shutdown: diff --git a/manifests/java.yml b/manifests/java.yml index 6f7fcc2d46f..808d74e935f 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3479,6 +3479,7 @@ manifest: weblog: [spring-boot] - declaration: bug (FFL-2184) excluded_weblog: [spring-boot] + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/java_lambda.yml b/manifests/java_lambda.yml index 83478d4eba8..bb44387817a 100644 --- a/manifests/java_lambda.yml +++ b/manifests/java_lambda.yml @@ -61,6 +61,7 @@ manifest: tests/appsec/waf/test_blocking.py::Test_Blocking: missing_feature tests/appsec/waf/test_blocking.py::Test_Blocking_strip_response_headers: missing_feature tests/appsec/waf/test_blocking.py::Test_CustomBlockingResponse: missing_feature + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (This server weblog scenario does not apply to Lambda) tests/ffe/test_flag_eval_evp.py: irrelevant (server-side FFE EVP weblog scenario is not applicable to lambda) tests/test_resource_renaming.py: missing_feature tests/test_rum_injection.py: irrelevant (RUM injection only supported for Java) diff --git a/manifests/java_otel.yml b/manifests/java_otel.yml index 0652748e59d..3a149b7e01e 100644 --- a/manifests/java_otel.yml +++ b/manifests/java_otel.yml @@ -12,6 +12,7 @@ manifest: tests/debugger/test_debugger_exception_replay.py::Test_Debugger_Exception_Replay::test_exception_replay_stackoverflow: missing_feature (Implemented only for dotnet) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (OpenTelemetry test library does not implement Datadog feature flag providers) tests/ffe/test_flag_eval_evp.py: irrelevant (OpenTelemetry test library does not implement server-side FFE EVP) tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_sqs.py::Test_SQS_PROPAGATION_VIA_AWS_XRAY_HEADERS: irrelevant (Localstack SQS does not support AWS Xray Header parsing) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 94b6334410c..4c4c55f78b4 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1871,6 +1871,7 @@ manifest: weblog: [express5] - declaration: bug (FFL-2182) excluded_weblog: [express5] + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": incomplete_test_app diff --git a/manifests/nodejs_lambda.yml b/manifests/nodejs_lambda.yml index f1ff3b763b9..3b0b4007d34 100644 --- a/manifests/nodejs_lambda.yml +++ b/manifests/nodejs_lambda.yml @@ -171,6 +171,7 @@ manifest: tests/appsec/waf/test_blocking.py::Test_Blocking: missing_feature tests/appsec/waf/test_blocking.py::Test_Blocking_strip_response_headers: missing_feature tests/appsec/waf/test_blocking.py::Test_CustomBlockingResponse: missing_feature + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (This server weblog scenario does not apply to Lambda) tests/ffe/test_flag_eval_evp.py: irrelevant (server-side FFE EVP weblog scenario is not applicable to lambda) tests/test_resource_renaming.py: missing_feature tests/test_rum_injection.py: irrelevant diff --git a/manifests/nodejs_otel.yml b/manifests/nodejs_otel.yml index 0852007bf5d..eaab8733ff0 100644 --- a/manifests/nodejs_otel.yml +++ b/manifests/nodejs_otel.yml @@ -11,6 +11,7 @@ manifest: tests/debugger/test_debugger_exception_replay.py::Test_Debugger_Exception_Replay::test_exception_replay_stackoverflow: missing_feature (Implemented only for dotnet) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (OpenTelemetry test library does not implement Datadog feature flag providers) tests/ffe/test_flag_eval_evp.py: irrelevant (OpenTelemetry test library does not implement server-side FFE EVP) tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_sqs.py::Test_SQS_PROPAGATION_VIA_AWS_XRAY_HEADERS: irrelevant (Localstack SQS does not support AWS Xray Header parsing) diff --git a/manifests/php.yml b/manifests/php.yml index c99643f4dc8..8eea6081b3d 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1240,6 +1240,7 @@ manifest: "*": v1.21.0-dev laravel11x: incomplete_test_app symfony7x: incomplete_test_app + tests/ffe/test_evaluation_telemetry_opt_out.py: incomplete_test_app (Deferred until the PHP OpenFeature test-app foundation is available) tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": v1.21.0-dev diff --git a/manifests/python.yml b/manifests/python.yml index 80f14f5e4aa..47ed7a10070 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2240,6 +2240,7 @@ manifest: flask-poc: v4.14.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v4.0.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: flaky (FFL-1622) + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_exposure_egress.py: v4.2.0-dev tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: - weblog_declaration: diff --git a/manifests/python_lambda.yml b/manifests/python_lambda.yml index 39432f50af5..7bff96a9e68 100644 --- a/manifests/python_lambda.yml +++ b/manifests/python_lambda.yml @@ -305,6 +305,7 @@ manifest: tests/debugger/test_debugger_exception_replay.py::Test_Debugger_Exception_Replay::test_exception_replay_stackoverflow: missing_feature (Implemented only for dotnet) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (This server weblog scenario does not apply to Lambda) tests/ffe/test_flag_eval_evp.py: irrelevant (server-side FFE EVP weblog scenario is not applicable to lambda) tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_sqs.py::Test_SQS_PROPAGATION_VIA_AWS_XRAY_HEADERS: irrelevant (Localstack SQS does not support AWS Xray Header parsing) diff --git a/manifests/python_otel.yml b/manifests/python_otel.yml index 241d0d3c919..7cf144e0493 100644 --- a/manifests/python_otel.yml +++ b/manifests/python_otel.yml @@ -11,6 +11,7 @@ manifest: tests/debugger/test_debugger_exception_replay.py::Test_Debugger_Exception_Replay::test_exception_replay_stackoverflow: missing_feature (Implemented only for dotnet) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (OpenTelemetry test library does not implement Datadog feature flag providers) tests/ffe/test_flag_eval_evp.py: irrelevant (OpenTelemetry test library does not implement server-side FFE EVP) tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_sqs.py::Test_SQS_PROPAGATION_VIA_AWS_XRAY_HEADERS: irrelevant (Localstack SQS does not support AWS Xray Header parsing) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 5a7cf90a6ae..6be3b6badd1 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2137,6 +2137,7 @@ manifest: - weblog_declaration: "*": irrelevant rails72: v2.23.0-dev + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/ruby_lambda.yml b/manifests/ruby_lambda.yml index 89f6602b30a..cd183c66b17 100644 --- a/manifests/ruby_lambda.yml +++ b/manifests/ruby_lambda.yml @@ -85,4 +85,5 @@ manifest: tests/appsec/waf/test_blocking.py::Test_Blocking::test_html_template_v2: missing_feature tests/appsec/waf/test_blocking.py::Test_Blocking_strip_response_headers: v3.30.0 tests/appsec/waf/test_blocking.py::Test_CustomBlockingResponse: v3.29.0 + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (This server weblog scenario does not apply to Lambda) tests/ffe/test_flag_eval_evp.py: irrelevant (server-side FFE EVP weblog scenario is not applicable to lambda) diff --git a/manifests/rust.yml b/manifests/rust.yml index 67215ed811e..6fdac3ee58a 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -23,6 +23,7 @@ manifest: tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature tests/ffe/test_agentless_configuration.py: missing_feature (Rust is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) diff --git a/tests/ffe/README.md b/tests/ffe/README.md index b1d8151715a..8188dd059ca 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -11,6 +11,7 @@ This directory contains system tests for the Feature Flags & Experimentation (FF | `test_exposures_datadog_agent.py` | Flag exposure tracking and reporting through the Datadog Agent | | `test_flag_eval_metrics.py` | Evaluation metrics (OTel counter) | | `test_flag_eval_evp.py` | Server-side EVP flagevaluation payloads, aggregation, and bounds | +| `test_evaluation_telemetry_opt_out.py` | Per-subject evaluation telemetry opt-out, default-off behavior, and unchanged exposures | ## Running FFE Tests @@ -18,14 +19,104 @@ This directory contains system tests for the Feature Flags & Experimentation (FF ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION --library ``` +## Per-subject evaluation telemetry opt-out + +All four test classes report under [#649: Per-subject evaluation telemetry opt-out](https://feature-parity.us1.prod.dog/#/?feature=649). + +The [customer solutions document](https://docs.google.com/document/d/15rai-S0NmGO2sKsMgy6cGf4963Y--9qdEfAj5MiytUg/edit) +describes the customer capability. The implemented name is `dd_evaluation_telemetry_opt_out`. +The [explicit scope decision](https://github.com/DataDog/dd-sdk-ios/pull/3305#discussion_r4326232944) +preserves experiment exposures, and the +[attribute decision](https://github.com/DataDog/dd-trace-go/pull/5611#discussion_r4307380659) +retains the control in targeting and exposure attributes. + +Candidate SDK implementations must pass the applicable local checks before delivery. +Released-version CI activation is a separate step. Both scenarios are excluded from the default +`all`, `end_to_end`, and `tracer_release` groups. They remain selectable by name or through the `ffe` group. +When activating released versions in manifests, restore the default scenario groups too. +Otherwise tracer CI that selects only `tracer_release` does not run these tests. +Manifests mark unsupported SDK versions as expected failures; this does not prevent their setup from running. +Use a local candidate SDK and force these tests to validate unreleased changes: + +```bash +./run.sh FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT tests/ffe/test_evaluation_telemetry_opt_out.py \ + -F tests/ffe/test_evaluation_telemetry_opt_out.py +./run.sh FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT_DISABLED tests/ffe/test_evaluation_telemetry_opt_out.py \ + -F tests/ffe/test_evaluation_telemetry_opt_out.py +``` + +The enabled scenario sets `DD_FEATURE_FLAGS_EVALUATION_TELEMETRY_PER_SUBJECT_OPT_OUT_ENABLED=true`. +The disabled scenario leaves the setting absent. Both use the same requests. + +| Context value | Enabled | Disabled | +| --- | --- | --- | +| Boolean `true` | No evaluation telemetry | Evaluation telemetry remains | +| Boolean `false`, missing, `null`, string `"true"`, or number `1` | Evaluation telemetry remains | Evaluation telemetry remains | +| Wrong-case key or legacy `dd_telemetry_opt_out` key | Evaluation telemetry remains | Evaluation telemetry remains | + +The SDK reads the attribute on every evaluation. A subject stays opted out only while each evaluation context carries it. +Set the attribute in the context supplied to the evaluation call. +These tests do not cover opt-outs added by application hooks; SDK behavior differs for those hooks. + +Separate tests check exact EVP counts, evaluation metric presence, and automatic trace annotations. +Metric checks accept delta exports and repeated cumulative exports. Opted-out flags must have no nonzero metric points. +Each test includes allowed subjects as positive controls. +Mixed subjects evaluate the same flag to detect a flag-scoped opt-out decision. +EVP counts match both the flag and the subject's raw or hashed targeting key. +Shared-flag subjects receive the same value from distinct fixture allocations. +Metric assertions use the allocation tags to check each subject separately, without adding subject tags to metrics. +One rule targets the opt-out attribute to verify that it remains available for evaluation. +One subject opts out, then removes the attribute; later evaluations must emit telemetry again. +Those two phases use distinct allocation tags with the same flag value, so resumed metrics cannot hide an earlier leak. +Missing flags test suppression on the error path, return the application default, and produce no exposures. +Configured flags must return the same flag value for every request. +Exposures must retain their context attributes and ordinary deduplication for an unchanged context. +After an attribute changes, exposure deduplication can differ between SDKs; these tests do not check it. +Validation reads final captures after the scenario stops its application and drains the Agent. +Remote Configuration and evaluation-request errors are reported during validation, without aborting the remaining setups. + +PHP coverage is deferred until its OpenFeature test-app foundation is available. +.NET EVP coverage requires a tracer release containing [dd-trace-dotnet#9351](https://github.com/DataDog/dd-trace-dotnet/pull/9351). +For older candidates without that producer, exclude the EVP class with `-k 'not EVP'`. +Browser and mobile SDKs are outside this server harness and retain their repository integration tests. + +### Release activation and follow-up coverage + +Replace each provisional manifest entry with the first released SDK version containing its opt-out implementation. +For Node.js, this must include [dd-trace-js#10776](https://github.com/DataDog/dd-trace-js/pull/10776); a merged PR is not a released version. +.NET needs a tracer release containing [dd-trace-dotnet#9426](https://github.com/DataDog/dd-trace-dotnet/pull/9426) +and a weblog using `Datadog.FeatureFlags.OpenFeature` 2.3.3 or later. +Its EVP class has a separate dependency on #9351. The tracer manifest does not gate the NuGet provider version. + +These scenarios activate opt-out through the process environment only. +Coverage for other configuration sources, such as `application_monitoring.yaml`, is separate follow-up work. + +### Local .NET candidate packages + +Put the `Datadog.Trace` and `Datadog.FeatureFlags.OpenFeature` packages in `binaries/`. +Also supply the matching tracer archive (`datadog-dotnet-apm-*.tar.gz`) or extracted tracer files. +For the `poc` weblog, pass exact NuGet version ranges through the Docker build arguments: + +```bash +./build.sh dotnet -w poc --extra-docker-args \ + "--build-arg DD_TRACE_PACKAGE_VERSION=[] --build-arg DD_OPENFEATURE_PACKAGE_VERSION=[]" +``` + +Brackets require that exact version. A plain version is a minimum; NuGet can select a higher version with warning NU1603. +Use unique candidate versions that are not published on nuget.org, so only `binaries/` can supply them. +Verify the resolved packages before reporting local validation. These arguments do not select the installed tracer binary. +The `uds` weblog does not support these build arguments or the local package source. + ## Test-contract self-tests -The self-tests in `tests/test_the_test/test_ffe_tests.py` validate the FFE assertions using synthetic -captures. They do not run SDKs or contact an intake. They are part of the -`TEST_THE_TEST` scenario, which runs on every PR: +The self-tests in `tests/test_the_test/test_ffe_tests.py` and +`tests/test_the_test/test_ffe_evaluation_telemetry_opt_out.py` validate the FFE assertions +using synthetic captures. They do not run SDKs or contact an intake. They are part of +the `TEST_THE_TEST` scenario, which runs on every PR. With the Python 3.12 runner +environment installed, run them explicitly: ```bash -./run.sh TEST_THE_TEST tests/test_the_test/test_ffe_tests.py +./run.sh TEST_THE_TEST tests/test_the_test/test_ffe_tests.py tests/test_the_test/test_ffe_evaluation_telemetry_opt_out.py ``` Framework mock-backend and lifecycle tests stay in `tests/test_the_test/`. diff --git a/tests/ffe/test_evaluation_telemetry_opt_out.py b/tests/ffe/test_evaluation_telemetry_opt_out.py new file mode 100644 index 00000000000..ee2245b0268 --- /dev/null +++ b/tests/ffe/test_evaluation_telemetry_opt_out.py @@ -0,0 +1,300 @@ +"""Per-subject opt-out suppresses evaluation telemetry, not flag results or exposures. + +Run the same assertions with support disabled (the default) and +enabled. Each signal has its own class so manifests can describe SDK support. +Validation reads the final captures after the scenario flushes and stops its +containers. Positive controls prevent missing telemetry from passing as opt-out. +""" + +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Literal, TypedDict, cast + +from tests.ffe.utils.evaluation_telemetry import ( + assert_count, + assert_evaluation_responses, + assert_opt_out_exposure_attribute, + assert_metric_activity, + assert_setup_succeeded, + evaluation_count, + evaluation_setup_error, +) +from tests.ffe.utils.exposures import exposure_events_from_data +from tests.ffe.utils.fixtures import make_ufc_fixture +from utils import HttpResponse, context, features, interfaces, remote_config as rc, rfc, scenarios, weblog + + +ATTRIBUTE = "dd_evaluation_telemetry_opt_out" +RFC = "https://docs.google.com/document/d/15rai-S0NmGO2sKsMgy6cGf4963Y--9qdEfAj5MiytUg/edit" +ATTRIBUTES: dict[str, dict[str, Any]] = { + "true": {ATTRIBUTE: True}, + "false": {ATTRIBUTE: False}, + "missing": {}, + "string-true": {ATTRIBUTE: "true"}, + "number-one": {ATTRIBUTE: 1}, + "null": {ATTRIBUTE: None}, + "wrong-case": {"DD_EVALUATION_TELEMETRY_OPT_OUT": True}, + "legacy-name": {"dd_telemetry_opt_out": True}, +} +ANNOTATIONS = {"ffe_flags_enc", "ffe_subjects_enc", "ffe_runtime_defaults"} + + +class EvaluationCase(TypedDict): + flag: str + subject: str + allocation_key: str | None + attributes: dict[str, Any] + expected_count: int + expected_value: str + expected_exposures: int | None + responses: list[HttpResponse] + + +class _TelemetryContract: + cases: list[EvaluationCase] + + def prepare(self, signal: str) -> None: + self.setup_error = evaluation_setup_error(lambda: self._prepare(signal)) + + def _prepare(self, signal: str) -> None: + enabled = context.scenario == scenarios.feature_flagging_evaluation_telemetry_opt_out + self.cases = [] + fixture = make_ufc_fixture(f"opt-out-{signal}-true", serial_id=101) + fixture["flags"] = {} # Keep only the envelope; flags are added below. + for index, (label, attributes) in enumerate(ATTRIBUTES.items()): + flag = f"opt-out-{signal}-{label}" + fixture["flags"].update(make_ufc_fixture(flag, serial_id=101 + index)["flags"]) + if label == "true": + allocations = fixture["flags"][flag]["allocations"] + allocations[0]["rules"] = [ + {"conditions": [{"attribute": ATTRIBUTE, "operator": "ONE_OF", "value": ["true"]}]} + ] + # Removing the attribute before evaluation selects off-value, + # even if the original context is retained for exposure logging. + allocations.extend( + make_ufc_fixture(flag, "off", allocation_key="fallback", serial_id=201)["flags"][flag][ + "allocations" + ] + ) + self.cases.append( + EvaluationCase( + flag=flag, + subject=f"subject-{signal}-{label}", + allocation_key="default-allocation", + attributes=attributes, + expected_count=0 if enabled and label == "true" else 3, + expected_value="on-value", + expected_exposures=1, + responses=[], + ) + ) + + # Absent flags return the application default without an allocation. + self.cases.extend( + EvaluationCase( + flag=f"opt-out-{signal}-missing-flag-{label}", + subject=f"subject-{signal}-missing-flag-{label}", + allocation_key=None, + attributes={ATTRIBUTE: opted_out}, + expected_count=0 if enabled and opted_out else 3, + expected_value="default", + expected_exposures=0, + responses=[], + ) + for label, opted_out in [("true", True), ("false", False)] + ) + independent_cases = list(self.cases) + shared_flag = f"opt-out-{signal}-shared" + fixture["flags"].update(make_ufc_fixture(shared_flag)["flags"]) + fixture["flags"][shared_flag]["allocations"] = [] + shared_cases = [ + EvaluationCase( + flag=shared_flag, + subject=f"subject-{signal}-shared-{label}", + allocation_key=f"shared-{label}", + attributes={ATTRIBUTE: opted_out, "test_subject": f"subject-{signal}-shared-{label}"}, + expected_count=0 if enabled and opted_out else 3, + expected_value="on-value", + expected_exposures=1, + responses=[], + ) + for label, opted_out in [("true", True), ("false", False)] + ] + # Metrics do not contain subject identifiers. Distinct allocations let + # their existing tags identify each subject without changing the value. + for index, case in enumerate(shared_cases): + allocation = make_ufc_fixture( + shared_flag, allocation_key=cast("str", case["allocation_key"]), serial_id=110 + index + )["flags"][shared_flag]["allocations"][0] + allocation["rules"] = [ + {"conditions": [{"attribute": "test_subject", "operator": "ONE_OF", "value": [case["subject"]]}]} + ] + fixture["flags"][shared_flag]["allocations"].append(allocation) + self.cases.extend(shared_cases) + + # The same subject opts out, then removes the attribute. Suppression is + # per evaluation, not a permanent decision cached for the subject. + per_evaluation_flag = f"opt-out-{signal}-per-evaluation" + fixture["flags"].update( + make_ufc_fixture(per_evaluation_flag, allocation_key="opted-out", serial_id=120)["flags"] + ) + per_evaluation_allocations = fixture["flags"][per_evaluation_flag]["allocations"] + per_evaluation_allocations[0]["rules"] = [ + {"conditions": [{"attribute": ATTRIBUTE, "operator": "ONE_OF", "value": ["true"]}]} + ] + # Separate allocation tags keep the later positive control from hiding + # metric leakage during the first phase. Both phases return the same value. + per_evaluation_allocations.extend( + make_ufc_fixture(per_evaluation_flag, allocation_key="resumed", serial_id=121)["flags"][ + per_evaluation_flag + ]["allocations"] + ) + per_evaluation_cases = [ + EvaluationCase( + flag=per_evaluation_flag, + subject=f"subject-{signal}-per-evaluation", + allocation_key="opted-out" if attributes else "resumed", + attributes=attributes, + expected_count=0 if enabled and attributes else 3, + expected_value="on-value", + # Exposure deduplication after an attribute change is SDK-defined. + expected_exposures=None, + responses=[], + ) + for attributes in [{ATTRIBUTE: True}, {}] + ] + self.cases.extend(per_evaluation_cases) + + rc.tracer_rc_state.reset().set_config(f"datadog/2/FFE_FLAGS/opt-out-{signal}/config", fixture).apply() + + # Different subjects overlap. Repeated evaluations also exercise aggregation + # and the ordinary exposure cache without changing the targeting context. + def evaluate(case: EvaluationCase) -> None: + response = weblog.post( + "/ffe", + json={ + "flag": case["flag"], + "targetingKey": case["subject"], + "attributes": case["attributes"], + "variationType": "STRING", + "defaultValue": "default", + }, + ) + case["responses"].append(response) + + with ThreadPoolExecutor(max_workers=len(independent_cases)) as executor: + for _ in range(3): + list(executor.map(evaluate, independent_cases)) + + # Alternate subjects on one flag. A flag-scoped opt-out cache must not + # suppress the allowed subject or permit the opted-out subject. + for _ in range(3): + for case in shared_cases: + evaluate(case) + + for case in per_evaluation_cases: + for _ in range(3): + evaluate(case) + + def assert_evaluations(self) -> None: + assert_setup_succeeded(self.setup_error) + for case in self.cases: + assert_evaluation_responses(case["responses"], expected_count=3, expected_value=case["expected_value"]) + + def expected_totals(self, key: Literal["subject", "allocation_key"]) -> dict[tuple[str, str | None], int]: + # Sequential evaluations of the same subject share the exported series. + totals: dict[tuple[str, str | None], int] = {} + for case in self.cases: + group = (case["flag"], case[key]) + totals[group] = totals.get(group, 0) + case["expected_count"] + return totals + + +@scenarios.feature_flagging_evaluation_telemetry_opt_out_disabled +@scenarios.feature_flagging_evaluation_telemetry_opt_out +@features.feature_flags_evaluation_telemetry_opt_out +@rfc(RFC) +class Test_FFE_Evaluation_Telemetry_Opt_Out_EVP(_TelemetryContract): + def setup_counts(self) -> None: + self.prepare("evp") + + def test_counts(self) -> None: + self.assert_evaluations() + captures = list(interfaces.agent.get_data(path_filters="/api/v2/flagevaluation")) + for (flag, subject), expected_count in self.expected_totals("subject").items(): + assert subject is not None + assert_count( + evaluation_count(captures, flag, subject=subject), + expected_count, + f"{flag} ({subject})", + ) + + +@scenarios.feature_flagging_evaluation_telemetry_opt_out_disabled +@scenarios.feature_flagging_evaluation_telemetry_opt_out +@features.feature_flags_evaluation_telemetry_opt_out +@rfc(RFC) +class Test_FFE_Evaluation_Telemetry_Opt_Out_Metrics(_TelemetryContract): + def setup_counts(self) -> None: + self.prepare("metrics") + + def test_counts(self) -> None: + self.assert_evaluations() + metrics = [metric for _, metric in interfaces.agent.get_metrics()] + for (flag, allocation_key), expected_count in self.expected_totals("allocation_key").items(): + assert_metric_activity( + metrics, + flag, + allocation_key=allocation_key, + suppressed=expected_count == 0, + ) + + +@scenarios.feature_flagging_evaluation_telemetry_opt_out_disabled +@scenarios.feature_flagging_evaluation_telemetry_opt_out +@features.feature_flags_evaluation_telemetry_opt_out +@rfc(RFC) +class Test_FFE_Evaluation_Telemetry_Opt_Out_Exposures(_TelemetryContract): + def setup_unchanged(self) -> None: + self.prepare("exposures") + + def test_unchanged(self) -> None: + self.assert_evaluations() + captures = list(interfaces.agent.get_data(path_filters="/api/v2/exposures")) + for case in self.cases: + if case["expected_exposures"] is None: + continue + events = [ + event for data in captures for event in exposure_events_from_data(data, {case["flag"]}, case["subject"]) + ] + assert len(events) == case["expected_exposures"], ( + f"{case['flag']}: expected {case['expected_exposures']} exposures, got {events}" + ) + if not events: + continue + assert events[0]["variant"]["key"] == "on" + assert events[0]["allocation"]["key"] == case["allocation_key"] + attributes = events[0]["subject"].get("attributes", {}) + # Some providers also include targetingKey in exposure attributes. + # Check the customer input without forbidding that existing behavior. + assert_opt_out_exposure_attribute(attributes, case["attributes"], ATTRIBUTE) + + +@scenarios.feature_flagging_evaluation_telemetry_opt_out_disabled +@scenarios.feature_flagging_evaluation_telemetry_opt_out +@features.feature_flags_evaluation_telemetry_opt_out +@rfc(RFC) +class Test_FFE_Evaluation_Telemetry_Opt_Out_Trace_Annotations(_TelemetryContract): + def setup_annotations(self) -> None: + self.prepare("traces") + + def test_annotations(self) -> None: + self.assert_evaluations() + for case in self.cases: + for response in case["responses"]: + spans = [span for _, _, span in interfaces.library.get_spans(request=response, full_trace=True)] + assert spans, f"{case['flag']}: unrelated application tracing must remain enabled" + annotated = [span for span in spans if ANNOTATIONS.intersection(span.get("meta", {}))] + if case["expected_count"]: + assert annotated, f"{case['flag']}: missing positive-control flag annotation" + else: + assert not annotated, f"{case['flag']}: opted-out evaluation annotated a trace" diff --git a/tests/ffe/utils/evaluation_telemetry.py b/tests/ffe/utils/evaluation_telemetry.py new file mode 100644 index 00000000000..7a7b59f3ec4 --- /dev/null +++ b/tests/ffe/utils/evaluation_telemetry.py @@ -0,0 +1,82 @@ +"""Validate evaluation responses and telemetry after the scenario stops.""" + +import hashlib +import json +import traceback +from collections.abc import Callable, Iterable +from typing import Any + +from utils import HttpResponse + + +def evaluation_setup_error(operation: Callable[[], None]) -> str | None: + """Defer failures as serializable text for validation and replay.""" + try: + operation() + except Exception: # Report setup failures during the test phase. + return traceback.format_exc() + return None + + +def assert_setup_succeeded(error: str | None) -> None: + assert error is None, f"Evaluation setup failed:\n{error}" + + +def assert_evaluation_responses(responses: list[HttpResponse], *, expected_count: int, expected_value: str) -> None: + assert len(responses) == expected_count, f"Expected {expected_count} evaluation responses, got {len(responses)}" + for response in responses: + assert response.status_code == 200, response.text + assert json.loads(response.text)["value"] == expected_value, response.text + + +def assert_count(actual: float, expected: int, flag: str) -> None: + assert actual == expected, f"{flag}: expected {expected} evaluations, captured {actual}" + + +def assert_opt_out_exposure_attribute(actual: dict[str, Any], expected: dict[str, Any], attribute: str) -> None: + if attribute not in expected: + assert attribute not in actual, actual + elif expected[attribute] is None: + # SDKs may omit null attributes or retain them as null. + assert actual.get(attribute) is None, actual + else: + assert actual[attribute] == expected[attribute], actual + # Python considers 1 == True. Preserve bool-ness in both directions. + assert isinstance(actual[attribute], bool) == isinstance(expected[attribute], bool), actual + + +def assert_metric_activity( + metrics: Iterable[dict[str, Any]], flag: str, *, allocation_key: str | None, suppressed: bool +) -> None: + # Agent series do not retain the source OTel aggregation temporality. Check + # presence, not a sum that can count repeated cumulative exports twice. + # EVP assertions separately check exact evaluation counts. + values = [ + float(point["value"] if isinstance(point, dict) else point[1]) + for metric in metrics + if metric.get("metric") == "feature_flag.evaluations" + and f"feature_flag.key:{flag}" in metric.get("tags", []) + and (allocation_key is None or f"feature_flag.result.allocation_key:{allocation_key}" in metric.get("tags", [])) + for point in metric.get("points", []) + ] + if suppressed: + assert all(value == 0 for value in values), ( + f"{flag} ({allocation_key}): opted-out evaluation emitted metrics: {values}" + ) + else: + assert any(value > 0 for value in values), ( + f"{flag} ({allocation_key}): missing positive-control evaluation metric: {values}" + ) + + +def evaluation_count(captures: Iterable[dict[str, Any]], flag: str, *, subject: str) -> int: + subject_keys = {subject, "sha256_" + hashlib.sha256(subject.encode()).hexdigest()} + total = 0 + for data in captures: + content = data.get("request", {}).get("content", {}) + if not isinstance(content, dict): + continue + for event in content.get("flagEvaluations", []): + if event.get("flag", {}).get("key") == flag and event.get("targeting_key") in subject_keys: + total += event["evaluation_count"] + return total diff --git a/tests/test_the_test/test_ffe_evaluation_telemetry_opt_out.py b/tests/test_the_test/test_ffe_evaluation_telemetry_opt_out.py new file mode 100644 index 00000000000..42efcc8e727 --- /dev/null +++ b/tests/test_the_test/test_ffe_evaluation_telemetry_opt_out.py @@ -0,0 +1,263 @@ +"""Self-tests ensure the opt-out oracle rejects leaks and missing positive controls.""" + +import hashlib +import json +from concurrent.futures import ThreadPoolExecutor +from typing import Any +from tests.ffe.utils.evaluation_telemetry import ( + assert_count, + assert_evaluation_responses, + assert_opt_out_exposure_attribute, + assert_metric_activity, + assert_setup_succeeded, + evaluation_count, + evaluation_setup_error, +) +from utils import HttpResponse, features, pytest, scenarios + + +@scenarios.test_the_test +@features.not_reported +@pytest.mark.parametrize( + ("status_code", "text", "count", "passes"), + [ + (200, '{"value": "on-value"}', 3, True), + (500, '{"value": "on-value"}', 3, False), + (200, '{"value": "default"}', 3, False), + (200, "not JSON", 3, False), + (200, "{}", 3, False), + (200, '{"value": "on-value"}', 0, False), + (200, '{"value": "on-value"}', 2, False), + (200, '{"value": "on-value"}', 4, False), + ], +) +def test_evaluation_response_oracle(status_code: int, text: str, count: int, *, passes: bool) -> None: + response = HttpResponse( + { + "request": {"method": "POST", "url": "http://weblog/ffe", "params": None}, + "status_code": status_code, + "text": text, + "cookies": {}, + } + ) + responses = [response] * count + if passes: + assert_evaluation_responses(responses, expected_count=3, expected_value="on-value") + else: + with pytest.raises((AssertionError, ValueError, KeyError)): + assert_evaluation_responses(responses, expected_count=3, expected_value="on-value") + + +@scenarios.test_the_test +@features.not_reported +@pytest.mark.parametrize( + ("suppressed_count", "allowed_count", "passes"), + [(0, 3, True), (3, 0, False), (3, 3, False), (0, 0, False), (0, 6, False)], +) +@pytest.mark.parametrize("shared_flag", [False, True]) +@pytest.mark.parametrize("hashed_subject", [False, True]) +def test_opt_out_evp_oracle( + suppressed_count: int, allowed_count: int, *, shared_flag: bool, hashed_subject: bool, passes: bool +) -> None: + events = [ + { + "flag": {"key": "shared" if shared_flag else subject}, + "targeting_key": "sha256_" + hashlib.sha256(subject.encode()).hexdigest() if hashed_subject else subject, + "evaluation_count": count, + } + for subject, count in [("suppressed", suppressed_count), ("allowed", allowed_count)] + if count + ] + captures: list[dict[str, Any]] = [ + {"path": "/api/v2/flagevaluation", "request": {"content": {"flagEvaluations": events}}} + ] + + def validate() -> None: + for subject, expected in [("suppressed", 0), ("allowed", 3)]: + flag = "shared" if shared_flag else subject + assert_count(evaluation_count(captures, flag, subject=subject), expected, subject) + + if passes: + validate() + else: + with pytest.raises(AssertionError, match="expected.*evaluations"): + validate() + + +@scenarios.test_the_test +@features.not_reported +@pytest.mark.parametrize( + ("values", "suppressed", "passes"), + [ + ([3, 3, 3], False, True), # Repeated cumulative snapshots are not three separate counts. + ([1, 2, 3, 3], False, True), + ([1, 1, 1], False, True), # Delta exports are also valid. + ([], False, False), + ([0, 0], False, False), + ([], True, True), + ([0, 0], True, True), + ([1, 0], True, False), # A later zero does not hide earlier leaked telemetry. + ([3, 3, 3], True, False), + ([-1, 1], True, False), # Points must not cancel each other out. + ], +) +@pytest.mark.parametrize("point_format", ["dict", "list"]) +def test_opt_out_metric_oracle(values: list[int], point_format: str, *, suppressed: bool, passes: bool) -> None: + points = [ + {"timestamp": index, "value": value} if point_format == "dict" else [index, value] + for index, value in enumerate(values) + ] + metrics: list[dict[str, Any]] = [ + { + "metric": "feature_flag.evaluations", + "tags": ["feature_flag.key:subject-flag", "feature_flag.result.allocation_key:subject-allocation"], + "points": points, + }, + { + "metric": "feature_flag.evaluations", + "tags": ["feature_flag.key:other-flag", "feature_flag.result.allocation_key:subject-allocation"], + "points": [[1, 99]], + }, + { + "metric": "feature_flag.evaluations", + "tags": ["feature_flag.key:subject-flag", "feature_flag.result.allocation_key:other-allocation"], + "points": [[1, 99]], + }, + { + "metric": "unrelated.metric", + "tags": ["feature_flag.key:subject-flag", "feature_flag.result.allocation_key:subject-allocation"], + "points": [[1, 99]], + }, + ] + if passes: + assert_metric_activity(metrics, "subject-flag", allocation_key="subject-allocation", suppressed=suppressed) + else: + with pytest.raises(AssertionError, match="subject-flag"): + assert_metric_activity(metrics, "subject-flag", allocation_key="subject-allocation", suppressed=suppressed) + + +@scenarios.test_the_test +@features.not_reported +@pytest.mark.parametrize( + ("suppressed_count", "allowed_count", "enabled", "passes"), + [ + (0, 3, True, True), + (3, 0, True, False), # Swapped subjects must not satisfy a shared-flag total. + (3, 3, True, False), + (0, 0, True, False), + (3, 3, False, True), # Default-disabled support preserves both subjects. + (0, 3, False, False), + (3, 0, False, False), + ], +) +def test_shared_flag_metric_oracle(suppressed_count: int, allowed_count: int, *, enabled: bool, passes: bool) -> None: + metrics: list[dict[str, Any]] = [ + { + "metric": "feature_flag.evaluations", + "tags": ["feature_flag.key:shared", f"feature_flag.result.allocation_key:{allocation}"], + "points": [[1, count]], + } + for allocation, count in [("suppressed", suppressed_count), ("allowed", allowed_count)] + if count + ] + + def validate() -> None: + assert_metric_activity(metrics, "shared", allocation_key="suppressed", suppressed=enabled) + assert_metric_activity(metrics, "shared", allocation_key="allowed", suppressed=False) + + if passes: + validate() + else: + with pytest.raises(AssertionError, match="shared"): + validate() + + +@scenarios.test_the_test +@features.not_reported +@pytest.mark.parametrize("failure_source", ["remote-config", "worker-request"]) +def test_evaluation_setup_failure_is_deferred(failure_source: str) -> None: + error = TimeoutError(f"{failure_source} timed out") + completed: list[str] = [] + + def fail() -> None: + raise error + + def prepare() -> None: + if failure_source == "worker-request": + with ThreadPoolExecutor(max_workers=1) as executor: + list(executor.map(lambda _: fail(), [None])) + else: + fail() + + # Setup errors must not stop the next setup or its validation. + setup_error = evaluation_setup_error(prepare) + later_error = evaluation_setup_error(lambda: completed.append("later setup")) + assert completed == ["later setup"] + assert_setup_succeeded(later_error) + # Setup properties must survive the harness's JSON replay storage. + restored_error = json.loads(json.dumps(setup_error)) + assert f"TimeoutError: {failure_source} timed out" in restored_error + with pytest.raises(AssertionError, match=failure_source): + assert_setup_succeeded(restored_error) + + +@scenarios.test_the_test +@features.not_reported +def test_evaluation_setup_does_not_swallow_interrupts() -> None: + def interrupt() -> None: + raise KeyboardInterrupt + + with pytest.raises(KeyboardInterrupt): + evaluation_setup_error(interrupt) + + +@scenarios.test_the_test +@features.not_reported +@pytest.mark.parametrize( + ("actual", "expected", "passes"), + [ + ({"opt_out": True}, {"opt_out": True}, True), + ({"opt_out": 1}, {"opt_out": 1}, True), + ({"opt_out": 1}, {"opt_out": True}, False), + ({"opt_out": True}, {"opt_out": 1}, False), + ({"opt_out": "true"}, {"opt_out": "true"}, True), + ({"opt_out": True}, {"opt_out": "true"}, False), + ({"opt_out": None}, {"opt_out": None}, True), + ({}, {"opt_out": None}, True), + ({"opt_out": False}, {"opt_out": None}, False), + ({"opt_out": True}, {"opt_out": None}, False), + ({}, {}, True), + ({"opt_out": True}, {}, False), + ], +) +def test_opt_out_exposure_attribute_oracle(actual: dict[str, Any], expected: dict[str, Any], *, passes: bool) -> None: + if passes: + assert_opt_out_exposure_attribute(actual, expected, "opt_out") + else: + with pytest.raises(AssertionError): + assert_opt_out_exposure_attribute(actual, expected, "opt_out") + + +@scenarios.test_the_test +@features.not_reported +@pytest.mark.parametrize( + ("count", "suppressed", "passes"), [(3, False, True), (0, False, False), (0, True, True), (3, True, False)] +) +def test_opt_out_error_metric_oracle(count: int, *, suppressed: bool, passes: bool) -> None: + metrics: list[dict[str, Any]] = [ + { + "metric": "feature_flag.evaluations", + "tags": ["feature_flag.key:missing-flag", "error.type:flag_not_found"], + "points": [[1, count]], + }, + { + "metric": "feature_flag.evaluations", + "tags": ["feature_flag.key:unrelated-flag"], + "points": [[1, 99]], + }, + ] + if passes: + assert_metric_activity(metrics, "missing-flag", allocation_key=None, suppressed=suppressed) + else: + with pytest.raises(AssertionError, match="missing-flag"): + assert_metric_activity(metrics, "missing-flag", allocation_key=None, suppressed=suppressed) diff --git a/tests/test_the_test/test_group_rules.py b/tests/test_the_test/test_group_rules.py index e3bfd947705..4b21b78596b 100644 --- a/tests/test_the_test/test_group_rules.py +++ b/tests/test_the_test/test_group_rules.py @@ -17,6 +17,18 @@ def test_appsec(): def test_tracer_release(): # make an exclusion list + provisional_opt_out_scenarios = [ + scenarios.feature_flagging_evaluation_telemetry_opt_out, + scenarios.feature_flagging_evaluation_telemetry_opt_out_disabled, + ] + for opt_out_scenario in provisional_opt_out_scenarios: + assert opt_out_scenario.github_workflow == "endtoend" + assert scenario_groups.ffe in opt_out_scenario.scenario_groups + assert opt_out_scenario in scenario_groups.ffe.scenarios + for group in (scenario_groups.all, scenario_groups.end_to_end, scenario_groups.tracer_release): + assert group not in opt_out_scenario.scenario_groups + assert opt_out_scenario not in group.scenarios + dormant_agentless_scenario = scenarios.feature_flagging_and_experimentation_agentless assert dormant_agentless_scenario.include_agent is False assert dormant_agentless_scenario.use_proxy is False @@ -45,6 +57,7 @@ def test_tracer_release(): scenarios.mock_the_test, scenarios.mock_the_test_2, scenarios.test_the_test, + *provisional_opt_out_scenarios, scenarios.todo, # targetting OTEL scenarios.otel_integrations, diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 93bc91c89b7..f8af39d6e0c 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -5,7 +5,7 @@ from utils.tools import update_environ_with_local_env from .aws_lambda import LambdaScenario -from .core import Scenario, scenario_groups +from .core import Scenario, ScenarioGroup, scenario_groups from .default import DefaultScenario from .agentless_endtoend import FeatureFlaggingAgentlessEndToEndScenario from .endtoend import ( @@ -62,6 +62,29 @@ "DD_APP_KEY": "mock_app_key", } +# Shared by the ordinary FFE scenario and both evaluation telemetry opt-out scenarios. +# Both opt-out scenarios also enable span enrichment; one enables the opt-out option. +_FFE_WEBLOG_ENV: dict[str, str | None] = { + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", + # Python's set_provider() waits for Remote Configuration, which arrives only + # after the weblog starts. The OpenFeature API has no nonblocking alternative. + # Fail initialization promptly so the weblog starts; the provider recovers + # when the tests supply Remote Configuration. + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS": "100", + "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", + "DD_METRICS_OTEL_ENABLED": "true", + "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": "http/protobuf", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://agent:4318/v1/metrics", + "OTEL_METRIC_EXPORT_INTERVAL": "1000", +} + + +class _ProvisionalFeatureFlaggingScenario(DdTraceEndToEndScenario): + """Allow explicit FFE runs without scheduling unsupported SDKs in default CI.""" + + _default_scenario_groups: tuple[ScenarioGroup, ...] = () + class _Scenarios: todo = Scenario("TODO", doc="scenario that skips tests not yet executed", github_workflow=None) @@ -878,25 +901,34 @@ class _Scenarios: rc_api_enabled=True, # Allow final EVP batches to reach the backend after the weblog flushes and stops. agent_interface_timeout=15, + weblog_env=_FFE_WEBLOG_ENV, + doc="", + scenario_groups=[scenario_groups.ffe], + ) + + feature_flagging_evaluation_telemetry_opt_out = _ProvisionalFeatureFlaggingScenario( + "FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT", + rc_api_enabled=True, + agent_interface_timeout=15, weblog_env={ - "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", - "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", - # set_provider() in Python blocks until we receive RC - # configuration for feature flags. But it is only sent - # after weblog sucessfully boots and tests start - # executing. Unfortunately, Python's OpenFeature SDK does - # not have "set provider and don't wait," so we reduce the - # timeout here, so that the provider initialization fails - # fast, weblog boots, and provider recovers when we set RC - # configuration later. - "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS": "100", - "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", - "DD_METRICS_OTEL_ENABLED": "true", - "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": "http/protobuf", - "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://agent:4318/v1/metrics", - "OTEL_METRIC_EXPORT_INTERVAL": "1000", + **_FFE_WEBLOG_ENV, + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": "true", + "DD_FEATURE_FLAGS_EVALUATION_TELEMETRY_PER_SUBJECT_OPT_OUT_ENABLED": "true", }, - doc="", + doc="Per-subject evaluation telemetry opt-out, with unchanged assignments and exposures.", + scenario_groups=[scenario_groups.ffe], + ) + + feature_flagging_evaluation_telemetry_opt_out_disabled = _ProvisionalFeatureFlaggingScenario( + "FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT_DISABLED", + rc_api_enabled=True, + agent_interface_timeout=15, + weblog_env={ + **_FFE_WEBLOG_ENV, + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": "true", + # Omit the opt-out option: an attribute must not activate the feature by itself. + }, + doc="Default-disabled control for per-subject evaluation telemetry opt-out.", scenario_groups=[scenario_groups.ffe], ) diff --git a/utils/_features.py b/utils/_features.py index 4a5ce48125b..50d506520b1 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -2727,6 +2727,14 @@ def feature_flags_serial_id_metadata(test_object): """ return _mark_test_object(test_object, feature_id=648, owner=_Owner.ffe) + @staticmethod + def feature_flags_evaluation_telemetry_opt_out(test_object): + """Per-subject evaluation telemetry opt-out + + https://feature-parity.us1.prod.dog/#/?feature=649 + """ + return _mark_test_object(test_object, feature_id=649, owner=_Owner.ffe) + @staticmethod def appsec_extended_data_collection(test_object): """AppSec supports extended data collection including headers and body diff --git a/utils/build/docker/dotnet/poc.Dockerfile b/utils/build/docker/dotnet/poc.Dockerfile index 44884499673..79ae1fb9a4b 100644 --- a/utils/build/docker/dotnet/poc.Dockerfile +++ b/utils/build/docker/dotnet/poc.Dockerfile @@ -1,6 +1,10 @@ FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-app WORKDIR /app +# Pass [version] to pin local SDK packages exactly; a plain version is a minimum. +ARG DD_TRACE_PACKAGE_VERSION=* +ARG DD_OPENFEATURE_PACKAGE_VERSION=* + # Copy binaries folder for local NuGet packages (for testing with local builds) COPY binaries/ /binaries/ RUN if ls /binaries/*.nupkg 1> /dev/null 2>&1; then \ @@ -9,11 +13,15 @@ RUN if ls /binaries/*.nupkg 1> /dev/null 2>&1; then \ # dotnet restore COPY utils/build/docker/dotnet/weblog/app.csproj app.csproj -RUN dotnet restore +RUN dotnet restore \ + -p:DatadogTracePackageVersion="$DD_TRACE_PACKAGE_VERSION" \ + -p:DatadogOpenFeaturePackageVersion="$DD_OPENFEATURE_PACKAGE_VERSION" # dotnet publish COPY utils/build/docker/dotnet/weblog/* . -RUN dotnet publish --no-restore -c Release -f net8.0 -o out +RUN dotnet publish --no-restore -c Release -f net8.0 -o out \ + -p:DatadogTracePackageVersion="$DD_TRACE_PACKAGE_VERSION" \ + -p:DatadogOpenFeaturePackageVersion="$DD_OPENFEATURE_PACKAGE_VERSION" ######### diff --git a/utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs b/utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs index 183cd1bcf9f..5aafeba2ff6 100644 --- a/utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs +++ b/utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs @@ -139,16 +139,22 @@ private static EvaluationContext CreateContext(EvaluateRequest request, string t { foreach (var attr in request.Attributes) { - // System.Text.Json deserializes to JsonElement, not string - var value = attr.Value switch + // Preserve primitive types so boolean context controls stay boolean. + switch (attr.Value) { - JsonElement jsonElement => jsonElement.ValueKind == JsonValueKind.String - ? jsonElement.GetString() - : jsonElement.ToString(), - string s => s, - _ => attr.Value?.ToString() - }; - builder.Set(attr.Key, value); + case JsonElement jsonElement when jsonElement.ValueKind is JsonValueKind.True or JsonValueKind.False: + builder.Set(attr.Key, jsonElement.GetBoolean()); + break; + case JsonElement jsonElement when jsonElement.ValueKind == JsonValueKind.Number: + builder.Set(attr.Key, jsonElement.GetDouble()); + break; + case JsonElement jsonElement when jsonElement.ValueKind == JsonValueKind.String: + builder.Set(attr.Key, jsonElement.GetString()); + break; + default: + builder.Set(attr.Key, attr.Value?.ToString()); + break; + } } } return builder.Build(); diff --git a/utils/build/docker/dotnet/weblog/app.csproj b/utils/build/docker/dotnet/weblog/app.csproj index 07e8575ae29..d141eec0841 100644 --- a/utils/build/docker/dotnet/weblog/app.csproj +++ b/utils/build/docker/dotnet/weblog/app.csproj @@ -8,6 +8,8 @@ false enable false + * + *