From f1a45f875e735cb2a554865f27ee762eced7e9bb Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 15 Sep 2026 15:58:30 +0300 Subject: [PATCH 01/12] feat(ffe): add settings and agentless endpoint resolution --- Cargo.lock | 1 + Cargo.toml | 1 + components-rs/ffe.rs | 4 + components-rs/ffe/settings.rs | 641 +++++++++++++++++++++++++ ext/configuration.h | 1 + metadata/supported-configurations.json | 54 ++- tracer/configuration.h | 6 + 7 files changed, 707 insertions(+), 1 deletion(-) create mode 100644 components-rs/ffe/settings.rs diff --git a/Cargo.lock b/Cargo.lock index 89c8002e20a..5be385cd0ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1298,6 +1298,7 @@ dependencies = [ "tracing", "tracing-core", "tracing-subscriber", + "url", "uuid", "zwohash", ] diff --git a/Cargo.toml b/Cargo.toml index 902e2827919..e35aec88808 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,6 +87,7 @@ serde_json = "1.0.113" regex = "1.10.5" regex-automata = "0.4.5" http = "1.4" +url = "2.5.4" libc = "0.2" bincode = "1.3.3" hashbrown = "0.15" diff --git a/components-rs/ffe.rs b/components-rs/ffe.rs index 4964bc7b0b9..78031bb5bd6 100644 --- a/components-rs/ffe.rs +++ b/components-rs/ffe.rs @@ -8,6 +8,10 @@ use std::cell::RefCell; use std::collections::HashMap; use std::sync::Arc; +// This module is wired into activation by the next change in the FFE stack. +#[allow(dead_code)] +pub(crate) mod settings; + struct FfeState { config: Option, version: u64, diff --git a/components-rs/ffe/settings.rs b/components-rs/ffe/settings.rs new file mode 100644 index 00000000000..07a300acbb9 --- /dev/null +++ b/components-rs/ffe/settings.rs @@ -0,0 +1,641 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::fmt; +use std::time::Duration; +use url::Url; + +pub const DEFAULT_POLL_INTERVAL_SECONDS: i64 = 30; +pub const MAX_POLL_INTERVAL_SECONDS: i64 = 3_600; +pub const DEFAULT_REQUEST_TIMEOUT_SECONDS: i64 = 5; +pub const MAX_REQUEST_TIMEOUT_SECONDS: i64 = 300; +pub const DEFAULT_INITIALIZATION_TIMEOUT_MS: i64 = 10_000; +pub const MAX_INITIALIZATION_TIMEOUT_MS: i64 = i32::MAX as i64; + +const DEFAULT_SITE: &str = "datadoghq.com"; +const AGENTLESS_HOST_PREFIX: &str = "ufc-server.ff-cdn."; +const AGENTLESS_CONFIGURATION_PATH: &str = "/api/v2/feature-flagging/config/rules-based/server"; + +/// The configured source is retained separately from enablement. In particular, +/// `Offline` is a reserved source whose delivery is disabled today, rather than +/// an alias for the absence of a configured source. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ConfigurationSource { + Agentless, + RemoteConfig, + Offline, + Invalid, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DisableReason { + KillSwitch, + InvalidSource, + LegacyDisabled, + Offline, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SourceResolution { + pub enabled: bool, + pub source: ConfigurationSource, + pub disable_reason: Option, + pub legacy_key_decided: bool, +} + +impl SourceResolution { + pub fn delivery_source(self) -> Option { + self.enabled.then_some(self.source) + } +} + +#[derive(Clone, Copy)] +pub struct SourceInput<'a> { + pub enabled: bool, + pub enabled_set: bool, + pub source: &'a str, + pub source_set: bool, + pub legacy_enabled: bool, + pub legacy_enabled_set: bool, +} + +pub fn resolve_source(input: SourceInput<'_>) -> SourceResolution { + let explicit_source = input + .source_set + .then(|| input.source.trim()) + .filter(|s| !s.is_empty()); + + let (mut enabled, source, disable_reason, legacy_key_decided) = + if let Some(source) = explicit_source { + match source.to_ascii_lowercase().as_str() { + "agentless" => (true, ConfigurationSource::Agentless, None, false), + "remote_config" => (true, ConfigurationSource::RemoteConfig, None, false), + "offline" => ( + false, + ConfigurationSource::Offline, + Some(DisableReason::Offline), + false, + ), + _ => ( + false, + ConfigurationSource::Invalid, + Some(DisableReason::InvalidSource), + false, + ), + } + } else if input.enabled_set { + (true, ConfigurationSource::Agentless, None, false) + } else if input.legacy_enabled_set { + if input.legacy_enabled { + (true, ConfigurationSource::RemoteConfig, None, true) + } else { + ( + false, + ConfigurationSource::RemoteConfig, + Some(DisableReason::LegacyDisabled), + true, + ) + } + } else { + (true, ConfigurationSource::Agentless, None, false) + }; + + let disable_reason = if input.enabled_set && !input.enabled { + enabled = false; + Some(DisableReason::KillSwitch) + } else { + disable_reason + }; + + SourceResolution { + enabled, + source, + disable_reason, + legacy_key_decided, + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SettingsIssue { + PollInterval, + RequestTimeout, + InitializationTimeout, +} + +pub struct SettingsInput<'a> { + pub source: SourceInput<'a>, + /// Sensitive: may contain credentials and must never be logged. + pub agentless_base_url: &'a str, + pub poll_interval_seconds: i64, + pub request_timeout_seconds: i64, + pub initialization_timeout_ms: i64, + pub site: &'a str, + /// Sensitive: must never be logged. + pub api_key: &'a str, + pub environment: &'a str, +} + +/// A validated, process-stable settings snapshot. Deliberately does not derive +/// `Debug`, because it owns the API key and potentially credential-bearing URL. +pub struct FeatureFlagsSettings { + pub resolution: SourceResolution, + agentless_base_url: String, + pub poll_interval: Duration, + pub request_timeout: Duration, + pub initialization_timeout: Duration, + site: String, + api_key: String, + environment: String, + pub issues: Vec, +} + +impl FeatureFlagsSettings { + pub fn resolve(input: SettingsInput<'_>) -> Self { + let mut issues = Vec::new(); + let poll_interval_seconds = validated_numeric( + input.poll_interval_seconds, + DEFAULT_POLL_INTERVAL_SECONDS, + MAX_POLL_INTERVAL_SECONDS, + SettingsIssue::PollInterval, + &mut issues, + ); + let request_timeout_seconds = validated_numeric( + input.request_timeout_seconds, + DEFAULT_REQUEST_TIMEOUT_SECONDS, + MAX_REQUEST_TIMEOUT_SECONDS, + SettingsIssue::RequestTimeout, + &mut issues, + ); + let initialization_timeout_ms = validated_numeric( + input.initialization_timeout_ms, + DEFAULT_INITIALIZATION_TIMEOUT_MS, + MAX_INITIALIZATION_TIMEOUT_MS, + SettingsIssue::InitializationTimeout, + &mut issues, + ); + + Self { + resolution: resolve_source(input.source), + agentless_base_url: input.agentless_base_url.to_owned(), + poll_interval: Duration::from_secs(poll_interval_seconds as u64), + request_timeout: Duration::from_secs(request_timeout_seconds as u64), + initialization_timeout: Duration::from_millis(initialization_timeout_ms as u64), + site: input.site.to_owned(), + api_key: input.api_key.to_owned(), + environment: input.environment.to_owned(), + issues, + } + } + + pub fn agentless_endpoint(&self) -> Result { + AgentlessEndpoint::build( + &self.agentless_base_url, + &self.site, + &self.environment, + &self.api_key, + ) + } +} + +fn validated_numeric( + value: i64, + default: i64, + maximum: i64, + issue: SettingsIssue, + issues: &mut Vec, +) -> i64 { + if value > 0 && value <= maximum { + value + } else { + issues.push(issue); + default + } +} + +/// A validated agentless endpoint. Only managed endpoints can own an API key, +/// ensuring a custom endpoint cannot accidentally receive the Datadog credential. +#[derive(Clone, Eq, PartialEq)] +pub struct AgentlessEndpoint { + url: String, + api_key: Option, +} + +impl AgentlessEndpoint { + pub fn build( + base_url: &str, + site: &str, + environment: &str, + api_key: &str, + ) -> Result { + if base_url.trim().is_empty() { + Self::managed(site, environment, api_key) + } else { + Self::custom(base_url) + } + } + + pub fn as_str(&self) -> &str { + &self.url + } + + pub fn is_managed(&self) -> bool { + self.api_key.is_some() + } + + pub fn api_key(&self) -> Option<&str> { + self.api_key.as_deref() + } + + fn managed(site: &str, environment: &str, api_key: &str) -> Result { + if api_key.is_empty() { + return Err(EndpointError::MissingApiKey); + } + + let normalized_site = normalize_site(site)?; + let expected_host = format!("{AGENTLESS_HOST_PREFIX}{normalized_site}"); + let mut url = Url::parse(&format!( + "https://{expected_host}{AGENTLESS_CONFIGURATION_PATH}" + )) + .map_err(|_| EndpointError::InvalidSite)?; + + // Compare the parsed host before this endpoint is allowed to carry + // managed authentication. This is defense in depth after rejecting + // URL-significant site characters in `normalize_site`. + if url.host_str() != Some(expected_host.as_str()) { + return Err(EndpointError::InvalidSite); + } + + if !environment.is_empty() { + url.query_pairs_mut().append_pair("dd_env", environment); + } + + Ok(Self { + url: url.to_string(), + api_key: Some(api_key.to_owned()), + }) + } + + fn custom(base_url: &str) -> Result { + if base_url.chars().any(char::is_whitespace) { + return Err(EndpointError::UrlContainsWhitespace); + } + + let mut url = Url::parse(base_url).map_err(|_| EndpointError::InvalidUrl)?; + if !matches!(url.scheme(), "http" | "https") { + return Err(EndpointError::UnsupportedScheme); + } + if url.host_str().is_none() || url.fragment().is_some() { + return Err(EndpointError::InvalidUrl); + } + + let url = if matches!(url.path(), "" | "/") { + url.set_path(AGENTLESS_CONFIGURATION_PATH); + url.to_string() + } else { + // A custom non-root URL is opaque and must be requested exactly as + // the operator supplied it, including its query string. + base_url.to_owned() + }; + + Ok(Self { url, api_key: None }) + } +} + +impl fmt::Debug for AgentlessEndpoint { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("AgentlessEndpoint") + .field("url", &"[REDACTED]") + .field("managed", &self.is_managed()) + .finish() + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum EndpointError { + MissingApiKey, + InvalidUrl, + UnsupportedScheme, + UrlContainsWhitespace, + InvalidSite, +} + +impl fmt::Display for EndpointError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::MissingApiKey => "no API key is configured for the managed endpoint", + Self::InvalidUrl => "the configured URL is not a valid absolute URL", + Self::UnsupportedScheme => "the configured URL must use HTTP or HTTPS", + Self::UrlContainsWhitespace => "the configured URL contains whitespace", + Self::InvalidSite => "the configured site is not a valid host", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for EndpointError {} + +fn normalize_site(site: &str) -> Result { + let site = if site.trim().is_empty() { + DEFAULT_SITE + } else { + site + }; + if !site.is_ascii() + || site.chars().any(|character| { + character.is_whitespace() || matches!(character, '/' | '\\' | '?' | '#' | '@' | ':') + }) + { + return Err(EndpointError::InvalidSite); + } + Ok(site.to_ascii_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn source_input() -> SourceInput<'static> { + SourceInput { + enabled: true, + enabled_set: false, + source: "", + source_set: false, + legacy_enabled: false, + legacy_enabled_set: false, + } + } + + #[test] + fn source_precedence_matches_the_cross_sdk_contract() { + let cases = [ + ( + source_input(), + true, + ConfigurationSource::Agentless, + None, + false, + ), + ( + SourceInput { + enabled: false, + enabled_set: true, + source: "remote_config", + source_set: true, + legacy_enabled: true, + legacy_enabled_set: true, + }, + false, + ConfigurationSource::RemoteConfig, + Some(DisableReason::KillSwitch), + false, + ), + ( + SourceInput { + source: " ReMoTe_CoNfIg ", + source_set: true, + legacy_enabled: false, + legacy_enabled_set: true, + ..source_input() + }, + true, + ConfigurationSource::RemoteConfig, + None, + false, + ), + ( + SourceInput { + source: "offline", + source_set: true, + ..source_input() + }, + false, + ConfigurationSource::Offline, + Some(DisableReason::Offline), + false, + ), + ( + SourceInput { + source: "typo", + source_set: true, + ..source_input() + }, + false, + ConfigurationSource::Invalid, + Some(DisableReason::InvalidSource), + false, + ), + ( + SourceInput { + enabled: true, + enabled_set: true, + legacy_enabled: false, + legacy_enabled_set: true, + ..source_input() + }, + true, + ConfigurationSource::Agentless, + None, + false, + ), + ( + SourceInput { + legacy_enabled: true, + legacy_enabled_set: true, + ..source_input() + }, + true, + ConfigurationSource::RemoteConfig, + None, + true, + ), + ( + SourceInput { + legacy_enabled: false, + legacy_enabled_set: true, + ..source_input() + }, + false, + ConfigurationSource::RemoteConfig, + Some(DisableReason::LegacyDisabled), + true, + ), + ]; + + for (input, enabled, source, disable_reason, legacy_key_decided) in cases { + let resolution = resolve_source(input); + assert_eq!(resolution.enabled, enabled); + assert_eq!(resolution.source, source); + assert_eq!(resolution.disable_reason, disable_reason); + assert_eq!(resolution.legacy_key_decided, legacy_key_decided); + } + } + + #[test] + fn blank_explicit_source_is_semantically_unset() { + for source in ["", " ", "\t\r\n"] { + let resolution = resolve_source(SourceInput { + source, + source_set: true, + legacy_enabled: true, + legacy_enabled_set: true, + ..source_input() + }); + assert_eq!( + resolution.delivery_source(), + Some(ConfigurationSource::RemoteConfig) + ); + assert!(resolution.legacy_key_decided); + } + } + + #[test] + fn invalid_numeric_values_use_defaults_and_report_issues() { + let settings = FeatureFlagsSettings::resolve(SettingsInput { + source: source_input(), + agentless_base_url: "", + poll_interval_seconds: 3_601, + request_timeout_seconds: 0, + initialization_timeout_ms: i64::MAX, + site: "", + api_key: "key", + environment: "", + }); + + assert_eq!(settings.poll_interval, Duration::from_secs(30)); + assert_eq!(settings.request_timeout, Duration::from_secs(5)); + assert_eq!(settings.initialization_timeout, Duration::from_secs(10)); + assert_eq!( + settings.issues, + vec![ + SettingsIssue::PollInterval, + SettingsIssue::RequestTimeout, + SettingsIssue::InitializationTimeout, + ] + ); + } + + #[test] + fn managed_endpoint_uses_normalized_site_and_encoded_environment() { + let endpoint = AgentlessEndpoint::build("", "DATADOGHQ.EU", "prod / blue", "key") + .expect("managed endpoint"); + + assert!(endpoint.is_managed()); + assert_eq!(endpoint.api_key(), Some("key")); + assert_eq!( + endpoint.as_str(), + "https://ufc-server.ff-cdn.datadoghq.eu/api/v2/feature-flagging/config/rules-based/server?dd_env=prod+%2F+blue" + ); + } + + #[test] + fn managed_endpoint_omits_an_absent_environment_and_defaults_blank_site() { + let endpoint = AgentlessEndpoint::build("", " ", "", "key").expect("managed endpoint"); + assert_eq!( + endpoint.as_str(), + "https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server" + ); + } + + #[test] + fn managed_endpoint_requires_an_api_key() { + assert_eq!( + AgentlessEndpoint::build("", "datadoghq.com", "", "").unwrap_err(), + EndpointError::MissingApiKey + ); + } + + #[test] + fn managed_endpoint_rejects_sites_that_can_change_the_url_meaning() { + for site in [ + "https://datadoghq.com", + "datadoghq.com@attacker.example", + "datadoghq.com/path", + "datadoghq.com?query", + "datadoghq.com#fragment", + "datadoghq.com:443", + " datadoghq.com", + "data doghq.com", + "dâtadoghq.com", + ] { + assert_eq!( + AgentlessEndpoint::build("", site, "", "key").unwrap_err(), + EndpointError::InvalidSite, + "site should be rejected" + ); + } + } + + #[test] + fn custom_root_endpoint_gets_the_canonical_path_and_keeps_its_query() { + for base_url in ["http://localhost:8080", "http://localhost:8080/"] { + let endpoint = AgentlessEndpoint::build(base_url, "ignored", "ignored", "secret") + .expect("custom endpoint"); + assert!(!endpoint.is_managed()); + assert_eq!( + endpoint.as_str(), + "http://localhost:8080/api/v2/feature-flagging/config/rules-based/server" + ); + } + + let endpoint = AgentlessEndpoint::build( + "https://user:pass@localhost/?tenant=a%2Fb", + "ignored", + "ignored", + "secret", + ) + .expect("custom endpoint"); + assert_eq!( + endpoint.as_str(), + "https://user:pass@localhost/api/v2/feature-flagging/config/rules-based/server?tenant=a%2Fb" + ); + } + + #[test] + fn custom_non_root_endpoint_is_preserved_without_environment() { + let endpoint = AgentlessEndpoint::build( + "HTTPS://LOCALHOST/custom/path?tenant=a%2Fb", + "ignored", + "must-not-be-added", + "must-not-be-sent", + ) + .expect("custom endpoint"); + assert_eq!( + endpoint.as_str(), + "HTTPS://LOCALHOST/custom/path?tenant=a%2Fb" + ); + assert!(!endpoint.is_managed()); + assert_eq!(endpoint.api_key(), None); + } + + #[test] + fn custom_endpoint_validation_errors_never_contain_the_url() { + let credential = "credential-that-must-not-leak"; + for (url, expected) in [ + ( + format!("ftp://{credential}@localhost/path"), + EndpointError::UnsupportedScheme, + ), + ( + format!("https://{credential}@local host/path"), + EndpointError::UrlContainsWhitespace, + ), + (format!("not-a-url-{credential}"), EndpointError::InvalidUrl), + ] { + let error = AgentlessEndpoint::build(&url, "", "", "key").unwrap_err(); + assert_eq!(error, expected); + assert!(!error.to_string().contains(credential)); + } + } + + #[test] + fn endpoint_debug_output_is_redacted() { + let endpoint = + AgentlessEndpoint::build("https://user:password@localhost/custom", "", "", "key") + .expect("custom endpoint"); + let debug = format!("{endpoint:?}"); + assert!(!debug.contains("user")); + assert!(!debug.contains("password")); + assert!(debug.contains("[REDACTED]")); + } +} diff --git a/ext/configuration.h b/ext/configuration.h index 667c9c49766..c1b06ff3818 100644 --- a/ext/configuration.h +++ b/ext/configuration.h @@ -46,6 +46,7 @@ enum datadog_sidecar_connection_mode { CONFIG(STRING, DD_DOGSTATSD_URL, "http://localhost:8125") \ CONFIG(STRING, DD_DOGSTATSD_HOST, "localhost") \ CONFIG(STRING, DD_API_KEY, "", .ini_change = zai_config_system_ini_change, .sensitive = true) \ + CONFIG(STRING, DD_SITE, "datadoghq.com") \ CONFIG(INT, DD_DOGSTATSD_PORT, "8125") \ CONFIG(STRING, DD_ENV, "", .ini_change = datadog_alter_dd_env, \ .env_config_fallback = ddtrace_conf_otel_resource_attributes_env) \ diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index b36ce1f4782..bc1990e22aa 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -397,6 +397,13 @@ "default": "false" } ], + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS": [ + { + "implementation": "B", + "type": "int", + "default": "10000" + } + ], "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": [ { "implementation": "A", @@ -411,6 +418,42 @@ "default": "true" } ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": [ + { + "implementation": "A", + "type": "string", + "default": "agentless" + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": [ + { + "implementation": "A", + "type": "string", + "default": "", + "sensitive": true + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": [ + { + "implementation": "A", + "type": "int", + "default": "30" + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": [ + { + "implementation": "A", + "type": "int", + "default": "5" + } + ], + "DD_FEATURE_FLAGS_ENABLED": [ + { + "implementation": "A", + "type": "boolean", + "default": "true" + } + ], "DD_GIT_COMMIT_SHA": [ { "implementation": "B", @@ -652,6 +695,13 @@ "default": "" } ], + "DD_SITE": [ + { + "implementation": "C", + "type": "string", + "default": "datadoghq.com" + } + ], "DD_SPAN_SAMPLING_RULES": [ { "implementation": "A", @@ -2869,5 +2919,7 @@ } ] }, - "deprecations": {} + "deprecations": { + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "DD_FEATURE_FLAGS_ENABLED" + } } diff --git a/tracer/configuration.h b/tracer/configuration.h index 892cdce0715..7fb3a1c06a4 100644 --- a/tracer/configuration.h +++ b/tracer/configuration.h @@ -163,7 +163,13 @@ CONFIG(BOOL, DD_TRACE_RESOURCE_RENAMING_ENABLED, "false") \ CONFIG(BOOL, DD_TRACE_RESOURCE_RENAMING_ALWAYS_SIMPLIFIED_ENDPOINT, "false") \ CONFIG(BOOL, DD_TRACE_STATS_COMPUTATION_ENABLED, "false") \ + CONFIG(BOOL, DD_FEATURE_FLAGS_ENABLED, "true") \ + CONFIG(STRING, DD_FEATURE_FLAGS_CONFIGURATION_SOURCE, "agentless") \ + CONFIG(STRING, DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, "", .sensitive = true) \ + CONFIG(INT, DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, "30") \ + CONFIG(INT, DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS, "5") \ CONFIG(BOOL, DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED, "false") \ + CONFIG(INT, DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS, "10000") \ DD_INTEGRATIONS #ifndef DDTRACE_CONFIGURATION From 9f17928ce8122768e73e783e3fdf494e221d3638 Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 15 Sep 2026 17:10:46 +0300 Subject: [PATCH 02/12] feat(ffe): add process-wide state and agentless poller --- Cargo.lock | 10 + Cargo.toml | 12 +- components-rs/ffe.rs | 162 ++-- components-rs/ffe/agentless.rs | 951 ++++++++++++++++++++++++ zend_abstract_interface/config/config.h | 4 +- 5 files changed, 1095 insertions(+), 44 deletions(-) create mode 100644 components-rs/ffe/agentless.rs diff --git a/Cargo.lock b/Cargo.lock index 5be385cd0ed..860062bb5e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1237,6 +1237,8 @@ name = "datadog-php" version = "0.0.1" dependencies = [ "anyhow", + "arc-swap", + "base64 0.22.1", "bincode", "bindgen 0.72.1", "cbindgen 0.27.0", @@ -1253,8 +1255,13 @@ dependencies = [ "ddtrace-sidecar", "dynasmrt", "env_logger", + "fastrand", + "flate2", + "futures-util", "hashbrown 0.15.2", "http 1.4.2", + "http-body-util", + "hyper", "itertools 0.11.0", "lazy_static", "libc 0.2.186", @@ -1281,6 +1288,7 @@ dependencies = [ "log", "mach2", "paste", + "percent-encoding", "perfcnt", "rand 0.9.4", "rand_distr", @@ -1295,6 +1303,8 @@ dependencies = [ "static_assertions", "tempfile", "thiserror 2.0.18", + "tokio", + "tokio-util", "tracing", "tracing-core", "tracing-subscriber", diff --git a/Cargo.toml b/Cargo.toml index e35aec88808..36b3098362c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ tracing-subscriber = ["tracing"] trigger_time_sample = ["profiling"] [dependencies] -libdd-common = { path = "libdatadog/libdd-common", features = ["cgroup_testing"] } +libdd-common = { path = "libdatadog/libdd-common", features = ["cgroup_testing", "hyper-proxy"] } libdd-common-ffi = { path = "libdatadog/libdd-common-ffi", default-features = false } libdd-telemetry = { path = "libdatadog/libdd-telemetry" } libdd-telemetry-ffi = { path = "libdatadog/libdd-telemetry-ffi", default-features = false } @@ -68,7 +68,14 @@ libdd-crashtracker-ffi = { path = "libdatadog/libdd-crashtracker-ffi", default-f libdd-library-config-ffi = { path = "libdatadog/libdd-library-config-ffi", default-features = false } spawn_worker = { path = "libdatadog/spawn_worker" } anyhow = "1.0" +arc-swap = "1.7.1" +base64 = "0.22" const-str = "0.5.6" +fastrand = "2" +flate2 = { version = "1.0", default-features = false, features = ["rust_backend"] } +futures-util = "0.3" +http-body-util = "0.1" +hyper = { version = "1.6", features = ["client", "http1"] } itertools = "0.11.0" serde = "1.0.196" simd-json = "0.14.1" @@ -79,10 +86,13 @@ env_logger = "0.11" zwohash = "0.1.2" uuid = { version = "1.0", features = ["v4"] } paste = "1" +percent-encoding = "2.1" tempfile = "3.3" tracing = { version = "0.1", default-features = false, features = ["std", "attributes"] } tracing-core = "0.1.32" tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "fmt", "env-filter", "smallvec"] } +tokio = { version = "1.36", features = ["macros", "rt", "sync", "time"] } +tokio-util = "0.7.11" serde_json = "1.0.113" regex = "1.10.5" regex-automata = "0.4.5" diff --git a/components-rs/ffe.rs b/components-rs/ffe.rs index 78031bb5bd6..a5a8493c3b1 100644 --- a/components-rs/ffe.rs +++ b/components-rs/ffe.rs @@ -1,43 +1,87 @@ use crate::bytes::MaybeOwnedZendString; +use arc_swap::ArcSwapOption; use datadog_ffe::rules_based::{ self as ffe, AssignmentReason, AssignmentValue, Attribute, Configuration, EvaluationContext, EvaluationError, ExpectedFlagType, Str, UniversalFlagConfig, }; use libdd_common_ffi::slice::{AsBytes, CharSlice}; -use std::cell::RefCell; use std::collections::HashMap; -use std::sync::Arc; +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::{Arc, LazyLock}; +// This module is wired into activation by the next change in the FFE stack. +#[allow(dead_code)] +pub(crate) mod agentless; // This module is wired into activation by the next change in the FFE stack. #[allow(dead_code)] pub(crate) mod settings; -struct FfeState { - config: Option, - version: u64, +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u8)] +#[allow(dead_code)] // Remaining states are connected by PR 3 activation wiring. +pub(crate) enum DeliveryState { + Inactive, + Starting, + Ready, + Stale, + Stopped, + PermanentError, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ConfigurationTransition { + Ready, + Recovered, + Changed, + Stale, + Unchanged, +} + +static FFE_CONFIG: LazyLock> = + LazyLock::new(|| ArcSwapOption::new(None)); +static FFE_VERSION: AtomicU64 = AtomicU64::new(0); +static DELIVERY_STATE: AtomicU8 = AtomicU8::new(DeliveryState::Inactive as u8); + +#[allow(dead_code)] // Connected by PR 3 activation wiring. +pub(crate) fn delivery_state() -> DeliveryState { + match DELIVERY_STATE.load(Ordering::Acquire) { + value if value == DeliveryState::Starting as u8 => DeliveryState::Starting, + value if value == DeliveryState::Ready as u8 => DeliveryState::Ready, + value if value == DeliveryState::Stale as u8 => DeliveryState::Stale, + value if value == DeliveryState::Stopped as u8 => DeliveryState::Stopped, + value if value == DeliveryState::PermanentError as u8 => DeliveryState::PermanentError, + _ => DeliveryState::Inactive, + } } -thread_local! { - static FFE_STATE: RefCell = const { RefCell::new(FfeState { - config: None, - version: 0, - }) }; +#[allow(dead_code)] // Connected by PR 3 activation wiring. +pub(crate) fn set_delivery_state(state: DeliveryState) { + DELIVERY_STATE.store(state as u8, Ordering::Release); } -pub fn store_config(config: Configuration) { - FFE_STATE.with(|state| { - let mut state = state.borrow_mut(); - state.config = Some(config); - state.version = state.version.wrapping_add(1); - }); +pub(crate) fn store_config(config: Configuration) -> ConfigurationTransition { + FFE_CONFIG.store(Some(Arc::new(config))); + FFE_VERSION.fetch_add(1, Ordering::AcqRel); + + let previous = DELIVERY_STATE.swap(DeliveryState::Ready as u8, Ordering::AcqRel); + if previous == DeliveryState::Ready as u8 { + ConfigurationTransition::Changed + } else if previous == DeliveryState::Stale as u8 { + ConfigurationTransition::Recovered + } else { + ConfigurationTransition::Ready + } } -pub fn clear_config() { - FFE_STATE.with(|state| { - let mut state = state.borrow_mut(); - state.config = None; - state.version = state.version.wrapping_add(1); - }); +pub(crate) fn clear_config() -> ConfigurationTransition { + FFE_CONFIG.store(None); + FFE_VERSION.fetch_add(1, Ordering::AcqRel); + let previous = DELIVERY_STATE.swap(DeliveryState::Stale as u8, Ordering::AcqRel); + if previous == DeliveryState::Stale as u8 { + ConfigurationTransition::Unchanged + } else { + ConfigurationTransition::Stale + } } #[no_mangle] @@ -62,12 +106,12 @@ pub extern "C" fn ddog_ffe_load_config(json: CharSlice<'_>) -> bool { #[no_mangle] pub extern "C" fn ddog_ffe_has_config() -> bool { - FFE_STATE.with(|state| state.borrow().config.is_some()) + FFE_CONFIG.load().is_some() } #[no_mangle] pub extern "C" fn ddog_ffe_config_version() -> u64 { - FFE_STATE.with(|state| state.borrow().version) + FFE_VERSION.load(Ordering::Acquire) } const REASON_STATIC: i32 = 0; @@ -160,18 +204,16 @@ pub extern "C" fn ddog_ffe_evaluate( let attributes = parse_attributes(attributes, attributes_count); let context = EvaluationContext::new(targeting_key, Arc::new(attributes)); - FFE_STATE.with(|state| { - let state = state.borrow(); - let assignment = ffe::get_assignment( - state.config.as_ref(), - flag_key, - &context, - expected_type, - ffe::now(), - ); + let config = FFE_CONFIG.load(); + let assignment = ffe::get_assignment( + config.as_deref(), + flag_key, + &context, + expected_type, + ffe::now(), + ); - result_from_assignment(assignment) - }) + result_from_assignment(assignment) } fn parse_attributes( @@ -302,9 +344,10 @@ mod tests { use std::mem; use std::ptr; use std::ptr::NonNull; - use std::sync::Once; + use std::sync::{Mutex, Once}; static INIT_ZEND_STRING_FUNCTIONS: Once = Once::new(); + static FFE_TEST_LOCK: Mutex<()> = Mutex::new(()); fn setup_zend_string_functions() { INIT_ZEND_STRING_FUNCTIONS.call_once(|| unsafe { @@ -410,6 +453,7 @@ mod tests { #[test] fn empty_targeting_key_is_not_dropped() { + let _guard = FFE_TEST_LOCK.lock().expect("FFE test lock"); setup_zend_string_functions(); clear_config(); let config = @@ -438,7 +482,8 @@ mod tests { } #[test] - fn configuration_state_is_thread_local() { + fn configuration_state_is_process_wide() { + let _guard = FFE_TEST_LOCK.lock().expect("FFE test lock"); clear_config(); let empty_version = ddog_ffe_config_version(); assert!(!ddog_ffe_has_config()); @@ -448,19 +493,52 @@ mod tests { let loaded_version = ddog_ffe_config_version(); assert_eq!(loaded_version, empty_version.wrapping_add(1)); - let child = std::thread::spawn(|| { - assert!(!ddog_ffe_has_config()); - assert_eq!(ddog_ffe_config_version(), 0); + let child = std::thread::spawn(move || { + assert!(ddog_ffe_has_config()); + assert_eq!(ddog_ffe_config_version(), loaded_version); assert!(load_empty_config()); assert!(ddog_ffe_has_config()); - assert_eq!(ddog_ffe_config_version(), 1); + assert_eq!(ddog_ffe_config_version(), loaded_version.wrapping_add(1)); }); child.join().expect("child thread should not panic"); assert!(ddog_ffe_has_config()); - assert_eq!(ddog_ffe_config_version(), loaded_version); + assert_eq!(ddog_ffe_config_version(), loaded_version.wrapping_add(1)); + clear_config(); + } + + #[test] + fn configuration_transitions_track_ready_stale_and_recovery() { + let _guard = FFE_TEST_LOCK.lock().expect("FFE test lock"); + set_delivery_state(DeliveryState::Inactive); + clear_config(); + assert_eq!(delivery_state(), DeliveryState::Stale); + assert_eq!(clear_config(), ConfigurationTransition::Unchanged); + + let json = UniversalFlagConfig::from_json(EMPTY_CONFIG.as_bytes().to_vec()) + .expect("test configuration"); + assert_eq!( + store_config(Configuration::from_server_response(json)), + ConfigurationTransition::Recovered + ); + assert_eq!(delivery_state(), DeliveryState::Ready); + + let json = UniversalFlagConfig::from_json(EMPTY_CONFIG.as_bytes().to_vec()) + .expect("test configuration"); + assert_eq!( + store_config(Configuration::from_server_response(json)), + ConfigurationTransition::Changed + ); + + set_delivery_state(DeliveryState::Starting); + let json = UniversalFlagConfig::from_json(EMPTY_CONFIG.as_bytes().to_vec()) + .expect("test configuration"); + assert_eq!( + store_config(Configuration::from_server_response(json)), + ConfigurationTransition::Ready + ); clear_config(); } } diff --git a/components-rs/ffe/agentless.rs b/components-rs/ffe/agentless.rs new file mode 100644 index 00000000000..1fe4663b676 --- /dev/null +++ b/components-rs/ffe/agentless.rs @@ -0,0 +1,951 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use super::settings::AgentlessEndpoint; +use super::{store_config, ConfigurationTransition}; +use crate::log::{self, Log}; +use base64::Engine; +use datadog_ffe::rules_based::{Configuration, UniversalFlagConfig}; +use flate2::read::GzDecoder; +use futures_util::FutureExt; +use http::header::{ACCEPT_ENCODING, CONTENT_ENCODING, CONTENT_LENGTH, ETAG, IF_NONE_MATCH}; +use http::{Method, Request}; +use http_body_util::BodyExt; +use libdd_common::http_common::{self, Body}; +use percent_encoding::percent_decode_str; +use std::future::Future; +use std::io::Read; +use std::panic::AssertUnwindSafe; +use std::pin::Pin; +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::thread::JoinHandle; +use std::time::Duration; +use tokio_util::sync::CancellationToken; + +const MAX_POLL_ATTEMPTS: usize = 3; +const MAX_RESPONSE_BODY_BYTES: usize = 10 << 20; +const CLIENT_LANGUAGE: &str = "php"; +const INTERNAL_UNTRACED_HEADER: &str = "DD-Internal-Untraced-Request"; +const API_KEY_HEADER: &str = "DD-API-KEY"; +const AUTHORIZATION_HEADER: &str = "Authorization"; + +type FetchFuture<'a> = + Pin> + Send + 'a>>; + +trait Transport: Send + Sync { + fn get<'a>( + &'a self, + endpoint: &'a AgentlessEndpoint, + timeout: Duration, + etag: Option<&'a str>, + cancellation: &'a CancellationToken, + ) -> FetchFuture<'a>; +} + +trait ConfigurationSink: Send + Sync { + fn apply(&self, configuration: Configuration) -> Result; +} + +struct GlobalConfigurationSink; + +impl ConfigurationSink for GlobalConfigurationSink { + fn apply(&self, configuration: Configuration) -> Result { + Ok(store_config(configuration)) + } +} + +#[derive(Debug, Eq, PartialEq)] +enum TransportFailure { + BuildRequest, + Cancelled, + TimedOut, + Request, + ResponseBody, + ResponseTooLarge, +} + +struct PollResponse { + status: u16, + etag: Option, + content_encoding: Option, + body: Vec, +} + +struct HyperTransport { + client: libdd_common::HttpClient, +} + +impl HyperTransport { + fn new() -> Self { + Self { + // This native client is outside PHP HTTP instrumentation, so it + // cannot recursively trace its own polling requests. Hyper also + // leaves 3xx responses visible instead of following redirects. + client: http_common::new_client_periodic(), + } + } + + async fn get_inner( + &self, + endpoint: &AgentlessEndpoint, + timeout: Duration, + etag: Option<&str>, + cancellation: &CancellationToken, + ) -> Result { + let request = build_request(endpoint, etag)?; + let operation = async { + let response = self + .client + .request(request) + .await + .map_err(|_| TransportFailure::Request)?; + let status = response.status().as_u16(); + let etag = header_value(response.headers(), ETAG); + let content_encoding = header_value(response.headers(), CONTENT_ENCODING); + + // A non-200 body can never contain configuration. Dropping it here + // also prevents a malicious error response from consuming memory. + if status != 200 { + return Ok(PollResponse { + status, + etag, + content_encoding, + body: Vec::new(), + }); + } + + if response + .headers() + .get(CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_some_and(|length| length > MAX_RESPONSE_BODY_BYTES as u64) + { + return Err(TransportFailure::ResponseTooLarge); + } + + let body = collect_bounded(response.into_body(), MAX_RESPONSE_BODY_BYTES).await?; + Ok(PollResponse { + status, + etag, + content_encoding, + body, + }) + }; + + tokio::select! { + _ = cancellation.cancelled() => Err(TransportFailure::Cancelled), + result = tokio::time::timeout(timeout, operation) => { + result.map_err(|_| TransportFailure::TimedOut)? + } + } + } +} + +impl Transport for HyperTransport { + fn get<'a>( + &'a self, + endpoint: &'a AgentlessEndpoint, + timeout: Duration, + etag: Option<&'a str>, + cancellation: &'a CancellationToken, + ) -> FetchFuture<'a> { + Box::pin(self.get_inner(endpoint, timeout, etag, cancellation)) + } +} + +fn build_request( + endpoint: &AgentlessEndpoint, + etag: Option<&str>, +) -> Result, TransportFailure> { + let (request_url, authorization) = request_url_and_authorization(endpoint.as_str())?; + let mut builder = Request::builder() + .method(Method::GET) + .uri(request_url) + .header(ACCEPT_ENCODING, "gzip") + .header("DD-Client-Library-Language", CLIENT_LANGUAGE) + .header( + "DD-Client-Library-Version", + include_str!("../../VERSION").trim(), + ) + .header(INTERNAL_UNTRACED_HEADER, "1"); + + if let Some(etag) = etag { + builder = builder.header(IF_NONE_MATCH, etag); + } + if let Some(api_key) = endpoint.api_key() { + builder = builder.header(API_KEY_HEADER, api_key); + } + if let Some(authorization) = authorization { + builder = builder.header(AUTHORIZATION_HEADER, authorization); + } + + builder + .body(Body::empty()) + .map_err(|_| TransportFailure::BuildRequest) +} + +fn request_url_and_authorization( + endpoint: &str, +) -> Result<(String, Option), TransportFailure> { + let mut url = url::Url::parse(endpoint).map_err(|_| TransportFailure::BuildRequest)?; + if url.username().is_empty() { + return Ok((endpoint.to_owned(), None)); + } + + let mut credentials = percent_decode_str(url.username()).collect::>(); + credentials.push(b':'); + if let Some(password) = url.password() { + credentials.extend(percent_decode_str(password)); + } + let authorization = format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode(credentials) + ); + + url.set_username("") + .map_err(|_| TransportFailure::BuildRequest)?; + url.set_password(None) + .map_err(|_| TransportFailure::BuildRequest)?; + Ok((url.to_string(), Some(authorization))) +} + +fn header_value(headers: &http::HeaderMap, name: http::header::HeaderName) -> Option { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) +} + +async fn collect_bounded( + mut body: hyper::body::Incoming, + limit: usize, +) -> Result, TransportFailure> { + let mut bytes = Vec::new(); + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|_| TransportFailure::ResponseBody)?; + if let Some(data) = frame.data_ref() { + let remaining = limit.saturating_sub(bytes.len()); + if data.len() > remaining { + return Err(TransportFailure::ResponseTooLarge); + } + bytes.extend_from_slice(data); + } + } + Ok(bytes) +} + +fn decode_response_body( + body: &[u8], + content_encoding: Option<&str>, + limit: usize, +) -> Result, TransportFailure> { + if !content_encoding.is_some_and(|value| value.trim().eq_ignore_ascii_case("gzip")) { + if body.len() > limit { + return Err(TransportFailure::ResponseTooLarge); + } + return Ok(body.to_vec()); + } + + let decoder = GzDecoder::new(body); + let mut decoded = Vec::new(); + decoder + .take(limit as u64 + 1) + .read_to_end(&mut decoded) + .map_err(|_| TransportFailure::ResponseBody)?; + if decoded.len() > limit { + return Err(TransportFailure::ResponseTooLarge); + } + Ok(decoded) +} + +#[derive(Clone, Copy)] +#[repr(u8)] +enum WarningCategory { + Authentication = 1 << 0, + RetryableHttp = 1 << 1, + UnexpectedHttp = 1 << 2, + Request = 1 << 3, + MalformedPayload = 1 << 4, + ApplyFailure = 1 << 5, + UnexpectedException = 1 << 6, +} + +#[derive(Default)] +struct WarningSlots(AtomicU8); + +impl WarningSlots { + fn take(&self, category: WarningCategory) -> bool { + self.0.fetch_or(category as u8, Ordering::Relaxed) & category as u8 == 0 + } + + fn reset(&self) { + self.0.store(0, Ordering::Relaxed); + } +} + +#[derive(Default)] +struct PollState { + etag: Mutex>, + warnings: WarningSlots, + poll_count: AtomicU64, +} + +impl PollState { + fn etag(&self) -> Option { + lock_unpoisoned(&self.etag).clone() + } + + fn set_etag(&self, etag: Option) { + *lock_unpoisoned(&self.etag) = etag; + } +} + +fn lock_unpoisoned(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PollOutcome { + Success, + Retryable, + Stop, + Cancelled, +} + +struct Poller { + endpoint: AgentlessEndpoint, + poll_interval: Duration, + request_timeout: Duration, + transport: T, + sink: S, + state: Arc, + retry_delay: fn(Duration, usize, f64) -> Duration, +} + +impl Poller { + async fn run(&mut self, cancellation: CancellationToken) { + loop { + if cancellation.is_cancelled() { + return; + } + + let poll = AssertUnwindSafe(self.poll(&cancellation)).catch_unwind(); + match poll.await { + Ok(PollOutcome::Cancelled) => return, + Ok(_) => {} + Err(_) => self.warn( + WarningCategory::UnexpectedException, + "Feature Flags agentless polling failed unexpectedly; polling continues", + ), + } + + tokio::select! { + _ = cancellation.cancelled() => return, + _ = tokio::time::sleep(self.poll_interval) => {} + } + } + } + + async fn poll(&mut self, cancellation: &CancellationToken) -> PollOutcome { + for attempt in 1..=MAX_POLL_ATTEMPTS { + let outcome = self.poll_once(cancellation).await; + if outcome != PollOutcome::Retryable || attempt == MAX_POLL_ATTEMPTS { + return outcome; + } + + let delay = (self.retry_delay)(self.poll_interval, attempt, fastrand::f64()); + tokio::select! { + _ = cancellation.cancelled() => return PollOutcome::Cancelled, + _ = tokio::time::sleep(delay) => {} + } + } + PollOutcome::Stop + } + + async fn poll_once(&mut self, cancellation: &CancellationToken) -> PollOutcome { + if cancellation.is_cancelled() { + return PollOutcome::Cancelled; + } + + self.state.poll_count.fetch_add(1, Ordering::Relaxed); + let etag = self.state.etag(); + match self + .transport + .get( + &self.endpoint, + self.request_timeout, + etag.as_deref(), + cancellation, + ) + .await + { + Ok(response) => self.process_response(response), + Err(TransportFailure::Cancelled) => PollOutcome::Cancelled, + Err(TransportFailure::BuildRequest) => { + self.warn( + WarningCategory::Request, + "Feature Flags agentless request could not be built", + ); + PollOutcome::Stop + } + Err(TransportFailure::ResponseTooLarge) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless response exceeds the 10 MiB limit; keeping the last known configuration", + ); + PollOutcome::Stop + } + Err(_) => { + self.warn( + WarningCategory::Request, + "Feature Flags agentless request failed; keeping the last known configuration", + ); + PollOutcome::Retryable + } + } + } + + fn process_response(&mut self, response: PollResponse) -> PollOutcome { + match response.status { + 304 => return PollOutcome::Success, + 401 | 403 => { + self.warn( + WarningCategory::Authentication, + "Feature Flags agentless endpoint rejected authentication; verify endpoint authentication", + ); + return PollOutcome::Stop; + } + 200 => {} + status if is_retryable_status(status) => { + self.warn( + WarningCategory::RetryableHttp, + &format!("Feature Flags agentless endpoint returned retryable HTTP {status}"), + ); + return PollOutcome::Retryable; + } + status => { + self.warn( + WarningCategory::UnexpectedHttp, + &format!("Feature Flags agentless endpoint returned unexpected HTTP {status}"), + ); + return PollOutcome::Stop; + } + } + + let body = match decode_response_body( + &response.body, + response.content_encoding.as_deref(), + MAX_RESPONSE_BODY_BYTES, + ) { + Ok(body) => body, + Err(TransportFailure::ResponseTooLarge) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless decoded response exceeds the 10 MiB limit; keeping the last known configuration", + ); + return PollOutcome::Stop; + } + Err(_) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless endpoint returned an invalid response body", + ); + return PollOutcome::Stop; + } + }; + + let configuration = match UniversalFlagConfig::from_json(body) { + Ok(configuration) => Configuration::from_server_response(configuration), + Err(_) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless endpoint returned malformed configuration", + ); + return PollOutcome::Stop; + } + }; + + if self.sink.apply(configuration).is_err() { + self.warn( + WarningCategory::ApplyFailure, + "Feature Flags agentless configuration could not be applied", + ); + return PollOutcome::Stop; + } + + // The response has now been parsed and applied. A blank or absent + // ETag intentionally clears an older value. + self.state.set_etag(response.etag); + PollOutcome::Success + } + + fn warn(&self, category: WarningCategory, message: &str) { + if self.state.warnings.take(category) { + log::log(Log::Warn, message); + } + } +} + +fn is_retryable_status(status: u16) -> bool { + matches!(status, 408 | 429 | 500..=599) +} + +fn retry_delay(poll_interval: Duration, attempt: usize, random: f64) -> Duration { + let base = if attempt <= 1 { + (poll_interval / 6).clamp(Duration::from_secs(2), Duration::from_secs(10)) + } else { + (poll_interval / 3).clamp(Duration::from_secs(5), Duration::from_secs(30)) + }; + base.mul_f64(0.8 + random.clamp(0.0, 1.0) * 0.4) + .max(Duration::from_secs(1)) +} + +#[derive(Clone)] +pub(crate) struct AgentlessWorkerConfig { + endpoint: AgentlessEndpoint, + poll_interval: Duration, + request_timeout: Duration, +} + +impl AgentlessWorkerConfig { + pub(crate) fn new( + endpoint: AgentlessEndpoint, + poll_interval: Duration, + request_timeout: Duration, + ) -> Self { + Self { + endpoint, + poll_interval, + request_timeout, + } + } +} + +struct RunningWorker { + cancellation: CancellationToken, + thread: JoinHandle<()>, +} + +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum WorkerStartError { + ThreadUnavailable, +} + +/// Owns the poll thread and makes its lifecycle explicit around `fork`. +/// PR 3 connects these methods to the tracer lifecycle hooks. +pub(crate) struct AgentlessWorker { + config: AgentlessWorkerConfig, + state: Arc, + running: Option, + restart_after_fork: bool, + permanently_stopped: bool, +} + +impl AgentlessWorker { + pub(crate) fn new(config: AgentlessWorkerConfig) -> Self { + Self { + config, + state: Arc::new(PollState::default()), + running: None, + restart_after_fork: false, + permanently_stopped: false, + } + } + + pub(crate) fn start(&mut self) -> Result { + if self.running.is_some() || self.permanently_stopped { + return Ok(false); + } + + let cancellation = CancellationToken::new(); + let thread_cancellation = cancellation.clone(); + let config = self.config.clone(); + let state = Arc::clone(&self.state); + let log_dispatch = tracing::dispatcher::get_default(Clone::clone); + let thread = std::thread::Builder::new() + .name("ddtrace-ffe-agentless".to_owned()) + .spawn(move || { + let _log_guard = tracing::dispatcher::set_default(&log_dispatch); + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(_) => { + log::log( + Log::Warn, + "Feature Flags agentless runtime could not be created", + ); + return; + } + }; + + runtime.block_on(async move { + let mut poller = Poller { + endpoint: config.endpoint, + poll_interval: config.poll_interval, + request_timeout: config.request_timeout, + transport: HyperTransport::new(), + sink: GlobalConfigurationSink, + state, + retry_delay, + }; + poller.run(thread_cancellation).await; + }); + }) + .map_err(|_| WorkerStartError::ThreadUnavailable)?; + + self.running = Some(RunningWorker { + cancellation, + thread, + }); + Ok(true) + } + + pub(crate) fn shutdown(&mut self) { + self.permanently_stopped = true; + self.stop_running(); + } + + pub(crate) fn prepare_for_fork(&mut self) { + self.restart_after_fork = self.running.is_some() && !self.permanently_stopped; + self.stop_running(); + } + + pub(crate) fn resume_after_fork_parent(&mut self) -> Result<(), WorkerStartError> { + self.restart_after_fork() + } + + pub(crate) fn reset_after_fork_child(&mut self) -> Result<(), WorkerStartError> { + // `prepare_for_fork` joined the only worker before the process split, + // so no synchronization primitive can be inherited while held. + self.state.warnings.reset(); + self.restart_after_fork() + } + + pub(crate) fn poll_count(&self) -> u64 { + self.state.poll_count.load(Ordering::Relaxed) + } + + fn restart_after_fork(&mut self) -> Result<(), WorkerStartError> { + if self.restart_after_fork && !self.permanently_stopped { + self.start()?; + self.restart_after_fork = false; + } + Ok(()) + } + + fn stop_running(&mut self) { + if let Some(running) = self.running.take() { + running.cancellation.cancel(); + let _ = running.thread.join(); + } + } +} + +impl Drop for AgentlessWorker { + fn drop(&mut self) { + self.stop_running(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::VecDeque; + use std::io::Write; + use std::sync::atomic::{AtomicBool, AtomicUsize}; + + const EMPTY_CONFIG: &[u8] = br#"{ + "createdAt": "2026-05-22T00:00:00.000Z", + "format": "SERVER", + "environment": {"name": "test"}, + "flags": {} + }"#; + + struct UnusedTransport; + + impl Transport for UnusedTransport { + fn get<'a>( + &'a self, + _endpoint: &'a AgentlessEndpoint, + _timeout: Duration, + _etag: Option<&'a str>, + _cancellation: &'a CancellationToken, + ) -> FetchFuture<'a> { + Box::pin(async { unreachable!("transport is not used by response-processing tests") }) + } + } + + struct TestSink { + fail: AtomicBool, + } + + struct QueueTransport { + responses: Mutex>>, + calls: AtomicUsize, + in_flight: AtomicUsize, + max_in_flight: AtomicUsize, + } + + impl QueueTransport { + fn new( + responses: impl IntoIterator>, + ) -> Self { + Self { + responses: Mutex::new(responses.into_iter().collect()), + calls: AtomicUsize::new(0), + in_flight: AtomicUsize::new(0), + max_in_flight: AtomicUsize::new(0), + } + } + } + + impl Transport for QueueTransport { + fn get<'a>( + &'a self, + _endpoint: &'a AgentlessEndpoint, + _timeout: Duration, + _etag: Option<&'a str>, + _cancellation: &'a CancellationToken, + ) -> FetchFuture<'a> { + Box::pin(async move { + self.calls.fetch_add(1, Ordering::Relaxed); + let in_flight = self.in_flight.fetch_add(1, Ordering::Relaxed) + 1; + self.max_in_flight.fetch_max(in_flight, Ordering::Relaxed); + tokio::task::yield_now().await; + let result = lock_unpoisoned(&self.responses) + .pop_front() + .unwrap_or(Err(TransportFailure::Request)); + self.in_flight.fetch_sub(1, Ordering::Relaxed); + result + }) + } + } + + impl ConfigurationSink for TestSink { + fn apply(&self, _configuration: Configuration) -> Result { + if self.fail.load(Ordering::Relaxed) { + Err(()) + } else { + Ok(ConfigurationTransition::Ready) + } + } + } + + fn test_poller(fail_apply: bool) -> Poller { + Poller { + endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") + .expect("test endpoint"), + poll_interval: Duration::from_secs(30), + request_timeout: Duration::from_secs(5), + transport: UnusedTransport, + sink: TestSink { + fail: AtomicBool::new(fail_apply), + }, + state: Arc::new(PollState::default()), + retry_delay, + } + } + + fn no_retry_delay(_: Duration, _: usize, _: f64) -> Duration { + Duration::ZERO + } + + fn response(status: u16, etag: Option<&str>, body: &[u8]) -> PollResponse { + PollResponse { + status, + etag: etag.map(str::to_owned), + content_encoding: None, + body: body.to_vec(), + } + } + + fn gzip(payload: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(payload).expect("compress test payload"); + encoder.finish().expect("finish test payload") + } + + #[test] + fn retries_only_transport_timeouts_and_selected_statuses() { + for status in [408, 429, 500, 503, 599] { + assert!(is_retryable_status(status), "status {status}"); + } + for status in [200, 204, 301, 400, 401, 403, 404, 499, 600] { + assert!(!is_retryable_status(status), "status {status}"); + } + } + + #[test] + fn retry_delay_uses_the_cross_sdk_clamps_and_jitter() { + assert_eq!( + retry_delay(Duration::from_secs(30), 1, 0.0), + Duration::from_secs(4) + ); + assert_eq!( + retry_delay(Duration::from_secs(30), 1, 1.0), + Duration::from_secs(6) + ); + assert_eq!( + retry_delay(Duration::from_secs(30), 2, 0.0), + Duration::from_secs(8) + ); + assert_eq!( + retry_delay(Duration::from_secs(30), 2, 1.0), + Duration::from_secs(12) + ); + assert_eq!( + retry_delay(Duration::from_secs(1), 1, 0.0), + Duration::from_millis(1_600) + ); + } + + #[test] + fn decoded_body_limit_applies_after_gzip() { + let payload = vec![b'a'; 256]; + let compressed = gzip(&payload); + assert!(compressed.len() < 64); + assert_eq!( + decode_response_body(&compressed, Some(" GZIP "), 64), + Err(TransportFailure::ResponseTooLarge) + ); + assert_eq!( + decode_response_body(&gzip(b"configuration"), Some("gzip"), 64).expect("decode gzip"), + b"configuration" + ); + } + + #[test] + fn etag_advances_only_after_parse_and_apply_succeed() { + let mut poller = test_poller(true); + poller.state.set_etag(Some("old".to_owned())); + + assert_eq!( + poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), + PollOutcome::Stop + ); + assert_eq!(poller.state.etag().as_deref(), Some("old")); + + poller.sink.fail.store(false, Ordering::Relaxed); + assert_eq!( + poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), + PollOutcome::Success + ); + assert_eq!(poller.state.etag().as_deref(), Some("new")); + + assert_eq!( + poller.process_response(response(200, Some("ignored"), b"not-json")), + PollOutcome::Stop + ); + assert_eq!(poller.state.etag().as_deref(), Some("new")); + } + + #[test] + fn non_200_responses_are_never_decoded() { + let mut poller = test_poller(false); + poller.state.set_etag(Some("current".to_owned())); + assert_eq!( + poller.process_response(response(304, Some("ignored"), b"not-json")), + PollOutcome::Success + ); + assert_eq!(poller.state.etag().as_deref(), Some("current")); + assert_eq!( + poller.process_response(response(401, None, EMPTY_CONFIG)), + PollOutcome::Stop + ); + assert_eq!(poller.state.etag().as_deref(), Some("current")); + } + + #[test] + fn request_authentication_is_managed_only() { + let managed = + AgentlessEndpoint::build("", "datadoghq.com", "", "secret").expect("managed endpoint"); + let custom = + AgentlessEndpoint::build("https://user:password@localhost/custom", "", "", "secret") + .expect("custom endpoint"); + + let managed_request = build_request(&managed, Some("etag")).expect("managed request"); + assert_eq!(managed_request.headers()[API_KEY_HEADER], "secret"); + assert_eq!(managed_request.headers()[IF_NONE_MATCH], "etag"); + assert_eq!(managed_request.headers()[INTERNAL_UNTRACED_HEADER], "1"); + + let custom_request = build_request(&custom, None).expect("custom request"); + assert!(!custom_request.headers().contains_key(API_KEY_HEADER)); + assert_eq!( + custom_request.headers()[AUTHORIZATION_HEADER], + "Basic dXNlcjpwYXNzd29yZA==" + ); + assert_eq!(custom_request.uri(), "https://localhost/custom"); + } + + #[test] + fn warning_categories_are_suppressed_independently() { + let warnings = WarningSlots::default(); + assert!(warnings.take(WarningCategory::Request)); + assert!(!warnings.take(WarningCategory::Request)); + assert!(warnings.take(WarningCategory::UnexpectedHttp)); + assert!(!warnings.take(WarningCategory::UnexpectedHttp)); + warnings.reset(); + assert!(warnings.take(WarningCategory::Request)); + } + + #[tokio::test] + async fn a_poll_retries_three_times_without_overlapping_requests() { + let transport = QueueTransport::new([ + Ok(response(503, None, b"ignored")), + Err(TransportFailure::TimedOut), + Ok(response(200, Some("accepted"), EMPTY_CONFIG)), + ]); + let state = Arc::new(PollState::default()); + let mut poller = Poller { + endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") + .expect("test endpoint"), + poll_interval: Duration::from_secs(30), + request_timeout: Duration::from_secs(5), + transport, + sink: TestSink { + fail: AtomicBool::new(false), + }, + state: Arc::clone(&state), + retry_delay: no_retry_delay, + }; + + assert_eq!( + poller.poll(&CancellationToken::new()).await, + PollOutcome::Success + ); + assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 3); + assert_eq!(poller.transport.max_in_flight.load(Ordering::Relaxed), 1); + assert_eq!(state.poll_count.load(Ordering::Relaxed), 3); + assert_eq!(state.etag().as_deref(), Some("accepted")); + } + + #[tokio::test] + async fn cancellation_before_a_poll_prevents_the_first_request() { + let transport = QueueTransport::new([]); + let mut poller = Poller { + endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") + .expect("test endpoint"), + poll_interval: Duration::from_secs(30), + request_timeout: Duration::from_secs(5), + transport, + sink: TestSink { + fail: AtomicBool::new(false), + }, + state: Arc::new(PollState::default()), + retry_delay: no_retry_delay, + }; + let cancellation = CancellationToken::new(); + cancellation.cancel(); + + assert_eq!(poller.poll(&cancellation).await, PollOutcome::Cancelled); + assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 0); + } +} diff --git a/zend_abstract_interface/config/config.h b/zend_abstract_interface/config/config.h index 41ea02e2fbb..ba5cd28e1d2 100644 --- a/zend_abstract_interface/config/config.h +++ b/zend_abstract_interface/config/config.h @@ -20,7 +20,9 @@ typedef uint16_t zai_config_id; #define ZAI_CONFIG_ENTRIES_COUNT_MAX 320 #define ZAI_CONFIG_NAMES_COUNT_MAX 4 -#define ZAI_CONFIG_NAME_BUFSIZ 72 +// Accommodate the longest supported environment name after its `DD_` prefix is +// expanded to the `datadog.` INI prefix. +#define ZAI_CONFIG_NAME_BUFSIZ 80 #define ZAI_CONFIG_ENTRY(_id, _name, _type, default, ...) \ { \ From 6969ce37f805dfb332ddd4fb1039ca4486eae450 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 18 Sep 2026 13:11:01 +0300 Subject: [PATCH 03/12] refactor(ffe): split agentless polling responsibilities --- components-rs/ffe/agentless.rs | 819 +---------------------- components-rs/ffe/agentless/poller.rs | 553 +++++++++++++++ components-rs/ffe/agentless/transport.rs | 291 ++++++++ 3 files changed, 859 insertions(+), 804 deletions(-) create mode 100644 components-rs/ffe/agentless/poller.rs create mode 100644 components-rs/ffe/agentless/transport.rs diff --git a/components-rs/ffe/agentless.rs b/components-rs/ffe/agentless.rs index 1fe4663b676..c7581285052 100644 --- a/components-rs/ffe/agentless.rs +++ b/components-rs/ffe/agentless.rs @@ -1,511 +1,18 @@ // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +mod poller; +mod transport; + +use self::poller::{GlobalConfigurationSink, PollState, Poller}; +use self::transport::HyperTransport; use super::settings::AgentlessEndpoint; -use super::{store_config, ConfigurationTransition}; use crate::log::{self, Log}; -use base64::Engine; -use datadog_ffe::rules_based::{Configuration, UniversalFlagConfig}; -use flate2::read::GzDecoder; -use futures_util::FutureExt; -use http::header::{ACCEPT_ENCODING, CONTENT_ENCODING, CONTENT_LENGTH, ETAG, IF_NONE_MATCH}; -use http::{Method, Request}; -use http_body_util::BodyExt; -use libdd_common::http_common::{self, Body}; -use percent_encoding::percent_decode_str; -use std::future::Future; -use std::io::Read; -use std::panic::AssertUnwindSafe; -use std::pin::Pin; -use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::Arc; use std::thread::JoinHandle; use std::time::Duration; use tokio_util::sync::CancellationToken; -const MAX_POLL_ATTEMPTS: usize = 3; -const MAX_RESPONSE_BODY_BYTES: usize = 10 << 20; -const CLIENT_LANGUAGE: &str = "php"; -const INTERNAL_UNTRACED_HEADER: &str = "DD-Internal-Untraced-Request"; -const API_KEY_HEADER: &str = "DD-API-KEY"; -const AUTHORIZATION_HEADER: &str = "Authorization"; - -type FetchFuture<'a> = - Pin> + Send + 'a>>; - -trait Transport: Send + Sync { - fn get<'a>( - &'a self, - endpoint: &'a AgentlessEndpoint, - timeout: Duration, - etag: Option<&'a str>, - cancellation: &'a CancellationToken, - ) -> FetchFuture<'a>; -} - -trait ConfigurationSink: Send + Sync { - fn apply(&self, configuration: Configuration) -> Result; -} - -struct GlobalConfigurationSink; - -impl ConfigurationSink for GlobalConfigurationSink { - fn apply(&self, configuration: Configuration) -> Result { - Ok(store_config(configuration)) - } -} - -#[derive(Debug, Eq, PartialEq)] -enum TransportFailure { - BuildRequest, - Cancelled, - TimedOut, - Request, - ResponseBody, - ResponseTooLarge, -} - -struct PollResponse { - status: u16, - etag: Option, - content_encoding: Option, - body: Vec, -} - -struct HyperTransport { - client: libdd_common::HttpClient, -} - -impl HyperTransport { - fn new() -> Self { - Self { - // This native client is outside PHP HTTP instrumentation, so it - // cannot recursively trace its own polling requests. Hyper also - // leaves 3xx responses visible instead of following redirects. - client: http_common::new_client_periodic(), - } - } - - async fn get_inner( - &self, - endpoint: &AgentlessEndpoint, - timeout: Duration, - etag: Option<&str>, - cancellation: &CancellationToken, - ) -> Result { - let request = build_request(endpoint, etag)?; - let operation = async { - let response = self - .client - .request(request) - .await - .map_err(|_| TransportFailure::Request)?; - let status = response.status().as_u16(); - let etag = header_value(response.headers(), ETAG); - let content_encoding = header_value(response.headers(), CONTENT_ENCODING); - - // A non-200 body can never contain configuration. Dropping it here - // also prevents a malicious error response from consuming memory. - if status != 200 { - return Ok(PollResponse { - status, - etag, - content_encoding, - body: Vec::new(), - }); - } - - if response - .headers() - .get(CONTENT_LENGTH) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.parse::().ok()) - .is_some_and(|length| length > MAX_RESPONSE_BODY_BYTES as u64) - { - return Err(TransportFailure::ResponseTooLarge); - } - - let body = collect_bounded(response.into_body(), MAX_RESPONSE_BODY_BYTES).await?; - Ok(PollResponse { - status, - etag, - content_encoding, - body, - }) - }; - - tokio::select! { - _ = cancellation.cancelled() => Err(TransportFailure::Cancelled), - result = tokio::time::timeout(timeout, operation) => { - result.map_err(|_| TransportFailure::TimedOut)? - } - } - } -} - -impl Transport for HyperTransport { - fn get<'a>( - &'a self, - endpoint: &'a AgentlessEndpoint, - timeout: Duration, - etag: Option<&'a str>, - cancellation: &'a CancellationToken, - ) -> FetchFuture<'a> { - Box::pin(self.get_inner(endpoint, timeout, etag, cancellation)) - } -} - -fn build_request( - endpoint: &AgentlessEndpoint, - etag: Option<&str>, -) -> Result, TransportFailure> { - let (request_url, authorization) = request_url_and_authorization(endpoint.as_str())?; - let mut builder = Request::builder() - .method(Method::GET) - .uri(request_url) - .header(ACCEPT_ENCODING, "gzip") - .header("DD-Client-Library-Language", CLIENT_LANGUAGE) - .header( - "DD-Client-Library-Version", - include_str!("../../VERSION").trim(), - ) - .header(INTERNAL_UNTRACED_HEADER, "1"); - - if let Some(etag) = etag { - builder = builder.header(IF_NONE_MATCH, etag); - } - if let Some(api_key) = endpoint.api_key() { - builder = builder.header(API_KEY_HEADER, api_key); - } - if let Some(authorization) = authorization { - builder = builder.header(AUTHORIZATION_HEADER, authorization); - } - - builder - .body(Body::empty()) - .map_err(|_| TransportFailure::BuildRequest) -} - -fn request_url_and_authorization( - endpoint: &str, -) -> Result<(String, Option), TransportFailure> { - let mut url = url::Url::parse(endpoint).map_err(|_| TransportFailure::BuildRequest)?; - if url.username().is_empty() { - return Ok((endpoint.to_owned(), None)); - } - - let mut credentials = percent_decode_str(url.username()).collect::>(); - credentials.push(b':'); - if let Some(password) = url.password() { - credentials.extend(percent_decode_str(password)); - } - let authorization = format!( - "Basic {}", - base64::engine::general_purpose::STANDARD.encode(credentials) - ); - - url.set_username("") - .map_err(|_| TransportFailure::BuildRequest)?; - url.set_password(None) - .map_err(|_| TransportFailure::BuildRequest)?; - Ok((url.to_string(), Some(authorization))) -} - -fn header_value(headers: &http::HeaderMap, name: http::header::HeaderName) -> Option { - headers - .get(name) - .and_then(|value| value.to_str().ok()) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) -} - -async fn collect_bounded( - mut body: hyper::body::Incoming, - limit: usize, -) -> Result, TransportFailure> { - let mut bytes = Vec::new(); - while let Some(frame) = body.frame().await { - let frame = frame.map_err(|_| TransportFailure::ResponseBody)?; - if let Some(data) = frame.data_ref() { - let remaining = limit.saturating_sub(bytes.len()); - if data.len() > remaining { - return Err(TransportFailure::ResponseTooLarge); - } - bytes.extend_from_slice(data); - } - } - Ok(bytes) -} - -fn decode_response_body( - body: &[u8], - content_encoding: Option<&str>, - limit: usize, -) -> Result, TransportFailure> { - if !content_encoding.is_some_and(|value| value.trim().eq_ignore_ascii_case("gzip")) { - if body.len() > limit { - return Err(TransportFailure::ResponseTooLarge); - } - return Ok(body.to_vec()); - } - - let decoder = GzDecoder::new(body); - let mut decoded = Vec::new(); - decoder - .take(limit as u64 + 1) - .read_to_end(&mut decoded) - .map_err(|_| TransportFailure::ResponseBody)?; - if decoded.len() > limit { - return Err(TransportFailure::ResponseTooLarge); - } - Ok(decoded) -} - -#[derive(Clone, Copy)] -#[repr(u8)] -enum WarningCategory { - Authentication = 1 << 0, - RetryableHttp = 1 << 1, - UnexpectedHttp = 1 << 2, - Request = 1 << 3, - MalformedPayload = 1 << 4, - ApplyFailure = 1 << 5, - UnexpectedException = 1 << 6, -} - -#[derive(Default)] -struct WarningSlots(AtomicU8); - -impl WarningSlots { - fn take(&self, category: WarningCategory) -> bool { - self.0.fetch_or(category as u8, Ordering::Relaxed) & category as u8 == 0 - } - - fn reset(&self) { - self.0.store(0, Ordering::Relaxed); - } -} - -#[derive(Default)] -struct PollState { - etag: Mutex>, - warnings: WarningSlots, - poll_count: AtomicU64, -} - -impl PollState { - fn etag(&self) -> Option { - lock_unpoisoned(&self.etag).clone() - } - - fn set_etag(&self, etag: Option) { - *lock_unpoisoned(&self.etag) = etag; - } -} - -fn lock_unpoisoned(mutex: &Mutex) -> MutexGuard<'_, T> { - mutex - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum PollOutcome { - Success, - Retryable, - Stop, - Cancelled, -} - -struct Poller { - endpoint: AgentlessEndpoint, - poll_interval: Duration, - request_timeout: Duration, - transport: T, - sink: S, - state: Arc, - retry_delay: fn(Duration, usize, f64) -> Duration, -} - -impl Poller { - async fn run(&mut self, cancellation: CancellationToken) { - loop { - if cancellation.is_cancelled() { - return; - } - - let poll = AssertUnwindSafe(self.poll(&cancellation)).catch_unwind(); - match poll.await { - Ok(PollOutcome::Cancelled) => return, - Ok(_) => {} - Err(_) => self.warn( - WarningCategory::UnexpectedException, - "Feature Flags agentless polling failed unexpectedly; polling continues", - ), - } - - tokio::select! { - _ = cancellation.cancelled() => return, - _ = tokio::time::sleep(self.poll_interval) => {} - } - } - } - - async fn poll(&mut self, cancellation: &CancellationToken) -> PollOutcome { - for attempt in 1..=MAX_POLL_ATTEMPTS { - let outcome = self.poll_once(cancellation).await; - if outcome != PollOutcome::Retryable || attempt == MAX_POLL_ATTEMPTS { - return outcome; - } - - let delay = (self.retry_delay)(self.poll_interval, attempt, fastrand::f64()); - tokio::select! { - _ = cancellation.cancelled() => return PollOutcome::Cancelled, - _ = tokio::time::sleep(delay) => {} - } - } - PollOutcome::Stop - } - - async fn poll_once(&mut self, cancellation: &CancellationToken) -> PollOutcome { - if cancellation.is_cancelled() { - return PollOutcome::Cancelled; - } - - self.state.poll_count.fetch_add(1, Ordering::Relaxed); - let etag = self.state.etag(); - match self - .transport - .get( - &self.endpoint, - self.request_timeout, - etag.as_deref(), - cancellation, - ) - .await - { - Ok(response) => self.process_response(response), - Err(TransportFailure::Cancelled) => PollOutcome::Cancelled, - Err(TransportFailure::BuildRequest) => { - self.warn( - WarningCategory::Request, - "Feature Flags agentless request could not be built", - ); - PollOutcome::Stop - } - Err(TransportFailure::ResponseTooLarge) => { - self.warn( - WarningCategory::MalformedPayload, - "Feature Flags agentless response exceeds the 10 MiB limit; keeping the last known configuration", - ); - PollOutcome::Stop - } - Err(_) => { - self.warn( - WarningCategory::Request, - "Feature Flags agentless request failed; keeping the last known configuration", - ); - PollOutcome::Retryable - } - } - } - - fn process_response(&mut self, response: PollResponse) -> PollOutcome { - match response.status { - 304 => return PollOutcome::Success, - 401 | 403 => { - self.warn( - WarningCategory::Authentication, - "Feature Flags agentless endpoint rejected authentication; verify endpoint authentication", - ); - return PollOutcome::Stop; - } - 200 => {} - status if is_retryable_status(status) => { - self.warn( - WarningCategory::RetryableHttp, - &format!("Feature Flags agentless endpoint returned retryable HTTP {status}"), - ); - return PollOutcome::Retryable; - } - status => { - self.warn( - WarningCategory::UnexpectedHttp, - &format!("Feature Flags agentless endpoint returned unexpected HTTP {status}"), - ); - return PollOutcome::Stop; - } - } - - let body = match decode_response_body( - &response.body, - response.content_encoding.as_deref(), - MAX_RESPONSE_BODY_BYTES, - ) { - Ok(body) => body, - Err(TransportFailure::ResponseTooLarge) => { - self.warn( - WarningCategory::MalformedPayload, - "Feature Flags agentless decoded response exceeds the 10 MiB limit; keeping the last known configuration", - ); - return PollOutcome::Stop; - } - Err(_) => { - self.warn( - WarningCategory::MalformedPayload, - "Feature Flags agentless endpoint returned an invalid response body", - ); - return PollOutcome::Stop; - } - }; - - let configuration = match UniversalFlagConfig::from_json(body) { - Ok(configuration) => Configuration::from_server_response(configuration), - Err(_) => { - self.warn( - WarningCategory::MalformedPayload, - "Feature Flags agentless endpoint returned malformed configuration", - ); - return PollOutcome::Stop; - } - }; - - if self.sink.apply(configuration).is_err() { - self.warn( - WarningCategory::ApplyFailure, - "Feature Flags agentless configuration could not be applied", - ); - return PollOutcome::Stop; - } - - // The response has now been parsed and applied. A blank or absent - // ETag intentionally clears an older value. - self.state.set_etag(response.etag); - PollOutcome::Success - } - - fn warn(&self, category: WarningCategory, message: &str) { - if self.state.warnings.take(category) { - log::log(Log::Warn, message); - } - } -} - -fn is_retryable_status(status: u16) -> bool { - matches!(status, 408 | 429 | 500..=599) -} - -fn retry_delay(poll_interval: Duration, attempt: usize, random: f64) -> Duration { - let base = if attempt <= 1 { - (poll_interval / 6).clamp(Duration::from_secs(2), Duration::from_secs(10)) - } else { - (poll_interval / 3).clamp(Duration::from_secs(5), Duration::from_secs(30)) - }; - base.mul_f64(0.8 + random.clamp(0.0, 1.0) * 0.4) - .max(Duration::from_secs(1)) -} - #[derive(Clone)] pub(crate) struct AgentlessWorkerConfig { endpoint: AgentlessEndpoint, @@ -587,15 +94,14 @@ impl AgentlessWorker { }; runtime.block_on(async move { - let mut poller = Poller { - endpoint: config.endpoint, - poll_interval: config.poll_interval, - request_timeout: config.request_timeout, - transport: HyperTransport::new(), - sink: GlobalConfigurationSink, + let mut poller = Poller::new( + config.endpoint, + config.poll_interval, + config.request_timeout, + HyperTransport::new(), + GlobalConfigurationSink, state, - retry_delay, - }; + ); poller.run(thread_cancellation).await; }); }) @@ -625,12 +131,12 @@ impl AgentlessWorker { pub(crate) fn reset_after_fork_child(&mut self) -> Result<(), WorkerStartError> { // `prepare_for_fork` joined the only worker before the process split, // so no synchronization primitive can be inherited while held. - self.state.warnings.reset(); + self.state.reset_warnings(); self.restart_after_fork() } pub(crate) fn poll_count(&self) -> u64 { - self.state.poll_count.load(Ordering::Relaxed) + self.state.poll_count() } fn restart_after_fork(&mut self) -> Result<(), WorkerStartError> { @@ -654,298 +160,3 @@ impl Drop for AgentlessWorker { self.stop_running(); } } - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::VecDeque; - use std::io::Write; - use std::sync::atomic::{AtomicBool, AtomicUsize}; - - const EMPTY_CONFIG: &[u8] = br#"{ - "createdAt": "2026-05-22T00:00:00.000Z", - "format": "SERVER", - "environment": {"name": "test"}, - "flags": {} - }"#; - - struct UnusedTransport; - - impl Transport for UnusedTransport { - fn get<'a>( - &'a self, - _endpoint: &'a AgentlessEndpoint, - _timeout: Duration, - _etag: Option<&'a str>, - _cancellation: &'a CancellationToken, - ) -> FetchFuture<'a> { - Box::pin(async { unreachable!("transport is not used by response-processing tests") }) - } - } - - struct TestSink { - fail: AtomicBool, - } - - struct QueueTransport { - responses: Mutex>>, - calls: AtomicUsize, - in_flight: AtomicUsize, - max_in_flight: AtomicUsize, - } - - impl QueueTransport { - fn new( - responses: impl IntoIterator>, - ) -> Self { - Self { - responses: Mutex::new(responses.into_iter().collect()), - calls: AtomicUsize::new(0), - in_flight: AtomicUsize::new(0), - max_in_flight: AtomicUsize::new(0), - } - } - } - - impl Transport for QueueTransport { - fn get<'a>( - &'a self, - _endpoint: &'a AgentlessEndpoint, - _timeout: Duration, - _etag: Option<&'a str>, - _cancellation: &'a CancellationToken, - ) -> FetchFuture<'a> { - Box::pin(async move { - self.calls.fetch_add(1, Ordering::Relaxed); - let in_flight = self.in_flight.fetch_add(1, Ordering::Relaxed) + 1; - self.max_in_flight.fetch_max(in_flight, Ordering::Relaxed); - tokio::task::yield_now().await; - let result = lock_unpoisoned(&self.responses) - .pop_front() - .unwrap_or(Err(TransportFailure::Request)); - self.in_flight.fetch_sub(1, Ordering::Relaxed); - result - }) - } - } - - impl ConfigurationSink for TestSink { - fn apply(&self, _configuration: Configuration) -> Result { - if self.fail.load(Ordering::Relaxed) { - Err(()) - } else { - Ok(ConfigurationTransition::Ready) - } - } - } - - fn test_poller(fail_apply: bool) -> Poller { - Poller { - endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") - .expect("test endpoint"), - poll_interval: Duration::from_secs(30), - request_timeout: Duration::from_secs(5), - transport: UnusedTransport, - sink: TestSink { - fail: AtomicBool::new(fail_apply), - }, - state: Arc::new(PollState::default()), - retry_delay, - } - } - - fn no_retry_delay(_: Duration, _: usize, _: f64) -> Duration { - Duration::ZERO - } - - fn response(status: u16, etag: Option<&str>, body: &[u8]) -> PollResponse { - PollResponse { - status, - etag: etag.map(str::to_owned), - content_encoding: None, - body: body.to_vec(), - } - } - - fn gzip(payload: &[u8]) -> Vec { - let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - encoder.write_all(payload).expect("compress test payload"); - encoder.finish().expect("finish test payload") - } - - #[test] - fn retries_only_transport_timeouts_and_selected_statuses() { - for status in [408, 429, 500, 503, 599] { - assert!(is_retryable_status(status), "status {status}"); - } - for status in [200, 204, 301, 400, 401, 403, 404, 499, 600] { - assert!(!is_retryable_status(status), "status {status}"); - } - } - - #[test] - fn retry_delay_uses_the_cross_sdk_clamps_and_jitter() { - assert_eq!( - retry_delay(Duration::from_secs(30), 1, 0.0), - Duration::from_secs(4) - ); - assert_eq!( - retry_delay(Duration::from_secs(30), 1, 1.0), - Duration::from_secs(6) - ); - assert_eq!( - retry_delay(Duration::from_secs(30), 2, 0.0), - Duration::from_secs(8) - ); - assert_eq!( - retry_delay(Duration::from_secs(30), 2, 1.0), - Duration::from_secs(12) - ); - assert_eq!( - retry_delay(Duration::from_secs(1), 1, 0.0), - Duration::from_millis(1_600) - ); - } - - #[test] - fn decoded_body_limit_applies_after_gzip() { - let payload = vec![b'a'; 256]; - let compressed = gzip(&payload); - assert!(compressed.len() < 64); - assert_eq!( - decode_response_body(&compressed, Some(" GZIP "), 64), - Err(TransportFailure::ResponseTooLarge) - ); - assert_eq!( - decode_response_body(&gzip(b"configuration"), Some("gzip"), 64).expect("decode gzip"), - b"configuration" - ); - } - - #[test] - fn etag_advances_only_after_parse_and_apply_succeed() { - let mut poller = test_poller(true); - poller.state.set_etag(Some("old".to_owned())); - - assert_eq!( - poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), - PollOutcome::Stop - ); - assert_eq!(poller.state.etag().as_deref(), Some("old")); - - poller.sink.fail.store(false, Ordering::Relaxed); - assert_eq!( - poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), - PollOutcome::Success - ); - assert_eq!(poller.state.etag().as_deref(), Some("new")); - - assert_eq!( - poller.process_response(response(200, Some("ignored"), b"not-json")), - PollOutcome::Stop - ); - assert_eq!(poller.state.etag().as_deref(), Some("new")); - } - - #[test] - fn non_200_responses_are_never_decoded() { - let mut poller = test_poller(false); - poller.state.set_etag(Some("current".to_owned())); - assert_eq!( - poller.process_response(response(304, Some("ignored"), b"not-json")), - PollOutcome::Success - ); - assert_eq!(poller.state.etag().as_deref(), Some("current")); - assert_eq!( - poller.process_response(response(401, None, EMPTY_CONFIG)), - PollOutcome::Stop - ); - assert_eq!(poller.state.etag().as_deref(), Some("current")); - } - - #[test] - fn request_authentication_is_managed_only() { - let managed = - AgentlessEndpoint::build("", "datadoghq.com", "", "secret").expect("managed endpoint"); - let custom = - AgentlessEndpoint::build("https://user:password@localhost/custom", "", "", "secret") - .expect("custom endpoint"); - - let managed_request = build_request(&managed, Some("etag")).expect("managed request"); - assert_eq!(managed_request.headers()[API_KEY_HEADER], "secret"); - assert_eq!(managed_request.headers()[IF_NONE_MATCH], "etag"); - assert_eq!(managed_request.headers()[INTERNAL_UNTRACED_HEADER], "1"); - - let custom_request = build_request(&custom, None).expect("custom request"); - assert!(!custom_request.headers().contains_key(API_KEY_HEADER)); - assert_eq!( - custom_request.headers()[AUTHORIZATION_HEADER], - "Basic dXNlcjpwYXNzd29yZA==" - ); - assert_eq!(custom_request.uri(), "https://localhost/custom"); - } - - #[test] - fn warning_categories_are_suppressed_independently() { - let warnings = WarningSlots::default(); - assert!(warnings.take(WarningCategory::Request)); - assert!(!warnings.take(WarningCategory::Request)); - assert!(warnings.take(WarningCategory::UnexpectedHttp)); - assert!(!warnings.take(WarningCategory::UnexpectedHttp)); - warnings.reset(); - assert!(warnings.take(WarningCategory::Request)); - } - - #[tokio::test] - async fn a_poll_retries_three_times_without_overlapping_requests() { - let transport = QueueTransport::new([ - Ok(response(503, None, b"ignored")), - Err(TransportFailure::TimedOut), - Ok(response(200, Some("accepted"), EMPTY_CONFIG)), - ]); - let state = Arc::new(PollState::default()); - let mut poller = Poller { - endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") - .expect("test endpoint"), - poll_interval: Duration::from_secs(30), - request_timeout: Duration::from_secs(5), - transport, - sink: TestSink { - fail: AtomicBool::new(false), - }, - state: Arc::clone(&state), - retry_delay: no_retry_delay, - }; - - assert_eq!( - poller.poll(&CancellationToken::new()).await, - PollOutcome::Success - ); - assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 3); - assert_eq!(poller.transport.max_in_flight.load(Ordering::Relaxed), 1); - assert_eq!(state.poll_count.load(Ordering::Relaxed), 3); - assert_eq!(state.etag().as_deref(), Some("accepted")); - } - - #[tokio::test] - async fn cancellation_before_a_poll_prevents_the_first_request() { - let transport = QueueTransport::new([]); - let mut poller = Poller { - endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") - .expect("test endpoint"), - poll_interval: Duration::from_secs(30), - request_timeout: Duration::from_secs(5), - transport, - sink: TestSink { - fail: AtomicBool::new(false), - }, - state: Arc::new(PollState::default()), - retry_delay: no_retry_delay, - }; - let cancellation = CancellationToken::new(); - cancellation.cancel(); - - assert_eq!(poller.poll(&cancellation).await, PollOutcome::Cancelled); - assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 0); - } -} diff --git a/components-rs/ffe/agentless/poller.rs b/components-rs/ffe/agentless/poller.rs new file mode 100644 index 00000000000..2865dc4ba21 --- /dev/null +++ b/components-rs/ffe/agentless/poller.rs @@ -0,0 +1,553 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use super::super::settings::AgentlessEndpoint; +use super::transport::{ + decode_response_body, PollResponse, Transport, TransportFailure, MAX_RESPONSE_BODY_BYTES, +}; +use crate::ffe::{store_config, ConfigurationTransition}; +use crate::log::{self, Log}; +use datadog_ffe::rules_based::{Configuration, UniversalFlagConfig}; +use futures_util::FutureExt; +use std::panic::AssertUnwindSafe; +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Duration; +use tokio_util::sync::CancellationToken; + +const MAX_POLL_ATTEMPTS: usize = 3; + +pub(super) trait ConfigurationSink: Send + Sync { + fn apply(&self, configuration: Configuration) -> Result; +} + +pub(super) struct GlobalConfigurationSink; + +impl ConfigurationSink for GlobalConfigurationSink { + fn apply(&self, configuration: Configuration) -> Result { + Ok(store_config(configuration)) + } +} + +#[derive(Clone, Copy)] +#[repr(u8)] +enum WarningCategory { + Authentication = 1 << 0, + RetryableHttp = 1 << 1, + UnexpectedHttp = 1 << 2, + Request = 1 << 3, + MalformedPayload = 1 << 4, + ApplyFailure = 1 << 5, + UnexpectedException = 1 << 6, +} + +#[derive(Default)] +struct WarningSlots(AtomicU8); + +impl WarningSlots { + fn take(&self, category: WarningCategory) -> bool { + self.0.fetch_or(category as u8, Ordering::Relaxed) & category as u8 == 0 + } + + fn reset(&self) { + self.0.store(0, Ordering::Relaxed); + } +} + +#[derive(Default)] +pub(super) struct PollState { + etag: Mutex>, + warnings: WarningSlots, + poll_count: AtomicU64, +} + +impl PollState { + fn etag(&self) -> Option { + lock_unpoisoned(&self.etag).clone() + } + + fn set_etag(&self, etag: Option) { + *lock_unpoisoned(&self.etag) = etag; + } + + pub(super) fn reset_warnings(&self) { + self.warnings.reset(); + } + + pub(super) fn poll_count(&self) -> u64 { + self.poll_count.load(Ordering::Relaxed) + } +} + +fn lock_unpoisoned(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PollOutcome { + Success, + Retryable, + Stop, + Cancelled, +} + +pub(super) struct Poller { + endpoint: AgentlessEndpoint, + poll_interval: Duration, + request_timeout: Duration, + transport: T, + sink: S, + state: Arc, + retry_delay: fn(Duration, usize, f64) -> Duration, +} + +impl Poller { + pub(super) fn new( + endpoint: AgentlessEndpoint, + poll_interval: Duration, + request_timeout: Duration, + transport: T, + sink: S, + state: Arc, + ) -> Self { + Self { + endpoint, + poll_interval, + request_timeout, + transport, + sink, + state, + retry_delay, + } + } + + pub(super) async fn run(&mut self, cancellation: CancellationToken) { + loop { + if cancellation.is_cancelled() { + return; + } + + let poll = AssertUnwindSafe(self.poll(&cancellation)).catch_unwind(); + match poll.await { + Ok(PollOutcome::Cancelled) => return, + Ok(_) => {} + Err(_) => self.warn( + WarningCategory::UnexpectedException, + "Feature Flags agentless polling failed unexpectedly; polling continues", + ), + } + + tokio::select! { + _ = cancellation.cancelled() => return, + _ = tokio::time::sleep(self.poll_interval) => {} + } + } + } + + async fn poll(&mut self, cancellation: &CancellationToken) -> PollOutcome { + for attempt in 1..=MAX_POLL_ATTEMPTS { + let outcome = self.poll_once(cancellation).await; + if outcome != PollOutcome::Retryable || attempt == MAX_POLL_ATTEMPTS { + return outcome; + } + + let delay = (self.retry_delay)(self.poll_interval, attempt, fastrand::f64()); + tokio::select! { + _ = cancellation.cancelled() => return PollOutcome::Cancelled, + _ = tokio::time::sleep(delay) => {} + } + } + PollOutcome::Stop + } + + async fn poll_once(&mut self, cancellation: &CancellationToken) -> PollOutcome { + if cancellation.is_cancelled() { + return PollOutcome::Cancelled; + } + + self.state.poll_count.fetch_add(1, Ordering::Relaxed); + let etag = self.state.etag(); + match self + .transport + .get( + &self.endpoint, + self.request_timeout, + etag.as_deref(), + cancellation, + ) + .await + { + Ok(response) => self.process_response(response), + Err(TransportFailure::Cancelled) => PollOutcome::Cancelled, + Err(TransportFailure::BuildRequest) => { + self.warn( + WarningCategory::Request, + "Feature Flags agentless request could not be built", + ); + PollOutcome::Stop + } + Err(TransportFailure::ResponseTooLarge) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless response exceeds the 10 MiB limit; keeping the last known configuration", + ); + PollOutcome::Stop + } + Err(_) => { + self.warn( + WarningCategory::Request, + "Feature Flags agentless request failed; keeping the last known configuration", + ); + PollOutcome::Retryable + } + } + } + + fn process_response(&mut self, response: PollResponse) -> PollOutcome { + match response.status { + 304 => return PollOutcome::Success, + 401 | 403 => { + self.warn( + WarningCategory::Authentication, + "Feature Flags agentless endpoint rejected authentication; verify endpoint authentication", + ); + return PollOutcome::Stop; + } + 200 => {} + status if is_retryable_status(status) => { + self.warn( + WarningCategory::RetryableHttp, + &format!("Feature Flags agentless endpoint returned retryable HTTP {status}"), + ); + return PollOutcome::Retryable; + } + status => { + self.warn( + WarningCategory::UnexpectedHttp, + &format!("Feature Flags agentless endpoint returned unexpected HTTP {status}"), + ); + return PollOutcome::Stop; + } + } + + let body = match decode_response_body( + &response.body, + response.content_encoding.as_deref(), + MAX_RESPONSE_BODY_BYTES, + ) { + Ok(body) => body, + Err(TransportFailure::ResponseTooLarge) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless decoded response exceeds the 10 MiB limit; keeping the last known configuration", + ); + return PollOutcome::Stop; + } + Err(_) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless endpoint returned an invalid response body", + ); + return PollOutcome::Stop; + } + }; + + let configuration = match UniversalFlagConfig::from_json(body) { + Ok(configuration) => Configuration::from_server_response(configuration), + Err(_) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless endpoint returned malformed configuration", + ); + return PollOutcome::Stop; + } + }; + + if self.sink.apply(configuration).is_err() { + self.warn( + WarningCategory::ApplyFailure, + "Feature Flags agentless configuration could not be applied", + ); + return PollOutcome::Stop; + } + + // The response has now been parsed and applied. A blank or absent + // ETag intentionally clears an older value. + self.state.set_etag(response.etag); + PollOutcome::Success + } + + fn warn(&self, category: WarningCategory, message: &str) { + if self.state.warnings.take(category) { + log::log(Log::Warn, message); + } + } +} + +fn is_retryable_status(status: u16) -> bool { + matches!(status, 408 | 429 | 500..=599) +} + +fn retry_delay(poll_interval: Duration, attempt: usize, random: f64) -> Duration { + let base = if attempt <= 1 { + (poll_interval / 6).clamp(Duration::from_secs(2), Duration::from_secs(10)) + } else { + (poll_interval / 3).clamp(Duration::from_secs(5), Duration::from_secs(30)) + }; + base.mul_f64(0.8 + random.clamp(0.0, 1.0) * 0.4) + .max(Duration::from_secs(1)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ffe::agentless::transport::FetchFuture; + use std::collections::VecDeque; + use std::sync::atomic::{AtomicBool, AtomicUsize}; + + const EMPTY_CONFIG: &[u8] = br#"{ + "createdAt": "2026-05-22T00:00:00.000Z", + "format": "SERVER", + "environment": {"name": "test"}, + "flags": {} + }"#; + + struct UnusedTransport; + + impl Transport for UnusedTransport { + fn get<'a>( + &'a self, + _endpoint: &'a AgentlessEndpoint, + _timeout: Duration, + _etag: Option<&'a str>, + _cancellation: &'a CancellationToken, + ) -> FetchFuture<'a> { + Box::pin(async { unreachable!("transport is not used by response-processing tests") }) + } + } + + struct TestSink { + fail: AtomicBool, + } + + struct QueueTransport { + responses: Mutex>>, + calls: AtomicUsize, + in_flight: AtomicUsize, + max_in_flight: AtomicUsize, + } + + impl QueueTransport { + fn new( + responses: impl IntoIterator>, + ) -> Self { + Self { + responses: Mutex::new(responses.into_iter().collect()), + calls: AtomicUsize::new(0), + in_flight: AtomicUsize::new(0), + max_in_flight: AtomicUsize::new(0), + } + } + } + + impl Transport for QueueTransport { + fn get<'a>( + &'a self, + _endpoint: &'a AgentlessEndpoint, + _timeout: Duration, + _etag: Option<&'a str>, + _cancellation: &'a CancellationToken, + ) -> FetchFuture<'a> { + Box::pin(async move { + self.calls.fetch_add(1, Ordering::Relaxed); + let in_flight = self.in_flight.fetch_add(1, Ordering::Relaxed) + 1; + self.max_in_flight.fetch_max(in_flight, Ordering::Relaxed); + tokio::task::yield_now().await; + let result = lock_unpoisoned(&self.responses) + .pop_front() + .unwrap_or(Err(TransportFailure::Request)); + self.in_flight.fetch_sub(1, Ordering::Relaxed); + result + }) + } + } + + impl ConfigurationSink for TestSink { + fn apply(&self, _configuration: Configuration) -> Result { + if self.fail.load(Ordering::Relaxed) { + Err(()) + } else { + Ok(ConfigurationTransition::Ready) + } + } + } + + fn test_poller(fail_apply: bool) -> Poller { + Poller { + endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") + .expect("test endpoint"), + poll_interval: Duration::from_secs(30), + request_timeout: Duration::from_secs(5), + transport: UnusedTransport, + sink: TestSink { + fail: AtomicBool::new(fail_apply), + }, + state: Arc::new(PollState::default()), + retry_delay, + } + } + + fn no_retry_delay(_: Duration, _: usize, _: f64) -> Duration { + Duration::ZERO + } + + fn response(status: u16, etag: Option<&str>, body: &[u8]) -> PollResponse { + PollResponse { + status, + etag: etag.map(str::to_owned), + content_encoding: None, + body: body.to_vec(), + } + } + + #[test] + fn retries_only_transport_timeouts_and_selected_statuses() { + for status in [408, 429, 500, 503, 599] { + assert!(is_retryable_status(status), "status {status}"); + } + for status in [200, 204, 301, 400, 401, 403, 404, 499, 600] { + assert!(!is_retryable_status(status), "status {status}"); + } + } + + #[test] + fn retry_delay_uses_the_cross_sdk_clamps_and_jitter() { + assert_eq!( + retry_delay(Duration::from_secs(30), 1, 0.0), + Duration::from_secs(4) + ); + assert_eq!( + retry_delay(Duration::from_secs(30), 1, 1.0), + Duration::from_secs(6) + ); + assert_eq!( + retry_delay(Duration::from_secs(30), 2, 0.0), + Duration::from_secs(8) + ); + assert_eq!( + retry_delay(Duration::from_secs(30), 2, 1.0), + Duration::from_secs(12) + ); + assert_eq!( + retry_delay(Duration::from_secs(1), 1, 0.0), + Duration::from_millis(1_600) + ); + } + + #[test] + fn etag_advances_only_after_parse_and_apply_succeed() { + let mut poller = test_poller(true); + poller.state.set_etag(Some("old".to_owned())); + + assert_eq!( + poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), + PollOutcome::Stop + ); + assert_eq!(poller.state.etag().as_deref(), Some("old")); + + poller.sink.fail.store(false, Ordering::Relaxed); + assert_eq!( + poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), + PollOutcome::Success + ); + assert_eq!(poller.state.etag().as_deref(), Some("new")); + + assert_eq!( + poller.process_response(response(200, Some("ignored"), b"not-json")), + PollOutcome::Stop + ); + assert_eq!(poller.state.etag().as_deref(), Some("new")); + } + + #[test] + fn non_200_responses_are_never_decoded() { + let mut poller = test_poller(false); + poller.state.set_etag(Some("current".to_owned())); + assert_eq!( + poller.process_response(response(304, Some("ignored"), b"not-json")), + PollOutcome::Success + ); + assert_eq!(poller.state.etag().as_deref(), Some("current")); + assert_eq!( + poller.process_response(response(401, None, EMPTY_CONFIG)), + PollOutcome::Stop + ); + assert_eq!(poller.state.etag().as_deref(), Some("current")); + } + + #[test] + fn warning_categories_are_suppressed_independently() { + let warnings = WarningSlots::default(); + assert!(warnings.take(WarningCategory::Request)); + assert!(!warnings.take(WarningCategory::Request)); + assert!(warnings.take(WarningCategory::UnexpectedHttp)); + assert!(!warnings.take(WarningCategory::UnexpectedHttp)); + warnings.reset(); + assert!(warnings.take(WarningCategory::Request)); + } + + #[tokio::test] + async fn a_poll_retries_three_times_without_overlapping_requests() { + let transport = QueueTransport::new([ + Ok(response(503, None, b"ignored")), + Err(TransportFailure::TimedOut), + Ok(response(200, Some("accepted"), EMPTY_CONFIG)), + ]); + let state = Arc::new(PollState::default()); + let mut poller = Poller { + endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") + .expect("test endpoint"), + poll_interval: Duration::from_secs(30), + request_timeout: Duration::from_secs(5), + transport, + sink: TestSink { + fail: AtomicBool::new(false), + }, + state: Arc::clone(&state), + retry_delay: no_retry_delay, + }; + + assert_eq!( + poller.poll(&CancellationToken::new()).await, + PollOutcome::Success + ); + assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 3); + assert_eq!(poller.transport.max_in_flight.load(Ordering::Relaxed), 1); + assert_eq!(state.poll_count.load(Ordering::Relaxed), 3); + assert_eq!(state.etag().as_deref(), Some("accepted")); + } + + #[tokio::test] + async fn cancellation_before_a_poll_prevents_the_first_request() { + let transport = QueueTransport::new([]); + let mut poller = Poller { + endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") + .expect("test endpoint"), + poll_interval: Duration::from_secs(30), + request_timeout: Duration::from_secs(5), + transport, + sink: TestSink { + fail: AtomicBool::new(false), + }, + state: Arc::new(PollState::default()), + retry_delay: no_retry_delay, + }; + let cancellation = CancellationToken::new(); + cancellation.cancel(); + + assert_eq!(poller.poll(&cancellation).await, PollOutcome::Cancelled); + assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 0); + } +} diff --git a/components-rs/ffe/agentless/transport.rs b/components-rs/ffe/agentless/transport.rs new file mode 100644 index 00000000000..f6571fba037 --- /dev/null +++ b/components-rs/ffe/agentless/transport.rs @@ -0,0 +1,291 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use super::super::settings::AgentlessEndpoint; +use base64::Engine; +use flate2::read::GzDecoder; +use http::header::{ACCEPT_ENCODING, CONTENT_ENCODING, CONTENT_LENGTH, ETAG, IF_NONE_MATCH}; +use http::{Method, Request}; +use http_body_util::BodyExt; +use libdd_common::http_common::{self, Body}; +use percent_encoding::percent_decode_str; +use std::future::Future; +use std::io::Read; +use std::pin::Pin; +use std::time::Duration; +use tokio_util::sync::CancellationToken; + +pub(super) const MAX_RESPONSE_BODY_BYTES: usize = 10 << 20; +const CLIENT_LANGUAGE: &str = "php"; +const INTERNAL_UNTRACED_HEADER: &str = "DD-Internal-Untraced-Request"; +const API_KEY_HEADER: &str = "DD-API-KEY"; +const AUTHORIZATION_HEADER: &str = "Authorization"; + +pub(super) type FetchFuture<'a> = + Pin> + Send + 'a>>; + +pub(super) trait Transport: Send + Sync { + fn get<'a>( + &'a self, + endpoint: &'a AgentlessEndpoint, + timeout: Duration, + etag: Option<&'a str>, + cancellation: &'a CancellationToken, + ) -> FetchFuture<'a>; +} + +#[derive(Debug, Eq, PartialEq)] +pub(super) enum TransportFailure { + BuildRequest, + Cancelled, + TimedOut, + Request, + ResponseBody, + ResponseTooLarge, +} + +pub(super) struct PollResponse { + pub(super) status: u16, + pub(super) etag: Option, + pub(super) content_encoding: Option, + pub(super) body: Vec, +} + +pub(super) struct HyperTransport { + client: libdd_common::HttpClient, +} + +impl HyperTransport { + pub(super) fn new() -> Self { + Self { + // This native client is outside PHP HTTP instrumentation, so it + // cannot recursively trace its own polling requests. Hyper also + // leaves 3xx responses visible instead of following redirects. + client: http_common::new_client_periodic(), + } + } + + async fn get_inner( + &self, + endpoint: &AgentlessEndpoint, + timeout: Duration, + etag: Option<&str>, + cancellation: &CancellationToken, + ) -> Result { + let request = build_request(endpoint, etag)?; + let operation = async { + let response = self + .client + .request(request) + .await + .map_err(|_| TransportFailure::Request)?; + let status = response.status().as_u16(); + let etag = header_value(response.headers(), ETAG); + let content_encoding = header_value(response.headers(), CONTENT_ENCODING); + + // A non-200 body can never contain configuration. Dropping it here + // also prevents a malicious error response from consuming memory. + if status != 200 { + return Ok(PollResponse { + status, + etag, + content_encoding, + body: Vec::new(), + }); + } + + if response + .headers() + .get(CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_some_and(|length| length > MAX_RESPONSE_BODY_BYTES as u64) + { + return Err(TransportFailure::ResponseTooLarge); + } + + let body = collect_bounded(response.into_body(), MAX_RESPONSE_BODY_BYTES).await?; + Ok(PollResponse { + status, + etag, + content_encoding, + body, + }) + }; + + tokio::select! { + _ = cancellation.cancelled() => Err(TransportFailure::Cancelled), + result = tokio::time::timeout(timeout, operation) => { + result.map_err(|_| TransportFailure::TimedOut)? + } + } + } +} + +impl Transport for HyperTransport { + fn get<'a>( + &'a self, + endpoint: &'a AgentlessEndpoint, + timeout: Duration, + etag: Option<&'a str>, + cancellation: &'a CancellationToken, + ) -> FetchFuture<'a> { + Box::pin(self.get_inner(endpoint, timeout, etag, cancellation)) + } +} + +fn build_request( + endpoint: &AgentlessEndpoint, + etag: Option<&str>, +) -> Result, TransportFailure> { + let (request_url, authorization) = request_url_and_authorization(endpoint.as_str())?; + let mut builder = Request::builder() + .method(Method::GET) + .uri(request_url) + .header(ACCEPT_ENCODING, "gzip") + .header("DD-Client-Library-Language", CLIENT_LANGUAGE) + .header( + "DD-Client-Library-Version", + include_str!("../../../VERSION").trim(), + ) + .header(INTERNAL_UNTRACED_HEADER, "1"); + + if let Some(etag) = etag { + builder = builder.header(IF_NONE_MATCH, etag); + } + if let Some(api_key) = endpoint.api_key() { + builder = builder.header(API_KEY_HEADER, api_key); + } + if let Some(authorization) = authorization { + builder = builder.header(AUTHORIZATION_HEADER, authorization); + } + + builder + .body(Body::empty()) + .map_err(|_| TransportFailure::BuildRequest) +} + +fn request_url_and_authorization( + endpoint: &str, +) -> Result<(String, Option), TransportFailure> { + let mut url = url::Url::parse(endpoint).map_err(|_| TransportFailure::BuildRequest)?; + if url.username().is_empty() { + return Ok((endpoint.to_owned(), None)); + } + + let mut credentials = percent_decode_str(url.username()).collect::>(); + credentials.push(b':'); + if let Some(password) = url.password() { + credentials.extend(percent_decode_str(password)); + } + let authorization = format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode(credentials) + ); + + url.set_username("") + .map_err(|_| TransportFailure::BuildRequest)?; + url.set_password(None) + .map_err(|_| TransportFailure::BuildRequest)?; + Ok((url.to_string(), Some(authorization))) +} + +fn header_value(headers: &http::HeaderMap, name: http::header::HeaderName) -> Option { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) +} + +async fn collect_bounded( + mut body: hyper::body::Incoming, + limit: usize, +) -> Result, TransportFailure> { + let mut bytes = Vec::new(); + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|_| TransportFailure::ResponseBody)?; + if let Some(data) = frame.data_ref() { + let remaining = limit.saturating_sub(bytes.len()); + if data.len() > remaining { + return Err(TransportFailure::ResponseTooLarge); + } + bytes.extend_from_slice(data); + } + } + Ok(bytes) +} + +pub(super) fn decode_response_body( + body: &[u8], + content_encoding: Option<&str>, + limit: usize, +) -> Result, TransportFailure> { + if !content_encoding.is_some_and(|value| value.trim().eq_ignore_ascii_case("gzip")) { + if body.len() > limit { + return Err(TransportFailure::ResponseTooLarge); + } + return Ok(body.to_vec()); + } + + let decoder = GzDecoder::new(body); + let mut decoded = Vec::new(); + decoder + .take(limit as u64 + 1) + .read_to_end(&mut decoded) + .map_err(|_| TransportFailure::ResponseBody)?; + if decoded.len() > limit { + return Err(TransportFailure::ResponseTooLarge); + } + Ok(decoded) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn gzip(payload: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(payload).expect("compress test payload"); + encoder.finish().expect("finish test payload") + } + + #[test] + fn decoded_body_limit_applies_after_gzip() { + let payload = vec![b'a'; 256]; + let compressed = gzip(&payload); + assert!(compressed.len() < 64); + assert_eq!( + decode_response_body(&compressed, Some(" GZIP "), 64), + Err(TransportFailure::ResponseTooLarge) + ); + assert_eq!( + decode_response_body(&gzip(b"configuration"), Some("gzip"), 64).expect("decode gzip"), + b"configuration" + ); + } + + #[test] + fn request_authentication_is_managed_only() { + let managed = + AgentlessEndpoint::build("", "datadoghq.com", "", "secret").expect("managed endpoint"); + let custom = + AgentlessEndpoint::build("https://user:password@localhost/custom", "", "", "secret") + .expect("custom endpoint"); + + let managed_request = build_request(&managed, Some("etag")).expect("managed request"); + assert_eq!(managed_request.headers()[API_KEY_HEADER], "secret"); + assert_eq!(managed_request.headers()[IF_NONE_MATCH], "etag"); + assert_eq!(managed_request.headers()[INTERNAL_UNTRACED_HEADER], "1"); + + let custom_request = build_request(&custom, None).expect("custom request"); + assert!(!custom_request.headers().contains_key(API_KEY_HEADER)); + assert_eq!( + custom_request.headers()[AUTHORIZATION_HEADER], + "Basic dXNlcjpwYXNzd29yZA==" + ); + assert_eq!(custom_request.uri(), "https://localhost/custom"); + } +} From 2dc2b63f67486a3b6f7068840e04a45b8869efe8 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 18 Sep 2026 14:02:34 +0300 Subject: [PATCH 04/12] refactor(ffe): remove PR-specific implementation comments --- components-rs/ffe.rs | 8 +++----- components-rs/ffe/agentless.rs | 1 - components-rs/ffe/agentless/poller.rs | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/components-rs/ffe.rs b/components-rs/ffe.rs index a5a8493c3b1..aa52de8c611 100644 --- a/components-rs/ffe.rs +++ b/components-rs/ffe.rs @@ -9,16 +9,14 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; use std::sync::{Arc, LazyLock}; -// This module is wired into activation by the next change in the FFE stack. #[allow(dead_code)] pub(crate) mod agentless; -// This module is wired into activation by the next change in the FFE stack. #[allow(dead_code)] pub(crate) mod settings; #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[repr(u8)] -#[allow(dead_code)] // Remaining states are connected by PR 3 activation wiring. +#[allow(dead_code)] pub(crate) enum DeliveryState { Inactive, Starting, @@ -42,7 +40,7 @@ static FFE_CONFIG: LazyLock> = static FFE_VERSION: AtomicU64 = AtomicU64::new(0); static DELIVERY_STATE: AtomicU8 = AtomicU8::new(DeliveryState::Inactive as u8); -#[allow(dead_code)] // Connected by PR 3 activation wiring. +#[allow(dead_code)] pub(crate) fn delivery_state() -> DeliveryState { match DELIVERY_STATE.load(Ordering::Acquire) { value if value == DeliveryState::Starting as u8 => DeliveryState::Starting, @@ -54,7 +52,7 @@ pub(crate) fn delivery_state() -> DeliveryState { } } -#[allow(dead_code)] // Connected by PR 3 activation wiring. +#[allow(dead_code)] pub(crate) fn set_delivery_state(state: DeliveryState) { DELIVERY_STATE.store(state as u8, Ordering::Release); } diff --git a/components-rs/ffe/agentless.rs b/components-rs/ffe/agentless.rs index c7581285052..888cdadf96a 100644 --- a/components-rs/ffe/agentless.rs +++ b/components-rs/ffe/agentless.rs @@ -45,7 +45,6 @@ pub(crate) enum WorkerStartError { } /// Owns the poll thread and makes its lifecycle explicit around `fork`. -/// PR 3 connects these methods to the tracer lifecycle hooks. pub(crate) struct AgentlessWorker { config: AgentlessWorkerConfig, state: Arc, diff --git a/components-rs/ffe/agentless/poller.rs b/components-rs/ffe/agentless/poller.rs index 2865dc4ba21..f9005494776 100644 --- a/components-rs/ffe/agentless/poller.rs +++ b/components-rs/ffe/agentless/poller.rs @@ -423,7 +423,7 @@ mod tests { } #[test] - fn retry_delay_uses_the_cross_sdk_clamps_and_jitter() { + fn retry_delay_applies_clamps_and_jitter() { assert_eq!( retry_delay(Duration::from_secs(30), 1, 0.0), Duration::from_secs(4) From 2b172c0255d2454f3012ae9c9999b6b345fc352b Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 8 Oct 2026 04:05:41 +0000 Subject: [PATCH 05/12] Activate agentless PHP feature flags with bounded process lifecycle --- components-rs/common.h | 45 +++ components-rs/datadog.h | 30 ++ components-rs/ffe.rs | 11 +- components-rs/ffe/agentless.rs | 39 +++ components-rs/ffe/agentless/poller.rs | 32 +- components-rs/ffe/lifecycle.rs | 297 ++++++++++++++++++ ext/datadog.c | 3 + .../FeatureFlags/Internal/NativeEvaluator.php | 10 +- tests/ext/ffe/agentless_fork.phpt | 35 +++ tests/ext/ffe/agentless_lifecycle.phpt | 37 +++ .../ffe/agentless_openfeature_lifecycle.phpt | 38 +++ tests/ext/ffe/agentless_recovery.phpt | 32 ++ tests/ext/ffe/agentless_shutdown.phpt | 32 ++ tests/ext/ffe/agentless_source_selection.phpt | 78 +++++ tests/ext/ffe/stubs/agentless_lifecycle.inc | 83 +++++ .../ext/ffe/stubs/agentless_lifecycle_app.php | 41 +++ tests/ext/ffe/stubs/agentless_process.inc | 80 +++++ tests/ext/ffe/stubs/agentless_process_app.php | 72 +++++ tracer/ddtrace.c | 8 +- tracer/ddtrace.stub.php | 2 + tracer/ddtrace_arginfo.h | 6 +- tracer/ffe.c | 20 ++ tracer/ffe.h | 3 + tracer/functions.c | 20 ++ 24 files changed, 1042 insertions(+), 12 deletions(-) create mode 100644 components-rs/ffe/lifecycle.rs create mode 100644 tests/ext/ffe/agentless_fork.phpt create mode 100644 tests/ext/ffe/agentless_lifecycle.phpt create mode 100644 tests/ext/ffe/agentless_openfeature_lifecycle.phpt create mode 100644 tests/ext/ffe/agentless_recovery.phpt create mode 100644 tests/ext/ffe/agentless_shutdown.phpt create mode 100644 tests/ext/ffe/agentless_source_selection.phpt create mode 100644 tests/ext/ffe/stubs/agentless_lifecycle.inc create mode 100644 tests/ext/ffe/stubs/agentless_lifecycle_app.php create mode 100644 tests/ext/ffe/stubs/agentless_process.inc create mode 100644 tests/ext/ffe/stubs/agentless_process_app.php diff --git a/components-rs/common.h b/components-rs/common.h index 60b9c5a93f8..54476ec92d7 100644 --- a/components-rs/common.h +++ b/components-rs/common.h @@ -293,6 +293,18 @@ typedef struct _zend_string _zend_string; #define ddog_DYANMIC_CONFIG_UPDATE_UNMODIFIED (_zend_string*)1 +#define ddog_DEFAULT_POLL_INTERVAL_SECONDS 30 + +#define ddog_MAX_POLL_INTERVAL_SECONDS 3600 + +#define ddog_DEFAULT_REQUEST_TIMEOUT_SECONDS 5 + +#define ddog_MAX_REQUEST_TIMEOUT_SECONDS 300 + +#define ddog_DEFAULT_INITIALIZATION_TIMEOUT_MS 10000 + +#define ddog_MAX_INITIALIZATION_TIMEOUT_MS (int64_t)INT32_MAX + #define ddog_LOG_ONCE (1 << 3) /** @@ -317,6 +329,8 @@ typedef enum ddog_ConfigurationOrigin { DDOG_CONFIGURATION_ORIGIN_UNKNOWN, } ddog_ConfigurationOrigin; +typedef uint8_t ddog_DeliveryState; + typedef enum ddog_DynamicConfigUpdateMode { DDOG_DYNAMIC_CONFIG_UPDATE_MODE_READ, DDOG_DYNAMIC_CONFIG_UPDATE_MODE_READ_WRITE, @@ -329,6 +343,12 @@ typedef enum ddog_EvaluateAt { DDOG_EVALUATE_AT_EXIT, } ddog_EvaluateAt; +typedef enum ddog_FfeConfigurationSource { + DDOG_FFE_CONFIGURATION_SOURCE_DISABLED, + DDOG_FFE_CONFIGURATION_SOURCE_REMOTE_CONFIG, + DDOG_FFE_CONFIGURATION_SOURCE_AGENTLESS, +} ddog_FfeConfigurationSource; + typedef enum ddog_InBodyLocation { DDOG_IN_BODY_LOCATION_NONE, DDOG_IN_BODY_LOCATION_START, @@ -549,6 +569,31 @@ typedef struct ddog_FfeAttribute { bool bool_value; } ddog_FfeAttribute; +typedef struct ddog_FfeRuntimeConfig { + enum ddog_FfeConfigurationSource source; + bool enabled; + bool endpoint_valid; +} ddog_FfeRuntimeConfig; + +/** + * Borrowed process settings. No pointer is retained and no credential is logged. + */ +typedef struct ddog_FfeSettingsInput { + bool enabled; + bool enabled_set; + ddog_CharSlice source; + bool source_set; + bool legacy_enabled; + bool legacy_enabled_set; + ddog_CharSlice agentless_base_url; + int64_t poll_interval_seconds; + int64_t request_timeout_seconds; + int64_t initialization_timeout_ms; + ddog_CharSlice site; + ddog_CharSlice api_key; + ddog_CharSlice environment; +} ddog_FfeSettingsInput; + /** * Flags selecting which Remote Config products/capabilities to subscribe to. * diff --git a/components-rs/datadog.h b/components-rs/datadog.h index db51f57e2d6..8416523ae99 100644 --- a/components-rs/datadog.h +++ b/components-rs/datadog.h @@ -8,6 +8,14 @@ struct _zend_string; #include "common.h" #include "telemetry.h" #include "sidecar.h" +enum ddog_DeliveryState { + DDOG_DELIVERY_STATE_INACTIVE, + DDOG_DELIVERY_STATE_STARTING, + DDOG_DELIVERY_STATE_READY, + DDOG_DELIVERY_STATE_STALE, + DDOG_DELIVERY_STATE_STOPPED, + DDOG_DELIVERY_STATE_PERMANENT_ERROR, +}; extern void (*ddog_log_callback)(ddog_CharSlice); @@ -132,6 +140,8 @@ void ddog_add_zstr_span_meta_struct_CharSlice(ddog_SpanBytes *ptr, struct _zend_string *key, ddog_CharSlice val); +ddog_DeliveryState ddog_ffe_delivery_state(void); + bool ddog_ffe_load_config(ddog_CharSlice json); bool ddog_ffe_has_config(void); @@ -144,6 +154,26 @@ struct ddog_FfeResult ddog_ffe_evaluate(ddog_CharSlice flag_key, const struct ddog_FfeAttribute *attributes, uintptr_t attributes_count); +/** + * Configure once during PHP's ordinary process setup, before sidecar startup. + * This does not start network polling; the first evaluation activates it. + */ +struct ddog_FfeRuntimeConfig ddog_ffe_configure(const struct ddog_FfeSettingsInput *input); + +struct ddog_FfeRuntimeConfig ddog_ffe_runtime_config(void); + +/** + * First-use compatibility path for PHP providers without an initialize API. + * Concurrent first callers share one deadline; later calls never renew it. + */ +void ddog_ffe_ensure_initialized(void); + +void ddog_ffe_shutdown(void); + +void ddog_ffe_prepare_for_fork(void); + +void ddog_ffe_resume_after_fork(bool child); + bool ddog_shall_log(enum ddog_Log category); void ddog_set_error_log_level(bool once); diff --git a/components-rs/ffe.rs b/components-rs/ffe.rs index f1eb1304c69..ea8bd1bd044 100644 --- a/components-rs/ffe.rs +++ b/components-rs/ffe.rs @@ -11,13 +11,14 @@ use std::sync::{Arc, LazyLock}; #[allow(dead_code)] pub(crate) mod agentless; +pub(crate) mod lifecycle; #[allow(dead_code)] pub(crate) mod settings; #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[repr(u8)] #[allow(dead_code)] -pub(crate) enum DeliveryState { +pub enum DeliveryState { Inactive, Starting, Ready, @@ -52,6 +53,11 @@ pub(crate) fn delivery_state() -> DeliveryState { } } +#[no_mangle] +pub extern "C" fn ddog_ffe_delivery_state() -> DeliveryState { + delivery_state() +} + #[allow(dead_code)] pub(crate) fn set_delivery_state(state: DeliveryState) { DELIVERY_STATE.store(state as u8, Ordering::Release); @@ -60,6 +66,7 @@ pub(crate) fn set_delivery_state(state: DeliveryState) { pub(crate) fn store_config(config: Configuration) -> ConfigurationTransition { FFE_CONFIG.store(Some(Arc::new(config))); FFE_VERSION.fetch_add(1, Ordering::AcqRel); + lifecycle::notify_configuration(); let previous = DELIVERY_STATE.swap(DeliveryState::Ready as u8, Ordering::AcqRel); if previous == DeliveryState::Ready as u8 { @@ -354,7 +361,7 @@ mod tests { use std::sync::{Mutex, Once}; static INIT_ZEND_STRING_FUNCTIONS: Once = Once::new(); - static FFE_TEST_LOCK: Mutex<()> = Mutex::new(()); + pub(super) static FFE_TEST_LOCK: Mutex<()> = Mutex::new(()); fn setup_zend_string_functions() { INIT_ZEND_STRING_FUNCTIONS.call_once(|| unsafe { diff --git a/components-rs/ffe/agentless.rs b/components-rs/ffe/agentless.rs index 888cdadf96a..ac3dca883e7 100644 --- a/components-rs/ffe/agentless.rs +++ b/components-rs/ffe/agentless.rs @@ -65,6 +65,15 @@ impl AgentlessWorker { } pub(crate) fn start(&mut self) -> Result { + // A runtime creation failure must not leave a finished thread looking + // like an active poller forever. The next ordinary evaluation may retry. + if self + .running + .as_ref() + .is_some_and(|worker| worker.thread.is_finished()) + { + self.stop_running(); + } if self.running.is_some() || self.permanently_stopped { return Ok(false); } @@ -159,3 +168,33 @@ impl Drop for AgentlessWorker { self.stop_running(); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_finished_worker_can_restart_but_shutdown_is_permanent() { + let endpoint = AgentlessEndpoint::build("http://127.0.0.1:1/config", "", "", "").unwrap(); + let mut worker = AgentlessWorker::new(AgentlessWorkerConfig::new( + endpoint, + Duration::from_secs(30), + Duration::from_secs(30), + )); + // Model the worker returning before polling (for example when Tokio + // cannot allocate its runtime). Its handle must not suppress retries. + let thread = std::thread::spawn(|| {}); + while !thread.is_finished() { + std::thread::yield_now(); + } + worker.running = Some(RunningWorker { + cancellation: CancellationToken::new(), + thread, + }); + assert_eq!(worker.start(), Ok(true)); + assert_eq!(worker.start(), Ok(false)); + worker.shutdown(); + assert!(worker.running.is_none()); + assert_eq!(worker.start(), Ok(false)); + } +} diff --git a/components-rs/ffe/agentless/poller.rs b/components-rs/ffe/agentless/poller.rs index f9005494776..96d83397f58 100644 --- a/components-rs/ffe/agentless/poller.rs +++ b/components-rs/ffe/agentless/poller.rs @@ -19,6 +19,8 @@ const MAX_POLL_ATTEMPTS: usize = 3; pub(super) trait ConfigurationSink: Send + Sync { fn apply(&self, configuration: Configuration) -> Result; + fn failed(&self) {} + fn confirmed(&self) {} } pub(super) struct GlobalConfigurationSink; @@ -27,6 +29,22 @@ impl ConfigurationSink for GlobalConfigurationSink { fn apply(&self, configuration: Configuration) -> Result { Ok(store_config(configuration)) } + + fn failed(&self) { + use crate::ffe::{ddog_ffe_has_config, set_delivery_state, DeliveryState}; + set_delivery_state(if ddog_ffe_has_config() { + DeliveryState::Stale + } else { + DeliveryState::Starting + }); + } + + fn confirmed(&self) { + use crate::ffe::{ddog_ffe_has_config, set_delivery_state, DeliveryState}; + if ddog_ffe_has_config() { + set_delivery_state(DeliveryState::Ready); + } + } } #[derive(Clone, Copy)] @@ -132,11 +150,15 @@ impl Poller { let poll = AssertUnwindSafe(self.poll(&cancellation)).catch_unwind(); match poll.await { Ok(PollOutcome::Cancelled) => return, - Ok(_) => {} - Err(_) => self.warn( - WarningCategory::UnexpectedException, - "Feature Flags agentless polling failed unexpectedly; polling continues", - ), + Ok(PollOutcome::Success) => self.sink.confirmed(), + Ok(_) => self.sink.failed(), + Err(_) => { + self.sink.failed(); + self.warn( + WarningCategory::UnexpectedException, + "Feature Flags agentless polling failed unexpectedly; polling continues", + ); + } } tokio::select! { diff --git a/components-rs/ffe/lifecycle.rs b/components-rs/ffe/lifecycle.rs new file mode 100644 index 00000000000..5383de97506 --- /dev/null +++ b/components-rs/ffe/lifecycle.rs @@ -0,0 +1,297 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use super::agentless::{AgentlessWorker, AgentlessWorkerConfig}; +use super::settings::{ConfigurationSource, FeatureFlagsSettings, SettingsInput, SourceInput}; +use super::{ddog_ffe_has_config, set_delivery_state, DeliveryState}; +use crate::log::{self, Log}; +use arc_swap::ArcSwap; +use libdd_common_ffi::slice::{AsBytes, CharSlice}; +use std::cell::RefCell; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, LazyLock, Mutex, MutexGuard}; +use std::time::Instant; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum FfeConfigurationSource { + #[default] + Disabled, + RemoteConfig, + Agentless, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct FfeRuntimeConfig { + pub source: FfeConfigurationSource, + pub enabled: bool, + pub endpoint_valid: bool, +} + +/// Borrowed process settings. No pointer is retained and no credential is logged. +#[repr(C)] +pub struct FfeSettingsInput<'a> { + pub enabled: bool, + pub enabled_set: bool, + pub source: CharSlice<'a>, + pub source_set: bool, + pub legacy_enabled: bool, + pub legacy_enabled_set: bool, + pub agentless_base_url: CharSlice<'a>, + pub poll_interval_seconds: i64, + pub request_timeout_seconds: i64, + pub initialization_timeout_ms: i64, + pub site: CharSlice<'a>, + pub api_key: CharSlice<'a>, + pub environment: CharSlice<'a>, +} + +#[derive(Default)] +struct Lifecycle { + settings: Option, + config: FfeRuntimeConfig, + worker: Option, + initialization_deadline: Option, + stopped: bool, +} + +static LIFECYCLE: LazyLock> = LazyLock::new(|| Mutex::new(Lifecycle::default())); +type ConfigurationSignal = (Mutex<()>, Condvar); +static CONFIGURATION_CHANGED: LazyLock> = + LazyLock::new(|| ArcSwap::from_pointee((Mutex::new(()), Condvar::new()))); +static INITIALIZATION_STOPPED: AtomicBool = AtomicBool::new(false); + +thread_local! { + // Keep lifecycle operations excluded across PHP's fork boundary. The + // forking thread releases this guard in both parent and child callbacks. + static FORK_GUARD: RefCell>> = const { RefCell::new(None) }; +} + +fn lifecycle() -> MutexGuard<'static, Lifecycle> { + LIFECYCLE.lock().unwrap_or_else(|error| error.into_inner()) +} + +pub(super) fn notify_configuration() { + let signal = CONFIGURATION_CHANGED.load(); + let _guard = signal.0.lock().unwrap_or_else(|error| error.into_inner()); + signal.1.notify_all(); +} + +/// Configure once during PHP's ordinary process setup, before sidecar startup. +/// This does not start network polling; the first evaluation activates it. +#[no_mangle] +pub extern "C" fn ddog_ffe_configure(input: &FfeSettingsInput<'_>) -> FfeRuntimeConfig { + let mut state = lifecycle(); + if state.settings.is_some() { + return state.config; + } + let settings = FeatureFlagsSettings::resolve(SettingsInput { + source: SourceInput { + enabled: input.enabled, + enabled_set: input.enabled_set, + source: input.source.try_to_utf8().unwrap_or("invalid"), + source_set: input.source_set, + legacy_enabled: input.legacy_enabled, + legacy_enabled_set: input.legacy_enabled_set, + }, + agentless_base_url: input.agentless_base_url.try_to_utf8().unwrap_or("invalid"), + poll_interval_seconds: input.poll_interval_seconds, + request_timeout_seconds: input.request_timeout_seconds, + initialization_timeout_ms: input.initialization_timeout_ms, + site: input.site.try_to_utf8().unwrap_or("invalid"), + api_key: input.api_key.try_to_utf8().unwrap_or(""), + environment: input.environment.try_to_utf8().unwrap_or(""), + }); + let source = match settings.resolution.source { + ConfigurationSource::RemoteConfig => FfeConfigurationSource::RemoteConfig, + ConfigurationSource::Agentless => FfeConfigurationSource::Agentless, + ConfigurationSource::Offline | ConfigurationSource::Invalid => { + FfeConfigurationSource::Disabled + } + }; + let mut endpoint_valid = false; + if settings.resolution.enabled && source == FfeConfigurationSource::Agentless { + if let Ok(endpoint) = settings.agentless_endpoint() { + endpoint_valid = true; + state.worker = Some(AgentlessWorker::new(AgentlessWorkerConfig::new( + endpoint, + settings.poll_interval, + settings.request_timeout, + ))); + } + } + state.config = FfeRuntimeConfig { + source, + enabled: settings.resolution.enabled, + endpoint_valid, + }; + state.settings = Some(settings); + state.config +} + +#[no_mangle] +pub extern "C" fn ddog_ffe_runtime_config() -> FfeRuntimeConfig { + lifecycle().config +} + +/// First-use compatibility path for PHP providers without an initialize API. +/// Concurrent first callers share one deadline; later calls never renew it. +#[no_mangle] +pub extern "C" fn ddog_ffe_ensure_initialized() { + let deadline = { + let mut state = lifecycle(); + if state.stopped || !state.config.enabled { + return; + } + if state.config.source != FfeConfigurationSource::Agentless { + return; + } + if state.worker.is_none() { + set_delivery_state(DeliveryState::PermanentError); + return; + } + if state.worker.as_mut().unwrap().start().is_err() { + log::log( + Log::Warn, + "Feature Flags agentless worker could not be started", + ); + return; + } + if state.initialization_deadline.is_none() { + if !ddog_ffe_has_config() { + set_delivery_state(DeliveryState::Starting); + } + state.initialization_deadline = + Some(Instant::now() + state.settings.as_ref().unwrap().initialization_timeout); + } + state.initialization_deadline.unwrap() + }; + let signal = CONFIGURATION_CHANGED.load_full(); + let mut guard = signal.0.lock().unwrap_or_else(|error| error.into_inner()); + while !ddog_ffe_has_config() && !INITIALIZATION_STOPPED.load(Ordering::Acquire) { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + break; + } + guard = signal + .1 + .wait_timeout(guard, remaining) + .unwrap_or_else(|error| error.into_inner()) + .0; + } +} + +#[no_mangle] +pub extern "C" fn ddog_ffe_shutdown() { + let mut state = lifecycle(); + state.stopped = true; + INITIALIZATION_STOPPED.store(true, Ordering::Release); + if let Some(worker) = &mut state.worker { + worker.shutdown(); + } + set_delivery_state(DeliveryState::Stopped); + notify_configuration(); +} + +#[no_mangle] +pub extern "C" fn ddog_ffe_prepare_for_fork() { + FORK_GUARD.with(|slot| { + let mut slot = slot.borrow_mut(); + if slot.is_none() { + let mut state = lifecycle(); + if let Some(worker) = &mut state.worker { + worker.prepare_for_fork(); + } + *slot = Some(state); + } + }); +} + +#[no_mangle] +pub extern "C" fn ddog_ffe_resume_after_fork(child: bool) { + FORK_GUARD.with(|slot| { + if let Some(mut state) = slot.borrow_mut().take() { + if child { + // Other parent threads may have been waiting on the old + // condition variable. Never reuse that synchronization state. + CONFIGURATION_CHANGED.store(Arc::new((Mutex::new(()), Condvar::new()))); + } + if let Some(worker) = &mut state.worker { + let result = if child { + worker.reset_after_fork_child() + } else { + worker.resume_after_fork_parent() + }; + if result.is_err() { + log::log( + Log::Warn, + "Feature Flags agentless worker could not restart after fork", + ); + } + } + if child && !ddog_ffe_has_config() { + state.initialization_deadline = None; + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{mpsc, Barrier}; + use std::time::Duration; + + #[test] + fn publishing_configuration_wakes_concurrent_initializers() { + let _test_guard = crate::ffe::tests::FFE_TEST_LOCK.lock().unwrap(); + crate::ffe::clear_config(); + let input = FfeSettingsInput { + enabled: true, + enabled_set: true, + source: CharSlice::from("agentless"), + source_set: true, + legacy_enabled: false, + legacy_enabled_set: false, + agentless_base_url: CharSlice::from("http://127.0.0.1:1/config"), + poll_interval_seconds: 30, + request_timeout_seconds: 30, + initialization_timeout_ms: 3000, + site: CharSlice::from(""), + api_key: CharSlice::from(""), + environment: CharSlice::from(""), + }; + assert!(ddog_ffe_configure(&input).endpoint_valid); + let barrier = Arc::new(Barrier::new(5)); + let (tx, rx) = mpsc::channel(); + let callers: Vec<_> = (0..4) + .map(|_| { + let barrier = Arc::clone(&barrier); + let tx = tx.clone(); + std::thread::spawn(move || { + barrier.wait(); + ddog_ffe_ensure_initialized(); + tx.send(()).unwrap(); + }) + }) + .collect(); + barrier.wait(); + // No caller may finish without a config while its deadline is open. + assert!(rx.recv_timeout(Duration::from_millis(100)).is_err()); + assert!(crate::ffe::ddog_ffe_load_config(CharSlice::from( + r#"{"createdAt":"2026-05-22T00:00:00Z","environment":{"name":"test"},"flags":{}}"#, + ))); + for _ in 0..4 { + rx.recv_timeout(Duration::from_secs(1)).unwrap(); + } + for caller in callers { + caller.join().unwrap(); + } + ddog_ffe_shutdown(); + assert_eq!(crate::ffe::delivery_state(), DeliveryState::Stopped); + *lifecycle() = Lifecycle::default(); + INITIALIZATION_STOPPED.store(false, Ordering::Release); + crate::ffe::clear_config(); + } +} diff --git a/ext/datadog.c b/ext/datadog.c index a2994b65e8c..aa4e9ec9027 100644 --- a/ext/datadog.c +++ b/ext/datadog.c @@ -728,6 +728,9 @@ static PHP_MINFO_FUNCTION(datadog) { void datadog_internal_handle_fork(void) { // CHILD PROCESS +#ifdef DDTRACE + ddog_ffe_resume_after_fork(true); +#endif datadog_force_new_instance_id(); datadog_sidecar_handle_fork(); #ifdef __linux__ diff --git a/src/api/FeatureFlags/Internal/NativeEvaluator.php b/src/api/FeatureFlags/Internal/NativeEvaluator.php index 419eeb0229e..51a77a2af0d 100644 --- a/src/api/FeatureFlags/Internal/NativeEvaluator.php +++ b/src/api/FeatureFlags/Internal/NativeEvaluator.php @@ -8,6 +8,7 @@ final class NativeEvaluator implements Evaluator { const WARNING_MESSAGE = 'Datadog-backed PHP feature flag evaluation has no Remote Configuration data loaded for this request. Returning default values.'; + const AGENTLESS_WARNING_MESSAGE = 'Datadog-backed PHP feature flag evaluation has no agentless configuration loaded. Returning default values.'; private $mapper; private $recordMetrics; @@ -137,6 +138,11 @@ private function withProviderState($rawResult) 'mode' => 'native_remote_config', 'reason' => $hasConfig ? 'ready' : 'configuration_missing', ); + if (function_exists('DDTrace\\Internal\\ffe_provider_state')) { + $providerState = array_merge($providerState, \DDTrace\Internal\ffe_provider_state()); + } + $warningMessage = $providerState['mode'] === 'native_agentless' + ? self::AGENTLESS_WARNING_MESSAGE : self::WARNING_MESSAGE; if (is_array($rawResult)) { if (isset($rawResult['provider_state']) && is_array($rawResult['provider_state'])) { @@ -144,7 +150,7 @@ private function withProviderState($rawResult) } if (!$hasConfig) { - $rawResult['error_message'] = self::WARNING_MESSAGE; + $rawResult['error_message'] = $warningMessage; } $rawResult['provider_state'] = $providerState; @@ -159,7 +165,7 @@ private function withProviderState($rawResult) } if (!$hasConfig) { - $rawResult->errorMessage = self::WARNING_MESSAGE; + $rawResult->errorMessage = $warningMessage; } $rawResult->providerState = $providerState; diff --git a/tests/ext/ffe/agentless_fork.phpt b/tests/ext/ffe/agentless_fork.phpt new file mode 100644 index 00000000000..024f03257d2 --- /dev/null +++ b/tests/ext/ffe/agentless_fork.phpt @@ -0,0 +1,35 @@ +--TEST-- +FFE agentless fork restarts independent parent and child configuration pollers +--SKIPIF-- + +--ENV-- +DD_FEATURE_FLAGS_ENABLED=1 +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS=1 +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS=30 +DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS=200 +DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED=0 +DD_TRACE_ENABLED=0 +DD_TRACE_CLI_ENABLED=1 +DD_REMOTE_CONFIG_ENABLED=0 +DD_INSTRUMENTATION_TELEMETRY_ENABLED=0 +DD_METRICS_OTEL_ENABLED=0 +DD_TRACE_AGENT_URL=http://127.0.0.1:1 +DD_API_KEY=custom-endpoint-must-not-receive-this +NO_PROXY=127.0.0.1,localhost +--FILE-- + +--EXPECT-- +child_forked +child_recovered +fork_clean_exit +parent_forked +parent_recovered diff --git a/tests/ext/ffe/agentless_lifecycle.phpt b/tests/ext/ffe/agentless_lifecycle.phpt new file mode 100644 index 00000000000..4c37f549323 --- /dev/null +++ b/tests/ext/ffe/agentless_lifecycle.phpt @@ -0,0 +1,37 @@ +--TEST-- +FFE agentless first evaluation, 304, last-good configuration and recovery with tracing disabled +--SKIPIF-- + +--ENV-- +DD_FEATURE_FLAGS_ENABLED=1 +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS=1 +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS=1 +DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS=1500 +DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED=0 +DD_TRACE_ENABLED=0 +DD_TRACE_CLI_ENABLED=1 +DD_REMOTE_CONFIG_ENABLED=0 +DD_INSTRUMENTATION_TELEMETRY_ENABLED=0 +DD_METRICS_OTEL_ENABLED=0 +DD_TRACE_AGENT_URL=http://127.0.0.1:1 +DD_API_KEY=custom-endpoint-must-not-receive-this +NO_PROXY=127.0.0.1,localhost +--FILE-- + +--EXPECT-- +first=blue +changed=green +stale_preserved=true +recovered_on_304=true +version_advanced_once=true +source=native_agentless +ready=true +request_protocol=true diff --git a/tests/ext/ffe/agentless_openfeature_lifecycle.phpt b/tests/ext/ffe/agentless_openfeature_lifecycle.phpt new file mode 100644 index 00000000000..961cdb5eccc --- /dev/null +++ b/tests/ext/ffe/agentless_openfeature_lifecycle.phpt @@ -0,0 +1,38 @@ +--TEST-- +FFE OpenFeature agentless first evaluation, 304, last-good configuration and recovery with tracing disabled +--SKIPIF-- + +--ENV-- +DD_FEATURE_FLAGS_ENABLED=1 +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS=1 +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS=1 +DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS=1500 +DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED=0 +DD_TRACE_ENABLED=0 +DD_TRACE_CLI_ENABLED=1 +DD_REMOTE_CONFIG_ENABLED=0 +DD_INSTRUMENTATION_TELEMETRY_ENABLED=0 +DD_METRICS_OTEL_ENABLED=0 +DD_TRACE_AGENT_URL=http://127.0.0.1:1 +DD_API_KEY=custom-endpoint-must-not-receive-this +NO_PROXY=127.0.0.1,localhost +--FILE-- + +--EXPECT-- +first=blue +changed=green +stale_preserved=true +recovered_on_304=true +version_advanced_once=true +source=native_agentless +ready=true +request_protocol=true diff --git a/tests/ext/ffe/agentless_recovery.phpt b/tests/ext/ffe/agentless_recovery.phpt new file mode 100644 index 00000000000..005c0d6c57f --- /dev/null +++ b/tests/ext/ffe/agentless_recovery.phpt @@ -0,0 +1,32 @@ +--TEST-- +FFE agentless initialization deadline is shared across evaluations and failed initial polling recovers +--SKIPIF-- + +--ENV-- +DD_FEATURE_FLAGS_ENABLED=1 +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS=1 +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS=30 +DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS=200 +DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED=0 +DD_TRACE_ENABLED=0 +DD_TRACE_CLI_ENABLED=1 +DD_REMOTE_CONFIG_ENABLED=0 +DD_INSTRUMENTATION_TELEMETRY_ENABLED=0 +DD_METRICS_OTEL_ENABLED=0 +DD_TRACE_AGENT_URL=http://127.0.0.1:1 +DD_API_KEY=custom-endpoint-must-not-receive-this +NO_PROXY=127.0.0.1,localhost +--FILE-- + +--EXPECT-- +initial_deadline_bounded +initial_failure_recovered +later_evaluations_do_not_wait diff --git a/tests/ext/ffe/agentless_shutdown.phpt b/tests/ext/ffe/agentless_shutdown.phpt new file mode 100644 index 00000000000..6b1cbe8567b --- /dev/null +++ b/tests/ext/ffe/agentless_shutdown.phpt @@ -0,0 +1,32 @@ +--TEST-- +FFE agentless shutdown cancels a configuration request before its HTTP deadline +--SKIPIF-- + +--ENV-- +DD_FEATURE_FLAGS_ENABLED=1 +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS=1 +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS=30 +DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS=200 +DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED=0 +DD_TRACE_ENABLED=0 +DD_TRACE_CLI_ENABLED=1 +DD_REMOTE_CONFIG_ENABLED=0 +DD_INSTRUMENTATION_TELEMETRY_ENABLED=0 +DD_METRICS_OTEL_ENABLED=0 +DD_TRACE_AGENT_URL=http://127.0.0.1:1 +DD_API_KEY=custom-endpoint-must-not-receive-this +NO_PROXY=127.0.0.1,localhost +--FILE-- + +--EXPECT-- +initial_deadline_bounded +later_evaluations_do_not_wait +shutdown_cancelled_inflight_request diff --git a/tests/ext/ffe/agentless_source_selection.phpt b/tests/ext/ffe/agentless_source_selection.phpt new file mode 100644 index 00000000000..4b47b32f174 --- /dev/null +++ b/tests/ext/ffe/agentless_source_selection.phpt @@ -0,0 +1,78 @@ +--TEST-- +FFE stable and legacy source settings control native activation without configuration traffic +--SKIPIF-- + +--ENV-- +DD_TRACE_ENABLED=0 +DD_TRACE_CLI_ENABLED=1 +DD_REMOTE_CONFIG_ENABLED=0 +DD_INSTRUMENTATION_TELEMETRY_ENABLED=0 +DD_METRICS_OTEL_ENABLED=0 +DD_TRACE_AGENT_URL=http://127.0.0.1:1 +--FILE-- + array('0', 'agentless', '1', false, 'native_agentless'), + 'legacy_kill' => array(null, null, '0', false, 'native_remote_config'), + 'legacy_remote_config' => array(null, null, '1', true, 'native_remote_config'), + 'explicit_remote_config' => array('1', 'remote_config', '0', true, 'native_remote_config'), + 'offline' => array('1', 'offline', '1', false, 'native_disabled'), + 'invalid' => array('1', 'unknown', '1', false, 'native_disabled'), +); +$keys = array('DD_FEATURE_FLAGS_ENABLED', 'DD_FEATURE_FLAGS_CONFIGURATION_SOURCE', 'DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED'); +$script = '\DDTrace\ffe_evaluate("flag", \DDTrace\FFE_STRING, null, array(), false); echo json_encode(\DDTrace\Internal\ffe_provider_state());'; +foreach ($cases as $name => $case) { + foreach ($keys as $i => $key) { + unset($environment[$key]); + if ($case[$i] !== null) $environment[$key] = $case[$i]; + } + $command = 'exec ' . escapeshellarg(getenv('TEST_PHP_EXECUTABLE')) . ' ' . getenv('TEST_PHP_EXTRA_ARGS') . ' -r ' . escapeshellarg($script); + $process = proc_open($command, array(array('pipe', 'r'), array('pipe', 'w'), array('pipe', 'w')), $pipes, null, $environment); + if (!is_resource($process)) throw new RuntimeException('cannot launch PHP'); + fclose($pipes[0]); + stream_set_blocking($pipes[1], false); + stream_set_blocking($pipes[2], false); + $stdout = $stderr = ''; + $start = microtime(true); + do { + $stdout .= stream_get_contents($pipes[1]); + $stderr .= stream_get_contents($pipes[2]); + $status = proc_get_status($process); + if (!$status['running']) break; + if ($connection = @stream_socket_accept($server, 0.05)) { + fclose($connection); + proc_terminate($process); + throw new RuntimeException($name . ' unexpectedly polled agentless configuration'); + } + } while (microtime(true) - $start < 3); + $stdout .= stream_get_contents($pipes[1]); + $stderr .= stream_get_contents($pipes[2]); + if ($status['running']) proc_terminate($process); + fclose($pipes[1]); + fclose($pipes[2]); + proc_close($process); + if ($status['running'] || $status['exitcode'] !== 0) throw new RuntimeException($name . ' failed: ' . $stderr); + $state = json_decode($stdout, true); + if ($state['enabled'] !== $case[3] || $state['mode'] !== $case[4] || $state['ready']) { + throw new RuntimeException($name . ' wrong source: ' . $stdout); + } + echo $name, "=ok\n"; +} +fclose($server); +?> +--EXPECT-- +stable_kill=ok +legacy_kill=ok +legacy_remote_config=ok +explicit_remote_config=ok +offline=ok +invalid=ok diff --git a/tests/ext/ffe/stubs/agentless_lifecycle.inc b/tests/ext/ffe/stubs/agentless_lifecycle.inc new file mode 100644 index 00000000000..b75983986ec --- /dev/null +++ b/tests/ext/ffe/stubs/agentless_lifecycle.inc @@ -0,0 +1,83 @@ + '2026-05-22T00:00:00Z', + 'environment' => array('name' => 'test'), + 'flags' => array('flag' => array( + 'key' => 'flag', 'enabled' => true, 'variationType' => 'STRING', + 'variations' => array('on' => array('key' => 'on', 'value' => $value)), + 'allocations' => array(array('key' => 'allocation', 'rules' => array(), 'doLog' => false, + 'splits' => array(array('variationKey' => 'on', 'shards' => array())))), + )), + )); + if ($notModified) $body = ''; + if ($index === 3) $body = '{malformed'; + fwrite($connection, "HTTP/1.1 $code\r\nContent-Type: application/json\r\nETag: $etag\r\nContent-Length: " + . strlen($body) . "\r\nConnection: close\r\n\r\n" . $body); + fclose($connection); + } while (microtime(true) < $deadline); + $stdout .= stream_get_contents($pipes[1]); + $stderr .= stream_get_contents($pipes[2]); + if ($exitCode !== 0) throw new RuntimeException('child failed or timed out: ' . $stdout . ' ' . $stderr); + echo $stdout; + if (count($requests) < 5) throw new RuntimeException('polling did not recover'); + $protocol = true; + foreach ($requests as $i => $request) { + list($route, $headers) = $request; + $protocol = $protocol && $route === 'GET /configuration?fixture=1 HTTP/1.1' + && !isset($headers['dd-api-key']) + && $headers['dd-client-library-language'] === 'php' + && $headers['dd-client-library-version'] === phpversion('ddtrace') + && $headers['dd-internal-untraced-request'] === '1'; + if ($i === 0) $protocol = $protocol && !isset($headers['if-none-match']); + if ($i > 0 && $i < 5) $protocol = $protocol && $headers['if-none-match'] === '"blue"'; + } + echo 'request_protocol=', json_encode($protocol), "\n"; +} finally { + if ($running) proc_terminate($process); + fclose($pipes[1]); + fclose($pipes[2]); + proc_close($process); + fclose($server); +} diff --git a/tests/ext/ffe/stubs/agentless_lifecycle_app.php b/tests/ext/ffe/stubs/agentless_lifecycle_app.php new file mode 100644 index 00000000000..5cce65a1b27 --- /dev/null +++ b/tests/ext/ffe/stubs/agentless_lifecycle_app.php @@ -0,0 +1,41 @@ +setProvider(new \DDTrace\OpenFeature\DataDogProvider()); + $client = $api->getClient('php-agentless-lifecycle'); +} +// No testing loader or Agent supplies this configuration. +$first = $client->getStringValue('flag', 'fallback'); +echo 'first=', $first, "\n"; +$version = \DDTrace\ffe_config_version(); +$stale = false; +$recoveredOn304 = false; +$deadline = microtime(true) + 8; +do { + $value = $client->getStringValue('flag', 'fallback'); + $state = \DDTrace\Internal\ffe_provider_state(); + if ($value === 'green') break; + if ($value !== 'blue') throw new RuntimeException('last known configuration lost'); + if (\DDTrace\ffe_config_version() !== $version) throw new RuntimeException('304 or malformed response changed config version'); + if ($state['reason'] === 'stale') $stale = true; + if ($stale && $state['reason'] === 'ready') $recoveredOn304 = true; + usleep(20000); +} while (microtime(true) < $deadline); +echo 'changed=', $value, "\n"; +echo 'stale_preserved=', json_encode($stale), "\n"; +echo 'recovered_on_304=', json_encode($recoveredOn304), "\n"; +echo 'version_advanced_once=', json_encode(\DDTrace\ffe_config_version() === $version + 1), "\n"; +echo 'source=', $state['mode'], "\n"; +echo 'ready=', json_encode($state['ready']), "\n"; diff --git a/tests/ext/ffe/stubs/agentless_process.inc b/tests/ext/ffe/stubs/agentless_process.inc new file mode 100644 index 00000000000..b74a7c0e9b4 --- /dev/null +++ b/tests/ext/ffe/stubs/agentless_process.inc @@ -0,0 +1,80 @@ + '2026-05-22T00:00:00Z', 'environment' => array('name' => 'test'), + 'flags' => array('flag' => array( + 'key' => 'flag', 'enabled' => true, 'variationType' => 'STRING', + 'variations' => array('on' => array('key' => 'on', 'value' => $value)), + 'allocations' => array(array('key' => 'allocation', 'rules' => array(), 'doLog' => false, + 'splits' => array(array('variationKey' => 'on', 'shards' => array())))), + )), + )); + } + fwrite($connection, "HTTP/1.1 $code\r\nContent-Type: application/json\r\nContent-Length: " + . strlen($body) . "\r\nConnection: close\r\n\r\n" . $body); + fclose($connection); + } while (microtime(true) - $start < 10); + $elapsed = microtime(true) - $start; + $stdout .= stream_get_contents($pipes[1]); + $stderr .= stream_get_contents($pipes[2]); + if ($exitCode !== 0) throw new RuntimeException('child failed or timed out: ' . $stdout . ' ' . $stderr); + if ($requests === 0) throw new RuntimeException('no configuration request'); + if ($scenario === 'shutdown' && $elapsed > 3) throw new RuntimeException('shutdown waited on HTTP timeout'); + // Fork output is intentionally concurrent; compare the complete set. + $lines = explode("\n", trim($stdout)); + sort($lines); + echo implode("\n", $lines), "\n"; + if ($scenario === 'shutdown') echo "shutdown_cancelled_inflight_request\n"; +} finally { + if ($running) proc_terminate($process); + fclose($pipes[1]); + fclose($pipes[2]); + proc_close($process); + foreach ($heldConnections as $connection) fclose($connection); + fclose($server); +} diff --git a/tests/ext/ffe/stubs/agentless_process_app.php b/tests/ext/ffe/stubs/agentless_process_app.php new file mode 100644 index 00000000000..88921ff8785 --- /dev/null +++ b/tests/ext/ffe/stubs/agentless_process_app.php @@ -0,0 +1,72 @@ +getStringValue('flag', 'fallback'); +$elapsed = microtime(true) - $start; + +if ($scenario === 'fork') { + if ($first !== 'blue') throw new RuntimeException('initial fetch failed'); + $version = \DDTrace\ffe_config_version(); + $pid = pcntl_fork(); + if ($pid < 0) throw new RuntimeException('fork failed'); + // The HTTP fixture only serves green after this marker. Both copies must + // fetch a new configuration after the split to observe it. + echo $pid === 0 ? "child_forked\n" : "parent_forked\n"; + $deadline = microtime(true) + 7; + $value = $first; + do { + // Merely evaluating would invoke first-use activation again and hide + // a broken fork callback. Require the background poller to publish a + // new snapshot before another evaluation is allowed to run. + $currentVersion = \DDTrace\ffe_config_version(); + if ($currentVersion !== $version) { + $version = $currentVersion; + $value = $client->getStringValue('flag', 'fallback'); + } + if ($value === 'green') break; + if ($value !== 'blue') throw new RuntimeException('fork lost configuration'); + usleep(20000); + } while (microtime(true) < $deadline); + if ($value !== 'green') throw new RuntimeException('forked worker did not resume'); + echo $pid === 0 ? "child_recovered\n" : "parent_recovered\n"; + if ($pid > 0) { + pcntl_waitpid($pid, $status); + if (!pcntl_wifexited($status) || pcntl_wexitstatus($status) !== 0) { + throw new RuntimeException('child failed'); + } + echo "fork_clean_exit\n"; + } + exit; +} + +if ($first !== 'fallback') throw new RuntimeException('unexpected initial value'); +if ($elapsed < 0.15 || $elapsed > 1) throw new RuntimeException('initialization deadline not honored'); +echo "initial_deadline_bounded\n"; +$start = microtime(true); +for ($i = 0; $i < 10; $i++) $client->getStringValue('flag', 'fallback'); +if (microtime(true) - $start > 0.5) throw new RuntimeException('initialization budget was renewed'); +echo "later_evaluations_do_not_wait\n"; +if ($scenario === 'shutdown') { + // Leave a HTTP request in flight: process shutdown must cancel it without + // waiting for the configured 30-second request timeout. + exit; +} +$deadline = microtime(true) + 7; +do { + $value = $client->getStringValue('flag', 'fallback'); + if ($value === 'blue') break; + usleep(20000); +} while (microtime(true) < $deadline); +if ($value !== 'blue') throw new RuntimeException('initial failure did not recover'); +echo "initial_failure_recovered\n"; diff --git a/tracer/ddtrace.c b/tracer/ddtrace.c index edb328bbeff..3c46c278419 100644 --- a/tracer/ddtrace.c +++ b/tracer/ddtrace.c @@ -376,6 +376,7 @@ void ddtrace_minit_late() { } void ddtrace_mshutdown() { + ddog_ffe_shutdown(); zai_uhook_mshutdown(); zai_hook_mshutdown(); @@ -697,10 +698,11 @@ bool datadog_alter_dd_trace_disabled_config(zval *old_value, zval *new_value, ze } bool ddtrace_update_remote_config_flags(ddog_RemoteConfigFlags *flags) { - flags->ffe_enabled = get_global_DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED(); + ddog_FfeRuntimeConfig ffe = ddtrace_ffe_configure(); + flags->ffe_enabled = ffe.enabled && ffe.source == DDOG_FFE_CONFIGURATION_SOURCE_REMOTE_CONFIG; flags->live_debugging_enabled = get_global_DD_DYNAMIC_INSTRUMENTATION_ENABLED(); return get_global_DD_TRACE_SIDECAR_TRACE_SENDER() - || get_global_DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED() + || (ffe.enabled && (flags->ffe_enabled || ffe.endpoint_valid)) || get_global_DD_METRICS_OTEL_ENABLED(); } @@ -714,9 +716,11 @@ void ddtrace_internal_handle_prefork(void) { ddtrace_coms_flush_shutdown_writer_synchronous(); } #endif + ddog_ffe_prepare_for_fork(); } void ddtrace_internal_handle_postfork(void) { + ddog_ffe_resume_after_fork(false); #if JOIN_BGS_BEFORE_FORK if (!get_global_DD_TRACE_SIDECAR_TRACE_SENDER()) { ddtrace_coms_restart_writer(); diff --git a/tracer/ddtrace.stub.php b/tracer/ddtrace.stub.php index 0ad50beed33..2b59af05e8b 100644 --- a/tracer/ddtrace.stub.php +++ b/tracer/ddtrace.stub.php @@ -1097,6 +1097,8 @@ function flush_ffe_evaluation_metrics(): bool {} /** @internal Submit one final evaluation without waiting or reconnecting. */ function record_ffe_flag_evaluation(string $flagKey, ?string $variant, ?string $allocationKey, ?string $targetingKey, array $attributes, ?string $errorType, bool $runtimeDefaultUsed, bool $observeFullEvaluationData): bool {} + function ffe_provider_state(): array {} + } namespace datadog\appsec\v2 { diff --git a/tracer/ddtrace_arginfo.h b/tracer/ddtrace_arginfo.h index c34449762b9..57e0ba39913 100644 --- a/tracer/ddtrace_arginfo.h +++ b/tracer/ddtrace_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: fabc819a43be4dc1ce12f823113a9dd3d48af7cd */ + * Stub hash: 4d752651844dcd36b0f181251672f19467640697 */ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_DDTrace_trace_method, 0, 3, _IS_BOOL, 0) ZEND_ARG_TYPE_INFO(0, className, IS_STRING, 0) @@ -267,6 +267,8 @@ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_DDTrace_Internal_record_ffe_flag ZEND_ARG_TYPE_INFO(0, observeFullEvaluationData, _IS_BOOL, 0) ZEND_END_ARG_INFO() +#define arginfo_DDTrace_Internal_ffe_provider_state arginfo_DDTrace_current_context + ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_datadog_appsec_v2_track_user_login_success, 0, 1, IS_VOID, 0) ZEND_ARG_TYPE_INFO(0, login, IS_STRING, 0) ZEND_ARG_TYPE_MASK(0, user, MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_NULL, "null") @@ -454,6 +456,7 @@ ZEND_FUNCTION(DDTrace_Internal_handle_fork); ZEND_FUNCTION(DDTrace_Internal_record_ffe_evaluation_metric); ZEND_FUNCTION(DDTrace_Internal_flush_ffe_evaluation_metrics); ZEND_FUNCTION(DDTrace_Internal_record_ffe_flag_evaluation); +ZEND_FUNCTION(DDTrace_Internal_ffe_provider_state); ZEND_FUNCTION(datadog_appsec_v2_track_user_login_success); ZEND_FUNCTION(datadog_appsec_v2_track_user_login_failure); ZEND_FUNCTION(dd_trace_env_config); @@ -557,6 +560,7 @@ static const zend_function_entry ext_functions[] = { ZEND_RAW_FENTRY(ZEND_NS_NAME("DDTrace\\Internal", "record_ffe_evaluation_metric"), zif_DDTrace_Internal_record_ffe_evaluation_metric, arginfo_DDTrace_Internal_record_ffe_evaluation_metric, 0, NULL, NULL) ZEND_RAW_FENTRY(ZEND_NS_NAME("DDTrace\\Internal", "flush_ffe_evaluation_metrics"), zif_DDTrace_Internal_flush_ffe_evaluation_metrics, arginfo_DDTrace_Internal_flush_ffe_evaluation_metrics, 0, NULL, NULL) ZEND_RAW_FENTRY(ZEND_NS_NAME("DDTrace\\Internal", "record_ffe_flag_evaluation"), zif_DDTrace_Internal_record_ffe_flag_evaluation, arginfo_DDTrace_Internal_record_ffe_flag_evaluation, 0, NULL, NULL) + ZEND_RAW_FENTRY(ZEND_NS_NAME("DDTrace\\Internal", "ffe_provider_state"), zif_DDTrace_Internal_ffe_provider_state, arginfo_DDTrace_Internal_ffe_provider_state, 0, NULL, NULL) ZEND_RAW_FENTRY(ZEND_NS_NAME("datadog\\appsec\\v2", "track_user_login_success"), zif_datadog_appsec_v2_track_user_login_success, arginfo_datadog_appsec_v2_track_user_login_success, 0, NULL, NULL) ZEND_RAW_FENTRY(ZEND_NS_NAME("datadog\\appsec\\v2", "track_user_login_failure"), zif_datadog_appsec_v2_track_user_login_failure, arginfo_datadog_appsec_v2_track_user_login_failure, 0, NULL, NULL) ZEND_FE(dd_trace_env_config, arginfo_dd_trace_env_config) diff --git a/tracer/ffe.c b/tracer/ffe.c index a364c60f774..38a81eb8cac 100644 --- a/tracer/ffe.c +++ b/tracer/ffe.c @@ -3,6 +3,7 @@ #include "configuration.h" #include "span.h" #include +#include #include #include #include @@ -17,6 +18,25 @@ ZEND_EXTERN_MODULE_GLOBALS(datadog); #define DD_FFE_EXPOSURE_BUFFER_LIMIT 1000 #define DD_FFE_CONTEXT_FIELD_LIMIT 256 +ddog_FfeRuntimeConfig ddtrace_ffe_configure(void) { + const ddog_FfeSettingsInput settings = { + .enabled = get_global_DD_FEATURE_FLAGS_ENABLED(), + .enabled_set = zai_config_memoized_entries[DATADOG_CONFIG_DD_FEATURE_FLAGS_ENABLED].name_index != ZAI_CONFIG_ORIGIN_DEFAULT, + .source = dd_zend_string_to_CharSlice(get_global_DD_FEATURE_FLAGS_CONFIGURATION_SOURCE()), + .source_set = zai_config_memoized_entries[DATADOG_CONFIG_DD_FEATURE_FLAGS_CONFIGURATION_SOURCE].name_index != ZAI_CONFIG_ORIGIN_DEFAULT, + .legacy_enabled = get_global_DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED(), + .legacy_enabled_set = zai_config_memoized_entries[DATADOG_CONFIG_DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED].name_index != ZAI_CONFIG_ORIGIN_DEFAULT, + .agentless_base_url = dd_zend_string_to_CharSlice(get_global_DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL()), + .poll_interval_seconds = get_global_DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS(), + .request_timeout_seconds = get_global_DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS(), + .initialization_timeout_ms = get_global_DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS(), + .site = dd_zend_string_to_CharSlice(get_global_DD_SITE()), + .api_key = dd_zend_string_to_CharSlice(get_global_DD_API_KEY()), + .environment = dd_zend_string_to_CharSlice(get_global_DD_ENV()), + }; + return ddog_ffe_configure(&settings); +} + bool ddtrace_ffe_record_flag_evaluation(zend_string *flag_key, zend_string *variant, zend_string *allocation_key, zend_string *targeting_key, HashTable *attributes, zend_string *error_type, bool runtime_default_used, bool observe_full_evaluation_data) { diff --git a/tracer/ffe.h b/tracer/ffe.h index 2353db57780..e542c11eb66 100644 --- a/tracer/ffe.h +++ b/tracer/ffe.h @@ -5,6 +5,9 @@ #include #include #include +#include + +ddog_FfeRuntimeConfig ddtrace_ffe_configure(void); bool ddtrace_ffe_record_evaluation_metric(zend_string *flag_key, zend_string *variant, const char *reason, const char *error_type, zend_string *allocation_key); bool ddtrace_ffe_flush_evaluation_metrics(void); diff --git a/tracer/functions.c b/tracer/functions.c index fb1257592f5..bd1f0591efc 100644 --- a/tracer/functions.c +++ b/tracer/functions.c @@ -1874,6 +1874,7 @@ PHP_FUNCTION(DDTrace_ffe_evaluate) { attrs_count = idx; } + ddog_ffe_ensure_initialized(); result = ddog_ffe_evaluate( dd_zend_string_to_CharSlice(flag_key), type_id, @@ -2954,6 +2955,25 @@ PHP_FUNCTION(DDTrace_Internal_record_ffe_flag_evaluation) { targeting_key, attributes, error_type, runtime_default_used, observe_full_evaluation_data)); } +PHP_FUNCTION(DDTrace_Internal_ffe_provider_state) { + ZEND_PARSE_PARAMETERS_NONE(); + ddog_FfeRuntimeConfig config = ddog_ffe_runtime_config(); + bool has_config = ddog_ffe_has_config(); + array_init(return_value); + add_assoc_bool(return_value, "ready", config.enabled && has_config); + add_assoc_bool(return_value, "hasConfig", has_config); + add_assoc_long(return_value, "configVersion", ddog_ffe_config_version()); + add_assoc_bool(return_value, "productionRuntime", true); + add_assoc_bool(return_value, "enabled", config.enabled); + const char *mode = config.source == DDOG_FFE_CONFIGURATION_SOURCE_AGENTLESS + ? "native_agentless" : config.source == DDOG_FFE_CONFIGURATION_SOURCE_REMOTE_CONFIG + ? "native_remote_config" : "native_disabled"; + add_assoc_string(return_value, "mode", (char *) mode); + add_assoc_string(return_value, "reason", !config.enabled ? "disabled" + : has_config ? (ddog_ffe_delivery_state() == DDOG_DELIVERY_STATE_STALE ? "stale" : "ready") + : "configuration_missing"); +} + /* {{{ proto array generate_distributed_tracing_headers() */ PHP_FUNCTION(DDTrace_generate_distributed_tracing_headers) { zend_array *inject = NULL; From 2d4bfb08daa7829ea1261c367e972218612d547a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 8 Oct 2026 04:56:05 +0000 Subject: [PATCH 06/12] Release FFE worker thread state and align activation fixtures --- components-rs/common.h | 4 +- components-rs/ffe/agentless.rs | 42 ++++++++++++++++++++ components-rs/ffe/lifecycle.rs | 23 ++++++++--- components-rs/ffe/settings.rs | 10 +++-- metadata/supported-configurations.json | 6 +-- tests/ext/ffe/stubs/agentless_lifecycle.inc | 2 +- tests/ext/ffe/stubs/agentless_process.inc | 2 +- tests/ext/telemetry/config.phpt | 24 +++++++---- tooling/generate-supported-configurations.sh | 6 +++ tracer/ffe.c | 3 ++ 10 files changed, 101 insertions(+), 21 deletions(-) diff --git a/components-rs/common.h b/components-rs/common.h index 54476ec92d7..3328e95882a 100644 --- a/components-rs/common.h +++ b/components-rs/common.h @@ -576,7 +576,8 @@ typedef struct ddog_FfeRuntimeConfig { } ddog_FfeRuntimeConfig; /** - * Borrowed process settings. No pointer is retained and no credential is logged. + * Borrowed process settings. Strings are copied; credentials are never logged. + * The optional cleanup callback must remain valid until shutdown joins workers. */ typedef struct ddog_FfeSettingsInput { bool enabled; @@ -592,6 +593,7 @@ typedef struct ddog_FfeSettingsInput { ddog_CharSlice site; ddog_CharSlice api_key; ddog_CharSlice environment; + void (*thread_cleanup)(void*); } ddog_FfeSettingsInput; /** diff --git a/components-rs/ffe/agentless.rs b/components-rs/ffe/agentless.rs index ac3dca883e7..60c177c79ef 100644 --- a/components-rs/ffe/agentless.rs +++ b/components-rs/ffe/agentless.rs @@ -8,6 +8,7 @@ use self::poller::{GlobalConfigurationSink, PollState, Poller}; use self::transport::HyperTransport; use super::settings::AgentlessEndpoint; use crate::log::{self, Log}; +use std::ffi::c_void; use std::sync::Arc; use std::thread::JoinHandle; use std::time::Duration; @@ -18,6 +19,7 @@ pub(crate) struct AgentlessWorkerConfig { endpoint: AgentlessEndpoint, poll_interval: Duration, request_timeout: Duration, + pub(crate) thread_cleanup: Option, } impl AgentlessWorkerConfig { @@ -30,6 +32,20 @@ impl AgentlessWorkerConfig { endpoint, poll_interval, request_timeout, + thread_cleanup: None, + } + } +} + +// PHP intercepts Rust TLS destructors to permit extension unloading. Threads +// without PHP GSHUTDOWN must explicitly drain that registry after their runtime +// and logging guards are dropped, including on an early return or unwind. +struct ThreadCleanup(Option); + +impl Drop for ThreadCleanup { + fn drop(&mut self) { + if let Some(cleanup) = self.0 { + cleanup(std::ptr::null_mut()); } } } @@ -86,6 +102,7 @@ impl AgentlessWorker { let thread = std::thread::Builder::new() .name("ddtrace-ffe-agentless".to_owned()) .spawn(move || { + let _thread_cleanup = ThreadCleanup(config.thread_cleanup); let _log_guard = tracing::dispatcher::set_default(&log_dispatch); let runtime = match tokio::runtime::Builder::new_current_thread() .enable_all() @@ -172,6 +189,31 @@ impl Drop for AgentlessWorker { #[cfg(test)] mod tests { use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[test] + fn worker_cleanup_runs_on_its_thread_after_runtime_shutdown() { + static CLEANUPS: AtomicUsize = AtomicUsize::new(0); + extern "C" fn cleanup(_: *mut c_void) { + // Runtime and tracing guards must have exited before TLS is drained. + assert!(tokio::runtime::Handle::try_current().is_err()); + assert_eq!(std::thread::current().name(), Some("ddtrace-ffe-agentless")); + CLEANUPS.fetch_add(1, Ordering::SeqCst); + } + let endpoint = AgentlessEndpoint::build("http://127.0.0.1:1/config", "", "", "").unwrap(); + let mut config = + AgentlessWorkerConfig::new(endpoint, Duration::from_secs(30), Duration::from_secs(30)); + config.thread_cleanup = Some(cleanup); + let mut worker = AgentlessWorker::new(config); + worker.start().unwrap(); + worker.prepare_for_fork(); + assert_eq!(CLEANUPS.load(Ordering::SeqCst), 1); + worker.resume_after_fork_parent().unwrap(); + worker.shutdown(); + assert_eq!(CLEANUPS.load(Ordering::SeqCst), 2); + worker.shutdown(); + assert_eq!(CLEANUPS.load(Ordering::SeqCst), 2); + } #[test] fn a_finished_worker_can_restart_but_shutdown_is_permanent() { diff --git a/components-rs/ffe/lifecycle.rs b/components-rs/ffe/lifecycle.rs index 5383de97506..08269a3c768 100644 --- a/components-rs/ffe/lifecycle.rs +++ b/components-rs/ffe/lifecycle.rs @@ -8,6 +8,7 @@ use crate::log::{self, Log}; use arc_swap::ArcSwap; use libdd_common_ffi::slice::{AsBytes, CharSlice}; use std::cell::RefCell; +use std::ffi::c_void; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Condvar, LazyLock, Mutex, MutexGuard}; use std::time::Instant; @@ -29,7 +30,8 @@ pub struct FfeRuntimeConfig { pub endpoint_valid: bool, } -/// Borrowed process settings. No pointer is retained and no credential is logged. +/// Borrowed process settings. Strings are copied; credentials are never logged. +/// The optional cleanup callback must remain valid until shutdown joins workers. #[repr(C)] pub struct FfeSettingsInput<'a> { pub enabled: bool, @@ -45,6 +47,7 @@ pub struct FfeSettingsInput<'a> { pub site: CharSlice<'a>, pub api_key: CharSlice<'a>, pub environment: CharSlice<'a>, + pub thread_cleanup: Option, } #[derive(Default)] @@ -83,7 +86,7 @@ pub(super) fn notify_configuration() { #[no_mangle] pub extern "C" fn ddog_ffe_configure(input: &FfeSettingsInput<'_>) -> FfeRuntimeConfig { let mut state = lifecycle(); - if state.settings.is_some() { + if state.settings.is_some() || state.stopped { return state.config; } let settings = FeatureFlagsSettings::resolve(SettingsInput { @@ -114,11 +117,13 @@ pub extern "C" fn ddog_ffe_configure(input: &FfeSettingsInput<'_>) -> FfeRuntime if settings.resolution.enabled && source == FfeConfigurationSource::Agentless { if let Ok(endpoint) = settings.agentless_endpoint() { endpoint_valid = true; - state.worker = Some(AgentlessWorker::new(AgentlessWorkerConfig::new( + let mut worker_config = AgentlessWorkerConfig::new( endpoint, settings.poll_interval, settings.request_timeout, - ))); + ); + worker_config.thread_cleanup = input.thread_cleanup; + state.worker = Some(AgentlessWorker::new(worker_config)); } } state.config = FfeRuntimeConfig { @@ -187,9 +192,13 @@ pub extern "C" fn ddog_ffe_shutdown() { let mut state = lifecycle(); state.stopped = true; INITIALIZATION_STOPPED.store(true, Ordering::Release); - if let Some(worker) = &mut state.worker { + if let Some(mut worker) = state.worker.take() { worker.shutdown(); } + state.settings = None; + // Static configuration has no automatic destructor at module shutdown. + // Release its owned evaluator data only after the publisher has joined. + super::clear_config(); set_delivery_state(DeliveryState::Stopped); notify_configuration(); } @@ -261,6 +270,7 @@ mod tests { site: CharSlice::from(""), api_key: CharSlice::from(""), environment: CharSlice::from(""), + thread_cleanup: None, }; assert!(ddog_ffe_configure(&input).endpoint_valid); let barrier = Arc::new(Barrier::new(5)); @@ -290,6 +300,9 @@ mod tests { } ddog_ffe_shutdown(); assert_eq!(crate::ffe::delivery_state(), DeliveryState::Stopped); + assert!(!ddog_ffe_has_config()); + assert!(lifecycle().worker.is_none()); + assert!(lifecycle().settings.is_none()); *lifecycle() = Lifecycle::default(); INITIALIZATION_STOPPED.store(false, Ordering::Release); crate::ffe::clear_config(); diff --git a/components-rs/ffe/settings.rs b/components-rs/ffe/settings.rs index 07a300acbb9..2b228586f01 100644 --- a/components-rs/ffe/settings.rs +++ b/components-rs/ffe/settings.rs @@ -139,7 +139,7 @@ pub struct SettingsInput<'a> { /// `Debug`, because it owns the API key and potentially credential-bearing URL. pub struct FeatureFlagsSettings { pub resolution: SourceResolution, - agentless_base_url: String, + agentless_base_url: Option, pub poll_interval: Duration, pub request_timeout: Duration, pub initialization_timeout: Duration, @@ -176,7 +176,9 @@ impl FeatureFlagsSettings { Self { resolution: resolve_source(input.source), - agentless_base_url: input.agentless_base_url.to_owned(), + // PHP INI uses an empty string for the optional URL's unset value. + agentless_base_url: (!input.agentless_base_url.trim().is_empty()) + .then(|| input.agentless_base_url.to_owned()), poll_interval: Duration::from_secs(poll_interval_seconds as u64), request_timeout: Duration::from_secs(request_timeout_seconds as u64), initialization_timeout: Duration::from_millis(initialization_timeout_ms as u64), @@ -189,7 +191,7 @@ impl FeatureFlagsSettings { pub fn agentless_endpoint(&self) -> Result { AgentlessEndpoint::build( - &self.agentless_base_url, + self.agentless_base_url.as_deref().unwrap_or_default(), &self.site, &self.environment, &self.api_key, @@ -502,6 +504,8 @@ mod tests { environment: "", }); + assert!(settings.agentless_base_url.is_none()); + assert!(settings.agentless_endpoint().unwrap().is_managed()); assert_eq!(settings.poll_interval, Duration::from_secs(30)); assert_eq!(settings.request_timeout, Duration::from_secs(5)); assert_eq!(settings.initialization_timeout, Duration::from_secs(10)); diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index e8cb4d572d4..772d554f5b3 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -399,7 +399,7 @@ ], "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS": [ { - "implementation": "A", + "implementation": "B", "type": "int", "default": "10000" } @@ -429,7 +429,7 @@ { "implementation": "A", "type": "string", - "default": "", + "default": null, "sensitive": true } ], @@ -704,7 +704,7 @@ ], "DD_SITE": [ { - "implementation": "A", + "implementation": "C", "type": "string", "default": "datadoghq.com" } diff --git a/tests/ext/ffe/stubs/agentless_lifecycle.inc b/tests/ext/ffe/stubs/agentless_lifecycle.inc index b75983986ec..beede542efb 100644 --- a/tests/ext/ffe/stubs/agentless_lifecycle.inc +++ b/tests/ext/ffe/stubs/agentless_lifecycle.inc @@ -4,7 +4,7 @@ $server = stream_socket_server('tcp://127.0.0.1:0', $errno, $error); if (!$server) throw new RuntimeException($error); $address = stream_socket_get_name($server, false); putenv('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL=http://' . $address . '/configuration?fixture=1'); -$command = 'exec ' . escapeshellarg(getenv('TEST_PHP_EXECUTABLE')) . ' ' +$command = (PHP_OS === 'WINNT' ? '' : 'exec ') . escapeshellarg(getenv('TEST_PHP_EXECUTABLE')) . ' ' . getenv('TEST_PHP_EXTRA_ARGS') . ' ' . escapeshellarg(__DIR__ . '/agentless_lifecycle_app.php'); $process = proc_open($command, array(array('pipe', 'r'), array('pipe', 'w'), array('pipe', 'w')), $pipes); if (!is_resource($process)) throw new RuntimeException('cannot launch agentless PHP'); diff --git a/tests/ext/ffe/stubs/agentless_process.inc b/tests/ext/ffe/stubs/agentless_process.inc index b74a7c0e9b4..8fb021067cd 100644 --- a/tests/ext/ffe/stubs/agentless_process.inc +++ b/tests/ext/ffe/stubs/agentless_process.inc @@ -4,7 +4,7 @@ $server = stream_socket_server('tcp://127.0.0.1:0', $errno, $error); if (!$server) throw new RuntimeException($error); $address = stream_socket_get_name($server, false); putenv('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL=http://' . $address . '/configuration'); -$command = 'exec ' . escapeshellarg(getenv('TEST_PHP_EXECUTABLE')) . ' ' +$command = (PHP_OS === 'WINNT' ? '' : 'exec ') . escapeshellarg(getenv('TEST_PHP_EXECUTABLE')) . ' ' . getenv('TEST_PHP_EXTRA_ARGS') . ' ' . escapeshellarg(__DIR__ . '/agentless_process_app.php'); $process = proc_open($command, array(array('pipe', 'r'), array('pipe', 'w'), array('pipe', 'w')), $pipes); if (!is_resource($process)) throw new RuntimeException('cannot launch agentless PHP'); diff --git a/tests/ext/telemetry/config.phpt b/tests/ext/telemetry/config.phpt index 30eb46f3ff5..47c86d165c4 100644 --- a/tests/ext/telemetry/config.phpt +++ b/tests/ext/telemetry/config.phpt @@ -18,6 +18,7 @@ DD_TRACE_GENERATE_ROOT_SPAN=0 DD_TRACE_AUTOFINISH_SPANS=1 DD_INSTRUMENTATION_TELEMETRY_ENABLED=1 DD_AGENT_HOST=127.0.0.2 +DD_SITE=datadoghq.eu DD_TRACE_AGENT_PORT=3 DD_AUTOLOAD_NO_COMPILE= DD_TRACE_AGENT_TIMEOUT=200 @@ -95,6 +96,15 @@ Array ) [2] => Array + ( + [name] => DD_SITE + [value] => datadoghq.eu + [origin] => env_var + [config_id] =>%w + [seq_id] =>%w + ) + + [3] => Array ( [name] => DD_TRACE_AGENT_PORT [value] => 3 @@ -103,7 +113,7 @@ Array [seq_id] => ) - [3] => Array + [4] => Array ( [name] => DD_INSTRUMENTATION_TELEMETRY_ENABLED [value] => 1 @@ -112,7 +122,7 @@ Array [seq_id] => ) - [4] => Array + [5] => Array ( [name] => DD_TRACE_GIT_METADATA_ENABLED [value] => 0 @@ -121,7 +131,7 @@ Array [seq_id] => ) - [5] => Array + [6] => Array ( [name] => DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED [value] => 0 @@ -130,7 +140,7 @@ Array [seq_id] => ) - [6] => Array + [7] => Array ( [name] => DD_TRACE_RATE_LIMIT [value] => 1000000 @@ -139,7 +149,7 @@ Array [seq_id] => ) - [7] => Array + [8] => Array ( [name] => DD_TRACE_IGNORE_AGENT_SAMPLING_RATES [value] => 1 @@ -148,7 +158,7 @@ Array [seq_id] => ) - [8] => Array + [9] => Array ( [name] => DD_TRACE_GENERATE_ROOT_SPAN [value] => 0 @@ -157,7 +167,7 @@ Array [seq_id] => ) - [9] => Array + [10] => Array ( [name] => ssi_forced_injection_enabled [value] => False diff --git a/tooling/generate-supported-configurations.sh b/tooling/generate-supported-configurations.sh index 0a70d2c2614..0ce7071c8b0 100755 --- a/tooling/generate-supported-configurations.sh +++ b/tooling/generate-supported-configurations.sh @@ -63,6 +63,12 @@ function map_type($raw) { } function normalize_default($v, $type, $name) { + // PHP INI represents an unset optional URL as an empty string. The FFE + // settings resolver converts it to None and selects the managed endpoint; + // report that semantic default rather than an empty custom endpoint. + if ($name === 'DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL' && $v === '') { + return null; + } if (strtoupper($v) === 'NULL') { // OTEL env vars are string-typed and use "" (not null) as their "unset" default. if (strpos($name, 'OTEL_') === 0) { diff --git a/tracer/ffe.c b/tracer/ffe.c index 38a81eb8cac..7f6e88f9a8a 100644 --- a/tracer/ffe.c +++ b/tracer/ffe.c @@ -33,6 +33,9 @@ ddog_FfeRuntimeConfig ddtrace_ffe_configure(void) { .site = dd_zend_string_to_CharSlice(get_global_DD_SITE()), .api_key = dd_zend_string_to_CharSlice(get_global_DD_API_KEY()), .environment = dd_zend_string_to_CharSlice(get_global_DD_ENV()), +#ifdef CXA_THREAD_ATEXIT_WRAPPER + .thread_cleanup = dd_run_rust_thread_destructors, +#endif }; return ddog_ffe_configure(&settings); } From e14c46638a1a8adb743c8468cbfea79f55e3a2fe Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 8 Oct 2026 05:21:49 +0000 Subject: [PATCH 07/12] test: synchronize agentless shutdown with an in-flight request --- tests/ext/ffe/stubs/agentless_process.inc | 16 +++++++++++++--- tests/ext/ffe/stubs/agentless_process_app.php | 9 +++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/ext/ffe/stubs/agentless_process.inc b/tests/ext/ffe/stubs/agentless_process.inc index 8fb021067cd..39e8deb105e 100644 --- a/tests/ext/ffe/stubs/agentless_process.inc +++ b/tests/ext/ffe/stubs/agentless_process.inc @@ -8,7 +8,7 @@ $command = (PHP_OS === 'WINNT' ? '' : 'exec ') . escapeshellarg(getenv('TEST_PHP . getenv('TEST_PHP_EXTRA_ARGS') . ' ' . escapeshellarg(__DIR__ . '/agentless_process_app.php'); $process = proc_open($command, array(array('pipe', 'r'), array('pipe', 'w'), array('pipe', 'w')), $pipes); if (!is_resource($process)) throw new RuntimeException('cannot launch agentless PHP'); -fclose($pipes[0]); +if ($scenario !== 'shutdown') fclose($pipes[0]); stream_set_blocking($pipes[1], false); stream_set_blocking($pipes[2], false); $stdout = $stderr = ''; @@ -17,6 +17,7 @@ $heldConnections = array(); $exitCode = null; $running = true; $start = microtime(true); +$shutdownStart = null; try { do { $stdout .= stream_get_contents($pipes[1]); @@ -34,6 +35,13 @@ try { $requests++; if ($scenario === 'shutdown') { $heldConnections[] = $connection; + if ($shutdownStart === null) { + // Release the child only once an HTTP request is in flight. + // Worker scheduling may outlast its initialization budget. + $shutdownStart = microtime(true); + fwrite($pipes[0], "request_received\n"); + fclose($pipes[0]); + } continue; } if ($scenario === 'recovery' && $requests === 1) { @@ -59,12 +67,13 @@ try { . strlen($body) . "\r\nConnection: close\r\n\r\n" . $body); fclose($connection); } while (microtime(true) - $start < 10); - $elapsed = microtime(true) - $start; $stdout .= stream_get_contents($pipes[1]); $stderr .= stream_get_contents($pipes[2]); if ($exitCode !== 0) throw new RuntimeException('child failed or timed out: ' . $stdout . ' ' . $stderr); if ($requests === 0) throw new RuntimeException('no configuration request'); - if ($scenario === 'shutdown' && $elapsed > 3) throw new RuntimeException('shutdown waited on HTTP timeout'); + if ($scenario === 'shutdown' && microtime(true) - $shutdownStart > 3) { + throw new RuntimeException('shutdown waited on HTTP timeout'); + } // Fork output is intentionally concurrent; compare the complete set. $lines = explode("\n", trim($stdout)); sort($lines); @@ -72,6 +81,7 @@ try { if ($scenario === 'shutdown') echo "shutdown_cancelled_inflight_request\n"; } finally { if ($running) proc_terminate($process); + if (is_resource($pipes[0])) fclose($pipes[0]); fclose($pipes[1]); fclose($pipes[2]); proc_close($process); diff --git a/tests/ext/ffe/stubs/agentless_process_app.php b/tests/ext/ffe/stubs/agentless_process_app.php index 88921ff8785..50e7ac3f5c2 100644 --- a/tests/ext/ffe/stubs/agentless_process_app.php +++ b/tests/ext/ffe/stubs/agentless_process_app.php @@ -58,8 +58,13 @@ if (microtime(true) - $start > 0.5) throw new RuntimeException('initialization budget was renewed'); echo "later_evaluations_do_not_wait\n"; if ($scenario === 'shutdown') { - // Leave a HTTP request in flight: process shutdown must cancel it without - // waiting for the configured 30-second request timeout. + // The initialization deadline can expire before the worker is scheduled. + // Confirm the fixture has a request before measuring its cancellation. + stream_set_timeout(STDIN, 5); + if (fgets(STDIN) !== "request_received\n") { + throw new RuntimeException('fixture did not observe an in-flight request'); + } + // Shutdown must not wait for the configured 30-second request timeout. exit; } $deadline = microtime(true) + 7; From e51cff765a9e731a49e3f737d89aa9c45c4e57a9 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 8 Oct 2026 05:50:44 +0000 Subject: [PATCH 08/12] fix: decode agentless configuration JSON API resources --- Cargo.toml | 2 +- components-rs/ffe/agentless/poller.rs | 69 ++++++++++++++++++++- tests/ext/ffe/stubs/agentless_lifecycle.inc | 8 ++- tests/ext/ffe/stubs/agentless_process.inc | 8 ++- 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 76990872f66..ff68c9aeacf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,7 +97,7 @@ tracing-core = "0.1.32" tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "fmt", "env-filter", "smallvec"] } tokio = { version = "1.36", features = ["macros", "rt", "sync", "time"] } tokio-util = "0.7.11" -serde_json = "1.0.113" +serde_json = { version = "1.0.113", features = ["raw_value"] } regex = "1.10.5" regex-automata = "0.4.5" http = "1.4" diff --git a/components-rs/ffe/agentless/poller.rs b/components-rs/ffe/agentless/poller.rs index 96d83397f58..3dec7dd4de8 100644 --- a/components-rs/ffe/agentless/poller.rs +++ b/components-rs/ffe/agentless/poller.rs @@ -17,6 +17,26 @@ use tokio_util::sync::CancellationToken; const MAX_POLL_ATTEMPTS: usize = 3; +#[derive(serde::Deserialize)] +struct ConfigurationResponse<'a> { + #[serde(borrow)] + data: ConfigurationResource<'a>, +} + +#[derive(serde::Deserialize)] +struct ConfigurationResource<'a> { + #[serde(rename = "type")] + _resource_type: ConfigurationResourceType, + #[serde(borrow)] + attributes: &'a serde_json::value::RawValue, +} + +#[derive(serde::Deserialize)] +enum ConfigurationResourceType { + #[serde(rename = "universal-flag-configuration")] + UniversalFlagConfiguration, +} + pub(super) trait ConfigurationSink: Send + Sync { fn apply(&self, configuration: Configuration) -> Result; fn failed(&self) {} @@ -276,7 +296,13 @@ impl Poller { } }; - let configuration = match UniversalFlagConfig::from_json(body) { + // The managed endpoint returns a JSON:API resource, not a bare UFC. + // Validate its type before handing the original attributes bytes to + // the shared evaluator; malformed resources must not advance the ETag. + let configuration = match serde_json::from_slice::>(&body) + .and_then(|response| { + UniversalFlagConfig::from_json(response.data.attributes.get().as_bytes().to_vec()) + }) { Ok(configuration) => Configuration::from_server_response(configuration), Err(_) => { self.warn( @@ -329,12 +355,14 @@ mod tests { use std::collections::VecDeque; use std::sync::atomic::{AtomicBool, AtomicUsize}; - const EMPTY_CONFIG: &[u8] = br#"{ + const EMPTY_CONFIG: &[u8] = br#"{"data": { + "type": "universal-flag-configuration", + "attributes": { "createdAt": "2026-05-22T00:00:00.000Z", "format": "SERVER", "environment": {"name": "test"}, "flags": {} - }"#; + }}}"#; struct UnusedTransport; @@ -352,6 +380,7 @@ mod tests { struct TestSink { fail: AtomicBool, + applications: AtomicUsize, } struct QueueTransport { @@ -398,6 +427,7 @@ mod tests { impl ConfigurationSink for TestSink { fn apply(&self, _configuration: Configuration) -> Result { + self.applications.fetch_add(1, Ordering::Relaxed); if self.fail.load(Ordering::Relaxed) { Err(()) } else { @@ -415,6 +445,7 @@ mod tests { transport: UnusedTransport, sink: TestSink { fail: AtomicBool::new(fail_apply), + applications: AtomicUsize::new(0), }, state: Arc::new(PollState::default()), retry_delay, @@ -493,6 +524,36 @@ mod tests { assert_eq!(poller.state.etag().as_deref(), Some("new")); } + #[test] + fn malformed_jsonapi_resources_keep_the_previous_configuration_and_etag() { + let mut poller = test_poller(false); + assert_eq!( + poller.process_response(response(200, Some("accepted"), EMPTY_CONFIG)), + PollOutcome::Success + ); + let valid: serde_json::Value = serde_json::from_slice(EMPTY_CONFIG).unwrap(); + let attributes = valid["data"]["attributes"].clone(); + for malformed in [ + attributes.clone(), + serde_json::json!({"data": {"type": "other-resource", "attributes": attributes}}), + serde_json::json!({"data": {"attributes": attributes}}), + serde_json::json!({"data": {"type": "universal-flag-configuration"}}), + serde_json::json!({"data": {"type": "universal-flag-configuration", "attributes": null}}), + serde_json::json!({"data": {"type": "universal-flag-configuration", "attributes": {}}}), + ] { + assert_eq!( + poller.process_response(response( + 200, + Some("rejected"), + &serde_json::to_vec(&malformed).unwrap() + )), + PollOutcome::Stop + ); + assert_eq!(poller.state.etag().as_deref(), Some("accepted")); + assert_eq!(poller.sink.applications.load(Ordering::Relaxed), 1); + } + } + #[test] fn non_200_responses_are_never_decoded() { let mut poller = test_poller(false); @@ -536,6 +597,7 @@ mod tests { transport, sink: TestSink { fail: AtomicBool::new(false), + applications: AtomicUsize::new(0), }, state: Arc::clone(&state), retry_delay: no_retry_delay, @@ -562,6 +624,7 @@ mod tests { transport, sink: TestSink { fail: AtomicBool::new(false), + applications: AtomicUsize::new(0), }, state: Arc::new(PollState::default()), retry_delay: no_retry_delay, diff --git a/tests/ext/ffe/stubs/agentless_lifecycle.inc b/tests/ext/ffe/stubs/agentless_lifecycle.inc index beede542efb..a3003511d5e 100644 --- a/tests/ext/ffe/stubs/agentless_lifecycle.inc +++ b/tests/ext/ffe/stubs/agentless_lifecycle.inc @@ -41,7 +41,7 @@ try { $code = $notModified ? '304 Not Modified' : '200 OK'; $etag = $index === 3 ? '"bad"' : ($index <= 4 ? '"blue"' : '"green"'); $value = $index <= 4 ? 'blue' : 'green'; - $body = json_encode(array( + $configuration = array( 'createdAt' => '2026-05-22T00:00:00Z', 'environment' => array('name' => 'test'), 'flags' => array('flag' => array( @@ -50,7 +50,11 @@ try { 'allocations' => array(array('key' => 'allocation', 'rules' => array(), 'doLog' => false, 'splits' => array(array('variationKey' => 'on', 'shards' => array())))), )), - )); + ); + $configuration['format'] = 'SERVER'; + $body = json_encode(array('data' => array( + 'type' => 'universal-flag-configuration', 'attributes' => $configuration, + ))); if ($notModified) $body = ''; if ($index === 3) $body = '{malformed'; fwrite($connection, "HTTP/1.1 $code\r\nContent-Type: application/json\r\nETag: $etag\r\nContent-Length: " diff --git a/tests/ext/ffe/stubs/agentless_process.inc b/tests/ext/ffe/stubs/agentless_process.inc index 39e8deb105e..26d1e52e498 100644 --- a/tests/ext/ffe/stubs/agentless_process.inc +++ b/tests/ext/ffe/stubs/agentless_process.inc @@ -53,7 +53,7 @@ try { $forked = strpos($stdout, 'parent_forked') !== false && strpos($stdout, 'child_forked') !== false; $value = $scenario === 'fork' && $forked ? 'green' : 'blue'; $code = '200 OK'; - $body = json_encode(array( + $configuration = array( 'createdAt' => '2026-05-22T00:00:00Z', 'environment' => array('name' => 'test'), 'flags' => array('flag' => array( 'key' => 'flag', 'enabled' => true, 'variationType' => 'STRING', @@ -61,7 +61,11 @@ try { 'allocations' => array(array('key' => 'allocation', 'rules' => array(), 'doLog' => false, 'splits' => array(array('variationKey' => 'on', 'shards' => array())))), )), - )); + ); + $configuration['format'] = 'SERVER'; + $body = json_encode(array('data' => array( + 'type' => 'universal-flag-configuration', 'attributes' => $configuration, + ))); } fwrite($connection, "HTTP/1.1 $code\r\nContent-Type: application/json\r\nContent-Length: " . strlen($body) . "\r\nConnection: close\r\n\r\n" . $body); From 8bcc7e005cfd9f62dd62ef0ca57d7284ddaf4a41 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 8 Oct 2026 06:13:12 +0000 Subject: [PATCH 09/12] Preserve existing configuration registry implementation labels --- metadata/supported-configurations.json | 134 ++++++++++++------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 772d554f5b3..b95baf6b5b5 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -10,7 +10,7 @@ ], "DD_API_KEY": [ { - "implementation": "A", + "implementation": "C", "type": "string", "default": "", "sensitive": true @@ -53,7 +53,7 @@ ], "DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE": [ { - "implementation": "A", + "implementation": "D", "type": "string", "default": "ident", "aliases": [ @@ -77,14 +77,14 @@ ], "DD_APPSEC_EXTRA_HEADERS": [ { - "implementation": "A", + "implementation": "B", "type": "array", "default": "" } ], "DD_APPSEC_HELPER_LOG_FILE": [ { - "implementation": "A", + "implementation": "B", "type": "string", "default": "" } @@ -98,14 +98,14 @@ ], "DD_APPSEC_HTTP_BLOCKED_TEMPLATE_HTML": [ { - "implementation": "A", + "implementation": "C", "type": "string", "default": "" } ], "DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON": [ { - "implementation": "A", + "implementation": "C", "type": "string", "default": "" } @@ -147,14 +147,14 @@ ], "DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP": [ { - "implementation": "A", + "implementation": "B", "type": "string", "default": "(?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|jsessionid|phpsessid|asp\\.net[_-]sessionid|sid|jwt" } ], "DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP": [ { - "implementation": "A", + "implementation": "G", "type": "string", "default": "(?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|(?:(?:api|private|public|access)[_-]?)key(?:[_-]?id)?|(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?|jsessionid|phpsessid|asp\\.net(?:[_-]|-)sessionid|sid|jwt)(?:\\s*=([^;&]+)|\"\\s*:\\s*(\"[^\"]+\"|\\d+))|bearer\\s+([a-z0-9\\._\\-]+)|token\\s*:\\s*([a-z0-9]{13})|gh[opsu]_([0-9a-zA-Z]{36})|ey[I-L][\\w=-]+\\.(ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?)|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}([^\\-]+)[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*([a-z0-9\\/\\.+]{100,})" } @@ -175,21 +175,21 @@ ], "DD_APPSEC_RULES": [ { - "implementation": "A", + "implementation": "C", "type": "string", "default": "" } ], "DD_APPSEC_SCA_ENABLED": [ { - "implementation": "A", + "implementation": "C", "type": "boolean", "default": "false" } ], "DD_APPSEC_STACK_TRACE_ENABLED": [ { - "implementation": "A", + "implementation": "B", "type": "boolean", "default": "true" } @@ -238,7 +238,7 @@ ], "DD_APPSEC_WAF_TIMEOUT": [ { - "implementation": "A", + "implementation": "D", "type": "int", "default": "10000" } @@ -357,7 +357,7 @@ ], "DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS": [ { - "implementation": "A", + "implementation": "C", "type": "array", "default": "" } @@ -378,7 +378,7 @@ ], "DD_EXCEPTION_REPLAY_CAPTURE_MAX_FRAMES": [ { - "implementation": "A", + "implementation": "C", "type": "int", "default": "-1" } @@ -413,7 +413,7 @@ ], "DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED": [ { - "implementation": "A", + "implementation": "B", "type": "boolean", "default": "true" } @@ -463,21 +463,21 @@ ], "DD_GIT_COMMIT_SHA": [ { - "implementation": "A", + "implementation": "B", "type": "string", "default": "" } ], "DD_GIT_REPOSITORY_URL": [ { - "implementation": "A", + "implementation": "B", "type": "string", "default": "" } ], "DD_HOSTNAME": [ { - "implementation": "A", + "implementation": "B", "type": "string", "default": "" } @@ -575,7 +575,7 @@ ], "DD_PROFILING_ALLOCATION_ENABLED": [ { - "implementation": "A", + "implementation": "B", "type": "boolean", "default": "true" } @@ -589,7 +589,7 @@ ], "DD_PROFILING_ENABLED": [ { - "implementation": "A", + "implementation": "D", "type": "boolean", "default": "true" } @@ -669,7 +669,7 @@ ], "DD_PROFILING_WALLTIME_ENABLED": [ { - "implementation": "A", + "implementation": "B", "type": "boolean", "default": "true" } @@ -690,14 +690,14 @@ ], "DD_SERVICE": [ { - "implementation": "A", + "implementation": "F", "type": "string", "default": "" } ], "DD_SERVICE_MAPPING": [ { - "implementation": "A", + "implementation": "C", "type": "map", "default": "" } @@ -718,7 +718,7 @@ ], "DD_SPAN_SAMPLING_RULES_FILE": [ { - "implementation": "A", + "implementation": "B", "type": "string", "default": "" } @@ -739,7 +739,7 @@ ], "DD_TELEMETRY_HEARTBEAT_INTERVAL": [ { - "implementation": "A", + "implementation": "C", "type": "int", "default": "60" } @@ -844,7 +844,7 @@ ], "DD_TRACE_AGENT_TIMEOUT": [ { - "implementation": "A", + "implementation": "C", "type": "int", "default": "3000" } @@ -885,7 +885,7 @@ ], "DD_TRACE_ANALYTICS_ENABLED": [ { - "implementation": "A", + "implementation": "B", "type": "boolean", "default": "false" } @@ -948,7 +948,7 @@ ], "DD_TRACE_BUFFER_SIZE": [ { - "implementation": "A", + "implementation": "B", "type": "int", "default": "2097152" } @@ -1064,7 +1064,7 @@ ], "DD_TRACE_DEBUG": [ { - "implementation": "A", + "implementation": "B", "type": "boolean", "default": "false" } @@ -1112,7 +1112,7 @@ ], "DD_TRACE_ELASTICSEARCH_ANALYTICS_ENABLED": [ { - "implementation": "A", + "implementation": "B", "type": "boolean", "default": "false", "aliases": [ @@ -1357,14 +1357,14 @@ ], "DD_TRACE_HEADER_TAGS": [ { - "implementation": "A", + "implementation": "B", "type": "map", "default": "" } ], "DD_TRACE_HEALTH_METRICS_ENABLED": [ { - "implementation": "A", + "implementation": "B", "type": "boolean", "default": "false" } @@ -1412,7 +1412,7 @@ ], "DD_TRACE_HTTP_CLIENT_ERROR_STATUSES": [ { - "implementation": "A", + "implementation": "B", "type": "array", "default": "400-499" } @@ -1433,7 +1433,7 @@ ], "DD_TRACE_HTTP_SERVER_ERROR_STATUSES": [ { - "implementation": "A", + "implementation": "C", "type": "array", "default": "500-599" } @@ -1461,7 +1461,7 @@ ], "DD_TRACE_KAFKA_ANALYTICS_ENABLED": [ { - "implementation": "A", + "implementation": "B", "type": "boolean", "default": "false", "aliases": [ @@ -1620,7 +1620,7 @@ ], "DD_TRACE_LOG_LEVEL": [ { - "implementation": "A", + "implementation": "B", "type": "string", "default": "error" } @@ -1729,7 +1729,7 @@ ], "DD_TRACE_MEMCACHE_ANALYTICS_ENABLED": [ { - "implementation": "A", + "implementation": "B", "type": "boolean", "default": "false", "aliases": [ @@ -1797,7 +1797,7 @@ ], "DD_TRACE_MONGO_ANALYTICS_ENABLED": [ { - "implementation": "A", + "implementation": "B", "type": "boolean", "default": "false", "aliases": [ @@ -1878,7 +1878,7 @@ ], "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP": [ { - "implementation": "A", + "implementation": "G", "type": "string", "default": "(?i)(?:(?:\"|%22)?)(?:(?:old[-_]?|new[-_]?)?p(?:ass)?w(?:or)?d(?:1|2)?|pass(?:[-_]?phrase)?|secret|(?:api[-_]?|private[-_]?|public[-_]?|access[-_]?|secret[-_]?|app(?:lication)?[-_]?)key(?:[-_]?id)?|token|consumer[-_]?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:(?:\\s|%20)*(?:=|%3D)[^&]+|(?:\"|%22)(?:\\s|%20)*(?::|%3A)(?:\\s|%20)*(?:\"|%22)(?:%2[^2]|%[^2]|[^\"%])+(?:\"|%22))|(?:bearer(?:\\s|%20)+[a-z0-9._\\-]+|token(?::|%3A)[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|(^|[^\\w%-]|%[0-9a-f]{2})ey[I-L][\\w-]+(?:=|%3D)*\\.ey[I-L][\\w-]+(?:=|%3D)*(?:\\.(?:[\\w.+/=-]|%3D|%2F|%2B)+)?|-{5}BEGIN(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY-{5}[^\\-]+-{5}END(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY(?:-{5})?(?:\\n|%0A)?|(?:ssh-(?:rsa|dss)|ecdsa-[a-z0-9]+-[a-z0-9]+)(?:\\s|%20|%09)+(?:[a-z0-9/.+]|%2F|%5C|%2B){100,}(?:=|%3D)*(?:(?:\\s|%20|%09)+[a-z0-9._-]+)?)" } @@ -2062,28 +2062,28 @@ ], "DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT": [ { - "implementation": "A", + "implementation": "B", "type": "string", "default": "continue" } ], "DD_TRACE_PROPAGATION_STYLE": [ { - "implementation": "A", + "implementation": "D", "type": "array", "default": "datadog,tracecontext,baggage" } ], "DD_TRACE_PROPAGATION_STYLE_EXTRACT": [ { - "implementation": "A", + "implementation": "E", "type": "array", "default": "datadog,tracecontext,B3,B3 single header,baggage" } ], "DD_TRACE_PROPAGATION_STYLE_INJECT": [ { - "implementation": "A", + "implementation": "D", "type": "array", "default": "datadog,tracecontext,baggage" } @@ -2242,7 +2242,7 @@ ], "DD_TRACE_RETRY_INTERVAL": [ { - "implementation": "A", + "implementation": "B", "type": "int", "default": "100" } @@ -2276,14 +2276,14 @@ ], "DD_TRACE_SAMPLE_RATE": [ { - "implementation": "A", + "implementation": "D", "type": "decimal", "default": "-1" } ], "DD_TRACE_SAMPLING_RULES": [ { - "implementation": "A", + "implementation": "D", "type": "array", "default": "[]" } @@ -2407,7 +2407,7 @@ ], "DD_TRACE_STARTUP_LOGS": [ { - "implementation": "A", + "implementation": "C", "type": "boolean", "default": "true" } @@ -2421,7 +2421,7 @@ ], "DD_TRACE_STRIPE_ANALYTICS_ENABLED": [ { - "implementation": "A", + "implementation": "B", "type": "boolean", "default": "false", "aliases": [ @@ -2431,7 +2431,7 @@ ], "DD_TRACE_STRIPE_ANALYTICS_SAMPLE_RATE": [ { - "implementation": "A", + "implementation": "B", "type": "decimal", "default": "1.0", "aliases": [ @@ -2756,7 +2756,7 @@ ], "OTEL_EXPORTER_OTLP_ENDPOINT": [ { - "implementation": "A", + "implementation": "B", "type": "string", "default": "http://localhost:4318" } @@ -2771,14 +2771,14 @@ ], "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT": [ { - "implementation": "A", + "implementation": "C", "type": "string", "default": "http://localhost:4318" } ], "OTEL_EXPORTER_OTLP_LOGS_HEADERS": [ { - "implementation": "A", + "implementation": "B", "type": "map", "default": null, "sensitive": true @@ -2786,7 +2786,7 @@ ], "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL": [ { - "implementation": "A", + "implementation": "D", "type": "string", "default": "http/protobuf" } @@ -2800,7 +2800,7 @@ ], "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": [ { - "implementation": "A", + "implementation": "B", "type": "string", "default": "http://localhost:4318" } @@ -2815,21 +2815,21 @@ ], "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": [ { - "implementation": "A", + "implementation": "B", "type": "string", "default": "http/protobuf" } ], "OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE": [ { - "implementation": "A", + "implementation": "C", "type": "string", "default": "cumulative" } ], "OTEL_EXPORTER_OTLP_METRICS_TIMEOUT": [ { - "implementation": "A", + "implementation": "B", "type": "int", "default": "10000" } @@ -2850,42 +2850,42 @@ ], "OTEL_LOGS_EXPORTER": [ { - "implementation": "A", + "implementation": "C", "type": "array", "default": "otlp" } ], "OTEL_LOG_LEVEL": [ { - "implementation": "A", + "implementation": "F", "type": "string", "default": "info" } ], "OTEL_METRICS_EXPORTER": [ { - "implementation": "A", + "implementation": "E", "type": "array", "default": "otlp" } ], "OTEL_METRIC_EXPORT_INTERVAL": [ { - "implementation": "A", + "implementation": "B", "type": "int", "default": "60000" } ], "OTEL_METRIC_EXPORT_TIMEOUT": [ { - "implementation": "A", + "implementation": "B", "type": "int", "default": "30000" } ], "OTEL_PROPAGATORS": [ { - "implementation": "A", + "implementation": "C", "type": "array", "default": "tracecontext,baggage" } @@ -2899,28 +2899,28 @@ ], "OTEL_SERVICE_NAME": [ { - "implementation": "A", + "implementation": "C", "type": "string", "default": "" } ], "OTEL_TRACES_EXPORTER": [ { - "implementation": "A", + "implementation": "I", "type": "array", "default": "otlp" } ], "OTEL_TRACES_SAMPLER": [ { - "implementation": "A", + "implementation": "E", "type": "string", "default": "parentbased_always_on" } ], "OTEL_TRACES_SAMPLER_ARG": [ { - "implementation": "A", + "implementation": "E", "type": "string", "default": "" } From 7f4fa4a557dab7b51c5242d29b6bf983f6ac39e5 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 8 Oct 2026 07:33:44 +0000 Subject: [PATCH 10/12] Make PHP agentless lifecycle fixtures deterministic across CI runtimes --- tests/ext/ffe/agentless_fork.phpt | 2 +- tests/ext/ffe/agentless_source_selection.phpt | 11 +++--- tests/ext/ffe/stubs/agentless_lifecycle.inc | 34 ++++++++++++------- .../ext/ffe/stubs/agentless_lifecycle_app.php | 30 +++++++++------- tests/ext/ffe/stubs/agentless_process.inc | 6 ++-- tests/ext/ffe/stubs/agentless_process_app.php | 11 ++---- .../stubs/agentless_source_selection_app.php | 5 +++ 7 files changed, 58 insertions(+), 41 deletions(-) create mode 100644 tests/ext/ffe/stubs/agentless_source_selection_app.php diff --git a/tests/ext/ffe/agentless_fork.phpt b/tests/ext/ffe/agentless_fork.phpt index 024f03257d2..4e0c8e19e91 100644 --- a/tests/ext/ffe/agentless_fork.phpt +++ b/tests/ext/ffe/agentless_fork.phpt @@ -12,7 +12,7 @@ DD_FEATURE_FLAGS_ENABLED=1 DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS=1 DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS=30 -DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS=200 +DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS=10000 DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED=0 DD_TRACE_ENABLED=0 DD_TRACE_CLI_ENABLED=1 diff --git a/tests/ext/ffe/agentless_source_selection.phpt b/tests/ext/ffe/agentless_source_selection.phpt index 4b47b32f174..829930bde3f 100644 --- a/tests/ext/ffe/agentless_source_selection.phpt +++ b/tests/ext/ffe/agentless_source_selection.phpt @@ -29,13 +29,14 @@ $cases = array( 'invalid' => array('1', 'unknown', '1', false, 'native_disabled'), ); $keys = array('DD_FEATURE_FLAGS_ENABLED', 'DD_FEATURE_FLAGS_CONFIGURATION_SOURCE', 'DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED'); -$script = '\DDTrace\ffe_evaluate("flag", \DDTrace\FFE_STRING, null, array(), false); echo json_encode(\DDTrace\Internal\ffe_provider_state());'; foreach ($cases as $name => $case) { foreach ($keys as $i => $key) { unset($environment[$key]); if ($case[$i] !== null) $environment[$key] = $case[$i]; } - $command = 'exec ' . escapeshellarg(getenv('TEST_PHP_EXECUTABLE')) . ' ' . getenv('TEST_PHP_EXTRA_ARGS') . ' -r ' . escapeshellarg($script); + // A file also avoids Windows cmd.exe changing quotes in inline PHP code. + $command = (PHP_OS === 'WINNT' ? '' : 'exec ') . escapeshellarg(getenv('TEST_PHP_EXECUTABLE')) . ' ' . getenv('TEST_PHP_EXTRA_ARGS') + . ' ' . escapeshellarg(__DIR__ . '/stubs/agentless_source_selection_app.php'); $process = proc_open($command, array(array('pipe', 'r'), array('pipe', 'w'), array('pipe', 'w')), $pipes, null, $environment); if (!is_resource($process)) throw new RuntimeException('cannot launch PHP'); fclose($pipes[0]); @@ -43,9 +44,11 @@ foreach ($cases as $name => $case) { stream_set_blocking($pipes[2], false); $stdout = $stderr = ''; $start = microtime(true); + $readyAt = null; do { $stdout .= stream_get_contents($pipes[1]); $stderr .= stream_get_contents($pipes[2]); + if ($readyAt === null && strpos($stdout, "fixture_child_ready\n") !== false) $readyAt = microtime(true); $status = proc_get_status($process); if (!$status['running']) break; if ($connection = @stream_socket_accept($server, 0.05)) { @@ -53,7 +56,7 @@ foreach ($cases as $name => $case) { proc_terminate($process); throw new RuntimeException($name . ' unexpectedly polled agentless configuration'); } - } while (microtime(true) - $start < 3); + } while (microtime(true) - ($readyAt === null ? $start : $readyAt) < ($readyAt === null ? 60 : 3)); $stdout .= stream_get_contents($pipes[1]); $stderr .= stream_get_contents($pipes[2]); if ($status['running']) proc_terminate($process); @@ -61,7 +64,7 @@ foreach ($cases as $name => $case) { fclose($pipes[2]); proc_close($process); if ($status['running'] || $status['exitcode'] !== 0) throw new RuntimeException($name . ' failed: ' . $stderr); - $state = json_decode($stdout, true); + $state = json_decode(str_replace("fixture_child_ready\n", '', $stdout), true); if ($state['enabled'] !== $case[3] || $state['mode'] !== $case[4] || $state['ready']) { throw new RuntimeException($name . ' wrong source: ' . $stdout); } diff --git a/tests/ext/ffe/stubs/agentless_lifecycle.inc b/tests/ext/ffe/stubs/agentless_lifecycle.inc index a3003511d5e..8d212d56d3a 100644 --- a/tests/ext/ffe/stubs/agentless_lifecycle.inc +++ b/tests/ext/ffe/stubs/agentless_lifecycle.inc @@ -15,11 +15,15 @@ $stdout = $stderr = ''; $requests = array(); $exitCode = null; $running = true; +$phase = 0; +$expectedEtag = null; try { - $deadline = microtime(true) + 12; + $startedAt = microtime(true); + $readyAt = null; do { $stdout .= stream_get_contents($pipes[1]); $stderr .= stream_get_contents($pipes[2]); + if ($readyAt === null && strpos($stdout, "fixture_child_ready\n") !== false) $readyAt = microtime(true); $status = proc_get_status($process); if (!$status['running']) { $running = false; @@ -35,12 +39,15 @@ try { $pair = explode(':', $line, 2); if (count($pair) === 2) $headers[strtolower($pair[0])] = trim($pair[1]); } - $requests[] = array($route, $headers); - $index = count($requests); - $notModified = $index === 2 || $index === 4; + $requests[] = array($route, $headers, $expectedEtag); + // Keep stale/304 states observable even when sanitizers preempt the + // child for an entire poll interval. The child must acknowledge each. + if ($phase === 3 && strpos($stdout, "fixture_stale_observed\n") !== false) $phase = 4; + if ($phase === 4 && strpos($stdout, "fixture_304_observed\n") !== false) $phase = 5; + $notModified = in_array($phase, array(1, 4, 6), true); $code = $notModified ? '304 Not Modified' : '200 OK'; - $etag = $index === 3 ? '"bad"' : ($index <= 4 ? '"blue"' : '"green"'); - $value = $index <= 4 ? 'blue' : 'green'; + $etag = $phase === 3 ? '"bad"' : ($phase < 5 ? '"blue"' : '"green"'); + $value = $phase < 5 ? 'blue' : 'green'; $configuration = array( 'createdAt' => '2026-05-22T00:00:00Z', 'environment' => array('name' => 'test'), @@ -56,26 +63,29 @@ try { 'type' => 'universal-flag-configuration', 'attributes' => $configuration, ))); if ($notModified) $body = ''; - if ($index === 3) $body = '{malformed'; + if ($phase === 3) $body = '{malformed'; fwrite($connection, "HTTP/1.1 $code\r\nContent-Type: application/json\r\nETag: $etag\r\nContent-Length: " . strlen($body) . "\r\nConnection: close\r\n\r\n" . $body); fclose($connection); - } while (microtime(true) < $deadline); + if ($phase === 0) { $expectedEtag = '"blue"'; $phase = 1; } + elseif ($phase === 1) $phase = 3; + elseif ($phase === 5) { $expectedEtag = '"green"'; $phase = 6; } + } while (microtime(true) - ($readyAt === null ? $startedAt : $readyAt) < ($readyAt === null ? 60 : 35)); $stdout .= stream_get_contents($pipes[1]); $stderr .= stream_get_contents($pipes[2]); if ($exitCode !== 0) throw new RuntimeException('child failed or timed out: ' . $stdout . ' ' . $stderr); - echo $stdout; + echo str_replace(array("fixture_child_ready\n", "fixture_stale_observed\n", "fixture_304_observed\n"), '', $stdout); if (count($requests) < 5) throw new RuntimeException('polling did not recover'); $protocol = true; foreach ($requests as $i => $request) { - list($route, $headers) = $request; + list($route, $headers, $expectedEtag) = $request; $protocol = $protocol && $route === 'GET /configuration?fixture=1 HTTP/1.1' && !isset($headers['dd-api-key']) && $headers['dd-client-library-language'] === 'php' && $headers['dd-client-library-version'] === phpversion('ddtrace') && $headers['dd-internal-untraced-request'] === '1'; - if ($i === 0) $protocol = $protocol && !isset($headers['if-none-match']); - if ($i > 0 && $i < 5) $protocol = $protocol && $headers['if-none-match'] === '"blue"'; + if ($expectedEtag === null) $protocol = $protocol && !isset($headers['if-none-match']); + else $protocol = $protocol && $headers['if-none-match'] === $expectedEtag; } echo 'request_protocol=', json_encode($protocol), "\n"; } finally { diff --git a/tests/ext/ffe/stubs/agentless_lifecycle_app.php b/tests/ext/ffe/stubs/agentless_lifecycle_app.php index 5cce65a1b27..ee38c5d9c9e 100644 --- a/tests/ext/ffe/stubs/agentless_lifecycle_app.php +++ b/tests/ext/ffe/stubs/agentless_lifecycle_app.php @@ -1,14 +1,6 @@ getClient('php-agentless-lifecycle'); } // No testing loader or Agent supplies this configuration. +echo "fixture_child_ready\n"; $first = $client->getStringValue('flag', 'fallback'); echo 'first=', $first, "\n"; $version = \DDTrace\ffe_config_version(); $stale = false; $recoveredOn304 = false; -$deadline = microtime(true) + 8; +$deadline = microtime(true) + 30; do { + $before = \DDTrace\ffe_config_version(); $value = $client->getStringValue('flag', 'fallback'); $state = \DDTrace\Internal\ffe_provider_state(); + $after = \DDTrace\ffe_config_version(); + // These APIs are independent reads, not an atomic snapshot. A poll can + // publish green after the blue evaluation and before the version read. + if ($before !== $after) continue; if ($value === 'green') break; if ($value !== 'blue') throw new RuntimeException('last known configuration lost'); - if (\DDTrace\ffe_config_version() !== $version) throw new RuntimeException('304 or malformed response changed config version'); - if ($state['reason'] === 'stale') $stale = true; - if ($stale && $state['reason'] === 'ready') $recoveredOn304 = true; + if ($after !== $version) throw new RuntimeException('304 or malformed response changed config version'); + if (!$stale && $state['reason'] === 'stale') { + $stale = true; + echo "fixture_stale_observed\n"; + } + if ($stale && !$recoveredOn304 && $state['reason'] === 'ready') { + $recoveredOn304 = true; + echo "fixture_304_observed\n"; + } usleep(20000); } while (microtime(true) < $deadline); echo 'changed=', $value, "\n"; diff --git a/tests/ext/ffe/stubs/agentless_process.inc b/tests/ext/ffe/stubs/agentless_process.inc index 26d1e52e498..c9e5c11725b 100644 --- a/tests/ext/ffe/stubs/agentless_process.inc +++ b/tests/ext/ffe/stubs/agentless_process.inc @@ -17,11 +17,13 @@ $heldConnections = array(); $exitCode = null; $running = true; $start = microtime(true); +$readyAt = null; $shutdownStart = null; try { do { $stdout .= stream_get_contents($pipes[1]); $stderr .= stream_get_contents($pipes[2]); + if ($readyAt === null && strpos($stdout, "fixture_child_ready\n") !== false) $readyAt = microtime(true); $status = proc_get_status($process); if (!$status['running']) { $running = false; @@ -70,7 +72,7 @@ try { fwrite($connection, "HTTP/1.1 $code\r\nContent-Type: application/json\r\nContent-Length: " . strlen($body) . "\r\nConnection: close\r\n\r\n" . $body); fclose($connection); - } while (microtime(true) - $start < 10); + } while (microtime(true) - ($readyAt === null ? $start : $readyAt) < ($readyAt === null ? 60 : 20)); $stdout .= stream_get_contents($pipes[1]); $stderr .= stream_get_contents($pipes[2]); if ($exitCode !== 0) throw new RuntimeException('child failed or timed out: ' . $stdout . ' ' . $stderr); @@ -79,7 +81,7 @@ try { throw new RuntimeException('shutdown waited on HTTP timeout'); } // Fork output is intentionally concurrent; compare the complete set. - $lines = explode("\n", trim($stdout)); + $lines = explode("\n", trim(str_replace("fixture_child_ready\n", '', $stdout))); sort($lines); echo implode("\n", $lines), "\n"; if ($scenario === 'shutdown') echo "shutdown_cancelled_inflight_request\n"; diff --git a/tests/ext/ffe/stubs/agentless_process_app.php b/tests/ext/ffe/stubs/agentless_process_app.php index 50e7ac3f5c2..310d4e53d8a 100644 --- a/tests/ext/ffe/stubs/agentless_process_app.php +++ b/tests/ext/ffe/stubs/agentless_process_app.php @@ -1,16 +1,9 @@ getStringValue('flag', 'fallback'); $elapsed = microtime(true) - $start; diff --git a/tests/ext/ffe/stubs/agentless_source_selection_app.php b/tests/ext/ffe/stubs/agentless_source_selection_app.php new file mode 100644 index 00000000000..dba027f8229 --- /dev/null +++ b/tests/ext/ffe/stubs/agentless_source_selection_app.php @@ -0,0 +1,5 @@ + Date: Thu, 8 Oct 2026 08:09:21 +0000 Subject: [PATCH 11/12] Keep lifecycle fixture HTTP serving independent of child output --- tests/ext/ffe/agentless_source_selection.phpt | 16 ++++++---------- tests/ext/ffe/stubs/agentless_lifecycle.inc | 15 ++++++--------- tests/ext/ffe/stubs/agentless_process.inc | 17 +++++++---------- 3 files changed, 19 insertions(+), 29 deletions(-) diff --git a/tests/ext/ffe/agentless_source_selection.phpt b/tests/ext/ffe/agentless_source_selection.phpt index 829930bde3f..75700a3bdf0 100644 --- a/tests/ext/ffe/agentless_source_selection.phpt +++ b/tests/ext/ffe/agentless_source_selection.phpt @@ -15,6 +15,7 @@ DD_METRICS_OTEL_ENABLED=0 DD_TRACE_AGENT_URL=http://127.0.0.1:1 --FILE-- $case) { unset($environment[$key]); if ($case[$i] !== null) $environment[$key] = $case[$i]; } - // A file also avoids Windows cmd.exe changing quotes in inline PHP code. - $command = (PHP_OS === 'WINNT' ? '' : 'exec ') . escapeshellarg(getenv('TEST_PHP_EXECUTABLE')) . ' ' . getenv('TEST_PHP_EXTRA_ARGS') - . ' ' . escapeshellarg(__DIR__ . '/stubs/agentless_source_selection_app.php'); - $process = proc_open($command, array(array('pipe', 'r'), array('pipe', 'w'), array('pipe', 'w')), $pipes, null, $environment); + $process = ffeTestProcess(__DIR__ . '/stubs/agentless_source_selection_app.php', $pipes, $environment); if (!is_resource($process)) throw new RuntimeException('cannot launch PHP'); fclose($pipes[0]); - stream_set_blocking($pipes[1], false); - stream_set_blocking($pipes[2], false); $stdout = $stderr = ''; $start = microtime(true); $readyAt = null; do { - $stdout .= stream_get_contents($pipes[1]); - $stderr .= stream_get_contents($pipes[2]); + $stdout .= ffeTestProcessRead($pipes[1]); + $stderr .= ffeTestProcessRead($pipes[2]); if ($readyAt === null && strpos($stdout, "fixture_child_ready\n") !== false) $readyAt = microtime(true); $status = proc_get_status($process); if (!$status['running']) break; @@ -57,8 +53,8 @@ foreach ($cases as $name => $case) { throw new RuntimeException($name . ' unexpectedly polled agentless configuration'); } } while (microtime(true) - ($readyAt === null ? $start : $readyAt) < ($readyAt === null ? 60 : 3)); - $stdout .= stream_get_contents($pipes[1]); - $stderr .= stream_get_contents($pipes[2]); + $stdout .= ffeTestProcessRead($pipes[1]); + $stderr .= ffeTestProcessRead($pipes[2]); if ($status['running']) proc_terminate($process); fclose($pipes[1]); fclose($pipes[2]); diff --git a/tests/ext/ffe/stubs/agentless_lifecycle.inc b/tests/ext/ffe/stubs/agentless_lifecycle.inc index 8d212d56d3a..06d1671fd18 100644 --- a/tests/ext/ffe/stubs/agentless_lifecycle.inc +++ b/tests/ext/ffe/stubs/agentless_lifecycle.inc @@ -1,16 +1,13 @@ 3) { From 72e78a5ff2a2247c7df99f78b09b32470145e723 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 8 Oct 2026 09:35:58 +0000 Subject: [PATCH 12/12] test(ffe): coordinate instrumented initialization and recovery fixtures --- tests/ext/ffe/stubs/agentless_lifecycle.inc | 7 ++++++- tests/ext/ffe/stubs/agentless_process.inc | 21 ++++++++++++++++--- tests/ext/ffe/stubs/agentless_process_app.php | 9 ++++---- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/tests/ext/ffe/stubs/agentless_lifecycle.inc b/tests/ext/ffe/stubs/agentless_lifecycle.inc index 06d1671fd18..eb184f11fb6 100644 --- a/tests/ext/ffe/stubs/agentless_lifecycle.inc +++ b/tests/ext/ffe/stubs/agentless_lifecycle.inc @@ -1,5 +1,10 @@ 3) { + if ($scenario === 'shutdown' && microtime(true) - $shutdownStart > ($instrumented ? 10 : 3)) { throw new RuntimeException('shutdown waited on HTTP timeout'); } // Fork output is intentionally concurrent; compare the complete set. diff --git a/tests/ext/ffe/stubs/agentless_process_app.php b/tests/ext/ffe/stubs/agentless_process_app.php index 310d4e53d8a..935495cf41d 100644 --- a/tests/ext/ffe/stubs/agentless_process_app.php +++ b/tests/ext/ffe/stubs/agentless_process_app.php @@ -3,6 +3,7 @@ require __DIR__ . '/ffe_api_bootstrap.inc'; $client = new \DDTrace\FeatureFlags\Client(); $scenario = getenv('PHP_FFE_PROCESS_SCENARIO'); +$instrumented = getenv('USE_ZEND_ALLOC') === '0'; echo "fixture_child_ready\n"; $start = microtime(true); $first = $client->getStringValue('flag', 'fallback'); @@ -44,23 +45,23 @@ } if ($first !== 'fallback') throw new RuntimeException('unexpected initial value'); -if ($elapsed < 0.15 || $elapsed > 1) throw new RuntimeException('initialization deadline not honored'); +if ($elapsed < ($instrumented ? 1.5 : 0.15) || $elapsed > ($instrumented ? 10 : 1)) throw new RuntimeException('initialization deadline not honored: elapsed=' . $elapsed); echo "initial_deadline_bounded\n"; $start = microtime(true); for ($i = 0; $i < 10; $i++) $client->getStringValue('flag', 'fallback'); -if (microtime(true) - $start > 0.5) throw new RuntimeException('initialization budget was renewed'); +if (microtime(true) - $start > ($instrumented ? 3 : 0.5)) throw new RuntimeException('initialization budget was renewed: elapsed=' . (microtime(true) - $start)); echo "later_evaluations_do_not_wait\n"; if ($scenario === 'shutdown') { // The initialization deadline can expire before the worker is scheduled. // Confirm the fixture has a request before measuring its cancellation. - stream_set_timeout(STDIN, 5); + stream_set_timeout(STDIN, $instrumented ? 20 : 5); if (fgets(STDIN) !== "request_received\n") { throw new RuntimeException('fixture did not observe an in-flight request'); } // Shutdown must not wait for the configured 30-second request timeout. exit; } -$deadline = microtime(true) + 7; +$deadline = microtime(true) + ($instrumented ? 20 : 7); do { $value = $client->getStringValue('flag', 'fallback'); if ($value === 'blue') break;