From f1a45f875e735cb2a554865f27ee762eced7e9bb Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 15 Sep 2026 15:58:30 +0300 Subject: [PATCH 01/10] feat(ffe): add settings and agentless endpoint resolution --- Cargo.lock | 1 + Cargo.toml | 1 + components-rs/ffe.rs | 4 + components-rs/ffe/settings.rs | 641 +++++++++++++++++++++++++ ext/configuration.h | 1 + metadata/supported-configurations.json | 54 ++- tracer/configuration.h | 6 + 7 files changed, 707 insertions(+), 1 deletion(-) create mode 100644 components-rs/ffe/settings.rs diff --git a/Cargo.lock b/Cargo.lock index 89c8002e20a..5be385cd0ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1298,6 +1298,7 @@ dependencies = [ "tracing", "tracing-core", "tracing-subscriber", + "url", "uuid", "zwohash", ] diff --git a/Cargo.toml b/Cargo.toml index 902e2827919..e35aec88808 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,6 +87,7 @@ serde_json = "1.0.113" regex = "1.10.5" regex-automata = "0.4.5" http = "1.4" +url = "2.5.4" libc = "0.2" bincode = "1.3.3" hashbrown = "0.15" diff --git a/components-rs/ffe.rs b/components-rs/ffe.rs index 4964bc7b0b9..78031bb5bd6 100644 --- a/components-rs/ffe.rs +++ b/components-rs/ffe.rs @@ -8,6 +8,10 @@ use std::cell::RefCell; use std::collections::HashMap; use std::sync::Arc; +// This module is wired into activation by the next change in the FFE stack. +#[allow(dead_code)] +pub(crate) mod settings; + struct FfeState { config: Option, version: u64, diff --git a/components-rs/ffe/settings.rs b/components-rs/ffe/settings.rs new file mode 100644 index 00000000000..07a300acbb9 --- /dev/null +++ b/components-rs/ffe/settings.rs @@ -0,0 +1,641 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::fmt; +use std::time::Duration; +use url::Url; + +pub const DEFAULT_POLL_INTERVAL_SECONDS: i64 = 30; +pub const MAX_POLL_INTERVAL_SECONDS: i64 = 3_600; +pub const DEFAULT_REQUEST_TIMEOUT_SECONDS: i64 = 5; +pub const MAX_REQUEST_TIMEOUT_SECONDS: i64 = 300; +pub const DEFAULT_INITIALIZATION_TIMEOUT_MS: i64 = 10_000; +pub const MAX_INITIALIZATION_TIMEOUT_MS: i64 = i32::MAX as i64; + +const DEFAULT_SITE: &str = "datadoghq.com"; +const AGENTLESS_HOST_PREFIX: &str = "ufc-server.ff-cdn."; +const AGENTLESS_CONFIGURATION_PATH: &str = "/api/v2/feature-flagging/config/rules-based/server"; + +/// The configured source is retained separately from enablement. In particular, +/// `Offline` is a reserved source whose delivery is disabled today, rather than +/// an alias for the absence of a configured source. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ConfigurationSource { + Agentless, + RemoteConfig, + Offline, + Invalid, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DisableReason { + KillSwitch, + InvalidSource, + LegacyDisabled, + Offline, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SourceResolution { + pub enabled: bool, + pub source: ConfigurationSource, + pub disable_reason: Option, + pub legacy_key_decided: bool, +} + +impl SourceResolution { + pub fn delivery_source(self) -> Option { + self.enabled.then_some(self.source) + } +} + +#[derive(Clone, Copy)] +pub struct SourceInput<'a> { + pub enabled: bool, + pub enabled_set: bool, + pub source: &'a str, + pub source_set: bool, + pub legacy_enabled: bool, + pub legacy_enabled_set: bool, +} + +pub fn resolve_source(input: SourceInput<'_>) -> SourceResolution { + let explicit_source = input + .source_set + .then(|| input.source.trim()) + .filter(|s| !s.is_empty()); + + let (mut enabled, source, disable_reason, legacy_key_decided) = + if let Some(source) = explicit_source { + match source.to_ascii_lowercase().as_str() { + "agentless" => (true, ConfigurationSource::Agentless, None, false), + "remote_config" => (true, ConfigurationSource::RemoteConfig, None, false), + "offline" => ( + false, + ConfigurationSource::Offline, + Some(DisableReason::Offline), + false, + ), + _ => ( + false, + ConfigurationSource::Invalid, + Some(DisableReason::InvalidSource), + false, + ), + } + } else if input.enabled_set { + (true, ConfigurationSource::Agentless, None, false) + } else if input.legacy_enabled_set { + if input.legacy_enabled { + (true, ConfigurationSource::RemoteConfig, None, true) + } else { + ( + false, + ConfigurationSource::RemoteConfig, + Some(DisableReason::LegacyDisabled), + true, + ) + } + } else { + (true, ConfigurationSource::Agentless, None, false) + }; + + let disable_reason = if input.enabled_set && !input.enabled { + enabled = false; + Some(DisableReason::KillSwitch) + } else { + disable_reason + }; + + SourceResolution { + enabled, + source, + disable_reason, + legacy_key_decided, + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SettingsIssue { + PollInterval, + RequestTimeout, + InitializationTimeout, +} + +pub struct SettingsInput<'a> { + pub source: SourceInput<'a>, + /// Sensitive: may contain credentials and must never be logged. + pub agentless_base_url: &'a str, + pub poll_interval_seconds: i64, + pub request_timeout_seconds: i64, + pub initialization_timeout_ms: i64, + pub site: &'a str, + /// Sensitive: must never be logged. + pub api_key: &'a str, + pub environment: &'a str, +} + +/// A validated, process-stable settings snapshot. Deliberately does not derive +/// `Debug`, because it owns the API key and potentially credential-bearing URL. +pub struct FeatureFlagsSettings { + pub resolution: SourceResolution, + agentless_base_url: String, + pub poll_interval: Duration, + pub request_timeout: Duration, + pub initialization_timeout: Duration, + site: String, + api_key: String, + environment: String, + pub issues: Vec, +} + +impl FeatureFlagsSettings { + pub fn resolve(input: SettingsInput<'_>) -> Self { + let mut issues = Vec::new(); + let poll_interval_seconds = validated_numeric( + input.poll_interval_seconds, + DEFAULT_POLL_INTERVAL_SECONDS, + MAX_POLL_INTERVAL_SECONDS, + SettingsIssue::PollInterval, + &mut issues, + ); + let request_timeout_seconds = validated_numeric( + input.request_timeout_seconds, + DEFAULT_REQUEST_TIMEOUT_SECONDS, + MAX_REQUEST_TIMEOUT_SECONDS, + SettingsIssue::RequestTimeout, + &mut issues, + ); + let initialization_timeout_ms = validated_numeric( + input.initialization_timeout_ms, + DEFAULT_INITIALIZATION_TIMEOUT_MS, + MAX_INITIALIZATION_TIMEOUT_MS, + SettingsIssue::InitializationTimeout, + &mut issues, + ); + + Self { + resolution: resolve_source(input.source), + agentless_base_url: input.agentless_base_url.to_owned(), + poll_interval: Duration::from_secs(poll_interval_seconds as u64), + request_timeout: Duration::from_secs(request_timeout_seconds as u64), + initialization_timeout: Duration::from_millis(initialization_timeout_ms as u64), + site: input.site.to_owned(), + api_key: input.api_key.to_owned(), + environment: input.environment.to_owned(), + issues, + } + } + + pub fn agentless_endpoint(&self) -> Result { + AgentlessEndpoint::build( + &self.agentless_base_url, + &self.site, + &self.environment, + &self.api_key, + ) + } +} + +fn validated_numeric( + value: i64, + default: i64, + maximum: i64, + issue: SettingsIssue, + issues: &mut Vec, +) -> i64 { + if value > 0 && value <= maximum { + value + } else { + issues.push(issue); + default + } +} + +/// A validated agentless endpoint. Only managed endpoints can own an API key, +/// ensuring a custom endpoint cannot accidentally receive the Datadog credential. +#[derive(Clone, Eq, PartialEq)] +pub struct AgentlessEndpoint { + url: String, + api_key: Option, +} + +impl AgentlessEndpoint { + pub fn build( + base_url: &str, + site: &str, + environment: &str, + api_key: &str, + ) -> Result { + if base_url.trim().is_empty() { + Self::managed(site, environment, api_key) + } else { + Self::custom(base_url) + } + } + + pub fn as_str(&self) -> &str { + &self.url + } + + pub fn is_managed(&self) -> bool { + self.api_key.is_some() + } + + pub fn api_key(&self) -> Option<&str> { + self.api_key.as_deref() + } + + fn managed(site: &str, environment: &str, api_key: &str) -> Result { + if api_key.is_empty() { + return Err(EndpointError::MissingApiKey); + } + + let normalized_site = normalize_site(site)?; + let expected_host = format!("{AGENTLESS_HOST_PREFIX}{normalized_site}"); + let mut url = Url::parse(&format!( + "https://{expected_host}{AGENTLESS_CONFIGURATION_PATH}" + )) + .map_err(|_| EndpointError::InvalidSite)?; + + // Compare the parsed host before this endpoint is allowed to carry + // managed authentication. This is defense in depth after rejecting + // URL-significant site characters in `normalize_site`. + if url.host_str() != Some(expected_host.as_str()) { + return Err(EndpointError::InvalidSite); + } + + if !environment.is_empty() { + url.query_pairs_mut().append_pair("dd_env", environment); + } + + Ok(Self { + url: url.to_string(), + api_key: Some(api_key.to_owned()), + }) + } + + fn custom(base_url: &str) -> Result { + if base_url.chars().any(char::is_whitespace) { + return Err(EndpointError::UrlContainsWhitespace); + } + + let mut url = Url::parse(base_url).map_err(|_| EndpointError::InvalidUrl)?; + if !matches!(url.scheme(), "http" | "https") { + return Err(EndpointError::UnsupportedScheme); + } + if url.host_str().is_none() || url.fragment().is_some() { + return Err(EndpointError::InvalidUrl); + } + + let url = if matches!(url.path(), "" | "/") { + url.set_path(AGENTLESS_CONFIGURATION_PATH); + url.to_string() + } else { + // A custom non-root URL is opaque and must be requested exactly as + // the operator supplied it, including its query string. + base_url.to_owned() + }; + + Ok(Self { url, api_key: None }) + } +} + +impl fmt::Debug for AgentlessEndpoint { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("AgentlessEndpoint") + .field("url", &"[REDACTED]") + .field("managed", &self.is_managed()) + .finish() + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum EndpointError { + MissingApiKey, + InvalidUrl, + UnsupportedScheme, + UrlContainsWhitespace, + InvalidSite, +} + +impl fmt::Display for EndpointError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let message = match self { + Self::MissingApiKey => "no API key is configured for the managed endpoint", + Self::InvalidUrl => "the configured URL is not a valid absolute URL", + Self::UnsupportedScheme => "the configured URL must use HTTP or HTTPS", + Self::UrlContainsWhitespace => "the configured URL contains whitespace", + Self::InvalidSite => "the configured site is not a valid host", + }; + formatter.write_str(message) + } +} + +impl std::error::Error for EndpointError {} + +fn normalize_site(site: &str) -> Result { + let site = if site.trim().is_empty() { + DEFAULT_SITE + } else { + site + }; + if !site.is_ascii() + || site.chars().any(|character| { + character.is_whitespace() || matches!(character, '/' | '\\' | '?' | '#' | '@' | ':') + }) + { + return Err(EndpointError::InvalidSite); + } + Ok(site.to_ascii_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn source_input() -> SourceInput<'static> { + SourceInput { + enabled: true, + enabled_set: false, + source: "", + source_set: false, + legacy_enabled: false, + legacy_enabled_set: false, + } + } + + #[test] + fn source_precedence_matches_the_cross_sdk_contract() { + let cases = [ + ( + source_input(), + true, + ConfigurationSource::Agentless, + None, + false, + ), + ( + SourceInput { + enabled: false, + enabled_set: true, + source: "remote_config", + source_set: true, + legacy_enabled: true, + legacy_enabled_set: true, + }, + false, + ConfigurationSource::RemoteConfig, + Some(DisableReason::KillSwitch), + false, + ), + ( + SourceInput { + source: " ReMoTe_CoNfIg ", + source_set: true, + legacy_enabled: false, + legacy_enabled_set: true, + ..source_input() + }, + true, + ConfigurationSource::RemoteConfig, + None, + false, + ), + ( + SourceInput { + source: "offline", + source_set: true, + ..source_input() + }, + false, + ConfigurationSource::Offline, + Some(DisableReason::Offline), + false, + ), + ( + SourceInput { + source: "typo", + source_set: true, + ..source_input() + }, + false, + ConfigurationSource::Invalid, + Some(DisableReason::InvalidSource), + false, + ), + ( + SourceInput { + enabled: true, + enabled_set: true, + legacy_enabled: false, + legacy_enabled_set: true, + ..source_input() + }, + true, + ConfigurationSource::Agentless, + None, + false, + ), + ( + SourceInput { + legacy_enabled: true, + legacy_enabled_set: true, + ..source_input() + }, + true, + ConfigurationSource::RemoteConfig, + None, + true, + ), + ( + SourceInput { + legacy_enabled: false, + legacy_enabled_set: true, + ..source_input() + }, + false, + ConfigurationSource::RemoteConfig, + Some(DisableReason::LegacyDisabled), + true, + ), + ]; + + for (input, enabled, source, disable_reason, legacy_key_decided) in cases { + let resolution = resolve_source(input); + assert_eq!(resolution.enabled, enabled); + assert_eq!(resolution.source, source); + assert_eq!(resolution.disable_reason, disable_reason); + assert_eq!(resolution.legacy_key_decided, legacy_key_decided); + } + } + + #[test] + fn blank_explicit_source_is_semantically_unset() { + for source in ["", " ", "\t\r\n"] { + let resolution = resolve_source(SourceInput { + source, + source_set: true, + legacy_enabled: true, + legacy_enabled_set: true, + ..source_input() + }); + assert_eq!( + resolution.delivery_source(), + Some(ConfigurationSource::RemoteConfig) + ); + assert!(resolution.legacy_key_decided); + } + } + + #[test] + fn invalid_numeric_values_use_defaults_and_report_issues() { + let settings = FeatureFlagsSettings::resolve(SettingsInput { + source: source_input(), + agentless_base_url: "", + poll_interval_seconds: 3_601, + request_timeout_seconds: 0, + initialization_timeout_ms: i64::MAX, + site: "", + api_key: "key", + environment: "", + }); + + assert_eq!(settings.poll_interval, Duration::from_secs(30)); + assert_eq!(settings.request_timeout, Duration::from_secs(5)); + assert_eq!(settings.initialization_timeout, Duration::from_secs(10)); + assert_eq!( + settings.issues, + vec![ + SettingsIssue::PollInterval, + SettingsIssue::RequestTimeout, + SettingsIssue::InitializationTimeout, + ] + ); + } + + #[test] + fn managed_endpoint_uses_normalized_site_and_encoded_environment() { + let endpoint = AgentlessEndpoint::build("", "DATADOGHQ.EU", "prod / blue", "key") + .expect("managed endpoint"); + + assert!(endpoint.is_managed()); + assert_eq!(endpoint.api_key(), Some("key")); + assert_eq!( + endpoint.as_str(), + "https://ufc-server.ff-cdn.datadoghq.eu/api/v2/feature-flagging/config/rules-based/server?dd_env=prod+%2F+blue" + ); + } + + #[test] + fn managed_endpoint_omits_an_absent_environment_and_defaults_blank_site() { + let endpoint = AgentlessEndpoint::build("", " ", "", "key").expect("managed endpoint"); + assert_eq!( + endpoint.as_str(), + "https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server" + ); + } + + #[test] + fn managed_endpoint_requires_an_api_key() { + assert_eq!( + AgentlessEndpoint::build("", "datadoghq.com", "", "").unwrap_err(), + EndpointError::MissingApiKey + ); + } + + #[test] + fn managed_endpoint_rejects_sites_that_can_change_the_url_meaning() { + for site in [ + "https://datadoghq.com", + "datadoghq.com@attacker.example", + "datadoghq.com/path", + "datadoghq.com?query", + "datadoghq.com#fragment", + "datadoghq.com:443", + " datadoghq.com", + "data doghq.com", + "dâtadoghq.com", + ] { + assert_eq!( + AgentlessEndpoint::build("", site, "", "key").unwrap_err(), + EndpointError::InvalidSite, + "site should be rejected" + ); + } + } + + #[test] + fn custom_root_endpoint_gets_the_canonical_path_and_keeps_its_query() { + for base_url in ["http://localhost:8080", "http://localhost:8080/"] { + let endpoint = AgentlessEndpoint::build(base_url, "ignored", "ignored", "secret") + .expect("custom endpoint"); + assert!(!endpoint.is_managed()); + assert_eq!( + endpoint.as_str(), + "http://localhost:8080/api/v2/feature-flagging/config/rules-based/server" + ); + } + + let endpoint = AgentlessEndpoint::build( + "https://user:pass@localhost/?tenant=a%2Fb", + "ignored", + "ignored", + "secret", + ) + .expect("custom endpoint"); + assert_eq!( + endpoint.as_str(), + "https://user:pass@localhost/api/v2/feature-flagging/config/rules-based/server?tenant=a%2Fb" + ); + } + + #[test] + fn custom_non_root_endpoint_is_preserved_without_environment() { + let endpoint = AgentlessEndpoint::build( + "HTTPS://LOCALHOST/custom/path?tenant=a%2Fb", + "ignored", + "must-not-be-added", + "must-not-be-sent", + ) + .expect("custom endpoint"); + assert_eq!( + endpoint.as_str(), + "HTTPS://LOCALHOST/custom/path?tenant=a%2Fb" + ); + assert!(!endpoint.is_managed()); + assert_eq!(endpoint.api_key(), None); + } + + #[test] + fn custom_endpoint_validation_errors_never_contain_the_url() { + let credential = "credential-that-must-not-leak"; + for (url, expected) in [ + ( + format!("ftp://{credential}@localhost/path"), + EndpointError::UnsupportedScheme, + ), + ( + format!("https://{credential}@local host/path"), + EndpointError::UrlContainsWhitespace, + ), + (format!("not-a-url-{credential}"), EndpointError::InvalidUrl), + ] { + let error = AgentlessEndpoint::build(&url, "", "", "key").unwrap_err(); + assert_eq!(error, expected); + assert!(!error.to_string().contains(credential)); + } + } + + #[test] + fn endpoint_debug_output_is_redacted() { + let endpoint = + AgentlessEndpoint::build("https://user:password@localhost/custom", "", "", "key") + .expect("custom endpoint"); + let debug = format!("{endpoint:?}"); + assert!(!debug.contains("user")); + assert!(!debug.contains("password")); + assert!(debug.contains("[REDACTED]")); + } +} diff --git a/ext/configuration.h b/ext/configuration.h index 667c9c49766..c1b06ff3818 100644 --- a/ext/configuration.h +++ b/ext/configuration.h @@ -46,6 +46,7 @@ enum datadog_sidecar_connection_mode { CONFIG(STRING, DD_DOGSTATSD_URL, "http://localhost:8125") \ CONFIG(STRING, DD_DOGSTATSD_HOST, "localhost") \ CONFIG(STRING, DD_API_KEY, "", .ini_change = zai_config_system_ini_change, .sensitive = true) \ + CONFIG(STRING, DD_SITE, "datadoghq.com") \ CONFIG(INT, DD_DOGSTATSD_PORT, "8125") \ CONFIG(STRING, DD_ENV, "", .ini_change = datadog_alter_dd_env, \ .env_config_fallback = ddtrace_conf_otel_resource_attributes_env) \ diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index b36ce1f4782..bc1990e22aa 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -397,6 +397,13 @@ "default": "false" } ], + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS": [ + { + "implementation": "B", + "type": "int", + "default": "10000" + } + ], "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": [ { "implementation": "A", @@ -411,6 +418,42 @@ "default": "true" } ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": [ + { + "implementation": "A", + "type": "string", + "default": "agentless" + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": [ + { + "implementation": "A", + "type": "string", + "default": "", + "sensitive": true + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": [ + { + "implementation": "A", + "type": "int", + "default": "30" + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": [ + { + "implementation": "A", + "type": "int", + "default": "5" + } + ], + "DD_FEATURE_FLAGS_ENABLED": [ + { + "implementation": "A", + "type": "boolean", + "default": "true" + } + ], "DD_GIT_COMMIT_SHA": [ { "implementation": "B", @@ -652,6 +695,13 @@ "default": "" } ], + "DD_SITE": [ + { + "implementation": "C", + "type": "string", + "default": "datadoghq.com" + } + ], "DD_SPAN_SAMPLING_RULES": [ { "implementation": "A", @@ -2869,5 +2919,7 @@ } ] }, - "deprecations": {} + "deprecations": { + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "DD_FEATURE_FLAGS_ENABLED" + } } diff --git a/tracer/configuration.h b/tracer/configuration.h index 892cdce0715..7fb3a1c06a4 100644 --- a/tracer/configuration.h +++ b/tracer/configuration.h @@ -163,7 +163,13 @@ CONFIG(BOOL, DD_TRACE_RESOURCE_RENAMING_ENABLED, "false") \ CONFIG(BOOL, DD_TRACE_RESOURCE_RENAMING_ALWAYS_SIMPLIFIED_ENDPOINT, "false") \ CONFIG(BOOL, DD_TRACE_STATS_COMPUTATION_ENABLED, "false") \ + CONFIG(BOOL, DD_FEATURE_FLAGS_ENABLED, "true") \ + CONFIG(STRING, DD_FEATURE_FLAGS_CONFIGURATION_SOURCE, "agentless") \ + CONFIG(STRING, DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, "", .sensitive = true) \ + CONFIG(INT, DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, "30") \ + CONFIG(INT, DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS, "5") \ CONFIG(BOOL, DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED, "false") \ + CONFIG(INT, DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS, "10000") \ DD_INTEGRATIONS #ifndef DDTRACE_CONFIGURATION From 9f17928ce8122768e73e783e3fdf494e221d3638 Mon Sep 17 00:00:00 2001 From: Pavel Date: Tue, 15 Sep 2026 17:10:46 +0300 Subject: [PATCH 02/10] feat(ffe): add process-wide state and agentless poller --- Cargo.lock | 10 + Cargo.toml | 12 +- components-rs/ffe.rs | 162 ++-- components-rs/ffe/agentless.rs | 951 ++++++++++++++++++++++++ zend_abstract_interface/config/config.h | 4 +- 5 files changed, 1095 insertions(+), 44 deletions(-) create mode 100644 components-rs/ffe/agentless.rs diff --git a/Cargo.lock b/Cargo.lock index 5be385cd0ed..860062bb5e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1237,6 +1237,8 @@ name = "datadog-php" version = "0.0.1" dependencies = [ "anyhow", + "arc-swap", + "base64 0.22.1", "bincode", "bindgen 0.72.1", "cbindgen 0.27.0", @@ -1253,8 +1255,13 @@ dependencies = [ "ddtrace-sidecar", "dynasmrt", "env_logger", + "fastrand", + "flate2", + "futures-util", "hashbrown 0.15.2", "http 1.4.2", + "http-body-util", + "hyper", "itertools 0.11.0", "lazy_static", "libc 0.2.186", @@ -1281,6 +1288,7 @@ dependencies = [ "log", "mach2", "paste", + "percent-encoding", "perfcnt", "rand 0.9.4", "rand_distr", @@ -1295,6 +1303,8 @@ dependencies = [ "static_assertions", "tempfile", "thiserror 2.0.18", + "tokio", + "tokio-util", "tracing", "tracing-core", "tracing-subscriber", diff --git a/Cargo.toml b/Cargo.toml index e35aec88808..36b3098362c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ tracing-subscriber = ["tracing"] trigger_time_sample = ["profiling"] [dependencies] -libdd-common = { path = "libdatadog/libdd-common", features = ["cgroup_testing"] } +libdd-common = { path = "libdatadog/libdd-common", features = ["cgroup_testing", "hyper-proxy"] } libdd-common-ffi = { path = "libdatadog/libdd-common-ffi", default-features = false } libdd-telemetry = { path = "libdatadog/libdd-telemetry" } libdd-telemetry-ffi = { path = "libdatadog/libdd-telemetry-ffi", default-features = false } @@ -68,7 +68,14 @@ libdd-crashtracker-ffi = { path = "libdatadog/libdd-crashtracker-ffi", default-f libdd-library-config-ffi = { path = "libdatadog/libdd-library-config-ffi", default-features = false } spawn_worker = { path = "libdatadog/spawn_worker" } anyhow = "1.0" +arc-swap = "1.7.1" +base64 = "0.22" const-str = "0.5.6" +fastrand = "2" +flate2 = { version = "1.0", default-features = false, features = ["rust_backend"] } +futures-util = "0.3" +http-body-util = "0.1" +hyper = { version = "1.6", features = ["client", "http1"] } itertools = "0.11.0" serde = "1.0.196" simd-json = "0.14.1" @@ -79,10 +86,13 @@ env_logger = "0.11" zwohash = "0.1.2" uuid = { version = "1.0", features = ["v4"] } paste = "1" +percent-encoding = "2.1" tempfile = "3.3" tracing = { version = "0.1", default-features = false, features = ["std", "attributes"] } tracing-core = "0.1.32" tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "fmt", "env-filter", "smallvec"] } +tokio = { version = "1.36", features = ["macros", "rt", "sync", "time"] } +tokio-util = "0.7.11" serde_json = "1.0.113" regex = "1.10.5" regex-automata = "0.4.5" diff --git a/components-rs/ffe.rs b/components-rs/ffe.rs index 78031bb5bd6..a5a8493c3b1 100644 --- a/components-rs/ffe.rs +++ b/components-rs/ffe.rs @@ -1,43 +1,87 @@ use crate::bytes::MaybeOwnedZendString; +use arc_swap::ArcSwapOption; use datadog_ffe::rules_based::{ self as ffe, AssignmentReason, AssignmentValue, Attribute, Configuration, EvaluationContext, EvaluationError, ExpectedFlagType, Str, UniversalFlagConfig, }; use libdd_common_ffi::slice::{AsBytes, CharSlice}; -use std::cell::RefCell; use std::collections::HashMap; -use std::sync::Arc; +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::{Arc, LazyLock}; +// This module is wired into activation by the next change in the FFE stack. +#[allow(dead_code)] +pub(crate) mod agentless; // This module is wired into activation by the next change in the FFE stack. #[allow(dead_code)] pub(crate) mod settings; -struct FfeState { - config: Option, - version: u64, +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u8)] +#[allow(dead_code)] // Remaining states are connected by PR 3 activation wiring. +pub(crate) enum DeliveryState { + Inactive, + Starting, + Ready, + Stale, + Stopped, + PermanentError, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ConfigurationTransition { + Ready, + Recovered, + Changed, + Stale, + Unchanged, +} + +static FFE_CONFIG: LazyLock> = + LazyLock::new(|| ArcSwapOption::new(None)); +static FFE_VERSION: AtomicU64 = AtomicU64::new(0); +static DELIVERY_STATE: AtomicU8 = AtomicU8::new(DeliveryState::Inactive as u8); + +#[allow(dead_code)] // Connected by PR 3 activation wiring. +pub(crate) fn delivery_state() -> DeliveryState { + match DELIVERY_STATE.load(Ordering::Acquire) { + value if value == DeliveryState::Starting as u8 => DeliveryState::Starting, + value if value == DeliveryState::Ready as u8 => DeliveryState::Ready, + value if value == DeliveryState::Stale as u8 => DeliveryState::Stale, + value if value == DeliveryState::Stopped as u8 => DeliveryState::Stopped, + value if value == DeliveryState::PermanentError as u8 => DeliveryState::PermanentError, + _ => DeliveryState::Inactive, + } } -thread_local! { - static FFE_STATE: RefCell = const { RefCell::new(FfeState { - config: None, - version: 0, - }) }; +#[allow(dead_code)] // Connected by PR 3 activation wiring. +pub(crate) fn set_delivery_state(state: DeliveryState) { + DELIVERY_STATE.store(state as u8, Ordering::Release); } -pub fn store_config(config: Configuration) { - FFE_STATE.with(|state| { - let mut state = state.borrow_mut(); - state.config = Some(config); - state.version = state.version.wrapping_add(1); - }); +pub(crate) fn store_config(config: Configuration) -> ConfigurationTransition { + FFE_CONFIG.store(Some(Arc::new(config))); + FFE_VERSION.fetch_add(1, Ordering::AcqRel); + + let previous = DELIVERY_STATE.swap(DeliveryState::Ready as u8, Ordering::AcqRel); + if previous == DeliveryState::Ready as u8 { + ConfigurationTransition::Changed + } else if previous == DeliveryState::Stale as u8 { + ConfigurationTransition::Recovered + } else { + ConfigurationTransition::Ready + } } -pub fn clear_config() { - FFE_STATE.with(|state| { - let mut state = state.borrow_mut(); - state.config = None; - state.version = state.version.wrapping_add(1); - }); +pub(crate) fn clear_config() -> ConfigurationTransition { + FFE_CONFIG.store(None); + FFE_VERSION.fetch_add(1, Ordering::AcqRel); + let previous = DELIVERY_STATE.swap(DeliveryState::Stale as u8, Ordering::AcqRel); + if previous == DeliveryState::Stale as u8 { + ConfigurationTransition::Unchanged + } else { + ConfigurationTransition::Stale + } } #[no_mangle] @@ -62,12 +106,12 @@ pub extern "C" fn ddog_ffe_load_config(json: CharSlice<'_>) -> bool { #[no_mangle] pub extern "C" fn ddog_ffe_has_config() -> bool { - FFE_STATE.with(|state| state.borrow().config.is_some()) + FFE_CONFIG.load().is_some() } #[no_mangle] pub extern "C" fn ddog_ffe_config_version() -> u64 { - FFE_STATE.with(|state| state.borrow().version) + FFE_VERSION.load(Ordering::Acquire) } const REASON_STATIC: i32 = 0; @@ -160,18 +204,16 @@ pub extern "C" fn ddog_ffe_evaluate( let attributes = parse_attributes(attributes, attributes_count); let context = EvaluationContext::new(targeting_key, Arc::new(attributes)); - FFE_STATE.with(|state| { - let state = state.borrow(); - let assignment = ffe::get_assignment( - state.config.as_ref(), - flag_key, - &context, - expected_type, - ffe::now(), - ); + let config = FFE_CONFIG.load(); + let assignment = ffe::get_assignment( + config.as_deref(), + flag_key, + &context, + expected_type, + ffe::now(), + ); - result_from_assignment(assignment) - }) + result_from_assignment(assignment) } fn parse_attributes( @@ -302,9 +344,10 @@ mod tests { use std::mem; use std::ptr; use std::ptr::NonNull; - use std::sync::Once; + use std::sync::{Mutex, Once}; static INIT_ZEND_STRING_FUNCTIONS: Once = Once::new(); + static FFE_TEST_LOCK: Mutex<()> = Mutex::new(()); fn setup_zend_string_functions() { INIT_ZEND_STRING_FUNCTIONS.call_once(|| unsafe { @@ -410,6 +453,7 @@ mod tests { #[test] fn empty_targeting_key_is_not_dropped() { + let _guard = FFE_TEST_LOCK.lock().expect("FFE test lock"); setup_zend_string_functions(); clear_config(); let config = @@ -438,7 +482,8 @@ mod tests { } #[test] - fn configuration_state_is_thread_local() { + fn configuration_state_is_process_wide() { + let _guard = FFE_TEST_LOCK.lock().expect("FFE test lock"); clear_config(); let empty_version = ddog_ffe_config_version(); assert!(!ddog_ffe_has_config()); @@ -448,19 +493,52 @@ mod tests { let loaded_version = ddog_ffe_config_version(); assert_eq!(loaded_version, empty_version.wrapping_add(1)); - let child = std::thread::spawn(|| { - assert!(!ddog_ffe_has_config()); - assert_eq!(ddog_ffe_config_version(), 0); + let child = std::thread::spawn(move || { + assert!(ddog_ffe_has_config()); + assert_eq!(ddog_ffe_config_version(), loaded_version); assert!(load_empty_config()); assert!(ddog_ffe_has_config()); - assert_eq!(ddog_ffe_config_version(), 1); + assert_eq!(ddog_ffe_config_version(), loaded_version.wrapping_add(1)); }); child.join().expect("child thread should not panic"); assert!(ddog_ffe_has_config()); - assert_eq!(ddog_ffe_config_version(), loaded_version); + assert_eq!(ddog_ffe_config_version(), loaded_version.wrapping_add(1)); + clear_config(); + } + + #[test] + fn configuration_transitions_track_ready_stale_and_recovery() { + let _guard = FFE_TEST_LOCK.lock().expect("FFE test lock"); + set_delivery_state(DeliveryState::Inactive); + clear_config(); + assert_eq!(delivery_state(), DeliveryState::Stale); + assert_eq!(clear_config(), ConfigurationTransition::Unchanged); + + let json = UniversalFlagConfig::from_json(EMPTY_CONFIG.as_bytes().to_vec()) + .expect("test configuration"); + assert_eq!( + store_config(Configuration::from_server_response(json)), + ConfigurationTransition::Recovered + ); + assert_eq!(delivery_state(), DeliveryState::Ready); + + let json = UniversalFlagConfig::from_json(EMPTY_CONFIG.as_bytes().to_vec()) + .expect("test configuration"); + assert_eq!( + store_config(Configuration::from_server_response(json)), + ConfigurationTransition::Changed + ); + + set_delivery_state(DeliveryState::Starting); + let json = UniversalFlagConfig::from_json(EMPTY_CONFIG.as_bytes().to_vec()) + .expect("test configuration"); + assert_eq!( + store_config(Configuration::from_server_response(json)), + ConfigurationTransition::Ready + ); clear_config(); } } diff --git a/components-rs/ffe/agentless.rs b/components-rs/ffe/agentless.rs new file mode 100644 index 00000000000..1fe4663b676 --- /dev/null +++ b/components-rs/ffe/agentless.rs @@ -0,0 +1,951 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use super::settings::AgentlessEndpoint; +use super::{store_config, ConfigurationTransition}; +use crate::log::{self, Log}; +use base64::Engine; +use datadog_ffe::rules_based::{Configuration, UniversalFlagConfig}; +use flate2::read::GzDecoder; +use futures_util::FutureExt; +use http::header::{ACCEPT_ENCODING, CONTENT_ENCODING, CONTENT_LENGTH, ETAG, IF_NONE_MATCH}; +use http::{Method, Request}; +use http_body_util::BodyExt; +use libdd_common::http_common::{self, Body}; +use percent_encoding::percent_decode_str; +use std::future::Future; +use std::io::Read; +use std::panic::AssertUnwindSafe; +use std::pin::Pin; +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::thread::JoinHandle; +use std::time::Duration; +use tokio_util::sync::CancellationToken; + +const MAX_POLL_ATTEMPTS: usize = 3; +const MAX_RESPONSE_BODY_BYTES: usize = 10 << 20; +const CLIENT_LANGUAGE: &str = "php"; +const INTERNAL_UNTRACED_HEADER: &str = "DD-Internal-Untraced-Request"; +const API_KEY_HEADER: &str = "DD-API-KEY"; +const AUTHORIZATION_HEADER: &str = "Authorization"; + +type FetchFuture<'a> = + Pin> + Send + 'a>>; + +trait Transport: Send + Sync { + fn get<'a>( + &'a self, + endpoint: &'a AgentlessEndpoint, + timeout: Duration, + etag: Option<&'a str>, + cancellation: &'a CancellationToken, + ) -> FetchFuture<'a>; +} + +trait ConfigurationSink: Send + Sync { + fn apply(&self, configuration: Configuration) -> Result; +} + +struct GlobalConfigurationSink; + +impl ConfigurationSink for GlobalConfigurationSink { + fn apply(&self, configuration: Configuration) -> Result { + Ok(store_config(configuration)) + } +} + +#[derive(Debug, Eq, PartialEq)] +enum TransportFailure { + BuildRequest, + Cancelled, + TimedOut, + Request, + ResponseBody, + ResponseTooLarge, +} + +struct PollResponse { + status: u16, + etag: Option, + content_encoding: Option, + body: Vec, +} + +struct HyperTransport { + client: libdd_common::HttpClient, +} + +impl HyperTransport { + fn new() -> Self { + Self { + // This native client is outside PHP HTTP instrumentation, so it + // cannot recursively trace its own polling requests. Hyper also + // leaves 3xx responses visible instead of following redirects. + client: http_common::new_client_periodic(), + } + } + + async fn get_inner( + &self, + endpoint: &AgentlessEndpoint, + timeout: Duration, + etag: Option<&str>, + cancellation: &CancellationToken, + ) -> Result { + let request = build_request(endpoint, etag)?; + let operation = async { + let response = self + .client + .request(request) + .await + .map_err(|_| TransportFailure::Request)?; + let status = response.status().as_u16(); + let etag = header_value(response.headers(), ETAG); + let content_encoding = header_value(response.headers(), CONTENT_ENCODING); + + // A non-200 body can never contain configuration. Dropping it here + // also prevents a malicious error response from consuming memory. + if status != 200 { + return Ok(PollResponse { + status, + etag, + content_encoding, + body: Vec::new(), + }); + } + + if response + .headers() + .get(CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_some_and(|length| length > MAX_RESPONSE_BODY_BYTES as u64) + { + return Err(TransportFailure::ResponseTooLarge); + } + + let body = collect_bounded(response.into_body(), MAX_RESPONSE_BODY_BYTES).await?; + Ok(PollResponse { + status, + etag, + content_encoding, + body, + }) + }; + + tokio::select! { + _ = cancellation.cancelled() => Err(TransportFailure::Cancelled), + result = tokio::time::timeout(timeout, operation) => { + result.map_err(|_| TransportFailure::TimedOut)? + } + } + } +} + +impl Transport for HyperTransport { + fn get<'a>( + &'a self, + endpoint: &'a AgentlessEndpoint, + timeout: Duration, + etag: Option<&'a str>, + cancellation: &'a CancellationToken, + ) -> FetchFuture<'a> { + Box::pin(self.get_inner(endpoint, timeout, etag, cancellation)) + } +} + +fn build_request( + endpoint: &AgentlessEndpoint, + etag: Option<&str>, +) -> Result, TransportFailure> { + let (request_url, authorization) = request_url_and_authorization(endpoint.as_str())?; + let mut builder = Request::builder() + .method(Method::GET) + .uri(request_url) + .header(ACCEPT_ENCODING, "gzip") + .header("DD-Client-Library-Language", CLIENT_LANGUAGE) + .header( + "DD-Client-Library-Version", + include_str!("../../VERSION").trim(), + ) + .header(INTERNAL_UNTRACED_HEADER, "1"); + + if let Some(etag) = etag { + builder = builder.header(IF_NONE_MATCH, etag); + } + if let Some(api_key) = endpoint.api_key() { + builder = builder.header(API_KEY_HEADER, api_key); + } + if let Some(authorization) = authorization { + builder = builder.header(AUTHORIZATION_HEADER, authorization); + } + + builder + .body(Body::empty()) + .map_err(|_| TransportFailure::BuildRequest) +} + +fn request_url_and_authorization( + endpoint: &str, +) -> Result<(String, Option), TransportFailure> { + let mut url = url::Url::parse(endpoint).map_err(|_| TransportFailure::BuildRequest)?; + if url.username().is_empty() { + return Ok((endpoint.to_owned(), None)); + } + + let mut credentials = percent_decode_str(url.username()).collect::>(); + credentials.push(b':'); + if let Some(password) = url.password() { + credentials.extend(percent_decode_str(password)); + } + let authorization = format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode(credentials) + ); + + url.set_username("") + .map_err(|_| TransportFailure::BuildRequest)?; + url.set_password(None) + .map_err(|_| TransportFailure::BuildRequest)?; + Ok((url.to_string(), Some(authorization))) +} + +fn header_value(headers: &http::HeaderMap, name: http::header::HeaderName) -> Option { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) +} + +async fn collect_bounded( + mut body: hyper::body::Incoming, + limit: usize, +) -> Result, TransportFailure> { + let mut bytes = Vec::new(); + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|_| TransportFailure::ResponseBody)?; + if let Some(data) = frame.data_ref() { + let remaining = limit.saturating_sub(bytes.len()); + if data.len() > remaining { + return Err(TransportFailure::ResponseTooLarge); + } + bytes.extend_from_slice(data); + } + } + Ok(bytes) +} + +fn decode_response_body( + body: &[u8], + content_encoding: Option<&str>, + limit: usize, +) -> Result, TransportFailure> { + if !content_encoding.is_some_and(|value| value.trim().eq_ignore_ascii_case("gzip")) { + if body.len() > limit { + return Err(TransportFailure::ResponseTooLarge); + } + return Ok(body.to_vec()); + } + + let decoder = GzDecoder::new(body); + let mut decoded = Vec::new(); + decoder + .take(limit as u64 + 1) + .read_to_end(&mut decoded) + .map_err(|_| TransportFailure::ResponseBody)?; + if decoded.len() > limit { + return Err(TransportFailure::ResponseTooLarge); + } + Ok(decoded) +} + +#[derive(Clone, Copy)] +#[repr(u8)] +enum WarningCategory { + Authentication = 1 << 0, + RetryableHttp = 1 << 1, + UnexpectedHttp = 1 << 2, + Request = 1 << 3, + MalformedPayload = 1 << 4, + ApplyFailure = 1 << 5, + UnexpectedException = 1 << 6, +} + +#[derive(Default)] +struct WarningSlots(AtomicU8); + +impl WarningSlots { + fn take(&self, category: WarningCategory) -> bool { + self.0.fetch_or(category as u8, Ordering::Relaxed) & category as u8 == 0 + } + + fn reset(&self) { + self.0.store(0, Ordering::Relaxed); + } +} + +#[derive(Default)] +struct PollState { + etag: Mutex>, + warnings: WarningSlots, + poll_count: AtomicU64, +} + +impl PollState { + fn etag(&self) -> Option { + lock_unpoisoned(&self.etag).clone() + } + + fn set_etag(&self, etag: Option) { + *lock_unpoisoned(&self.etag) = etag; + } +} + +fn lock_unpoisoned(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PollOutcome { + Success, + Retryable, + Stop, + Cancelled, +} + +struct Poller { + endpoint: AgentlessEndpoint, + poll_interval: Duration, + request_timeout: Duration, + transport: T, + sink: S, + state: Arc, + retry_delay: fn(Duration, usize, f64) -> Duration, +} + +impl Poller { + async fn run(&mut self, cancellation: CancellationToken) { + loop { + if cancellation.is_cancelled() { + return; + } + + let poll = AssertUnwindSafe(self.poll(&cancellation)).catch_unwind(); + match poll.await { + Ok(PollOutcome::Cancelled) => return, + Ok(_) => {} + Err(_) => self.warn( + WarningCategory::UnexpectedException, + "Feature Flags agentless polling failed unexpectedly; polling continues", + ), + } + + tokio::select! { + _ = cancellation.cancelled() => return, + _ = tokio::time::sleep(self.poll_interval) => {} + } + } + } + + async fn poll(&mut self, cancellation: &CancellationToken) -> PollOutcome { + for attempt in 1..=MAX_POLL_ATTEMPTS { + let outcome = self.poll_once(cancellation).await; + if outcome != PollOutcome::Retryable || attempt == MAX_POLL_ATTEMPTS { + return outcome; + } + + let delay = (self.retry_delay)(self.poll_interval, attempt, fastrand::f64()); + tokio::select! { + _ = cancellation.cancelled() => return PollOutcome::Cancelled, + _ = tokio::time::sleep(delay) => {} + } + } + PollOutcome::Stop + } + + async fn poll_once(&mut self, cancellation: &CancellationToken) -> PollOutcome { + if cancellation.is_cancelled() { + return PollOutcome::Cancelled; + } + + self.state.poll_count.fetch_add(1, Ordering::Relaxed); + let etag = self.state.etag(); + match self + .transport + .get( + &self.endpoint, + self.request_timeout, + etag.as_deref(), + cancellation, + ) + .await + { + Ok(response) => self.process_response(response), + Err(TransportFailure::Cancelled) => PollOutcome::Cancelled, + Err(TransportFailure::BuildRequest) => { + self.warn( + WarningCategory::Request, + "Feature Flags agentless request could not be built", + ); + PollOutcome::Stop + } + Err(TransportFailure::ResponseTooLarge) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless response exceeds the 10 MiB limit; keeping the last known configuration", + ); + PollOutcome::Stop + } + Err(_) => { + self.warn( + WarningCategory::Request, + "Feature Flags agentless request failed; keeping the last known configuration", + ); + PollOutcome::Retryable + } + } + } + + fn process_response(&mut self, response: PollResponse) -> PollOutcome { + match response.status { + 304 => return PollOutcome::Success, + 401 | 403 => { + self.warn( + WarningCategory::Authentication, + "Feature Flags agentless endpoint rejected authentication; verify endpoint authentication", + ); + return PollOutcome::Stop; + } + 200 => {} + status if is_retryable_status(status) => { + self.warn( + WarningCategory::RetryableHttp, + &format!("Feature Flags agentless endpoint returned retryable HTTP {status}"), + ); + return PollOutcome::Retryable; + } + status => { + self.warn( + WarningCategory::UnexpectedHttp, + &format!("Feature Flags agentless endpoint returned unexpected HTTP {status}"), + ); + return PollOutcome::Stop; + } + } + + let body = match decode_response_body( + &response.body, + response.content_encoding.as_deref(), + MAX_RESPONSE_BODY_BYTES, + ) { + Ok(body) => body, + Err(TransportFailure::ResponseTooLarge) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless decoded response exceeds the 10 MiB limit; keeping the last known configuration", + ); + return PollOutcome::Stop; + } + Err(_) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless endpoint returned an invalid response body", + ); + return PollOutcome::Stop; + } + }; + + let configuration = match UniversalFlagConfig::from_json(body) { + Ok(configuration) => Configuration::from_server_response(configuration), + Err(_) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless endpoint returned malformed configuration", + ); + return PollOutcome::Stop; + } + }; + + if self.sink.apply(configuration).is_err() { + self.warn( + WarningCategory::ApplyFailure, + "Feature Flags agentless configuration could not be applied", + ); + return PollOutcome::Stop; + } + + // The response has now been parsed and applied. A blank or absent + // ETag intentionally clears an older value. + self.state.set_etag(response.etag); + PollOutcome::Success + } + + fn warn(&self, category: WarningCategory, message: &str) { + if self.state.warnings.take(category) { + log::log(Log::Warn, message); + } + } +} + +fn is_retryable_status(status: u16) -> bool { + matches!(status, 408 | 429 | 500..=599) +} + +fn retry_delay(poll_interval: Duration, attempt: usize, random: f64) -> Duration { + let base = if attempt <= 1 { + (poll_interval / 6).clamp(Duration::from_secs(2), Duration::from_secs(10)) + } else { + (poll_interval / 3).clamp(Duration::from_secs(5), Duration::from_secs(30)) + }; + base.mul_f64(0.8 + random.clamp(0.0, 1.0) * 0.4) + .max(Duration::from_secs(1)) +} + +#[derive(Clone)] +pub(crate) struct AgentlessWorkerConfig { + endpoint: AgentlessEndpoint, + poll_interval: Duration, + request_timeout: Duration, +} + +impl AgentlessWorkerConfig { + pub(crate) fn new( + endpoint: AgentlessEndpoint, + poll_interval: Duration, + request_timeout: Duration, + ) -> Self { + Self { + endpoint, + poll_interval, + request_timeout, + } + } +} + +struct RunningWorker { + cancellation: CancellationToken, + thread: JoinHandle<()>, +} + +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum WorkerStartError { + ThreadUnavailable, +} + +/// Owns the poll thread and makes its lifecycle explicit around `fork`. +/// PR 3 connects these methods to the tracer lifecycle hooks. +pub(crate) struct AgentlessWorker { + config: AgentlessWorkerConfig, + state: Arc, + running: Option, + restart_after_fork: bool, + permanently_stopped: bool, +} + +impl AgentlessWorker { + pub(crate) fn new(config: AgentlessWorkerConfig) -> Self { + Self { + config, + state: Arc::new(PollState::default()), + running: None, + restart_after_fork: false, + permanently_stopped: false, + } + } + + pub(crate) fn start(&mut self) -> Result { + if self.running.is_some() || self.permanently_stopped { + return Ok(false); + } + + let cancellation = CancellationToken::new(); + let thread_cancellation = cancellation.clone(); + let config = self.config.clone(); + let state = Arc::clone(&self.state); + let log_dispatch = tracing::dispatcher::get_default(Clone::clone); + let thread = std::thread::Builder::new() + .name("ddtrace-ffe-agentless".to_owned()) + .spawn(move || { + let _log_guard = tracing::dispatcher::set_default(&log_dispatch); + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(_) => { + log::log( + Log::Warn, + "Feature Flags agentless runtime could not be created", + ); + return; + } + }; + + runtime.block_on(async move { + let mut poller = Poller { + endpoint: config.endpoint, + poll_interval: config.poll_interval, + request_timeout: config.request_timeout, + transport: HyperTransport::new(), + sink: GlobalConfigurationSink, + state, + retry_delay, + }; + poller.run(thread_cancellation).await; + }); + }) + .map_err(|_| WorkerStartError::ThreadUnavailable)?; + + self.running = Some(RunningWorker { + cancellation, + thread, + }); + Ok(true) + } + + pub(crate) fn shutdown(&mut self) { + self.permanently_stopped = true; + self.stop_running(); + } + + pub(crate) fn prepare_for_fork(&mut self) { + self.restart_after_fork = self.running.is_some() && !self.permanently_stopped; + self.stop_running(); + } + + pub(crate) fn resume_after_fork_parent(&mut self) -> Result<(), WorkerStartError> { + self.restart_after_fork() + } + + pub(crate) fn reset_after_fork_child(&mut self) -> Result<(), WorkerStartError> { + // `prepare_for_fork` joined the only worker before the process split, + // so no synchronization primitive can be inherited while held. + self.state.warnings.reset(); + self.restart_after_fork() + } + + pub(crate) fn poll_count(&self) -> u64 { + self.state.poll_count.load(Ordering::Relaxed) + } + + fn restart_after_fork(&mut self) -> Result<(), WorkerStartError> { + if self.restart_after_fork && !self.permanently_stopped { + self.start()?; + self.restart_after_fork = false; + } + Ok(()) + } + + fn stop_running(&mut self) { + if let Some(running) = self.running.take() { + running.cancellation.cancel(); + let _ = running.thread.join(); + } + } +} + +impl Drop for AgentlessWorker { + fn drop(&mut self) { + self.stop_running(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::VecDeque; + use std::io::Write; + use std::sync::atomic::{AtomicBool, AtomicUsize}; + + const EMPTY_CONFIG: &[u8] = br#"{ + "createdAt": "2026-05-22T00:00:00.000Z", + "format": "SERVER", + "environment": {"name": "test"}, + "flags": {} + }"#; + + struct UnusedTransport; + + impl Transport for UnusedTransport { + fn get<'a>( + &'a self, + _endpoint: &'a AgentlessEndpoint, + _timeout: Duration, + _etag: Option<&'a str>, + _cancellation: &'a CancellationToken, + ) -> FetchFuture<'a> { + Box::pin(async { unreachable!("transport is not used by response-processing tests") }) + } + } + + struct TestSink { + fail: AtomicBool, + } + + struct QueueTransport { + responses: Mutex>>, + calls: AtomicUsize, + in_flight: AtomicUsize, + max_in_flight: AtomicUsize, + } + + impl QueueTransport { + fn new( + responses: impl IntoIterator>, + ) -> Self { + Self { + responses: Mutex::new(responses.into_iter().collect()), + calls: AtomicUsize::new(0), + in_flight: AtomicUsize::new(0), + max_in_flight: AtomicUsize::new(0), + } + } + } + + impl Transport for QueueTransport { + fn get<'a>( + &'a self, + _endpoint: &'a AgentlessEndpoint, + _timeout: Duration, + _etag: Option<&'a str>, + _cancellation: &'a CancellationToken, + ) -> FetchFuture<'a> { + Box::pin(async move { + self.calls.fetch_add(1, Ordering::Relaxed); + let in_flight = self.in_flight.fetch_add(1, Ordering::Relaxed) + 1; + self.max_in_flight.fetch_max(in_flight, Ordering::Relaxed); + tokio::task::yield_now().await; + let result = lock_unpoisoned(&self.responses) + .pop_front() + .unwrap_or(Err(TransportFailure::Request)); + self.in_flight.fetch_sub(1, Ordering::Relaxed); + result + }) + } + } + + impl ConfigurationSink for TestSink { + fn apply(&self, _configuration: Configuration) -> Result { + if self.fail.load(Ordering::Relaxed) { + Err(()) + } else { + Ok(ConfigurationTransition::Ready) + } + } + } + + fn test_poller(fail_apply: bool) -> Poller { + Poller { + endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") + .expect("test endpoint"), + poll_interval: Duration::from_secs(30), + request_timeout: Duration::from_secs(5), + transport: UnusedTransport, + sink: TestSink { + fail: AtomicBool::new(fail_apply), + }, + state: Arc::new(PollState::default()), + retry_delay, + } + } + + fn no_retry_delay(_: Duration, _: usize, _: f64) -> Duration { + Duration::ZERO + } + + fn response(status: u16, etag: Option<&str>, body: &[u8]) -> PollResponse { + PollResponse { + status, + etag: etag.map(str::to_owned), + content_encoding: None, + body: body.to_vec(), + } + } + + fn gzip(payload: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(payload).expect("compress test payload"); + encoder.finish().expect("finish test payload") + } + + #[test] + fn retries_only_transport_timeouts_and_selected_statuses() { + for status in [408, 429, 500, 503, 599] { + assert!(is_retryable_status(status), "status {status}"); + } + for status in [200, 204, 301, 400, 401, 403, 404, 499, 600] { + assert!(!is_retryable_status(status), "status {status}"); + } + } + + #[test] + fn retry_delay_uses_the_cross_sdk_clamps_and_jitter() { + assert_eq!( + retry_delay(Duration::from_secs(30), 1, 0.0), + Duration::from_secs(4) + ); + assert_eq!( + retry_delay(Duration::from_secs(30), 1, 1.0), + Duration::from_secs(6) + ); + assert_eq!( + retry_delay(Duration::from_secs(30), 2, 0.0), + Duration::from_secs(8) + ); + assert_eq!( + retry_delay(Duration::from_secs(30), 2, 1.0), + Duration::from_secs(12) + ); + assert_eq!( + retry_delay(Duration::from_secs(1), 1, 0.0), + Duration::from_millis(1_600) + ); + } + + #[test] + fn decoded_body_limit_applies_after_gzip() { + let payload = vec![b'a'; 256]; + let compressed = gzip(&payload); + assert!(compressed.len() < 64); + assert_eq!( + decode_response_body(&compressed, Some(" GZIP "), 64), + Err(TransportFailure::ResponseTooLarge) + ); + assert_eq!( + decode_response_body(&gzip(b"configuration"), Some("gzip"), 64).expect("decode gzip"), + b"configuration" + ); + } + + #[test] + fn etag_advances_only_after_parse_and_apply_succeed() { + let mut poller = test_poller(true); + poller.state.set_etag(Some("old".to_owned())); + + assert_eq!( + poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), + PollOutcome::Stop + ); + assert_eq!(poller.state.etag().as_deref(), Some("old")); + + poller.sink.fail.store(false, Ordering::Relaxed); + assert_eq!( + poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), + PollOutcome::Success + ); + assert_eq!(poller.state.etag().as_deref(), Some("new")); + + assert_eq!( + poller.process_response(response(200, Some("ignored"), b"not-json")), + PollOutcome::Stop + ); + assert_eq!(poller.state.etag().as_deref(), Some("new")); + } + + #[test] + fn non_200_responses_are_never_decoded() { + let mut poller = test_poller(false); + poller.state.set_etag(Some("current".to_owned())); + assert_eq!( + poller.process_response(response(304, Some("ignored"), b"not-json")), + PollOutcome::Success + ); + assert_eq!(poller.state.etag().as_deref(), Some("current")); + assert_eq!( + poller.process_response(response(401, None, EMPTY_CONFIG)), + PollOutcome::Stop + ); + assert_eq!(poller.state.etag().as_deref(), Some("current")); + } + + #[test] + fn request_authentication_is_managed_only() { + let managed = + AgentlessEndpoint::build("", "datadoghq.com", "", "secret").expect("managed endpoint"); + let custom = + AgentlessEndpoint::build("https://user:password@localhost/custom", "", "", "secret") + .expect("custom endpoint"); + + let managed_request = build_request(&managed, Some("etag")).expect("managed request"); + assert_eq!(managed_request.headers()[API_KEY_HEADER], "secret"); + assert_eq!(managed_request.headers()[IF_NONE_MATCH], "etag"); + assert_eq!(managed_request.headers()[INTERNAL_UNTRACED_HEADER], "1"); + + let custom_request = build_request(&custom, None).expect("custom request"); + assert!(!custom_request.headers().contains_key(API_KEY_HEADER)); + assert_eq!( + custom_request.headers()[AUTHORIZATION_HEADER], + "Basic dXNlcjpwYXNzd29yZA==" + ); + assert_eq!(custom_request.uri(), "https://localhost/custom"); + } + + #[test] + fn warning_categories_are_suppressed_independently() { + let warnings = WarningSlots::default(); + assert!(warnings.take(WarningCategory::Request)); + assert!(!warnings.take(WarningCategory::Request)); + assert!(warnings.take(WarningCategory::UnexpectedHttp)); + assert!(!warnings.take(WarningCategory::UnexpectedHttp)); + warnings.reset(); + assert!(warnings.take(WarningCategory::Request)); + } + + #[tokio::test] + async fn a_poll_retries_three_times_without_overlapping_requests() { + let transport = QueueTransport::new([ + Ok(response(503, None, b"ignored")), + Err(TransportFailure::TimedOut), + Ok(response(200, Some("accepted"), EMPTY_CONFIG)), + ]); + let state = Arc::new(PollState::default()); + let mut poller = Poller { + endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") + .expect("test endpoint"), + poll_interval: Duration::from_secs(30), + request_timeout: Duration::from_secs(5), + transport, + sink: TestSink { + fail: AtomicBool::new(false), + }, + state: Arc::clone(&state), + retry_delay: no_retry_delay, + }; + + assert_eq!( + poller.poll(&CancellationToken::new()).await, + PollOutcome::Success + ); + assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 3); + assert_eq!(poller.transport.max_in_flight.load(Ordering::Relaxed), 1); + assert_eq!(state.poll_count.load(Ordering::Relaxed), 3); + assert_eq!(state.etag().as_deref(), Some("accepted")); + } + + #[tokio::test] + async fn cancellation_before_a_poll_prevents_the_first_request() { + let transport = QueueTransport::new([]); + let mut poller = Poller { + endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") + .expect("test endpoint"), + poll_interval: Duration::from_secs(30), + request_timeout: Duration::from_secs(5), + transport, + sink: TestSink { + fail: AtomicBool::new(false), + }, + state: Arc::new(PollState::default()), + retry_delay: no_retry_delay, + }; + let cancellation = CancellationToken::new(); + cancellation.cancel(); + + assert_eq!(poller.poll(&cancellation).await, PollOutcome::Cancelled); + assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 0); + } +} diff --git a/zend_abstract_interface/config/config.h b/zend_abstract_interface/config/config.h index 41ea02e2fbb..ba5cd28e1d2 100644 --- a/zend_abstract_interface/config/config.h +++ b/zend_abstract_interface/config/config.h @@ -20,7 +20,9 @@ typedef uint16_t zai_config_id; #define ZAI_CONFIG_ENTRIES_COUNT_MAX 320 #define ZAI_CONFIG_NAMES_COUNT_MAX 4 -#define ZAI_CONFIG_NAME_BUFSIZ 72 +// Accommodate the longest supported environment name after its `DD_` prefix is +// expanded to the `datadog.` INI prefix. +#define ZAI_CONFIG_NAME_BUFSIZ 80 #define ZAI_CONFIG_ENTRY(_id, _name, _type, default, ...) \ { \ From 6969ce37f805dfb332ddd4fb1039ca4486eae450 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 18 Sep 2026 13:11:01 +0300 Subject: [PATCH 03/10] refactor(ffe): split agentless polling responsibilities --- components-rs/ffe/agentless.rs | 819 +---------------------- components-rs/ffe/agentless/poller.rs | 553 +++++++++++++++ components-rs/ffe/agentless/transport.rs | 291 ++++++++ 3 files changed, 859 insertions(+), 804 deletions(-) create mode 100644 components-rs/ffe/agentless/poller.rs create mode 100644 components-rs/ffe/agentless/transport.rs diff --git a/components-rs/ffe/agentless.rs b/components-rs/ffe/agentless.rs index 1fe4663b676..c7581285052 100644 --- a/components-rs/ffe/agentless.rs +++ b/components-rs/ffe/agentless.rs @@ -1,511 +1,18 @@ // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +mod poller; +mod transport; + +use self::poller::{GlobalConfigurationSink, PollState, Poller}; +use self::transport::HyperTransport; use super::settings::AgentlessEndpoint; -use super::{store_config, ConfigurationTransition}; use crate::log::{self, Log}; -use base64::Engine; -use datadog_ffe::rules_based::{Configuration, UniversalFlagConfig}; -use flate2::read::GzDecoder; -use futures_util::FutureExt; -use http::header::{ACCEPT_ENCODING, CONTENT_ENCODING, CONTENT_LENGTH, ETAG, IF_NONE_MATCH}; -use http::{Method, Request}; -use http_body_util::BodyExt; -use libdd_common::http_common::{self, Body}; -use percent_encoding::percent_decode_str; -use std::future::Future; -use std::io::Read; -use std::panic::AssertUnwindSafe; -use std::pin::Pin; -use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::Arc; use std::thread::JoinHandle; use std::time::Duration; use tokio_util::sync::CancellationToken; -const MAX_POLL_ATTEMPTS: usize = 3; -const MAX_RESPONSE_BODY_BYTES: usize = 10 << 20; -const CLIENT_LANGUAGE: &str = "php"; -const INTERNAL_UNTRACED_HEADER: &str = "DD-Internal-Untraced-Request"; -const API_KEY_HEADER: &str = "DD-API-KEY"; -const AUTHORIZATION_HEADER: &str = "Authorization"; - -type FetchFuture<'a> = - Pin> + Send + 'a>>; - -trait Transport: Send + Sync { - fn get<'a>( - &'a self, - endpoint: &'a AgentlessEndpoint, - timeout: Duration, - etag: Option<&'a str>, - cancellation: &'a CancellationToken, - ) -> FetchFuture<'a>; -} - -trait ConfigurationSink: Send + Sync { - fn apply(&self, configuration: Configuration) -> Result; -} - -struct GlobalConfigurationSink; - -impl ConfigurationSink for GlobalConfigurationSink { - fn apply(&self, configuration: Configuration) -> Result { - Ok(store_config(configuration)) - } -} - -#[derive(Debug, Eq, PartialEq)] -enum TransportFailure { - BuildRequest, - Cancelled, - TimedOut, - Request, - ResponseBody, - ResponseTooLarge, -} - -struct PollResponse { - status: u16, - etag: Option, - content_encoding: Option, - body: Vec, -} - -struct HyperTransport { - client: libdd_common::HttpClient, -} - -impl HyperTransport { - fn new() -> Self { - Self { - // This native client is outside PHP HTTP instrumentation, so it - // cannot recursively trace its own polling requests. Hyper also - // leaves 3xx responses visible instead of following redirects. - client: http_common::new_client_periodic(), - } - } - - async fn get_inner( - &self, - endpoint: &AgentlessEndpoint, - timeout: Duration, - etag: Option<&str>, - cancellation: &CancellationToken, - ) -> Result { - let request = build_request(endpoint, etag)?; - let operation = async { - let response = self - .client - .request(request) - .await - .map_err(|_| TransportFailure::Request)?; - let status = response.status().as_u16(); - let etag = header_value(response.headers(), ETAG); - let content_encoding = header_value(response.headers(), CONTENT_ENCODING); - - // A non-200 body can never contain configuration. Dropping it here - // also prevents a malicious error response from consuming memory. - if status != 200 { - return Ok(PollResponse { - status, - etag, - content_encoding, - body: Vec::new(), - }); - } - - if response - .headers() - .get(CONTENT_LENGTH) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.parse::().ok()) - .is_some_and(|length| length > MAX_RESPONSE_BODY_BYTES as u64) - { - return Err(TransportFailure::ResponseTooLarge); - } - - let body = collect_bounded(response.into_body(), MAX_RESPONSE_BODY_BYTES).await?; - Ok(PollResponse { - status, - etag, - content_encoding, - body, - }) - }; - - tokio::select! { - _ = cancellation.cancelled() => Err(TransportFailure::Cancelled), - result = tokio::time::timeout(timeout, operation) => { - result.map_err(|_| TransportFailure::TimedOut)? - } - } - } -} - -impl Transport for HyperTransport { - fn get<'a>( - &'a self, - endpoint: &'a AgentlessEndpoint, - timeout: Duration, - etag: Option<&'a str>, - cancellation: &'a CancellationToken, - ) -> FetchFuture<'a> { - Box::pin(self.get_inner(endpoint, timeout, etag, cancellation)) - } -} - -fn build_request( - endpoint: &AgentlessEndpoint, - etag: Option<&str>, -) -> Result, TransportFailure> { - let (request_url, authorization) = request_url_and_authorization(endpoint.as_str())?; - let mut builder = Request::builder() - .method(Method::GET) - .uri(request_url) - .header(ACCEPT_ENCODING, "gzip") - .header("DD-Client-Library-Language", CLIENT_LANGUAGE) - .header( - "DD-Client-Library-Version", - include_str!("../../VERSION").trim(), - ) - .header(INTERNAL_UNTRACED_HEADER, "1"); - - if let Some(etag) = etag { - builder = builder.header(IF_NONE_MATCH, etag); - } - if let Some(api_key) = endpoint.api_key() { - builder = builder.header(API_KEY_HEADER, api_key); - } - if let Some(authorization) = authorization { - builder = builder.header(AUTHORIZATION_HEADER, authorization); - } - - builder - .body(Body::empty()) - .map_err(|_| TransportFailure::BuildRequest) -} - -fn request_url_and_authorization( - endpoint: &str, -) -> Result<(String, Option), TransportFailure> { - let mut url = url::Url::parse(endpoint).map_err(|_| TransportFailure::BuildRequest)?; - if url.username().is_empty() { - return Ok((endpoint.to_owned(), None)); - } - - let mut credentials = percent_decode_str(url.username()).collect::>(); - credentials.push(b':'); - if let Some(password) = url.password() { - credentials.extend(percent_decode_str(password)); - } - let authorization = format!( - "Basic {}", - base64::engine::general_purpose::STANDARD.encode(credentials) - ); - - url.set_username("") - .map_err(|_| TransportFailure::BuildRequest)?; - url.set_password(None) - .map_err(|_| TransportFailure::BuildRequest)?; - Ok((url.to_string(), Some(authorization))) -} - -fn header_value(headers: &http::HeaderMap, name: http::header::HeaderName) -> Option { - headers - .get(name) - .and_then(|value| value.to_str().ok()) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) -} - -async fn collect_bounded( - mut body: hyper::body::Incoming, - limit: usize, -) -> Result, TransportFailure> { - let mut bytes = Vec::new(); - while let Some(frame) = body.frame().await { - let frame = frame.map_err(|_| TransportFailure::ResponseBody)?; - if let Some(data) = frame.data_ref() { - let remaining = limit.saturating_sub(bytes.len()); - if data.len() > remaining { - return Err(TransportFailure::ResponseTooLarge); - } - bytes.extend_from_slice(data); - } - } - Ok(bytes) -} - -fn decode_response_body( - body: &[u8], - content_encoding: Option<&str>, - limit: usize, -) -> Result, TransportFailure> { - if !content_encoding.is_some_and(|value| value.trim().eq_ignore_ascii_case("gzip")) { - if body.len() > limit { - return Err(TransportFailure::ResponseTooLarge); - } - return Ok(body.to_vec()); - } - - let decoder = GzDecoder::new(body); - let mut decoded = Vec::new(); - decoder - .take(limit as u64 + 1) - .read_to_end(&mut decoded) - .map_err(|_| TransportFailure::ResponseBody)?; - if decoded.len() > limit { - return Err(TransportFailure::ResponseTooLarge); - } - Ok(decoded) -} - -#[derive(Clone, Copy)] -#[repr(u8)] -enum WarningCategory { - Authentication = 1 << 0, - RetryableHttp = 1 << 1, - UnexpectedHttp = 1 << 2, - Request = 1 << 3, - MalformedPayload = 1 << 4, - ApplyFailure = 1 << 5, - UnexpectedException = 1 << 6, -} - -#[derive(Default)] -struct WarningSlots(AtomicU8); - -impl WarningSlots { - fn take(&self, category: WarningCategory) -> bool { - self.0.fetch_or(category as u8, Ordering::Relaxed) & category as u8 == 0 - } - - fn reset(&self) { - self.0.store(0, Ordering::Relaxed); - } -} - -#[derive(Default)] -struct PollState { - etag: Mutex>, - warnings: WarningSlots, - poll_count: AtomicU64, -} - -impl PollState { - fn etag(&self) -> Option { - lock_unpoisoned(&self.etag).clone() - } - - fn set_etag(&self, etag: Option) { - *lock_unpoisoned(&self.etag) = etag; - } -} - -fn lock_unpoisoned(mutex: &Mutex) -> MutexGuard<'_, T> { - mutex - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum PollOutcome { - Success, - Retryable, - Stop, - Cancelled, -} - -struct Poller { - endpoint: AgentlessEndpoint, - poll_interval: Duration, - request_timeout: Duration, - transport: T, - sink: S, - state: Arc, - retry_delay: fn(Duration, usize, f64) -> Duration, -} - -impl Poller { - async fn run(&mut self, cancellation: CancellationToken) { - loop { - if cancellation.is_cancelled() { - return; - } - - let poll = AssertUnwindSafe(self.poll(&cancellation)).catch_unwind(); - match poll.await { - Ok(PollOutcome::Cancelled) => return, - Ok(_) => {} - Err(_) => self.warn( - WarningCategory::UnexpectedException, - "Feature Flags agentless polling failed unexpectedly; polling continues", - ), - } - - tokio::select! { - _ = cancellation.cancelled() => return, - _ = tokio::time::sleep(self.poll_interval) => {} - } - } - } - - async fn poll(&mut self, cancellation: &CancellationToken) -> PollOutcome { - for attempt in 1..=MAX_POLL_ATTEMPTS { - let outcome = self.poll_once(cancellation).await; - if outcome != PollOutcome::Retryable || attempt == MAX_POLL_ATTEMPTS { - return outcome; - } - - let delay = (self.retry_delay)(self.poll_interval, attempt, fastrand::f64()); - tokio::select! { - _ = cancellation.cancelled() => return PollOutcome::Cancelled, - _ = tokio::time::sleep(delay) => {} - } - } - PollOutcome::Stop - } - - async fn poll_once(&mut self, cancellation: &CancellationToken) -> PollOutcome { - if cancellation.is_cancelled() { - return PollOutcome::Cancelled; - } - - self.state.poll_count.fetch_add(1, Ordering::Relaxed); - let etag = self.state.etag(); - match self - .transport - .get( - &self.endpoint, - self.request_timeout, - etag.as_deref(), - cancellation, - ) - .await - { - Ok(response) => self.process_response(response), - Err(TransportFailure::Cancelled) => PollOutcome::Cancelled, - Err(TransportFailure::BuildRequest) => { - self.warn( - WarningCategory::Request, - "Feature Flags agentless request could not be built", - ); - PollOutcome::Stop - } - Err(TransportFailure::ResponseTooLarge) => { - self.warn( - WarningCategory::MalformedPayload, - "Feature Flags agentless response exceeds the 10 MiB limit; keeping the last known configuration", - ); - PollOutcome::Stop - } - Err(_) => { - self.warn( - WarningCategory::Request, - "Feature Flags agentless request failed; keeping the last known configuration", - ); - PollOutcome::Retryable - } - } - } - - fn process_response(&mut self, response: PollResponse) -> PollOutcome { - match response.status { - 304 => return PollOutcome::Success, - 401 | 403 => { - self.warn( - WarningCategory::Authentication, - "Feature Flags agentless endpoint rejected authentication; verify endpoint authentication", - ); - return PollOutcome::Stop; - } - 200 => {} - status if is_retryable_status(status) => { - self.warn( - WarningCategory::RetryableHttp, - &format!("Feature Flags agentless endpoint returned retryable HTTP {status}"), - ); - return PollOutcome::Retryable; - } - status => { - self.warn( - WarningCategory::UnexpectedHttp, - &format!("Feature Flags agentless endpoint returned unexpected HTTP {status}"), - ); - return PollOutcome::Stop; - } - } - - let body = match decode_response_body( - &response.body, - response.content_encoding.as_deref(), - MAX_RESPONSE_BODY_BYTES, - ) { - Ok(body) => body, - Err(TransportFailure::ResponseTooLarge) => { - self.warn( - WarningCategory::MalformedPayload, - "Feature Flags agentless decoded response exceeds the 10 MiB limit; keeping the last known configuration", - ); - return PollOutcome::Stop; - } - Err(_) => { - self.warn( - WarningCategory::MalformedPayload, - "Feature Flags agentless endpoint returned an invalid response body", - ); - return PollOutcome::Stop; - } - }; - - let configuration = match UniversalFlagConfig::from_json(body) { - Ok(configuration) => Configuration::from_server_response(configuration), - Err(_) => { - self.warn( - WarningCategory::MalformedPayload, - "Feature Flags agentless endpoint returned malformed configuration", - ); - return PollOutcome::Stop; - } - }; - - if self.sink.apply(configuration).is_err() { - self.warn( - WarningCategory::ApplyFailure, - "Feature Flags agentless configuration could not be applied", - ); - return PollOutcome::Stop; - } - - // The response has now been parsed and applied. A blank or absent - // ETag intentionally clears an older value. - self.state.set_etag(response.etag); - PollOutcome::Success - } - - fn warn(&self, category: WarningCategory, message: &str) { - if self.state.warnings.take(category) { - log::log(Log::Warn, message); - } - } -} - -fn is_retryable_status(status: u16) -> bool { - matches!(status, 408 | 429 | 500..=599) -} - -fn retry_delay(poll_interval: Duration, attempt: usize, random: f64) -> Duration { - let base = if attempt <= 1 { - (poll_interval / 6).clamp(Duration::from_secs(2), Duration::from_secs(10)) - } else { - (poll_interval / 3).clamp(Duration::from_secs(5), Duration::from_secs(30)) - }; - base.mul_f64(0.8 + random.clamp(0.0, 1.0) * 0.4) - .max(Duration::from_secs(1)) -} - #[derive(Clone)] pub(crate) struct AgentlessWorkerConfig { endpoint: AgentlessEndpoint, @@ -587,15 +94,14 @@ impl AgentlessWorker { }; runtime.block_on(async move { - let mut poller = Poller { - endpoint: config.endpoint, - poll_interval: config.poll_interval, - request_timeout: config.request_timeout, - transport: HyperTransport::new(), - sink: GlobalConfigurationSink, + let mut poller = Poller::new( + config.endpoint, + config.poll_interval, + config.request_timeout, + HyperTransport::new(), + GlobalConfigurationSink, state, - retry_delay, - }; + ); poller.run(thread_cancellation).await; }); }) @@ -625,12 +131,12 @@ impl AgentlessWorker { pub(crate) fn reset_after_fork_child(&mut self) -> Result<(), WorkerStartError> { // `prepare_for_fork` joined the only worker before the process split, // so no synchronization primitive can be inherited while held. - self.state.warnings.reset(); + self.state.reset_warnings(); self.restart_after_fork() } pub(crate) fn poll_count(&self) -> u64 { - self.state.poll_count.load(Ordering::Relaxed) + self.state.poll_count() } fn restart_after_fork(&mut self) -> Result<(), WorkerStartError> { @@ -654,298 +160,3 @@ impl Drop for AgentlessWorker { self.stop_running(); } } - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::VecDeque; - use std::io::Write; - use std::sync::atomic::{AtomicBool, AtomicUsize}; - - const EMPTY_CONFIG: &[u8] = br#"{ - "createdAt": "2026-05-22T00:00:00.000Z", - "format": "SERVER", - "environment": {"name": "test"}, - "flags": {} - }"#; - - struct UnusedTransport; - - impl Transport for UnusedTransport { - fn get<'a>( - &'a self, - _endpoint: &'a AgentlessEndpoint, - _timeout: Duration, - _etag: Option<&'a str>, - _cancellation: &'a CancellationToken, - ) -> FetchFuture<'a> { - Box::pin(async { unreachable!("transport is not used by response-processing tests") }) - } - } - - struct TestSink { - fail: AtomicBool, - } - - struct QueueTransport { - responses: Mutex>>, - calls: AtomicUsize, - in_flight: AtomicUsize, - max_in_flight: AtomicUsize, - } - - impl QueueTransport { - fn new( - responses: impl IntoIterator>, - ) -> Self { - Self { - responses: Mutex::new(responses.into_iter().collect()), - calls: AtomicUsize::new(0), - in_flight: AtomicUsize::new(0), - max_in_flight: AtomicUsize::new(0), - } - } - } - - impl Transport for QueueTransport { - fn get<'a>( - &'a self, - _endpoint: &'a AgentlessEndpoint, - _timeout: Duration, - _etag: Option<&'a str>, - _cancellation: &'a CancellationToken, - ) -> FetchFuture<'a> { - Box::pin(async move { - self.calls.fetch_add(1, Ordering::Relaxed); - let in_flight = self.in_flight.fetch_add(1, Ordering::Relaxed) + 1; - self.max_in_flight.fetch_max(in_flight, Ordering::Relaxed); - tokio::task::yield_now().await; - let result = lock_unpoisoned(&self.responses) - .pop_front() - .unwrap_or(Err(TransportFailure::Request)); - self.in_flight.fetch_sub(1, Ordering::Relaxed); - result - }) - } - } - - impl ConfigurationSink for TestSink { - fn apply(&self, _configuration: Configuration) -> Result { - if self.fail.load(Ordering::Relaxed) { - Err(()) - } else { - Ok(ConfigurationTransition::Ready) - } - } - } - - fn test_poller(fail_apply: bool) -> Poller { - Poller { - endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") - .expect("test endpoint"), - poll_interval: Duration::from_secs(30), - request_timeout: Duration::from_secs(5), - transport: UnusedTransport, - sink: TestSink { - fail: AtomicBool::new(fail_apply), - }, - state: Arc::new(PollState::default()), - retry_delay, - } - } - - fn no_retry_delay(_: Duration, _: usize, _: f64) -> Duration { - Duration::ZERO - } - - fn response(status: u16, etag: Option<&str>, body: &[u8]) -> PollResponse { - PollResponse { - status, - etag: etag.map(str::to_owned), - content_encoding: None, - body: body.to_vec(), - } - } - - fn gzip(payload: &[u8]) -> Vec { - let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - encoder.write_all(payload).expect("compress test payload"); - encoder.finish().expect("finish test payload") - } - - #[test] - fn retries_only_transport_timeouts_and_selected_statuses() { - for status in [408, 429, 500, 503, 599] { - assert!(is_retryable_status(status), "status {status}"); - } - for status in [200, 204, 301, 400, 401, 403, 404, 499, 600] { - assert!(!is_retryable_status(status), "status {status}"); - } - } - - #[test] - fn retry_delay_uses_the_cross_sdk_clamps_and_jitter() { - assert_eq!( - retry_delay(Duration::from_secs(30), 1, 0.0), - Duration::from_secs(4) - ); - assert_eq!( - retry_delay(Duration::from_secs(30), 1, 1.0), - Duration::from_secs(6) - ); - assert_eq!( - retry_delay(Duration::from_secs(30), 2, 0.0), - Duration::from_secs(8) - ); - assert_eq!( - retry_delay(Duration::from_secs(30), 2, 1.0), - Duration::from_secs(12) - ); - assert_eq!( - retry_delay(Duration::from_secs(1), 1, 0.0), - Duration::from_millis(1_600) - ); - } - - #[test] - fn decoded_body_limit_applies_after_gzip() { - let payload = vec![b'a'; 256]; - let compressed = gzip(&payload); - assert!(compressed.len() < 64); - assert_eq!( - decode_response_body(&compressed, Some(" GZIP "), 64), - Err(TransportFailure::ResponseTooLarge) - ); - assert_eq!( - decode_response_body(&gzip(b"configuration"), Some("gzip"), 64).expect("decode gzip"), - b"configuration" - ); - } - - #[test] - fn etag_advances_only_after_parse_and_apply_succeed() { - let mut poller = test_poller(true); - poller.state.set_etag(Some("old".to_owned())); - - assert_eq!( - poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), - PollOutcome::Stop - ); - assert_eq!(poller.state.etag().as_deref(), Some("old")); - - poller.sink.fail.store(false, Ordering::Relaxed); - assert_eq!( - poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), - PollOutcome::Success - ); - assert_eq!(poller.state.etag().as_deref(), Some("new")); - - assert_eq!( - poller.process_response(response(200, Some("ignored"), b"not-json")), - PollOutcome::Stop - ); - assert_eq!(poller.state.etag().as_deref(), Some("new")); - } - - #[test] - fn non_200_responses_are_never_decoded() { - let mut poller = test_poller(false); - poller.state.set_etag(Some("current".to_owned())); - assert_eq!( - poller.process_response(response(304, Some("ignored"), b"not-json")), - PollOutcome::Success - ); - assert_eq!(poller.state.etag().as_deref(), Some("current")); - assert_eq!( - poller.process_response(response(401, None, EMPTY_CONFIG)), - PollOutcome::Stop - ); - assert_eq!(poller.state.etag().as_deref(), Some("current")); - } - - #[test] - fn request_authentication_is_managed_only() { - let managed = - AgentlessEndpoint::build("", "datadoghq.com", "", "secret").expect("managed endpoint"); - let custom = - AgentlessEndpoint::build("https://user:password@localhost/custom", "", "", "secret") - .expect("custom endpoint"); - - let managed_request = build_request(&managed, Some("etag")).expect("managed request"); - assert_eq!(managed_request.headers()[API_KEY_HEADER], "secret"); - assert_eq!(managed_request.headers()[IF_NONE_MATCH], "etag"); - assert_eq!(managed_request.headers()[INTERNAL_UNTRACED_HEADER], "1"); - - let custom_request = build_request(&custom, None).expect("custom request"); - assert!(!custom_request.headers().contains_key(API_KEY_HEADER)); - assert_eq!( - custom_request.headers()[AUTHORIZATION_HEADER], - "Basic dXNlcjpwYXNzd29yZA==" - ); - assert_eq!(custom_request.uri(), "https://localhost/custom"); - } - - #[test] - fn warning_categories_are_suppressed_independently() { - let warnings = WarningSlots::default(); - assert!(warnings.take(WarningCategory::Request)); - assert!(!warnings.take(WarningCategory::Request)); - assert!(warnings.take(WarningCategory::UnexpectedHttp)); - assert!(!warnings.take(WarningCategory::UnexpectedHttp)); - warnings.reset(); - assert!(warnings.take(WarningCategory::Request)); - } - - #[tokio::test] - async fn a_poll_retries_three_times_without_overlapping_requests() { - let transport = QueueTransport::new([ - Ok(response(503, None, b"ignored")), - Err(TransportFailure::TimedOut), - Ok(response(200, Some("accepted"), EMPTY_CONFIG)), - ]); - let state = Arc::new(PollState::default()); - let mut poller = Poller { - endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") - .expect("test endpoint"), - poll_interval: Duration::from_secs(30), - request_timeout: Duration::from_secs(5), - transport, - sink: TestSink { - fail: AtomicBool::new(false), - }, - state: Arc::clone(&state), - retry_delay: no_retry_delay, - }; - - assert_eq!( - poller.poll(&CancellationToken::new()).await, - PollOutcome::Success - ); - assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 3); - assert_eq!(poller.transport.max_in_flight.load(Ordering::Relaxed), 1); - assert_eq!(state.poll_count.load(Ordering::Relaxed), 3); - assert_eq!(state.etag().as_deref(), Some("accepted")); - } - - #[tokio::test] - async fn cancellation_before_a_poll_prevents_the_first_request() { - let transport = QueueTransport::new([]); - let mut poller = Poller { - endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") - .expect("test endpoint"), - poll_interval: Duration::from_secs(30), - request_timeout: Duration::from_secs(5), - transport, - sink: TestSink { - fail: AtomicBool::new(false), - }, - state: Arc::new(PollState::default()), - retry_delay: no_retry_delay, - }; - let cancellation = CancellationToken::new(); - cancellation.cancel(); - - assert_eq!(poller.poll(&cancellation).await, PollOutcome::Cancelled); - assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 0); - } -} diff --git a/components-rs/ffe/agentless/poller.rs b/components-rs/ffe/agentless/poller.rs new file mode 100644 index 00000000000..2865dc4ba21 --- /dev/null +++ b/components-rs/ffe/agentless/poller.rs @@ -0,0 +1,553 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use super::super::settings::AgentlessEndpoint; +use super::transport::{ + decode_response_body, PollResponse, Transport, TransportFailure, MAX_RESPONSE_BODY_BYTES, +}; +use crate::ffe::{store_config, ConfigurationTransition}; +use crate::log::{self, Log}; +use datadog_ffe::rules_based::{Configuration, UniversalFlagConfig}; +use futures_util::FutureExt; +use std::panic::AssertUnwindSafe; +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Duration; +use tokio_util::sync::CancellationToken; + +const MAX_POLL_ATTEMPTS: usize = 3; + +pub(super) trait ConfigurationSink: Send + Sync { + fn apply(&self, configuration: Configuration) -> Result; +} + +pub(super) struct GlobalConfigurationSink; + +impl ConfigurationSink for GlobalConfigurationSink { + fn apply(&self, configuration: Configuration) -> Result { + Ok(store_config(configuration)) + } +} + +#[derive(Clone, Copy)] +#[repr(u8)] +enum WarningCategory { + Authentication = 1 << 0, + RetryableHttp = 1 << 1, + UnexpectedHttp = 1 << 2, + Request = 1 << 3, + MalformedPayload = 1 << 4, + ApplyFailure = 1 << 5, + UnexpectedException = 1 << 6, +} + +#[derive(Default)] +struct WarningSlots(AtomicU8); + +impl WarningSlots { + fn take(&self, category: WarningCategory) -> bool { + self.0.fetch_or(category as u8, Ordering::Relaxed) & category as u8 == 0 + } + + fn reset(&self) { + self.0.store(0, Ordering::Relaxed); + } +} + +#[derive(Default)] +pub(super) struct PollState { + etag: Mutex>, + warnings: WarningSlots, + poll_count: AtomicU64, +} + +impl PollState { + fn etag(&self) -> Option { + lock_unpoisoned(&self.etag).clone() + } + + fn set_etag(&self, etag: Option) { + *lock_unpoisoned(&self.etag) = etag; + } + + pub(super) fn reset_warnings(&self) { + self.warnings.reset(); + } + + pub(super) fn poll_count(&self) -> u64 { + self.poll_count.load(Ordering::Relaxed) + } +} + +fn lock_unpoisoned(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PollOutcome { + Success, + Retryable, + Stop, + Cancelled, +} + +pub(super) struct Poller { + endpoint: AgentlessEndpoint, + poll_interval: Duration, + request_timeout: Duration, + transport: T, + sink: S, + state: Arc, + retry_delay: fn(Duration, usize, f64) -> Duration, +} + +impl Poller { + pub(super) fn new( + endpoint: AgentlessEndpoint, + poll_interval: Duration, + request_timeout: Duration, + transport: T, + sink: S, + state: Arc, + ) -> Self { + Self { + endpoint, + poll_interval, + request_timeout, + transport, + sink, + state, + retry_delay, + } + } + + pub(super) async fn run(&mut self, cancellation: CancellationToken) { + loop { + if cancellation.is_cancelled() { + return; + } + + let poll = AssertUnwindSafe(self.poll(&cancellation)).catch_unwind(); + match poll.await { + Ok(PollOutcome::Cancelled) => return, + Ok(_) => {} + Err(_) => self.warn( + WarningCategory::UnexpectedException, + "Feature Flags agentless polling failed unexpectedly; polling continues", + ), + } + + tokio::select! { + _ = cancellation.cancelled() => return, + _ = tokio::time::sleep(self.poll_interval) => {} + } + } + } + + async fn poll(&mut self, cancellation: &CancellationToken) -> PollOutcome { + for attempt in 1..=MAX_POLL_ATTEMPTS { + let outcome = self.poll_once(cancellation).await; + if outcome != PollOutcome::Retryable || attempt == MAX_POLL_ATTEMPTS { + return outcome; + } + + let delay = (self.retry_delay)(self.poll_interval, attempt, fastrand::f64()); + tokio::select! { + _ = cancellation.cancelled() => return PollOutcome::Cancelled, + _ = tokio::time::sleep(delay) => {} + } + } + PollOutcome::Stop + } + + async fn poll_once(&mut self, cancellation: &CancellationToken) -> PollOutcome { + if cancellation.is_cancelled() { + return PollOutcome::Cancelled; + } + + self.state.poll_count.fetch_add(1, Ordering::Relaxed); + let etag = self.state.etag(); + match self + .transport + .get( + &self.endpoint, + self.request_timeout, + etag.as_deref(), + cancellation, + ) + .await + { + Ok(response) => self.process_response(response), + Err(TransportFailure::Cancelled) => PollOutcome::Cancelled, + Err(TransportFailure::BuildRequest) => { + self.warn( + WarningCategory::Request, + "Feature Flags agentless request could not be built", + ); + PollOutcome::Stop + } + Err(TransportFailure::ResponseTooLarge) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless response exceeds the 10 MiB limit; keeping the last known configuration", + ); + PollOutcome::Stop + } + Err(_) => { + self.warn( + WarningCategory::Request, + "Feature Flags agentless request failed; keeping the last known configuration", + ); + PollOutcome::Retryable + } + } + } + + fn process_response(&mut self, response: PollResponse) -> PollOutcome { + match response.status { + 304 => return PollOutcome::Success, + 401 | 403 => { + self.warn( + WarningCategory::Authentication, + "Feature Flags agentless endpoint rejected authentication; verify endpoint authentication", + ); + return PollOutcome::Stop; + } + 200 => {} + status if is_retryable_status(status) => { + self.warn( + WarningCategory::RetryableHttp, + &format!("Feature Flags agentless endpoint returned retryable HTTP {status}"), + ); + return PollOutcome::Retryable; + } + status => { + self.warn( + WarningCategory::UnexpectedHttp, + &format!("Feature Flags agentless endpoint returned unexpected HTTP {status}"), + ); + return PollOutcome::Stop; + } + } + + let body = match decode_response_body( + &response.body, + response.content_encoding.as_deref(), + MAX_RESPONSE_BODY_BYTES, + ) { + Ok(body) => body, + Err(TransportFailure::ResponseTooLarge) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless decoded response exceeds the 10 MiB limit; keeping the last known configuration", + ); + return PollOutcome::Stop; + } + Err(_) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless endpoint returned an invalid response body", + ); + return PollOutcome::Stop; + } + }; + + let configuration = match UniversalFlagConfig::from_json(body) { + Ok(configuration) => Configuration::from_server_response(configuration), + Err(_) => { + self.warn( + WarningCategory::MalformedPayload, + "Feature Flags agentless endpoint returned malformed configuration", + ); + return PollOutcome::Stop; + } + }; + + if self.sink.apply(configuration).is_err() { + self.warn( + WarningCategory::ApplyFailure, + "Feature Flags agentless configuration could not be applied", + ); + return PollOutcome::Stop; + } + + // The response has now been parsed and applied. A blank or absent + // ETag intentionally clears an older value. + self.state.set_etag(response.etag); + PollOutcome::Success + } + + fn warn(&self, category: WarningCategory, message: &str) { + if self.state.warnings.take(category) { + log::log(Log::Warn, message); + } + } +} + +fn is_retryable_status(status: u16) -> bool { + matches!(status, 408 | 429 | 500..=599) +} + +fn retry_delay(poll_interval: Duration, attempt: usize, random: f64) -> Duration { + let base = if attempt <= 1 { + (poll_interval / 6).clamp(Duration::from_secs(2), Duration::from_secs(10)) + } else { + (poll_interval / 3).clamp(Duration::from_secs(5), Duration::from_secs(30)) + }; + base.mul_f64(0.8 + random.clamp(0.0, 1.0) * 0.4) + .max(Duration::from_secs(1)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ffe::agentless::transport::FetchFuture; + use std::collections::VecDeque; + use std::sync::atomic::{AtomicBool, AtomicUsize}; + + const EMPTY_CONFIG: &[u8] = br#"{ + "createdAt": "2026-05-22T00:00:00.000Z", + "format": "SERVER", + "environment": {"name": "test"}, + "flags": {} + }"#; + + struct UnusedTransport; + + impl Transport for UnusedTransport { + fn get<'a>( + &'a self, + _endpoint: &'a AgentlessEndpoint, + _timeout: Duration, + _etag: Option<&'a str>, + _cancellation: &'a CancellationToken, + ) -> FetchFuture<'a> { + Box::pin(async { unreachable!("transport is not used by response-processing tests") }) + } + } + + struct TestSink { + fail: AtomicBool, + } + + struct QueueTransport { + responses: Mutex>>, + calls: AtomicUsize, + in_flight: AtomicUsize, + max_in_flight: AtomicUsize, + } + + impl QueueTransport { + fn new( + responses: impl IntoIterator>, + ) -> Self { + Self { + responses: Mutex::new(responses.into_iter().collect()), + calls: AtomicUsize::new(0), + in_flight: AtomicUsize::new(0), + max_in_flight: AtomicUsize::new(0), + } + } + } + + impl Transport for QueueTransport { + fn get<'a>( + &'a self, + _endpoint: &'a AgentlessEndpoint, + _timeout: Duration, + _etag: Option<&'a str>, + _cancellation: &'a CancellationToken, + ) -> FetchFuture<'a> { + Box::pin(async move { + self.calls.fetch_add(1, Ordering::Relaxed); + let in_flight = self.in_flight.fetch_add(1, Ordering::Relaxed) + 1; + self.max_in_flight.fetch_max(in_flight, Ordering::Relaxed); + tokio::task::yield_now().await; + let result = lock_unpoisoned(&self.responses) + .pop_front() + .unwrap_or(Err(TransportFailure::Request)); + self.in_flight.fetch_sub(1, Ordering::Relaxed); + result + }) + } + } + + impl ConfigurationSink for TestSink { + fn apply(&self, _configuration: Configuration) -> Result { + if self.fail.load(Ordering::Relaxed) { + Err(()) + } else { + Ok(ConfigurationTransition::Ready) + } + } + } + + fn test_poller(fail_apply: bool) -> Poller { + Poller { + endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") + .expect("test endpoint"), + poll_interval: Duration::from_secs(30), + request_timeout: Duration::from_secs(5), + transport: UnusedTransport, + sink: TestSink { + fail: AtomicBool::new(fail_apply), + }, + state: Arc::new(PollState::default()), + retry_delay, + } + } + + fn no_retry_delay(_: Duration, _: usize, _: f64) -> Duration { + Duration::ZERO + } + + fn response(status: u16, etag: Option<&str>, body: &[u8]) -> PollResponse { + PollResponse { + status, + etag: etag.map(str::to_owned), + content_encoding: None, + body: body.to_vec(), + } + } + + #[test] + fn retries_only_transport_timeouts_and_selected_statuses() { + for status in [408, 429, 500, 503, 599] { + assert!(is_retryable_status(status), "status {status}"); + } + for status in [200, 204, 301, 400, 401, 403, 404, 499, 600] { + assert!(!is_retryable_status(status), "status {status}"); + } + } + + #[test] + fn retry_delay_uses_the_cross_sdk_clamps_and_jitter() { + assert_eq!( + retry_delay(Duration::from_secs(30), 1, 0.0), + Duration::from_secs(4) + ); + assert_eq!( + retry_delay(Duration::from_secs(30), 1, 1.0), + Duration::from_secs(6) + ); + assert_eq!( + retry_delay(Duration::from_secs(30), 2, 0.0), + Duration::from_secs(8) + ); + assert_eq!( + retry_delay(Duration::from_secs(30), 2, 1.0), + Duration::from_secs(12) + ); + assert_eq!( + retry_delay(Duration::from_secs(1), 1, 0.0), + Duration::from_millis(1_600) + ); + } + + #[test] + fn etag_advances_only_after_parse_and_apply_succeed() { + let mut poller = test_poller(true); + poller.state.set_etag(Some("old".to_owned())); + + assert_eq!( + poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), + PollOutcome::Stop + ); + assert_eq!(poller.state.etag().as_deref(), Some("old")); + + poller.sink.fail.store(false, Ordering::Relaxed); + assert_eq!( + poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), + PollOutcome::Success + ); + assert_eq!(poller.state.etag().as_deref(), Some("new")); + + assert_eq!( + poller.process_response(response(200, Some("ignored"), b"not-json")), + PollOutcome::Stop + ); + assert_eq!(poller.state.etag().as_deref(), Some("new")); + } + + #[test] + fn non_200_responses_are_never_decoded() { + let mut poller = test_poller(false); + poller.state.set_etag(Some("current".to_owned())); + assert_eq!( + poller.process_response(response(304, Some("ignored"), b"not-json")), + PollOutcome::Success + ); + assert_eq!(poller.state.etag().as_deref(), Some("current")); + assert_eq!( + poller.process_response(response(401, None, EMPTY_CONFIG)), + PollOutcome::Stop + ); + assert_eq!(poller.state.etag().as_deref(), Some("current")); + } + + #[test] + fn warning_categories_are_suppressed_independently() { + let warnings = WarningSlots::default(); + assert!(warnings.take(WarningCategory::Request)); + assert!(!warnings.take(WarningCategory::Request)); + assert!(warnings.take(WarningCategory::UnexpectedHttp)); + assert!(!warnings.take(WarningCategory::UnexpectedHttp)); + warnings.reset(); + assert!(warnings.take(WarningCategory::Request)); + } + + #[tokio::test] + async fn a_poll_retries_three_times_without_overlapping_requests() { + let transport = QueueTransport::new([ + Ok(response(503, None, b"ignored")), + Err(TransportFailure::TimedOut), + Ok(response(200, Some("accepted"), EMPTY_CONFIG)), + ]); + let state = Arc::new(PollState::default()); + let mut poller = Poller { + endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") + .expect("test endpoint"), + poll_interval: Duration::from_secs(30), + request_timeout: Duration::from_secs(5), + transport, + sink: TestSink { + fail: AtomicBool::new(false), + }, + state: Arc::clone(&state), + retry_delay: no_retry_delay, + }; + + assert_eq!( + poller.poll(&CancellationToken::new()).await, + PollOutcome::Success + ); + assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 3); + assert_eq!(poller.transport.max_in_flight.load(Ordering::Relaxed), 1); + assert_eq!(state.poll_count.load(Ordering::Relaxed), 3); + assert_eq!(state.etag().as_deref(), Some("accepted")); + } + + #[tokio::test] + async fn cancellation_before_a_poll_prevents_the_first_request() { + let transport = QueueTransport::new([]); + let mut poller = Poller { + endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") + .expect("test endpoint"), + poll_interval: Duration::from_secs(30), + request_timeout: Duration::from_secs(5), + transport, + sink: TestSink { + fail: AtomicBool::new(false), + }, + state: Arc::new(PollState::default()), + retry_delay: no_retry_delay, + }; + let cancellation = CancellationToken::new(); + cancellation.cancel(); + + assert_eq!(poller.poll(&cancellation).await, PollOutcome::Cancelled); + assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 0); + } +} diff --git a/components-rs/ffe/agentless/transport.rs b/components-rs/ffe/agentless/transport.rs new file mode 100644 index 00000000000..f6571fba037 --- /dev/null +++ b/components-rs/ffe/agentless/transport.rs @@ -0,0 +1,291 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use super::super::settings::AgentlessEndpoint; +use base64::Engine; +use flate2::read::GzDecoder; +use http::header::{ACCEPT_ENCODING, CONTENT_ENCODING, CONTENT_LENGTH, ETAG, IF_NONE_MATCH}; +use http::{Method, Request}; +use http_body_util::BodyExt; +use libdd_common::http_common::{self, Body}; +use percent_encoding::percent_decode_str; +use std::future::Future; +use std::io::Read; +use std::pin::Pin; +use std::time::Duration; +use tokio_util::sync::CancellationToken; + +pub(super) const MAX_RESPONSE_BODY_BYTES: usize = 10 << 20; +const CLIENT_LANGUAGE: &str = "php"; +const INTERNAL_UNTRACED_HEADER: &str = "DD-Internal-Untraced-Request"; +const API_KEY_HEADER: &str = "DD-API-KEY"; +const AUTHORIZATION_HEADER: &str = "Authorization"; + +pub(super) type FetchFuture<'a> = + Pin> + Send + 'a>>; + +pub(super) trait Transport: Send + Sync { + fn get<'a>( + &'a self, + endpoint: &'a AgentlessEndpoint, + timeout: Duration, + etag: Option<&'a str>, + cancellation: &'a CancellationToken, + ) -> FetchFuture<'a>; +} + +#[derive(Debug, Eq, PartialEq)] +pub(super) enum TransportFailure { + BuildRequest, + Cancelled, + TimedOut, + Request, + ResponseBody, + ResponseTooLarge, +} + +pub(super) struct PollResponse { + pub(super) status: u16, + pub(super) etag: Option, + pub(super) content_encoding: Option, + pub(super) body: Vec, +} + +pub(super) struct HyperTransport { + client: libdd_common::HttpClient, +} + +impl HyperTransport { + pub(super) fn new() -> Self { + Self { + // This native client is outside PHP HTTP instrumentation, so it + // cannot recursively trace its own polling requests. Hyper also + // leaves 3xx responses visible instead of following redirects. + client: http_common::new_client_periodic(), + } + } + + async fn get_inner( + &self, + endpoint: &AgentlessEndpoint, + timeout: Duration, + etag: Option<&str>, + cancellation: &CancellationToken, + ) -> Result { + let request = build_request(endpoint, etag)?; + let operation = async { + let response = self + .client + .request(request) + .await + .map_err(|_| TransportFailure::Request)?; + let status = response.status().as_u16(); + let etag = header_value(response.headers(), ETAG); + let content_encoding = header_value(response.headers(), CONTENT_ENCODING); + + // A non-200 body can never contain configuration. Dropping it here + // also prevents a malicious error response from consuming memory. + if status != 200 { + return Ok(PollResponse { + status, + etag, + content_encoding, + body: Vec::new(), + }); + } + + if response + .headers() + .get(CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_some_and(|length| length > MAX_RESPONSE_BODY_BYTES as u64) + { + return Err(TransportFailure::ResponseTooLarge); + } + + let body = collect_bounded(response.into_body(), MAX_RESPONSE_BODY_BYTES).await?; + Ok(PollResponse { + status, + etag, + content_encoding, + body, + }) + }; + + tokio::select! { + _ = cancellation.cancelled() => Err(TransportFailure::Cancelled), + result = tokio::time::timeout(timeout, operation) => { + result.map_err(|_| TransportFailure::TimedOut)? + } + } + } +} + +impl Transport for HyperTransport { + fn get<'a>( + &'a self, + endpoint: &'a AgentlessEndpoint, + timeout: Duration, + etag: Option<&'a str>, + cancellation: &'a CancellationToken, + ) -> FetchFuture<'a> { + Box::pin(self.get_inner(endpoint, timeout, etag, cancellation)) + } +} + +fn build_request( + endpoint: &AgentlessEndpoint, + etag: Option<&str>, +) -> Result, TransportFailure> { + let (request_url, authorization) = request_url_and_authorization(endpoint.as_str())?; + let mut builder = Request::builder() + .method(Method::GET) + .uri(request_url) + .header(ACCEPT_ENCODING, "gzip") + .header("DD-Client-Library-Language", CLIENT_LANGUAGE) + .header( + "DD-Client-Library-Version", + include_str!("../../../VERSION").trim(), + ) + .header(INTERNAL_UNTRACED_HEADER, "1"); + + if let Some(etag) = etag { + builder = builder.header(IF_NONE_MATCH, etag); + } + if let Some(api_key) = endpoint.api_key() { + builder = builder.header(API_KEY_HEADER, api_key); + } + if let Some(authorization) = authorization { + builder = builder.header(AUTHORIZATION_HEADER, authorization); + } + + builder + .body(Body::empty()) + .map_err(|_| TransportFailure::BuildRequest) +} + +fn request_url_and_authorization( + endpoint: &str, +) -> Result<(String, Option), TransportFailure> { + let mut url = url::Url::parse(endpoint).map_err(|_| TransportFailure::BuildRequest)?; + if url.username().is_empty() { + return Ok((endpoint.to_owned(), None)); + } + + let mut credentials = percent_decode_str(url.username()).collect::>(); + credentials.push(b':'); + if let Some(password) = url.password() { + credentials.extend(percent_decode_str(password)); + } + let authorization = format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode(credentials) + ); + + url.set_username("") + .map_err(|_| TransportFailure::BuildRequest)?; + url.set_password(None) + .map_err(|_| TransportFailure::BuildRequest)?; + Ok((url.to_string(), Some(authorization))) +} + +fn header_value(headers: &http::HeaderMap, name: http::header::HeaderName) -> Option { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) +} + +async fn collect_bounded( + mut body: hyper::body::Incoming, + limit: usize, +) -> Result, TransportFailure> { + let mut bytes = Vec::new(); + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|_| TransportFailure::ResponseBody)?; + if let Some(data) = frame.data_ref() { + let remaining = limit.saturating_sub(bytes.len()); + if data.len() > remaining { + return Err(TransportFailure::ResponseTooLarge); + } + bytes.extend_from_slice(data); + } + } + Ok(bytes) +} + +pub(super) fn decode_response_body( + body: &[u8], + content_encoding: Option<&str>, + limit: usize, +) -> Result, TransportFailure> { + if !content_encoding.is_some_and(|value| value.trim().eq_ignore_ascii_case("gzip")) { + if body.len() > limit { + return Err(TransportFailure::ResponseTooLarge); + } + return Ok(body.to_vec()); + } + + let decoder = GzDecoder::new(body); + let mut decoded = Vec::new(); + decoder + .take(limit as u64 + 1) + .read_to_end(&mut decoded) + .map_err(|_| TransportFailure::ResponseBody)?; + if decoded.len() > limit { + return Err(TransportFailure::ResponseTooLarge); + } + Ok(decoded) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn gzip(payload: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(payload).expect("compress test payload"); + encoder.finish().expect("finish test payload") + } + + #[test] + fn decoded_body_limit_applies_after_gzip() { + let payload = vec![b'a'; 256]; + let compressed = gzip(&payload); + assert!(compressed.len() < 64); + assert_eq!( + decode_response_body(&compressed, Some(" GZIP "), 64), + Err(TransportFailure::ResponseTooLarge) + ); + assert_eq!( + decode_response_body(&gzip(b"configuration"), Some("gzip"), 64).expect("decode gzip"), + b"configuration" + ); + } + + #[test] + fn request_authentication_is_managed_only() { + let managed = + AgentlessEndpoint::build("", "datadoghq.com", "", "secret").expect("managed endpoint"); + let custom = + AgentlessEndpoint::build("https://user:password@localhost/custom", "", "", "secret") + .expect("custom endpoint"); + + let managed_request = build_request(&managed, Some("etag")).expect("managed request"); + assert_eq!(managed_request.headers()[API_KEY_HEADER], "secret"); + assert_eq!(managed_request.headers()[IF_NONE_MATCH], "etag"); + assert_eq!(managed_request.headers()[INTERNAL_UNTRACED_HEADER], "1"); + + let custom_request = build_request(&custom, None).expect("custom request"); + assert!(!custom_request.headers().contains_key(API_KEY_HEADER)); + assert_eq!( + custom_request.headers()[AUTHORIZATION_HEADER], + "Basic dXNlcjpwYXNzd29yZA==" + ); + assert_eq!(custom_request.uri(), "https://localhost/custom"); + } +} From 2dc2b63f67486a3b6f7068840e04a45b8869efe8 Mon Sep 17 00:00:00 2001 From: Pavel Date: Fri, 18 Sep 2026 14:02:34 +0300 Subject: [PATCH 04/10] refactor(ffe): remove PR-specific implementation comments --- components-rs/ffe.rs | 8 +++----- components-rs/ffe/agentless.rs | 1 - components-rs/ffe/agentless/poller.rs | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/components-rs/ffe.rs b/components-rs/ffe.rs index a5a8493c3b1..aa52de8c611 100644 --- a/components-rs/ffe.rs +++ b/components-rs/ffe.rs @@ -9,16 +9,14 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; use std::sync::{Arc, LazyLock}; -// This module is wired into activation by the next change in the FFE stack. #[allow(dead_code)] pub(crate) mod agentless; -// This module is wired into activation by the next change in the FFE stack. #[allow(dead_code)] pub(crate) mod settings; #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[repr(u8)] -#[allow(dead_code)] // Remaining states are connected by PR 3 activation wiring. +#[allow(dead_code)] pub(crate) enum DeliveryState { Inactive, Starting, @@ -42,7 +40,7 @@ static FFE_CONFIG: LazyLock> = static FFE_VERSION: AtomicU64 = AtomicU64::new(0); static DELIVERY_STATE: AtomicU8 = AtomicU8::new(DeliveryState::Inactive as u8); -#[allow(dead_code)] // Connected by PR 3 activation wiring. +#[allow(dead_code)] pub(crate) fn delivery_state() -> DeliveryState { match DELIVERY_STATE.load(Ordering::Acquire) { value if value == DeliveryState::Starting as u8 => DeliveryState::Starting, @@ -54,7 +52,7 @@ pub(crate) fn delivery_state() -> DeliveryState { } } -#[allow(dead_code)] // Connected by PR 3 activation wiring. +#[allow(dead_code)] pub(crate) fn set_delivery_state(state: DeliveryState) { DELIVERY_STATE.store(state as u8, Ordering::Release); } diff --git a/components-rs/ffe/agentless.rs b/components-rs/ffe/agentless.rs index c7581285052..888cdadf96a 100644 --- a/components-rs/ffe/agentless.rs +++ b/components-rs/ffe/agentless.rs @@ -45,7 +45,6 @@ pub(crate) enum WorkerStartError { } /// Owns the poll thread and makes its lifecycle explicit around `fork`. -/// PR 3 connects these methods to the tracer lifecycle hooks. pub(crate) struct AgentlessWorker { config: AgentlessWorkerConfig, state: Arc, diff --git a/components-rs/ffe/agentless/poller.rs b/components-rs/ffe/agentless/poller.rs index 2865dc4ba21..f9005494776 100644 --- a/components-rs/ffe/agentless/poller.rs +++ b/components-rs/ffe/agentless/poller.rs @@ -423,7 +423,7 @@ mod tests { } #[test] - fn retry_delay_uses_the_cross_sdk_clamps_and_jitter() { + fn retry_delay_applies_clamps_and_jitter() { assert_eq!( retry_delay(Duration::from_secs(30), 1, 0.0), Duration::from_secs(4) From 8ec8c3213a618080277cb000fddc6451d12e6da8 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 13 Oct 2026 23:37:51 +0000 Subject: [PATCH 05/10] Use registered CDN configuration implementation labels --- metadata/supported-configurations.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 5f163c31347..5214138f216 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -399,7 +399,7 @@ ], "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS": [ { - "implementation": "A", + "implementation": "B", "type": "int", "default": "10000" } @@ -697,7 +697,7 @@ ], "DD_SITE": [ { - "implementation": "A", + "implementation": "C", "type": "string", "default": "datadoghq.com" } From a3bd62a1beac453c79542a22c9faeea9252cf618 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 14 Oct 2026 14:47:48 +0000 Subject: [PATCH 06/10] Fetch CDN configuration through the libdatadog sidecar Read shared-memory snapshots in PHP and resubscribe after sidecar restart. Preserve bounded initialization, last-good state and fork lifecycle. Environment: Datadog workspace --- Cargo.lock | 55 +- Cargo.toml | 10 +- components-rs/common.h | 35 + components-rs/ffe/agentless.rs | 136 ++-- components-rs/ffe/agentless/poller.rs | 638 ------------------ components-rs/ffe/agentless/transport.rs | 291 -------- components-rs/ffe/lifecycle.rs | 15 +- components-rs/ffe/settings.rs | 270 +------- components-rs/library-config.h | 71 +- components-rs/sidecar.h | 40 +- components-rs/sidecar.rs | 6 +- libdatadog | 2 +- tests/ext/ffe/agentless_fork.phpt | 2 +- .../ext/ffe/agentless_process_isolation.phpt | 28 + tests/ext/ffe/agentless_sidecar_restart.phpt | 32 + tests/ext/ffe/agentless_source_selection.phpt | 3 + tests/ext/ffe/stubs/agentless_process.inc | 72 +- tests/ext/ffe/stubs/agentless_process_app.php | 31 +- tracer/ffe.c | 6 + 19 files changed, 434 insertions(+), 1309 deletions(-) delete mode 100644 components-rs/ffe/agentless/poller.rs delete mode 100644 components-rs/ffe/agentless/transport.rs create mode 100644 tests/ext/ffe/agentless_process_isolation.phpt create mode 100644 tests/ext/ffe/agentless_sidecar_restart.phpt diff --git a/Cargo.lock b/Cargo.lock index 60f58886897..2710a790af2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -803,6 +803,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "colored" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" +dependencies = [ + "lazy_static", + "windows-sys 0.59.0", +] + [[package]] name = "combine" version = "4.6.7" @@ -1201,7 +1211,6 @@ version = "0.0.1" dependencies = [ "anyhow", "arc-swap", - "base64 0.22.1", "bincode", "bindgen", "cbindgen 0.27.0", @@ -1216,13 +1225,8 @@ dependencies = [ "ddtrace-sidecar", "dynasmrt", "env_logger", - "fastrand", - "flate2", - "futures-util", "hashbrown 0.15.2", "http 1.4.2", - "http-body-util", - "hyper", "itertools 0.11.0", "lazy_static", "libc", @@ -1249,7 +1253,6 @@ dependencies = [ "log", "mach2", "paste", - "percent-encoding", "rand 0.9.4", "rand_distr", "regex", @@ -1268,7 +1271,6 @@ dependencies = [ "tracing", "tracing-core", "tracing-subscriber", - "url", "uuid", "zwohash", ] @@ -1334,6 +1336,8 @@ dependencies = [ name = "datadog-sidecar-ffi" version = "0.0.1" dependencies = [ + "build_common", + "cc", "datadog-sidecar", "http 1.4.2", "libc", @@ -1351,6 +1355,8 @@ dependencies = [ "libdd-trace-utils", "rmp-serde", "serde_json", + "tempfile", + "tools", "tracing", ] @@ -2976,12 +2982,18 @@ dependencies = [ name = "libdd-ffe" version = "2.0.0" dependencies = [ + "base64 0.22.1", "bincode", "chrono", "derive_more", + "fastrand", "faststr", + "flate2", + "futures-util", "http 1.4.2", + "http-body-util", "httpmock", + "hyper", "libdd-capabilities", "libdd-capabilities-impl", "libdd-common", @@ -2990,6 +3002,7 @@ dependencies = [ "log", "lru", "md5", + "percent-encoding", "prost", "pyo3", "semver", @@ -3000,6 +3013,8 @@ dependencies = [ "strum_macros", "thiserror 2.0.18", "tokio", + "tokio-util", + "url", ] [[package]] @@ -3399,6 +3414,7 @@ dependencies = [ "flate2", "futures", "getrandom 0.2.15", + "hashbrown 0.17.1", "hex", "http 1.4.2", "http-body", @@ -4546,6 +4562,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.46" @@ -6083,6 +6108,20 @@ dependencies = [ "tonic", ] +[[package]] +name = "tools" +version = "0.0.1" +dependencies = [ + "anyhow", + "cargo_metadata", + "clap", + "colored", + "libdd-common", + "quick-xml", + "toml", + "wait-timeout", +] + [[package]] name = "tower" version = "0.5.2" diff --git a/Cargo.toml b/Cargo.toml index 38f435a15f8..8f6d44af093 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ libdd-telemetry-ffi = { path = "libdatadog/libdd-telemetry-ffi", default-feature libdd-live-debugger = { path = "libdatadog/libdd-live-debugger" } libdd-live-debugger-ffi = { path = "libdatadog/libdd-live-debugger-ffi", default-features = false } libdd-ipc = { path = "libdatadog/libdd-ipc" } -datadog-ffe = { package = "libdd-ffe", path = "libdatadog/libdd-ffe" } +datadog-ffe = { package = "libdd-ffe", path = "libdatadog/libdd-ffe", features = ["cdn"] } libdd-remote-config = { path = "libdatadog/libdd-remote-config" } datadog-sidecar = { path = "libdatadog/datadog-sidecar" } datadog-sidecar-ffi = { path = "libdatadog/datadog-sidecar-ffi" } @@ -72,13 +72,7 @@ libdd-library-config-ffi = { path = "libdatadog/libdd-library-config-ffi", defau spawn_worker = { path = "libdatadog/spawn_worker" } anyhow = "1.0" arc-swap.workspace = true -base64 = "0.22" const-str = "0.5.6" -fastrand = "2" -flate2 = { version = "1.0", default-features = false, features = ["rust_backend"] } -futures-util = "0.3" -http-body-util = "0.1" -hyper = { version = "1.6", features = ["client", "http1"] } itertools = "0.11.0" serde = "1.0.196" simd-json = "0.14.1" @@ -89,7 +83,6 @@ env_logger = "0.11" zwohash = "0.1.2" uuid = { version = "1.0", features = ["v4"] } paste = "1" -percent-encoding = "2.1" tempfile = "3.3" tracing = { version = "0.1", default-features = false, features = ["std", "attributes"] } tracing-core = "0.1.32" @@ -100,7 +93,6 @@ serde_json = { version = "1.0.113", features = ["raw_value"] } regex = "1.10.5" regex-automata = "0.4.5" http = "1.4" -url = "2.5.4" libc = "0.2" bincode = "1.3.3" hashbrown = "0.15" diff --git a/components-rs/common.h b/components-rs/common.h index df191e08d2d..595c54d671f 100644 --- a/components-rs/common.h +++ b/components-rs/common.h @@ -486,6 +486,7 @@ typedef enum ddog_RemoteConfigProduct { DDOG_REMOTE_CONFIG_PRODUCT_LIVE_DEBUGGING, DDOG_REMOTE_CONFIG_PRODUCT_LIVE_DEBUGGING_SYMBOL_DB, DDOG_REMOTE_CONFIG_PRODUCT_DEBUG, + DDOG_REMOTE_CONFIG_PRODUCT_APM_PROFILING, } ddog_RemoteConfigProduct; typedef enum ddog_SpanProbeTarget { @@ -593,6 +594,8 @@ typedef struct ddog_FfeSettingsInput { ddog_CharSlice site; ddog_CharSlice api_key; ddog_CharSlice environment; + bool sidecar_subprocess_enabled; + bool sidecar_self_telemetry; void (*thread_cleanup)(void*); } ddog_FfeSettingsInput; @@ -1253,6 +1256,15 @@ typedef enum ddog_DynamicInstrumentationConfigState { DDOG_DYNAMIC_INSTRUMENTATION_CONFIG_STATE_NOT_SET, } ddog_DynamicInstrumentationConfigState; +/** + * Controls whether an EVP client stays fixed to the Agent or opts into local + * discovery and direct-intake fallback. + */ +typedef enum ddog_EvpTransportMode { + DDOG_EVP_TRANSPORT_MODE_AGENT_ONLY, + DDOG_EVP_TRANSPORT_MODE_PREFER_LOCAL_THEN_DIRECT, +} ddog_EvpTransportMode; + typedef struct ddog_AgentRemoteConfigReader ddog_AgentRemoteConfigReader; typedef struct ddog_AgentRemoteConfigWriter_ShmHandle ddog_AgentRemoteConfigWriter_ShmHandle; @@ -1314,12 +1326,29 @@ typedef struct ddog_TracerHeaderTags { bool client_computed_stats; } ddog_TracerHeaderTags; +/** + * Logical tracer identity attached to EVP requests. + * + * Both fields must be non-empty, valid HTTP header values no longer than 256 + * bytes. This identifies the producing SDK, not the sidecar transport. + */ +typedef struct ddog_EvpProducerIdentity { + ddog_CharSlice origin; + ddog_CharSlice version; +} ddog_EvpProducerIdentity; + +/** + * C representation of the context attached to Feature Flags telemetry. + */ typedef struct ddog_FfeTelemetryContext { ddog_CharSlice service; ddog_CharSlice env; ddog_CharSlice version; } ddog_FfeTelemetryContext; +/** + * C representation of a Feature Flags exposure event. + */ typedef struct ddog_FfeExposure { uint64_t timestamp_ms; ddog_CharSlice flag_key; @@ -1348,6 +1377,9 @@ typedef struct ddog_Slice_FfeExposure { */ uintptr_t len; } ddog_Slice_FfeExposure; +/** + * C representation of a Feature Flags evaluation event. + */ typedef struct ddog_FfeFlagEvaluation { int64_t timestamp_ms; ddog_CharSlice flag_key; @@ -1386,6 +1418,9 @@ typedef struct ddog_Slice_FfeFlagEvaluation { */ uintptr_t len; } ddog_Slice_FfeFlagEvaluation; +/** + * C representation of a Feature Flags evaluation metric. + */ typedef struct ddog_FfeEvaluationMetric { ddog_CharSlice flag_key; ddog_CharSlice variant; diff --git a/components-rs/ffe/agentless.rs b/components-rs/ffe/agentless.rs index 60c177c79ef..004e5cc1128 100644 --- a/components-rs/ffe/agentless.rs +++ b/components-rs/ffe/agentless.rs @@ -1,24 +1,22 @@ // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -mod poller; -mod transport; - -use self::poller::{GlobalConfigurationSink, PollState, Poller}; -use self::transport::HyperTransport; use super::settings::AgentlessEndpoint; +use super::{ddog_ffe_has_config, set_delivery_state, store_config, DeliveryState}; use crate::log::{self, Log}; -use std::ffi::c_void; -use std::sync::Arc; +use datadog_ffe::cdn::CdnConfig; +use datadog_ffe::rules_based::{Configuration, UniversalFlagConfig}; +use datadog_sidecar::service::blocking::{ping, subscribe_cdn_configuration}; +use datadog_sidecar::shm_remote_config::cdn::{CdnDeliveryState, CdnReader}; +use std::ffi::{c_void, CString}; use std::thread::JoinHandle; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; #[derive(Clone)] pub(crate) struct AgentlessWorkerConfig { - endpoint: AgentlessEndpoint, - poll_interval: Duration, - request_timeout: Duration, + subscription: CdnConfig, + pub(crate) sidecar_self_telemetry: bool, pub(crate) thread_cleanup: Option, } @@ -29,10 +27,15 @@ impl AgentlessWorkerConfig { request_timeout: Duration, ) -> Self { Self { - endpoint, - poll_interval, - request_timeout, + subscription: CdnConfig { + endpoint, + poll_interval, + request_timeout, + language: "php".to_owned(), + version: include_str!("../../VERSION").trim().to_owned(), + }, thread_cleanup: None, + sidecar_self_telemetry: false, } } } @@ -60,10 +63,9 @@ pub(crate) enum WorkerStartError { ThreadUnavailable, } -/// Owns the poll thread and makes its lifecycle explicit around `fork`. +/// Owns a local IPC/shared-memory reader. CDN HTTP runs in the sidecar subprocess. pub(crate) struct AgentlessWorker { config: AgentlessWorkerConfig, - state: Arc, running: Option, restart_after_fork: bool, permanently_stopped: bool, @@ -73,7 +75,6 @@ impl AgentlessWorker { pub(crate) fn new(config: AgentlessWorkerConfig) -> Self { Self { config, - state: Arc::new(PollState::default()), running: None, restart_after_fork: false, permanently_stopped: false, @@ -81,8 +82,7 @@ impl AgentlessWorker { } pub(crate) fn start(&mut self) -> Result { - // A runtime creation failure must not leave a finished thread looking - // like an active poller forever. The next ordinary evaluation may retry. + // A reader that exits unexpectedly must not suppress later retries. if self .running .as_ref() @@ -97,38 +97,29 @@ impl AgentlessWorker { let cancellation = CancellationToken::new(); let thread_cancellation = cancellation.clone(); let config = self.config.clone(); - let state = Arc::clone(&self.state); let log_dispatch = tracing::dispatcher::get_default(Clone::clone); let thread = std::thread::Builder::new() .name("ddtrace-ffe-agentless".to_owned()) .spawn(move || { let _thread_cleanup = ThreadCleanup(config.thread_cleanup); let _log_guard = tracing::dispatcher::set_default(&log_dispatch); - let runtime = match tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - { - Ok(runtime) => runtime, - Err(_) => { - log::log( - Log::Warn, - "Feature Flags agentless runtime could not be created", - ); - return; + let mut warned = false; + while !thread_cancellation.is_cancelled() { + if read_configuration(&config, &thread_cancellation).is_err() { + set_delivery_state(if ddog_ffe_has_config() { + DeliveryState::Stale + } else { + DeliveryState::Starting + }); + if !warned { + log::log(Log::Warn, "Feature Flags CDN sidecar is unavailable; retaining the last configuration"); + warned = true; + } + } + if !thread_cancellation.is_cancelled() { + std::thread::park_timeout(Duration::from_secs(1)); } - }; - - runtime.block_on(async move { - let mut poller = Poller::new( - config.endpoint, - config.poll_interval, - config.request_timeout, - HyperTransport::new(), - GlobalConfigurationSink, - state, - ); - poller.run(thread_cancellation).await; - }); + } }) .map_err(|_| WorkerStartError::ThreadUnavailable)?; @@ -156,14 +147,9 @@ impl AgentlessWorker { pub(crate) fn reset_after_fork_child(&mut self) -> Result<(), WorkerStartError> { // `prepare_for_fork` joined the only worker before the process split, // so no synchronization primitive can be inherited while held. - self.state.reset_warnings(); self.restart_after_fork() } - pub(crate) fn poll_count(&self) -> u64 { - self.state.poll_count() - } - fn restart_after_fork(&mut self) -> Result<(), WorkerStartError> { if self.restart_after_fork && !self.permanently_stopped { self.start()?; @@ -175,11 +161,58 @@ impl AgentlessWorker { fn stop_running(&mut self) { if let Some(running) = self.running.take() { running.cancellation.cancel(); + running.thread.thread().unpark(); let _ = running.thread.join(); } } } +/// Establish an independently owned connection so request/thread teardown does +/// not invalidate the process-wide CDN subscription. Always use subprocess +/// startup: a connection failure must never move HTTP into the PHP process. +fn read_configuration( + config: &AgentlessWorkerConfig, + cancellation: &CancellationToken, +) -> anyhow::Result<()> { + if cancellation.is_cancelled() { + return Ok(()); + } + let mut sidecar = datadog_sidecar::config::FromEnv::config(); + sidecar.self_telemetry = config.sidecar_self_telemetry; + let mut transport = crate::sidecar::run_sidecar(sidecar)?; + transport.set_read_timeout(Some(Duration::from_millis(200)))?; + transport.set_write_timeout(Some(Duration::from_millis(200)))?; + let path = subscribe_cdn_configuration(&mut transport, Some(config.subscription.clone()))? + .ok_or_else(|| anyhow::anyhow!("CDN subscription unavailable"))?; + let mut reader = CdnReader::new(&CString::new(path)?); + let mut revision = 0; + let mut heartbeat = Instant::now(); + while !cancellation.is_cancelled() { + if let Some(snapshot) = reader.read()? { + if snapshot.revision != revision && !snapshot.configuration.is_empty() { + let config = UniversalFlagConfig::from_json(snapshot.configuration)?; + store_config(Configuration::from_server_response(config)); + revision = snapshot.revision; + } + set_delivery_state(match snapshot.state { + CdnDeliveryState::Ready if ddog_ffe_has_config() => DeliveryState::Ready, + CdnDeliveryState::Stale | CdnDeliveryState::Stopped if ddog_ffe_has_config() => { + DeliveryState::Stale + } + _ => DeliveryState::Starting, + }); + } + // Detect a dead sidecar even when the last shared-memory snapshot is + // unchanged. A fresh connection obtains a fresh subscription and name. + if heartbeat.elapsed() >= Duration::from_secs(1) { + ping(&mut transport)?; + heartbeat = Instant::now(); + } + reader.wait_for_change(Duration::from_millis(50)); + } + Ok(()) +} + impl Drop for AgentlessWorker { fn drop(&mut self) { self.stop_running(); @@ -192,7 +225,7 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; #[test] - fn worker_cleanup_runs_on_its_thread_after_runtime_shutdown() { + fn worker_cleanup_runs_on_its_thread_after_reader_shutdown() { static CLEANUPS: AtomicUsize = AtomicUsize::new(0); extern "C" fn cleanup(_: *mut c_void) { // Runtime and tracing guards must have exited before TLS is drained. @@ -223,8 +256,7 @@ mod tests { Duration::from_secs(30), Duration::from_secs(30), )); - // Model the worker returning before polling (for example when Tokio - // cannot allocate its runtime). Its handle must not suppress retries. + // Model a reader returning unexpectedly. Its handle must not suppress retries. let thread = std::thread::spawn(|| {}); while !thread.is_finished() { std::thread::yield_now(); diff --git a/components-rs/ffe/agentless/poller.rs b/components-rs/ffe/agentless/poller.rs deleted file mode 100644 index 3dec7dd4de8..00000000000 --- a/components-rs/ffe/agentless/poller.rs +++ /dev/null @@ -1,638 +0,0 @@ -// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ -// SPDX-License-Identifier: Apache-2.0 - -use super::super::settings::AgentlessEndpoint; -use super::transport::{ - decode_response_body, PollResponse, Transport, TransportFailure, MAX_RESPONSE_BODY_BYTES, -}; -use crate::ffe::{store_config, ConfigurationTransition}; -use crate::log::{self, Log}; -use datadog_ffe::rules_based::{Configuration, UniversalFlagConfig}; -use futures_util::FutureExt; -use std::panic::AssertUnwindSafe; -use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; -use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::Duration; -use tokio_util::sync::CancellationToken; - -const MAX_POLL_ATTEMPTS: usize = 3; - -#[derive(serde::Deserialize)] -struct ConfigurationResponse<'a> { - #[serde(borrow)] - data: ConfigurationResource<'a>, -} - -#[derive(serde::Deserialize)] -struct ConfigurationResource<'a> { - #[serde(rename = "type")] - _resource_type: ConfigurationResourceType, - #[serde(borrow)] - attributes: &'a serde_json::value::RawValue, -} - -#[derive(serde::Deserialize)] -enum ConfigurationResourceType { - #[serde(rename = "universal-flag-configuration")] - UniversalFlagConfiguration, -} - -pub(super) trait ConfigurationSink: Send + Sync { - fn apply(&self, configuration: Configuration) -> Result; - fn failed(&self) {} - fn confirmed(&self) {} -} - -pub(super) struct GlobalConfigurationSink; - -impl ConfigurationSink for GlobalConfigurationSink { - fn apply(&self, configuration: Configuration) -> Result { - Ok(store_config(configuration)) - } - - fn failed(&self) { - use crate::ffe::{ddog_ffe_has_config, set_delivery_state, DeliveryState}; - set_delivery_state(if ddog_ffe_has_config() { - DeliveryState::Stale - } else { - DeliveryState::Starting - }); - } - - fn confirmed(&self) { - use crate::ffe::{ddog_ffe_has_config, set_delivery_state, DeliveryState}; - if ddog_ffe_has_config() { - set_delivery_state(DeliveryState::Ready); - } - } -} - -#[derive(Clone, Copy)] -#[repr(u8)] -enum WarningCategory { - Authentication = 1 << 0, - RetryableHttp = 1 << 1, - UnexpectedHttp = 1 << 2, - Request = 1 << 3, - MalformedPayload = 1 << 4, - ApplyFailure = 1 << 5, - UnexpectedException = 1 << 6, -} - -#[derive(Default)] -struct WarningSlots(AtomicU8); - -impl WarningSlots { - fn take(&self, category: WarningCategory) -> bool { - self.0.fetch_or(category as u8, Ordering::Relaxed) & category as u8 == 0 - } - - fn reset(&self) { - self.0.store(0, Ordering::Relaxed); - } -} - -#[derive(Default)] -pub(super) struct PollState { - etag: Mutex>, - warnings: WarningSlots, - poll_count: AtomicU64, -} - -impl PollState { - fn etag(&self) -> Option { - lock_unpoisoned(&self.etag).clone() - } - - fn set_etag(&self, etag: Option) { - *lock_unpoisoned(&self.etag) = etag; - } - - pub(super) fn reset_warnings(&self) { - self.warnings.reset(); - } - - pub(super) fn poll_count(&self) -> u64 { - self.poll_count.load(Ordering::Relaxed) - } -} - -fn lock_unpoisoned(mutex: &Mutex) -> MutexGuard<'_, T> { - mutex - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum PollOutcome { - Success, - Retryable, - Stop, - Cancelled, -} - -pub(super) struct Poller { - endpoint: AgentlessEndpoint, - poll_interval: Duration, - request_timeout: Duration, - transport: T, - sink: S, - state: Arc, - retry_delay: fn(Duration, usize, f64) -> Duration, -} - -impl Poller { - pub(super) fn new( - endpoint: AgentlessEndpoint, - poll_interval: Duration, - request_timeout: Duration, - transport: T, - sink: S, - state: Arc, - ) -> Self { - Self { - endpoint, - poll_interval, - request_timeout, - transport, - sink, - state, - retry_delay, - } - } - - pub(super) async fn run(&mut self, cancellation: CancellationToken) { - loop { - if cancellation.is_cancelled() { - return; - } - - let poll = AssertUnwindSafe(self.poll(&cancellation)).catch_unwind(); - match poll.await { - Ok(PollOutcome::Cancelled) => return, - Ok(PollOutcome::Success) => self.sink.confirmed(), - Ok(_) => self.sink.failed(), - Err(_) => { - self.sink.failed(); - self.warn( - WarningCategory::UnexpectedException, - "Feature Flags agentless polling failed unexpectedly; polling continues", - ); - } - } - - tokio::select! { - _ = cancellation.cancelled() => return, - _ = tokio::time::sleep(self.poll_interval) => {} - } - } - } - - async fn poll(&mut self, cancellation: &CancellationToken) -> PollOutcome { - for attempt in 1..=MAX_POLL_ATTEMPTS { - let outcome = self.poll_once(cancellation).await; - if outcome != PollOutcome::Retryable || attempt == MAX_POLL_ATTEMPTS { - return outcome; - } - - let delay = (self.retry_delay)(self.poll_interval, attempt, fastrand::f64()); - tokio::select! { - _ = cancellation.cancelled() => return PollOutcome::Cancelled, - _ = tokio::time::sleep(delay) => {} - } - } - PollOutcome::Stop - } - - async fn poll_once(&mut self, cancellation: &CancellationToken) -> PollOutcome { - if cancellation.is_cancelled() { - return PollOutcome::Cancelled; - } - - self.state.poll_count.fetch_add(1, Ordering::Relaxed); - let etag = self.state.etag(); - match self - .transport - .get( - &self.endpoint, - self.request_timeout, - etag.as_deref(), - cancellation, - ) - .await - { - Ok(response) => self.process_response(response), - Err(TransportFailure::Cancelled) => PollOutcome::Cancelled, - Err(TransportFailure::BuildRequest) => { - self.warn( - WarningCategory::Request, - "Feature Flags agentless request could not be built", - ); - PollOutcome::Stop - } - Err(TransportFailure::ResponseTooLarge) => { - self.warn( - WarningCategory::MalformedPayload, - "Feature Flags agentless response exceeds the 10 MiB limit; keeping the last known configuration", - ); - PollOutcome::Stop - } - Err(_) => { - self.warn( - WarningCategory::Request, - "Feature Flags agentless request failed; keeping the last known configuration", - ); - PollOutcome::Retryable - } - } - } - - fn process_response(&mut self, response: PollResponse) -> PollOutcome { - match response.status { - 304 => return PollOutcome::Success, - 401 | 403 => { - self.warn( - WarningCategory::Authentication, - "Feature Flags agentless endpoint rejected authentication; verify endpoint authentication", - ); - return PollOutcome::Stop; - } - 200 => {} - status if is_retryable_status(status) => { - self.warn( - WarningCategory::RetryableHttp, - &format!("Feature Flags agentless endpoint returned retryable HTTP {status}"), - ); - return PollOutcome::Retryable; - } - status => { - self.warn( - WarningCategory::UnexpectedHttp, - &format!("Feature Flags agentless endpoint returned unexpected HTTP {status}"), - ); - return PollOutcome::Stop; - } - } - - let body = match decode_response_body( - &response.body, - response.content_encoding.as_deref(), - MAX_RESPONSE_BODY_BYTES, - ) { - Ok(body) => body, - Err(TransportFailure::ResponseTooLarge) => { - self.warn( - WarningCategory::MalformedPayload, - "Feature Flags agentless decoded response exceeds the 10 MiB limit; keeping the last known configuration", - ); - return PollOutcome::Stop; - } - Err(_) => { - self.warn( - WarningCategory::MalformedPayload, - "Feature Flags agentless endpoint returned an invalid response body", - ); - return PollOutcome::Stop; - } - }; - - // The managed endpoint returns a JSON:API resource, not a bare UFC. - // Validate its type before handing the original attributes bytes to - // the shared evaluator; malformed resources must not advance the ETag. - let configuration = match serde_json::from_slice::>(&body) - .and_then(|response| { - UniversalFlagConfig::from_json(response.data.attributes.get().as_bytes().to_vec()) - }) { - Ok(configuration) => Configuration::from_server_response(configuration), - Err(_) => { - self.warn( - WarningCategory::MalformedPayload, - "Feature Flags agentless endpoint returned malformed configuration", - ); - return PollOutcome::Stop; - } - }; - - if self.sink.apply(configuration).is_err() { - self.warn( - WarningCategory::ApplyFailure, - "Feature Flags agentless configuration could not be applied", - ); - return PollOutcome::Stop; - } - - // The response has now been parsed and applied. A blank or absent - // ETag intentionally clears an older value. - self.state.set_etag(response.etag); - PollOutcome::Success - } - - fn warn(&self, category: WarningCategory, message: &str) { - if self.state.warnings.take(category) { - log::log(Log::Warn, message); - } - } -} - -fn is_retryable_status(status: u16) -> bool { - matches!(status, 408 | 429 | 500..=599) -} - -fn retry_delay(poll_interval: Duration, attempt: usize, random: f64) -> Duration { - let base = if attempt <= 1 { - (poll_interval / 6).clamp(Duration::from_secs(2), Duration::from_secs(10)) - } else { - (poll_interval / 3).clamp(Duration::from_secs(5), Duration::from_secs(30)) - }; - base.mul_f64(0.8 + random.clamp(0.0, 1.0) * 0.4) - .max(Duration::from_secs(1)) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ffe::agentless::transport::FetchFuture; - use std::collections::VecDeque; - use std::sync::atomic::{AtomicBool, AtomicUsize}; - - const EMPTY_CONFIG: &[u8] = br#"{"data": { - "type": "universal-flag-configuration", - "attributes": { - "createdAt": "2026-05-22T00:00:00.000Z", - "format": "SERVER", - "environment": {"name": "test"}, - "flags": {} - }}}"#; - - struct UnusedTransport; - - impl Transport for UnusedTransport { - fn get<'a>( - &'a self, - _endpoint: &'a AgentlessEndpoint, - _timeout: Duration, - _etag: Option<&'a str>, - _cancellation: &'a CancellationToken, - ) -> FetchFuture<'a> { - Box::pin(async { unreachable!("transport is not used by response-processing tests") }) - } - } - - struct TestSink { - fail: AtomicBool, - applications: AtomicUsize, - } - - struct QueueTransport { - responses: Mutex>>, - calls: AtomicUsize, - in_flight: AtomicUsize, - max_in_flight: AtomicUsize, - } - - impl QueueTransport { - fn new( - responses: impl IntoIterator>, - ) -> Self { - Self { - responses: Mutex::new(responses.into_iter().collect()), - calls: AtomicUsize::new(0), - in_flight: AtomicUsize::new(0), - max_in_flight: AtomicUsize::new(0), - } - } - } - - impl Transport for QueueTransport { - fn get<'a>( - &'a self, - _endpoint: &'a AgentlessEndpoint, - _timeout: Duration, - _etag: Option<&'a str>, - _cancellation: &'a CancellationToken, - ) -> FetchFuture<'a> { - Box::pin(async move { - self.calls.fetch_add(1, Ordering::Relaxed); - let in_flight = self.in_flight.fetch_add(1, Ordering::Relaxed) + 1; - self.max_in_flight.fetch_max(in_flight, Ordering::Relaxed); - tokio::task::yield_now().await; - let result = lock_unpoisoned(&self.responses) - .pop_front() - .unwrap_or(Err(TransportFailure::Request)); - self.in_flight.fetch_sub(1, Ordering::Relaxed); - result - }) - } - } - - impl ConfigurationSink for TestSink { - fn apply(&self, _configuration: Configuration) -> Result { - self.applications.fetch_add(1, Ordering::Relaxed); - if self.fail.load(Ordering::Relaxed) { - Err(()) - } else { - Ok(ConfigurationTransition::Ready) - } - } - } - - fn test_poller(fail_apply: bool) -> Poller { - Poller { - endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") - .expect("test endpoint"), - poll_interval: Duration::from_secs(30), - request_timeout: Duration::from_secs(5), - transport: UnusedTransport, - sink: TestSink { - fail: AtomicBool::new(fail_apply), - applications: AtomicUsize::new(0), - }, - state: Arc::new(PollState::default()), - retry_delay, - } - } - - fn no_retry_delay(_: Duration, _: usize, _: f64) -> Duration { - Duration::ZERO - } - - fn response(status: u16, etag: Option<&str>, body: &[u8]) -> PollResponse { - PollResponse { - status, - etag: etag.map(str::to_owned), - content_encoding: None, - body: body.to_vec(), - } - } - - #[test] - fn retries_only_transport_timeouts_and_selected_statuses() { - for status in [408, 429, 500, 503, 599] { - assert!(is_retryable_status(status), "status {status}"); - } - for status in [200, 204, 301, 400, 401, 403, 404, 499, 600] { - assert!(!is_retryable_status(status), "status {status}"); - } - } - - #[test] - fn retry_delay_applies_clamps_and_jitter() { - assert_eq!( - retry_delay(Duration::from_secs(30), 1, 0.0), - Duration::from_secs(4) - ); - assert_eq!( - retry_delay(Duration::from_secs(30), 1, 1.0), - Duration::from_secs(6) - ); - assert_eq!( - retry_delay(Duration::from_secs(30), 2, 0.0), - Duration::from_secs(8) - ); - assert_eq!( - retry_delay(Duration::from_secs(30), 2, 1.0), - Duration::from_secs(12) - ); - assert_eq!( - retry_delay(Duration::from_secs(1), 1, 0.0), - Duration::from_millis(1_600) - ); - } - - #[test] - fn etag_advances_only_after_parse_and_apply_succeed() { - let mut poller = test_poller(true); - poller.state.set_etag(Some("old".to_owned())); - - assert_eq!( - poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), - PollOutcome::Stop - ); - assert_eq!(poller.state.etag().as_deref(), Some("old")); - - poller.sink.fail.store(false, Ordering::Relaxed); - assert_eq!( - poller.process_response(response(200, Some("new"), EMPTY_CONFIG)), - PollOutcome::Success - ); - assert_eq!(poller.state.etag().as_deref(), Some("new")); - - assert_eq!( - poller.process_response(response(200, Some("ignored"), b"not-json")), - PollOutcome::Stop - ); - assert_eq!(poller.state.etag().as_deref(), Some("new")); - } - - #[test] - fn malformed_jsonapi_resources_keep_the_previous_configuration_and_etag() { - let mut poller = test_poller(false); - assert_eq!( - poller.process_response(response(200, Some("accepted"), EMPTY_CONFIG)), - PollOutcome::Success - ); - let valid: serde_json::Value = serde_json::from_slice(EMPTY_CONFIG).unwrap(); - let attributes = valid["data"]["attributes"].clone(); - for malformed in [ - attributes.clone(), - serde_json::json!({"data": {"type": "other-resource", "attributes": attributes}}), - serde_json::json!({"data": {"attributes": attributes}}), - serde_json::json!({"data": {"type": "universal-flag-configuration"}}), - serde_json::json!({"data": {"type": "universal-flag-configuration", "attributes": null}}), - serde_json::json!({"data": {"type": "universal-flag-configuration", "attributes": {}}}), - ] { - assert_eq!( - poller.process_response(response( - 200, - Some("rejected"), - &serde_json::to_vec(&malformed).unwrap() - )), - PollOutcome::Stop - ); - assert_eq!(poller.state.etag().as_deref(), Some("accepted")); - assert_eq!(poller.sink.applications.load(Ordering::Relaxed), 1); - } - } - - #[test] - fn non_200_responses_are_never_decoded() { - let mut poller = test_poller(false); - poller.state.set_etag(Some("current".to_owned())); - assert_eq!( - poller.process_response(response(304, Some("ignored"), b"not-json")), - PollOutcome::Success - ); - assert_eq!(poller.state.etag().as_deref(), Some("current")); - assert_eq!( - poller.process_response(response(401, None, EMPTY_CONFIG)), - PollOutcome::Stop - ); - assert_eq!(poller.state.etag().as_deref(), Some("current")); - } - - #[test] - fn warning_categories_are_suppressed_independently() { - let warnings = WarningSlots::default(); - assert!(warnings.take(WarningCategory::Request)); - assert!(!warnings.take(WarningCategory::Request)); - assert!(warnings.take(WarningCategory::UnexpectedHttp)); - assert!(!warnings.take(WarningCategory::UnexpectedHttp)); - warnings.reset(); - assert!(warnings.take(WarningCategory::Request)); - } - - #[tokio::test] - async fn a_poll_retries_three_times_without_overlapping_requests() { - let transport = QueueTransport::new([ - Ok(response(503, None, b"ignored")), - Err(TransportFailure::TimedOut), - Ok(response(200, Some("accepted"), EMPTY_CONFIG)), - ]); - let state = Arc::new(PollState::default()); - let mut poller = Poller { - endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") - .expect("test endpoint"), - poll_interval: Duration::from_secs(30), - request_timeout: Duration::from_secs(5), - transport, - sink: TestSink { - fail: AtomicBool::new(false), - applications: AtomicUsize::new(0), - }, - state: Arc::clone(&state), - retry_delay: no_retry_delay, - }; - - assert_eq!( - poller.poll(&CancellationToken::new()).await, - PollOutcome::Success - ); - assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 3); - assert_eq!(poller.transport.max_in_flight.load(Ordering::Relaxed), 1); - assert_eq!(state.poll_count.load(Ordering::Relaxed), 3); - assert_eq!(state.etag().as_deref(), Some("accepted")); - } - - #[tokio::test] - async fn cancellation_before_a_poll_prevents_the_first_request() { - let transport = QueueTransport::new([]); - let mut poller = Poller { - endpoint: AgentlessEndpoint::build("http://localhost/custom", "", "", "") - .expect("test endpoint"), - poll_interval: Duration::from_secs(30), - request_timeout: Duration::from_secs(5), - transport, - sink: TestSink { - fail: AtomicBool::new(false), - applications: AtomicUsize::new(0), - }, - state: Arc::new(PollState::default()), - retry_delay: no_retry_delay, - }; - let cancellation = CancellationToken::new(); - cancellation.cancel(); - - assert_eq!(poller.poll(&cancellation).await, PollOutcome::Cancelled); - assert_eq!(poller.transport.calls.load(Ordering::Relaxed), 0); - } -} diff --git a/components-rs/ffe/agentless/transport.rs b/components-rs/ffe/agentless/transport.rs deleted file mode 100644 index f6571fba037..00000000000 --- a/components-rs/ffe/agentless/transport.rs +++ /dev/null @@ -1,291 +0,0 @@ -// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ -// SPDX-License-Identifier: Apache-2.0 - -use super::super::settings::AgentlessEndpoint; -use base64::Engine; -use flate2::read::GzDecoder; -use http::header::{ACCEPT_ENCODING, CONTENT_ENCODING, CONTENT_LENGTH, ETAG, IF_NONE_MATCH}; -use http::{Method, Request}; -use http_body_util::BodyExt; -use libdd_common::http_common::{self, Body}; -use percent_encoding::percent_decode_str; -use std::future::Future; -use std::io::Read; -use std::pin::Pin; -use std::time::Duration; -use tokio_util::sync::CancellationToken; - -pub(super) const MAX_RESPONSE_BODY_BYTES: usize = 10 << 20; -const CLIENT_LANGUAGE: &str = "php"; -const INTERNAL_UNTRACED_HEADER: &str = "DD-Internal-Untraced-Request"; -const API_KEY_HEADER: &str = "DD-API-KEY"; -const AUTHORIZATION_HEADER: &str = "Authorization"; - -pub(super) type FetchFuture<'a> = - Pin> + Send + 'a>>; - -pub(super) trait Transport: Send + Sync { - fn get<'a>( - &'a self, - endpoint: &'a AgentlessEndpoint, - timeout: Duration, - etag: Option<&'a str>, - cancellation: &'a CancellationToken, - ) -> FetchFuture<'a>; -} - -#[derive(Debug, Eq, PartialEq)] -pub(super) enum TransportFailure { - BuildRequest, - Cancelled, - TimedOut, - Request, - ResponseBody, - ResponseTooLarge, -} - -pub(super) struct PollResponse { - pub(super) status: u16, - pub(super) etag: Option, - pub(super) content_encoding: Option, - pub(super) body: Vec, -} - -pub(super) struct HyperTransport { - client: libdd_common::HttpClient, -} - -impl HyperTransport { - pub(super) fn new() -> Self { - Self { - // This native client is outside PHP HTTP instrumentation, so it - // cannot recursively trace its own polling requests. Hyper also - // leaves 3xx responses visible instead of following redirects. - client: http_common::new_client_periodic(), - } - } - - async fn get_inner( - &self, - endpoint: &AgentlessEndpoint, - timeout: Duration, - etag: Option<&str>, - cancellation: &CancellationToken, - ) -> Result { - let request = build_request(endpoint, etag)?; - let operation = async { - let response = self - .client - .request(request) - .await - .map_err(|_| TransportFailure::Request)?; - let status = response.status().as_u16(); - let etag = header_value(response.headers(), ETAG); - let content_encoding = header_value(response.headers(), CONTENT_ENCODING); - - // A non-200 body can never contain configuration. Dropping it here - // also prevents a malicious error response from consuming memory. - if status != 200 { - return Ok(PollResponse { - status, - etag, - content_encoding, - body: Vec::new(), - }); - } - - if response - .headers() - .get(CONTENT_LENGTH) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.parse::().ok()) - .is_some_and(|length| length > MAX_RESPONSE_BODY_BYTES as u64) - { - return Err(TransportFailure::ResponseTooLarge); - } - - let body = collect_bounded(response.into_body(), MAX_RESPONSE_BODY_BYTES).await?; - Ok(PollResponse { - status, - etag, - content_encoding, - body, - }) - }; - - tokio::select! { - _ = cancellation.cancelled() => Err(TransportFailure::Cancelled), - result = tokio::time::timeout(timeout, operation) => { - result.map_err(|_| TransportFailure::TimedOut)? - } - } - } -} - -impl Transport for HyperTransport { - fn get<'a>( - &'a self, - endpoint: &'a AgentlessEndpoint, - timeout: Duration, - etag: Option<&'a str>, - cancellation: &'a CancellationToken, - ) -> FetchFuture<'a> { - Box::pin(self.get_inner(endpoint, timeout, etag, cancellation)) - } -} - -fn build_request( - endpoint: &AgentlessEndpoint, - etag: Option<&str>, -) -> Result, TransportFailure> { - let (request_url, authorization) = request_url_and_authorization(endpoint.as_str())?; - let mut builder = Request::builder() - .method(Method::GET) - .uri(request_url) - .header(ACCEPT_ENCODING, "gzip") - .header("DD-Client-Library-Language", CLIENT_LANGUAGE) - .header( - "DD-Client-Library-Version", - include_str!("../../../VERSION").trim(), - ) - .header(INTERNAL_UNTRACED_HEADER, "1"); - - if let Some(etag) = etag { - builder = builder.header(IF_NONE_MATCH, etag); - } - if let Some(api_key) = endpoint.api_key() { - builder = builder.header(API_KEY_HEADER, api_key); - } - if let Some(authorization) = authorization { - builder = builder.header(AUTHORIZATION_HEADER, authorization); - } - - builder - .body(Body::empty()) - .map_err(|_| TransportFailure::BuildRequest) -} - -fn request_url_and_authorization( - endpoint: &str, -) -> Result<(String, Option), TransportFailure> { - let mut url = url::Url::parse(endpoint).map_err(|_| TransportFailure::BuildRequest)?; - if url.username().is_empty() { - return Ok((endpoint.to_owned(), None)); - } - - let mut credentials = percent_decode_str(url.username()).collect::>(); - credentials.push(b':'); - if let Some(password) = url.password() { - credentials.extend(percent_decode_str(password)); - } - let authorization = format!( - "Basic {}", - base64::engine::general_purpose::STANDARD.encode(credentials) - ); - - url.set_username("") - .map_err(|_| TransportFailure::BuildRequest)?; - url.set_password(None) - .map_err(|_| TransportFailure::BuildRequest)?; - Ok((url.to_string(), Some(authorization))) -} - -fn header_value(headers: &http::HeaderMap, name: http::header::HeaderName) -> Option { - headers - .get(name) - .and_then(|value| value.to_str().ok()) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) -} - -async fn collect_bounded( - mut body: hyper::body::Incoming, - limit: usize, -) -> Result, TransportFailure> { - let mut bytes = Vec::new(); - while let Some(frame) = body.frame().await { - let frame = frame.map_err(|_| TransportFailure::ResponseBody)?; - if let Some(data) = frame.data_ref() { - let remaining = limit.saturating_sub(bytes.len()); - if data.len() > remaining { - return Err(TransportFailure::ResponseTooLarge); - } - bytes.extend_from_slice(data); - } - } - Ok(bytes) -} - -pub(super) fn decode_response_body( - body: &[u8], - content_encoding: Option<&str>, - limit: usize, -) -> Result, TransportFailure> { - if !content_encoding.is_some_and(|value| value.trim().eq_ignore_ascii_case("gzip")) { - if body.len() > limit { - return Err(TransportFailure::ResponseTooLarge); - } - return Ok(body.to_vec()); - } - - let decoder = GzDecoder::new(body); - let mut decoded = Vec::new(); - decoder - .take(limit as u64 + 1) - .read_to_end(&mut decoded) - .map_err(|_| TransportFailure::ResponseBody)?; - if decoded.len() > limit { - return Err(TransportFailure::ResponseTooLarge); - } - Ok(decoded) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - - fn gzip(payload: &[u8]) -> Vec { - let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - encoder.write_all(payload).expect("compress test payload"); - encoder.finish().expect("finish test payload") - } - - #[test] - fn decoded_body_limit_applies_after_gzip() { - let payload = vec![b'a'; 256]; - let compressed = gzip(&payload); - assert!(compressed.len() < 64); - assert_eq!( - decode_response_body(&compressed, Some(" GZIP "), 64), - Err(TransportFailure::ResponseTooLarge) - ); - assert_eq!( - decode_response_body(&gzip(b"configuration"), Some("gzip"), 64).expect("decode gzip"), - b"configuration" - ); - } - - #[test] - fn request_authentication_is_managed_only() { - let managed = - AgentlessEndpoint::build("", "datadoghq.com", "", "secret").expect("managed endpoint"); - let custom = - AgentlessEndpoint::build("https://user:password@localhost/custom", "", "", "secret") - .expect("custom endpoint"); - - let managed_request = build_request(&managed, Some("etag")).expect("managed request"); - assert_eq!(managed_request.headers()[API_KEY_HEADER], "secret"); - assert_eq!(managed_request.headers()[IF_NONE_MATCH], "etag"); - assert_eq!(managed_request.headers()[INTERNAL_UNTRACED_HEADER], "1"); - - let custom_request = build_request(&custom, None).expect("custom request"); - assert!(!custom_request.headers().contains_key(API_KEY_HEADER)); - assert_eq!( - custom_request.headers()[AUTHORIZATION_HEADER], - "Basic dXNlcjpwYXNzd29yZA==" - ); - assert_eq!(custom_request.uri(), "https://localhost/custom"); - } -} diff --git a/components-rs/ffe/lifecycle.rs b/components-rs/ffe/lifecycle.rs index eb59e181779..2ab8317edcf 100644 --- a/components-rs/ffe/lifecycle.rs +++ b/components-rs/ffe/lifecycle.rs @@ -47,6 +47,8 @@ pub struct FfeSettingsInput<'a> { pub site: CharSlice<'a>, pub api_key: CharSlice<'a>, pub environment: CharSlice<'a>, + pub sidecar_subprocess_enabled: bool, + pub sidecar_self_telemetry: bool, pub thread_cleanup: Option, } @@ -124,7 +126,16 @@ pub extern "C" fn ddog_ffe_configure(input: &FfeSettingsInput<'_>) -> FfeRuntime settings.request_timeout, ); worker_config.thread_cleanup = input.thread_cleanup; - state.worker = Some(AgentlessWorker::new(worker_config)); + worker_config.sidecar_self_telemetry = input.sidecar_self_telemetry; + if input.sidecar_subprocess_enabled { + state.worker = Some(AgentlessWorker::new(worker_config)); + } else { + log::log( + Log::Warn, + "Feature Flags CDN configuration requires sidecar subprocess mode", + ); + set_delivery_state(DeliveryState::PermanentError); + } } } state.config = FfeRuntimeConfig { @@ -271,6 +282,8 @@ mod tests { site: CharSlice::from(""), api_key: CharSlice::from(""), environment: CharSlice::from(""), + sidecar_subprocess_enabled: true, + sidecar_self_telemetry: false, thread_cleanup: None, }; assert!(ddog_ffe_configure(&input).endpoint_valid); diff --git a/components-rs/ffe/settings.rs b/components-rs/ffe/settings.rs index 2b228586f01..0e1656e7265 100644 --- a/components-rs/ffe/settings.rs +++ b/components-rs/ffe/settings.rs @@ -1,9 +1,8 @@ // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -use std::fmt; +pub use datadog_ffe::cdn::{CdnEndpoint as AgentlessEndpoint, EndpointError}; use std::time::Duration; -use url::Url; pub const DEFAULT_POLL_INTERVAL_SECONDS: i64 = 30; pub const MAX_POLL_INTERVAL_SECONDS: i64 = 3_600; @@ -12,10 +11,6 @@ pub const MAX_REQUEST_TIMEOUT_SECONDS: i64 = 300; pub const DEFAULT_INITIALIZATION_TIMEOUT_MS: i64 = 10_000; pub const MAX_INITIALIZATION_TIMEOUT_MS: i64 = i32::MAX as i64; -const DEFAULT_SITE: &str = "datadoghq.com"; -const AGENTLESS_HOST_PREFIX: &str = "ufc-server.ff-cdn."; -const AGENTLESS_CONFIGURATION_PATH: &str = "/api/v2/feature-flagging/config/rules-based/server"; - /// The configured source is retained separately from enablement. In particular, /// `Offline` is a reserved source whose delivery is disabled today, rather than /// an alias for the absence of a configured source. @@ -214,145 +209,6 @@ fn validated_numeric( } } -/// A validated agentless endpoint. Only managed endpoints can own an API key, -/// ensuring a custom endpoint cannot accidentally receive the Datadog credential. -#[derive(Clone, Eq, PartialEq)] -pub struct AgentlessEndpoint { - url: String, - api_key: Option, -} - -impl AgentlessEndpoint { - pub fn build( - base_url: &str, - site: &str, - environment: &str, - api_key: &str, - ) -> Result { - if base_url.trim().is_empty() { - Self::managed(site, environment, api_key) - } else { - Self::custom(base_url) - } - } - - pub fn as_str(&self) -> &str { - &self.url - } - - pub fn is_managed(&self) -> bool { - self.api_key.is_some() - } - - pub fn api_key(&self) -> Option<&str> { - self.api_key.as_deref() - } - - fn managed(site: &str, environment: &str, api_key: &str) -> Result { - if api_key.is_empty() { - return Err(EndpointError::MissingApiKey); - } - - let normalized_site = normalize_site(site)?; - let expected_host = format!("{AGENTLESS_HOST_PREFIX}{normalized_site}"); - let mut url = Url::parse(&format!( - "https://{expected_host}{AGENTLESS_CONFIGURATION_PATH}" - )) - .map_err(|_| EndpointError::InvalidSite)?; - - // Compare the parsed host before this endpoint is allowed to carry - // managed authentication. This is defense in depth after rejecting - // URL-significant site characters in `normalize_site`. - if url.host_str() != Some(expected_host.as_str()) { - return Err(EndpointError::InvalidSite); - } - - if !environment.is_empty() { - url.query_pairs_mut().append_pair("dd_env", environment); - } - - Ok(Self { - url: url.to_string(), - api_key: Some(api_key.to_owned()), - }) - } - - fn custom(base_url: &str) -> Result { - if base_url.chars().any(char::is_whitespace) { - return Err(EndpointError::UrlContainsWhitespace); - } - - let mut url = Url::parse(base_url).map_err(|_| EndpointError::InvalidUrl)?; - if !matches!(url.scheme(), "http" | "https") { - return Err(EndpointError::UnsupportedScheme); - } - if url.host_str().is_none() || url.fragment().is_some() { - return Err(EndpointError::InvalidUrl); - } - - let url = if matches!(url.path(), "" | "/") { - url.set_path(AGENTLESS_CONFIGURATION_PATH); - url.to_string() - } else { - // A custom non-root URL is opaque and must be requested exactly as - // the operator supplied it, including its query string. - base_url.to_owned() - }; - - Ok(Self { url, api_key: None }) - } -} - -impl fmt::Debug for AgentlessEndpoint { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("AgentlessEndpoint") - .field("url", &"[REDACTED]") - .field("managed", &self.is_managed()) - .finish() - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum EndpointError { - MissingApiKey, - InvalidUrl, - UnsupportedScheme, - UrlContainsWhitespace, - InvalidSite, -} - -impl fmt::Display for EndpointError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - let message = match self { - Self::MissingApiKey => "no API key is configured for the managed endpoint", - Self::InvalidUrl => "the configured URL is not a valid absolute URL", - Self::UnsupportedScheme => "the configured URL must use HTTP or HTTPS", - Self::UrlContainsWhitespace => "the configured URL contains whitespace", - Self::InvalidSite => "the configured site is not a valid host", - }; - formatter.write_str(message) - } -} - -impl std::error::Error for EndpointError {} - -fn normalize_site(site: &str) -> Result { - let site = if site.trim().is_empty() { - DEFAULT_SITE - } else { - site - }; - if !site.is_ascii() - || site.chars().any(|character| { - character.is_whitespace() || matches!(character, '/' | '\\' | '?' | '#' | '@' | ':') - }) - { - return Err(EndpointError::InvalidSite); - } - Ok(site.to_ascii_lowercase()) -} - #[cfg(test)] mod tests { use super::*; @@ -518,128 +374,4 @@ mod tests { ] ); } - - #[test] - fn managed_endpoint_uses_normalized_site_and_encoded_environment() { - let endpoint = AgentlessEndpoint::build("", "DATADOGHQ.EU", "prod / blue", "key") - .expect("managed endpoint"); - - assert!(endpoint.is_managed()); - assert_eq!(endpoint.api_key(), Some("key")); - assert_eq!( - endpoint.as_str(), - "https://ufc-server.ff-cdn.datadoghq.eu/api/v2/feature-flagging/config/rules-based/server?dd_env=prod+%2F+blue" - ); - } - - #[test] - fn managed_endpoint_omits_an_absent_environment_and_defaults_blank_site() { - let endpoint = AgentlessEndpoint::build("", " ", "", "key").expect("managed endpoint"); - assert_eq!( - endpoint.as_str(), - "https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server" - ); - } - - #[test] - fn managed_endpoint_requires_an_api_key() { - assert_eq!( - AgentlessEndpoint::build("", "datadoghq.com", "", "").unwrap_err(), - EndpointError::MissingApiKey - ); - } - - #[test] - fn managed_endpoint_rejects_sites_that_can_change_the_url_meaning() { - for site in [ - "https://datadoghq.com", - "datadoghq.com@attacker.example", - "datadoghq.com/path", - "datadoghq.com?query", - "datadoghq.com#fragment", - "datadoghq.com:443", - " datadoghq.com", - "data doghq.com", - "dâtadoghq.com", - ] { - assert_eq!( - AgentlessEndpoint::build("", site, "", "key").unwrap_err(), - EndpointError::InvalidSite, - "site should be rejected" - ); - } - } - - #[test] - fn custom_root_endpoint_gets_the_canonical_path_and_keeps_its_query() { - for base_url in ["http://localhost:8080", "http://localhost:8080/"] { - let endpoint = AgentlessEndpoint::build(base_url, "ignored", "ignored", "secret") - .expect("custom endpoint"); - assert!(!endpoint.is_managed()); - assert_eq!( - endpoint.as_str(), - "http://localhost:8080/api/v2/feature-flagging/config/rules-based/server" - ); - } - - let endpoint = AgentlessEndpoint::build( - "https://user:pass@localhost/?tenant=a%2Fb", - "ignored", - "ignored", - "secret", - ) - .expect("custom endpoint"); - assert_eq!( - endpoint.as_str(), - "https://user:pass@localhost/api/v2/feature-flagging/config/rules-based/server?tenant=a%2Fb" - ); - } - - #[test] - fn custom_non_root_endpoint_is_preserved_without_environment() { - let endpoint = AgentlessEndpoint::build( - "HTTPS://LOCALHOST/custom/path?tenant=a%2Fb", - "ignored", - "must-not-be-added", - "must-not-be-sent", - ) - .expect("custom endpoint"); - assert_eq!( - endpoint.as_str(), - "HTTPS://LOCALHOST/custom/path?tenant=a%2Fb" - ); - assert!(!endpoint.is_managed()); - assert_eq!(endpoint.api_key(), None); - } - - #[test] - fn custom_endpoint_validation_errors_never_contain_the_url() { - let credential = "credential-that-must-not-leak"; - for (url, expected) in [ - ( - format!("ftp://{credential}@localhost/path"), - EndpointError::UnsupportedScheme, - ), - ( - format!("https://{credential}@local host/path"), - EndpointError::UrlContainsWhitespace, - ), - (format!("not-a-url-{credential}"), EndpointError::InvalidUrl), - ] { - let error = AgentlessEndpoint::build(&url, "", "", "key").unwrap_err(); - assert_eq!(error, expected); - assert!(!error.to_string().contains(credential)); - } - } - - #[test] - fn endpoint_debug_output_is_redacted() { - let endpoint = - AgentlessEndpoint::build("https://user:password@localhost/custom", "", "", "key") - .expect("custom endpoint"); - let debug = format!("{endpoint:?}"); - assert!(!debug.contains("user")); - assert!(!debug.contains("password")); - assert!(debug.contains("[REDACTED]")); - } } diff --git a/components-rs/library-config.h b/components-rs/library-config.h index c29b0cf3e55..fe1e026585f 100644 --- a/components-rs/library-config.h +++ b/components-rs/library-config.h @@ -24,32 +24,8 @@ void ddog_library_configurator_with_fleet_path(struct ddog_Configurator *c, void ddog_library_configurator_with_process_info(struct ddog_Configurator *c, struct ddog_ProcessInfo p); -void ddog_library_configurator_with_detect_process_info(struct ddog_Configurator *c); - void ddog_library_configurator_drop(struct ddog_Configurator*); -struct ddog_LibraryConfigLoggedResult ddog_library_configurator_get(const struct ddog_Configurator *configurator); - -/** - * Returns a static null-terminated string, containing the name of the environment variable - * associated with the library configuration - */ -struct ddog_CStr ddog_library_config_source_to_string(enum ddog_LibraryConfigSource name); - -/** - * Returns a static null-terminated string with the path to the managed stable config yaml config - * file - */ -struct ddog_CStr ddog_library_config_fleet_stable_config_path(void); - -/** - * Returns a static null-terminated string with the path to the local stable config yaml config - * file - */ -struct ddog_CStr ddog_library_config_local_stable_config_path(void); - -void ddog_library_config_drop(struct ddog_LibraryConfigLoggedResult config_result); - /** * Allocates and returns a pointer to a new `TracerMetadata` object on the heap. * @@ -89,6 +65,29 @@ void ddog_tracer_metadata_set(struct ddog_TracerMetadata *ptr, enum ddog_MetadataKind kind, const char *value); +/** + * Includes thread-context discovery metadata in the `TracerMetadata` object pointed to by `ptr`. + * + * If the builder has no thread-context metadata, configures it to publish + * the `tlsdesc_v1_dev` schema and a key map containing `datadog.local_root_span_id`. + * If the thread-context metadata is already set, leaves it unchanged. + * + * This function only updates the builder. Call `ddog_tracer_metadata_store` to publish it. + * + * # Arguments + * - `ptr`: Pointer to a `TracerMetadata` instance. + * + * # Safety + * - If non-null, `ptr` must point to a valid, properly aligned, live `TracerMetadata`. + * - The caller must ensure exclusive access to the instance for the duration of the call. + * - Ownership remains with the caller + * + * # Returns + * - On success: `VoidResult::Ok`, also when thread-context metadata is already present + * - An error if `ptr` is null. + */ +struct ddog_VoidResult ddog_tracer_metadata_include_otel_thread_context(struct ddog_TracerMetadata *ptr); + /** * Serializes the `TracerMetadata` into a platform-specific memory handle (e.g., memfd on Linux). * This function also attempts to publish the tracer metadata as an OTel process context @@ -108,6 +107,30 @@ void ddog_tracer_metadata_set(struct ddog_TracerMetadata *ptr, */ struct ddog_Result_TracerMemfdHandle ddog_tracer_metadata_store(struct ddog_TracerMetadata *ptr); +void ddog_library_configurator_with_detect_process_info(struct ddog_Configurator *c); + +struct ddog_LibraryConfigLoggedResult ddog_library_configurator_get(const struct ddog_Configurator *configurator); + +/** + * Returns a static null-terminated string, containing the name of the environment variable + * associated with the library configuration + */ +struct ddog_CStr ddog_library_config_source_to_string(enum ddog_LibraryConfigSource name); + +/** + * Returns a static null-terminated string with the path to the managed stable config yaml + * config file + */ +struct ddog_CStr ddog_library_config_fleet_stable_config_path(void); + +/** + * Returns a static null-terminated string with the path to the local stable config yaml config + * file + */ +struct ddog_CStr ddog_library_config_local_stable_config_path(void); + +void ddog_library_config_drop(struct ddog_LibraryConfigLoggedResult config_result); + #ifdef __cplusplus } // extern "C" #endif // __cplusplus diff --git a/components-rs/sidecar.h b/components-rs/sidecar.h index a6036f56feb..bb0d20f9601 100644 --- a/components-rs/sidecar.h +++ b/components-rs/sidecar.h @@ -292,10 +292,33 @@ ddog_MaybeError ddog_sidecar_send_debugger_datum(struct ddog_SidecarTransport ** struct ddog_DebuggerPayload *payload); /** - * Send structured FFE exposure events to the sidecar. The sidecar owns - * deduplication, JSON serialization, and Agent EVP delivery. This function is - * caller-driven; shared libdatadog evaluator calls do not log unless an SDK - * explicitly sends this action. + * Configure one shared EVP intake target in this connection's sidecar session. + * `AgentOnly` preserves the historical fixed EVP v2 route. + * `PreferLocalThenDirect` explicitly opts the client into local discovery and + * authenticated direct fallback. + * + * Call after setting the connection configuration. Other intake targets are + * unaffected. Identical configuration preserves routing and deduplication + * state; changed configuration replaces this target's transport. The client + * retains the latest configuration per target across sidecar reconnects and + * forks. Each new connection or forked instance starts with fresh route state. + * + * # Safety + * `direct_endpoint` must be null or point to a valid `Endpoint`, and all + * string slices must remain valid for the duration of this call. + */ +ddog_MaybeError ddog_sidecar_session_set_evp_transport(struct ddog_SidecarTransport **transport, + enum ddog_EvpTransportMode mode, + const struct ddog_Endpoint *agent_endpoint, + const struct ddog_Endpoint *direct_endpoint, + ddog_CharSlice intake_subdomain, + const struct ddog_EvpProducerIdentity *producer); + +/** + * Send structured Feature Flags exposure events to the sidecar. The sidecar owns + * route selection, deduplication, JSON serialization, and EVP delivery. This + * function is caller-driven; shared libdatadog evaluator calls do not log unless + * an SDK explicitly sends this action. * * # Safety * `context` and every element in `exposures` must contain valid UTF-8 @@ -306,9 +329,10 @@ ddog_MaybeError ddog_sidecar_send_ffe_exposure_batch(struct ddog_SidecarTranspor struct ddog_Slice_FfeExposure exposures); /** - * Send structured FFE flag evaluation events to the sidecar. The sidecar owns - * JSON serialization and Agent EVP delivery. This function is caller-driven; - * callers must aggregate and bound event cardinality before passing a batch. + * Send structured Feature Flags evaluation events to the sidecar. The sidecar owns + * route selection, JSON serialization, and EVP delivery. This function is + * caller-driven; callers must aggregate and bound event cardinality before passing + * a batch. * * # Safety * All slices must reference valid memory under the CharSlice contract. Context @@ -322,7 +346,7 @@ ddog_MaybeError ddog_sidecar_send_ffe_flag_evaluation_batch(struct ddog_SidecarT struct ddog_Slice_FfeFlagEvaluation flag_evaluations); /** - * Send structured FFE evaluation metric events to the sidecar. The sidecar + * Send structured Feature Flags evaluation metric events to the sidecar. The sidecar * owns aggregation, OTLP/protobuf serialization, and OTLP HTTP delivery. This * function is caller-driven so SDKs with existing host-language hooks can * safely coexist until they explicitly migrate. diff --git a/components-rs/sidecar.rs b/components-rs/sidecar.rs index bd6b09e50bc..8c7b3b22e1f 100644 --- a/components-rs/sidecar.rs +++ b/components-rs/sidecar.rs @@ -25,7 +25,7 @@ use std::sync::Mutex; use std::time::Duration; #[cfg(php_shared_build)] -fn run_sidecar(mut cfg: config::Config) -> anyhow::Result { +pub(crate) fn run_sidecar(mut cfg: config::Config) -> anyhow::Result { #[cfg(target_os = "linux")] if std::env::var_os("DD_SIDECAR_DISABLE_DIRECT_EXEC") .map(|s| s.is_empty()) @@ -40,7 +40,7 @@ fn run_sidecar(mut cfg: config::Config) -> anyhow::Result { } #[cfg(not(any(windows, php_shared_build)))] -fn run_sidecar(cfg: config::Config) -> anyhow::Result { +pub(crate) fn run_sidecar(cfg: config::Config) -> anyhow::Result { ddtrace_sidecar::start_or_connect_to_sidecar(cfg) } @@ -49,7 +49,7 @@ fn run_sidecar(cfg: config::Config) -> anyhow::Result { pub static mut DDOG_PHP_FUNCTION: *const u8 = std::ptr::null(); #[cfg(windows)] -fn run_sidecar(mut cfg: config::Config) -> anyhow::Result { +pub(crate) fn run_sidecar(mut cfg: config::Config) -> anyhow::Result { let php_dll = get_trampoline_target_data(unsafe { DDOG_PHP_FUNCTION })?; cfg.library_dependencies .push(LibDependency::Path(php_dll.into())); diff --git a/libdatadog b/libdatadog index ed81443e10c..6d3330a6df2 160000 --- a/libdatadog +++ b/libdatadog @@ -1 +1 @@ -Subproject commit ed81443e10c4642c5a952a0a1b36596f7df576d1 +Subproject commit 6d3330a6df20bfed6537de38f02f33a50ce90e25 diff --git a/tests/ext/ffe/agentless_fork.phpt b/tests/ext/ffe/agentless_fork.phpt index 4e0c8e19e91..1bab2317f26 100644 --- a/tests/ext/ffe/agentless_fork.phpt +++ b/tests/ext/ffe/agentless_fork.phpt @@ -1,5 +1,5 @@ --TEST-- -FFE agentless fork restarts independent parent and child configuration pollers +FFE agentless fork restarts parent and child shared-memory configuration readers --SKIPIF-- +--ENV-- +DD_FEATURE_FLAGS_ENABLED=1 +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless +DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS=10000 +DD_TRACE_ENABLED=0 +DD_TRACE_CLI_ENABLED=1 +DD_REMOTE_CONFIG_ENABLED=0 +DD_INSTRUMENTATION_TELEMETRY_ENABLED=0 +DD_METRICS_OTEL_ENABLED=0 +DD_TRACE_AGENT_URL=http://127.0.0.1:1 +NO_PROXY=127.0.0.1,localhost +--FILE-- + +--EXPECT-- +cdn_configuration_evaluated +cdn_request_outside_php diff --git a/tests/ext/ffe/agentless_sidecar_restart.phpt b/tests/ext/ffe/agentless_sidecar_restart.phpt new file mode 100644 index 00000000000..1cf685d7f85 --- /dev/null +++ b/tests/ext/ffe/agentless_sidecar_restart.phpt @@ -0,0 +1,32 @@ +--TEST-- +FFE agentless retains configuration and resubscribes after a sidecar subprocess restart +--SKIPIF-- + +--ENV-- +_DD_DEBUG_SIDECAR_IPC_MODE=instance_per_process +_DD_DEBUG_SIDECAR_IDLE_LINGER_TIME_SECS=1 +DD_FEATURE_FLAGS_ENABLED=1 +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS=1 +DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless +DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS=10000 +DD_TRACE_ENABLED=0 +DD_TRACE_CLI_ENABLED=1 +DD_REMOTE_CONFIG_ENABLED=0 +DD_INSTRUMENTATION_TELEMETRY_ENABLED=0 +DD_METRICS_OTEL_ENABLED=0 +DD_TRACE_AGENT_URL=http://127.0.0.1:1 +NO_PROXY=127.0.0.1,localhost +--FILE-- + +--EXPECT-- +last_configuration_retained +sidecar_restarted_and_refreshed diff --git a/tests/ext/ffe/agentless_source_selection.phpt b/tests/ext/ffe/agentless_source_selection.phpt index 75700a3bdf0..e86b7e05dae 100644 --- a/tests/ext/ffe/agentless_source_selection.phpt +++ b/tests/ext/ffe/agentless_source_selection.phpt @@ -28,9 +28,11 @@ $cases = array( 'explicit_remote_config' => array('1', 'remote_config', '0', true, 'native_remote_config'), 'offline' => array('1', 'offline', '1', false, 'native_disabled'), 'invalid' => array('1', 'unknown', '1', false, 'native_disabled'), + 'thread_mode' => array('1', 'agentless', '0', true, 'native_agentless'), ); $keys = array('DD_FEATURE_FLAGS_ENABLED', 'DD_FEATURE_FLAGS_CONFIGURATION_SOURCE', 'DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED'); foreach ($cases as $name => $case) { + $environment['DD_TRACE_SIDECAR_CONNECTION_MODE'] = $name === 'thread_mode' ? 'thread' : 'auto'; foreach ($keys as $i => $key) { unset($environment[$key]); if ($case[$i] !== null) $environment[$key] = $case[$i]; @@ -75,3 +77,4 @@ legacy_remote_config=ok explicit_remote_config=ok offline=ok invalid=ok +thread_mode=ok diff --git a/tests/ext/ffe/stubs/agentless_process.inc b/tests/ext/ffe/stubs/agentless_process.inc index cbfdce9640d..d05593541e0 100644 --- a/tests/ext/ffe/stubs/agentless_process.inc +++ b/tests/ext/ffe/stubs/agentless_process.inc @@ -21,6 +21,7 @@ $running = true; $start = microtime(true); $readyAt = null; $shutdownStart = null; +$restarted = false; try { do { $stdout .= ffeTestProcessRead($pipes[1]); @@ -32,11 +33,77 @@ try { $exitCode = $status['exitcode']; break; } + if ($scenario === 'restart' && !$restarted && strpos($stdout, "fixture_restart_ready\n") !== false) { + // Kill only this child's per-process sidecar: match its unique + // listening socket, then find the process that owns that socket. + $socketInode = null; + foreach (file('/proc/net/unix') as $row) { + $columns = preg_split('/\s+/', trim($row)); + if (isset($columns[7]) && $columns[3] === '00010000' + && strpos($columns[7], '@libdatadog/') === 0 + && substr($columns[7], -strlen('.' . $status['pid'] . '.sock')) === '.' . $status['pid'] . '.sock') { + $socketInode = $columns[6]; + break; + } + } + if ($socketInode === null) throw new RuntimeException('cannot locate the dedicated sidecar socket'); + foreach (glob('/proc/[0-9]*', GLOB_ONLYDIR) as $directory) { + if (@fileowner($directory) !== posix_geteuid()) continue; + foreach (glob($directory . '/fd/*') as $descriptor) { + if (@readlink($descriptor) !== 'socket:[' . $socketInode . ']') continue; + $sidecarPid = (int)basename($directory); + if ($sidecarPid === $status['pid'] || $sidecarPid === getmypid()) { + throw new RuntimeException('sidecar listener is in a PHP application process'); + } + if (!posix_kill($sidecarPid, SIGKILL)) throw new RuntimeException('cannot stop dedicated sidecar'); + $restarted = true; + break 2; + } + } + if (!$restarted) throw new RuntimeException('cannot find the dedicated sidecar owner'); + } $connection = @stream_socket_accept($server, 0.1); if (!$connection) continue; stream_set_timeout($connection, 1); while (($line = fgets($connection)) !== false && trim($line) !== '') {} $requests++; + if ($scenario === 'isolation') { + // The server holds the response while we inspect the live client + // connection. A background thread still belongs to the PHP PID. + $peer = stream_socket_get_name($connection, true); + $clientPort = strtoupper(sprintf('%04X', substr(strrchr($peer, ':'), 1))); + $serverPort = strtoupper(sprintf('%04X', substr(strrchr($address, ':'), 1))); + $inode = null; + foreach (file('/proc/' . $status['pid'] . '/net/tcp') as $row) { + $columns = preg_split('/\s+/', trim($row)); + if (isset($columns[9]) && $columns[1] === '0100007F:' . $clientPort + && $columns[2] === '0100007F:' . $serverPort) { + $inode = $columns[9]; + break; + } + } + if ($inode === null) throw new RuntimeException('cannot locate the live CDN socket'); + $descriptors = glob('/proc/' . $status['pid'] . '/fd/*'); + if (!$descriptors) throw new RuntimeException('cannot inspect the PHP descriptors'); + foreach ($descriptors as $descriptor) { + if (@readlink($descriptor) === 'socket:[' . $inode . ']') { + throw new RuntimeException('CDN request originated in the PHP application process'); + } + } + } + if ($scenario === 'restart' && $restarted) { + // Keep the replacement response pending until PHP has observed + // stale readiness while still evaluating the last good snapshot. + $staleDeadline = microtime(true) + 5; + do { + $stdout .= ffeTestProcessRead($pipes[1]); + if (strpos($stdout, "last_configuration_retained\n") !== false) break; + usleep(10000); + } while (microtime(true) < $staleDeadline); + if (strpos($stdout, "last_configuration_retained\n") === false) { + throw new RuntimeException('sidecar loss did not preserve stale configuration'); + } + } if ($scenario === 'shutdown') { $heldConnections[] = $connection; if ($shutdownStart === null) { @@ -65,7 +132,7 @@ try { } else { $stdout .= ffeTestProcessRead($pipes[1]); $forked = strpos($stdout, 'parent_forked') !== false && strpos($stdout, 'child_forked') !== false; - $value = $scenario === 'fork' && $forked ? 'green' : 'blue'; + $value = ($scenario === 'fork' && $forked) || ($scenario === 'restart' && $restarted) ? 'green' : 'blue'; $code = '200 OK'; $configuration = array( 'createdAt' => '2026-05-22T00:00:00Z', 'environment' => array('name' => 'test'), @@ -93,10 +160,11 @@ try { throw new RuntimeException('shutdown waited on HTTP timeout'); } // Fork output is intentionally concurrent; compare the complete set. - $lines = explode("\n", trim(str_replace("fixture_child_ready\n", '', $stdout))); + $lines = explode("\n", trim(str_replace(array("fixture_child_ready\n", "fixture_restart_ready\n"), '', $stdout))); sort($lines); echo implode("\n", $lines), "\n"; if ($scenario === 'shutdown') echo "shutdown_cancelled_inflight_request\n"; + if ($scenario === 'isolation') echo "cdn_request_outside_php\n"; } finally { if ($running) proc_terminate($process); if (is_resource($pipes[0])) fclose($pipes[0]); diff --git a/tests/ext/ffe/stubs/agentless_process_app.php b/tests/ext/ffe/stubs/agentless_process_app.php index 935495cf41d..c5cd67d9d42 100644 --- a/tests/ext/ffe/stubs/agentless_process_app.php +++ b/tests/ext/ffe/stubs/agentless_process_app.php @@ -9,19 +9,46 @@ $first = $client->getStringValue('flag', 'fallback'); $elapsed = microtime(true) - $start; +if ($scenario === 'isolation') { + if ($first !== 'blue') throw new RuntimeException('configuration did not reach the PHP evaluator'); + echo "cdn_configuration_evaluated\n"; + exit; +} + +if ($scenario === 'restart') { + if ($first !== 'blue') throw new RuntimeException('initial fetch failed'); + echo "fixture_restart_ready\n"; + $retained = false; + $deadline = microtime(true) + 12; + do { + $value = $client->getStringValue('flag', 'fallback'); + $state = \DDTrace\Internal\ffe_provider_state(); + if (!$retained && $value === 'blue' && $state['reason'] === 'stale') { + $retained = true; + echo "last_configuration_retained\n"; + } + if ($value === 'green' && $state['ready']) break; + if ($value !== 'blue') throw new RuntimeException('sidecar restart lost configuration'); + usleep(20000); + } while (microtime(true) < $deadline); + if (!$retained || $value !== 'green') throw new RuntimeException('sidecar restart did not recover'); + echo "sidecar_restarted_and_refreshed\n"; + exit; +} + if ($scenario === 'fork') { if ($first !== 'blue') throw new RuntimeException('initial fetch failed'); $version = \DDTrace\ffe_config_version(); $pid = pcntl_fork(); if ($pid < 0) throw new RuntimeException('fork failed'); // The HTTP fixture only serves green after this marker. Both copies must - // fetch a new configuration after the split to observe it. + // read a new configuration after the split to observe it. echo $pid === 0 ? "child_forked\n" : "parent_forked\n"; $deadline = microtime(true) + 7; $value = $first; do { // Merely evaluating would invoke first-use activation again and hide - // a broken fork callback. Require the background poller to publish a + // a broken fork callback. Require the background reader to publish a // new snapshot before another evaluation is allowed to run. $currentVersion = \DDTrace\ffe_config_version(); if ($currentVersion !== $version) { diff --git a/tracer/ffe.c b/tracer/ffe.c index c21658d563f..74a1bdab1e4 100644 --- a/tracer/ffe.c +++ b/tracer/ffe.c @@ -17,6 +17,10 @@ ZEND_EXTERN_MODULE_GLOBALS(datadog); #define DD_FFE_EXPOSURE_BUFFER_LIMIT 1000 ddog_FfeRuntimeConfig ddtrace_ffe_configure(void) { +#ifdef _WIN32 + // CDN can own a subprocess connection even when other products are disabled. + DDOG_PHP_FUNCTION = (const uint8_t *)zend_hash_func; +#endif const ddog_FfeSettingsInput settings = { .enabled = get_global_DD_FEATURE_FLAGS_ENABLED(), .enabled_set = zai_config_memoized_entries[DATADOG_CONFIG_DD_FEATURE_FLAGS_ENABLED].name_index != ZAI_CONFIG_ORIGIN_DEFAULT, @@ -31,6 +35,8 @@ ddog_FfeRuntimeConfig ddtrace_ffe_configure(void) { .site = dd_zend_string_to_CharSlice(get_global_DD_SITE()), .api_key = dd_zend_string_to_CharSlice(get_global_DD_API_KEY()), .environment = dd_zend_string_to_CharSlice(get_global_DD_ENV()), + .sidecar_subprocess_enabled = get_global_DD_TRACE_SIDECAR_CONNECTION_MODE() != DD_TRACE_SIDECAR_CONNECTION_MODE_THREAD, + .sidecar_self_telemetry = get_global_DD_INSTRUMENTATION_TELEMETRY_ENABLED(), #ifdef CXA_THREAD_ATEXIT_WRAPPER .thread_cleanup = dd_run_rust_thread_destructors, #endif From 00a6b8f095212b10fc73a5aaddca3ebbe01f7885 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 14 Oct 2026 14:58:22 +0000 Subject: [PATCH 07/10] Update libdatadog CDN test portability fixes Environment: Datadog workspace --- libdatadog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libdatadog b/libdatadog index 6d3330a6df2..6c44d50813b 160000 --- a/libdatadog +++ b/libdatadog @@ -1 +1 @@ -Subproject commit 6d3330a6df20bfed6537de38f02f33a50ce90e25 +Subproject commit 6c44d50813b1a64e1614647c22527871c88774e3 From fc7b8e72a4c705e2fcbc80952e94567f36359413 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 14 Oct 2026 18:13:58 +0000 Subject: [PATCH 08/10] Complete CDN diagnostics and lifecycle regression coverage --- Cargo.lock | 1 + Cargo.toml | 1 + components-rs/datadog.h | 7 ++ components-rs/ffe/lifecycle.rs | 115 +++++++++++++++--- libdatadog | 2 +- .../FeatureFlags/Internal/NativeEvaluator.php | 3 + .../ffe/agentless_settings_diagnostics.phpt | 81 ++++++++++++ tests/ext/ffe/agentless_source_selection.phpt | 48 +------- tests/ext/ffe/agentless_thread_mode.phpt | 24 ++++ tests/ext/ffe/stubs/agentless_process.inc | 2 +- .../ext/ffe/stubs/agentless_settings_app.php | 7 ++ .../ffe/stubs/agentless_source_selection.inc | 49 ++++++++ tracer/functions.c | 13 +- 13 files changed, 285 insertions(+), 68 deletions(-) create mode 100644 tests/ext/ffe/agentless_settings_diagnostics.phpt create mode 100644 tests/ext/ffe/agentless_thread_mode.phpt create mode 100644 tests/ext/ffe/stubs/agentless_settings_app.php create mode 100644 tests/ext/ffe/stubs/agentless_source_selection.inc diff --git a/Cargo.lock b/Cargo.lock index 2710a790af2..a9902c8472b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2992,6 +2992,7 @@ dependencies = [ "futures-util", "http 1.4.2", "http-body-util", + "httpdate", "httpmock", "hyper", "libdd-capabilities", diff --git a/Cargo.toml b/Cargo.toml index 8f6d44af093..ffa59381d97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -219,6 +219,7 @@ hex = { version = "0.4", default-features = false } http = { version = "1.1", default-features = false } http-body = { version = "1", default-features = false } http-body-util = { version = "0.1", default-features = false } +httpdate = { version = "1.0", default-features = false } # allow(workspace-deps-features): httpmock 0.8.0-alpha.1 does not compile # without the `cookies` feature: its own `readers.rs` calls `req.cookies()` # unconditionally while that method is gated behind `cookies`. Keep it as a diff --git a/components-rs/datadog.h b/components-rs/datadog.h index 86c8de153f1..7bbd0de93ee 100644 --- a/components-rs/datadog.h +++ b/components-rs/datadog.h @@ -162,6 +162,13 @@ struct ddog_FfeRuntimeConfig ddog_ffe_configure(const struct ddog_FfeSettingsInp struct ddog_FfeRuntimeConfig ddog_ffe_runtime_config(void); +/** + * Static, redacted configuration failure. Empty when initialization can retry. + */ +ddog_CharSlice ddog_ffe_initialization_error_reason(void); + +ddog_CharSlice ddog_ffe_initialization_error_message(void); + /** * First-use compatibility path for PHP providers without an initialize API. * Concurrent first callers share one deadline; later calls never renew it. diff --git a/components-rs/ffe/lifecycle.rs b/components-rs/ffe/lifecycle.rs index 2ab8317edcf..0b9aceec129 100644 --- a/components-rs/ffe/lifecycle.rs +++ b/components-rs/ffe/lifecycle.rs @@ -2,7 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 use super::agentless::{AgentlessWorker, AgentlessWorkerConfig}; -use super::settings::{ConfigurationSource, FeatureFlagsSettings, SettingsInput, SourceInput}; +use super::settings::{ + ConfigurationSource, DisableReason, EndpointError, FeatureFlagsSettings, SettingsInput, + SettingsIssue, SourceInput, +}; use super::{ddog_ffe_has_config, set_delivery_state, DeliveryState}; use crate::log::{self, Log}; use arc_swap::ArcSwap; @@ -59,6 +62,62 @@ struct Lifecycle { worker: Option, initialization_deadline: Option, stopped: bool, + failure: Option, + diagnostics_reported: bool, +} + +#[derive(Clone, Copy)] +enum InitializationFailure { + InvalidSource, + Endpoint(EndpointError), + SidecarMode, +} + +impl InitializationFailure { + fn reason(self) -> &'static str { + match self { + Self::InvalidSource => "invalid_source", + Self::Endpoint(EndpointError::MissingApiKey) => "missing_api_key", + Self::Endpoint(EndpointError::InvalidSite) => "invalid_site", + Self::Endpoint(_) => "invalid_endpoint", + Self::SidecarMode => "unsupported_sidecar_mode", + } + } + + fn message(self) -> &'static str { + match self { + Self::InvalidSource => "Feature Flags configuration source is invalid; use agentless, remote_config or offline", + Self::Endpoint(EndpointError::MissingApiKey) => "Feature Flags CDN configuration requires DD_API_KEY for the managed endpoint", + Self::Endpoint(EndpointError::InvalidSite) => "Feature Flags CDN configuration requires a valid DD_SITE hostname", + Self::Endpoint(_) => "Feature Flags CDN base URL must be an absolute HTTP or HTTPS URL without whitespace or a fragment", + Self::SidecarMode => "Feature Flags CDN configuration requires sidecar subprocess mode", + } + } +} + +impl Lifecycle { + fn report_diagnostics(&mut self) { + if self.diagnostics_reported { + return; + } + self.diagnostics_reported = true; + if let Some(settings) = &self.settings { + if settings.resolution.legacy_key_decided { + log::log(Log::Warn, "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED is deprecated; use DD_FEATURE_FLAGS_ENABLED and DD_FEATURE_FLAGS_CONFIGURATION_SOURCE"); + } + for issue in &settings.issues { + let message = match issue { + SettingsIssue::PollInterval => "Invalid DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS; using 30 seconds (valid range: 1-3600)", + SettingsIssue::RequestTimeout => "Invalid DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS; using 5 seconds (valid range: 1-300)", + SettingsIssue::InitializationTimeout => "Invalid DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS; using 10000 milliseconds (valid range: 1-2147483647)", + }; + log::log(Log::Warn, message); + } + } + if let Some(failure) = self.failure { + log::log(Log::Warn, failure.message()); + } + } } static LIFECYCLE: LazyLock> = LazyLock::new(|| Mutex::new(Lifecycle::default())); @@ -116,24 +175,30 @@ pub extern "C" fn ddog_ffe_configure(input: &FfeSettingsInput<'_>) -> FfeRuntime } }; super::use_agentless_config(source == FfeConfigurationSource::Agentless); + if settings.resolution.disable_reason == Some(DisableReason::InvalidSource) { + state.failure = Some(InitializationFailure::InvalidSource); + } let mut endpoint_valid = false; if settings.resolution.enabled && source == FfeConfigurationSource::Agentless { - if let Ok(endpoint) = settings.agentless_endpoint() { - endpoint_valid = true; - let mut worker_config = AgentlessWorkerConfig::new( - endpoint, - settings.poll_interval, - settings.request_timeout, - ); - worker_config.thread_cleanup = input.thread_cleanup; - worker_config.sidecar_self_telemetry = input.sidecar_self_telemetry; - if input.sidecar_subprocess_enabled { - state.worker = Some(AgentlessWorker::new(worker_config)); - } else { - log::log( - Log::Warn, - "Feature Flags CDN configuration requires sidecar subprocess mode", + match settings.agentless_endpoint() { + Ok(endpoint) => { + endpoint_valid = true; + let mut worker_config = AgentlessWorkerConfig::new( + endpoint, + settings.poll_interval, + settings.request_timeout, ); + worker_config.thread_cleanup = input.thread_cleanup; + worker_config.sidecar_self_telemetry = input.sidecar_self_telemetry; + if input.sidecar_subprocess_enabled { + state.worker = Some(AgentlessWorker::new(worker_config)); + } else { + state.failure = Some(InitializationFailure::SidecarMode); + set_delivery_state(DeliveryState::PermanentError); + } + } + Err(error) => { + state.failure = Some(InitializationFailure::Endpoint(error)); set_delivery_state(DeliveryState::PermanentError); } } @@ -152,12 +217,30 @@ pub extern "C" fn ddog_ffe_runtime_config() -> FfeRuntimeConfig { lifecycle().config } +/// Static, redacted configuration failure. Empty when initialization can retry. +#[no_mangle] +pub extern "C" fn ddog_ffe_initialization_error_reason() -> CharSlice<'static> { + lifecycle() + .failure + .map_or("", InitializationFailure::reason) + .into() +} + +#[no_mangle] +pub extern "C" fn ddog_ffe_initialization_error_message() -> CharSlice<'static> { + lifecycle() + .failure + .map_or("", InitializationFailure::message) + .into() +} + /// First-use compatibility path for PHP providers without an initialize API. /// Concurrent first callers share one deadline; later calls never renew it. #[no_mangle] pub extern "C" fn ddog_ffe_ensure_initialized() { let deadline = { let mut state = lifecycle(); + state.report_diagnostics(); if state.stopped || !state.config.enabled { return; } diff --git a/libdatadog b/libdatadog index 6c44d50813b..f670da53e85 160000 --- a/libdatadog +++ b/libdatadog @@ -1 +1 @@ -Subproject commit 6c44d50813b1a64e1614647c22527871c88774e3 +Subproject commit f670da53e85179c3b3767a67c5a0cf1aa3a3615e diff --git a/src/api/FeatureFlags/Internal/NativeEvaluator.php b/src/api/FeatureFlags/Internal/NativeEvaluator.php index eac444f4d57..9c1a46966b2 100644 --- a/src/api/FeatureFlags/Internal/NativeEvaluator.php +++ b/src/api/FeatureFlags/Internal/NativeEvaluator.php @@ -113,6 +113,9 @@ private function withProviderState($rawResult) } $warningMessage = $providerState['mode'] === 'native_agentless' ? self::AGENTLESS_WARNING_MESSAGE : self::WARNING_MESSAGE; + if (!empty($providerState['errorMessage'])) { + $warningMessage = $providerState['errorMessage']; + } if (is_array($rawResult)) { if (isset($rawResult['provider_state']) && is_array($rawResult['provider_state'])) { diff --git a/tests/ext/ffe/agentless_settings_diagnostics.phpt b/tests/ext/ffe/agentless_settings_diagnostics.phpt new file mode 100644 index 00000000000..ac81f81e6bb --- /dev/null +++ b/tests/ext/ffe/agentless_settings_diagnostics.phpt @@ -0,0 +1,81 @@ +--TEST-- +FFE reports redacted configuration failures and once-only diagnostics on first use +--SKIPIF-- + +--ENV-- +DD_TRACE_ENABLED=0 +DD_TRACE_CLI_ENABLED=1 +DD_REMOTE_CONFIG_ENABLED=0 +DD_INSTRUMENTATION_TELEMETRY_ENABLED=0 +DD_METRICS_OTEL_ENABLED=0 +DD_TRACE_STARTUP_LOGS=0 +DD_TRACE_AGENT_URL=http://127.0.0.1:1 +--FILE-- + array(array(), 'missing_api_key', array('requires DD_API_KEY')), + 'source' => array(array('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE' => 'secret-source'), 'invalid_source', array('configuration source is invalid')), + 'endpoint' => array(array('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL' => 'ftp://secret-user:secret-password@example.com/path?key=secret-query'), 'invalid_endpoint', array('CDN base URL must')), + 'site' => array(array('DD_API_KEY' => 'secret-key', 'DD_SITE' => 'secret-site/path'), 'invalid_site', array('requires a valid DD_SITE')), + 'timings' => array(array('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE' => 'offline', 'DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS' => '0', 'DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS' => '-1', 'DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS' => '0'), 'disabled', array('using 30 seconds', 'using 5 seconds', 'using 10000 milliseconds')), + 'legacy' => array(array('DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED' => '0'), 'disabled', array('DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED is deprecated')), +); +foreach ($cases as $name => $case) { + $log = tempnam(sys_get_temp_dir(), 'ffe-settings-'); + $environment = array_merge($base, $case[0], array('DD_TRACE_LOG_FILE' => $log)); + $process = ffeTestProcess(__DIR__ . '/stubs/agentless_settings_app.php', $pipes, $environment); + $stdout = $stderr = ''; + $deadline = microtime(true) + 60; + do { + $stdout .= ffeTestProcessRead($pipes[1]); + if (strpos($stdout, "fixture_child_ready\n") !== false) break; + usleep(10000); + } while (microtime(true) < $deadline); + if (strpos($stdout, "fixture_child_ready\n") === false) throw new RuntimeException('child startup timed out'); + foreach ($case[2] as $message) { + if (strpos(file_get_contents($log), $message) !== false) throw new RuntimeException('diagnostic emitted before feature use'); + } + fwrite($pipes[0], "evaluate\n"); + fclose($pipes[0]); + do { + $stdout .= ffeTestProcessRead($pipes[1]); + $stderr .= ffeTestProcessRead($pipes[2]); + $status = proc_get_status($process); + if (!$status['running']) break; + usleep(10000); + } while (microtime(true) < $deadline); + $stdout .= ffeTestProcessRead($pipes[1]); + if ($status['running']) proc_terminate($process); + fclose($pipes[1]); + fclose($pipes[2]); + proc_close($process); + if ($status['running'] || $status['exitcode'] !== 0) throw new RuntimeException($name . ' failed: ' . $stderr); + $result = json_decode(str_replace("fixture_child_ready\n", '', $stdout), true); + if ($result['state']['reason'] !== $case[1] || $result['value'] !== 'fallback') throw new RuntimeException($name . ' wrong result: ' . $stdout); + if ($case[1] !== 'disabled' && $result['error'] !== $result['state']['errorMessage']) throw new RuntimeException('caller lost the failure reason'); + $diagnostics = file_get_contents($log); + unlink($log); + foreach ($case[2] as $message) { + if (substr_count($diagnostics, $message) !== 1) throw new RuntimeException($name . ' diagnostic missing or repeated: ' . $diagnostics); + } + if (strpos($diagnostics . $stdout . $stderr, 'secret-') !== false) throw new RuntimeException('configuration secret leaked'); + echo $name, "=ok\n"; +} +?> +--EXPECT-- +missing_key=ok +source=ok +endpoint=ok +site=ok +timings=ok +legacy=ok diff --git a/tests/ext/ffe/agentless_source_selection.phpt b/tests/ext/ffe/agentless_source_selection.phpt index e86b7e05dae..2011337ccc1 100644 --- a/tests/ext/ffe/agentless_source_selection.phpt +++ b/tests/ext/ffe/agentless_source_selection.phpt @@ -15,12 +15,6 @@ DD_METRICS_OTEL_ENABLED=0 DD_TRACE_AGENT_URL=http://127.0.0.1:1 --FILE-- array('0', 'agentless', '1', false, 'native_agentless'), 'legacy_kill' => array(null, null, '0', false, 'native_remote_config'), @@ -28,47 +22,8 @@ $cases = array( 'explicit_remote_config' => array('1', 'remote_config', '0', true, 'native_remote_config'), 'offline' => array('1', 'offline', '1', false, 'native_disabled'), 'invalid' => array('1', 'unknown', '1', false, 'native_disabled'), - 'thread_mode' => array('1', 'agentless', '0', true, 'native_agentless'), ); -$keys = array('DD_FEATURE_FLAGS_ENABLED', 'DD_FEATURE_FLAGS_CONFIGURATION_SOURCE', 'DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED'); -foreach ($cases as $name => $case) { - $environment['DD_TRACE_SIDECAR_CONNECTION_MODE'] = $name === 'thread_mode' ? 'thread' : 'auto'; - foreach ($keys as $i => $key) { - unset($environment[$key]); - if ($case[$i] !== null) $environment[$key] = $case[$i]; - } - $process = ffeTestProcess(__DIR__ . '/stubs/agentless_source_selection_app.php', $pipes, $environment); - if (!is_resource($process)) throw new RuntimeException('cannot launch PHP'); - fclose($pipes[0]); - $stdout = $stderr = ''; - $start = microtime(true); - $readyAt = null; - do { - $stdout .= ffeTestProcessRead($pipes[1]); - $stderr .= ffeTestProcessRead($pipes[2]); - if ($readyAt === null && strpos($stdout, "fixture_child_ready\n") !== false) $readyAt = microtime(true); - $status = proc_get_status($process); - if (!$status['running']) break; - if ($connection = @stream_socket_accept($server, 0.05)) { - fclose($connection); - proc_terminate($process); - throw new RuntimeException($name . ' unexpectedly polled agentless configuration'); - } - } while (microtime(true) - ($readyAt === null ? $start : $readyAt) < ($readyAt === null ? 60 : 3)); - $stdout .= ffeTestProcessRead($pipes[1]); - $stderr .= ffeTestProcessRead($pipes[2]); - if ($status['running']) proc_terminate($process); - fclose($pipes[1]); - fclose($pipes[2]); - proc_close($process); - if ($status['running'] || $status['exitcode'] !== 0) throw new RuntimeException($name . ' failed: ' . $stderr); - $state = json_decode(str_replace("fixture_child_ready\n", '', $stdout), true); - if ($state['enabled'] !== $case[3] || $state['mode'] !== $case[4] || $state['ready']) { - throw new RuntimeException($name . ' wrong source: ' . $stdout); - } - echo $name, "=ok\n"; -} -fclose($server); +require __DIR__ . '/stubs/agentless_source_selection.inc'; ?> --EXPECT-- stable_kill=ok @@ -77,4 +32,3 @@ legacy_remote_config=ok explicit_remote_config=ok offline=ok invalid=ok -thread_mode=ok diff --git a/tests/ext/ffe/agentless_thread_mode.phpt b/tests/ext/ffe/agentless_thread_mode.phpt new file mode 100644 index 00000000000..1fd0c87e9ac --- /dev/null +++ b/tests/ext/ffe/agentless_thread_mode.phpt @@ -0,0 +1,24 @@ +--TEST-- +FFE thread mode fails closed without CDN traffic +--SKIPIF-- + + + +--ENV-- +DD_TRACE_ENABLED=0 +DD_TRACE_CLI_ENABLED=1 +DD_REMOTE_CONFIG_ENABLED=0 +DD_INSTRUMENTATION_TELEMETRY_ENABLED=0 +DD_METRICS_OTEL_ENABLED=0 +DD_TRACE_AGENT_URL=http://127.0.0.1:1 +--FILE-- + array('1', 'agentless', '0', true, 'native_agentless')); +require __DIR__ . '/stubs/agentless_source_selection.inc'; +?> +--EXPECT-- +thread_mode=ok diff --git a/tests/ext/ffe/stubs/agentless_process.inc b/tests/ext/ffe/stubs/agentless_process.inc index d05593541e0..7fc341e32cb 100644 --- a/tests/ext/ffe/stubs/agentless_process.inc +++ b/tests/ext/ffe/stubs/agentless_process.inc @@ -2,7 +2,7 @@ require_once __DIR__ . '/ffe_process.inc'; $instrumented = getenv('USE_ZEND_ALLOC') === '0'; // Keep the no-renewal assertion decisive: ten renewed 2s waits exceed 3s. -if ($instrumented && $scenario !== 'fork') { +if ($instrumented && in_array($scenario, array('recovery', 'shutdown'), true)) { putenv('DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS=2000'); } putenv('PHP_FFE_PROCESS_SCENARIO=' . $scenario); diff --git a/tests/ext/ffe/stubs/agentless_settings_app.php b/tests/ext/ffe/stubs/agentless_settings_app.php new file mode 100644 index 00000000000..920a2088fc7 --- /dev/null +++ b/tests/ext/ffe/stubs/agentless_settings_app.php @@ -0,0 +1,7 @@ +getStringDetails('flag', 'fallback'); +echo json_encode(array('value' => $details->getValue(), 'state' => $details->getProviderState(), 'error' => $details->getErrorMessage())); diff --git a/tests/ext/ffe/stubs/agentless_source_selection.inc b/tests/ext/ffe/stubs/agentless_source_selection.inc new file mode 100644 index 00000000000..31bc3bd5d24 --- /dev/null +++ b/tests/ext/ffe/stubs/agentless_source_selection.inc @@ -0,0 +1,49 @@ + $case) { + $environment['DD_TRACE_SIDECAR_CONNECTION_MODE'] = $name === 'thread_mode' ? 'thread' : 'auto'; + foreach ($keys as $i => $key) { + unset($environment[$key]); + if ($case[$i] !== null) $environment[$key] = $case[$i]; + } + $process = ffeTestProcess(__DIR__ . '/agentless_source_selection_app.php', $pipes, $environment); + if (!is_resource($process)) throw new RuntimeException('cannot launch PHP'); + fclose($pipes[0]); + $stdout = $stderr = ''; + $start = microtime(true); + $readyAt = null; + do { + $stdout .= ffeTestProcessRead($pipes[1]); + $stderr .= ffeTestProcessRead($pipes[2]); + if ($readyAt === null && strpos($stdout, "fixture_child_ready\n") !== false) $readyAt = microtime(true); + $status = proc_get_status($process); + if (!$status['running']) break; + if ($connection = @stream_socket_accept($server, 0.05)) { + fclose($connection); + proc_terminate($process); + throw new RuntimeException($name . ' unexpectedly polled agentless configuration'); + } + } while (microtime(true) - ($readyAt === null ? $start : $readyAt) < ($readyAt === null ? 60 : 3)); + $stdout .= ffeTestProcessRead($pipes[1]); + $stderr .= ffeTestProcessRead($pipes[2]); + if ($status['running']) proc_terminate($process); + fclose($pipes[1]); + fclose($pipes[2]); + proc_close($process); + if ($status['running'] || $status['exitcode'] !== 0) throw new RuntimeException($name . ' failed: ' . $stderr); + $state = json_decode(str_replace("fixture_child_ready\n", '', $stdout), true); + if ($state['enabled'] !== $case[3] || $state['mode'] !== $case[4] || $state['ready']) { + throw new RuntimeException($name . ' wrong source: ' . $stdout); + } + if ($name === 'thread_mode' && $state['reason'] !== 'unsupported_sidecar_mode') { + throw new RuntimeException('thread mode did not report its permanent failure'); + } + echo $name, "=ok\n"; +} +fclose($server); diff --git a/tracer/functions.c b/tracer/functions.c index 094b61d0291..0741a830e20 100644 --- a/tracer/functions.c +++ b/tracer/functions.c @@ -2948,9 +2948,16 @@ PHP_FUNCTION(DDTrace_Internal_ffe_provider_state) { ? "native_agentless" : config.source == DDOG_FFE_CONFIGURATION_SOURCE_REMOTE_CONFIG ? "native_remote_config" : "native_disabled"; add_assoc_string(return_value, "mode", (char *) mode); - add_assoc_string(return_value, "reason", !config.enabled ? "disabled" - : has_config ? (ddog_ffe_delivery_state() == DDOG_DELIVERY_STATE_STALE ? "stale" : "ready") - : "configuration_missing"); + ddog_CharSlice failure_reason = ddog_ffe_initialization_error_reason(); + ddog_CharSlice failure_message = ddog_ffe_initialization_error_message(); + if (!has_config && failure_reason.len) { + add_assoc_stringl(return_value, "reason", (char *)failure_reason.ptr, failure_reason.len); + add_assoc_stringl(return_value, "errorMessage", (char *)failure_message.ptr, failure_message.len); + } else { + add_assoc_string(return_value, "reason", !config.enabled ? "disabled" + : has_config ? (ddog_ffe_delivery_state() == DDOG_DELIVERY_STATE_STALE ? "stale" : "ready") + : "configuration_missing"); + } } From 1e4bc910a173862b769a2c19c27344099731658c Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 14 Oct 2026 18:39:00 +0000 Subject: [PATCH 09/10] Wait for initial configuration before testing sidecar restart --- tests/ext/ffe/stubs/agentless_process_app.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/ext/ffe/stubs/agentless_process_app.php b/tests/ext/ffe/stubs/agentless_process_app.php index c5cd67d9d42..d970379652b 100644 --- a/tests/ext/ffe/stubs/agentless_process_app.php +++ b/tests/ext/ffe/stubs/agentless_process_app.php @@ -16,7 +16,15 @@ } if ($scenario === 'restart') { - if ($first !== 'blue') throw new RuntimeException('initial fetch failed'); + // Establish a last-good snapshot before testing its survival. Instrumented + // subprocess startup may exceed the first-use budget and legitimately + // return the default; initialization timing is covered by recovery/shutdown. + $readyDeadline = microtime(true) + ($instrumented ? 20 : 5); + while ($first !== 'blue' && microtime(true) < $readyDeadline) { + usleep(20000); + $first = $client->getStringValue('flag', 'fallback'); + } + if ($first !== 'blue') throw new RuntimeException('initial configuration unavailable: ' . json_encode(\DDTrace\Internal\ffe_provider_state())); echo "fixture_restart_ready\n"; $retained = false; $deadline = microtime(true) + 12; From 3c453585e5fa240b2bd62c2c157b0d2ab0cc207f Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Thu, 22 Oct 2026 16:31:48 +0000 Subject: [PATCH 10/10] fix(ffe): apply CDN configuration replacements atomically Consume CDN publications through RemoteConfigManager and stage FFE changes until each update batch is drained. Keep accepted flags available during replacements and preserve genuine removals. Pin the sidecar fix that keeps local publications independent of agent directory restarts; cover concurrent evaluation and the native CDN lifecycle. Environment: Datadog workspace --- Cargo.lock | 17 +- Cargo.toml | 2 + components-rs/ffe.rs | 1 + components-rs/ffe/agentless.rs | 74 +++++++-- components-rs/ffe/remote_config.rs | 147 ++++++++++++++++++ components-rs/remote_config.rs | 14 +- components-rs/sidecar.rs | 1 + libdatadog | 2 +- .../ext/ffe/stubs/agentless_lifecycle_app.php | 5 + 9 files changed, 233 insertions(+), 30 deletions(-) create mode 100644 components-rs/ffe/remote_config.rs diff --git a/Cargo.lock b/Cargo.lock index a9902c8472b..ef0fcab5319 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2803,6 +2803,8 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tower-service", + "webpki-root-certs", + "webpki-roots", "windows-sys 0.52.0", ] @@ -3316,6 +3318,7 @@ name = "libdd-tinybytes" version = "1.1.5" dependencies = [ "libdd-tinybytes", + "libdd-trace-model", "once_cell", "pretty_assertions", "proptest", @@ -3325,6 +3328,10 @@ dependencies = [ "test-case", ] +[[package]] +name = "libdd-trace-model" +version = "0.1.0" + [[package]] name = "libdd-trace-normalization" version = "4.2.0" @@ -3404,6 +3411,7 @@ dependencies = [ name = "libdd-trace-utils" version = "14.0.0" dependencies = [ + "allocator-api2", "anyhow", "base64 0.22.1", "bolero", @@ -3428,6 +3436,7 @@ dependencies = [ "libdd-capabilities-impl", "libdd-common", "libdd-tinybytes", + "libdd-trace-model", "libdd-trace-normalization", "libdd-trace-protobuf", "libdd-trace-utils", @@ -6569,18 +6578,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.6" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.6" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] diff --git a/Cargo.toml b/Cargo.toml index ffa59381d97..c4ee26619d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -344,6 +344,8 @@ walkdir = { version = "2.4", default-features = false } wasm-bindgen-futures = { version = "0.4", default-features = false } wasm-bindgen-test = { version = "=0.3.50", default-features = false } web-time = { version = "1", default-features = false } +webpki-roots = { version = "1.0.10", default-features = false } +webpki-root-certs = { version = "1.0.10", default-features = false } windows = { version = "0.59.0", default-features = false } windows-sys = { version = "0.52", default-features = false } winver = { version = "1.0.0", default-features = false } diff --git a/components-rs/ffe.rs b/components-rs/ffe.rs index 67a0ec5562b..3f45f4c1120 100644 --- a/components-rs/ffe.rs +++ b/components-rs/ffe.rs @@ -13,6 +13,7 @@ use std::sync::{Arc, LazyLock}; #[allow(dead_code)] pub(crate) mod agentless; pub(crate) mod lifecycle; +pub(crate) mod remote_config; #[allow(dead_code)] pub(crate) mod settings; diff --git a/components-rs/ffe/agentless.rs b/components-rs/ffe/agentless.rs index 004e5cc1128..a94d592c157 100644 --- a/components-rs/ffe/agentless.rs +++ b/components-rs/ffe/agentless.rs @@ -1,14 +1,20 @@ // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +use super::remote_config::ConfigurationUpdate; use super::settings::AgentlessEndpoint; -use super::{ddog_ffe_has_config, set_delivery_state, store_config, DeliveryState}; +use super::{ddog_ffe_has_config, set_delivery_state, DeliveryState}; use crate::log::{self, Log}; use datadog_ffe::cdn::CdnConfig; -use datadog_ffe::rules_based::{Configuration, UniversalFlagConfig}; +use datadog_ffe::rules_based::UniversalFlagConfig; use datadog_sidecar::service::blocking::{ping, subscribe_cdn_configuration}; -use datadog_sidecar::shm_remote_config::cdn::{CdnDeliveryState, CdnReader}; -use std::ffi::{c_void, CString}; +use datadog_sidecar::shm_remote_config::{ + RemoteConfigDeliveryState, RemoteConfigManager, RemoteConfigUpdate, +}; +use libdd_remote_config::fetch::ConfigInvariants; +use libdd_remote_config::{default_registry, RemoteConfigProduct}; +use std::ffi::c_void; +use std::sync::Arc; use std::thread::JoinHandle; use std::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; @@ -182,21 +188,29 @@ fn read_configuration( let mut transport = crate::sidecar::run_sidecar(sidecar)?; transport.set_read_timeout(Some(Duration::from_millis(200)))?; transport.set_write_timeout(Some(Duration::from_millis(200)))?; - let path = subscribe_cdn_configuration(&mut transport, Some(config.subscription.clone()))? + let handle = subscribe_cdn_configuration(&mut transport, Some(config.subscription.clone()))? .ok_or_else(|| anyhow::anyhow!("CDN subscription unavailable"))?; - let mut reader = CdnReader::new(&CString::new(path)?); - let mut revision = 0; + let mut manager = RemoteConfigManager::new_with_registry( + ConfigInvariants { + language: config.subscription.language.clone(), + tracer_version: config.subscription.version.clone(), + // This manager has only the explicitly attached local source: it never + // tracks an agent target or makes requests to this endpoint. + endpoint: libdd_common::Endpoint::default(), + agentless: None, + }, + Arc::new(default_registry().with::()?), + ); + manager.set_local_configuration(Some(handle))?; let mut heartbeat = Instant::now(); while !cancellation.is_cancelled() { - if let Some(snapshot) = reader.read()? { - if snapshot.revision != revision && !snapshot.configuration.is_empty() { - let config = UniversalFlagConfig::from_json(snapshot.configuration)?; - store_config(Configuration::from_server_response(config)); - revision = snapshot.revision; - } - set_delivery_state(match snapshot.state { - CdnDeliveryState::Ready if ddog_ffe_has_config() => DeliveryState::Ready, - CdnDeliveryState::Stale | CdnDeliveryState::Stopped if ddog_ffe_has_config() => { + apply_configuration_updates(&mut manager)?; + if let Some(state) = manager.local_configuration_state() { + set_delivery_state(match state { + RemoteConfigDeliveryState::Ready if ddog_ffe_has_config() => DeliveryState::Ready, + RemoteConfigDeliveryState::Stale | RemoteConfigDeliveryState::Stopped + if ddog_ffe_has_config() => + { DeliveryState::Stale } _ => DeliveryState::Starting, @@ -208,8 +222,34 @@ fn read_configuration( ping(&mut transport)?; heartbeat = Instant::now(); } - reader.wait_for_change(Duration::from_millis(50)); + manager.wait_for_change(Duration::from_millis(50)); + } + Ok(()) +} + +fn apply_configuration_updates(manager: &mut RemoteConfigManager) -> anyhow::Result<()> { + let mut update = ConfigurationUpdate::default(); + loop { + match manager.fetch_update() { + RemoteConfigUpdate::None => break, + RemoteConfigUpdate::Add { value, .. } + if value.path.product() == RemoteConfigProduct::FfeFlags => + { + let data = value + .data + .ok_or_else(|| anyhow::anyhow!("FFE parser unavailable"))?; + let flags = data + .downcast::() + .ok_or_else(|| anyhow::anyhow!("Unexpected FFE configuration type"))?; + update.add(flags)?; + } + RemoteConfigUpdate::Remove(path) if path.product() == RemoteConfigProduct::FfeFlags => { + update.remove(); + } + _ => {} + } } + update.apply(); Ok(()) } diff --git a/components-rs/ffe/remote_config.rs b/components-rs/ffe/remote_config.rs new file mode 100644 index 00000000000..37257e5e830 --- /dev/null +++ b/components-rs/ffe/remote_config.rs @@ -0,0 +1,147 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use datadog_ffe::rules_based::{Configuration, UniversalFlagConfig}; + +/// Stage a complete RC update batch before changing the evaluator's configuration. +/// RC replaces a path with Remove followed by Add. Publishing the Remove immediately +/// would expose an empty configuration to concurrent CDN evaluations. +#[derive(Default)] +pub(crate) struct ConfigurationUpdate { + pending: Option>, +} + +impl ConfigurationUpdate { + pub(crate) fn add(&mut self, flags: &UniversalFlagConfig) -> anyhow::Result<()> { + // If preparing the replacement fails, retain the installed configuration. + self.pending = None; + let flags = UniversalFlagConfig::from_json(flags.to_json().to_vec())?; + self.pending = Some(Some(Configuration::from_server_response(flags))); + Ok(()) + } + + pub(crate) fn remove(&mut self) { + self.pending = Some(None); + } + + /// Apply only after the manager has returned None. Dropping an unfinished batch + /// leaves the installed configuration untouched. + pub(crate) fn apply(self) { + match self.pending { + Some(Some(config)) => { + super::store_config(config); + } + Some(None) => { + super::clear_config(); + } + None => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ffe::{self, DeliveryState}; + use datadog_ffe::rules_based::{ + get_assignment, AssignmentValue, EvaluationContext, EvaluationError, ExpectedFlagType, + }; + use std::collections::HashMap; + use std::sync::Arc; + + struct SharedConfiguration; + + impl SharedConfiguration { + fn new() -> Self { + ffe::use_agentless_config(true); + ffe::clear_config(); + Self + } + } + + impl Drop for SharedConfiguration { + fn drop(&mut self) { + ffe::clear_config(); + ffe::use_agentless_config(false); + } + } + + fn flags(value: &str) -> UniversalFlagConfig { + UniversalFlagConfig::from_json( + serde_json::to_vec(&serde_json::json!({ + "createdAt": "2026-10-21T00:00:00Z", "environment": {"name": "test"}, + "flags": {"flag": { + "key": "flag", "enabled": true, "variationType": "STRING", + "variations": {"v": {"key": "v", "value": value}}, + "allocations": [{"key": "static", "splits": [{"variationKey": "v", "shards": []}]}] + }} + })) + .unwrap(), + ) + .unwrap() + } + + fn evaluate_on_another_thread() -> Result { + std::thread::spawn(|| { + let configuration = ffe::current_config(); + get_assignment( + configuration.as_deref(), + "flag", + &EvaluationContext::new(None, Arc::new(HashMap::new())), + ExpectedFlagType::String, + chrono::Utc::now(), + ) + .map(|assignment| match assignment.value { + AssignmentValue::String(value) => value.as_str().to_owned(), + other => panic!("unexpected assignment {other:?}"), + }) + }) + .join() + .unwrap() + } + + fn install(value: &str) { + let mut update = ConfigurationUpdate::default(); + update.add(&flags(value)).unwrap(); + update.apply(); + } + + #[test] + fn cdn_replacement_keeps_evaluations_available_until_atomic_publication() { + let _lock = ffe::tests::FFE_TEST_LOCK.lock().unwrap(); + let _configuration = SharedConfiguration::new(); + install("old"); + let version = ffe::ddog_ffe_config_version(); + + let mut update = ConfigurationUpdate::default(); + update.remove(); + assert_eq!(evaluate_on_another_thread().unwrap(), "old"); + update.add(&flags("new")).unwrap(); + assert_eq!(evaluate_on_another_thread().unwrap(), "old"); + assert_eq!(ffe::ddog_ffe_config_version(), version); + update.apply(); + assert_eq!(evaluate_on_another_thread().unwrap(), "new"); + assert_eq!(ffe::ddog_ffe_config_version(), version + 1); + assert_eq!(ffe::delivery_state(), DeliveryState::Ready); + } + + #[test] + fn a_completed_removal_clears_configuration_but_an_abandoned_batch_does_not() { + let _lock = ffe::tests::FFE_TEST_LOCK.lock().unwrap(); + let _configuration = SharedConfiguration::new(); + install("old"); + let mut abandoned = ConfigurationUpdate::default(); + abandoned.remove(); + drop(abandoned); + assert_eq!(evaluate_on_another_thread().unwrap(), "old"); + + let mut removal = ConfigurationUpdate::default(); + removal.remove(); + removal.apply(); + assert_eq!( + evaluate_on_another_thread(), + Err(EvaluationError::ConfigurationMissing) + ); + assert_eq!(ffe::delivery_state(), DeliveryState::Stale); + } +} diff --git a/components-rs/remote_config.rs b/components-rs/remote_config.rs index 299c1d2bd80..9c961d9eba9 100644 --- a/components-rs/remote_config.rs +++ b/components-rs/remote_config.rs @@ -1,6 +1,6 @@ use crate::bytes::{dangling_zend_string, OwnedZendString, ZendString}; use crate::sidecar::MaybeShmLimiter; -use datadog_ffe::rules_based::{Configuration, UniversalFlagConfig}; +use datadog_ffe::rules_based::UniversalFlagConfig; use datadog_sidecar::service::blocking::SidecarTransport; use datadog_sidecar::service::InstanceId; use datadog_sidecar::shm_remote_config::{RemoteConfigManager, RemoteConfigUpdate}; @@ -423,6 +423,7 @@ pub extern "C" fn ddog_remote_config_get_path(remote_config: &RemoteConfigState) #[no_mangle] pub extern "C" fn ddog_process_remote_configs(remote_config: &mut RemoteConfigState) -> bool { let mut has_updates = false; + let mut ffe_update = crate::ffe::remote_config::ConfigurationUpdate::default(); loop { match remote_config.manager.fetch_update() { RemoteConfigUpdate::None => break, @@ -479,12 +480,8 @@ pub extern "C" fn ddog_process_remote_configs(remote_config: &mut RemoteConfigSt RemoteConfigProduct::FfeFlags => { debug!("Received FFE flags configuration"); if let Some(ufc) = data.downcast::() { - if let Ok(ufc_owned) = - UniversalFlagConfig::from_json(ufc.to_json().to_vec()) - { - crate::ffe::store_config(Configuration::from_server_response( - ufc_owned, - )); + if let Err(error) = ffe_update.add(ufc) { + debug!("Failed preparing FFE configuration: {error}"); } } } @@ -524,13 +521,14 @@ pub extern "C" fn ddog_process_remote_configs(remote_config: &mut RemoteConfigSt } RemoteConfigProduct::FfeFlags => { debug!("FFE flags configuration removed"); - crate::ffe::clear_config(); + ffe_update.remove(); } _ => (), }, } has_updates = true } + ffe_update.apply(); has_updates } diff --git a/components-rs/sidecar.rs b/components-rs/sidecar.rs index 8c7b3b22e1f..0be514b9476 100644 --- a/components-rs/sidecar.rs +++ b/components-rs/sidecar.rs @@ -338,6 +338,7 @@ pub extern "C" fn ddog_sidecar_reconnect_readers( /// The fd of the transport's current connection; it changes on reconnect. #[cfg(unix)] #[no_mangle] +#[allow(clippy::borrowed_box)] // The C ABI takes a pointer to the transport handle. pub extern "C" fn ddog_sidecar_transport_raw_fd(transport: &Box) -> i32 { transport.as_raw_fd() } diff --git a/libdatadog b/libdatadog index f670da53e85..765058a0f67 160000 --- a/libdatadog +++ b/libdatadog @@ -1 +1 @@ -Subproject commit f670da53e85179c3b3767a67c5a0cf1aa3a3615e +Subproject commit 765058a0f6747709dba2edcb990e2a517f315f07 diff --git a/tests/ext/ffe/stubs/agentless_lifecycle_app.php b/tests/ext/ffe/stubs/agentless_lifecycle_app.php index ee38c5d9c9e..5baf38e7699 100644 --- a/tests/ext/ffe/stubs/agentless_lifecycle_app.php +++ b/tests/ext/ffe/stubs/agentless_lifecycle_app.php @@ -21,6 +21,11 @@ $value = $client->getStringValue('flag', 'fallback'); $state = \DDTrace\Internal\ffe_provider_state(); $after = \DDTrace\ffe_config_version(); + // A concurrent replacement may change the version, but must never expose + // an empty configuration between the old and new accepted values. + if ($value !== 'blue' && $value !== 'green') { + throw new RuntimeException('configuration missing during CDN replacement'); + } // These APIs are independent reads, not an atomic snapshot. A poll can // publish green after the blue evaluation and before the version read. if ($before !== $after) continue;