diff --git a/.cargo/config.toml b/.cargo/config.toml index 1302091e0..f8fcfadaf 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -64,6 +64,13 @@ run_cli_macos = "run --package trusted-server-cli --target aarch64-apple-darwin test_cli_linux = "test --package trusted-server-cli --target x86_64-unknown-linux-gnu" test_cli_macos = "test --package trusted-server-cli --target aarch64-apple-darwin" +# --- Host-target lint gates that no adapter alias covers --- +# CI lints these two crates explicitly (see .github/workflows/format.yml), but +# pins the Linux triple, so there was no command a developer could run locally +# to reproduce them. These omit --target and therefore build for the host. +clippy-cli = "clippy -p trusted-server-cli --all-targets --all-features -- -D warnings" +clippy-codegen = "clippy -p trusted-server-openrtb-codegen --all-targets -- -D warnings" + # When a wasm binary IS built, run it under Viceroy. [target.'cfg(all(target_arch = "wasm32"))'] runner = "viceroy run -C ../../fastly.toml -- " diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 97402e6f4..1f1bbe27a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -231,9 +231,14 @@ jobs: run: | cargo clippy --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" --all-targets -- -D warnings + - name: Set up Chrome for browser fixture tests + id: setup-chrome + uses: browser-actions/setup-chrome@v1 + - name: cargo test - run: | - cargo test --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" + run: ./scripts/test-cli.sh + env: + CHROME: ${{ steps.setup-chrome.outputs.chrome-path }} test-typescript: name: vitest diff --git a/.gitignore b/.gitignore index 24b9e06aa..96ffa2a5c 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,4 @@ src/*.html # leftover local build artifacts (node_modules, target, dist) that remain on disk. /crates/js/ /crates/integration-tests/ +wrangler.integration.generated.toml diff --git a/CLAUDE.md b/CLAUDE.md index 546a3bf52..0447da1cd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -120,6 +120,12 @@ cargo clippy-cloudflare-wasm cargo clippy-spin-native cargo clippy-spin-wasm +# The CLI and the OpenRTB codegen crate are host-target members that no adapter +# alias covers. CI lints both with the Linux triple pinned; these aliases omit +# `--target` so they reproduce it on any host. +cargo clippy-cli +cargo clippy-codegen + # Check compilation (per-target aliases — bare `cargo check` fails at the workspace root) cargo check-fastly && cargo check-axum && cargo check-cloudflare @@ -336,7 +342,7 @@ IntegrationRegistration::builder(ID) Every PR must pass: 1. `cargo fmt --all -- --check` -2. `cargo clippy-fastly && cargo clippy-axum && cargo clippy-cloudflare && cargo clippy-cloudflare-wasm && cargo clippy-spin-native && cargo clippy-spin-wasm` +2. `cargo clippy-fastly && cargo clippy-axum && cargo clippy-cloudflare && cargo clippy-cloudflare-wasm && cargo clippy-spin-native && cargo clippy-spin-wasm && cargo clippy-cli && cargo clippy-codegen` 3. `cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin` 4. `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity` 5. JS build and test (`cd crates/trusted-server-js/lib && npx vitest run`) diff --git a/Cargo.lock b/Cargo.lock index 311597aae..f79bc24b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -787,7 +787,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -5396,6 +5396,7 @@ dependencies = [ "futures", "log", "log-fastly", + "rand 0.8.6", "serde", "serde_json", "toml", @@ -5437,8 +5438,11 @@ dependencies = [ "derive_more", "directories", "edgezero-cli", + "edgezero-core", "error-stack", "futures", + "glob", + "http", "http-body-util", "hyper", "hyper-util", @@ -5451,12 +5455,15 @@ dependencies = [ "scraper", "serde", "serde_json", + "similar", + "temp-env", "tempfile", "time", "tokio", "tokio-rustls", "toml", "toml_edit 0.23.10+spec-1.0.0", + "tracing", "trusted-server-core", "url", "webpki-roots", @@ -6014,7 +6021,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index dfd94d0c4..ac0cac621 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,6 +97,7 @@ scraper = "0.24.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.149" sha2 = "0.10.9" +similar = "2.7" simple_logger = "5" spin-sdk = { version = "~6.0", default-features = false, features = ["http", "key-value", "variables"] } subtle = "2.6" @@ -109,6 +110,7 @@ tokio-rustls = "0.26" toml = "1.1" toml_edit = "0.23.10" tower = "0.4" +tracing = "0.1" trusted-server-core = { path = "crates/trusted-server-core" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } diff --git a/README.md b/README.md index c606b340a..0ad5b0351 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ ts config init ts config validate # Audit a public page with Chrome/Chromium to bootstrap a draft config -ts audit https://publisher.example +ts audit generate https://publisher.example # Run tests (Fastly/WASM crates — requires Viceroy) cargo test-fastly diff --git a/crates/trusted-server-adapter-axum/Cargo.toml b/crates/trusted-server-adapter-axum/Cargo.toml index 15b6ee59d..09e8c77d2 100644 --- a/crates/trusted-server-adapter-axum/Cargo.toml +++ b/crates/trusted-server-adapter-axum/Cargo.toml @@ -20,6 +20,7 @@ path = "src/main.rs" [dependencies] async-trait = { workspace = true } +axum = { workspace = true } edgezero-adapter-axum = { workspace = true, features = ["axum"] } edgezero-core = { workspace = true } error-stack = { workspace = true } @@ -27,12 +28,11 @@ futures = { workspace = true } log = { workspace = true } reqwest = { workspace = true } simple_logger = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "sync", "time"] } +tower = { workspace = true, features = ["util"] } trusted-server-core = { workspace = true } [dev-dependencies] -axum = { workspace = true } base64 = { workspace = true } temp-env = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } -tower = { workspace = true, features = ["util"] } diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 38776eb95..fcf8bf98f 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -1,6 +1,7 @@ use core::future::Future; use std::sync::Arc; +use edgezero_adapter_axum::service::EdgeZeroAxumService; use edgezero_core::app::Hooks; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; @@ -574,15 +575,7 @@ impl Hooks for TrustedServerApp { } fn routes() -> RouterService { - let state = match build_state() { - Ok(s) => s, - Err(ref e) => { - log::error!("failed to build application state: {:?}", e); - return startup_error_router(e); - } - }; - - build_router(&state) + Self::routes_with_server_timing_flag().0 } } @@ -603,6 +596,44 @@ impl TrustedServerApp { let state = build_state_with_settings(settings)?; Ok(build_router(&state)) } + + /// The dev server's fully configured tower service: the application + /// router wrapped in the terminal timing layer + /// ([`crate::timing::TimingService`]), with `server_timing_enabled` + /// read from the same settings snapshot that built the router. + /// + /// This is the standard construction path for serving this adapter. + /// [`Hooks::routes`] satisfies the `Hooks` trait contract and returns + /// the bare router without the timing layer; callers who serve traffic + /// should use this instead so `server_timing_enabled` is never + /// silently discarded. + #[must_use] + pub fn dev_server_service() -> crate::timing::TimingService { + let (router, server_timing_enabled) = Self::routes_with_server_timing_flag(); + crate::timing::TimingService::new(EdgeZeroAxumService::new(router), server_timing_enabled) + } + + /// Build the router alongside whether `Server-Timing` emission is + /// enabled, read from the same settings snapshot used to build the + /// router. + /// + /// The Axum dev server's terminal timing layer ([`crate::timing`]) needs + /// this flag once at startup: unlike the Fastly adapter, which rebuilds + /// `Settings` per request, the Axum dev server builds its application + /// state once and reuses the same [`RouterService`] for every request. + #[must_use] + fn routes_with_server_timing_flag() -> (RouterService, bool) { + let state = match build_state() { + Ok(s) => s, + Err(ref e) => { + log::error!("failed to build application state: {:?}", e); + return (startup_error_router(e), false); + } + }; + + let server_timing_enabled = state.settings.observability.server_timing_enabled; + (build_router(&state), server_timing_enabled) + } } fn build_router(state: &Arc) -> RouterService { diff --git a/crates/trusted-server-adapter-axum/src/lib.rs b/crates/trusted-server-adapter-axum/src/lib.rs index 2f15e566d..b1d4c3dd8 100644 --- a/crates/trusted-server-adapter-axum/src/lib.rs +++ b/crates/trusted-server-adapter-axum/src/lib.rs @@ -10,3 +10,6 @@ pub mod app; pub mod middleware; /// Platform-trait implementations backed by env vars and `reqwest`. pub mod platform; +/// Terminal timing layer wrapping the Axum dev server's tower `Service` +/// boundary with the request-phase `Server-Timing` freeze point. +pub mod timing; diff --git a/crates/trusted-server-adapter-axum/src/main.rs b/crates/trusted-server-adapter-axum/src/main.rs index 960982176..4e360ea41 100644 --- a/crates/trusted-server-adapter-axum/src/main.rs +++ b/crates/trusted-server-adapter-axum/src/main.rs @@ -1,6 +1,15 @@ -use edgezero_adapter_axum::dev_server::{AxumDevServer, AxumDevServerConfig}; -use edgezero_core::app::Hooks as _; +use std::net::SocketAddr; + +use axum::Router; +use edgezero_adapter_axum::dev_server::AxumDevServerConfig; +use edgezero_adapter_axum::service::EdgeZeroAxumService; +use tokio::net::TcpListener; +use tokio::runtime::Builder as RuntimeBuilder; +use tokio::signal; +use tower::Service as _; +use tower::service_fn; use trusted_server_adapter_axum::app::TrustedServerApp; +use trusted_server_adapter_axum::timing::TimingService; #[allow(clippy::print_stderr)] fn main() { @@ -20,13 +29,63 @@ fn main() { }; log::info!("Listening on http://{}", config.addr); - let router = TrustedServerApp::routes(); - if let Err(err) = AxumDevServer::with_config(router, config).run() { + let service = TrustedServerApp::dev_server_service(); + if let Err(err) = run(service, config) { log::error!("trusted-server-adapter-axum failed: {err}"); std::process::exit(1); } } +/// Runs the Axum dev server with the request-phase timing terminal layer +/// ([`trusted_server_adapter_axum::timing::TimingService`]) wrapped around +/// `EdgeZeroAxumService`, ahead of `axum::serve`. +/// +/// This does not use `edgezero_adapter_axum::dev_server::AxumDevServer::run`: +/// that helper only accepts a bare [`RouterService`] and builds its own +/// `EdgeZeroAxumService` and `axum::Router` internally, with no seam for an +/// outer service wrapper. Router-generated 404/405 responses bypass +/// `RouterBuilder::middleware` (see `trusted_server_adapter_axum::timing`), +/// so the freeze point has to wrap the tower `Service` boundary itself. +/// Driving `axum::serve` directly here mirrors that helper's own internal +/// bind/wrap/serve/shutdown sequence closely enough to keep behavior +/// identical for callers (`PORT` env var, ctrl-c graceful shutdown). +/// +/// # Errors +/// +/// Returns an error if the Tokio runtime fails to start, the listener fails +/// to bind, or the underlying serve loop errors. +fn run( + service: TimingService, + config: AxumDevServerConfig, +) -> std::io::Result<()> { + let runtime = RuntimeBuilder::new_multi_thread().enable_all().build()?; + runtime.block_on(serve(service, config)) +} + +async fn serve( + service: TimingService, + config: AxumDevServerConfig, +) -> std::io::Result<()> { + let listener = TcpListener::bind(config.addr).await?; + + let axum_router = Router::new().fallback_service(service_fn(move |req| { + let mut svc = service.clone(); + async move { svc.call(req).await } + })); + let make_service = axum_router.into_make_service_with_connect_info::(); + + let server = axum::serve(listener, make_service); + if config.enable_ctrl_c { + server + .with_graceful_shutdown(async { + let _ctrl_c = signal::ctrl_c().await; + }) + .await + } else { + server.await + } +} + /// Read a port number from the `PORT` environment variable. /// /// Returns `None` when the variable is unset. Exits non-zero if the value diff --git a/crates/trusted-server-adapter-axum/src/timing.rs b/crates/trusted-server-adapter-axum/src/timing.rs new file mode 100644 index 000000000..832823c15 --- /dev/null +++ b/crates/trusted-server-adapter-axum/src/timing.rs @@ -0,0 +1,242 @@ +//! Terminal timing layer for the Axum dev server. +//! +//! [`TimingService`](crate::timing::TimingService) wraps the tower `Service` +//! boundary the Axum dev server's router sits behind: it creates a +//! [`RequestTimings`](trusted_server_core::request_timing::RequestTimings) +//! collector per request, threads it through request extensions so +//! downstream core handlers can record into it, and on the way back stamps +//! `mark_headers_ready` and appends the `Server-Timing` header via +//! [`append_server_timing_if_private`](trusted_server_core::request_timing::append_server_timing_if_private). +//! +//! This wraps *outside* `RouterService` rather than registering as +//! `RouterBuilder::middleware`. A router-generated 404/405 short-circuits +//! `RouterInner::dispatch` before its middleware chain ever runs, so +//! middleware never sees those responses. By the time a response reaches +//! this layer -- after `RouterService::oneshot` inside +//! `EdgeZeroAxumService::call` has already converted any dispatch error into +//! a plain response -- every response is covered uniformly, router-generated +//! or not. +//! +//! `/health` is excluded by path match before a +//! [`RequestTimings`](trusted_server_core::request_timing::RequestTimings) +//! collector is even created: health checks never carry timing data on any +//! adapter. +//! +//! Unlike the Fastly adapter (state built per request, adding +//! `Phase::AppBuild` to the rendered header), the Axum dev server builds its +//! application state once at startup. There is no per-request app-build +//! interval to measure, so `ts-appbuild` never appears in the header here. + +use std::convert::Infallible; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use axum::body::Body as AxumBody; +use axum::http::{Request, Response}; +use tower::Service; +use trusted_server_core::request_timing::{RequestTimings, append_server_timing_if_private}; + +/// Path excluded from timing collection and `Server-Timing` emission: health +/// checks never carry timing data on any adapter. +const HEALTH_PATH: &str = "/health"; + +/// Wraps an inner Axum tower service with the request-phase timing freeze +/// point described in the module docs. +#[derive(Clone)] +pub struct TimingService { + inner: S, + server_timing_enabled: bool, +} + +impl TimingService { + /// Wraps `inner`, appending `Server-Timing` when `server_timing_enabled` + /// is set and the response is conclusively private. + #[must_use] + pub fn new(inner: S, server_timing_enabled: bool) -> Self { + Self { + inner, + server_timing_enabled, + } + } +} + +impl Service> for TimingService +where + S: Service, Response = Response, Error = Infallible> + + Clone + + Send + + 'static, + S::Future: Send + 'static, +{ + type Error = Infallible; + type Future = Pin> + Send>>; + type Response = Response; + + fn call(&mut self, mut req: Request) -> Self::Future { + let mut inner = self.inner.clone(); + + // Excluded before a collector is even created: `/health` never + // carries timing data, on any adapter. + if req.uri().path() == HEALTH_PATH { + return Box::pin(async move { inner.call(req).await }); + } + + let server_timing_enabled = self.server_timing_enabled; + let timings = RequestTimings::new(); + req.extensions_mut().insert(timings.clone()); + + Box::pin(async move { + let mut response = inner.call(req).await?; + append_server_timing_if_private(&mut response, &timings, server_timing_enabled); + Ok(response) + }) + } + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::header::CACHE_CONTROL; + use axum::http::{HeaderValue, StatusCode}; + use edgezero_adapter_axum::service::EdgeZeroAxumService; + use edgezero_core::body::Body as EdgeBody; + use edgezero_core::context::RequestContext; + use edgezero_core::error::EdgeError; + use edgezero_core::http::response_builder; + use edgezero_core::router::RouterService; + use tower::{ServiceExt as _, service_fn}; + + /// Builds a private (`cache-control: private, no-store`) response for a + /// handler under test. + fn private_ok_response() -> Result { + Ok(response_builder() + .status(StatusCode::OK) + .header("cache-control", "private, no-store") + .body(EdgeBody::from("ok")) + .expect("should build a private response fixture")) + } + + /// Reads a response header as a UTF-8 string, or `None` if absent. + fn header(response: &Response, name: &str) -> Option { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .map(ToOwned::to_owned) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn axum_emits_header_on_private_response() { + let router = RouterService::builder() + .get("/private", |_ctx: RequestContext| async { + private_ok_response() + }) + .build(); + let mut service = TimingService::new(EdgeZeroAxumService::new(router), true); + + let request = Request::builder() + .uri("/private") + .body(AxumBody::empty()) + .expect("should build request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should not fail"); + + let server_timing = header(&response, "server-timing").expect("should emit header"); + assert!( + server_timing.contains("ts-total;dur="), + "should carry the collected total: {server_timing}" + ); + assert!( + !server_timing.contains("ts-appbuild"), + "the Axum dev server builds state once at startup, so there is no \ + per-request app-build interval to render: {server_timing}" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn axum_404_carries_header_when_private() { + // An empty router has no routes at all, so any path dispatches + // through `RouterInner::dispatch`'s `NotFound` branch -- exactly the + // path that bypasses `RouterBuilder::middleware`. The router's own + // `EdgeError::into_response` does not attach `Cache-Control`, so a + // small wrapping service forces the response private here, standing + // in for whatever upstream layer would normally mark a genuinely + // private 404. This proves the freeze point still runs for a + // router-generated response without weakening + // `append_server_timing_if_private`'s real gating logic. + let empty_router = RouterService::builder().build(); + let inner = EdgeZeroAxumService::new(empty_router); + let force_private = service_fn(move |req: Request| { + let mut svc = inner.clone(); + async move { + let mut response = svc.call(req).await?; + response + .headers_mut() + .insert(CACHE_CONTROL, HeaderValue::from_static("private, no-store")); + Ok::<_, Infallible>(response) + } + }); + let mut service = TimingService::new(force_private, true); + + let request = Request::builder() + .uri("/does-not-exist") + .body(AxumBody::empty()) + .expect("should build request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should not fail"); + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should still be the router's own not-found response" + ); + let server_timing = header(&response, "server-timing") + .expect("a router-generated 404 must still carry the header when private"); + assert!( + server_timing.contains("ts-total;dur="), + "should carry the collected total: {server_timing}" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn axum_health_is_excluded() { + let router = RouterService::builder() + .get("/health", |_ctx: RequestContext| async { + private_ok_response() + }) + .build(); + let mut service = TimingService::new(EdgeZeroAxumService::new(router), true); + + let request = Request::builder() + .uri("/health") + .body(AxumBody::empty()) + .expect("should build request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should not fail"); + + assert!( + header(&response, "server-timing").is_none(), + "/health must never carry a server-timing header" + ); + } +} diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 65320faa6..584085e76 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -28,6 +28,7 @@ log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } trusted-server-core = { workspace = true } +rand = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 190be505c..c5e8bf9d4 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -100,6 +100,7 @@ use edgezero_core::http::{ }; use edgezero_core::router::RouterService; use error_stack::Report; +use trusted_server_core::access_telemetry::{RouteClass, RouteMetadata, publisher_route_template}; use trusted_server_core::auction::AuctionTelemetrySink; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ @@ -119,13 +120,14 @@ use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; +use trusted_server_core::geo::GeoLookupState; use trusted_server_core::http_util::is_navigation_request; use trusted_server_core::integrations::{ IntegrationRegistry, ProxyDispatchInput, RequestFilterEffects, RequestFilterRegistryInput, RequestFilterRegistryOutcome, }; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, PlatformKvStore, RuntimeServices, StoreName, + ClientInfo, GeoInfo, PlatformKvStore, RuntimeServices, StoreName, TimedKvStore, }; use trusted_server_core::proxy::{ AssetProxyCachePolicy, handle_asset_proxy_request, handle_first_party_click, @@ -140,8 +142,11 @@ use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, handle_verify_signature, }; +use trusted_server_core::request_timing::{Phase, RequestTimings}; use trusted_server_core::settings::{ProxyAssetRoute, Settings}; -use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; +use trusted_server_core::settings_data::{ + DEFAULT_CONFIG_STORE_ID, config_key, config_store_name, get_settings_from_config_store, +}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; @@ -164,8 +169,8 @@ pub(crate) struct RuntimeStoreConfig { impl RuntimeStoreConfig { pub(crate) fn from_env(env: &EnvConfig) -> Self { Self { - config_store_name: StoreName::from(env.store_name("config", DEFAULT_CONFIG_STORE_ID)), - config_key: env.store_key("config", DEFAULT_CONFIG_STORE_ID), + config_store_name: config_store_name(env), + config_key: config_key(env), secret_store_name: StoreName::from(env.store_name("secrets", DEFAULT_SECRET_STORE_ID)), } } @@ -252,13 +257,18 @@ fn warn_if_certificate_check_disabled(settings: &Settings) { pub(crate) fn runtime_services_for_consent_route( settings: &Settings, runtime_services: &RuntimeServices, + timings: &RequestTimings, ) -> Result> { let Some(store_name) = settings.consent.consent_store.as_deref() else { return Ok(runtime_services.clone()); }; open_kv_store(store_name) - .map(|store| runtime_services.clone().with_kv_store(store)) + .map(|store| { + let timed_store = + Arc::new(TimedKvStore::new(store, timings.clone())) as Arc; + runtime_services.clone().with_kv_store(timed_store) + }) .map_err(|e| { Report::new(TrustedServerError::KvStore { store_name: store_name.to_string(), @@ -326,6 +336,12 @@ fn uses_dynamic_tsjs_fallback(method: &Method, path: &str) -> bool { *method == Method::GET && path.starts_with("/static/tsjs=") } +/// Coarse route template for every `tsjs` bundle request, used as the +/// `route_template` in the [`RouteMetadata`] attached by the tsjs branch of +/// [`dispatch_fallback`]. Actual filenames vary by module/hash; the prefix +/// alone is the route identity that matters for access telemetry. +const TSJS_ROUTE_TEMPLATE: &str = "/static/tsjs=*"; + // --------------------------------------------------------------------------- // EC request state // --------------------------------------------------------------------------- @@ -387,6 +403,17 @@ impl EcRequestState { services: self.services, } } + + /// Derives the carried [`GeoLookupState`] from this request's geo lookup + /// outcome, so response-phase finalize can reuse it instead of repeating + /// the lookup. `build_ec_request_state` always attempts the lookup, so + /// `None` here means the lookup ran and failed, not that it was skipped. + fn geo_lookup_state(&self) -> GeoLookupState { + match &self.geo_info { + Some(info) => GeoLookupState::Resolved(info.clone()), + None => GeoLookupState::Attempted, + } + } } /// Derives device signals from the request's `User-Agent` header. @@ -442,13 +469,21 @@ fn build_ec_request_state( let eids_cookie = crate::extract_cookie_value(req, COOKIE_TS_EIDS); let sharedid_cookie = crate::extract_cookie_value(req, COOKIE_SHAREDID); - let geo_info = services - .geo() - .lookup(services.client_info().client_ip) - .unwrap_or_else(|e| { - log::warn!("geo lookup failed during EC setup: {e}"); - None - }); + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let geo_info = { + let _span = timings.span(Phase::Geo); + services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed during EC setup: {e}"); + None + }) + }; let (ec_context, setup_error) = match EcContext::read_from_request_with_geo(settings, req, services, geo_info.as_ref()) { @@ -469,7 +504,7 @@ fn build_ec_request_state( // Bot gate: suppress KV-backed EC writes for unrecognized clients, except // consent withdrawals. Revocations keep the write path so tombstones stay // authoritative even for privacy-extension-heavy clients. - let kv_graph = crate::maybe_identity_graph(settings); + let kv_graph = crate::identity_graph_with_timing(settings, &timings); let finalize_kv_graph = if setup_error.is_none() && (is_real_browser || ec_consent_withdrawn(ec_context.consent())) { @@ -521,6 +556,18 @@ async fn run_pre_route_filters( req: &mut Request, geo_info: Option<&GeoInfo>, ) -> PreRoute { + // Only recorded when a filter is actually registered, so unconfigured + // deployments omit ts-filter from the Server-Timing header entirely. + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let _span = state + .registry + .has_request_filters() + .then(|| timings.span(Phase::Filter)); + match state .registry .filter_request(RequestFilterRegistryInput { @@ -554,6 +601,7 @@ fn attach_dispatch_extensions( ec: EcRequestState, effects: RequestFilterEffects, ) -> Response { + response.extensions_mut().insert(ec.geo_lookup_state()); response.extensions_mut().insert(ec.into_finalize_state()); if !effects.response_headers.is_empty() { response.extensions_mut().insert(effects); @@ -590,7 +638,12 @@ async fn execute_named( // Deliberately do not use an EC request-state graph: that // copy is bot-gated, while operators use curl for this // authenticated diagnostic. - let kv = crate::maybe_identity_graph(&state.settings); + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let kv = crate::identity_graph_with_timing(&state.settings, &timings); handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) } NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), @@ -661,7 +714,12 @@ async fn run_named_route( if req.method() == Method::OPTIONS { cors_preflight_identify(&state.settings, &req) } else { - let kv = crate::require_identity_graph(&state.settings)?; + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let kv = crate::require_identity_graph_with_timing(&state.settings, &timings)?; let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; handle_identify( &state.settings, @@ -678,7 +736,13 @@ async fn run_named_route( // The auction reads consent data, so the consent KV store must be // available — fail closed with 503 when it is configured but // cannot be opened, matching legacy behavior. - let consent_services = runtime_services_for_consent_route(&state.settings, services)?; + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let consent_services = + runtime_services_for_consent_route(&state.settings, services, &timings)?; let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; let registry_ref = if partner_registry.is_empty() { None @@ -706,7 +770,13 @@ async fn run_named_route( // Like the auction, page-bids reads consent data, so the consent KV // store must be available — fail closed with 503 when configured but // unopenable, matching legacy. - let consent_services = runtime_services_for_consent_route(&state.settings, services)?; + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let consent_services = + runtime_services_for_consent_route(&state.settings, services, &timings)?; let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; let registry_ref = if partner_registry.is_empty() { None @@ -751,12 +821,18 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> let is_real_browser = device_signals.looks_like_browser(); let eids_cookie = crate::extract_cookie_value(&req, COOKIE_TS_EIDS); let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); - let result = crate::require_identity_graph(&state.settings).and_then(|kv| { - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; - let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - handle_batch_sync(&kv, &partner_registry, &limiter, req) - }); + let result = + crate::require_identity_graph_with_timing(&state.settings, &timings).and_then(|kv| { + let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + handle_batch_sync(&kv, &partner_registry, &limiter, req) + }); let mut response = result.unwrap_or_else(|e| http_error(&e)); // Legacy parity: batch-sync responses still pass through @@ -816,12 +892,28 @@ async fn dispatch_fallback( PreRoute::Continue { effects } => effects, }; + // Assigned exactly once, per branch below, alongside the routing + // decision itself, so the access-telemetry route identity always + // reflects which branch actually dispatched the request — including + // when that branch's handler errors. The asset-route sub-branch is an + // early return handled separately by `dispatch_asset_fallback`, so it + // never reaches (or needs to assign) this binding. + let route_metadata: Option; + let result = if uses_dynamic_tsjs_fallback(&method, &path) { + route_metadata = Some(RouteMetadata { + route_class: RouteClass::Tsjs, + route_template: TSJS_ROUTE_TEMPLATE.to_owned(), + }); handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SurrogateControl) } else if state.registry.has_route(&method, &path) { // Integration-proxy responses are not bounded by // publisher.max_buffered_body_bytes. Publisher fallback below uses the // publisher-specific streaming finalizer instead. + route_metadata = Some(RouteMetadata { + route_class: RouteClass::IntegrationProxy, + route_template: publisher_route_template(&path), + }); state .registry .handle_proxy(ProxyDispatchInput { @@ -849,9 +941,31 @@ async fn dispatch_fallback( .then(|| state.settings.asset_route_for_path(&path)) .flatten(); if let Some(asset_route) = matched_asset_route { - return dispatch_asset_fallback(state, services, req, asset_route, &effects).await; + // The template is the operator-configured route prefix, so it + // is bounded and content-free by construction (unlike request + // paths, which need `publisher_route_template`). + let asset_metadata = RouteMetadata { + route_class: RouteClass::Asset, + route_template: format!("{}/*", asset_route.prefix.trim_end_matches('/')), + }; + let mut response = dispatch_asset_fallback( + state, + services, + req, + asset_route, + &effects, + ec.geo_lookup_state(), + ) + .await; + response.extensions_mut().insert(asset_metadata); + return response; } + route_metadata = Some(RouteMetadata { + route_class: RouteClass::PublisherHtml, + route_template: publisher_route_template(&path), + }); + // Generate an EC ID if needed — mirrors the legacy catch-all arm. // Only for document navigations by recognised browsers; subresource // requests may lack consent signals such as Sec-GPC. @@ -867,7 +981,12 @@ async fn dispatch_fallback( // Publisher pages read consent data, so the consent KV store must be // available — fail closed with 503 when it is configured but cannot // be opened, matching legacy behavior. - match runtime_services_for_consent_route(&state.settings, services) { + let timings = req + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + match runtime_services_for_consent_route(&state.settings, services, &timings) { Ok(publisher_services) => { // Run the server-side auction with the configured creative- // opportunity slots and collect dispatched bids from the lazy @@ -923,7 +1042,10 @@ async fn dispatch_fallback( } }; - let response = result.unwrap_or_else(|e| http_error(&e)); + let mut response = result.unwrap_or_else(|e| http_error(&e)); + if let Some(metadata) = route_metadata { + response.extensions_mut().insert(metadata); + } attach_dispatch_extensions(response, ec, effects) } @@ -947,7 +1069,10 @@ fn asset_response_carries_body(method: &Method, status: StatusCode) -> bool { /// [`AssetProxyCachePolicy`] out via response extensions so `edgezero_main` /// can reapply protected cache directives after finalization. EC finalization /// is intentionally skipped: no [`EcFinalizeState`] is attached, matching the -/// legacy `should_finalize_ec = false` behavior for asset responses. +/// legacy `should_finalize_ec = false` behavior for asset responses. The +/// caller's [`GeoLookupState`] is still attached, since `build_ec_request_state` +/// already attempted the lookup before the asset route was matched — this is +/// the one exit path that carries geo state without an `EcFinalizeState`. /// /// Like legacy `route_request`, asset bodies are streamed straight to the client /// with no cap: the origin stream is attached to the response and `edgezero_main` @@ -962,6 +1087,7 @@ async fn dispatch_asset_fallback( req: Request, asset_route: &ProxyAssetRoute, effects: &RequestFilterEffects, + geo_state: GeoLookupState, ) -> Response { log::info!("No explicit route matched; proxying via configured asset route"); @@ -983,6 +1109,7 @@ async fn dispatch_asset_fallback( } response.extensions_mut().insert(cache_policy); + response.extensions_mut().insert(geo_state); attach_request_filter_effects(&mut response, effects); response } @@ -991,6 +1118,7 @@ async fn dispatch_asset_fallback( response .extensions_mut() .insert(AssetProxyCachePolicy::NoStorePrivate); + response.extensions_mut().insert(geo_state); attach_request_filter_effects(&mut response, effects); response } @@ -1113,6 +1241,10 @@ struct NamedRoute { path: &'static str, primary_methods: &'static [Method], handler: NamedRouteHandler, + /// Access-telemetry traffic category for this row. Attached verbatim + /// alongside `path` (the route-table pattern) to every response this + /// route produces — see [`named_route_handler`]. + route_class: RouteClass, } const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ @@ -1130,21 +1262,25 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: "/.well-known/trusted-server.json", primary_methods: &[Method::GET], handler: NamedRouteHandler::TrustedServerDiscovery, + route_class: RouteClass::Other, }, NamedRoute { path: "/verify-signature", primary_methods: &[Method::POST], handler: NamedRouteHandler::VerifySignature, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/admin/keys/rotate", primary_methods: &[Method::POST], handler: NamedRouteHandler::RotateKey, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/admin/keys/deactivate", primary_methods: &[Method::POST], handler: NamedRouteHandler::DeactivateKey, + route_class: RouteClass::Ec, }, // Admin EC lookup: the bare route reads the EC ID from the caller's // `ts-ec` cookie; the parameterized route takes an explicit EC ID. @@ -1152,11 +1288,13 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: "/_ts/admin/ec", primary_methods: &[Method::GET], handler: NamedRouteHandler::AdminEcLookup, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/admin/ec/{id}", primary_methods: &[Method::GET], handler: NamedRouteHandler::AdminEcLookup, + route_class: RouteClass::Ec, }, // Admin EIDs echo: decodes the request's ts-eids/sharedId cookies with // an ingestion preview. Pure request inspection — no KV access. @@ -1164,6 +1302,7 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: "/_ts/admin/eids", primary_methods: &[Method::GET], handler: NamedRouteHandler::AdminEidsLookup, + route_class: RouteClass::Ec, }, // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with a // 404 instead of executing key operations: the production basic-auth handler @@ -1175,36 +1314,43 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: "/admin/keys/rotate", primary_methods: LEGACY_ADMIN_DENY_METHODS, handler: NamedRouteHandler::LegacyAdminDenied, + route_class: RouteClass::Other, }, NamedRoute { path: "/admin/keys/deactivate", primary_methods: LEGACY_ADMIN_DENY_METHODS, handler: NamedRouteHandler::LegacyAdminDenied, + route_class: RouteClass::Other, }, NamedRoute { path: "/_ts/api/v1/batch-sync", primary_methods: &[Method::POST], handler: NamedRouteHandler::BatchSync, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/api/v1/identify", primary_methods: &[Method::GET, Method::OPTIONS], handler: NamedRouteHandler::Identify, + route_class: RouteClass::Ec, }, NamedRoute { path: "/_ts/set-tester", primary_methods: &[Method::GET], handler: NamedRouteHandler::SetTester, + route_class: RouteClass::Other, }, NamedRoute { path: "/_ts/clear-tester", primary_methods: &[Method::GET], handler: NamedRouteHandler::ClearTester, + route_class: RouteClass::Other, }, NamedRoute { path: "/auction", primary_methods: &[Method::POST], handler: NamedRouteHandler::Auction, + route_class: RouteClass::AuctionApi, }, // GET runs the SPA re-auction; OPTIONS is denied in-handler as a CORS // preflight guard for this side-effecting endpoint. @@ -1212,6 +1358,7 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: PAGE_BIDS_PATH, primary_methods: &[Method::GET, Method::OPTIONS], handler: NamedRouteHandler::PageBids, + route_class: RouteClass::AuctionApi, }, // Deprecated double-underscore alias. tsjs bundles served before the // `/_ts/page-bids` rename keep requesting this path from already-loaded @@ -1222,21 +1369,29 @@ const NAMED_ROUTES: &[NamedRoute] = &[ path: PAGE_BIDS_LEGACY_PATH, primary_methods: &[Method::GET, Method::OPTIONS], handler: NamedRouteHandler::PageBids, + route_class: RouteClass::AuctionApi, }, + // Classified `Other` rather than `IntegrationProxy`: that class is + // reserved for `state.registry.handle_proxy` (the js-integration proxy + // dispatch in `dispatch_fallback`), which these first-party proxy routes + // do not go through. NamedRoute { path: "/first-party/proxy", primary_methods: &[Method::GET], handler: NamedRouteHandler::FirstPartyProxy, + route_class: RouteClass::Other, }, NamedRoute { path: "/first-party/click", primary_methods: &[Method::GET], handler: NamedRouteHandler::FirstPartyClick, + route_class: RouteClass::Other, }, NamedRoute { path: "/first-party/sign", primary_methods: &[Method::GET, Method::POST], handler: NamedRouteHandler::FirstPartySign, + route_class: RouteClass::Other, }, NamedRoute { path: "/first-party/proxy-rebuild", @@ -1245,16 +1400,35 @@ const NAMED_ROUTES: &[NamedRoute] = &[ // POST is blocked by CORS and the guard navigates here for a 302 instead. primary_methods: &[Method::GET, Method::POST], handler: NamedRouteHandler::FirstPartyProxyRebuild, + route_class: RouteClass::Other, }, ]; +/// Wraps [`execute_named`], attaching a [`RouteMetadata`] extension carrying +/// `route_class` and the route-table pattern (`route_template`, verbatim, +/// with parameters left as placeholders) to every response the handler +/// produces — including its early-return diagnostic and setup-error arms, +/// since the attachment happens once around the whole future rather than in +/// each branch. fn named_route_handler( state: Arc, handler: NamedRouteHandler, + route_class: RouteClass, + route_template: &'static str, ) -> impl Fn(RequestContext) -> HandlerFuture + Clone + Send + Sync + 'static { move |ctx: RequestContext| { let state = Arc::clone(&state); - Box::pin(execute_named(state, ctx, handler)) + Box::pin(async move { + execute_named(state, ctx, handler) + .await + .map(|mut response| { + response.extensions_mut().insert(RouteMetadata { + route_class, + route_template: route_template.to_owned(), + }); + response + }) + }) } } @@ -1315,7 +1489,12 @@ impl TrustedServerApp { router = router.route( route.path, method.clone(), - named_route_handler(Arc::clone(state), route.handler), + named_route_handler( + Arc::clone(state), + route.handler, + route.route_class, + route.path, + ), ); } @@ -1375,18 +1554,21 @@ mod tests { use super::{ AppState, AuctionDispatch, EcContext, EdgeCacheHeader, HandlerFuture, NAMED_ROUTES, - NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, RuntimeStoreConfig, - TrustedServerApp, build_orchestrator_with_plan, build_per_request_services, - build_state_from_settings, compile_auction_plan, handle_publisher_request, - publisher_response_into_streaming_response, startup_error_router, + NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, RouteClass, RouteMetadata, + RuntimeStoreConfig, TSJS_ROUTE_TEMPLATE, TrustedServerApp, build_orchestrator_with_plan, + build_per_request_services, build_state_from_settings, compile_auction_plan, + handle_publisher_request, publisher_response_into_streaming_response, + publisher_route_template, startup_error_router, }; use base64::Engine as _; use bytes::Bytes; - use edgezero_core::app::Hooks as _; + use edgezero_core::app::{Hooks as _, StoreMetadata}; use edgezero_core::body::Body; use edgezero_core::context::RequestContext; use edgezero_core::env_config::EnvConfig; - use edgezero_core::http::{Method, Response, StatusCode, header, request_builder}; + use edgezero_core::http::{ + Method, Response, StatusCode, header, request_builder, response_builder, + }; use edgezero_core::key_value_store::NoopKvStore; use edgezero_core::params::PathParams; use edgezero_core::router::RouterService; @@ -1395,20 +1577,23 @@ mod tests { use error_stack::Report; use futures::executor::block_on; use serde_json::json; - use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; + use trusted_server_core::constants::{HEADER_X_GEO_COUNTRY, HEADER_X_GEO_INFO_AVAILABLE}; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::error::TrustedServerError; + use trusted_server_core::geo::GeoLookupState; use trusted_server_core::integrations::{ HeaderMutation, IntegrationRegistry, IntegrationRequestFilter, RequestFilterDecision, RequestFilterEffects, RequestFilterInput, }; use trusted_server_core::platform::{ - ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformError, PlatformHttpClient, - PlatformHttpRequest, PlatformKvStore, PlatformPendingRequest, PlatformResponse, - PlatformSelectResult, PlatformTemplateCache, PlatformTemplateCacheReservation, - RuntimeServices, TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, - TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, TemplateMetadata, + ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformError, PlatformGeo, + PlatformHttpClient, PlatformHttpRequest, PlatformKvStore, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, PlatformTemplateCache, + PlatformTemplateCacheReservation, RuntimeServices, TemplateCacheError, TemplateCacheKey, + TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, + TemplateMetadata, }; + use trusted_server_core::request_timing::RequestTimings; use trusted_server_core::settings::Settings; #[test] @@ -1604,6 +1789,33 @@ mod tests { TrustedServerApp::routes_for_state(&state) } + #[test] + fn trusted_server_app_declares_runtime_store_metadata() { + let stores = TrustedServerApp::stores(); + + assert_eq!( + stores.config, + Some(StoreMetadata { + default: "trusted_server_config", + ids: &["trusted_server_config"], + }) + ); + assert_eq!( + stores.kv, + Some(StoreMetadata { + default: "trusted_server_kv", + ids: &["trusted_server_kv"], + }) + ); + assert_eq!( + stores.secrets, + Some(StoreMetadata { + default: "trusted_server_secrets", + ids: &["trusted_server_secrets"], + }) + ); + } + #[test] fn per_request_services_register_the_fastly_template_assembler() { let state = build_state_from_settings(test_settings()).expect("should build test state"); @@ -1634,12 +1846,12 @@ mod tests { ); } - /// Builds a router whose `AppState` uses a registry containing the given - /// request filters (and no routes), so dispatch-level request-filter - /// behavior can be exercised without a real integration. - fn router_with_request_filters( + /// Builds an `AppState` whose registry contains the given request + /// filters (and no routes), so dispatch-level request-filter behavior can + /// be exercised without a real integration. + fn state_with_request_filters( filters: Vec>, - ) -> RouterService { + ) -> Arc { let settings = test_settings(); let plan = Arc::new( trusted_server_core::auction::compile_auction_plan(&settings) @@ -1651,7 +1863,7 @@ mod tests { let registry = IntegrationRegistry::from_request_filters(filters); let default_kv_store = Arc::new(crate::platform::UnavailableKvStore) as Arc; - let state = Arc::new(super::AppState { + Arc::new(super::AppState { auction_telemetry_sink: Arc::new( trusted_server_core::auction::NoopAuctionTelemetrySink, ), @@ -1659,8 +1871,15 @@ mod tests { orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, - }); - TrustedServerApp::routes_for_state(&state) + }) + } + + /// Builds a router on top of [`state_with_request_filters`] so + /// dispatch-level request-filter behavior can be exercised end-to-end. + fn router_with_request_filters( + filters: Vec>, + ) -> RouterService { + TrustedServerApp::routes_for_state(&state_with_request_filters(filters)) } /// Continues routing while mutating the request and emitting a response @@ -2312,6 +2531,111 @@ mod tests { ); } + /// `Authorization: Basic` header value for `test_settings()`'s + /// `^/_ts/admin` handler (`admin` / `admin-pass`). + fn admin_basic_auth_header() -> edgezero_core::http::HeaderValue { + let credentials = base64::engine::general_purpose::STANDARD.encode("admin:admin-pass"); + format!("Basic {credentials}") + .parse() + .expect("should parse basic-auth header value") + } + + #[test] + fn named_route_attaches_the_table_pattern_verbatim_even_with_a_real_id_in_the_path() { + // A named-route response must carry the route-TABLE pattern + // (`{id}` left as a placeholder), never the caller's actual matched + // path segment — this is what keeps a real EC identifier out of + // access telemetry, independent of anything the row-serialization + // layer does. + let router = test_router(); + let ec_id = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01"; + let mut req = empty_request(Method::GET, &format!("/_ts/admin/ec/{ec_id}")); + req.headers_mut() + .insert(header::AUTHORIZATION, admin_basic_auth_header()); + let response = route(&router, req); + + let metadata = response + .extensions() + .get::() + .expect("named-route responses should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::Ec); + assert_eq!(metadata.route_template, "/_ts/admin/ec/{id}"); + assert!( + !metadata.route_template.contains(ec_id), + "the attached template must never contain the matched id" + ); + } + + #[test] + fn named_route_attaches_metadata_even_on_a_read_only_diagnostic_early_return() { + // AdminEidsLookup is handled by an early-return arm inside + // execute_named, before the normal EC lifecycle runs (see the + // "read-only diagnostics" comment there). named_route_handler wraps + // the whole future, so the attachment must still happen here too. + let router = test_router(); + let mut req = empty_request(Method::GET, "/_ts/admin/eids"); + req.headers_mut() + .insert(header::AUTHORIZATION, admin_basic_auth_header()); + let response = route(&router, req); + + let metadata = response + .extensions() + .get::() + .expect("even a read-only diagnostic early-return response should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::Ec); + assert_eq!(metadata.route_template, "/_ts/admin/eids"); + } + + #[test] + fn tsjs_fallback_attaches_tsjs_route_metadata() { + let router = test_router(); + let response = route( + &router, + empty_request(Method::GET, "/static/tsjs=tsjs-unified.min.js"), + ); + + let metadata = response + .extensions() + .get::() + .expect("tsjs fallback responses should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::Tsjs); + assert_eq!(metadata.route_template, TSJS_ROUTE_TEMPLATE); + } + + #[test] + fn integration_proxy_fallback_attaches_integration_proxy_route_metadata() { + // test_settings() enables the prebid integration, which registers a + // proxy route at /integrations/prebid/bundle.js. + let router = test_router(); + let response = route( + &router, + empty_request(Method::GET, "/integrations/prebid/bundle.js"), + ); + + let metadata = response + .extensions() + .get::() + .expect("integration-proxy fallback responses should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::IntegrationProxy); + assert_eq!( + metadata.route_template, + publisher_route_template("/integrations/prebid/bundle.js") + ); + } + + #[test] + fn publisher_fallback_attaches_publisher_html_route_metadata() { + let router = test_router(); + let response = route(&router, empty_request(Method::GET, "/news/some-article")); + + let metadata = response + .extensions() + .get::() + .expect("publisher fallback responses should carry RouteMetadata"); + assert_eq!(metadata.route_class, RouteClass::PublisherHtml); + assert_eq!(metadata.route_template, "/news/*"); + } + #[test] fn browser_device_signals_from_extension_reach_ec_finalize_state() { // Regression guard for the EdgeZero JA4/H2 signal loss: `edgezero_main` @@ -2732,6 +3056,61 @@ mod tests { ); } + #[test] + fn asset_fallback_carries_geo_state_without_ec_finalize_state() { + // The asset-route fallback is the one exit path that skips + // EcFinalizeState but must still carry GeoLookupState, since + // build_ec_request_state (and its geo lookup) already ran before the + // asset route was matched. Without this, the finalize step would + // silently repeat the lookup for every asset request. + let settings = Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + + [request_signing] + enabled = false + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + + [proxy] + + [[proxy.asset_routes]] + prefix = "/.image/" + origin_url = "https://assets.example.com" + "#, + ) + .expect("should parse asset-route settings"); + let state = build_state_from_settings(settings).expect("should build state"); + let router = TrustedServerApp::routes_for_state(&state); + + let response = route(&router, empty_request(Method::GET, "/.image/banner.png")); + + assert!( + response.extensions().get::().is_some(), + "asset-route responses should still carry GeoLookupState even though \ + EC finalization is skipped" + ); + assert!( + response + .extensions() + .get::() + .is_none(), + "asset-route responses must skip EC finalization (no EcFinalizeState)" + ); + } + struct FixedBackend; impl PlatformBackend for FixedBackend { @@ -3148,6 +3527,7 @@ mod tests { req, asset_route, &effects, + trusted_server_core::geo::GeoLookupState::NotAttempted, )); assert_eq!( @@ -3191,6 +3571,259 @@ mod tests { ); } + /// A [`PlatformGeo`] stub that counts every `lookup` call and always + /// returns the same canned result, used to prove the request-phase geo + /// lookup is never repeated during finalize. + struct CountingGeo { + calls: Arc, + result: Option, + } + + impl PlatformGeo for CountingGeo { + fn lookup(&self, _: Option) -> Result, Report> { + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(self.result.clone()) + } + } + + fn sample_geo_info() -> GeoInfo { + GeoInfo { + city: "Testville".to_string(), + country: "US".to_string(), + continent: "NorthAmerica".to_string(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + } + } + + fn runtime_services_with_geo(geo: Arc) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(StreamingHttpClient)) + .geo(geo) + .client_info(ClientInfo::default()) + .build() + } + + #[test] + fn finalize_reuses_request_phase_geo_without_second_lookup() { + // Dispatching a publisher route runs build_ec_request_state, which + // attempts the geo lookup once and carries the result via + // GeoLookupState. The finalize step (resolve_geo_for_response) must + // reuse that carried value instead of calling the geo backend again. + let calls = Arc::new(AtomicUsize::new(0)); + let geo = Arc::new(CountingGeo { + calls: Arc::clone(&calls), + result: Some(sample_geo_info()), + }); + let state = app_state_for_settings(test_settings()); + let services = runtime_services_with_geo(geo); + let req = empty_request(Method::GET, "/some-page"); + + let response = block_on(super::dispatch_fallback(&state, &services, req)); + + let carried = response + .extensions() + .get::() + .cloned() + .expect("dispatch should attach GeoLookupState"); + assert!( + matches!(carried, GeoLookupState::Resolved(_)), + "a successful lookup should carry Resolved" + ); + + let geo_info = + crate::middleware::resolve_geo_for_response(&response, &carried, None, |_| { + panic!("finalize must not repeat a resolved geo lookup"); + }); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "only the request-phase lookup should have run" + ); + + let mut response = response; + geo_info + .expect("geo info should have resolved") + .set_response_headers(&mut response); + assert!( + response.headers().get(HEADER_X_GEO_COUNTRY).is_some(), + "x-geo-country should still be set on the response after reusing the carried geo" + ); + } + + #[test] + fn failed_lookup_is_not_retried() { + // When the request-phase lookup fails (returns None), dispatch must + // carry GeoLookupState::Attempted rather than NotAttempted, and + // finalize must not retry it. + let calls = Arc::new(AtomicUsize::new(0)); + let geo = Arc::new(CountingGeo { + calls: Arc::clone(&calls), + result: None, + }); + let state = app_state_for_settings(test_settings()); + let services = runtime_services_with_geo(geo); + let req = empty_request(Method::GET, "/some-page"); + + let response = block_on(super::dispatch_fallback(&state, &services, req)); + + let carried = response + .extensions() + .get::() + .cloned() + .expect("dispatch should attach GeoLookupState even for a failed lookup"); + assert!( + matches!(carried, GeoLookupState::Attempted), + "a failed lookup should carry Attempted, not Resolved or NotAttempted" + ); + + let geo_info = + crate::middleware::resolve_geo_for_response(&response, &carried, None, |_| { + panic!("finalize must not retry a failed geo lookup"); + }); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "only the request-phase lookup should have run" + ); + assert!( + geo_info.is_none(), + "no geo info should be available after a failed lookup" + ); + } + + #[test] + fn filter_span_recorded_when_request_filter_runs() { + // The Filter phase span should only be recorded when the registry + // actually has a request filter registered, so unconfigured + // deployments omit ts-filter from the Server-Timing header entirely. + let state = state_with_request_filters(vec![Arc::new(RecordingRequestFilter)]); + let services = RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(StreamingHttpClient)) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(ClientInfo::default()) + .build(); + let mut req = empty_request(Method::GET, "/some-page"); + let timings = RequestTimings::new(); + req.extensions_mut().insert(timings.clone()); + + let _ = block_on(super::run_pre_route_filters( + &state, &services, &mut req, None, + )); + + assert!( + timings.snapshot().filter_ms.is_some(), + "should record the Filter phase span when a request filter is registered and runs" + ); + } + + #[test] + fn filter_span_not_recorded_when_no_request_filters_registered() { + // Mirror test: an empty registry must never record the Filter span, + // even though run_pre_route_filters still runs (as a no-op loop). + let state = state_with_request_filters(Vec::new()); + let services = RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(StreamingHttpClient)) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(ClientInfo::default()) + .build(); + let mut req = empty_request(Method::GET, "/some-page"); + let timings = RequestTimings::new(); + req.extensions_mut().insert(timings.clone()); + + let _ = block_on(super::run_pre_route_filters( + &state, &services, &mut req, None, + )); + + assert!( + timings.snapshot().filter_ms.is_none(), + "should omit the Filter phase span when no request filters are registered" + ); + } + + fn settings_with_consent_and_ec_store() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + ec_store = "ec_identity_store" + + [consent] + consent_store = "consent_store" + + [request_signing] + enabled = false + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + "#, + ) + .expect("should parse settings with consent and EC KV stores configured") + } + + #[test] + fn consent_store_reads_are_timed_and_pull_sync_is_not() { + // Consent-store access threaded through RuntimeServices uses the same + // TimedKvStore decorator as request-path KvIdentityGraph + // construction, so a read through it records Phase::EcKv. + let settings = settings_with_consent_and_ec_store(); + let services = streaming_runtime_services(); + let timings = RequestTimings::new(); + + let consent_services = + super::runtime_services_for_consent_route(&settings, &services, &timings) + .expect("should open the configured consent store"); + let _ = block_on(consent_services.kv_store().get_bytes("consent-read-key")); + + timings.mark_headers_ready(); + assert!( + timings.snapshot().kv_ms.is_some(), + "a consent-store read through the decorated RuntimeServices store should record Phase::EcKv" + ); + + // Pull-sync's identity graph is built by `require_identity_graph`, + // which takes no `timings` parameter at all — the untimed store it + // constructs cannot record into any handle, including a fresh one. + let graph = crate::require_identity_graph(&settings) + .expect("should construct the pull-sync identity graph"); + let ec_id = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01"; + let _ = graph.get(ec_id); + + let pull_sync_timings = RequestTimings::new(); + pull_sync_timings.mark_headers_ready(); + assert!( + pull_sync_timings.snapshot().kv_ms.is_none(), + "pull-sync's untimed graph construction has no timings handle to record into" + ); + } + #[test] fn dispatch_runs_request_filter_and_threads_response_effects() { // Regression guard for the EdgeZero request-filter bypass: the publisher @@ -3312,6 +3945,45 @@ mod tests { ); } + /// Joins every instance of a response header into one comma-separated + /// string (mirroring how a client sees repeated header fields), or + /// `None` if the header is absent. + fn response_header(response: &Response, name: &str) -> Option { + let values: Vec<&str> = response + .headers() + .get_all(name) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect(); + if values.is_empty() { + None + } else { + Some(values.join(", ")) + } + } + + #[test] + fn server_timing_emitted_on_private_response_when_enabled() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .body(Body::empty()) + .expect("should build a private response fixture"); + let timings = RequestTimings::new(); + + crate::apply_server_timing_header(&mut response, &timings, true); + + let header = response_header(&response, "server-timing").expect("should emit header"); + assert!( + header.contains("ts-total;dur="), + "should carry the stored total: {header}" + ); + assert_eq!( + header.matches("ts-total").count(), + 1, + "should emit exactly one TS-owned metric set" + ); + } + #[test] fn publisher_navigation_origin_start_failure_is_not_recovery_eligible() { // Recovery is authorized only after a successful origin start. With no @@ -3325,4 +3997,69 @@ mod tests { "an origin-start failure must not authorize orphan recovery" ); } + + #[test] + fn server_timing_absent_when_flag_off() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .body(Body::empty()) + .expect("should build a private response fixture"); + let timings = RequestTimings::new(); + + crate::apply_server_timing_header(&mut response, &timings, false); + + assert!( + response_header(&response, "server-timing").is_none(), + "should not emit server-timing when the flag is off" + ); + } + + #[test] + fn server_timing_absent_on_cacheable_responses() { + // tsjs route policy: public, long max-age, immutable. + let mut tsjs_response = response_builder() + .header("cache-control", "public, max-age=31536000, immutable") + .body(Body::empty()) + .expect("should build a tsjs-style response fixture"); + // A bare shared-cacheable response with no private/no-store directive. + let mut public_response = response_builder() + .header("cache-control", "max-age=60") + .body(Body::empty()) + .expect("should build a bare max-age response fixture"); + + crate::apply_server_timing_header(&mut tsjs_response, &RequestTimings::new(), true); + crate::apply_server_timing_header(&mut public_response, &RequestTimings::new(), true); + + assert!( + response_header(&tsjs_response, "server-timing").is_none(), + "should not emit on the public immutable tsjs cache policy" + ); + assert!( + response_header(&public_response, "server-timing").is_none(), + "should not emit on a bare shared-cacheable max-age response" + ); + } + + #[test] + fn preexisting_server_timing_values_survive() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .header("server-timing", "upstream;dur=1") + .body(Body::empty()) + .expect("should build a private response fixture carrying an upstream Server-Timing"); + let timings = RequestTimings::new(); + + crate::apply_server_timing_header(&mut response, &timings, true); + + let header = + response_header(&response, "server-timing").expect("should still carry a header"); + assert!( + header.contains("upstream;dur=1"), + "should preserve the pre-existing entry: {header}" + ); + assert!( + header.contains("ts-total"), + "should append the TS-owned set: {header}" + ); + } } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 367a3e33f..15ca6fcf9 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -1,5 +1,8 @@ use std::sync::Arc; +use rand::Rng as _; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + use edgezero_adapter_fastly::config_store::FastlyConfigStore as EdgeZeroFastlyConfigStore; use edgezero_adapter_fastly::request::into_core_request; use edgezero_adapter_fastly::runtime_env_config; @@ -13,7 +16,13 @@ use error_stack::Report; use fastly::http::Method as FastlyMethod; use fastly::{Request as FastlyRequest, Response as FastlyResponse}; +use trusted_server_core::access_telemetry::{ + AccessTelemetrySnapshot, RouteClass, RouteMetadata, access_event_row, +}; use trusted_server_core::cache_policy::EdgeCacheHeader; +use trusted_server_core::constants::{ + ENV_FASTLY_IS_STAGING, ENV_FASTLY_POP, ENV_FASTLY_SERVICE_ID, ENV_FASTLY_SERVICE_VERSION, +}; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; @@ -22,10 +31,13 @@ use trusted_server_core::ec::pull_sync::{ }; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; +use trusted_server_core::geo::GeoLookupState; use trusted_server_core::integrations::RequestFilterEffects; use trusted_server_core::platform::PlatformGeo as _; -use trusted_server_core::platform::RuntimeServices; +use trusted_server_core::platform::{RuntimeServices, TimedKvStore}; use trusted_server_core::proxy::{AssetProxyCachePolicy, stream_asset_body}; +use trusted_server_core::publisher::TemplateCacheResponseState; +use trusted_server_core::request_timing::{Phase, RequestTimings, append_server_timing_if_private}; use trusted_server_core::response_privacy::TerminalPrivateResponse; use trusted_server_core::settings::Settings; @@ -85,19 +97,18 @@ fn main() { } logging::init_logger(); - edgezero_main(req); + let env = runtime_env_config(TrustedServerApp::stores()); + let runtime_stores = RuntimeStoreConfig::from_env(&env); + edgezero_main(req, &runtime_stores); } /// Handles a request through the `EdgeZero` router path. -fn edgezero_main(mut req: FastlyRequest) { - let runtime_env = runtime_env_config(TrustedServerApp::stores()); - let runtime_stores = RuntimeStoreConfig::from_env(&runtime_env); - +fn edgezero_main(mut req: FastlyRequest, runtime_stores: &RuntimeStoreConfig) { // Short-circuit the JA4 debug probe before app construction. Must run here // because TLS/JA4 accessors are only available on FastlyRequest before // conversion to edgezero types. if req.get_method() == FastlyMethod::GET && req.get_path() == "/_ts/debug/ja4" { - match load_settings_from_config_store(&runtime_stores) { + match load_settings_from_config_store(runtime_stores) { Ok(settings) if settings.debug.ja4_endpoint_enabled => { build_ja4_debug_response(&req).send_to_client(); } @@ -114,20 +125,43 @@ fn edgezero_main(mut req: FastlyRequest) { return; } - let config_store = - match open_trusted_server_config_store(runtime_stores.config_store_name.as_ref()) { - Ok(cs) => cs, - Err(e) => { - log::error!("failed to open config store: {e}"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - .with_body_text_plain("Internal Server Error") - .send_to_client(); - return; - } - }; + let timings = RequestTimings::new(); - let (app, app_state) = TrustedServerApp::build_app_with_state(&runtime_stores); + let (config_store, app, app_state) = { + let _appbuild = timings.span(Phase::AppBuild); + let config_store = + match open_trusted_server_config_store(runtime_stores.config_store_name.as_ref()) { + Ok(cs) => cs, + Err(e) => { + log::error!("failed to open config store: {e}"); + FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) + .with_body_text_plain("Internal Server Error") + .send_to_client(); + return; + } + }; + let (app, app_state) = TrustedServerApp::build_app_with_state(runtime_stores); + (config_store, app, app_state) + }; let settings_snapshot = app_state.as_ref().map(|state| Arc::clone(&state.settings)); + let server_timing_enabled = settings_snapshot + .as_deref() + .is_some_and(|settings| settings.observability.server_timing_enabled); + // Both read once here rather than at each `send_edgezero_response` call + // site: if `app_state` failed to build, there is no settings snapshot to + // read them from at all, so every call site would need the same + // degraded-mode fallback. `access_sample_rate` defaults to `0.0` (never + // sampled in) and `publisher_domain` to `"unknown"` in that case. + let access_sample_rate = settings_snapshot + .as_deref() + .map_or(0.0, |settings| settings.tinybird.access_sample_rate); + let access_telemetry_enabled = settings_snapshot + .as_deref() + .is_some_and(|settings| settings.tinybird.enabled && settings.tinybird.access_enabled); + let publisher_domain = settings_snapshot.as_deref().map_or_else( + || "unknown".to_owned(), + |settings| settings.publisher.domain.clone(), + ); let trusted_client_ip = settings_snapshot .as_deref() .and_then(|settings| settings.trusted_client_ip.as_ref()); @@ -147,6 +181,10 @@ fn edgezero_main(mut req: FastlyRequest) { req.set_header("fastly-ssl", "1"); } + // Capture the method before dispatch consumes the request. The resolved + // client IP is retained below in `ClientInfo`. + let request_method = req.get_method_str().to_owned(); + // Strip any client-supplied x-ts-tls-* headers before injecting the trusted // values from the Fastly SDK. Must run after sanitize_fastly_forwarded_headers. req.remove_header("x-ts-tls-protocol"); @@ -178,6 +216,7 @@ fn edgezero_main(mut req: FastlyRequest) { core_req.extensions_mut().insert(config_store); core_req.extensions_mut().insert(device_signals); core_req.extensions_mut().insert(client_info); + core_req.extensions_mut().insert(timings.clone()); match futures::executor::block_on(app.router().oneshot(core_req)) { Ok(response) => response, Err(error) => edge_error_response(error), @@ -196,14 +235,34 @@ fn edgezero_main(mut req: FastlyRequest) { let ec_state = response.extensions_mut().remove::(); let asset_cache_policy = response.extensions_mut().remove::(); let request_filter_effects = response.extensions_mut().remove::(); + // Read rather than pop: the access-telemetry snapshot built later in + // `send_edgezero_response` reads this same extension, so it must still + // be attached to `response` at that point. + let geo_lookup_state = response + .extensions() + .get::() + .cloned() + .unwrap_or(GeoLookupState::NotAttempted); if !take_finalize_sentinel(&mut response) { if let Some(settings) = settings_snapshot.as_deref() { - apply_entry_point_finalize_headers(settings, &mut response, client_ip); + apply_entry_point_finalize_headers( + settings, + &mut response, + client_ip, + &geo_lookup_state, + &timings, + ); } else { - match load_settings_from_config_store(&runtime_stores) { + match load_settings_from_config_store(runtime_stores) { Ok(settings) => { - apply_entry_point_finalize_headers(&settings, &mut response, client_ip); + apply_entry_point_finalize_headers( + &settings, + &mut response, + client_ip, + &geo_lookup_state, + &timings, + ); } Err(e) => { log::warn!("entry-point finalize skipped: failed to reload settings: {e:?}"); @@ -218,10 +277,22 @@ fn edgezero_main(mut req: FastlyRequest) { if let Some(mut ec_state) = ec_state { if let Some(settings) = settings_snapshot.as_deref() { - match apply_edgezero_ec_finalize(settings, &mut ec_state, &mut response) { + match apply_edgezero_ec_finalize(settings, &mut ec_state, &mut response, &timings) { Ok(partner_registry) => { - send_edgezero_response(response, request_filter_effects.as_ref()); + let outcome = send_edgezero_response( + response, + request_filter_effects.as_ref(), + &SendContext { + timings: timings.clone(), + server_timing_enabled, + method: request_method.clone(), + publisher_domain: publisher_domain.clone(), + access_sample_rate, + access_telemetry_enabled, + }, + ); run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); + emit_access_telemetry_after_send(settings, &outcome, &timings); return; } Err(e) => { @@ -231,16 +302,33 @@ fn edgezero_main(mut req: FastlyRequest) { } } } else { - match load_settings_from_config_store(&runtime_stores) { + match load_settings_from_config_store(runtime_stores) { Ok(settings) => { - match apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response) { + match apply_edgezero_ec_finalize( + &settings, + &mut ec_state, + &mut response, + &timings, + ) { Ok(partner_registry) => { - send_edgezero_response(response, request_filter_effects.as_ref()); + let outcome = send_edgezero_response( + response, + request_filter_effects.as_ref(), + &SendContext { + timings: timings.clone(), + server_timing_enabled, + method: request_method.clone(), + publisher_domain: publisher_domain.clone(), + access_sample_rate, + access_telemetry_enabled, + }, + ); run_edgezero_pull_sync_after_send( &settings, &partner_registry, &ec_state, ); + emit_access_telemetry_after_send(&settings, &outcome, &timings); return; } Err(e) => { @@ -257,7 +345,27 @@ fn edgezero_main(mut req: FastlyRequest) { } } - send_edgezero_response(response, request_filter_effects.as_ref()); + let outcome = send_edgezero_response( + response, + request_filter_effects.as_ref(), + &SendContext { + timings: timings.clone(), + server_timing_enabled, + method: request_method, + publisher_domain, + access_sample_rate, + access_telemetry_enabled, + }, + ); + // The asset/admin/error fallback path: no `EcFinalizeState` (or the ec + // finalize branch above failed), so there is no pull-sync dispatch here + // at all — telemetry is the only post-send step. When `app_state` never + // built there is nothing to emit either: `access_telemetry_enabled` was + // necessarily false without a settings snapshot, so the outcome carries + // no access snapshot, and reloading settings here could not change that. + if let Some(settings) = settings_snapshot.as_deref() { + emit_access_telemetry_after_send(settings, &outcome, &timings); + } } fn edge_error_response(error: EdgeError) -> HttpResponse { @@ -285,24 +393,40 @@ fn apply_entry_point_finalize_headers( settings: &Settings, response: &mut HttpResponse, client_ip: Option, + geo_state: &GeoLookupState, + timings: &RequestTimings, ) { - let geo_info = resolve_geo_for_response(response, client_ip, |client_ip| { + let geo_info = resolve_geo_for_response(response, geo_state, client_ip, |client_ip| { + let _span = timings.span(Phase::Geo); FastlyPlatformGeo.lookup(client_ip).unwrap_or_else(|e| { log::warn!("entry-point geo lookup failed: {e}"); None }) }); apply_finalize_headers(settings, geo_info.as_ref(), response); + + // This path runs only when the middleware chain was bypassed (e.g. a + // router-level 404/405 for an unregistered method), so `geo_state` may + // still be `NotAttempted` even after a fresh lookup just ran above. + // Write the resolved outcome back so the access-telemetry snapshot built + // later in `send_edgezero_response` sees what was actually looked up, + // not the stale carried-in state. + let resolved_state = match &geo_info { + Some(info) => GeoLookupState::Resolved(info.clone()), + None => GeoLookupState::Attempted, + }; + response.extensions_mut().insert(resolved_state); } fn apply_edgezero_ec_finalize( settings: &Settings, ec_state: &mut EcFinalizeState, response: &mut HttpResponse, + timings: &RequestTimings, ) -> Result> { let partner_registry = PartnerRegistry::from_config(&settings.ec.partners)?; let finalize_kv_graph = if ec_state.use_finalize_kv { - maybe_identity_graph(settings) + identity_graph_with_timing(settings, timings) } else { None }; @@ -330,6 +454,219 @@ fn run_edgezero_pull_sync_after_send( } } +/// Builds and emits the access-telemetry row for one delivered response, +/// when access telemetry is enabled and this request is sampled in. +/// +/// Called last at every `send_edgezero_response` call site in +/// [`edgezero_main`] — after `run_edgezero_pull_sync_after_send` on the two +/// EC-finalized paths, and directly after send on the asset/admin/error +/// fallback path, which never builds an [`EcFinalizeState`] or route-scoped +/// `RuntimeServices` at all. The Tinybird transport context is therefore +/// constructed fresh from `settings` here rather than threaded through +/// either of those per-route types, so every response class can emit. +/// +/// Sampled-out requests return silently — that is the expected, high-volume +/// case and not worth a log line. The sampling roll uses the rate stored on +/// the snapshot itself, so the emission probability always matches the +/// row's `sample_rate` column by construction. Every other drop (row +/// build, token load, send, or non-2xx status — all folded into +/// `emit_access_event`'s `Result`) logs exactly one warning naming the +/// reason. +fn emit_access_telemetry_after_send( + settings: &Settings, + outcome: &DeliveryOutcome, + timings: &RequestTimings, +) { + if !settings.tinybird.enabled || !settings.tinybird.access_enabled { + return; + } + + // No snapshot means access telemetry was disabled when the response + // was sent (the flag is read once, before dispatch); nothing to emit. + let Some(snapshot) = &outcome.snapshot else { + return; + }; + + let since_epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + let epoch_ms = u64::try_from(since_epoch.as_millis()).unwrap_or(u64::MAX); + // Sample with the rate stored on the snapshot itself — the same value + // serialized into the row's `sample_rate` column — so the emission + // probability and the row's claimed rate cannot diverge, which the + // documented `sum(1.0 / sample_rate)` volume estimator depends on. + let roll = rand::thread_rng().r#gen::(); + if !tinybird::sampled_in(snapshot.sample_rate, roll) { + return; + } + + let row = access_event_row(snapshot, &timings.snapshot(), epoch_ms); + let target = tinybird::TinybirdEventsTarget::from_access_config(settings.tinybird.clone()); + let result = futures::executor::block_on(tinybird::emit_access_event( + &platform::FastlyPlatformHttpClient, + &target, + row, + )); + if let Err(error) = result { + log::warn!("access telemetry emission dropped: {error:?}"); + } +} + +/// Per-response context threaded into [`send_edgezero_response`] so the +/// function stays at or under seven parameters. +struct SendContext { + /// The request's phase-timing collector. + timings: RequestTimings, + /// Whether `observability.server_timing_enabled` is set. + server_timing_enabled: bool, + /// The request's HTTP method, captured before the request was consumed + /// by dispatch. + method: String, + /// The configured publisher domain. + publisher_domain: String, + /// The configured access-telemetry sample rate. + access_sample_rate: f64, + /// Whether `tinybird.enabled` and `tinybird.access_enabled` were both + /// set when settings were first read. Gates building the + /// [`AccessTelemetrySnapshot`] at all: the snapshot costs env reads and + /// `String` allocations on the pre-send path, which a disabled + /// deployment (the default) should not pay. + access_telemetry_enabled: bool, +} + +/// Outcome of handing a finalized response to the client. +pub(crate) struct DeliveryOutcome { + /// Response body size in bytes. + #[allow(dead_code)] + pub bytes: u64, + /// Whether delivery completed or failed partway. Collected as + /// groundwork; not yet emitted on any surface. + #[allow(dead_code)] + pub result: DeliveryResult, + /// Access-telemetry dimensions captured for this response at the + /// freeze point. `None` when access telemetry was disabled at snapshot + /// time; the emitter treats that as nothing to send. + pub snapshot: Option, +} + +/// Whether [`send_edgezero_response`] completed delivery or failed partway. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum DeliveryResult { + /// The response was handed to the client in full. + Complete, + /// Delivery started but did not finish cleanly: some bytes reached the + /// client's transport before a stream error, or the transport could not + /// be closed cleanly after every byte was written. + Partial, + /// Delivery failed before any bytes reached the client. + Error, +} + +/// Thin Fastly-adapter wrapper around +/// [`append_server_timing_if_private`], the freeze point shared with the +/// Axum adapter's terminal timing layer. See that function's doc for the +/// emission rules (always stamps `mark_headers_ready`; appends rather than +/// overwrites; never promotes a response to shared-cacheable). +pub(crate) fn apply_server_timing_header( + response: &mut HttpResponse, + timings: &RequestTimings, + server_timing_enabled: bool, +) { + append_server_timing_if_private(response, timings, server_timing_enabled); +} + +/// A [`Write`](std::io::Write) wrapper that tallies bytes successfully written +/// to the inner writer. +/// +/// Wraps the client transport during a streaming drive so a truncated or +/// failed drive still reports how many bytes actually reached it, instead of +/// the placeholder `0` a failed/aborted drive would otherwise report. +struct CountingWriter { + inner: W, + bytes: u64, +} + +impl CountingWriter { + fn new(inner: W) -> Self { + Self { inner, bytes: 0 } + } + + /// Bytes successfully written to the inner writer so far. + fn bytes(&self) -> u64 { + self.bytes + } + + fn into_inner(self) -> W { + self.inner + } +} + +impl std::io::Write for CountingWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let written = self.inner.write(buf)?; + self.bytes = self.bytes.saturating_add(written as u64); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +/// Drives a streaming `EdgeZero` body through `output`, tallying bytes written +/// and timing the drive into `timings`. +/// +/// Stamps `resp_bytes` and `request_elapsed` immediately once the drive +/// returns — before the caller does anything transport-specific (finishing +/// the streaming body, logging) — so `request_elapsed` never includes that +/// work. Returns the counting writer (so the caller can recover both the +/// tallied byte count and the wrapped transport) alongside the drive's +/// result. +fn drive_streaming_body( + body: EdgeBody, + output: W, + timings: &RequestTimings, +) -> (CountingWriter, Result<(), Report>) { + let mut counting = CountingWriter::new(output); + let drive_started = Instant::now(); + let result = futures::executor::block_on(stream_asset_body(body, &mut counting)); + timings.record(Phase::Stream, drive_started.elapsed()); + timings.set_resp_bytes(counting.bytes()); + timings.mark_request_elapsed(); + (counting, result) +} + +/// Classifies a completed streaming drive into a [`DeliveryResult`]. +/// +/// A drive that failed after writing at least one byte delivered a truncated +/// response rather than nothing at all, so it is [`DeliveryResult::Partial`], +/// not [`DeliveryResult::Error`]. +/// +/// The `Ok(())` arm exists for the classifier's totality, not for the +/// production caller: `send_edgezero_response` consumes this value only in +/// its `Err` branch and re-derives the success outcome from +/// `streaming_body.finish()`. +fn classify_stream_delivery( + drive_result: &Result<(), Report>, + bytes: u64, +) -> DeliveryResult { + match drive_result { + Ok(()) => DeliveryResult::Complete, + Err(_) if bytes > 0 => DeliveryResult::Partial, + Err(_) => DeliveryResult::Error, + } +} + +/// Stamps `resp_bytes`/`request_elapsed` for an already-materialized body, +/// immediately before it is handed to the Fastly client transport, and +/// returns its byte length. +fn record_buffered_delivery(body: &EdgeBody, timings: &RequestTimings) -> u64 { + let bytes = u64::try_from(body.as_bytes().map(<[u8]>::len).unwrap_or(0)).unwrap_or(u64::MAX); + timings.set_resp_bytes(bytes); + timings.mark_request_elapsed(); + bytes +} + /// Sends a finalized `EdgeZero` response to the client. /// /// Streaming `EdgeZero` bodies commit headers first, then pipe chunks to Fastly's @@ -338,8 +675,24 @@ fn run_edgezero_pull_sync_after_send( fn send_edgezero_response( mut response: HttpResponse, request_filter_effects: Option<&RequestFilterEffects>, -) { + context: &SendContext, +) -> DeliveryOutcome { apply_terminal_response_effects(&mut response, request_filter_effects); + apply_server_timing_header( + &mut response, + &context.timings, + context.server_timing_enabled, + ); + + // Built right after the freeze point and before `into_parts()` + // consumes `response`: nothing else survives to post-send on every + // path (the request was consumed by dispatch, and `EcFinalizeState` + // is absent on asset, admin, and error paths). Skipped entirely when + // access telemetry is disabled, so the default configuration pays no + // env reads or allocations here. + let snapshot = context + .access_telemetry_enabled + .then(|| build_access_telemetry_snapshot(&response, context)); let (parts, body) = response.into_parts(); @@ -349,25 +702,133 @@ fn send_edgezero_response( parts, EdgeBody::empty(), )); - let mut streaming_body = skeleton.stream_to_client(); - match futures::executor::block_on(stream_asset_body(body, &mut streaming_body)) { - Ok(()) => { - if let Err(e) = streaming_body.finish() { + let (counting, drive_result) = + drive_streaming_body(body, skeleton.stream_to_client(), &context.timings); + let bytes = counting.bytes(); + let streaming_body = counting.into_inner(); + // Computed before `drive_result` is matched by value below, since + // the `Err` arm there moves its `Report` out. + let result = classify_stream_delivery(&drive_result, bytes); + match drive_result { + Ok(()) => match streaming_body.finish() { + Ok(()) => DeliveryOutcome { + bytes, + result: DeliveryResult::Complete, + snapshot, + }, + Err(e) => { + // Every byte was handed to the transport (the drive + // above returned Ok), but the transport itself could + // not close cleanly — the client may still see a + // truncated response. log::error!("failed to finish EdgeZero streaming body: {e}"); + DeliveryOutcome { + bytes, + result: DeliveryResult::Partial, + snapshot, + } } - } + }, Err(e) => { log::error!("EdgeZero streaming failed: {e:?}"); drop(streaming_body); + DeliveryOutcome { + bytes, + result, + snapshot, + } } } } once => { + let bytes = record_buffered_delivery(&once, &context.timings); compat::to_fastly_response(HttpResponse::from_parts(parts, once)).send_to_client(); + DeliveryOutcome { + bytes, + result: DeliveryResult::Complete, + snapshot, + } } } } +/// Builds the [`AccessTelemetrySnapshot`] for `response` at the +/// `Server-Timing` freeze point. +/// +/// Reads route identity, geo country, and template-cache state from typed +/// response extensions rather than the headers those extensions back — +/// operator-configured response headers can override a managed header, so +/// reading a header here could silently drift from what actually happened. +/// Falls back to `"unknown"`/[`RouteClass::Other`] sentinels when an +/// extension was never attached (router-generated, asset, and other +/// responses that never passed through a `RouteMetadata`-attaching +/// wrapper). +fn build_access_telemetry_snapshot( + response: &HttpResponse, + context: &SendContext, +) -> AccessTelemetrySnapshot { + let (route_class, route_template) = match response.extensions().get::() { + Some(metadata) => (metadata.route_class, metadata.route_template.clone()), + None => (RouteClass::Other, "unknown".to_owned()), + }; + + let country = match response.extensions().get::() { + Some(GeoLookupState::Resolved(info)) => info.country.clone(), + Some(GeoLookupState::Attempted | GeoLookupState::NotAttempted) | None => { + "unknown".to_owned() + } + }; + + let template_cache_state = response + .extensions() + .get::() + .map_or_else(|| "unknown".to_owned(), |state| state.as_str().to_owned()); + + let body_mode = if matches!(response.body(), EdgeBody::Stream(_)) { + "streamed" + } else { + "buffered" + }; + + AccessTelemetrySnapshot { + method: context.method.clone(), + status: response.status().as_u16(), + route_class, + route_template, + publisher_domain: context.publisher_domain.clone(), + env: resolve_env_dimension(), + service_id: env_var_or_unknown(ENV_FASTLY_SERVICE_ID), + pop: env_var_or_unknown(ENV_FASTLY_POP), + ts_version: env_var_or_unknown(ENV_FASTLY_SERVICE_VERSION), + country, + template_cache_state, + body_mode, + sample_rate: context.access_sample_rate, + } +} + +/// Derives the `env` access-telemetry dimension from the same +/// `FASTLY_IS_STAGING` input that drives the `x-ts-env` response header +/// (see [`apply_finalize_headers`]), never from [`Settings`] — `Settings` +/// has no environment field and does not gain one for this. +/// +/// `"unknown"` covers contexts where the variable is entirely absent (for +/// example native unit tests run outside Fastly Compute); on the Fastly +/// platform the variable is always present, as either `"1"` or not. +fn resolve_env_dimension() -> String { + match std::env::var(ENV_FASTLY_IS_STAGING) { + Ok(value) if value == "1" => "staging".to_owned(), + Ok(_) => "production".to_owned(), + Err(_) => "unknown".to_owned(), + } +} + +/// Reads a Fastly-provided environment variable, defaulting to `"unknown"` +/// when unset. +fn env_var_or_unknown(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()) +} + /// Apply every late response mutation, then restore privacy invariants before headers commit. fn apply_terminal_response_effects( response: &mut HttpResponse, @@ -439,12 +900,21 @@ fn build_ja4_debug_response(req: &FastlyRequest) -> FastlyResponse { .with_body(body) } -pub(crate) fn maybe_identity_graph(settings: &Settings) -> Option { - settings - .ec - .ec_store - .as_ref() - .map(|store_name| KvIdentityGraph::new(FastlyEcKvStore::new(store_name))) +/// Constructs a `KvIdentityGraph` wrapped in the [`Phase::EcKv`] timing +/// decorator, for request-path callers with a `RequestTimings` handle. +/// +/// Returns `None` when `ec.ec_store` is not configured, matching +/// [`require_identity_graph_with_timing`]'s contract on every other axis. +pub(crate) fn identity_graph_with_timing( + settings: &Settings, + timings: &RequestTimings, +) -> Option { + settings.ec.ec_store.as_ref().map(|store_name| { + KvIdentityGraph::new(TimedKvStore::new( + FastlyEcKvStore::new(store_name), + timings.clone(), + )) + }) } fn run_pull_sync_after_send( @@ -467,6 +937,12 @@ fn run_pull_sync_after_send( /// Constructs a `KvIdentityGraph` from settings, or returns an error if the /// `ec_store` config is not set. +/// +/// Deliberately untimed: pull-sync (this function's only caller) runs after +/// `send_edgezero_response`'s Server-Timing freeze point, so a decorated +/// store here would record into a handle nothing ever renders. +/// Request-path callers with a `RequestTimings` handle use +/// [`require_identity_graph_with_timing`] instead. pub(crate) fn require_identity_graph( settings: &Settings, ) -> Result> { @@ -479,6 +955,27 @@ pub(crate) fn require_identity_graph( Ok(KvIdentityGraph::new(FastlyEcKvStore::new(store_name))) } +/// Constructs a `KvIdentityGraph` wrapped in the [`Phase::EcKv`] timing +/// decorator, or returns an error if the `ec_store` config is not set. +/// +/// Request-path sibling of [`require_identity_graph`], which pull-sync uses +/// unwrapped because pull-sync runs after the Server-Timing freeze point. +pub(crate) fn require_identity_graph_with_timing( + settings: &Settings, + timings: &RequestTimings, +) -> Result> { + let store_name = settings.ec.ec_store.as_deref().ok_or_else(|| { + Report::new(TrustedServerError::KvStore { + store_name: "ec.ec_store".to_owned(), + message: "ec.ec_store is not configured".to_owned(), + }) + })?; + Ok(KvIdentityGraph::new(TimedKvStore::new( + FastlyEcKvStore::new(store_name), + timings.clone(), + ))) +} + /// Extracts a named cookie value from the request's `Cookie` header. pub(crate) fn extract_cookie_value(req: &HttpRequest, name: &str) -> Option { let cookie_header = req.headers().get("cookie").and_then(|v| v.to_str().ok())?; @@ -508,11 +1005,14 @@ pub(crate) fn derive_device_signals(req: &FastlyRequest) -> DeviceSignals { #[cfg(test)] mod tests { use super::*; + use base64::Engine as _; use edgezero_core::body::Body as EdgeBody; use edgezero_core::http::HeaderValue; use edgezero_core::http::response_builder; use fastly::mime; + use std::time::Duration; use trusted_server_core::integrations::HeaderMutation; + use trusted_server_core::request_timing::AuctionWaitPlacement; fn test_settings() -> Settings { Settings::from_toml( @@ -540,6 +1040,26 @@ mod tests { .expect("should parse test settings") } + /// A minimal [`AccessTelemetrySnapshot`] fixture for tests that only + /// need a `DeliveryOutcome` to exist, not its telemetry content. + fn sample_access_snapshot() -> AccessTelemetrySnapshot { + AccessTelemetrySnapshot { + method: "GET".to_owned(), + status: 200, + route_class: RouteClass::Other, + route_template: "/other/*".to_owned(), + publisher_domain: "unknown".to_owned(), + env: "unknown".to_owned(), + service_id: "unknown".to_owned(), + pop: "unknown".to_owned(), + ts_version: "unknown".to_owned(), + country: "unknown".to_owned(), + template_cache_state: "unknown".to_owned(), + body_mode: "buffered", + sample_rate: 0.0, + } + } + #[test] fn health_response_short_circuits_get_health() { let req = FastlyRequest::get("https://example.com/health"); @@ -775,9 +1295,10 @@ mod tests { .body(EdgeBody::empty()) .expect("should build response"); - let geo_info = resolve_geo_for_response(&response, None, |_| { - panic!("should skip entry-point geo lookup for 401 responses"); - }); + let geo_info = + resolve_geo_for_response(&response, &GeoLookupState::NotAttempted, None, |_| { + panic!("should skip entry-point geo lookup for 401 responses"); + }); apply_finalize_headers(&settings, geo_info.as_ref(), &mut response); assert_eq!( @@ -843,4 +1364,408 @@ mod tests { "should include sec-ch-ua-platform fallback" ); } + + fn ec_finalize_settings() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + ec_store = "ec_identity_store" + + [[ec.partners]] + name = "Example Partner" + source_domain = "example.com" + api_token = "test-vendor-token-32-bytes-minimum" + + [request_signing] + enabled = false + config_store_id = "test-config-store-id" + secret_store_id = "test-secret-store-id" + "#, + ) + .expect("should parse EC finalize test settings") + } + + /// Minimal `RuntimeServices` for `EcFinalizeState.services`. Real + /// `FastlyPlatform*` handles are used as inert placeholders: EC + /// finalization never calls through them, it only satisfies the field. + fn inert_runtime_services() -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore) + as Arc) + .backend(Arc::new(crate::platform::FastlyPlatformBackend)) + .http_client(Arc::new(crate::platform::FastlyPlatformHttpClient)) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(trusted_server_core::platform::ClientInfo::default()) + .build() + } + + #[test] + fn ec_finalize_kv_lands_before_freeze() { + // A pre-seeded EC entry (see fastly.toml's ec_identity_store fixture) + // for a returning user carrying an eids cookie that matches the + // configured partner. This drives ec_finalize_response into + // ingest_eid_cookies, which reads and writes the KV identity graph. + let settings = ec_finalize_settings(); + let ec_id = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01"; + let eids = serde_json::json!([{ + "source": "example.com", + "uids": [{ "id": "example-uid", "atype": 1 }] + }]); + let eids_cookie = base64::engine::general_purpose::STANDARD.encode(eids.to_string()); + let request = edgezero_core::http::request_builder() + .method(fastly::http::Method::GET) + .uri("https://test-publisher.com/article") + .header("cookie", format!("ts-ec={ec_id}; ts-eids={eids_cookie}")) + .body(EdgeBody::empty()) + .expect("should build EC finalize test request"); + + let services = inert_runtime_services(); + let geo_info = trusted_server_core::platform::GeoInfo { + city: String::new(), + country: "US".to_owned(), + continent: "NorthAmerica".to_owned(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + }; + let ec_context = trusted_server_core::ec::EcContext::read_from_request_with_geo( + &settings, + &request, + &services, + Some(&geo_info), + ) + .expect("should read EC context from a non-regulated request"); + assert!( + ec_context.ec_was_present(), + "the pre-seeded ts-ec cookie should be recognized" + ); + + let mut ec_state = EcFinalizeState { + ec_context, + use_finalize_kv: true, + eids_cookie: Some(eids_cookie), + sharedid_cookie: None, + is_real_browser: true, + services, + }; + let mut response = response_builder() + .header("cache-control", "private, no-store") + .body(EdgeBody::empty()) + .expect("should build EC finalize response fixture"); + let timings = RequestTimings::new(); + + // Mirrors edgezero_main's ordering: EC finalize runs, then the freeze + // point (apply_server_timing_header, called just before + // response.into_parts() inside send_edgezero_response) renders the + // header. Calling both directly exercises exactly this order without + // requiring a live Fastly client connection. + apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response, &timings) + .expect("should finalize EC response"); + apply_server_timing_header(&mut response, &timings, true); + + let header = response + .headers() + .get("server-timing") + .and_then(|v| v.to_str().ok()) + .expect("should emit a Server-Timing header"); + assert!( + header.contains("ts-kv"), + "the freeze point must run after EC finalization recorded KV time: {header}" + ); + } + + #[test] + fn delivery_outcome_reports_bytes_and_request_elapsed_set() { + let timings = RequestTimings::new(); + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"hello "), + bytes::Bytes::from_static(b"world"), + ])); + + let (counting, drive_result) = drive_streaming_body(body, Vec::new(), &timings); + drive_result.expect("streaming a well-formed body should not fail"); + let bytes = counting.bytes(); + let outcome = DeliveryOutcome { + bytes, + result: DeliveryResult::Complete, + snapshot: Some(sample_access_snapshot()), + }; + + assert_eq!( + counting.into_inner(), + b"hello world", + "should write every byte to the underlying transport" + ); + assert_eq!( + outcome.bytes, + "hello world".len() as u64, + "DeliveryOutcome.bytes should equal the streamed body length" + ); + + let snapshot = timings.snapshot(); + assert_eq!( + snapshot.resp_bytes, + Some("hello world".len() as u64), + "should stamp resp_bytes to the tallied byte count" + ); + assert!( + snapshot.request_elapsed_ms.is_some(), + "should stamp request_elapsed once the drive returns" + ); + } + + #[test] + fn buffered_delivery_stamps_bytes_and_request_elapsed() { + let timings = RequestTimings::new(); + let body = EdgeBody::from(b"a buffered body".to_vec()); + + let bytes = record_buffered_delivery(&body, &timings); + + assert_eq!( + bytes, + "a buffered body".len() as u64, + "should report the buffered body length" + ); + let snapshot = timings.snapshot(); + assert_eq!( + snapshot.resp_bytes, + Some("a buffered body".len() as u64), + "should stamp resp_bytes for the buffered path too" + ); + assert!( + snapshot.request_elapsed_ms.is_some(), + "should stamp request_elapsed for the buffered path too" + ); + } + + fn send_context_fixture() -> SendContext { + SendContext { + timings: RequestTimings::new(), + server_timing_enabled: false, + method: "GET".to_owned(), + publisher_domain: "test-publisher.com".to_owned(), + access_sample_rate: 0.25, + access_telemetry_enabled: true, + } + } + + #[test] + fn access_snapshot_defaults_when_no_extensions_are_attached() { + // Router-generated 404/405 responses and other paths that never pass + // through a RouteMetadata-attaching wrapper must still produce a + // usable snapshot: RouteClass::Other and "unknown" sentinels, never + // a missing/panicking build. + let response = response_builder() + .status(404) + .body(EdgeBody::empty()) + .expect("should build response"); + let context = send_context_fixture(); + + let snapshot = build_access_telemetry_snapshot(&response, &context); + + assert_eq!(snapshot.status, 404); + assert_eq!(snapshot.method, "GET"); + assert!(matches!(snapshot.route_class, RouteClass::Other)); + assert_eq!(snapshot.route_template, "unknown"); + assert_eq!(snapshot.country, "unknown"); + assert_eq!(snapshot.template_cache_state, "unknown"); + assert_eq!(snapshot.body_mode, "buffered"); + assert_eq!(snapshot.publisher_domain, "test-publisher.com"); + assert_eq!(snapshot.sample_rate, 0.25); + } + + #[test] + fn access_snapshot_reads_route_geo_and_template_cache_extensions() { + let mut response = response_builder() + .status(200) + .body(EdgeBody::empty()) + .expect("should build response"); + response.extensions_mut().insert(RouteMetadata { + route_class: RouteClass::AuctionApi, + route_template: "/auction".to_owned(), + }); + response.extensions_mut().insert(GeoLookupState::Resolved( + trusted_server_core::platform::GeoInfo { + city: String::new(), + country: "US".to_owned(), + continent: "NorthAmerica".to_owned(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + }, + )); + response + .extensions_mut() + .insert(TemplateCacheResponseState::Hit); + let context = send_context_fixture(); + + let snapshot = build_access_telemetry_snapshot(&response, &context); + + assert!(matches!(snapshot.route_class, RouteClass::AuctionApi)); + assert_eq!(snapshot.route_template, "/auction"); + assert_eq!(snapshot.country, "US"); + assert_eq!(snapshot.template_cache_state, "hit"); + } + + #[test] + fn access_snapshot_treats_attempted_geo_lookup_as_unknown_country() { + let mut response = response_builder() + .status(200) + .body(EdgeBody::empty()) + .expect("should build response"); + response.extensions_mut().insert(GeoLookupState::Attempted); + let context = send_context_fixture(); + + let snapshot = build_access_telemetry_snapshot(&response, &context); + + assert_eq!( + snapshot.country, "unknown", + "an attempted-but-unresolved lookup must not surface a stale country" + ); + } + + #[test] + fn access_snapshot_body_mode_reflects_the_response_body_variant() { + let streamed = response_builder() + .status(200) + .body(EdgeBody::stream(futures::stream::empty())) + .expect("should build streaming response"); + let buffered = response_builder() + .status(200) + .body(EdgeBody::from(b"hi".to_vec())) + .expect("should build buffered response"); + let context = send_context_fixture(); + + assert_eq!( + build_access_telemetry_snapshot(&streamed, &context).body_mode, + "streamed" + ); + assert_eq!( + build_access_telemetry_snapshot(&buffered, &context).body_mode, + "buffered" + ); + } + + #[test] + fn stream_drive_records_stream_ms_covering_the_in_stream_auction_wait() { + // A streaming seam wait (Task 6, publisher.rs) records into the same + // `RequestTimings` handle the adapter drives with. `Phase::Stream` + // wraps the entire drive, so it must cover — and therefore be at + // least as large as — any `AuctionWait` recorded while the body was + // being polled. + let timings = RequestTimings::new(); + let wait_timings = timings.clone(); + let stream = futures::stream::once(async move { + let waited = Duration::from_millis(5); + std::thread::sleep(waited); + wait_timings.record_auction_wait(AuctionWaitPlacement::InStream, waited); + bytes::Bytes::from_static(b"") + }); + let body = EdgeBody::stream(stream); + + let (_counting, drive_result) = drive_streaming_body(body, Vec::new(), &timings); + drive_result.expect("streaming a well-formed body should not fail"); + + let snapshot = timings.snapshot(); + assert_eq!( + snapshot.auction_wait_placement, + Some(AuctionWaitPlacement::InStream), + "should preserve the placement recorded from inside the polled body" + ); + let auction_wait_ms = snapshot + .auction_wait_ms + .expect("should record the auction wait"); + let stream_ms = snapshot.stream_ms.expect("should record the stream drive"); + assert!( + stream_ms >= auction_wait_ms, + "the drive's Phase::Stream span must cover the in-stream auction wait: \ + stream_ms={stream_ms} auction_wait_ms={auction_wait_ms}" + ); + } + + #[test] + fn classify_stream_delivery_treats_bytes_written_before_an_error_as_partial() { + let err = Report::new(TrustedServerError::Proxy { + message: "boom".to_string(), + }); + assert_eq!( + classify_stream_delivery(&Err(err), 42), + DeliveryResult::Partial, + "bytes already on the wire before a stream error is a truncated delivery" + ); + } + + #[test] + fn classify_stream_delivery_treats_an_error_with_no_bytes_as_error() { + let err = Report::new(TrustedServerError::Proxy { + message: "boom".to_string(), + }); + assert_eq!( + classify_stream_delivery(&Err(err), 0), + DeliveryResult::Error, + "a failure before any byte reached the client is a clean failure, not a truncation" + ); + } + + #[test] + fn classify_stream_delivery_treats_ok_as_complete() { + assert_eq!( + classify_stream_delivery(&Ok(()), 123), + DeliveryResult::Complete + ); + } + + #[test] + fn request_elapsed_is_stamped_when_send_returns() { + // `edgezero_main`'s post-send ordering (pull-sync before telemetry) + // is a source-order invariant with no injectable seam, so this test + // deliberately proves only the leg that has one: by the time + // `send_edgezero_response` returns, `request_elapsed` is already + // stamped, so everything `edgezero_main` runs afterwards (pull-sync, + // telemetry emission) is excluded from `request_elapsed_ms`. + let timings = RequestTimings::new(); + let response = response_builder() + .body(EdgeBody::from("ok")) + .expect("should build response"); + + let outcome = send_edgezero_response( + response, + None, + &SendContext { + timings: timings.clone(), + server_timing_enabled: false, + method: "GET".to_owned(), + publisher_domain: "test-publisher.com".to_owned(), + access_sample_rate: 1.0, + access_telemetry_enabled: true, + }, + ); + + assert!( + timings.snapshot().request_elapsed_ms.is_some(), + "request_elapsed should be stamped by the time send returns" + ); + assert!( + outcome.snapshot.is_some(), + "the access snapshot should exist for the enabled context" + ); + } } diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 283f16255..79ded8362 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -24,8 +24,9 @@ use trusted_server_core::constants::{ ENV_FASTLY_IS_STAGING, ENV_FASTLY_SERVICE_VERSION, HEADER_X_GEO_INFO_AVAILABLE, HEADER_X_TS_ENV, HEADER_X_TS_VERSION, }; -use trusted_server_core::geo::GeoInfo; +use trusted_server_core::geo::{GeoInfo, GeoLookupState}; use trusted_server_core::platform::{ClientInfo, PlatformGeo}; +use trusted_server_core::request_timing::{Phase, RequestTimings}; use trusted_server_core::settings::Settings; pub(crate) const HEADER_X_TS_FINALIZED: &str = "x-ts-finalized"; @@ -71,6 +72,12 @@ impl Middleware for FinalizeResponseMiddleware { || FastlyRequestContext::get(ctx.request()).and_then(|c| c.client_ip), |info| info.client_ip, ); + let timings = ctx + .request() + .extensions() + .get::() + .cloned() + .unwrap_or_default(); let mut response = match next.run(ctx).await { Ok(r) => r, @@ -80,13 +87,31 @@ impl Middleware for FinalizeResponseMiddleware { } }; - let geo_info = resolve_geo_for_response(&response, client_ip, |ip| { + let carried = response + .extensions() + .get::() + .cloned() + .unwrap_or(GeoLookupState::NotAttempted); + let geo_info = resolve_geo_for_response(&response, &carried, client_ip, |ip| { + let _span = timings.span(Phase::Geo); self.geo.lookup(ip).unwrap_or_else(|e| { log::warn!("geo lookup failed: {e}"); None }) }); + // Write the resolved outcome back so a downstream access-telemetry + // snapshot (built from response extensions after finalize) sees + // what was actually looked up here rather than the stale carried-in + // state — mirrors the entry-point finalize site in `main.rs` + // (`apply_entry_point_finalize_headers`), which writes back for the + // same reason. + let resolved_state = match &geo_info { + Some(geo) => GeoLookupState::Resolved(geo.clone()), + None => GeoLookupState::Attempted, + }; + response.extensions_mut().insert(resolved_state); + apply_finalize_headers(&self.settings, geo_info.as_ref(), &mut response); response .headers_mut() @@ -145,14 +170,20 @@ impl Middleware for AuthMiddleware { // Shared geo resolution helper // --------------------------------------------------------------------------- -/// Resolves geo for a response, skipping the lookup for 401 responses. +/// Resolves geo for a response, skipping the lookup for 401 responses and +/// reusing a request-phase lookup when one was already carried. /// -/// Returns `None` for authentication rejections (401) without calling `lookup_geo` -/// to avoid unnecessary work and exposing geo data to unauthenticated callers. -/// All other responses call `lookup_geo` and return its result. +/// Returns `None` for authentication rejections (401) without consulting +/// `carried` or calling `lookup_geo`, to avoid unnecessary work and exposing +/// geo data to unauthenticated callers. Otherwise dispatches on `carried`: +/// a [`GeoLookupState::Resolved`] value is reused as-is, a +/// [`GeoLookupState::Attempted`] value is treated as no geo info without +/// retrying the lookup, and [`GeoLookupState::NotAttempted`] falls back to +/// calling `lookup_geo`. /// /// Used by both [`FinalizeResponseMiddleware`] and the entry-point finalization -/// in `main.rs` so the 401-skip rule is defined in one place. +/// in `main.rs` so the 401-skip rule and the dedupe rule are each defined in +/// one place. /// /// # Parity note /// @@ -164,6 +195,7 @@ impl Middleware for AuthMiddleware { /// server or the upstream origin. pub(crate) fn resolve_geo_for_response( response: &Response, + carried: &GeoLookupState, client_ip: Option, lookup_geo: F, ) -> Option @@ -171,9 +203,12 @@ where F: FnOnce(Option) -> Option, { if response.status() == StatusCode::UNAUTHORIZED { - None - } else { - lookup_geo(client_ip) + return None; + } + match carried { + GeoLookupState::Resolved(geo) => Some(geo.clone()), + GeoLookupState::Attempted => None, + GeoLookupState::NotAttempted => lookup_geo(client_ip), } } @@ -280,6 +315,19 @@ mod tests { RequestContext::new(req, PathParams::new(HashMap::new())) } + fn sample_geo_info() -> GeoInfo { + GeoInfo { + city: "Testville".to_string(), + country: "US".to_string(), + continent: "NorthAmerica".to_string(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + } + } + struct FixedGeo(Option); impl PlatformGeo for FixedGeo { @@ -682,6 +730,67 @@ mod tests { ); } + #[test] + fn finalize_handle_writes_back_resolved_geo_state_after_fallback_lookup() { + // The request phase never attempted a geo lookup (no GeoLookupState + // extension on the handler's response), so the middleware resolves + // one via the fallback closure. That resolved outcome must be + // written back into response extensions -- mirroring + // apply_entry_point_finalize_headers in main.rs -- so a downstream + // access-telemetry snapshot sees the freshly resolved country + // instead of a stale/missing GeoLookupState. + let settings = settings_with_response_headers(vec![]); + let middleware = FinalizeResponseMiddleware::new( + Arc::new(settings), + Arc::new(FixedGeo(Some(sample_geo_info()))), + ); + let handler = + Arc::new( + |_ctx: RequestContext| async move { Ok::(empty_response()) }, + ); + + let response = block_on(middleware.handle(empty_ctx(), Next::new(&[], &*handler))) + .expect("should succeed"); + + match response.extensions().get::() { + Some(GeoLookupState::Resolved(info)) => { + assert_eq!( + info.country, "US", + "should carry the fallback-resolved geo info" + ); + } + other => { + panic!("expected GeoLookupState::Resolved after a fallback lookup, got {other:?}") + } + } + } + + #[test] + fn finalize_handle_writes_back_attempted_geo_state_when_fallback_finds_nothing() { + // The fallback lookup ran but resolved no geo info. The middleware + // must still record that the lookup was attempted, so a later + // consumer of the extension does not mistake this for + // GeoLookupState::NotAttempted and retry the lookup. + let settings = settings_with_response_headers(vec![]); + let middleware = + FinalizeResponseMiddleware::new(Arc::new(settings), Arc::new(FixedGeo(None))); + let handler = + Arc::new( + |_ctx: RequestContext| async move { Ok::(empty_response()) }, + ); + + let response = block_on(middleware.handle(empty_ctx(), Next::new(&[], &*handler))) + .expect("should succeed"); + + assert!( + matches!( + response.extensions().get::(), + Some(GeoLookupState::Attempted) + ), + "should write back Attempted when the fallback lookup finds no geo info" + ); + } + #[test] fn finalize_handle_marks_response_as_finalized() { let settings = settings_with_response_headers(vec![]); @@ -763,6 +872,30 @@ mod tests { ); } + #[test] + #[allow(clippy::panic)] + fn geo_lookup_skipped_for_unauthorized_responses() { + // The 401 short-circuit in resolve_geo_for_response must win + // regardless of what state the request phase carried in, and must + // never invoke the fallback lookup closure. + let mut response = empty_response(); + *response.status_mut() = StatusCode::UNAUTHORIZED; + + for carried in [ + GeoLookupState::NotAttempted, + GeoLookupState::Attempted, + GeoLookupState::Resolved(sample_geo_info()), + ] { + let geo_info = resolve_geo_for_response(&response, &carried, None, |_| { + panic!("401 responses must never trigger a geo lookup"); + }); + assert!( + geo_info.is_none(), + "401 responses should never resolve geo info, regardless of carried state" + ); + } + } + // --------------------------------------------------------------------------- // AuthMiddleware::handle tests // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-fastly/src/tinybird.rs b/crates/trusted-server-adapter-fastly/src/tinybird.rs index f315a7b56..85c77e53b 100644 --- a/crates/trusted-server-adapter-fastly/src/tinybird.rs +++ b/crates/trusted-server-adapter-fastly/src/tinybird.rs @@ -10,10 +10,15 @@ use trusted_server_core::auction::telemetry::{ AuctionEventBatch, AuctionTelemetrySink, NoopAuctionTelemetrySink, }; use trusted_server_core::error::TrustedServerError; -use trusted_server_core::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use trusted_server_core::platform::{ + PlatformBackend as _, PlatformBackendSpec, PlatformHttpClient, PlatformHttpRequest, + RuntimeServices, +}; use trusted_server_core::redacted::Redacted; use trusted_server_core::settings::{Settings, TinybirdSettings}; +use crate::platform::FastlyPlatformBackend; + const TINYBIRD_EVENTS_PATH: &str = "/v0/events"; const TINYBIRD_NDJSON_CONTENT_TYPE: &str = "application/x-ndjson"; const TINYBIRD_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(2); @@ -21,9 +26,14 @@ const TINYBIRD_BETWEEN_BYTES_TIMEOUT: Duration = Duration::from_secs(2); const TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH: usize = 512; /// Build the configured auction telemetry sink. +/// +/// Auction emission requires both the Tinybird master toggle +/// (`tinybird.enabled`) and the auction-specific toggle +/// (`tinybird.auction_enabled`), so access-log telemetry can be enabled +/// independently without also emitting auction events. #[must_use] pub(crate) fn auction_sink_from_settings(settings: &Settings) -> Arc { - if settings.tinybird.enabled { + if settings.tinybird.enabled && settings.tinybird.auction_enabled { Arc::new(FastlyTinybirdAuctionTelemetrySink::new( settings.tinybird.clone(), )) @@ -39,7 +49,7 @@ struct FastlyTinybirdAuctionTelemetrySink { } #[derive(Debug, Clone)] -struct TinybirdEventsTarget { +pub(crate) struct TinybirdEventsTarget { api_host: String, dataset: String, append_token: Redacted, @@ -63,6 +73,28 @@ impl TinybirdEventsTarget { max_body_bytes: config.max_body_bytes, } } + + /// Builds the Events API target for the access-log datasource. + /// + /// Shares [`from_config`](Self::from_config)'s host, resolved-token, and + /// body-size-limit derivation, but points at `access_dataset` and + /// `access_token_secret` instead of the auction pair, so access-log + /// emission never shares a datasource or token with auction telemetry + /// even though both configs come from the same [`TinybirdSettings`]. + pub(crate) fn from_access_config(config: TinybirdSettings) -> Self { + let uri = tinybird_events_uri(&config.api_host, &config.access_dataset); + let backend_spec = tinybird_backend_spec(&config.api_host); + Self { + api_host: config.api_host, + dataset: config.access_dataset, + append_token: config + .access_token_secret + .expect("should contain a resolved Tinybird access token when enabled"), + uri, + backend_spec, + max_body_bytes: config.max_body_bytes, + } + } } impl FastlyTinybirdAuctionTelemetrySink { @@ -182,6 +214,124 @@ impl AuctionTelemetrySink for FastlyTinybirdAuctionTelemetrySink { } } +// --------------------------------------------------------------------------- +// Access telemetry: confirmed-delivery emitter +// --------------------------------------------------------------------------- + +/// Decides whether one request's access-telemetry row should be emitted. +/// +/// `roll` is a uniform draw from `[0, 1)`; callers pass +/// `rand::thread_rng().r#gen::()`, which the wasm32-wasip1 guest backs with +/// real WASI randomness (the EC generation path already relies on this and +/// the CI wasm release build verifies it). Comparing the draw directly +/// against `rate` keeps the sampling probability exactly `rate` for every +/// positive value: there is no bucket quantization, so rates below one in a +/// million sample proportionally instead of never, and emitted rows' +/// `sample_rate` matches the probability they were sampled at, which the +/// `sum(1.0 / sample_rate)` volume estimator depends on. +/// +/// `rate <= 0.0` never samples and `rate >= 1.0` always samples, for any +/// `roll` in `[0, 1)`. `0.0` cannot actually occur while `access_enabled` +/// is `true` (`Settings` validation requires `access_sample_rate > 0.0` in +/// that case), but this function stays total rather than leaning on that +/// invariant. +#[must_use] +pub(crate) fn sampled_in(rate: f64, roll: f64) -> bool { + roll < rate +} + +/// Builds the Events API POST request for one access-log row. +fn build_access_events_request( + target: &TinybirdEventsTarget, + body: String, + auth_header: HeaderValue, +) -> Result> { + request_builder() + .method(Method::POST) + .uri(target.uri.as_str()) + .header(header::AUTHORIZATION, auth_header) + .header(header::CONTENT_TYPE, TINYBIRD_NDJSON_CONTENT_TYPE) + .body(Body::from(body)) + .change_context(TrustedServerError::Proxy { + message: "failed to build Tinybird Events API request".to_owned(), + }) +} + +/// Sends one confirmed access-log row to the Tinybird Events API and waits +/// for the response. +/// +/// Unlike [`FastlyTinybirdAuctionTelemetrySink::emit_auction_events`] (fire- +/// and-forget, dispatched mid-request so it never adds latency to the +/// response), this runs post-delivery: the response has already reached the +/// client, so there is no latency budget left to protect, and the send can +/// afford to wait for — and validate — the reply. `client` is the adapter's +/// stateless platform HTTP client in production +/// ([`crate::platform::FastlyPlatformHttpClient`]); accepting it as `&dyn +/// PlatformHttpClient` here (rather than that concrete type) is what lets +/// tests substitute a recording double instead of performing a real network +/// send, matching how [`RuntimeServices::http_client`] is consumed +/// elsewhere. `target` is derived from settings once at the post-send call +/// site rather than threaded through any per-route state. +/// +/// A non-2xx status is reported as `Err` naming the status; there is no +/// retry — the caller logs exactly one warning and moves on. +/// +/// # Errors +/// +/// Returns `Err` when the row exceeds the configured request-body limit, the +/// resolved access-log APPEND token is invalid, the backend cannot be registered, +/// the request cannot be built or sent, or the Tinybird Events API responds +/// with a non-2xx status. +pub(crate) async fn emit_access_event( + client: &dyn PlatformHttpClient, + target: &TinybirdEventsTarget, + row: String, +) -> Result<(), Report> { + let body_len = row.len(); + if body_len > target.max_body_bytes { + return Err(Report::new(TrustedServerError::Proxy { + message: format!( + "Tinybird access telemetry request body has {body_len} bytes, exceeding {} byte limit", + target.max_body_bytes + ), + })); + } + + let auth_header = + FastlyTinybirdAuctionTelemetrySink::authorization_header(target.append_token.expose())?; + let backend_name = FastlyPlatformBackend + .ensure(&target.backend_spec) + .change_context(TrustedServerError::Proxy { + message: "Tinybird backend registration failed".to_owned(), + })?; + let request = build_access_events_request(target, row, auth_header)?; + + log::info!( + "sending access telemetry to Tinybird dataset={} host={} backend={}", + target.dataset, + target.api_host, + backend_name + ); + + let response = client + .send(PlatformHttpRequest::new(request, backend_name)) + .await + .change_context(TrustedServerError::Proxy { + message: "failed to send Tinybird access telemetry request".to_owned(), + })?; + + if response.response.status().is_success() { + Ok(()) + } else { + Err(Report::new(TrustedServerError::Proxy { + message: format!( + "Tinybird access telemetry request failed with status {}", + response.response.status() + ), + })) + } +} + fn tinybird_backend_spec(api_host: &str) -> PlatformBackendSpec { PlatformBackendSpec { scheme: "https".to_owned(), @@ -308,25 +458,28 @@ mod tests { body: Vec, } + /// Records outbound requests and, for [`PlatformHttpClient::send`] (the + /// blocking variant `emit_access_event` uses), returns a synthetic + /// response carrying `respond_status` instead of performing a real + /// network send. #[derive(Default)] struct RecordingHttpClient { requests: Mutex>, select_calls: Mutex, + respond_status: Mutex, } - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for RecordingHttpClient { - async fn send( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) + impl RecordingHttpClient { + /// Status [`PlatformHttpClient::send`] should reply with. Irrelevant + /// to auction-sink tests, which only exercise `send_async`. + fn respond_with(status: u16) -> Self { + Self { + respond_status: Mutex::new(status), + ..Self::default() + } } - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { + fn record(&self, request: PlatformHttpRequest) { let backend_name = request.backend_name; let (parts, body) = request.request.into_parts(); let headers = parts @@ -350,6 +503,35 @@ mod tests { .lock() .expect("should lock recorded requests") .push(recorded); + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for RecordingHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.record(request); + let status = *self + .respond_status + .lock() + .expect("should lock configured response status"); + let response = edgezero_core::http::response_builder() + .status( + edgezero_core::http::StatusCode::from_u16(status) + .expect("should build a valid test status code"), + ) + .body(edgezero_core::body::Body::empty()) + .expect("should build test response"); + Ok(PlatformResponse::new(response)) + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.record(request); Ok(PlatformPendingRequest::new(()).with_backend_name("tinybird-backend")) } @@ -430,18 +612,52 @@ mod tests { fn enabled_config() -> TinybirdSettings { TinybirdSettings { enabled: true, + auction_enabled: true, api_host: "api.us-east.aws.tinybird.co".to_owned(), secret_store: None, auction_dataset: "auction_events_raw".to_owned(), auction_token_secret: Some(Redacted::new("append-token".to_owned())), access_enabled: false, access_dataset: "access_logs_raw".to_owned(), - access_token_secret: None, + access_token_secret: Some(Redacted::new("access-append-token".to_owned())), access_sample_rate: 0.0, max_body_bytes: 1024 * 1024, } } + #[test] + fn sink_from_settings_disables_when_auction_enabled_is_false() { + let settings = Settings { + tinybird: TinybirdSettings { + auction_enabled: false, + ..enabled_config() + }, + ..Settings::default() + }; + + let sink = auction_sink_from_settings(&settings); + + assert!( + !sink.is_enabled(), + "auction telemetry should stay off when auction_enabled is false, even if tinybird.enabled is true" + ); + } + + #[test] + fn sink_from_settings_enables_when_both_toggles_are_true() { + let settings = Settings { + tinybird: enabled_config(), + ..Settings::default() + }; + + let sink = auction_sink_from_settings(&settings); + + assert!( + sink.is_enabled(), + "auction telemetry should be on when both tinybird.enabled and tinybird.auction_enabled are true" + ); + } + #[test] fn events_uri_targets_dataset_on_region_host() { assert_eq!( @@ -618,6 +834,134 @@ mod tests { ); } + #[test] + fn access_emitter_rejects_oversized_row_before_sending() { + let mut config = enabled_config(); + config.max_body_bytes = 1024; + let target = TinybirdEventsTarget::from_access_config(config); + let http_client = RecordingHttpClient::respond_with(202); + let row = "x".repeat(1025); + + let result = futures::executor::block_on(emit_access_event(&http_client, &target, row)); + + let error = result.expect_err("should reject a row above the configured body limit"); + assert!( + error.to_string().contains("1024"), + "error should name the configured body limit: {error}" + ); + assert!( + http_client + .requests + .lock() + .expect("should lock recorded requests") + .is_empty(), + "should not send an oversized access row" + ); + } + + #[test] + fn access_emitter_posts_ndjson_and_validates_2xx() { + // Runtime settings carry the access APPEND token after startup secret + // resolution, so post-delivery emission does not reopen a secret store. + let target = TinybirdEventsTarget::from_access_config(enabled_config()); + let http_client = RecordingHttpClient::respond_with(202); + let row = r#"{"status":200}"#.to_owned(); + + futures::executor::block_on(emit_access_event(&http_client, &target, row.clone())) + .expect("should accept a 202 response"); + + let requests = http_client + .requests + .lock() + .expect("should lock recorded requests"); + assert_eq!(requests.len(), 1, "should send exactly one request"); + assert_eq!( + requests[0].uri, + "https://api.us-east.aws.tinybird.co/v0/events?name=access_logs_raw" + ); + assert_eq!(requests[0].method, Method::POST.to_string()); + assert_eq!( + header_value(&requests[0].headers, header::AUTHORIZATION.as_str()), + Some("Bearer access-append-token") + ); + assert_eq!( + std::str::from_utf8(&requests[0].body).expect("should record utf8 body"), + row, + "should send the row verbatim as the request body" + ); + } + + #[test] + fn access_emitter_warns_and_drops_on_non_2xx() { + let target = TinybirdEventsTarget::from_access_config(enabled_config()); + let http_client = RecordingHttpClient::respond_with(422); + + let result = futures::executor::block_on(emit_access_event( + &http_client, + &target, + r#"{"status":422}"#.to_owned(), + )); + + let error = result.expect_err("a 422 response should be reported as an error"); + assert!( + error.to_string().contains("422"), + "error should name the failing status: {error}" + ); + assert_eq!( + http_client + .requests + .lock() + .expect("should lock recorded requests") + .len(), + 1, + "should not retry after a non-2xx response" + ); + } + + #[test] + fn sampled_in_boundary_rates_are_unconditional() { + assert!( + sampled_in(1.0, 0.0), + "a 1.0 sample rate should always sample in" + ); + assert!( + sampled_in(1.0, 0.999_999), + "a 1.0 sample rate should sample in for the largest roll" + ); + assert!( + !sampled_in(0.0, 0.0), + "a 0.0 sample rate should never sample in, even on a zero roll" + ); + assert!( + !sampled_in(-1.0, 0.0), + "a negative rate should never sample in" + ); + } + + #[test] + fn sampled_in_keeps_exact_probability_for_tiny_rates() { + // The previous bucket-quantized sampler truncated rates below one + // in a million to a zero threshold, silently emitting nothing. + // Direct comparison keeps every positive rate proportional. + let rate = 0.000_000_1; + assert!( + sampled_in(rate, rate / 2.0), + "a roll below a tiny positive rate should sample in" + ); + assert!( + !sampled_in(rate, rate * 2.0), + "a roll above a tiny positive rate should sample out" + ); + assert!( + !sampled_in(0.000_001_9, 0.000_001_95), + "no downward quantization: the boundary sits exactly at the rate" + ); + assert!( + sampled_in(0.000_001_9, 0.000_001_85), + "rolls just under the rate should sample in" + ); + } + fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { headers .iter() diff --git a/crates/trusted-server-adapter-spin/spin.toml b/crates/trusted-server-adapter-spin/spin.toml index 7055f9563..76737cea8 100644 --- a/crates/trusted-server-adapter-spin/spin.toml +++ b/crates/trusted-server-adapter-spin/spin.toml @@ -18,6 +18,8 @@ version = "0.1.0" # Operators enabling request_signing must declare one encoded secret variable for # each private signing key. Public signing metadata remains in the KV store. [variables] +v_current_x2dkid = { default = "" } +v_active_x2dkids = { default = "" } # These declared variables match the example config's secret key names. Regenerate # or extend them for deployment-specific keys, including handler key names such as # `admin_password` or `api_handler_password`. Replace the empty defaults with values @@ -44,6 +46,8 @@ allowed_outbound_hosts = ["https://*:*", "http://*:*"] key_value_stores = ["default"] [component.trusted-server.variables] +v_current_x2dkid = "{{ v_current_x2dkid }}" +v_active_x2dkids = "{{ v_active_x2dkids }}" v_trusted_x5fserver_x5fsecrets_v_publisher_x5fproxy_x5fsecret = "{{ v_trusted_x5fserver_x5fsecrets_v_publisher_x5fproxy_x5fsecret }}" v_trusted_x5fserver_x5fsecrets_v_trusted_x5fclient_x5fip_x5fshared_x5fsecret = "{{ v_trusted_x5fserver_x5fsecrets_v_trusted_x5fclient_x5fip_x5fshared_x5fsecret }}" v_trusted_x5fserver_x5fsecrets_v_ec_x5fpassphrase = "{{ v_trusted_x5fserver_x5fsecrets_v_ec_x5fpassphrase }}" diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index a81911186..9120b0fab 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -788,6 +788,12 @@ mod tests { use edgezero_core::body::Body; use edgezero_core::config_store::{ConfigStore, ConfigStoreError}; + use edgezero_core::context::RequestContext; + use edgezero_core::http::request_builder; + use edgezero_core::params::PathParams; + use flate2::Compression; + use flate2::write::GzEncoder; + use std::io::Write as _; use trusted_server_core::platform::AuctionTargetId; #[test] @@ -799,12 +805,6 @@ mod tests { "Spin outbound HTTP does not expose an enforceable hard total request deadline" ); } - use edgezero_core::context::RequestContext; - use edgezero_core::http::request_builder; - use edgezero_core::params::PathParams; - use flate2::Compression; - use flate2::write::GzEncoder; - use std::io::Write as _; struct InMemoryConfigStore(std::collections::BTreeMap); @@ -954,6 +954,30 @@ mod tests { ); } + #[test] + fn spin_variable_name_encodes_trusted_server_keys() { + assert_eq!( + spin_variable_name("current-kid", PlatformError::ConfigStore) + .expect("should encode current kid key"), + "v_current_x2dkid" + ); + assert_eq!( + spin_variable_name("active-kids", PlatformError::ConfigStore) + .expect("should encode active kids key"), + "v_active_x2dkids" + ); + assert_eq!( + spin_variable_name("ts-2026-05-25", PlatformError::ConfigStore) + .expect("should encode generated kid"), + "v_ts_x2d2026_x2d05_x2d25" + ); + // Digit-leading keys are rejected at the encoder boundary. + assert!( + spin_variable_name("2026-key", PlatformError::ConfigStore).is_err(), + "should reject digit-leading key" + ); + } + #[test] fn spin_variable_name_encodes_secret_key_components() { assert_eq!( diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index 44ad1d443..247d179a7 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -17,18 +17,24 @@ workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] chromiumoxide = { workspace = true } clap = { workspace = true } +derive_more = { workspace = true } edgezero-cli = { workspace = true } +edgezero-core = { workspace = true } futures = { workspace = true } +glob = { workspace = true } +http = { workspace = true } log = { workspace = true } rand = { workspace = true } regex = { workspace = true } scraper = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +similar = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true } toml = { workspace = true } toml_edit = { workspace = true } +tracing = { workspace = true } trusted-server-core = { workspace = true } url = { workspace = true } which = { workspace = true } @@ -42,7 +48,6 @@ which = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] base64 = { workspace = true } bytes = { workspace = true } -derive_more = { workspace = true } directories = { workspace = true } error-stack = { workspace = true } http-body-util = { workspace = true } @@ -63,4 +68,5 @@ tokio = { workspace = true, features = ["test-util"] } x509-parser = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +temp-env = { workspace = true } tempfile = { workspace = true } diff --git a/crates/trusted-server-cli/src/ad_templates/compare.rs b/crates/trusted-server-cli/src/ad_templates/compare.rs new file mode 100644 index 000000000..48ab71f3e --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/compare.rs @@ -0,0 +1,830 @@ +//! Pure comparison of configured expected slots against browser ad evidence. +//! +//! This module is collector-independent and Chrome-free: it takes decoded +//! [`BrowserAdEvidence`] plus the [`ExpectedSlot`] set and produces a +//! [`PageVerificationResult`] with per-slot statuses, warnings, and unmatched +//! extra evidence, mirroring spec §5.3–§5.6. +//! +use serde::Deserialize; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +use crate::ad_templates::expected::ExpectedSlot; +use crate::ad_templates::output::Warning; + +/// The phase in which a piece of evidence was observed. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhase { + /// Observed during the initial load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A DOM element ID observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct DomEvidence { + /// The element ID. + pub dom_id: String, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// A GPT slot observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct GptSlotEvidence { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `(width, height)` pairs (non-numeric dropped upstream). + pub sizes: Vec<(u32, u32)>, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// An `apstag.fetchBids` call the page made, if any were recorded. +/// +/// The collector no longer hooks `apstag`: server-side APS configuration is +/// metadata rather than a client assertion, so a missing client call is not a +/// finding. The field and this shape stay for the evidence payload's schema, and +/// the list arrives empty. +#[derive(Debug, Clone, Deserialize)] +#[allow( + dead_code, + reason = "decoded for schema stability; the collector records no APS calls" +)] +pub struct ApsFetchBidsEvidence { + /// The APS slot ID requested. + pub slot_id: String, + /// Sizes requested for the slot. + pub sizes: Vec<(u32, u32)>, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// A `/__ts/page-bids` observation for SPA routes (spec §5.2). +/// +/// DEFERRED in Phase 1: kept as forward scaffolding so the decoded evidence shape +/// stays forward-compatible. Not populated by the collector or surfaced in JSON. +#[derive(Debug, Clone, Deserialize)] +#[allow( + dead_code, + reason = "reserved decoded shape for the optional bids phase" +)] +pub struct PageBidsEvidence { + /// The slot ID present in the page-bids response. + pub slot_id: String, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// All read-only ad evidence decoded from a single browser page. +#[derive(Debug, Clone, Deserialize)] +pub struct BrowserAdEvidence { + /// DOM element IDs matching configured prefixes. + pub dom_ids: Vec, + /// GPT slots observed via `defineSlot` and `getSlots()`. + pub gpt_slots: Vec, + /// `apstag.fetchBids` calls observed. + pub aps_calls: Vec, + /// `/__ts/page-bids` observations (deferred; default empty). + #[serde(default)] + #[allow(dead_code, reason = "reserved for the optional bids phase")] + pub page_bids: Vec, + /// Collector-level warnings (no page HTML/cookies/storage). + #[serde(default)] + pub warnings: Vec, +} + +/// Summary of the runtime ad-stack gate for a page. +#[derive(Debug, Clone, Copy)] +pub struct RuntimeGateSummary { + /// The three-state ad-stack expectation. + pub expected: RuntimeAdStackExpected, +} + +impl RuntimeGateSummary { + /// Builds a summary from a computed runtime expectation. + #[must_use] + pub fn from_expected(expected: RuntimeAdStackExpected) -> Self { + Self { expected } + } + + #[cfg(test)] + fn unknown_allowed() -> Self { + Self::from_expected(RuntimeAdStackExpected::Unknown) + } + + #[cfg(test)] + fn auction_disabled() -> Self { + Self::from_expected(RuntimeAdStackExpected::No) + } +} + +/// Confirmation status for a single configured slot (compare-side mirror of the +/// output `SlotStatus`). +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, + /// The checker cannot confirm this slot type; this is not page drift. + Unconfirmable, +} + +/// The verification result for one audited page. +#[derive(Debug, Clone)] +pub struct PageVerificationResult { + /// Whether the runtime ad stack was expected to run for this page. + pub runtime_ad_stack_expected: RuntimeAdStackExpected, + /// Per-slot results, in expected-slot order. + pub slots: Vec, + /// Live evidence that matched no configured slot. + pub extra_evidence: Vec, +} + +impl PageVerificationResult { + /// Whether `--strict` should fail for this page. + /// + /// False when the runtime ad stack is not expected to run (a known gate + /// suppressed it); otherwise true if any slot is missing or partial. Provider + /// warnings and extra evidence alone never fail strict. + #[must_use] + pub fn strict_failed(&self) -> bool { + if self.runtime_ad_stack_expected == RuntimeAdStackExpected::No { + return false; + } + self.slots + .iter() + .any(|slot| matches!(slot.status, SlotStatus::Missing | SlotStatus::Partial)) + } +} + +/// Per-slot verification result. +#[derive(Debug, Clone)] +pub struct SlotResult { + /// The configured slot id. + pub id: String, + /// The confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + pub phase: Option, + /// The live evidence observed for this slot. + pub evidence: SlotEvidence, + /// Slot-level warnings (size, provider, etc.). + pub warnings: Vec, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone)] +pub struct SlotEvidence { + /// The resolved DOM element ID, if any. + pub dom_id: Option, + /// The matched GPT slot, if any. + pub gpt: Option, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone)] +pub struct ExtraEvidence { + /// Evidence kind. Only `gpt` is produced today; the field is a string so a + /// later evidence source can be added without changing the JSON schema. + pub kind: String, + /// The phase it was observed in. + pub phase: EvidencePhase, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes. + pub sizes: Vec<(u32, u32)>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +fn warning(code: &str, message: String) -> Warning { + Warning { + code: code.to_string(), + message, + } +} + +/// Resolves the slot root DOM element per spec §5.3. +/// +/// Exact `div_id` match first, then the first element whose ID starts with +/// `div_id`, ignoring `-container` wrappers. +fn resolve_dom<'a>(dom_ids: &'a [DomEvidence], div_id: &str) -> Option<&'a DomEvidence> { + if let Some(exact) = dom_ids.iter().find(|dom| dom.dom_id == div_id) { + return Some(exact); + } + dom_ids + .iter() + .find(|dom| dom.dom_id.starts_with(div_id) && !dom.dom_id.ends_with("-container")) +} + +/// Returns true when a GPT slot's element ID matches the resolved DOM id (or its +/// `-container`), per spec §5.4. +fn gpt_div_matches(gpt_div: &str, expected: &ExpectedSlot, resolved_dom_id: Option<&str>) -> bool { + match resolved_dom_id { + Some(dom_id) => gpt_div == dom_id || gpt_div == format!("{dom_id}-container"), + None => { + gpt_div == expected.div_id + || (gpt_div.starts_with(&expected.div_id) && !gpt_div.ends_with("-container")) + } + } +} + +fn banner_sizes(expected: &ExpectedSlot) -> Vec<(u32, u32)> { + expected + .formats + .iter() + .filter(|format| format.media_type == MediaType::Banner) + .map(|format| (format.width, format.height)) + .collect() +} + +/// Compares configured expected slots against decoded browser evidence. +#[must_use] +pub fn compare_page_evidence( + expected: &[ExpectedSlot], + evidence: &BrowserAdEvidence, + gate: RuntimeGateSummary, +) -> PageVerificationResult { + let mut consumed_gpt = vec![false; evidence.gpt_slots.len()]; + let mut slots = Vec::with_capacity(expected.len()); + + for slot in expected { + let resolved = resolve_dom(&evidence.dom_ids, &slot.div_id); + let resolved_id = resolved.map(|dom| dom.dom_id.clone()); + // An unrenderable (`None`) configured path can never match live GPT + // evidence; matching on anything else would confirm the wrong unit. + let gpt_idx = slot.gam_unit_path.as_deref().and_then(|unit_path| { + evidence.gpt_slots.iter().position(|gpt| { + gpt.gam_unit_path == unit_path + && gpt_div_matches(&gpt.div_id, slot, resolved_id.as_deref()) + }) + }); + + let banner = banner_sizes(slot); + let mut warnings = Vec::new(); + // `expected_slots_for_path` drops a slot whose template does not render, + // so on the verify path this arm is unreachable; it exists for callers + // that build expected slots directly, and as a guard if that filter ever + // changes. + if slot.gam_unit_path.is_none() { + warnings.push(warning( + "gam_unit_path_unrenderable", + format!( + "slot `{}` gam_unit_path template renders past GAM's unit-path byte limit \ + for this page's section; the runtime omits this slot on this path", + slot.id + ), + )); + } + + let (status, dom_for_evidence, gpt_for_evidence, phase) = if let Some(idx) = gpt_idx { + consumed_gpt[idx] = true; + let gpt = &evidence.gpt_slots[idx]; + let dom_id = resolved_id.clone().or_else(|| Some(gpt.div_id.clone())); + if banner.is_empty() { + warnings.push(warning( + "unsupported_format", + format!( + "slot `{}` has only non-banner formats; not confirmable in Phase 1", + slot.id + ), + )); + ( + SlotStatus::Unconfirmable, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else if gpt.sizes.is_empty() { + warnings.push(warning( + "out_of_page_slot", + format!( + "slot `{}` matched an out-of-page GPT slot with no sizes", + slot.id + ), + )); + ( + SlotStatus::Partial, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else if banner.iter().any(|size| gpt.sizes.contains(size)) { + let extra: Vec<(u32, u32)> = gpt + .sizes + .iter() + .copied() + .filter(|size| !banner.contains(size)) + .collect(); + if !extra.is_empty() { + warnings.push(warning( + "extra_observed_size", + format!("slot `{}` observed extra GPT sizes {extra:?}", slot.id), + )); + } + let missing: Vec<(u32, u32)> = banner + .iter() + .copied() + .filter(|size| !gpt.sizes.contains(size)) + .collect(); + if !missing.is_empty() { + warnings.push(warning( + "configured_size_not_observed", + format!( + "slot `{}` configured sizes {missing:?} were not observed", + slot.id + ), + )); + } + ( + SlotStatus::Confirmed, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else { + warnings.push(warning( + "incompatible_sizes", + format!( + "slot `{}` GPT path and div matched but no configured size overlapped", + slot.id + ), + )); + ( + SlotStatus::Partial, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } + } else if let Some(dom) = resolved { + warnings.push(warning( + "dom_without_gpt", + "DOM element matched, but no GPT slot evidence was observed".to_string(), + )); + ( + SlotStatus::Partial, + Some(dom.dom_id.clone()), + None, + Some(dom.phase), + ) + } else { + (SlotStatus::Missing, None, None, None) + }; + + slots.push(SlotResult { + id: slot.id.clone(), + status, + phase, + evidence: SlotEvidence { + dom_id: dom_for_evidence, + gpt: gpt_for_evidence, + }, + warnings, + }); + } + + let extra_evidence = evidence + .gpt_slots + .iter() + .enumerate() + .filter(|(idx, _)| !consumed_gpt[*idx]) + .map(|(_, gpt)| ExtraEvidence { + kind: "gpt".to_string(), + phase: gpt.phase, + dom_id: Some(gpt.div_id.clone()), + gam_unit_path: Some(gpt.gam_unit_path.clone()), + sizes: gpt.sizes.clone(), + reason: "no_configured_slot_matched".to_string(), + }) + .collect(); + + PageVerificationResult { + runtime_ad_stack_expected: gate.expected, + slots, + extra_evidence, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ad_templates::expected::ExpectedFormat; + + fn dom(id: &str) -> DomEvidence { + DomEvidence { + dom_id: id.to_string(), + phase: EvidencePhase::InitialLoad, + } + } + + fn gpt_slot(gam_unit_path: &str, div_id: &str, sizes: &[(u32, u32)]) -> GptSlotEvidence { + GptSlotEvidence { + gam_unit_path: gam_unit_path.to_string(), + div_id: div_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn aps(slot_id: &str, sizes: &[(u32, u32)]) -> ApsFetchBidsEvidence { + ApsFetchBidsEvidence { + slot_id: slot_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn evidence( + doms: Vec, + gpts: Vec, + aps: Vec, + ) -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: doms, + gpt_slots: gpts, + aps_calls: aps, + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn expected_slot( + id: &str, + div_id: &str, + gam_unit_path: &str, + sizes: &[(u32, u32)], + providers: &[&str], + ) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: sizes + .iter() + .map(|&(width, height)| ExpectedFormat { + width, + height, + media_type: MediaType::Banner, + }) + .collect(), + providers: providers.iter().copied().map(String::from).collect(), + page_patterns: Vec::new(), + } + } + + fn expected_slot_video(id: &str, div_id: &str, gam_unit_path: &str) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: vec![ExpectedFormat { + width: 0, + height: 0, + media_type: MediaType::Video, + }], + providers: Vec::new(), + page_patterns: Vec::new(), + } + } + + #[test] + fn gpt_path_div_and_size_overlap_confirms_slot() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + result.slots[0].warnings.is_empty(), + "confirmed slot should carry no warnings" + ); + } + + #[test] + fn unrenderable_gam_unit_path_never_confirms() { + let mut expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + expected.gam_unit_path = None; + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Partial, + "an unrenderable configured path must not confirm against GPT evidence" + ); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "gam_unit_path_unrenderable"), + "should explain why the slot cannot be confirmed" + ); + } + + #[test] + fn dom_only_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(vec![dom("ad-atf-0")], Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "dom_without_gpt") + ); + } + + #[test] + fn no_dom_or_gpt_is_missing() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Missing); + } + + #[test] + fn prefix_dom_resolution_ignores_container_suffix() { + let expected = expected_slot( + "header", + "ad-header-0-", + "/123/homepage/header", + &[(728, 90)], + &[], + ); + let evidence = evidence( + vec![dom("ad-header-0--container"), dom("ad-header-0-_R_abc123")], + Vec::new(), + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].evidence.dom_id.as_deref(), + Some("ad-header-0-_R_abc123"), + "prefix match should skip -container" + ); + assert_eq!(result.slots[0].status, SlotStatus::Partial); + } + + #[test] + fn unmatched_gpt_slot_becomes_extra_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![ + gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)]), + gpt_slot( + "/123/publisher/right-rail", + "ad-right-rail-0", + &[(300, 250)], + ), + ], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert_eq!(result.extra_evidence.len(), 1); + assert_eq!(result.extra_evidence[0].kind, "gpt"); + assert!( + !result.strict_failed(), + "extra evidence alone must not fail strict" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::auction_disabled(), + ); + + assert_eq!(result.runtime_ad_stack_expected, RuntimeAdStackExpected::No); + assert_eq!(result.slots[0].status, SlotStatus::Missing); + assert!( + !result.strict_failed(), + "missing slot must not fail strict when ad stack is No" + ); + } + + #[test] + fn gpt_incompatible_sizes_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(728, 90)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "incompatible_sizes") + ); + } + + #[test] + fn non_banner_only_slot_is_unconfirmable_and_does_not_fail_strict() { + let expected = expected_slot_video("video", "ad-video-", "/123/news/video"); + let evidence = evidence( + vec![dom("ad-video-0")], + vec![gpt_slot("/123/news/video", "ad-video-0", &[(640, 480)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Unconfirmable); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "unsupported_format") + ); + assert!( + !result.strict_failed(), + "checker limitations should not fail strict" + ); + } + + #[test] + fn gpt_container_element_id_confirms() { + let expected = expected_slot("atf", "ad-atf-0", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0"), dom("ad-atf-0-container")], + vec![gpt_slot( + "/123/news/atf", + "ad-atf-0-container", + &[(300, 250)], + )], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "container element id is a valid GPT div match" + ); + } + + #[test] + fn sizeless_live_slot_is_partial_when_config_declares_banner_sizes() { + let expected = expected_slot( + "interstitial", + "ad-oop-", + "/123/news/oop", + &[(300, 250)], + &[], + ); + let evidence = evidence( + vec![dom("ad-oop-0")], + vec![gpt_slot("/123/news/oop", "ad-oop-0", &[])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "out_of_page_slot") + ); + assert!( + result.strict_failed(), + "a live sizeless slot drifting from configured banner sizes must fail strict" + ); + } + + #[test] + fn aps_match_adds_no_warning() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + vec![aps("atf", &[(300, 250)])], + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + !result.slots[0] + .warnings + .iter() + .any(|w| w.code.starts_with("aps_")), + "matching APS should not warn" + ); + } + + #[test] + fn server_side_aps_config_does_not_require_client_fetch_bids_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "missing APS does not flip status" + ); + assert!(result.slots[0].warnings.is_empty()); + assert!( + !result.strict_failed(), + "provider warning alone must not fail strict" + ); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/expected.rs b/crates/trusted-server-cli/src/ad_templates/expected.rs new file mode 100644 index 000000000..9392963ff --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/expected.rs @@ -0,0 +1,336 @@ +//! Pure expected-slot projection from the runtime creative-opportunity matcher. +//! +//! This module owns path/URL normalization and converts the slots matched by +//! [`match_slots`] into stable, owned [`ExpectedSlot`] records for output and +//! browser-evidence comparison. It must not duplicate glob-matching semantics. + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{CreativeOpportunitiesConfig, match_slots}; +use url::Url; + +/// The expected slots for a single page path, in configured slot order. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlots { + /// The page path the slots were matched against. + pub path: String, + /// Matched slots projected into stable records, in configured order. + pub slots: Vec, +} + +/// A single configured slot expected to appear for a page path. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlot { + /// The slot identifier. + pub id: String, + /// Resolved HTML `div` element ID (override or the slot id). + pub div_id: String, + /// Resolved GAM unit path: the rendered `gam_unit_path` template (or + /// `//` when the slot has none). + /// + /// `None` only for manually constructed comparison fixtures. Projection + /// omits a slot when the runtime cannot render it for this path. + pub gam_unit_path: Option, + /// Configured ad formats. + pub formats: Vec, + /// Configured provider names, in `aps`, `prebid` order. + pub providers: Vec, + /// Glob patterns configured for this slot. + pub page_patterns: Vec, +} + +/// A configured ad format as a stable width/height/media-type record. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedFormat { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Configured media type. + pub media_type: MediaType, +} + +/// Projects the slots matching `path` into stable expected-slot records. +/// +/// Uses [`match_slots`] so glob semantics stay identical to the runtime, and +/// preserves configured slot order. `path` is assumed already normalized via +/// [`normalize_path_or_url`]. +/// +/// `gam_unit_path` templates are rendered against the section the runtime would +/// derive from `path` (per the config's `section_root`/`section_segment` +/// policy), so `{section}`-bearing configs project the same unit path the live +/// page requests. +// Shared projection used by the audit verifier; the static commands match slots +// directly against the runtime matcher. +#[must_use] +pub fn expected_slots_for_path(path: &str, config: &CreativeOpportunitiesConfig) -> ExpectedSlots { + let section = config.section_for_path(path); + let slots = match_slots(&config.slot, path) + .into_iter() + .filter_map(|slot| { + let gam_unit_path = slot.render_gam_unit_path(&config.gam_network_id, §ion)?; + Some(ExpectedSlot { + id: slot.id.clone(), + div_id: slot.resolved_div_id().to_string(), + gam_unit_path: Some(gam_unit_path), + formats: slot + .formats + .iter() + .map(|format| ExpectedFormat { + width: format.width, + height: format.height, + media_type: format.media_type.clone(), + }) + .collect(), + providers: provider_names(slot), + page_patterns: slot.page_patterns.clone(), + }) + }) + .collect(); + + ExpectedSlots { + path: path.to_string(), + slots, + } +} + +fn provider_names( + slot: &trusted_server_core::creative_opportunities::CreativeOpportunitySlot, +) -> Vec { + let mut providers = Vec::new(); + if slot.providers.aps.is_some() { + providers.push("aps".to_string()); + } + if slot.providers.prebid.is_some() { + providers.push("prebid".to_string()); + } + providers +} + +/// Normalizes a page path or full URL into a request path. +/// +/// Full `scheme://` inputs are parsed and reduced to their path; bare inputs have +/// query and fragment stripped and a leading `/` ensured. Empty paths become `/`. +/// +/// # Errors +/// +/// Returns a user-facing string when a `scheme://` input cannot be parsed as a URL. +pub fn normalize_path_or_url(input: &str) -> Result { + let path_input = input.split(['?', '#']).next().unwrap_or(input); + let scheme_prefix = path_input.split_once("://").map(|(scheme, _)| scheme); + let has_url_scheme = scheme_prefix.is_some_and(|scheme| { + let mut chars = scheme.chars(); + chars.next().is_some_and(|ch| ch.is_ascii_alphabetic()) + && chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '+' | '-' | '.')) + }); + if has_url_scheme { + let url = Url::parse(input).map_err(|err| format!("invalid URL `{input}`: {err}"))?; + let path = url.path(); + return Ok(if path.is_empty() { + "/".to_string() + } else { + path.to_string() + }); + } + + let base = Url::parse("https://path-normalizer.example/") + .expect("should parse static path normalization base"); + let relative = input.trim_start_matches('/'); + let normalized = base + .join(&format!("./{relative}")) + .map_err(|error| format!("invalid path `{input}`: {error}"))?; + Ok(normalized.path().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn creative_config_with_slots(patterns: &[&str]) -> CreativeOpportunitiesConfig { + let page_patterns = patterns + .iter() + .map(|pattern| format!("\"{pattern}\"")) + .collect::>() + .join(", "); + let toml = format!( + "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [{page_patterns}]\n\ + formats = [{{ width = 300, height = 250 }}]\n\ + \n\ + [slot.providers.prebid]\n\ + bidders = {{}}\n" + ); + let mut config = toml::from_str::(&toml) + .expect("should deserialize creative opportunities config"); + config.compile_slots(); + config + } + + #[test] + fn expected_slots_use_runtime_matcher_and_config_order() { + let config = creative_config_with_slots(&["/news/*", "/"]); + let expected = expected_slots_for_path("/news/story", &config); + + assert_eq!(expected.path, "/news/story"); + assert_eq!( + expected + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(), + ["atf"] + ); + assert_eq!(expected.slots[0].div_id, "ad-atf-"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/123/news/atf") + ); + assert_eq!(expected.slots[0].providers, ["prebid"]); + assert_eq!( + expected.slots[0].formats, + vec![ExpectedFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + }] + ); + } + + #[test] + fn expected_slots_default_resolution_without_overrides() { + let toml = "gam_network_id = \"42\"\n\ + \n\ + [[slot]]\n\ + id = \"footer\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let expected = expected_slots_for_path("/", &config); + assert_eq!(expected.slots[0].div_id, "footer"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/42/footer") + ); + assert!(expected.slots[0].providers.is_empty()); + } + + #[test] + fn expected_slots_render_section_templates_per_path() { + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\", \"/news\", \"/news/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + // A path with a section segment renders that segment. + assert_eq!( + expected_slots_for_path("/news/story", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/news"), + "a section template should render the path's section" + ); + // The site root falls back to the configured section_root. + assert_eq!( + expected_slots_for_path("/", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/homepage"), + "the root path should render section_root" + ); + } + + #[test] + fn expected_slots_omit_dynamic_template_the_runtime_cannot_render() { + // A `{section}` template that renders past GAM's 100-byte unit-path + // limit. The runtime omits this slot for the request path, so diagnostics + // must not match it against a truncated or otherwise different path. + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{section}/{section}\"\n\ + page_patterns = [\"/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let long_path = format!("/{}", "a".repeat(60)); + let expected = expected_slots_for_path(&long_path, &config); + + assert!( + expected.slots.is_empty(), + "the runtime omits an over-limit dynamic slot on this path" + ); + } + + #[test] + fn normalize_path_or_url_strips_query_and_fragment() { + assert_eq!( + normalize_path_or_url("https://www.example.com/news/story?x=1#top") + .expect("should normalize"), + "/news/story" + ); + assert_eq!( + normalize_path_or_url("news/story?x=1").expect("should normalize"), + "/news/story" + ); + } + + #[test] + fn normalize_path_or_url_roots_empty_input() { + assert_eq!( + normalize_path_or_url("https://www.example.com").expect("should normalize"), + "/" + ); + assert_eq!(normalize_path_or_url("").expect("should normalize"), "/"); + } + + #[test] + fn normalize_path_or_url_uses_identical_url_rules_for_bare_paths() { + assert_eq!( + normalize_path_or_url("/a/../b").expect("should normalize bare dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("https://example.com/a/../b") + .expect("should normalize URL dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("/a b").expect("should encode bare path"), + "/a%20b" + ); + assert_eq!( + normalize_path_or_url("/r?to=https://example.com") + .expect("query URL should not change input classification"), + "/r" + ); + assert_eq!( + normalize_path_or_url("/news:latest").expect("colon should stay in bare path"), + "/news:latest", + "a colon in the first segment must not be parsed as a URL scheme" + ); + assert_eq!( + normalize_path_or_url("https://example.com/news:latest") + .expect("colon should stay in URL path"), + "/news:latest", + "bare and absolute forms should normalize identically" + ); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/mod.rs b/crates/trusted-server-cli/src/ad_templates/mod.rs new file mode 100644 index 000000000..3c26bf121 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/mod.rs @@ -0,0 +1,7 @@ +//! Pure, host-only ad-template CLI logic shared by the static `ts config +//! ad-templates ...` commands and the browser-backed `ts audit ad-templates +//! verify` command. + +pub mod compare; +pub mod expected; +pub mod output; diff --git a/crates/trusted-server-cli/src/ad_templates/output.rs b/crates/trusted-server-cli/src/ad_templates/output.rs new file mode 100644 index 000000000..afcf78ed0 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/output.rs @@ -0,0 +1,483 @@ +//! Stable, serializable output model for ad-template diagnostics. +//! +//! These types mirror the `--json` contract in +//! `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` §8. +//! Field names and declaration order are load-bearing: `serde` serializes struct +//! fields in declaration order, so the order here must match the spec examples. +//! +//! The model is consumed by the `ts audit ad-templates verify` orchestrator, +//! which assembles these wire types from the URL/gate context and comparison result. + +use std::borrow::Cow; + +use serde::{Deserialize, Serialize}; + +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +/// Escapes control characters in page-controlled text bound for a terminal. +/// +/// Page titles and collector warning messages are attacker-controlled: an +/// audited page can put ANSI/OSC escape sequences in `document.title` and drive +/// the operator's terminal (cursor movement, clipboard writes, forged output) +/// when the value is printed verbatim. Every C0 control (including ESC), DEL, +/// and the C1 range are rendered as `\u{XXXX}` so the text stays inert. JSON +/// output is unaffected — `serde_json` escapes these already. +/// +/// Returns a borrowed `Cow` when the input needs no escaping. +#[must_use] +pub fn escape_terminal_text(value: &str) -> Cow<'_, str> { + if !value.chars().any(is_terminal_control) { + return Cow::Borrowed(value); + } + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + if is_terminal_control(ch) { + escaped.push_str(&format!("\\u{{{:04X}}}", ch as u32)); + } else { + escaped.push(ch); + } + } + Cow::Owned(escaped) +} + +/// Whether `ch` can act as a terminal control code (C0, DEL, or C1). +fn is_terminal_control(ch: char) -> bool { + let code = ch as u32; + code < 0x20 + || (0x7f..=0x9f).contains(&code) + || (0x202a..=0x202e).contains(&code) + || (0x2066..=0x2069).contains(&code) +} + +/// Confirmation status for a single configured slot. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, + /// The checker does not support confirming this slot type. + Unconfirmable, +} + +/// JSON rendering of the runtime ad-stack expectation. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeAdStackExpectedJson { + /// The server-side ad stack is expected to run. + Yes, + /// A known gate blocks the server-side ad stack. + No, + /// Consent or another gate is unprovable. + Unknown, +} + +impl From for RuntimeAdStackExpectedJson { + fn from(value: RuntimeAdStackExpected) -> Self { + match value { + RuntimeAdStackExpected::Yes => Self::Yes, + RuntimeAdStackExpected::No => Self::No, + RuntimeAdStackExpected::Unknown => Self::Unknown, + } + } +} + +/// State of a single runtime gate. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GateState { + /// The gate passed. + Pass, + /// The gate blocked the ad stack. + Fail, + /// The gate state could not be proven. + Unknown, +} + +/// Evidence-collection phase, rendered for JSON output. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhaseJson { + /// Observed during the initial page load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A structured warning with a stable machine code and human message. +/// +/// `Serialize` for output; `Deserialize` because the browser collector payload +/// carries warning objects decoded into the comparison input. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct Warning { + /// Stable machine-readable code (e.g. `dom_without_gpt`). + pub code: String, + /// Human-readable message; JSON consumers must not parse this. + pub message: String, +} + +/// Top-level `--json` document for `ts audit ad-templates verify`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct VerificationReport { + /// True when no strict failure and no page-level error occurred. + pub ok: bool, + /// Whether `--strict` was set. + pub strict: bool, + /// One entry per requested URL, in input order. + pub pages: Vec, + /// Run-level warnings not attributable to a single page. + /// + /// Always empty today — every warning the verifier raises belongs to a page + /// or a slot. Kept because the JSON schema declares it, so a consumer can + /// read it unconditionally. + pub warnings: Vec, +} + +/// A single audited page result. +/// +/// `error` is declared immediately after `path` so the serialized key order +/// matches the spec §8 `navigation_failed` shape; on normal pages it is `None` +/// and skipped, leaving the runtime/gates fields in §8 order. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PageJson { + /// The requested URL. + pub url: String, + /// The final URL after redirects, or `null` on navigation failure. + pub final_url: Option, + /// The requested URL's path. + pub requested_path: String, + /// The final path used for matching, or `null` on navigation failure. + pub path: Option, + /// Present only on a page-level collection failure. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Three-state runtime ad-stack expectation; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_ad_stack_expected: Option, + /// Per-gate evidence; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub gates: Option, + /// Number of configured slots matched for the final path; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_slot_count: Option, + /// Per-slot verification results. + pub slots: Vec, + /// Live ad-slot evidence with no matching configured slot. + pub extra_evidence: Vec, + /// Page-level warnings. + pub warnings: Vec, +} + +/// Runtime gate states for a page, one field per spec §5.2 gate. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Gates { + /// Request method is `GET`. + pub method_get: GateState, + /// Request is a top-level navigation. + pub navigation: GateState, + /// Request is not a prefetch. + pub not_prefetch: GateState, + /// Request is not from a known bot. + pub not_bot: GateState, + /// At least one configured slot matched the final path. + pub matched_slots: GateState, + /// The `[auction].enabled` kill switch is on. + pub auction_enabled: GateState, + /// The `[creative_opportunities].enabled` template switch is on. + pub ad_templates_enabled: GateState, + /// Consent allows the auction (often `unknown` for live requests). + pub consent_allows_auction: GateState, +} + +/// A single configured slot's verification result. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotJson { + /// The configured slot id. + pub id: String, + /// The slot's confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + #[serde(skip_serializing_if = "Option::is_none")] + pub phase: Option, + /// The configured shape of the slot (no `id`/`page_patterns` per §8). + pub configured: ConfiguredJson, + /// The live evidence observed for this slot. + pub evidence: SlotEvidenceJson, + /// Slot-level warnings (e.g. provider or size warnings). + pub warnings: Vec, +} + +/// The configured shape of a slot, as rendered in §8 `configured`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ConfiguredJson { + /// Resolved div element ID. + pub div_id: String, + /// Resolved GAM unit path, or `null` when a dynamic template renders past + /// GAM's unit-path byte limit for this page's section. + pub gam_unit_path: Option, + /// Configured formats. + pub formats: Vec, + /// Configured provider names. + pub providers: Vec, +} + +/// A configured format, as rendered in §8. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct FormatJson { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Media type string (`banner`, `video`, `native`). + pub media_type: String, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotEvidenceJson { + /// The resolved DOM element ID observed, if any. + pub dom_id: Option, + /// GPT slot evidence, if any (no `phase` key per §8). + pub gpt: Option, +} + +/// GPT slot evidence, as rendered in §8 `evidence.gpt`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct GptEvidenceJson { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ExtraEvidenceJson { + /// Evidence kind: `dom`, `gpt`, or `aps`. + pub kind: String, + /// The phase the evidence was observed in. + pub phase: EvidencePhaseJson, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +#[cfg(test)] +impl VerificationReport { + fn example_confirmed_with_extra_evidence() -> Self { + VerificationReport { + ok: true, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/news/story".to_string(), + final_url: Some("https://www.example.com/news/story".to_string()), + requested_path: "/news/story".to_string(), + path: Some("/news/story".to_string()), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::Unknown), + gates: Some(Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: GateState::Pass, + auction_enabled: GateState::Pass, + ad_templates_enabled: GateState::Pass, + consent_allows_auction: GateState::Unknown, + }), + matched_slot_count: Some(1), + slots: vec![SlotJson { + id: "atf".to_string(), + status: SlotStatus::Confirmed, + phase: Some(EvidencePhaseJson::InitialLoad), + configured: ConfiguredJson { + div_id: "ad-atf-".to_string(), + gam_unit_path: Some("/123/news/atf".to_string()), + formats: vec![FormatJson { + width: 300, + height: 250, + media_type: "banner".to_string(), + }], + providers: vec!["aps".to_string()], + }, + evidence: SlotEvidenceJson { + dom_id: Some("ad-atf-0".to_string()), + gpt: Some(GptEvidenceJson { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![[300, 250]], + }), + }, + warnings: Vec::new(), + }], + extra_evidence: vec![ExtraEvidenceJson { + kind: "gpt".to_string(), + phase: EvidencePhaseJson::InitialLoad, + dom_id: Some("ad-right-rail-0".to_string()), + gam_unit_path: Some("/123/publisher/right-rail".to_string()), + sizes: vec![[300, 250]], + reason: "no_configured_slot_matched".to_string(), + }], + warnings: vec![Warning { + code: "redirected".to_string(), + message: "navigation redirected to the final path".to_string(), + }], + }], + warnings: Vec::new(), + } + } + + fn example_navigation_failed() -> Self { + VerificationReport { + ok: false, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/broken".to_string(), + final_url: None, + requested_path: "/broken".to_string(), + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: "failed to read main document navigation response".to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + }], + warnings: Vec::new(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escape_terminal_text_passes_through_ordinary_titles() { + assert!( + matches!( + escape_terminal_text("Example News — Story"), + Cow::Borrowed(_) + ), + "text with no control characters should not allocate" + ); + assert_eq!( + escape_terminal_text("Example News — Story"), + "Example News — Story" + ); + } + + #[test] + fn escape_terminal_text_neutralizes_control_sequences() { + // ESC-based CSI/OSC sequences and a raw newline are the terminal-driving + // primitives a hostile page would put in `document.title`. + assert_eq!( + escape_terminal_text("a\u{1b}]0;pwned\u{7}b"), + "a\\u{001B}]0;pwned\\u{0007}b", + "ESC and BEL should be rendered inert" + ); + assert_eq!( + escape_terminal_text("line\nforged: ok"), + "line\\u{000A}forged: ok", + "a newline should not let a title forge an output line" + ); + assert_eq!( + escape_terminal_text("del\u{7f}c1\u{9b}"), + "del\\u{007F}c1\\u{009B}", + "DEL and the C1 range should be escaped too" + ); + assert_eq!( + escape_terminal_text("safe\u{202E}forged\u{2066}tail"), + "safe\\u{202E}forged\\u{2066}tail", + "Unicode bidi controls should be rendered inert" + ); + } + + #[test] + fn verification_json_contains_gate_state_and_extra_evidence() { + let result = VerificationReport::example_confirmed_with_extra_evidence(); + let value = serde_json::to_value(&result).expect("should serialize"); + + assert_eq!(value["ok"], true); + assert_eq!(value["pages"][0]["requested_path"], "/news/story"); + assert_eq!(value["pages"][0]["runtime_ad_stack_expected"], "unknown"); + assert_eq!( + value["pages"][0]["gates"]["consent_allows_auction"], + "unknown" + ); + assert_eq!(value["pages"][0]["slots"][0]["status"], "confirmed"); + assert_eq!( + value["pages"][0]["slots"][0]["evidence"]["gpt"]["sizes"][0][0], + 300 + ); + assert_eq!(value["pages"][0]["extra_evidence"][0]["kind"], "gpt"); + assert_eq!(value["pages"][0]["warnings"][0]["code"], "redirected"); + // `configured` excludes id/page_patterns per §8. + assert!(value["pages"][0]["slots"][0]["configured"]["id"].is_null()); + assert!(value["pages"][0]["slots"][0]["configured"]["page_patterns"].is_null()); + // `evidence.gpt` has no `phase` key per §8. + assert!(value["pages"][0]["slots"][0]["evidence"]["gpt"]["phase"].is_null()); + } + + #[test] + fn page_error_json_matches_navigation_failed_shape() { + let result = VerificationReport::example_navigation_failed(); + let value = serde_json::to_value(&result).expect("should serialize"); + let page = &value["pages"][0]; + + assert_eq!(page["error"]["code"], "navigation_failed"); + assert!(page["final_url"].is_null(), "final_url should be null"); + assert!(page["path"].is_null(), "path should be null"); + assert!( + page.get("runtime_ad_stack_expected").is_none(), + "runtime field absent on error page" + ); + assert!(page.get("gates").is_none(), "gates absent on error page"); + assert!( + page.get("matched_slot_count").is_none(), + "matched_slot_count absent on error page" + ); + assert_eq!(value["ok"], false); + } + + #[test] + fn missing_slot_json_omits_evidence_phase() { + let slot = SlotJson { + id: "missing".to_string(), + status: SlotStatus::Missing, + phase: None, + configured: ConfiguredJson { + div_id: "ad-missing-".to_string(), + gam_unit_path: Some("/123/publisher/missing".to_string()), + formats: Vec::new(), + providers: Vec::new(), + }, + evidence: SlotEvidenceJson { + dom_id: None, + gpt: None, + }, + warnings: Vec::new(), + }; + + let value = serde_json::to_value(slot).expect("should serialize missing slot"); + + assert!( + value.get("phase").is_none(), + "missing evidence should not claim an initial-load phase" + ); + } +} diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs new file mode 100644 index 000000000..bee536146 --- /dev/null +++ b/crates/trusted-server-cli/src/app_config.rs @@ -0,0 +1,171 @@ +//! Shared effective Trusted Server app-config loading for the `ts` CLI. +//! +//! Both the static `ts config ad-templates ...` commands and the browser-backed +//! `ts audit ad-templates verify` command load the same effective app config +//! through [`load_settings`], so config-path resolution and the `EdgeZero` +//! environment overlay stay consistent across command families. + +use std::path::{Path, PathBuf}; + +use clap::Args; +use edgezero_core::app_config::{self, AppConfigLoadOptions}; +use edgezero_core::manifest::ManifestLoader; +use trusted_server_core::config::TrustedServerAppConfig; +use trusted_server_core::settings::Settings; + +/// Shared local app-config flags accepted by every config/audit ad-template command. +#[derive(Clone, Debug, Args)] +pub struct AppConfigArgs { + /// Path to `trusted-server.toml`. Defaults to `.toml` beside `edgezero.toml`. + #[arg(long)] + pub app_config: Option, + /// Path to `edgezero.toml`. + #[arg(long, default_value = "edgezero.toml")] + pub manifest: PathBuf, + /// Skip app-config environment overlay. + #[arg(long)] + pub no_env: bool, +} + +/// Effective settings plus the resolved app-config path they were loaded from. +#[derive(Debug)] +pub struct LoadedSettings { + /// The `trusted-server.toml` path the settings were loaded from. + pub app_config_path: PathBuf, + /// The deserialized effective settings. + pub settings: Settings, +} + +/// Loads the effective Trusted Server settings described by `args`. +/// +/// Resolves the app-config path from `args` (or the manifest's `.toml` +/// default), applies the `EdgeZero` environment overlay unless `no_env` is set, and +/// returns the deserialized [`Settings`]. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded, has no +/// `[app].name`, or the resolved app-config file cannot be read or parsed. When an +/// explicit `--app-config` path is given and is missing, the error names that +/// exact path rather than silently falling back. +pub fn load_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, !args.no_env) +} + +/// Loads Trusted Server settings from the resolved app-config file without +/// applying environment overlays. +/// +/// Mutating commands use this path so environment-only values are never +/// persisted into the operator-owned TOML file. +/// +/// # Errors +/// +/// Returns the same path-resolution, read, and parse errors as +/// [`load_settings`]. +#[cfg(test)] +pub(crate) fn load_file_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, false) +} + +/// Resolves the operator-owned app-config path without deserializing settings. +/// +/// Mutating recovery commands use this when the existing config may already be +/// invalid but still needs a narrowly scoped structural repair. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded or has no +/// `[app].name` and no explicit config path was supplied. +pub fn resolve_app_config_file(args: &AppConfigArgs) -> Result { + if let Some(path) = &args.app_config { + return Ok(path.clone()); + } + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + Ok(resolve_app_config_path(None, &args.manifest, &app_name)) +} + +fn load_settings_with_env_overlay( + args: &AppConfigArgs, + env_overlay: bool, +) -> Result { + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + let app_config_path = + resolve_app_config_path(args.app_config.as_deref(), &args.manifest, &app_name); + + let mut opts = AppConfigLoadOptions::default(); + opts.env_overlay = env_overlay; + let app_config = app_config::deserialize_app_config_with_options::( + &app_config_path, + &app_name, + &opts, + ) + .map_err(|err| format!("failed to load {}: {err}", app_config_path.display()))?; + + Ok(LoadedSettings { + app_config_path, + settings: app_config.into_settings(), + }) +} + +fn resolve_app_config_path( + explicit: Option<&Path>, + manifest_path: &Path, + app_name: &str, +) -> PathBuf { + if let Some(path) = explicit { + return path.to_path_buf(); + } + let file_name = format!("{app_name}.toml"); + if let Some(parent) = manifest_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + parent.join(file_name) + } else { + PathBuf::from(file_name) + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + + #[test] + fn explicit_missing_app_config_does_not_fall_back() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let missing_path = temp.path().join("missing.toml"); + + let args = AppConfigArgs { + app_config: Some(missing_path.clone()), + manifest: manifest_path, + no_env: true, + }; + + let err = load_settings(&args).expect_err("should reject missing explicit config"); + assert!( + err.contains(missing_path.to_string_lossy().as_ref()), + "error should mention the explicit missing path" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js new file mode 100644 index 000000000..6938808f5 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -0,0 +1,241 @@ +// Bounded ad-template evidence collector, injected before publisher scripts run. +// +// This body runs inside an IIFE that defines `__TS_CONFIG` (the configured div +// prefixes). It records evidence into `window.__tsAdTemplateEvidence` +// and never captures page HTML, cookies, storage, request bodies, or arbitrary DOM. +// It always calls original page functions with unchanged arguments and never +// spoofs the browser automation flag. + +const __ts_config = typeof __TS_CONFIG === "object" && __TS_CONFIG ? __TS_CONFIG : {} +const __ts_prefixes = Array.isArray(__ts_config.div_prefixes) ? __ts_config.div_prefixes : [] + +const __ts_ev = (window.__tsAdTemplateEvidence = window.__tsAdTemplateEvidence || { + dom_ids: [], + gpt_slots: [], + aps_calls: [], + warnings: [] +}) + +const __ts_phase = () => (window.__tsScrollPhase ? "scroll" : "initial_load") + +// Hard cap per evidence list so a hostile page cannot grow the store without +// bound; the page controls how many slots/elements/warnings it produces. +const __ts_max_entries = 128 +const __ts_max_string_length = 512 +const __ts_wrapped_googletags = new WeakSet() + +function __ts_text(value) { + return String(value).slice(0, __ts_max_string_length) +} + +// Truncation has to be visible: surplus configured slots classify Missing, and +// `--strict` counts that, so a silent drop is indistinguishable from real drift. +let __ts_truncated = false +function __ts_push(list, entry) { + if (list.length < __ts_max_entries) { + list.push(entry) + return + } + if (__ts_truncated) return + __ts_truncated = true + if (__ts_ev.warnings.length < __ts_max_entries) { + __ts_ev.warnings.push({ + code: "evidence_truncated", + message: "an evidence list hit the " + __ts_max_entries + "-entry cap; results are incomplete" + }) + } +} + +function __ts_warn(code, error) { + __ts_push(__ts_ev.warnings, { code, message: __ts_text(error) }) +} + +// GPT sizes reach Rust as u32 pairs, so anything non-integral (fluid slots, +// NaN, negative or fractional dimensions) must be dropped here — a single bad +// pair would fail deserialization of the whole evidence payload and discard +// every other slot's otherwise valid evidence. +function __ts_size_pair(width, height) { + if (!Number.isInteger(width) || !Number.isInteger(height)) return null + if (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) return null + return [width, height] +} + +function __ts_warn_ignored_size(width, height) { + const numeric = Number.isInteger(width) && Number.isInteger(height) + const outOfRange = + numeric && (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) + __ts_push(__ts_ev.warnings, { + code: outOfRange ? "size_out_of_range" : "fluid_size_ignored", + message: outOfRange ? "GPT size outside u32 range ignored" : "non-integer GPT size ignored" + }) +} + +function __ts_normalize_sizes(sizes) { + const out = [] + if (!Array.isArray(sizes)) return out + // Accept [w, h] or [[w, h], ...]; treat numeric-leading arrays as a single pair. + const pairs = typeof sizes[0] === "number" ? [sizes] : sizes + for (const size of pairs) { + if (out.length >= __ts_max_entries) break + const pair = Array.isArray(size) ? __ts_size_pair(size[0], size[1]) : null + if (pair) { + out.push(pair) + } else { + __ts_warn_ignored_size( + Array.isArray(size) ? size[0] : undefined, + Array.isArray(size) ? size[1] : undefined + ) + } + } + return out +} + +function __ts_record_define_slot(adUnitPath, sizes, divId) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: __ts_text(adUnitPath), + div_id: __ts_text(divId), + sizes: __ts_normalize_sizes(sizes), + phase: __ts_phase() + }) +} + +function __ts_wrap_googletag(googletag) { + if (!googletag || (typeof googletag !== "object" && typeof googletag !== "function")) { + return googletag + } + if (__ts_wrapped_googletags.has(googletag)) return googletag + __ts_wrapped_googletags.add(googletag) + // Wrap defineSlot so both direct calls and calls dispatched from the cmd queue + // are recorded (queued callbacks call this same wrapped function). + const originalDefineSlot = googletag.defineSlot + if (typeof originalDefineSlot === "function") { + try { + const descriptor = Object.getOwnPropertyDescriptor(googletag, "defineSlot") + Object.defineProperty(googletag, "defineSlot", { + configurable: true, + enumerable: descriptor ? descriptor.enumerable : true, + writable: true, + value: function (adUnitPath, sizes, divId) { + const slot = originalDefineSlot.apply(this, arguments) + try { + __ts_record_define_slot(adUnitPath, sizes, divId) + } catch (error) { + __ts_warn("define_slot_capture_failed", error) + } + return slot + } + }) + } catch (error) { + __ts_warn("define_slot_wrap_failed", error) + } + } + return googletag +} + +// Wrap an existing global or intercept a later assignment of it. +function __ts_install(name, wrap) { + if (window[name]) { + try { + wrap(window[name]) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } + return + } + let internal + Object.defineProperty(window, name, { + configurable: true, + // A real `window.googletag` is an ordinary enumerable global; matching that + // keeps `Object.keys(window)` identical with and without the collector. + enumerable: true, + get() { + return internal + }, + set(value) { + internal = value + try { + internal = wrap(value) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } + } + }) +} + +__ts_install("googletag", __ts_wrap_googletag) + +// On-demand DOM + getSlots scrape, invoked by the collector after settle/scroll. +window.__tsCollectAdTemplateEvidence = function () { + try { + const seen = new Set(__ts_ev.dom_ids.map((entry) => entry.dom_id)) + for (const element of document.querySelectorAll("[id]")) { + const id = __ts_text(element.id) + if (id.endsWith("-container")) continue + if (__ts_prefixes.some((prefix) => id.startsWith(prefix)) && !seen.has(id)) { + __ts_push(__ts_ev.dom_ids, { dom_id: id, phase: __ts_phase() }) + seen.add(id) + } + } + const googletag = window.googletag + if (googletag && typeof googletag.pubads === "function") { + const pubads = googletag.pubads() + const slots = typeof pubads.getSlots === "function" ? pubads.getSlots() : [] + for (const slot of slots) { + try { + const path = typeof slot.getAdUnitPath === "function" ? slot.getAdUnitPath() : "" + const divId = typeof slot.getSlotElementId === "function" ? slot.getSlotElementId() : "" + const rawSizes = typeof slot.getSizes === "function" ? slot.getSizes() : [] + const sizes = [] + for (const size of rawSizes) { + if (sizes.length >= __ts_max_entries) break + let pair = null + if ( + size && + typeof size.getWidth === "function" && + typeof size.getHeight === "function" + ) { + // A fluid GPT size answers getWidth()/getHeight() with a + // non-numeric value rather than throwing. + pair = __ts_size_pair(size.getWidth(), size.getHeight()) + } else if (Array.isArray(size)) { + pair = __ts_size_pair(size[0], size[1]) + } + if (pair) { + sizes.push(pair) + } else { + const width = + size && typeof size.getWidth === "function" + ? size.getWidth() + : Array.isArray(size) + ? size[0] + : undefined + const height = + size && typeof size.getHeight === "function" + ? size.getHeight() + : Array.isArray(size) + ? size[1] + : undefined + __ts_warn_ignored_size(width, height) + } + } + const exists = __ts_ev.gpt_slots.some( + (entry) => entry.gam_unit_path === __ts_text(path) && entry.div_id === __ts_text(divId) + ) + if (!exists) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: __ts_text(path), + div_id: __ts_text(divId), + sizes, + phase: __ts_phase() + }) + } + } catch (error) { + __ts_warn("gpt_scrape_failed", error) + } + } + } + } catch (error) { + __ts_warn("collect_failed", error) + } + return __ts_ev +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs new file mode 100644 index 000000000..0e2b51371 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -0,0 +1,1014 @@ +//! Browser-backed `ts audit ad-templates verify` orchestration. +//! +//! For each URL: collect live evidence through an [`AuditCollector`], match +//! configured slots against the **final** (post-redirect) path, evaluate the +//! runtime gate, compare evidence, and assemble the stable §8 wire result. The +//! orchestration is collector-agnostic so it is fully tested with an in-memory +//! fake collector, with no Chrome dependency. + +use std::io::{self, Write}; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + AdStackGateInput, CreativeOpportunitiesConfig, evaluate_ad_stack_gate, +}; + +use crate::ad_templates::compare::{ + BrowserAdEvidence, EvidencePhase, ExtraEvidence, RuntimeGateSummary, SlotEvidence, SlotResult, + SlotStatus as CompareStatus, compare_page_evidence, +}; +use crate::ad_templates::expected::{ExpectedSlot, expected_slots_for_path, normalize_path_or_url}; +use crate::ad_templates::output::{ + ConfiguredJson, EvidencePhaseJson, ExtraEvidenceJson, FormatJson, GateState, Gates, + GptEvidenceJson, PageJson, RuntimeAdStackExpectedJson, SlotEvidenceJson, SlotJson, SlotStatus, + VerificationReport, Warning, escape_terminal_text, +}; +use crate::commands::audit::AuditAdTemplatesVerifyArgs; +use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, AuditCollector, BrowserCollectRequest, build_ad_template_init_script, +}; +use crate::run::RunOutcome; + +/// Verifies configured ad-template slots against live page evidence. +/// +/// # Errors +/// +/// Returns a user-facing string when config loading fails, or when verification +/// surfaces a page-level error or a `--strict` failure (after writing output). +pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result { + args.browser.validate()?; + validate_cookie_scope(&args.urls, &args.cookies)?; + let loaded = crate::app_config::load_settings(&args.config)?; + let collector = crate::commands::audit::browser::BrowserCollector::from_opts(&args.browser); + let report = build_report( + &collector, + loaded.settings.creative_opportunities.as_ref(), + loaded.settings.auction.enabled, + &args.urls, + VerifyOptions { + strict: args.strict, + scroll: args.scroll, + allow_cross_origin_redirect: args.allow_cross_origin_redirect, + }, + &args.cookies, + )?; + + let stdout = io::stdout(); + let mut out = stdout.lock(); + if args.json { + write_json(&mut out, &report)?; + } else { + write_human(&mut out, &report)?; + } + + if report.pages.iter().any(|page| page.error.is_some()) { + Err("ad-template verification reported problems".to_string()) + } else if report.ok { + Ok(RunOutcome::Success) + } else { + Ok(RunOutcome::AssertionFailed) + } +} + +fn validate_cookie_scope(urls: &[url::Url], cookies: &[(String, String)]) -> Result<(), String> { + if cookies.is_empty() { + return Ok(()); + } + let origins: std::collections::BTreeSet = urls + .iter() + .map(|url| url.origin().ascii_serialization()) + .collect(); + if origins.len() > 1 { + return Err( + "--cookie may be used only when every verification URL has one origin; split this run so credentials are never copied to another origin" + .to_string(), + ); + } + Ok(()) +} + +/// Run-level verification switches. +#[derive(Debug, Clone, Copy)] +struct VerifyOptions { + /// Exit non-zero when a matched slot is missing or only partially confirmed. + strict: bool, + /// Perform a deterministic scroll pass after the initial settle. + scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + allow_cross_origin_redirect: bool, +} + +/// Builds the verification report for `urls` using `collector`. +/// +/// `creative` is the effective `[creative_opportunities]` config (if any) and +/// `auction_enabled` is the `[auction].enabled` kill switch. +fn build_report( + collector: &dyn AuditCollector, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, + urls: &[url::Url], + options: VerifyOptions, + cookies: &[(String, String)], +) -> Result { + let init_script = build_init_script(creative)?; + + let requests: Vec<_> = urls + .iter() + .map(|url| BrowserCollectRequest { + url: url.clone(), + init_scripts: vec![init_script.clone()], + scroll: options.scroll, + collect_ad_evidence: true, + cookies: cookies.to_vec(), + }) + .collect(); + let collected_pages = collector.collect_pages(&requests); + + let mut pages = Vec::with_capacity(urls.len()); + let mut any_error = false; + let mut any_strict_fail = false; + + for (url, collected) in urls.iter().zip(collected_pages) { + match collected { + Err(message) => { + any_error = true; + pages.push(error_page(url, &message)); + } + // Slots are matched on the *final* path, so a redirect to a + // different origin would let an unrelated site's evidence satisfy + // `--strict` — and the path-equality redirect warning would not even + // fire when the paths happen to agree. Reject unless opted in. + Ok(collected) + if !options.allow_cross_origin_redirect + && origin_changed(url, &collected.final_url) => + { + any_error = true; + pages.push(cross_origin_page(url, &collected.final_url)); + } + Ok(collected) => { + let (page, strict_failed) = build_page(url, &collected, creative, auction_enabled); + if options.strict && strict_failed { + any_strict_fail = true; + } + pages.push(page); + } + } + } + + let ok = !(any_error || (options.strict && any_strict_fail)); + Ok(VerificationReport { + ok, + strict: options.strict, + pages, + warnings: Vec::new(), + }) +} + +/// The URL without its fragment, for comparisons the server can observe. +pub(super) fn without_fragment(url: &url::Url) -> url::Url { + let mut url = url.clone(); + url.set_fragment(None); + url +} + +/// Whether navigation left the requested URL's origin (scheme, host, or port). +/// +/// A same-host default-port `http:80` to `https:443` redirect is *not* a change: +/// the host is the cookie boundary, and that upgrade is the ordinary canonical +/// redirect. Host changes, port changes, and HTTPS downgrades all are. +pub(super) fn origin_changed(requested: &url::Url, final_url: &url::Url) -> bool { + if requested.host_str() != final_url.host_str() { + return true; + } + + match (requested.scheme(), final_url.scheme()) { + ("http", "https") => { + requested.port_or_known_default() != Some(80) + || final_url.port_or_known_default() != Some(443) + } + (requested_scheme @ ("http" | "https"), final_scheme) + if requested_scheme == final_scheme => + { + requested.port_or_known_default() != final_url.port_or_known_default() + } + // Refuse HTTPS downgrades and any unexpected scheme transition. + _ => true, + } +} + +/// Builds the read-only collector init script from the configured slots. +fn build_init_script(creative: Option<&CreativeOpportunitiesConfig>) -> Result { + let config = AdTemplateCollectorConfig { + div_prefixes: creative + .map(|creative| { + creative + .slot + .iter() + .map(|slot| slot.resolved_div_id().to_string()) + .collect() + }) + .unwrap_or_default(), + }; + build_ad_template_init_script(&config) +} + +/// Assembles a successful page result, returning the wire `PageJson` and whether +/// the page would fail `--strict`. +fn build_page( + requested: &url::Url, + collected: &crate::commands::audit::collector::CollectedPage, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, +) -> (PageJson, bool) { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + let final_url = &collected.final_url; + let final_path = normalize_path_or_url(final_url.as_str()).unwrap_or_else(|_| "/".into()); + + let expected = creative + .map(|creative| expected_slots_for_path(&final_path, creative).slots) + .unwrap_or_default(); + let matched = !expected.is_empty(); + + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: matched, + consent_allows_auction: None, + auction_enabled, + // Absent creative opportunities block here as they do at runtime. + ad_templates_enabled: creative.is_some_and(|creative| creative.enabled), + }); + + let evidence = collected.ad_evidence.clone().unwrap_or_else(empty_evidence); + let result = compare_page_evidence( + &expected, + &evidence, + RuntimeGateSummary::from_expected(gate.expected), + ); + let strict_failed = result.strict_failed(); + + let mut warnings: Vec = collected.warnings.to_vec(); + warnings.extend(evidence.warnings.iter().map(|warning| Warning { + code: format!("page_{}", warning.code), + message: warning.message.clone(), + })); + // Fragments never reach the server, so a fragment-only difference is not a + // redirect and slots match on the path either way. + if without_fragment(requested) != without_fragment(final_url) { + warnings.push(Warning { + code: "redirected".to_string(), + message: format!("navigation redirected from {requested} to {final_url}"), + }); + } + + let slots = expected + .iter() + .zip(result.slots.iter()) + .map(|(expected_slot, slot_result)| to_slot_json(expected_slot, slot_result)) + .collect(); + let extra_evidence = result.extra_evidence.iter().map(to_extra_json).collect(); + + let page = PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: Some(final_path), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::from( + result.runtime_ad_stack_expected, + )), + gates: Some(to_gates( + matched, + auction_enabled, + creative.is_some_and(|creative| creative.enabled), + )), + matched_slot_count: Some(expected.len()), + slots, + extra_evidence, + warnings, + }; + (page, strict_failed) +} + +/// Builds a page-level navigation-failure result (spec §8 `navigation_failed`). +fn error_page(requested: &url::Url, message: &str) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: None, + requested_path, + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: message.to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +/// Builds a page-level cross-origin-redirect refusal. +/// +/// The final URL is reported so the operator can re-run against it explicitly +/// (or pass `--allow-cross-origin-redirect`) once they have confirmed it is +/// their own property. +fn cross_origin_page(requested: &url::Url, final_url: &url::Url) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: None, + error: Some(Warning { + code: "cross_origin_redirect".to_string(), + message: format!( + "navigation left the requested origin ({} -> {}); \ + evidence from another origin is not accepted as verification. \ + Re-run against the final URL, or pass --allow-cross-origin-redirect", + requested.origin().ascii_serialization(), + final_url.origin().ascii_serialization(), + ), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +fn empty_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: Vec::new(), + gpt_slots: Vec::new(), + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } +} + +fn to_gates(matched: bool, auction_enabled: bool, ad_templates_enabled: bool) -> Gates { + let pass_if = |cond: bool| { + if cond { + GateState::Pass + } else { + GateState::Fail + } + }; + Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: pass_if(matched), + auction_enabled: pass_if(auction_enabled), + ad_templates_enabled: pass_if(ad_templates_enabled), + // Live consent is not provable from a browser navigation in Phase 1. + consent_allows_auction: GateState::Unknown, + } +} + +fn to_slot_json(expected: &ExpectedSlot, result: &SlotResult) -> SlotJson { + SlotJson { + id: result.id.clone(), + status: to_status(result.status), + phase: result.phase.map(to_phase), + configured: ConfiguredJson { + div_id: expected.div_id.clone(), + gam_unit_path: expected.gam_unit_path.clone(), + formats: expected + .formats + .iter() + .map(|format| FormatJson { + width: format.width, + height: format.height, + media_type: media_type_label(&format.media_type).to_string(), + }) + .collect(), + providers: expected.providers.clone(), + }, + evidence: to_slot_evidence(&result.evidence), + warnings: result.warnings.clone(), + } +} + +fn to_slot_evidence(evidence: &SlotEvidence) -> SlotEvidenceJson { + SlotEvidenceJson { + dom_id: evidence.dom_id.clone(), + gpt: evidence.gpt.as_ref().map(|gpt| GptEvidenceJson { + gam_unit_path: gpt.gam_unit_path.clone(), + div_id: gpt.div_id.clone(), + sizes: gpt.sizes.iter().map(|&(w, h)| [w, h]).collect(), + }), + } +} + +fn to_extra_json(extra: &ExtraEvidence) -> ExtraEvidenceJson { + ExtraEvidenceJson { + kind: extra.kind.clone(), + phase: to_phase(extra.phase), + dom_id: extra.dom_id.clone(), + gam_unit_path: extra.gam_unit_path.clone(), + sizes: extra.sizes.iter().map(|&(w, h)| [w, h]).collect(), + reason: extra.reason.clone(), + } +} + +fn to_status(status: CompareStatus) -> SlotStatus { + match status { + CompareStatus::Confirmed => SlotStatus::Confirmed, + CompareStatus::Partial => SlotStatus::Partial, + CompareStatus::Missing => SlotStatus::Missing, + CompareStatus::Unconfirmable => SlotStatus::Unconfirmable, + } +} + +fn to_phase(phase: EvidencePhase) -> EvidencePhaseJson { + match phase { + EvidencePhase::InitialLoad => EvidencePhaseJson::InitialLoad, + EvidencePhase::Scroll => EvidencePhaseJson::Scroll, + } +} + +fn media_type_label(media_type: &MediaType) -> &'static str { + match media_type { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + } +} + +fn write_json(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + let json = serde_json::to_string_pretty(report) + .map_err(|error| format!("failed to serialize verification report: {error}"))?; + writeln!(out, "{json}").map_err(write_err) +} + +fn write_human(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + // Warning codes and messages can originate in the audited page (the + // collector forwards `String(error)` from page scripts), so escape control + // characters before writing them to the operator's terminal. + let write_warning = |out: &mut dyn Write, indent: &str, warning: &Warning| { + writeln!( + out, + "{indent}warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(write_err) + }; + + for warning in &report.warnings { + write_warning(out, "", warning)?; + } + for page in &report.pages { + writeln!(out, "url: {}", escape_terminal_text(&page.url)).map_err(write_err)?; + if let Some(error) = &page.error { + writeln!( + out, + " error [{}]: {}", + escape_terminal_text(&error.code), + escape_terminal_text(&error.message) + ) + .map_err(write_err)?; + continue; + } + if let Some(path) = &page.path { + writeln!(out, " path: {}", escape_terminal_text(path)).map_err(write_err)?; + } + if let Some(expected) = page.runtime_ad_stack_expected { + writeln!(out, " runtime ad stack: {}", runtime_label(expected)).map_err(write_err)?; + } + if let Some(count) = page.matched_slot_count { + writeln!(out, " matched slots: {count}").map_err(write_err)?; + } + if let Some(gates) = &page.gates { + writeln!(out, " gates: {}", gates_label(gates)).map_err(write_err)?; + } + for slot in &page.slots { + writeln!( + out, + " slot {}: {}", + escape_terminal_text(&slot.id), + status_label(slot.status) + ) + .map_err(write_err)?; + for warning in &slot.warnings { + write_warning(out, " ", warning)?; + } + } + for extra in &page.extra_evidence { + writeln!( + out, + " extra {} evidence: div={} gam={} sizes={:?} ({})", + escape_terminal_text(&extra.kind), + escape_terminal_text(extra.dom_id.as_deref().unwrap_or("-")), + escape_terminal_text(extra.gam_unit_path.as_deref().unwrap_or("-")), + extra.sizes, + escape_terminal_text(&extra.reason), + ) + .map_err(write_err)?; + } + for warning in &page.warnings { + write_warning(out, " ", warning)?; + } + } + writeln!(out, "ok: {}", report.ok).map_err(write_err) +} + +fn status_label(status: SlotStatus) -> &'static str { + match status { + SlotStatus::Confirmed => "confirmed", + SlotStatus::Partial => "partial", + SlotStatus::Missing => "missing", + SlotStatus::Unconfirmable => "unconfirmable", + } +} + +fn runtime_label(expected: RuntimeAdStackExpectedJson) -> &'static str { + match expected { + RuntimeAdStackExpectedJson::Yes => "yes", + RuntimeAdStackExpectedJson::No => "no", + RuntimeAdStackExpectedJson::Unknown => "unknown", + } +} + +fn gate_label(gate: GateState) -> &'static str { + match gate { + GateState::Pass => "pass", + GateState::Fail => "fail", + GateState::Unknown => "unknown", + } +} + +fn gates_label(gates: &Gates) -> String { + format!( + "method_get={} navigation={} not_prefetch={} not_bot={} matched_slots={} auction_enabled={} consent={}", + gate_label(gates.method_get), + gate_label(gates.navigation), + gate_label(gates.not_prefetch), + gate_label(gates.not_bot), + gate_label(gates.matched_slots), + gate_label(gates.auction_enabled), + gate_label(gates.consent_allows_auction), + ) +} + +#[allow( + clippy::needless_pass_by_value, + reason = "used as a map_err fn that receives io::Error by value" +)] +fn write_err(error: io::Error) -> String { + format!("failed to write command output: {error}") +} + +#[cfg(test)] +mod tests { + use std::cell::Cell; + use std::collections::HashMap; + + use super::*; + use crate::ad_templates::compare::{DomEvidence, GptSlotEvidence}; + use crate::commands::audit::collector::CollectedPage; + + struct FakeCollector { + pages: HashMap>, + batch_calls: Cell, + } + + impl FakeCollector { + fn page(requested: &str, final_url: &str, evidence: BrowserAdEvidence) -> Self { + let mut pages = HashMap::new(); + pages.insert( + requested.to_string(), + Ok(CollectedPage { + final_url: url::Url::parse(final_url).expect("should parse final URL"), + title: String::new(), + script_count: 0, + resource_count: 0, + warnings: Vec::new(), + ad_evidence: Some(evidence), + }), + ); + Self { + pages, + batch_calls: Cell::new(0), + } + } + + fn with_error(mut self, requested: &str, message: &str) -> Self { + self.pages + .insert(requested.to_string(), Err(message.to_string())); + self + } + } + + impl AuditCollector for FakeCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.pages + .get(request.url.as_str()) + .cloned() + .unwrap_or_else(|| Err(format!("no fake page for {}", request.url))) + } + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + self.batch_calls.set(self.batch_calls.get() + 1); + requests + .iter() + .cloned() + .map(|request| self.collect_page(request)) + .collect() + } + } + + fn news_config() -> CreativeOpportunitiesConfig { + let toml = "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + config + } + + fn confirmed_news_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: vec![DomEvidence { + dom_id: "ad-atf-0".to_string(), + phase: EvidencePhase::InitialLoad, + }], + gpt_slots: vec![GptSlotEvidence { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![(300, 250)], + phase: EvidencePhase::InitialLoad, + }], + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn report_for( + collector: &dyn AuditCollector, + auction_enabled: bool, + strict: bool, + urls: &[&str], + ) -> VerificationReport { + report_for_with_options( + collector, + auction_enabled, + urls, + VerifyOptions { + strict, + scroll: false, + allow_cross_origin_redirect: false, + }, + ) + } + + fn report_for_with_options( + collector: &dyn AuditCollector, + auction_enabled: bool, + urls: &[&str], + options: VerifyOptions, + ) -> VerificationReport { + let config = news_config(); + let parsed: Vec = urls + .iter() + .map(|url| url::Url::parse(url).expect("should parse URL")) + .collect(); + build_report( + collector, + Some(&config), + auction_enabled, + &parsed, + options, + &[], + ) + .expect("typed collector configuration should serialize") + } + + #[test] + fn verify_uses_final_url_for_matching_after_redirect() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, false, &["https://www.example.com/"]); + let json = serde_json::to_value(&report).expect("should serialize"); + + assert_eq!(json["pages"][0]["path"], "/news/story"); + assert_eq!(json["pages"][0]["slots"][0]["status"], "confirmed"); + let warnings = json["pages"][0]["warnings"] + .as_array() + .expect("should have warnings array"); + assert!( + warnings.iter().any(|w| w["code"] == "redirected"), + "redirect should emit a `redirected` warning" + ); + } + + #[test] + fn cross_origin_redirect_is_rejected_even_when_paths_match() { + // Same path on a different origin: the redirect warning would not fire, + // so without the origin check this unrelated page's evidence would + // satisfy --strict. + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://impostor.example.net/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!(!report.ok, "a cross-origin redirect must not report ok"); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][0]["error"]["code"], "cross_origin_redirect"); + assert!( + json["pages"][0]["slots"] + .as_array() + .expect("should have slots array") + .is_empty(), + "off-origin evidence must not be reported as slot verification" + ); + } + + #[test] + fn cross_origin_redirect_is_accepted_with_explicit_opt_in() { + let collector = FakeCollector::page( + "https://example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for_with_options( + &collector, + true, + &["https://example.com/news/story"], + VerifyOptions { + strict: true, + scroll: false, + allow_cross_origin_redirect: true, + }, + ); + + assert!( + report.ok, + "an opted-in apex -> www redirect should verify normally" + ); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn same_origin_path_redirect_still_verifies() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, true, &["https://www.example.com/"]); + + assert!( + report.ok, + "a same-origin redirect should still be verified, not refused" + ); + } + + #[test] + fn same_host_http_to_https_upgrade_is_accepted() { + let collector = FakeCollector::page( + "http://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["http://www.example.com/news/story"], + ); + + assert!(report.ok, "a default-port HTTPS upgrade should be accepted"); + } + + #[test] + fn downgrade_and_port_changes_are_rejected() { + for (requested, final_url) in [ + ( + "https://www.example.com/news/story", + "http://www.example.com/news/story", + ), + ( + "https://www.example.com:8443/news/story", + "https://www.example.com:9443/news/story", + ), + ( + "http://www.example.com:8080/news/story", + "https://www.example.com:8443/news/story", + ), + ] { + let collector = FakeCollector::page(requested, final_url, confirmed_news_evidence()); + let report = report_for(&collector, true, true, &[requested]); + assert!(!report.ok, "redirect {requested} -> {final_url} must fail"); + } + } + + #[test] + fn confirmed_page_is_ok_in_default_mode() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!(report.ok, "confirmed page should be ok"); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn verifier_surfaces_injected_collector_warnings() { + let mut evidence = confirmed_news_evidence(); + evidence.warnings.push(Warning { + code: "fluid_size_ignored".to_string(), + message: "a fluid size could not be compared".to_string(), + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!( + report.pages[0] + .warnings + .iter() + .any(|warning| warning.code == "page_fluid_size_ignored"), + "collector warning should be visible in the page report" + ); + } + + #[test] + fn human_output_includes_runtime_and_extra_evidence_diagnostics() { + let mut evidence = confirmed_news_evidence(); + evidence.gpt_slots.push(GptSlotEvidence { + gam_unit_path: "/123/publisher/extra".to_string(), + div_id: "ad-extra-0".to_string(), + sizes: vec![(728, 90)], + phase: EvidencePhase::InitialLoad, + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + let mut output = Vec::new(); + + write_human(&mut output, &report).expect("should write human report"); + let output = String::from_utf8(output).expect("should be UTF-8 output"); + + assert!(output.contains("runtime ad stack: unknown")); + assert!(output.contains("matched slots: 1")); + assert!(output.contains("gates: method_get=pass")); + assert!(output.contains("extra gpt evidence")); + } + + #[test] + fn strict_missing_slot_fails() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + !report.ok, + "strict mode with a missing slot should not be ok" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + // auction disabled -> runtime expected No -> strict does not fail on missing. + let report = report_for( + &collector, + false, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + report.ok, + "missing slot must not fail strict when auction is disabled" + ); + assert_eq!( + report.pages[0].runtime_ad_stack_expected, + Some(RuntimeAdStackExpectedJson::No) + ); + } + + #[test] + fn multi_url_page_error_sets_ok_false() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ) + .with_error("https://www.example.com/broken", "navigation failed"); + let report = report_for( + &collector, + true, + false, + &[ + "https://www.example.com/news/story", + "https://www.example.com/broken", + ], + ); + + assert!(!report.ok, "a page-level error sets ok=false"); + assert_eq!( + collector.batch_calls.get(), + 1, + "all verifier URLs should use one collector batch" + ); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][1]["error"]["code"], "navigation_failed"); + assert!(json["pages"][1]["final_url"].is_null()); + } + + #[test] + fn supplied_cookies_are_rejected_for_multiple_origins() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://b.example/y").expect("should parse second URL"), + ]; + + let error = validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect_err("should not replicate one cookie across origins"); + + assert!( + error.contains("one origin"), + "the refusal should explain cookie scope, got {error}" + ); + } + + #[test] + fn supplied_cookies_are_allowed_for_same_origin_urls() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://a.example/y").expect("should parse second URL"), + ]; + + validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect("same-origin URLs share the intended cookie scope"); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs new file mode 100644 index 000000000..fd79155f0 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -0,0 +1,1194 @@ +//! Chrome/Chromium-backed implementation of [`AuditCollector`] using +//! `chromiumoxide` (CDP). +//! +//! The collector installs optional pre-navigation init scripts, sets any +//! operator-supplied cookies, navigates, waits for the page to settle, optionally +//! scrolls, and reads back a bounded set of evidence. It never *captures* page +//! HTML, cookies, or storage; supplied cookies are only *sent* to carry an +//! existing session past origin gates. + +use std::time::Duration; + +use chromiumoxide::browser::{Browser, BrowserConfig}; +use chromiumoxide::cdp::browser_protocol::network::CookieParam; +use chromiumoxide::handler::viewport::Viewport; +use chromiumoxide::page::Page; +use futures::StreamExt as _; + +use crate::ad_templates::compare::BrowserAdEvidence; +use crate::ad_templates::output::Warning; +use crate::commands::audit::browser_scroll::{self, CDP_OPERATION_TIMEOUT}; +use crate::commands::audit::collector::{ + AuditCollector, BrowserCollectRequest, BrowserOpts, BrowserProfile, CollectedPage, + PAGE_SETTLE_MAX_MS, PAGE_SETTLE_QUIET_MS, +}; + +/// Candidate Chrome/Chromium executable names searched on `PATH`. +pub(crate) const CHROME_NAMES: &[&str] = &[ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + "chrome", + "Google Chrome", + "Google Chrome for Testing", +]; + +/// Poll interval while waiting for the page network to settle, in milliseconds. +const SETTLE_POLL_MS: u64 = 250; +/// Hard cap on page navigation so a stalled load cannot hang the audit. +const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); +/// Hard cap per decoded evidence list, so a hostile page cannot inflate CLI +/// memory. +/// +/// Must equal `__ts_max_entries` in `ad_template_collector.js`. The collector +/// already caps each list, but the evidence object lives on `window`, so a page +/// that appends to it directly is bounded here instead. Anything the collector +/// itself dropped is reported as an `evidence_truncated` warning. +const MAX_EVIDENCE_ENTRIES: usize = 128; +/// Hard cap on the UTF-8 JSON payload before CDP transfers it back to Rust. +const MAX_EVIDENCE_PAYLOAD_BYTES: usize = 1024 * 1024; +/// Hard cap on browser teardown so a wedged Chrome cannot hang the audit. +const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Page-settle timing thresholds. +#[derive(Debug, Clone, Copy)] +struct SettleConfig { + /// Quiet window with no new resources marking the page settled. + quiet: Duration, + /// Hard cap on total settle time. + max: Duration, +} + +/// Immutable browser/session settings shared by every URL in one audit batch. +struct BrowserSessionOptions<'a> { + chrome: &'a std::path::Path, + profile_dir: &'a std::path::Path, + settle: SettleConfig, + accept_invalid_certs: bool, + headful: bool, + assume_consent: bool, + proxy: Option<&'a str>, + profile: BrowserProfile, +} + +/// A `chromiumoxide`-backed page collector launching a local Chrome/Chromium. +#[derive(Debug, Clone)] +pub struct BrowserCollector { + /// Explicit Chrome/Chromium executable override (else `$CHROME`, else auto-detect). + chrome: Option, + /// Quiet window marking the page settled. + settle_quiet: Duration, + /// Hard cap on settling. + settle_max: Duration, + /// Navigate to origins with invalid TLS certificates (dangerous opt-in). + accept_invalid_certs: bool, + /// Run visible Chrome rather than new headless Chrome. + headful: bool, + /// Install the standard consent API stub before publisher scripts. + assume_consent: bool, + /// Optional browser proxy endpoint. + proxy: Option, + /// Device viewport/user-agent profile. + profile: BrowserProfile, +} + +impl Default for BrowserCollector { + fn default() -> Self { + Self::new() + } +} + +impl BrowserCollector { + /// Creates a collector with default tuning and auto-detected Chrome. + #[must_use] + pub fn new() -> Self { + Self { + chrome: None, + settle_quiet: Duration::from_millis(PAGE_SETTLE_QUIET_MS), + settle_max: Duration::from_millis(PAGE_SETTLE_MAX_MS), + accept_invalid_certs: false, + headful: false, + assume_consent: true, + proxy: None, + profile: BrowserProfile::Desktop, + } + } + + /// Creates a collector from operator-supplied browser options. + #[must_use] + pub fn from_opts(opts: &BrowserOpts) -> Self { + Self { + chrome: opts.chrome.clone(), + settle_quiet: Duration::from_millis(opts.settle_quiet_ms), + settle_max: Duration::from_millis(opts.settle_max_ms), + accept_invalid_certs: opts.danger_accept_invalid_certs, + headful: opts.headful, + assume_consent: !opts.no_assume_consent, + proxy: opts.browser_proxy.clone(), + profile: opts.profile, + } + } +} + +/// Pre-document consent behavior shared with the generation crawler. +pub(crate) const CONSENT_STUB_SCRIPT: &str = include_str!("consent_stub.js"); + +/// Shared browser launch inputs used by both audit collectors. +pub(crate) struct BrowserLaunchOptions<'a> { + pub(crate) chrome: &'a std::path::Path, + pub(crate) profile_dir: &'a std::path::Path, + pub(crate) headful: bool, + pub(crate) proxy: Option<&'a str>, + pub(crate) accept_invalid_certs: bool, + pub(crate) viewport: Viewport, + pub(crate) user_agent: Option<&'a str>, +} + +/// Builds the common Chrome configuration for all browser-backed audits. +pub(crate) fn build_browser_config( + options: BrowserLaunchOptions<'_>, +) -> Result { + let mut builder = BrowserConfig::builder() + .chrome_executable(options.chrome) + .user_data_dir(options.profile_dir); + if !options.accept_invalid_certs { + builder = builder.respect_https_errors(); + } + if let Some(proxy) = options.proxy { + let endpoint = if proxy.contains("://") { + proxy.to_string() + } else { + format!("http://{proxy}") + }; + builder = builder + .arg(("proxy-server", endpoint.as_str())) + .arg(("proxy-bypass-list", "<-loopback>")); + } + builder = if options.headful { + builder.with_head() + } else { + builder.new_headless_mode() + }; + builder = builder + .window_size(options.viewport.width, options.viewport.height) + .viewport(options.viewport); + if let Some(user_agent) = options.user_agent { + builder = builder.arg(("user-agent", user_agent)); + } + builder + .build() + .map_err(|error| format!("failed to build browser config: {error}")) +} + +fn browser_profile(profile: BrowserProfile) -> (Viewport, Option<&'static str>) { + match profile { + BrowserProfile::Desktop => ( + Viewport { + width: 1280, + height: 800, + device_scale_factor: Some(1.0), + emulating_mobile: false, + is_landscape: true, + has_touch: false, + }, + None, + ), + BrowserProfile::Mobile => ( + Viewport { + width: 390, + height: 844, + device_scale_factor: Some(3.0), + emulating_mobile: true, + is_landscape: false, + has_touch: true, + }, + Some( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) \ + AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1", + ), + ), + } +} + +/// Resolves the Chrome/Chromium executable to launch. +/// +/// Precedence: explicit `--chrome` override, then the `CHROME` environment +/// variable, then auto-detection on `PATH` and standard install locations. +pub(crate) fn resolve_chrome( + override_path: Option<&std::path::Path>, +) -> Result { + if let Some(path) = override_path { + return if path.is_file() { + Ok(path.to_path_buf()) + } else { + Err(format!( + "--chrome path does not point to a file: {}", + path.display() + )) + }; + } + if let Ok(env_path) = std::env::var("CHROME") { + let path = std::path::PathBuf::from(&env_path); + return if path.is_file() { + Ok(path) + } else { + Err(format!("CHROME={env_path} does not point to a file")) + }; + } + find_chrome() +} + +/// Builds a host-only cookie that applies to every path on `url`'s host. +/// +/// Scoped by origin rather than by the full URL: only the origin is load-bearing +/// for a host-only cookie, and a full URL would carry the path, query, and any +/// `user:password@` into CDP and into this function's error message. +pub(crate) fn host_cookie(name: &str, value: &str, url: &url::Url) -> Result { + let origin = url.origin(); + if !origin.is_tuple() { + return Err(format!( + "cannot scope cookie `{name}` because the audited URL has no host" + )); + } + let mut cookie = CookieParam::new(name.to_string(), value.to_string()); + cookie.url = Some(origin.ascii_serialization()); + cookie.path = Some("/".to_string()); + cookie.secure = Some(url.scheme() == "https"); + Ok(cookie) +} + +fn format_cookie_install_error(name: &str, _error: impl std::fmt::Display) -> String { + // Do not forward the CDP error: a browser implementation may include the + // rejected cookie value in its diagnostic. + format!("failed to set cookie `{name}`") +} + +/// Installs host-only, root-scoped cookies before a page has an origin. +pub(crate) async fn set_browser_cookies( + browser: &Browser, + cookies: &[(String, String)], + url: &url::Url, +) -> Result<(), String> { + for (name, value) in cookies { + let cookie = host_cookie(name, value, url)?; + browser + .set_cookies(vec![cookie]) + .await + .map_err(|error| format_cookie_install_error(name, error))?; + } + Ok(()) +} + +/// Auto-detects a Chrome/Chromium executable. +/// +/// Searches `PATH` by common names first, then well-known per-OS install +/// locations (e.g. the macOS `.app` bundle, which is not on `PATH`). +fn find_chrome() -> Result { + if let Some(path) = CHROME_NAMES.iter().find_map(|name| which::which(name).ok()) { + return Ok(path); + } + if let Some(path) = well_known_chrome_paths() + .into_iter() + .find(|path| path.is_file()) + { + return Ok(path); + } + Err(format!( + "could not find Chrome/Chromium on PATH or in standard install locations (looked for: {})", + CHROME_NAMES.join(", ") + )) +} + +/// Well-known absolute Chrome/Chromium install locations for the host OS. +fn well_known_chrome_paths() -> Vec { + let mut paths = Vec::new(); + + #[cfg(target_os = "macos")] + { + const APPS: &[&str] = &[ + "Google Chrome.app/Contents/MacOS/Google Chrome", + "Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + "Chromium.app/Contents/MacOS/Chromium", + ]; + for app in APPS { + paths.push(std::path::PathBuf::from(format!("/Applications/{app}"))); + if let Ok(home) = std::env::var("HOME") { + paths.push(std::path::PathBuf::from(format!( + "{home}/Applications/{app}" + ))); + } + } + } + + #[cfg(target_os = "linux")] + { + for path in [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/snap/bin/chromium", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + #[cfg(target_os = "windows")] + { + for path in [ + r"C:\Program Files\Google\Chrome\Application\chrome.exe", + r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + paths +} + +impl AuditCollector for BrowserCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.collect_pages(std::slice::from_ref(&request)) + .into_iter() + .next() + .expect("should return one result for one browser request") + } + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + if requests.is_empty() { + return Vec::new(); + } + // HTTP(S) scheme is enforced by the CLI value parser before we get here. + let chrome = match resolve_chrome(self.chrome.as_deref()) { + Ok(chrome) => chrome, + Err(error) => return vec![Err(error); requests.len()], + }; + let profile = match tempfile::tempdir() { + Ok(profile) => profile, + Err(error) => { + let error = format!("failed to create browser profile dir: {error}"); + return vec![Err(error); requests.len()]; + } + }; + + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let error = format!("failed to build browser runtime: {error}"); + return vec![Err(error); requests.len()]; + } + }; + + let settle = SettleConfig { + quiet: self.settle_quiet, + max: self.settle_max, + }; + + let accept_invalid_certs = self.accept_invalid_certs; + let headful = self.headful; + let assume_consent = self.assume_consent; + let proxy = self.proxy.clone(); + let browser_profile = self.profile; + let request_count = requests.len(); + let requests = requests.to_vec(); + let result = runtime.block_on(async move { + let options = BrowserSessionOptions { + chrome: &chrome, + profile_dir: profile.path(), + settle, + accept_invalid_certs, + headful, + assume_consent, + proxy: proxy.as_deref(), + profile: browser_profile, + }; + collect(requests, &options).await + }); + match result { + Ok(results) => results, + Err(error) => vec![Err(error); request_count], + } + } +} + +/// Drives a single page collection on the current-thread runtime. +async fn collect( + requests: Vec, + options: &BrowserSessionOptions<'_>, +) -> Result>, String> { + // chromiumoxide defaults to ignoring TLS errors. The audit sends + // operator-supplied session cookies and treats what it reads back as + // verification evidence, so a certificate-invalid impersonator could both + // harvest the session and fabricate the evidence. Validate certificates + // unless the operator explicitly opts out. + let (viewport, user_agent) = browser_profile(options.profile); + let config = build_browser_config(BrowserLaunchOptions { + chrome: options.chrome, + profile_dir: options.profile_dir, + headful: options.headful, + proxy: options.proxy, + accept_invalid_certs: options.accept_invalid_certs, + viewport, + user_agent, + })?; + + let (mut browser, mut handler) = Browser::launch(config) + .await + .map_err(|error| format!("failed to launch browser: {error}"))?; + + // Drive the CDP event loop for the duration of the session. + let handler_task = tokio::spawn(async move { while handler.next().await.is_some() {} }); + + let mut results = Vec::with_capacity(requests.len()); + for request in requests { + results.push( + collect_with_browser(&browser, request, options.settle, options.assume_consent).await, + ); + } + + // Best-effort teardown; ignore errors since we already have a result, but + // bound it so a Chrome that ignores `close` cannot hang the command. + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.close()).await; + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.wait()).await; + handler_task.abort(); + + Ok(results) +} + +async fn collect_with_browser( + browser: &Browser, + request: BrowserCollectRequest, + settle_config: SettleConfig, + assume_consent: bool, +) -> Result { + set_browser_cookies(browser, &request.cookies, &request.url).await?; + + // Open a blank page first so init scripts are installed before the real + // document loads (evaluate-on-new-document applies to subsequent navigations). + let page = browser + .new_page("about:blank") + .await + .map_err(|error| format!("failed to open browser page: {error}"))?; + + let result = collect_open_page(&page, &request, settle_config, assume_consent).await; + let close_result = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, page.close()).await; + + match (result, close_result) { + (Err(error), _) => Err(error), + (Ok(mut collected), Err(_)) => { + collected.warnings.push(Warning { + code: "page_close_timeout".to_string(), + message: "timed out closing the browser tab after collection".to_string(), + }); + Ok(collected) + } + (Ok(mut collected), Ok(Err(error))) => { + collected.warnings.push(Warning { + code: "page_close_failed".to_string(), + message: format!("failed to close the browser tab after collection: {error}"), + }); + Ok(collected) + } + (Ok(collected), Ok(Ok(_))) => Ok(collected), + } +} + +/// Collects from an open tab. The caller owns tab teardown so every return path, +/// including an error from this function, closes the page before continuing. +async fn collect_open_page( + page: &Page, + request: &BrowserCollectRequest, + settle_config: SettleConfig, + assume_consent: bool, +) -> Result { + let mut warnings = Vec::new(); + + if assume_consent { + page.evaluate_on_new_document(CONSENT_STUB_SCRIPT) + .await + .map_err(|error| format!("failed to install consent init script: {error}"))?; + warnings.push(Warning { + code: "consent_stub_active".to_string(), + message: "audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution".to_string(), + }); + } + page.evaluate_on_new_document("performance.setResourceTimingBufferSize(100000)") + .await + .map_err(|error| format!("failed to increase resource timing buffer: {error}"))?; + + for script in &request.init_scripts { + page.evaluate_on_new_document(script.clone()) + .await + .map_err(|error| format!("failed to install init script: {error}"))?; + } + + tokio::time::timeout(NAVIGATION_TIMEOUT, page.goto(request.url.as_str())) + .await + .map_err(|_| format!("navigation to {} timed out", request.url))? + .map_err(|error| format!("failed to navigate to {}: {error}", request.url))?; + match tokio::time::timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation()).await { + Ok(Ok(_)) => {} + Ok(Err(error)) => warnings.push(Warning { + code: "navigation_wait_failed".to_string(), + message: format!( + "navigation load event could not be read ({error}); continuing with settled page evidence" + ), + }), + Err(_) => warnings.push(Warning { + code: "navigation_wait_timeout".to_string(), + message: format!( + "navigation did not fire its load event within {} seconds; continuing with settled page evidence", + NAVIGATION_TIMEOUT.as_secs() + ), + }), + } + + settle(page, settle_config, &mut warnings).await; + + if request.scroll { + if request.collect_ad_evidence { + // Snapshot evidence before scrolling so entries already present at + // initial load keep phase "load"; the store dedups first-seen, so + // the post-scroll scrape only adds genuinely scroll-phase entries. + if tokio::time::timeout( + CDP_OPERATION_TIMEOUT, + page.evaluate( + "(typeof window.__tsCollectAdTemplateEvidence === 'function' \ + && window.__tsCollectAdTemplateEvidence(), null)", + ), + ) + .await + .is_err() + { + warnings.push(Warning { + code: "ad_evidence_snapshot_timeout".to_string(), + message: "timed out snapshotting ad evidence before scroll".to_string(), + }); + } + } + // Mark subsequent observations as scroll-phase for the verifier's + // injected evidence collector before shared scrolling begins. + eval_discard(page, "window.__tsScrollPhase = true", &mut warnings).await; + warnings.extend( + browser_scroll::scroll_page(page) + .await + .into_iter() + .map(|failure| Warning { + code: failure.code().to_string(), + message: failure.to_string(), + }), + ); + settle(page, settle_config, &mut warnings).await; + } + + let final_url_text = tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.url()) + .await + .map_err(|_| "timed out reading final page URL".to_string())? + .map_err(|error| format!("failed to read final page URL: {error}"))? + .ok_or_else(|| "browser page URL was empty after navigation".to_string())?; + let final_url = url::Url::parse(&final_url_text).map_err(|error| { + format!("browser returned invalid final URL `{final_url_text}`: {error}") + })?; + let title = match tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.get_title()).await { + Ok(Ok(title)) => title.unwrap_or_default(), + Ok(Err(error)) => { + warnings.push(Warning { + code: "page_title_failed".to_string(), + message: format!("failed to read page title: {error}"), + }); + String::new() + } + Err(_) => { + warnings.push(Warning { + code: "page_title_timeout".to_string(), + message: "timed out reading page title".to_string(), + }); + String::new() + } + }; + let script_count = eval_usize(page, "document.querySelectorAll('script').length") + .await + .unwrap_or_else(|message| { + warnings.push(Warning { + code: "script_count_failed".to_string(), + message, + }); + 0 + }); + let resource_count = resource_count(page).await.unwrap_or_else(|message| { + warnings.push(Warning { + code: "resource_count_failed".to_string(), + message, + }); + 0 + }); + + if resource_count >= 250 { + warnings.push(Warning { + code: "resource_timing_heavy".to_string(), + message: format!("page recorded {resource_count} network resources"), + }); + } + + if let Ok(Ok(frames)) = tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.frames()).await + && frames.len() > 1 + { + warnings.push(Warning { + code: "child_frames_not_inspected".to_string(), + message: format!( + "ad-template evidence inspected only the main frame; {} child frame(s) were present", + frames.len() - 1 + ), + }); + } + + let ad_evidence = if request.collect_ad_evidence { + extract_ad_evidence(page, &mut warnings).await + } else { + None + }; + + Ok(CollectedPage { + final_url, + title, + script_count, + resource_count, + warnings, + ad_evidence, + }) +} + +/// Waits for the page network to go quiet after navigation or scroll. +/// +/// Polls the resource-entry count and returns once it stays unchanged for a +/// quiet window, or when the hard cap elapses — so ad-heavy pages finish loading +/// before evidence is read, without hanging on pages that never go idle. +async fn settle(page: &Page, config: SettleConfig, warnings: &mut Vec) { + let start = std::time::Instant::now(); + let mut last = None; + let mut quiet_since = None; + + loop { + if start.elapsed() >= config.max { + warnings.push(Warning { + code: "settle_timeout".to_string(), + message: "page did not settle before the configured maximum wait".to_string(), + }); + return; + } + + let ready_state = match eval_string(page, "document.readyState").await { + Ok(state) => state, + Err(message) => { + warnings.push(Warning { + code: "settle_read_failed".to_string(), + message, + }); + return; + } + }; + let current = match resource_count(page).await { + Ok(count) => count, + Err(message) => { + warnings.push(Warning { + code: "settle_read_failed".to_string(), + message, + }); + return; + } + }; + let ready = matches!(ready_state.as_str(), "interactive" | "complete"); + if ready && last == Some(current) { + let quiet_start = quiet_since.get_or_insert_with(std::time::Instant::now); + if quiet_start.elapsed() >= config.quiet { + return; + } + } else { + quiet_since = None; + } + last = Some(current); + + let remaining_max = config.max.saturating_sub(start.elapsed()); + let remaining_quiet = quiet_since + .map(|quiet_start| config.quiet.saturating_sub(quiet_start.elapsed())) + .unwrap_or(config.quiet); + let sleep_for = Duration::from_millis(SETTLE_POLL_MS) + .min(remaining_max) + .min(remaining_quiet.max(Duration::from_millis(1))); + tokio::time::sleep(sleep_for).await; + } +} + +/// Reads the number of resource timing entries observed so far. +async fn resource_count(page: &Page) -> Result { + eval_usize(page, "performance.getEntriesByType('resource').length").await +} + +async fn eval_discard(page: &Page, expression: impl Into, warnings: &mut Vec) { + if let Err(failure) = browser_scroll::evaluate(page, expression).await { + warnings.push(Warning { + code: failure.code().to_string(), + message: failure.to_string(), + }); + } +} + +async fn eval_usize(page: &Page, expression: &str) -> Result { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression)) + .await + .map_err(|_| format!("timed out evaluating `{expression}`"))? + .map_err(|error| format!("failed to evaluate `{expression}`: {error}"))? + .into_value::() + .map_err(|error| format!("failed to decode `{expression}`: {error}")) +} + +async fn eval_string(page: &Page, expression: &str) -> Result { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression)) + .await + .map_err(|_| format!("timed out evaluating `{expression}`"))? + .map_err(|error| format!("failed to evaluate `{expression}`: {error}"))? + .into_value::() + .map_err(|error| format!("failed to decode `{expression}`: {error}")) +} + +/// Reads and decodes `window.__tsAdTemplateEvidence`, warning (not failing) on a +/// decode error. +async fn extract_ad_evidence( + page: &Page, + warnings: &mut Vec, +) -> Option { + // Serialize and size-check in the page so a hostile publisher-controlled + // evidence object cannot force an unbounded CDP response and Rust decode. + let evaluation = tokio::time::timeout( + CDP_OPERATION_TIMEOUT, + page.evaluate(format!( + r#"(() => {{ + const evidence = typeof window.__tsCollectAdTemplateEvidence === 'function' + ? window.__tsCollectAdTemplateEvidence() + : (window.__tsAdTemplateEvidence || null) + if (evidence === null) return {{ kind: 'absent' }} + try {{ + const json = JSON.stringify(evidence) + const bytes = new TextEncoder().encode(json).byteLength + if (bytes > {MAX_EVIDENCE_PAYLOAD_BYTES}) return {{ kind: 'too_large' }} + return {{ kind: 'evidence', json }} + }} catch (error) {{ + return {{ + kind: 'serialization_failed', + message: String(error).slice(0, 512), + }} + }} + }})()"# + )), + ) + .await; + + let envelope = match evaluation { + Ok(Ok(result)) => match result.into_value::() { + Ok(envelope) => Some(envelope), + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence envelope: {error}"), + }); + return None; + } + }, + Ok(Err(error)) => { + warnings.push(Warning { + code: "ad_evidence_read_failed".to_string(), + message: format!("failed to read ad-template evidence: {error}"), + }); + return None; + } + Err(_) => { + warnings.push(Warning { + code: "ad_evidence_read_timeout".to_string(), + message: "timed out reading ad-template evidence".to_string(), + }); + return None; + } + }; + + match envelope { + Some(envelope) => decode_ad_evidence_envelope(envelope, warnings), + None => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + } +} + +#[derive(Debug, serde::Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum EvidenceEnvelope { + Absent, + TooLarge, + Evidence { json: String }, + SerializationFailed { message: String }, +} + +fn decode_ad_evidence_envelope( + envelope: EvidenceEnvelope, + warnings: &mut Vec, +) -> Option { + match envelope { + EvidenceEnvelope::Absent => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + EvidenceEnvelope::TooLarge => { + warnings.push(Warning { + code: "ad_evidence_too_large".to_string(), + message: format!( + "ad-template evidence exceeded the {MAX_EVIDENCE_PAYLOAD_BYTES}-byte limit" + ), + }); + None + } + EvidenceEnvelope::SerializationFailed { message } => { + warnings.push(Warning { + code: "ad_evidence_encode_failed".to_string(), + message: format!("failed to serialize ad-template evidence in the page: {message}"), + }); + None + } + EvidenceEnvelope::Evidence { json } => { + match serde_json::from_str::(&json) { + Ok(mut evidence) => { + // Defense in depth: the injected script caps these lists, but the + // page owns that store, so re-cap after decode. + evidence.dom_ids.truncate(MAX_EVIDENCE_ENTRIES); + evidence.gpt_slots.truncate(MAX_EVIDENCE_ENTRIES); + evidence.aps_calls.truncate(MAX_EVIDENCE_ENTRIES); + evidence.warnings.truncate(MAX_EVIDENCE_ENTRIES); + Some(evidence) + } + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence: {error}"), + }); + None + } + } + } + } +} + +/// Whether a Chrome/Chromium fixture is available for browser-backed tests. +/// +/// Skips optional local runs, but makes the scripted/CI contract fail loudly. +/// Shared with the generation collector's tests so the contract has one +/// definition. +#[cfg(test)] +pub(crate) fn browser_fixture_available() -> bool { + if resolve_chrome(None).is_ok() { + return true; + } + assert!( + std::env::var_os("TS_AUDIT_BROWSER_TESTS").is_none(), + "TS_AUDIT_BROWSER_TESTS requires Chrome/Chromium; set CHROME to its executable" + ); + false +} + +#[cfg(test)] +mod tests { + use std::io::{Read as _, Write as _}; + use std::net::TcpListener; + use std::sync::mpsc; + + use super::*; + use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, build_ad_template_init_script, + }; + + const AD_TEMPLATE_COLLECTOR_JS: &str = include_str!("ad_template_collector.js"); + + #[test] + fn rust_and_javascript_evidence_entry_caps_match() { + // Parse the declared value rather than matching the whole line, so JS + // punctuation or spacing cannot false-alarm on a still-correct cap. + let declared = AD_TEMPLATE_COLLECTOR_JS + .lines() + .find_map(|line| line.trim().strip_prefix("const __ts_max_entries =")) + .and_then(|value| value.trim().trim_end_matches(';').parse::().ok()) + .expect("should declare __ts_max_entries in the collector script"); + + assert_eq!( + declared, MAX_EVIDENCE_ENTRIES, + "should keep the JS cap equal to MAX_EVIDENCE_ENTRIES" + ); + } + + #[test] + fn well_known_chrome_paths_are_known_for_this_os() { + // macOS/Linux/Windows each have candidate paths; guards the cfg branches. + assert!( + !well_known_chrome_paths().is_empty(), + "supported OSes should list candidate Chrome install paths" + ); + } + + #[test] + fn oversized_ad_evidence_is_an_explicit_warning() { + let mut warnings = Vec::new(); + let evidence = decode_ad_evidence_envelope(EvidenceEnvelope::TooLarge, &mut warnings); + + assert!(evidence.is_none()); + assert_eq!(warnings.len(), 1); + assert_eq!(warnings[0].code, "ad_evidence_too_large"); + } + + #[test] + fn supplied_cookie_is_host_only_and_root_scoped() { + let url = + url::Url::parse("https://publisher.example/news/story").expect("should parse test URL"); + let cookie = host_cookie("clearance", "token", &url).expect("should build cookie"); + + assert!(cookie.domain.is_none(), "host-only cookies omit Domain"); + assert_eq!(cookie.path.as_deref(), Some("/")); + assert_eq!( + cookie.url.as_deref(), + Some("https://publisher.example"), + "the origin scopes a host-only cookie before first navigation" + ); + assert_eq!(cookie.secure, Some(true), "HTTPS cookies must be Secure"); + } + + #[test] + fn cookie_install_error_identifies_name_without_a_value() { + let error = format_cookie_install_error( + "datadome", + "invalid cookie value operator-secret-cookie-value", + ); + + assert_eq!(error, "failed to set cookie `datadome`"); + assert!(!error.contains("operator-secret-cookie-value")); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn supplied_cookie_reaches_first_navigation() { + if !browser_fixture_available() { + return; + } + + let listener = TcpListener::bind("127.0.0.1:0").expect("should bind fixture server"); + let address = listener.local_addr().expect("should read fixture address"); + let (request_tx, request_rx) = mpsc::channel(); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("should accept browser request"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("should set fixture read timeout"); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let chunk_len = stream.read(&mut chunk).expect("should read HTTP request"); + assert!(chunk_len > 0, "request should contain complete headers"); + request.extend_from_slice(&chunk[..chunk_len]); + assert!( + request.len() <= 16 * 1024, + "request headers should be bounded" + ); + } + request_tx + .send(String::from_utf8_lossy(&request).into_owned()) + .expect("should send captured request"); + + let body = b"cookie fixture"; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .expect("should write fixture headers"); + stream.write_all(body).expect("should write fixture body"); + }); + + let collector = BrowserCollector { + settle_quiet: Duration::from_millis(100), + settle_max: Duration::from_secs(1), + ..BrowserCollector::new() + }; + collector + .collect_page(BrowserCollectRequest { + url: url::Url::parse(&format!("http://{address}/")) + .expect("should parse fixture URL"), + init_scripts: Vec::new(), + scroll: false, + collect_ad_evidence: false, + cookies: vec![("clearance".to_string(), "token".to_string())], + }) + .expect("cookie should be installed before first navigation"); + + let request = request_rx + .recv_timeout(Duration::from_secs(5)) + .expect("fixture should receive the first navigation"); + assert!( + request.lines().any(|line| { + line.split_once(':').is_some_and(|(name, value)| { + name.eq_ignore_ascii_case("cookie") + && value + .trim() + .split(';') + .any(|cookie| cookie.trim() == "clearance=token") + }) + }), + "first navigation should carry the supplied cookie; request was {request:?}" + ); + } + + /// A self-contained page that stubs just enough of GPT (no network) for the + /// collector to observe a defined slot via the wrapped `defineSlot` and the + /// `getSlots()` scrape. + const GPT_FIXTURE: &str = r#" + + + +
+ + + +"#; + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn collects_gpt_slot_from_local_fixture() { + if !browser_fixture_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: false, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence + .gpt_slots + .iter() + .any(|slot| slot.gam_unit_path == "/123/news/atf"), + "should capture the defined GPT slot" + ); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0"), + "should capture the configured-prefix DOM id" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn scroll_pass_keeps_initial_load_phase_for_load_time_evidence() { + if !browser_fixture_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: true, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + // The slot and DOM id exist at load time, so the pre-scroll snapshot + // must record them as initial-load even though a scroll pass ran. + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0" + && dom.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad), + "load-time DOM id should keep phase initial_load under --scroll" + ); + assert!( + evidence.gpt_slots.iter().any(|slot| { + slot.gam_unit_path == "/123/news/atf" + && slot.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad + }), + "load-time GPT slot should keep phase initial_load under --scroll" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/browser_collector.rs deleted file mode 100644 index 87a2ccc2c..000000000 --- a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs +++ /dev/null @@ -1,435 +0,0 @@ -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use chromiumoxide::ArcHttpRequest; -use chromiumoxide::browser::{Browser, BrowserConfig}; -use futures::StreamExt as _; -use serde::Deserialize; -use tempfile::TempDir; -use tokio::runtime::Builder; -use tokio::time::{sleep, timeout}; -use url::Url; -use which::which; - -use crate::commands::audit::collector::{ - AuditCollector, CollectedPage, CollectedRequest, CollectedScriptTag, -}; -use crate::error::{CliResult, report_error}; - -const SETTLE_QUIET_PERIOD: Duration = Duration::from_millis(750); -const SETTLE_POLL_INTERVAL: Duration = Duration::from_millis(250); -const SETTLE_MAX_WAIT: Duration = Duration::from_secs(6); -const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); -const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); -const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 250; -const RESOURCE_TIMING_BUFFER_WARNING: &str = - "browser resource timing buffer reached its default size; some network assets may be missing"; - -#[derive(Default)] -pub(crate) struct BrowserAuditCollector; - -impl AuditCollector for BrowserAuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult { - let runtime = Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Tokio runtime for browser audit: {error}" - )) - })?; - - runtime.block_on(collect_page_via_browser_async(target_url)) - } -} - -async fn collect_page_via_browser_async(target_url: &Url) -> CliResult { - let chrome_executable = find_browser_executable()?; - let user_data_dir = TempDir::new().map_err(|error| { - report_error(format!( - "failed to create temporary browser profile for audit: {error}" - )) - })?; - let config = BrowserConfig::builder() - .chrome_executable(chrome_executable) - .user_data_dir(user_data_dir.path()) - .new_headless_mode() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Chromium configuration for audit: {error}" - )) - })?; - - let (mut browser, mut handler) = Browser::launch(config).await.map_err(|error| { - report_error(format!( - "failed to launch Chrome/Chromium for audit: {error}" - )) - })?; - - let handler_task = tokio::spawn(async move { - while let Some(event) = handler.next().await { - if event.is_err() { - break; - } - } - }); - - let result = collect_page_from_browser(&mut browser, target_url).await; - - let close_result = timeout(BROWSER_CLOSE_TIMEOUT, browser.close()) - .await - .map_err(|_| report_error("timed out closing browser after audit")) - .and_then(|result| { - result.map_err(|error| { - report_error(format!("failed to close browser after audit: {error}")) - }) - }); - if close_result.is_err() { - handler_task.abort(); - } - let _ = handler_task.await; - - match (result, close_result) { - (Ok(collected), Ok(_)) => Ok(collected), - (Ok(_), Err(error)) | (Err(error), _) => Err(error), - } -} - -async fn collect_page_from_browser( - browser: &mut Browser, - target_url: &Url, -) -> CliResult { - let page = browser.new_page("about:blank").await.map_err(|error| { - report_error(format!("failed to create browser page for audit: {error}")) - })?; - - timeout(NAVIGATION_TIMEOUT, page.goto(target_url.as_str())) - .await - .map_err(|_| report_error(format!("timed out navigating to `{target_url}`")))? - .map_err(|error| report_error(format!("failed to navigate to `{target_url}`: {error}")))?; - - let navigation_response = timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation_response()) - .await - .map_err(|_| { - report_error(format!( - "timed out waiting for main document navigation response from `{target_url}`" - )) - })? - .map_err(|error| { - report_error(format!( - "failed to read main document navigation response: {error}" - )) - })?; - - let mut warnings = Vec::new(); - if let Some(warning) = validate_navigation_response(navigation_response)? { - warnings.push(warning); - } - if !wait_for_page_settle(&page).await? { - warnings.push( - "browser audit timed out while waiting for the page to settle; results may be partial" - .to_string(), - ); - } - - let final_url = page - .url() - .await - .map_err(|error| report_error(format!("failed to read final page URL: {error}")))? - .ok_or_else(|| report_error("browser page URL was empty after navigation"))?; - let page_title = page - .get_title() - .await - .map_err(|error| report_error(format!("failed to read page title: {error}")))?; - let html = page - .content() - .await - .map_err(|error| report_error(format!("failed to read rendered page HTML: {error}")))?; - - let script_tags: Vec = page - .evaluate( - r#"() => Array.from(document.scripts).map((script) => ({ - src: script.src || null, - inline_text: script.src ? null : (script.textContent || null), - }))"#, - ) - .await - .map_err(|error| report_error(format!("failed to read rendered script tags: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode rendered script tag data: {error}" - )) - })?; - - let network_requests: Vec = page - .evaluate( - r#"() => performance.getEntriesByType('resource').map((entry) => ({ - url: entry.name, - initiator_type: entry.initiatorType || null, - }))"#, - ) - .await - .map_err(|error| { - report_error(format!( - "failed to read browser performance resource entries: {error}" - )) - })? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode browser performance resource data: {error}" - )) - })?; - - if let Some(warning) = resource_timing_buffer_warning(network_requests.len()) { - warnings.push(warning.to_string()); - } - - Ok(CollectedPage { - requested_url: target_url.to_string(), - final_url, - page_title: page_title.filter(|title| !title.trim().is_empty()), - html, - script_tags: script_tags - .into_iter() - .map(|script| CollectedScriptTag { - src: script.src, - inline_text: script.inline_text.filter(|text| !text.trim().is_empty()), - }) - .collect(), - network_requests: network_requests - .into_iter() - .map(|entry| CollectedRequest { - url: entry.url, - resource_type: entry.initiator_type, - }) - .collect(), - warnings, - }) -} - -async fn wait_for_page_settle(page: &chromiumoxide::Page) -> CliResult { - let mut elapsed = Duration::ZERO; - let mut previous_count = None; - let mut stable_for = Duration::ZERO; - - while elapsed < SETTLE_MAX_WAIT { - let ready_state: String = page - .evaluate("document.readyState") - .await - .map_err(|error| report_error(format!("failed to read document ready state: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!("failed to decode document ready state: {error}")) - })?; - let resource_count: usize = page - .evaluate("performance.getEntriesByType('resource').length") - .await - .map_err(|error| report_error(format!("failed to read resource count: {error}")))? - .into_value() - .map_err(|error| report_error(format!("failed to decode resource count: {error}")))?; - - if ready_state == "complete" { - if previous_count == Some(resource_count) { - stable_for += SETTLE_POLL_INTERVAL; - } else { - stable_for = Duration::ZERO; - } - - if stable_for >= SETTLE_QUIET_PERIOD { - return Ok(true); - } - } - - previous_count = Some(resource_count); - sleep(SETTLE_POLL_INTERVAL).await; - elapsed += SETTLE_POLL_INTERVAL; - } - - Ok(false) -} - -fn validate_navigation_response(navigation_response: ArcHttpRequest) -> CliResult> { - let request = navigation_response - .ok_or_else(|| report_error("browser audit did not capture the main document response"))?; - - if let Some(failure_text) = &request.failure_text { - return Err(report_error(format!( - "main document request failed: {failure_text}" - ))); - } - - let response = request.response.as_ref().ok_or_else(|| { - report_error("browser audit did not capture the main document HTTP response") - })?; - - if is_successful_navigation_status(response.status) { - return Ok(None); - } - - Ok(Some(format!( - "audit request returned HTTP {} {} for `{}`; results may be partial", - response.status, response.status_text, response.url - ))) -} - -fn is_successful_navigation_status(status: i64) -> bool { - (200..400).contains(&status) -} - -fn resource_timing_buffer_warning(resource_count: usize) -> Option<&'static str> { - (resource_count >= RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD) - .then_some(RESOURCE_TIMING_BUFFER_WARNING) -} - -fn find_browser_executable() -> CliResult { - for candidate in browser_executable_path_candidates() { - if let Ok(path) = which(candidate) { - return Ok(path); - } - } - - for candidate in browser_executable_fallbacks() { - let candidate_path = Path::new(candidate); - if candidate_path.is_file() { - return Ok(candidate_path.to_path_buf()); - } - } - - Err(report_error( - "Chrome/Chromium was not found on PATH or in the standard local install locations checked by `ts audit`. Install a local Chrome or Chromium binary before running `ts audit`.", - )) -} - -fn browser_executable_path_candidates() -> &'static [&'static str] { - &[ - "google-chrome", - "google-chrome-stable", - "chromium", - "chromium-browser", - "chrome", - "Google Chrome", - "Google Chrome for Testing", - ] -} - -fn browser_executable_fallbacks() -> &'static [&'static str] { - #[cfg(target_os = "macos")] - { - &[ - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - "/Applications/Chromium.app/Contents/MacOS/Chromium", - "/Applications/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", - ] - } - - #[cfg(target_os = "linux")] - { - &[ - "/usr/bin/google-chrome", - "/usr/bin/google-chrome-stable", - "/usr/bin/chromium", - "/usr/bin/chromium-browser", - "/snap/bin/chromium", - ] - } - - #[cfg(not(any(target_os = "macos", target_os = "linux")))] - { - &[] - } -} - -#[derive(Debug, Deserialize)] -struct BrowserScriptTag { - src: Option, - inline_text: Option, -} - -#[derive(Debug, Deserialize)] -struct BrowserPerformanceEntry { - url: String, - initiator_type: Option, -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use chromiumoxide::cdp::browser_protocol::network::{Headers, RequestId, Response}; - use chromiumoxide::cdp::browser_protocol::security::SecurityState; - use chromiumoxide::handler::http::HttpRequest; - - use super::*; - - #[test] - fn successful_navigation_status_allows_redirects_but_rejects_errors() { - assert!(is_successful_navigation_status(200)); - assert!(is_successful_navigation_status(302)); - assert!(is_successful_navigation_status(399)); - assert!(!is_successful_navigation_status(199)); - assert!(!is_successful_navigation_status(400)); - assert!(!is_successful_navigation_status(500)); - } - - #[test] - fn navigation_response_returns_warning_for_http_error_status() { - let warning = - validate_navigation_response(navigation_response_with_status(403, "Forbidden")) - .expect("should validate navigation response") - .expect("should return warning for HTTP error status"); - - assert_eq!( - warning, - "audit request returned HTTP 403 Forbidden for `https://example.com/`; results may be partial", - "should warn and continue when the main document returns an HTTP error" - ); - } - - #[test] - fn resource_timing_buffer_warning_starts_at_threshold() { - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD - 1), - None, - "should not warn before the resource timing buffer threshold" - ); - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD), - Some(RESOURCE_TIMING_BUFFER_WARNING), - "should warn when the resource timing buffer reaches the threshold" - ); - } - - #[test] - fn browser_path_candidates_include_common_names() { - let candidates = browser_executable_path_candidates(); - - assert!(candidates.contains(&"google-chrome")); - assert!(candidates.contains(&"chromium")); - assert!(candidates.contains(&"Google Chrome for Testing")); - } - - fn navigation_response_with_status(status: i64, status_text: &str) -> ArcHttpRequest { - let mut request = - HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); - request.response = Some( - Response::builder() - .url("https://example.com/") - .status(status) - .status_text(status_text) - .headers(Headers::default()) - .mime_type("text/html") - .charset("utf-8") - .connection_reused(false) - .connection_id(1.0) - .encoded_data_length(0.0) - .security_state(SecurityState::Secure) - .build() - .expect("should build navigation response"), - ); - - Some(Arc::new(request)) - } -} diff --git a/crates/trusted-server-cli/src/commands/audit/browser_scroll.rs b/crates/trusted-server-cli/src/commands/audit/browser_scroll.rs new file mode 100644 index 000000000..17b05a491 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/browser_scroll.rs @@ -0,0 +1,83 @@ +//! Shared deterministic browser scrolling for audit commands. + +use std::time::Duration; + +use chromiumoxide::Page; + +const SCROLL_STEP_DELAY: Duration = Duration::from_millis(250); +/// Bound for each CDP operation after navigation. +pub(crate) const CDP_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); + +/// A best-effort browser scroll operation that could not be completed. +#[derive(Debug, derive_more::Display)] +pub(crate) enum ScrollFailure { + /// Chrome rejected the page evaluation. + #[display("browser page evaluation failed: {_0}")] + Evaluation(String), + /// Chrome did not complete the page evaluation within the operation bound. + #[display("browser page evaluation timed out")] + Timeout, +} + +impl core::error::Error for ScrollFailure {} + +impl ScrollFailure { + /// Stable warning code used by structured audit output. + pub(crate) const fn code(&self) -> &'static str { + match self { + Self::Evaluation(_) => "page_evaluation_failed", + Self::Timeout => "page_evaluation_timeout", + } + } +} + +/// Scrolls a page through deterministic fractions to trigger lazy content. +pub(crate) async fn scroll_page(page: &chromiumoxide::Page) -> Vec { + let mut failures = Vec::new(); + for fraction in ["0.33", "0.66", "1"] { + let script = format!( + "window.scrollTo(0, Math.floor(Math.max(document.body.scrollHeight, \ + document.documentElement.scrollHeight) * {fraction}))" + ); + if let Err(failure) = evaluate(page, script).await { + failures.push(failure); + } + tokio::time::sleep(SCROLL_STEP_DELAY).await; + } + if let Err(failure) = evaluate(page, "window.scrollTo(0, 0)").await { + failures.push(failure); + } + failures +} + +/// Evaluates a browser expression with the shared operation bound and errors. +pub(crate) async fn evaluate( + page: &Page, + expression: impl Into, +) -> Result<(), ScrollFailure> { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression.into())) + .await + .map_err(|_| ScrollFailure::Timeout)? + .map(|_| ()) + .map_err(|error| ScrollFailure::Evaluation(error.to_string())) +} + +#[cfg(test)] +mod tests { + use super::ScrollFailure; + + #[test] + fn scroll_failures_have_stable_messages() { + assert_eq!( + ScrollFailure::Evaluation("execution context was destroyed".to_string()).to_string(), + "browser page evaluation failed: execution context was destroyed" + ); + assert_eq!( + ScrollFailure::Timeout.to_string(), + "browser page evaluation timed out" + ); + + fn assert_error() {} + assert_error::(); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/collector.rs b/crates/trusted-server-cli/src/commands/audit/collector.rs index 314ae54fc..25aa9236f 100644 --- a/crates/trusted-server-cli/src/commands/audit/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/collector.rs @@ -1,41 +1,350 @@ -use serde::{Deserialize, Serialize}; -use url::Url; +//! Collector abstraction shared by the generic page audit and the ad-template +//! verifier. +//! +//! Decoupling collection behind [`AuditCollector`] lets the verifier orchestration +//! (Task 9) be tested with an in-memory fake collector, with no Chrome dependency. -use crate::error::CliResult; +use std::path::PathBuf; -pub(crate) trait AuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult; +use clap::{Args, ValueEnum}; + +use crate::ad_templates::compare::BrowserAdEvidence; + +/// Default quiet window for generation's browser collector. +pub(crate) const GENERATE_SETTLE_QUIET_MS: u64 = 750; +/// Default maximum settle wait for generation's browser collector. +pub(crate) const GENERATE_SETTLE_MAX_MS: u64 = 12_000; +/// Default quiet window for `ts audit page` and `ts audit ad-templates verify`. +/// +/// [`BrowserOpts`] and `BrowserCollector::new` must agree, or a collector built +/// in code drifts from the parsed flags without anything failing. +pub(crate) const PAGE_SETTLE_QUIET_MS: u64 = 750; +/// Default maximum settle wait for `ts audit page` and +/// `ts audit ad-templates verify`. +/// +/// See [`PAGE_SETTLE_QUIET_MS`] for why this is shared rather than duplicated. +pub(crate) const PAGE_SETTLE_MAX_MS: u64 = 10_000; + +/// Operator-tunable browser options shared by `ts audit page` and +/// `ts audit ad-templates verify`. +/// +/// These are audit-tool knobs, not publisher runtime config, so they live on the +/// CLI (flags / `CHROME` env) rather than in `trusted-server.toml`. +#[derive(Debug, Clone, Args)] +pub struct BrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then + /// auto-detection on `PATH` and standard install locations. + #[arg(long)] + pub chrome: Option, + /// Browser device profile used for viewport and user-agent emulation. + #[arg(long = "browser-profile", value_enum, default_value_t = BrowserProfile::Desktop)] + pub profile: BrowserProfile, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long)] + pub headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long)] + pub no_assume_consent: bool, + /// Route the browser through this proxy, as `host:port` or a full URL. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Quiet window in milliseconds (no new network resources) that marks the + /// page settled. + #[arg(long, default_value_t = PAGE_SETTLE_QUIET_MS)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = PAGE_SETTLE_MAX_MS)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as verification evidence, so an invalid + /// certificate could mean an impersonator is harvesting the session and + /// fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +/// Browser options for generation, whose device selection is controlled by +/// `--profiles` rather than the verifier's singular `--browser-profile`. +#[derive(Debug, Clone, Args)] +pub struct GenerateBrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then auto-detection. + #[arg(long)] + pub chrome: Option, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long)] + pub headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long)] + pub no_assume_consent: bool, + /// Route the browser through this proxy, as `host:port` or a full URL. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Quiet window in milliseconds that marks the page settled. + #[arg(long, default_value_t = GENERATE_SETTLE_QUIET_MS)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = GENERATE_SETTLE_MAX_MS)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as the evidence it writes config from, so an + /// invalid certificate could mean an impersonator is harvesting the session + /// and fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +/// Defaults mirroring the `#[arg(default_value_t)]` values above, so a path that +/// builds these options in code (the legacy `ts audit ` form) behaves like +/// the parsed command. +impl Default for GenerateBrowserOpts { + fn default() -> Self { + Self { + chrome: None, + headful: false, + no_assume_consent: false, + browser_proxy: None, + settle_quiet_ms: GENERATE_SETTLE_QUIET_MS, + settle_max_ms: GENERATE_SETTLE_MAX_MS, + danger_accept_invalid_certs: false, + } + } +} + +impl GenerateBrowserOpts { + /// Validates relationships between independently parsed browser flags. + pub fn validate(&self) -> Result<(), String> { + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } +} + +/// Browser device profile shared by page audits and ad-template verification. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] +pub enum BrowserProfile { + /// Desktop Chrome at 1280×800. + #[default] + Desktop, + /// Mobile-sized viewport with a mobile user agent. + Mobile, +} + +impl BrowserOpts { + /// Validates relationships between independently parsed browser flags. + pub fn validate(&self) -> Result<(), String> { + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } } -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] -pub(crate) struct CollectedPage { - pub(crate) requested_url: String, - pub(crate) final_url: String, - pub(crate) page_title: Option, - pub(crate) html: String, - pub(crate) script_tags: Vec, - pub(crate) network_requests: Vec, - pub(crate) warnings: Vec, +fn validate_settle_window(quiet_ms: u64, max_ms: u64) -> Result<(), String> { + if quiet_ms > max_ms { + return Err(format!( + "--settle-quiet-ms ({quiet_ms}) cannot exceed --settle-max-ms ({max_ms})" + )); + } + Ok(()) +} + +/// A request to collect a single page. +#[derive(Debug, Clone)] +pub struct BrowserCollectRequest { + /// The URL to navigate to. + pub url: url::Url, + /// Pre-navigation init scripts (evaluate-on-new-document). Empty for a plain + /// page audit; the ad-template verifier supplies the read-only collector here. + pub init_scripts: Vec, + /// Whether to perform the deterministic scroll pass after settle. + pub scroll: bool, + /// Whether to extract `window.__tsAdTemplateEvidence` after settle/scroll. + pub collect_ad_evidence: bool, + /// Operator-supplied `(name, value)` cookies set on the browser context + /// before navigation, scoped to the request URL. Used to carry an existing + /// authenticated session (e.g. a valid bot-protection clearance cookie) so + /// the origin serves the real page instead of a challenge. The collector + /// only sends these; it never reads cookies back. + pub cookies: Vec<(String, String)>, +} + +/// The result of collecting a single page. +#[derive(Debug, Clone)] +pub struct CollectedPage { + /// The final URL after redirects. + pub final_url: url::Url, + /// The page title. + pub title: String, + /// Number of ` + +"#; + + const DELAYED_GPT_FIXTURE: &str = r#" + + +
+
+ + +"#; + + fn gpt_fixture_url(html: &'static str) -> Url { + let listener = TcpListener::bind("127.0.0.1:0").expect("should bind fixture server"); + let address = listener.local_addr().expect("should read fixture address"); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("should accept browser request"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("should set fixture read timeout"); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let chunk_len = stream.read(&mut chunk).expect("should read HTTP request"); + assert!(chunk_len > 0, "request should contain complete headers"); + request.extend_from_slice(&chunk[..chunk_len]); + assert!( + request.len() <= 16 * 1024, + "request headers should be bounded" + ); + } + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + html.len(), + html, + ) + .expect("should write fixture response"); + }); + Url::parse(&format!("http://{address}/")).expect("should parse fixture URL") + } + + #[test] + fn successful_navigation_status_allows_redirects_but_rejects_errors() { + assert!(is_successful_navigation_status(200)); + assert!(is_successful_navigation_status(302)); + assert!(is_successful_navigation_status(399)); + assert!(!is_successful_navigation_status(199)); + assert!(!is_successful_navigation_status(400)); + assert!(!is_successful_navigation_status(500)); + } + + #[test] + fn navigation_response_returns_warning_for_http_error_status() { + let warning = + validate_navigation_response(navigation_response_with_status(403, "Forbidden")) + .expect("should validate navigation response") + .expect("should return warning for HTTP error status"); + + assert_eq!( + warning, + "audit request returned HTTP 403 Forbidden for `https://example.com/`; results may be partial", + "should warn and continue when the main document returns an HTTP error" + ); + } + + #[test] + fn navigation_response_reports_chromium_request_failure() { + let mut request = + HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); + request.failure_text = Some("net::ERR_BLOCKED_BY_ORB".to_string()); + + let error = validate_navigation_response(Some(Arc::new(request))) + .expect_err("should reject Chromium request failures"); + + assert_eq!( + error, "main document request failed: net::ERR_BLOCKED_BY_ORB", + "the crawl should retain the browser failure for its final skipped-page note" + ); + } + + #[test] + fn resource_timing_buffer_warning_starts_at_threshold() { + assert_eq!( + resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_SIZE - 1), + None, + "should not warn before the resource timing buffer threshold" + ); + assert_eq!( + resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_SIZE), + Some(RESOURCE_TIMING_BUFFER_WARNING), + "should warn when the resource timing buffer reaches the threshold" + ); + } + + #[test] + fn browser_path_candidates_include_common_names() { + let candidates = crate::commands::audit::browser::CHROME_NAMES; + + assert!(candidates.contains(&"google-chrome")); + assert!(candidates.contains(&"chromium")); + assert!(candidates.contains(&"Google Chrome for Testing")); + } + + #[test] + fn browser_run_reports_close_error_before_wait_error() { + let result = combine_browser_run_results( + Ok(()), + Ok(()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve teardown error"), + "close failed", + "the close failure is the first teardown failure" + ); + } + + #[test] + fn browser_run_reports_wait_error_when_close_succeeds() { + let result = + combine_browser_run_results(Ok(()), Ok(()), Ok(()), Err("wait failed".to_string())); + + assert_eq!( + result.expect_err("should preserve wait error"), + "wait failed", + "a wait failure must not be mislabeled as a close failure" + ); + } + + #[test] + fn browser_run_preserves_collection_error_over_later_failures() { + let result = combine_browser_run_results( + Err("collection failed".to_string()), + Err("finalization progress failed".to_string()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve first browser run error"), + "collection failed" + ); + } + + #[test] + fn browser_run_reports_finalization_progress_before_teardown_errors() { + let result = combine_browser_run_results( + Ok(()), + Err("finalization progress failed".to_string()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve finalization progress error"), + "finalization progress failed" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn progress_failure_still_finalizes_browser_session() { + if !browser_fixture_available() { + return; + } + + let collector = BrowserAuditCollector::default(); + let target = Url::parse("http://127.0.0.1:9/").expect("should parse fixture URL"); + let mut phases = Vec::new(); + let error = collector + .collect_pages( + &[target], + &[], + &mut |progress| match progress { + CollectionProgress::Launching => { + phases.push("launching"); + Ok(()) + } + CollectionProgress::Loading { .. } => { + phases.push("loading"); + Err(report_error("simulated progress failure")) + } + CollectionProgress::Planning => { + phases.push("planning"); + Ok(()) + } + CollectionProgress::Finalizing => { + phases.push("finalizing"); + Ok(()) + } + }, + &mut |_, _| panic!("page sink should not run after progress failure"), + ) + .expect_err("should return progress failure after browser teardown"); + + let rendered_error = format!("{error:?}"); + assert!( + rendered_error.contains("simulated progress failure"), + "should preserve progress failure, got {rendered_error}" + ); + assert_eq!(phases, ["launching", "loading", "finalizing"]); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn collects_lazy_gpt_slot_only_when_scroll_is_enabled() { + if !browser_fixture_available() { + return; + } + + let without_scroll = BrowserAuditCollector::default() + .collect_page(&gpt_fixture_url(LAZY_GPT_FIXTURE), &[]) + .expect("should collect without scrolling"); + let with_scroll = BrowserAuditCollector::default() + .with_scroll(true) + .collect_page(&gpt_fixture_url(LAZY_GPT_FIXTURE), &[]) + .expect("should collect with scrolling"); + + assert!( + without_scroll.gpt_slots.is_empty(), + "lazy GPT slot should not exist before scrolling" + ); + assert!( + with_scroll + .gpt_slots + .iter() + .any(|slot| { slot.gam_unit_path == "/123/lazy" && slot.div_id == "ad-lazy-0" }), + "scrolling should trigger and collect the lazy GPT slot" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn waits_for_delayed_gpt_registry_to_stabilize_in_definition_order() { + if !browser_fixture_available() { + return; + } + + let collected = BrowserAuditCollector::default() + .collect_page(&gpt_fixture_url(DELAYED_GPT_FIXTURE), &[]) + .expect("should collect delayed GPT registry"); + + assert_eq!( + collected.gpt_slots, + vec![ + CollectedGptSlot { + gam_unit_path: "/123/z-delayed".to_string(), + div_id: "ad-z-delayed-0".to_string(), + sizes: vec![(300, 250)], + }, + CollectedGptSlot { + gam_unit_path: "/123/a-delayed".to_string(), + div_id: "ad-a-delayed-0".to_string(), + sizes: vec![(728, 90)], + }, + ], + "collector should wait for stable registration without reordering slots" + ); + } + + fn navigation_response_with_status(status: i64, status_text: &str) -> ArcHttpRequest { + let mut request = + HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); + request.response = Some( + Response::builder() + .url("https://example.com/") + .status(status) + .status_text(status_text) + .headers(Headers::default()) + .mime_type("text/html") + .charset("utf-8") + .connection_reused(false) + .connection_id(1.0) + .encoded_data_length(0.0) + .security_state(SecurityState::Secure) + .build() + .expect("should build navigation response"), + ); + + Some(Arc::new(request)) + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs new file mode 100644 index 000000000..dc23af09c --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs @@ -0,0 +1,366 @@ +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::error::CliResult; + +/// Warning recorded on a page collected with the audit consent stub installed. +/// +/// A whole-run fact rather than a property of one page, so consumers report it +/// once and unscoped instead of once per page and per profile. +pub(crate) const CONSENT_STUB_WARNING: &str = "consent_stub_active: audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution"; + +/// A user-visible phase reached while collecting browser audit evidence. +#[derive(Debug, Clone, Copy)] +pub(crate) enum CollectionProgress<'a> { + /// The browser process is about to launch. + Launching, + /// A page navigation is about to begin. + Loading { + /// One-based position of this attempted page in the crawl. + current: usize, + /// Total pages when planning has completed, or `None` for the root. + total: Option, + /// Target page; renderers must omit credentials, query, and fragment. + url: &'a Url, + }, + /// Follow-up pages are being selected from the collected root page. + Planning, + /// The browser session is being closed and its process reaped. + Finalizing, +} + +/// Sink invoked synchronously when browser collection reaches a visible phase. +/// +/// Returning an error stops new collection work. An already-launched browser +/// must still be finalized, closed, and waited on before that error is returned. +pub(crate) type ProgressSink<'a> = + &'a mut dyn for<'event> FnMut(CollectionProgress<'event>) -> CliResult<()>; + +/// Sink invoked once per collected page during a batch crawl. +/// +/// Receives the per-page outcome so a failed page can be folded into the run as +/// a warning rather than aborting it; returning `Err` stops the crawl. +pub(crate) type PageSink<'a> = + &'a mut dyn FnMut(&Url, CliResult) -> CliResult; + +/// Plans follow-up URLs from the successfully collected root page. +pub(crate) type RootPlanner<'a> = &'a mut dyn FnMut(&Url, &CollectedPage) -> CliResult>; + +/// Whether a batch crawl should keep going after a page. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ControlFlow { + /// Collect the next target. + Continue, + /// Stop the crawl without an error (budget reached, challenge rate exceeded). + /// + /// What this can prevent depends on the collector: a sequential one loads no + /// further pages, while the browser collector has already finished + /// navigating by the time it folds, so there it only stops the fold. + Stop, +} + +pub(crate) trait AuditCollector { + /// Collects a live page. `cookies` are `(name, value)` pairs set on the + /// browser context before navigation (scoped to `target_url`) so an existing + /// session — e.g. a valid bot-protection clearance cookie — can carry the + /// audit past an origin challenge. + fn collect_page( + &self, + target_url: &Url, + cookies: &[(String, String)], + ) -> CliResult; + + /// Collects several pages in one session, handing each result to `on_page`. + /// + /// The default implementation loops over [`collect_page`](Self::collect_page), + /// which keeps every existing implementor working unchanged. The browser + /// collector overrides it to reuse one Chrome instance and profile across the + /// crawl — a fresh launch per page dominates the cost of a multi-page run, + /// and a shared profile carries bot-protection clearance cookies site-wide. + /// + /// Collectors may buffer results until the browser session closes so CPU-heavy + /// HTML analysis cannot starve a single-threaded CDP event pump. The sink API + /// keeps that buffering policy private and lets simple collectors stream. + /// + /// # Errors + /// + /// Returns an error when `on_page` does, or when the session itself cannot + /// be established. Individual page failures are delivered to `on_page`. + fn collect_pages( + &self, + targets: &[Url], + cookies: &[(String, String)], + on_progress: ProgressSink<'_>, + on_page: PageSink<'_>, + ) -> CliResult<()> { + for (index, target) in targets.iter().enumerate() { + on_progress(CollectionProgress::Loading { + current: index + 1, + total: Some(targets.len()), + url: target, + })?; + let collected = self.collect_page(target, cookies); + if on_page(target, collected)? == ControlFlow::Stop { + break; + } + } + Ok(()) + } + + /// Collects a root and follow-up URLs planned from it in one logical crawl. + /// + /// The browser implementation overrides this so planning happens while the + /// root's browser/profile remains open. Simple collectors retain equivalent + /// behavior through the default implementation. + fn collect_site( + &self, + root: &Url, + cookies: &[(String, String)], + on_progress: ProgressSink<'_>, + planner: RootPlanner<'_>, + on_page: PageSink<'_>, + ) -> CliResult<()> { + on_progress(CollectionProgress::Loading { + current: 1, + total: None, + url: root, + })?; + // A root failure is reported through `on_page` rather than returned, so + // the caller sees the reason as a per-page note exactly as it does from + // the browser collector. With no root page there is nothing to plan + // from, so the crawl ends here. + let root_page = match self.collect_page(root, cookies) { + Ok(page) => page, + Err(error) => { + on_page(root, Err(error))?; + return Ok(()); + } + }; + on_progress(CollectionProgress::Planning)?; + let targets = planner(root, &root_page)?; + if on_page(root, Ok(root_page))? == ControlFlow::Stop { + return Ok(()); + } + let total = targets.len() + 1; + for (index, target) in targets.iter().enumerate() { + on_progress(CollectionProgress::Loading { + current: index + 2, + total: Some(total), + url: target, + })?; + let collected = self.collect_page(target, cookies); + if on_page(target, collected)? == ControlFlow::Stop { + break; + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedPage { + pub(crate) requested_url: String, + pub(crate) final_url: String, + pub(crate) page_title: Option, + pub(crate) html: String, + pub(crate) script_tags: Vec, + pub(crate) network_requests: Vec, + /// Slots read from the live GPT registry (`googletag.pubads().getSlots()`). + /// + /// Populated at `defineSlot` time, so this captures configured slots even + /// when the ad request never fires (consent-gated or iframe-issued). + #[serde(default)] + pub(crate) gpt_slots: Vec, + /// Same-origin `a[href]` targets read from the hydrated DOM, absolutized. + /// + /// Read from the live DOM rather than the served HTML on purpose: an + /// app-router page keeps its link graph in the framework payload, so parsing + /// the raw markup finds only a fraction of the site's sections. + #[serde(default)] + pub(crate) links: Vec, + /// Sitemap `` entries discovered from `robots.txt`, when fetched. + /// + /// Empty unless sitemap discovery ran (root page only). + #[serde(default)] + pub(crate) sitemap_locs: Vec, + pub(crate) warnings: Vec, +} + +/// A same-origin link observed in the hydrated DOM. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedLink { + /// Absolute URL of the link target. + pub(crate) url: String, + /// Whether the anchor sits inside site navigation (`nav`, `header`, + /// `[role="navigation"]`). Nav links are the publisher's own declaration of + /// its taxonomy, so they rank above body links when choosing sections. + pub(crate) in_nav: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::{cli_error, report_error}; + + struct ProgressCollector; + + impl AuditCollector for ProgressCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + if target_url.path() == "/broken" { + return cli_error("simulated page failure"); + } + Ok(CollectedPage { + requested_url: target_url.to_string(), + final_url: target_url.to_string(), + page_title: None, + html: String::new(), + script_tags: Vec::new(), + network_requests: Vec::new(), + gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), + warnings: Vec::new(), + }) + } + } + + fn record_progress(event: CollectionProgress<'_>) -> String { + match event { + CollectionProgress::Launching => "launching".to_string(), + CollectionProgress::Loading { + current, + total, + url, + } => format!( + "loading:{current}/{}:{}", + total.map_or_else(|| "?".to_string(), |total| total.to_string()), + url.path() + ), + CollectionProgress::Planning => "planning".to_string(), + CollectionProgress::Finalizing => "finalizing".to_string(), + } + } + + #[test] + fn default_collect_site_reports_root_planning_and_offset_followups() { + let collector = ProgressCollector; + let root = Url::parse("https://publisher.example/").expect("should parse root URL"); + let news = Url::parse("https://publisher.example/news").expect("should parse news URL"); + let broken = + Url::parse("https://publisher.example/broken").expect("should parse broken URL"); + let mut events = Vec::new(); + let mut outcomes = Vec::new(); + + collector + .collect_site( + &root, + &[], + &mut |event| { + events.push(record_progress(event)); + Ok(()) + }, + &mut |_, _| Ok(vec![news.clone(), broken.clone()]), + &mut |url, result| { + outcomes.push((url.path().to_string(), result.is_ok())); + Ok(ControlFlow::Continue) + }, + ) + .expect("should collect site despite one page outcome failing"); + + assert_eq!( + events, + [ + "loading:1/?:/", + "planning", + "loading:2/3:/news", + "loading:3/3:/broken", + ] + ); + assert_eq!( + outcomes, + [ + ("/".to_string(), true), + ("/news".to_string(), true), + ("/broken".to_string(), false) + ] + ); + } + + #[test] + fn default_collect_pages_reports_a_fixed_total() { + let collector = ProgressCollector; + let targets = [ + Url::parse("https://publisher.example/").expect("should parse root URL"), + Url::parse("https://publisher.example/broken").expect("should parse broken URL"), + ]; + let mut events = Vec::new(); + + collector + .collect_pages( + &targets, + &[], + &mut |event| { + events.push(record_progress(event)); + Ok(()) + }, + &mut |_, _| Ok(ControlFlow::Continue), + ) + .expect("should deliver failed page as an outcome"); + + assert_eq!(events, ["loading:1/2:/", "loading:2/2:/broken"]); + } + + #[test] + fn default_collection_stops_when_progress_fails() { + let collector = ProgressCollector; + let targets = [Url::parse("https://publisher.example/").expect("should parse root URL")]; + + let error = collector + .collect_pages( + &targets, + &[], + &mut |_| Err(report_error("simulated progress failure")), + &mut |_, _| panic!("page sink should not run after progress failure"), + ) + .expect_err("should return progress failure"); + + assert!(format!("{error:?}").contains("simulated progress failure")); + } +} + +/// A single slot read from the page's live GPT registry. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedGptSlot { + /// The GAM ad-unit path (`slot.getAdUnitPath()`). + pub(crate) gam_unit_path: String, + /// The slot's div element id (`slot.getSlotElementId()`). + pub(crate) div_id: String, + /// Numeric `[width, height]` sizes (`slot.getSizes()`, fluid entries dropped). + pub(crate) sizes: Vec<(u32, u32)>, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedScriptTag { + pub(crate) src: Option, + pub(crate) inline_text: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedRequest { + pub(crate) url: String, + pub(crate) resource_type: Option, +} + +impl CollectedPage { + pub(crate) fn requested_url(&self) -> Result { + Url::parse(&self.requested_url) + } + + pub(crate) fn final_url(&self) -> Result { + Url::parse(&self.final_url) + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs new file mode 100644 index 000000000..76d57cf9e --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs @@ -0,0 +1,864 @@ +//! Pure crawl planning: turn discovered links and sitemap entries into the +//! bounded set of pages worth loading in a browser. +//! +//! The goal is deliberately *not* site coverage. Ad slots repeat per site +//! section, and the generated config needs one glob pair per section +//! (`/news` and `/news/*`), so one representative page per section is enough. +//! That keeps the crawl proportional to the publisher's taxonomy (a dozen +//! sections) rather than its catalog (tens of thousands of articles). +//! +//! Two sources feed the plan and each supplies a half the other cannot: +//! +//! - **Navigation links** give section *landing* paths (`/news`), which +//! sitemaps routinely omit, and are the publisher's own taxonomy declaration. +//! - **Sitemap entries** give a real *article* per section (`/news/story-abc`), +//! which is where in-content slots live, and reveal sections hidden behind a +//! navigation overflow menu. + +use std::collections::BTreeMap; + +use url::Url; + +use super::collector::CollectedLink; + +/// Path segments that are never a content section worth sampling. +/// +/// These carry either no ad stack at all or an unrepresentative one, and +/// crawling them spends budget that a real section needs. +const NOISE_SEGMENTS: &[&str] = &[ + "about", + "about-us", + "account", + "author", + "cart", + "contact", + "editorial-policy", + "login", + "logout", + "newsletter", + "page", + "press", + "privacy", + "register", + "search", + "sitemap", + "subscribe", + "terms", +]; + +/// File extensions that are assets rather than pages. +const NON_PAGE_EXTENSIONS: &[&str] = &[ + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".svg", ".ico", ".css", ".js", ".json", + ".xml", ".pdf", ".zip", ".mp4", ".mp3", ".rss", +]; + +/// ISO 639-1 alpha-2 language codes, sorted for binary search. +/// +/// Country codes are deliberately absent: `/us` and `/tv` are section roots on +/// plenty of publishers, and only the language form appears as a URL locale +/// prefix on its own. +const ISO_639_1_CODES: &[&str] = &[ + "aa", "ab", "ae", "af", "ak", "am", "an", "ar", "as", "av", "ay", "az", "ba", "be", "bg", "bh", + "bi", "bm", "bn", "bo", "br", "bs", "ca", "ce", "ch", "co", "cr", "cs", "cu", "cv", "cy", "da", + "de", "dv", "dz", "ee", "el", "en", "eo", "es", "et", "eu", "fa", "ff", "fi", "fj", "fo", "fr", + "fy", "ga", "gd", "gl", "gn", "gu", "gv", "ha", "he", "hi", "ho", "hr", "ht", "hu", "hy", "hz", + "ia", "id", "ie", "ig", "ii", "ik", "io", "is", "it", "iu", "ja", "jv", "ka", "kg", "ki", "kj", + "kk", "kl", "km", "kn", "ko", "kr", "ks", "ku", "kv", "kw", "ky", "la", "lb", "lg", "li", "ln", + "lo", "lt", "lu", "lv", "mg", "mh", "mi", "mk", "ml", "mn", "mr", "ms", "mt", "my", "na", "nb", + "nd", "ne", "ng", "nl", "nn", "no", "nr", "nv", "ny", "oc", "oj", "om", "or", "os", "pa", "pi", + "pl", "ps", "pt", "qu", "rm", "rn", "ro", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", + "sl", "sm", "sn", "so", "sq", "sr", "ss", "st", "su", "sv", "sw", "ta", "te", "tg", "th", "ti", + "tk", "tl", "tn", "to", "tr", "ts", "tt", "tw", "ty", "ug", "uk", "ur", "uz", "ve", "vi", "vo", + "wa", "wo", "xh", "yi", "yo", "za", "zh", "zu", +]; + +/// Filenames that name a directory's index document rather than a page of their +/// own, so a link to one is treated as a link to the parent directory. +const DIRECTORY_INDEX_NAMES: &[&str] = &[ + "index.html", + "index.htm", + "index.php", + "default.html", + "default.htm", + "default.php", + "home.html", + "home.htm", + "home.php", +]; + +/// Bounds on how much of a site a single run will load. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct CrawlBudget { + /// Maximum number of sections to sample. + pub(crate) max_sections: usize, + /// Maximum number of pages to load in total, including the root. + pub(crate) max_pages: usize, +} + +impl Default for CrawlBudget { + fn default() -> Self { + Self { + max_sections: 8, + max_pages: 17, + } + } +} + +/// One section selected for sampling. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct PlannedSection { + /// The path segment at [`CrawlPlan::section_segment`] identifying the section. + pub(super) segment: String, + /// The section landing page, when one was observed. + pub(super) landing: Option, + /// A representative content page inside the section, when one was observed. + pub(super) article: Option, +} + +impl PlannedSection { + /// The pages to load for this section, landing first. + fn targets(&self) -> impl Iterator { + self.landing.iter().chain(self.article.iter()) + } +} + +/// The bounded outcome of planning. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct CrawlPlan { + /// Sections selected for sampling, highest confidence first. + pub(super) sections: Vec, + /// Sections found but dropped because the budget was already spent. + pub(super) dropped_sections: Vec, + /// Human-readable notes about how the plan was reached. + pub(super) notes: Vec, + /// Path segment used to distinguish sections in this crawl. + pub(super) section_segment: usize, +} + +impl CrawlPlan { + /// Page URLs to load, in crawl order. The root is *not* included — the + /// caller has already collected it in order to plan at all. + pub(super) fn targets(&self) -> Vec { + self.sections + .iter() + .flat_map(PlannedSection::targets) + .cloned() + .collect() + } +} + +/// Evidence gathered about one candidate section before ranking. +#[derive(Debug, Default)] +struct SectionCandidate { + landing: Option, + article: Option, + in_nav: bool, + in_sitemap: bool, + link_count: usize, +} + +impl SectionCandidate { + /// Confidence ordering: corroborated by both sources beats either alone, + /// and navigation beats a sitemap-only hit because navigation is the + /// publisher's own statement of what its sections are. + fn rank(&self) -> u8 { + match (self.in_nav, self.in_sitemap) { + (true, true) => 3, + (true, false) => 2, + (false, true) => 1, + (false, false) => 0, + } + } +} + +/// Plans the crawl from the root page's links and any sitemap entries. +/// +/// `root` bounds the crawl: every candidate must share its origin, which also +/// stops a hostile or misconfigured `robots.txt` from redirecting the crawl (and +/// the operator's cookies) at an unrelated host. +pub(super) fn plan_crawl( + root: &Url, + links: &[CollectedLink], + sitemap_locs: &[String], + budget: CrawlBudget, +) -> CrawlPlan { + let section_segment = usize::from(root_is_locale_prefix(root)); + let mut candidates: BTreeMap = BTreeMap::new(); + let mut notes = Vec::new(); + + for link in links { + let Some(url) = same_origin_page_url(root, &link.url, section_segment) else { + continue; + }; + let Some(segment) = section_at(&url, section_segment) else { + continue; + }; + let entry = candidates.entry(segment).or_default(); + entry.in_nav |= link.in_nav; + entry.link_count += 1; + record_url(entry, &url, section_segment); + } + + let mut sitemap_pages = 0_usize; + for loc in sitemap_locs { + let Some(url) = same_origin_page_url(root, loc, section_segment) else { + continue; + }; + let Some(segment) = section_at(&url, section_segment) else { + continue; + }; + sitemap_pages += 1; + let entry = candidates.entry(segment).or_default(); + entry.in_sitemap = true; + record_url(entry, &url, section_segment); + } + + if !sitemap_locs.is_empty() { + notes.push(format!( + "sitemap contributed {sitemap_pages} same-origin page(s) across {} section(s)", + candidates.values().filter(|c| c.in_sitemap).count() + )); + } + if links.iter().all(|link| !link.in_nav) && !links.is_empty() { + notes.push( + "no navigation links were found; sections were inferred from body links only" + .to_string(), + ); + } + + // Rank before truncating: confidence first, then how heavily the section is + // linked, then the segment name so runs are reproducible. + let mut ranked: Vec<(String, SectionCandidate)> = candidates.into_iter().collect(); + ranked.sort_by(|(left_segment, left), (right_segment, right)| { + right + .rank() + .cmp(&left.rank()) + .then(right.link_count.cmp(&left.link_count)) + .then(left_segment.cmp(right_segment)) + }); + + let mut sections = Vec::new(); + let mut dropped_sections = Vec::new(); + // The root page is already collected and counts against the page budget. + let mut pages_used = 1_usize; + for (segment, candidate) in ranked { + let planned = PlannedSection { + segment: segment.clone(), + landing: candidate.landing, + article: candidate.article, + }; + let cost = planned.targets().count(); + if cost == 0 { + continue; + } + if sections.len() >= budget.max_sections || pages_used + cost > budget.max_pages { + dropped_sections.push(segment); + continue; + } + pages_used += cost; + sections.push(planned); + } + + if !dropped_sections.is_empty() { + let shown = dropped_sections + .iter() + .take(10) + .cloned() + .collect::>() + .join(", "); + let remainder = dropped_sections.len().saturating_sub(10); + let suffix = if remainder == 0 { + String::new() + } else { + format!(", and {remainder} more") + }; + notes.push(format!( + "budget reached: {} section(s) not sampled ({shown}{suffix}); raise --max-sections/--max-pages to include them", + dropped_sections.len(), + )); + } + + CrawlPlan { + sections, + dropped_sections, + notes, + section_segment, + } +} + +/// Files a URL as the section's landing page or its representative article. +/// +/// The first candidate of each kind wins, so a run is stable given stable input. +fn record_url(entry: &mut SectionCandidate, url: &Url, section_segment: usize) { + if segment_count(url) == section_segment + 1 { + if entry.landing.is_none() { + entry.landing = Some(url.clone()); + } + } else if entry.article.is_none() { + entry.article = Some(url.clone()); + } +} + +/// Parses `raw` against `root` and keeps it only if it is a same-origin page. +/// +/// Rejects other origins, non-HTTP schemes, asset extensions, and paginated or +/// utility paths. Query and fragment are dropped so `/news?page=2` and +/// `/news#top` collapse onto `/news`. +fn same_origin_page_url(root: &Url, raw: &str, section_segment: usize) -> Option { + let mut url = root.join(raw).ok()?; + if !matches!(url.scheme(), "http" | "https") || url.origin() != root.origin() { + return None; + } + url.set_query(None); + url.set_fragment(None); + + let path = percent_decode_for_filtering(url.path()).to_ascii_lowercase(); + if NON_PAGE_EXTENSIONS + .iter() + .any(|extension| path.ends_with(extension)) + { + return None; + } + // A section reachable only through its index document is still that section: + // `/news/index.html` is `/news`. Rejecting the URL outright loses the + // section; dropping the filename keeps it. + if path + .split('/') + .rfind(|part| !part.is_empty()) + .is_some_and(|last| DIRECTORY_INDEX_NAMES.contains(&last)) + { + url.path_segments_mut().ok()?.pop(); + } + let path = percent_decode_for_filtering(url.path()).to_ascii_lowercase(); + let segments: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + if segments.is_empty() { + return None; + } + if NOISE_SEGMENTS.contains(&segments.get(section_segment).copied().unwrap_or_default()) { + return None; + } + if section_segment > 0 { + let root_path = percent_decode_for_filtering(root.path()).to_ascii_lowercase(); + let root_segments: Vec<&str> = root_path + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + if !segments.starts_with(&root_segments) { + return None; + } + } + // `/news/page/2` is the same inventory as `/news`, so it is not a second + // sample worth spending a page load on. + if segments.contains(&"page") { + return None; + } + Some(url) +} + +/// The non-empty path segment at `index`, percent-decoded and lowercased. +fn section_at(url: &Url, index: usize) -> Option { + percent_decode_for_filtering(url.path()) + .split('/') + .filter(|part| !part.is_empty()) + .nth(index) + .map(str::to_ascii_lowercase) +} + +/// Whether the requested root is nothing but a locale prefix, which puts +/// sections one segment deeper than usual. +fn root_is_locale_prefix(root: &Url) -> bool { + let segments: Vec<&str> = root + .path() + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + matches!(segments.as_slice(), [locale] if is_locale_segment(locale)) +} + +/// Whether a root's single path segment is a locale prefix (`/en`, `/en-gb`) +/// rather than a content section. +/// +/// The language half must be a real ISO 639-1 code. Accepting any two letters +/// read ordinary section roots — `/tv`, `/ai`, `/us` — as locales, which shifts +/// `section_segment` by one: article slugs then become "sections" and the +/// containment check below discards the root's real siblings. +fn is_locale_segment(segment: &str) -> bool { + let segment = segment.to_ascii_lowercase(); + match segment.as_bytes() { + [_, _] => is_language_code(&segment), + [_, _, b'-', c, d] => { + is_language_code(&segment[..2]) && c.is_ascii_alphabetic() && d.is_ascii_alphabetic() + } + _ => false, + } +} + +/// Whether `segment` is an ISO 639-1 alpha-2 language code. +fn is_language_code(segment: &str) -> bool { + ISO_639_1_CODES.binary_search(&segment).is_ok() +} + +/// Decodes percent escapes solely for normalized path classification. +fn percent_decode_for_filtering(path: &str) -> String { + let bytes = path.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == b'%' + && index + 2 < bytes.len() + && let (Some(high), Some(low)) = + (hex_value(bytes[index + 1]), hex_value(bytes[index + 2])) + { + decoded.push((high << 4) | low); + index += 3; + } else { + decoded.push(bytes[index]); + index += 1; + } + } + String::from_utf8_lossy(&decoded).into_owned() +} + +/// Converts one ASCII hexadecimal digit to its numeric value. +fn hex_value(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +/// Count of non-empty path segments. +fn segment_count(url: &Url) -> usize { + url.path() + .split('/') + .filter(|part| !part.is_empty()) + .count() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn root() -> Url { + Url::parse("https://publisher.example/").expect("valid root") + } + + fn nav(path: &str) -> CollectedLink { + CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: true, + } + } + + fn body(path: &str) -> CollectedLink { + CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: false, + } + } + + fn segments(plan: &CrawlPlan) -> Vec<&str> { + plan.sections + .iter() + .map(|section| section.segment.as_str()) + .collect() + } + + #[test] + fn pairs_a_landing_page_with_an_article_from_the_sitemap() { + let plan = plan_crawl( + &root(), + &[nav("/news")], + &["https://publisher.example/news/story-abc".to_string()], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the witnessed section should be planned" + ); + let section = &plan.sections[0]; + assert_eq!( + section.landing.as_ref().map(Url::as_str), + Some("https://publisher.example/news") + ); + assert_eq!( + section.article.as_ref().map(Url::as_str), + Some("https://publisher.example/news/story-abc") + ); + assert_eq!(plan.targets().len(), 2, "should load landing then article"); + } + + #[test] + fn cross_origin_candidates_are_dropped() { + // Guards both the sitemap (a `Sitemap:` directive can point anywhere) + // and links: the crawl carries operator cookies, so it must not leave + // the requested origin. + let plan = plan_crawl( + &root(), + &[CollectedLink { + url: "https://tracker.example/news".to_string(), + in_nav: true, + }], + &["https://other.example/deals/x".to_string()], + CrawlBudget::default(), + ); + + assert!( + plan.sections.is_empty(), + "no off-origin section should survive, got {:?}", + segments(&plan) + ); + } + + #[test] + fn utility_paths_and_assets_are_filtered() { + let plan = plan_crawl( + &root(), + &[ + nav("/about-us"), + nav("/search"), + nav("/editorial-policy"), + nav("/logo.png"), + nav("/feed.xml"), + nav("/news/page/2"), + nav("/news"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the real content section should remain" + ); + } + + #[test] + fn query_and_fragment_collapse_onto_one_landing_page() { + let plan = plan_crawl( + &root(), + &[nav("/news?utm_source=x"), nav("/news#top"), nav("/news")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the witnessed section should be planned" + ); + assert_eq!( + plan.sections[0].landing.as_ref().map(Url::as_str), + Some("https://publisher.example/news"), + "tracking query and fragment should be stripped" + ); + } + + #[test] + fn nav_and_sitemap_corroboration_outranks_either_alone() { + let plan = plan_crawl( + &root(), + &[nav("/features"), body("/reviews")], + &[ + "https://publisher.example/features/story".to_string(), + "https://publisher.example/deals/x".to_string(), + ], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan)[0], + "features", + "nav + sitemap should rank first, got {:?}", + segments(&plan) + ); + } + + #[test] + fn budget_truncates_and_reports_what_was_dropped() { + let links: Vec = ["a", "b", "c", "d"] + .iter() + .map(|segment| nav(&format!("/{segment}"))) + .collect(); + + let plan = plan_crawl( + &root(), + &links, + &[], + CrawlBudget { + max_sections: 2, + max_pages: 17, + }, + ); + + assert_eq!(plan.sections.len(), 2, "section cap should be honoured"); + assert_eq!( + plan.dropped_sections.len(), + 2, + "sections past the budget should be reported as dropped" + ); + assert!( + plan.notes + .iter() + .any(|note| note.contains("budget reached")), + "dropping sections must be reported, not silent: {:?}", + plan.notes + ); + } + + #[test] + fn page_budget_counts_the_already_collected_root() { + // max_pages = 3 leaves room for exactly one landing+article pair on top + // of the root page the caller already loaded. + let plan = plan_crawl( + &root(), + &[nav("/news"), nav("/deals")], + &[ + "https://publisher.example/news/a".to_string(), + "https://publisher.example/deals/b".to_string(), + ], + CrawlBudget { + max_sections: 8, + max_pages: 3, + }, + ); + + assert_eq!( + plan.targets().len(), + 2, + "root + 2 pages fills max_pages = 3" + ); + assert_eq!( + plan.dropped_sections.len(), + 1, + "the section past the budget should be reported as dropped" + ); + } + + #[test] + fn body_only_links_still_yield_sections_with_a_note() { + let plan = plan_crawl( + &root(), + &[body("/news"), body("/deals")], + &[], + CrawlBudget::default(), + ); + + assert_eq!(segments(&plan), ["deals", "news"]); + assert!( + plan.notes + .iter() + .any(|note| note.contains("no navigation links")), + "a nav-less page should say so: {:?}", + plan.notes + ); + } + + #[test] + fn empty_input_plans_nothing_rather_than_panicking() { + let plan = plan_crawl(&root(), &[], &[], CrawlBudget::default()); + + assert!( + plan.sections.is_empty(), + "no input means no sections to sample" + ); + assert!( + plan.targets().is_empty(), + "no sections means nothing to load" + ); + } + + #[test] + fn locale_root_plans_sections_from_the_second_segment() { + let locale_root = Url::parse("https://publisher.example/en").expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav("/en/news"), nav("/en/deals")], + &[ + "https://publisher.example/en/news/story".to_string(), + "https://publisher.example/en/deals/item".to_string(), + ], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 1, + "a locale root puts sections one segment deeper" + ); + assert_eq!(segments(&plan), ["deals", "news"]); + assert_eq!( + plan.targets().len(), + 4, + "each section contributes a landing page and an article" + ); + } + + #[test] + fn encoded_noise_and_page_extensions_are_filtered() { + let plan = plan_crawl( + &root(), + &[ + nav("/%70rivacy"), + nav("/index.html"), + nav("/archive.htm"), + nav("/story.php"), + nav("/news"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["archive.htm", "news", "story.php"], + "only directory-index documents should be excluded by extension" + ); + } + + #[test] + fn a_two_letter_section_root_is_not_read_as_a_locale() { + // `/tv`, `/ai` and `/us` are section roots, not locales. Reading them as + // locales moves the section segment to 1, so article slugs become + // "sections" and the root's real siblings are discarded. + for root_path in ["/tv", "/ai", "/us"] { + let section_root = Url::parse(&format!("https://publisher.example{root_path}")) + .expect("should parse root"); + let plan = plan_crawl( + §ion_root, + &[ + nav(&format!("{root_path}/story-one")), + nav(&format!("{root_path}/story-two")), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 0, + "`{root_path}` should be a section root, not a locale prefix" + ); + assert_eq!( + segments(&plan), + [root_path.trim_start_matches('/')], + "articles below `{root_path}` should stay one section" + ); + } + } + + #[test] + fn a_real_language_prefix_is_still_read_as_a_locale() { + for root_path in ["/en", "/fr", "/pt-br"] { + let locale_root = Url::parse(&format!("https://publisher.example{root_path}")) + .expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav(&format!("{root_path}/news"))], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 1, + "`{root_path}` is a locale prefix, so sections start one segment in" + ); + assert_eq!(segments(&plan), ["news"]); + } + } + + #[test] + fn a_section_reachable_only_by_its_index_document_collapses_to_the_parent() { + let plan = plan_crawl( + &root(), + &[ + nav("/news/index.html"), + nav("/deals/index.php"), + nav("/sport/home.htm"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["deals", "news", "sport"], + "an index document names its section rather than disqualifying it" + ); + let targets: Vec = plan + .targets() + .iter() + .map(|url| url.path().to_string()) + .collect(); + assert_eq!( + targets, + ["/deals", "/news", "/sport"], + "the parent directory is what gets loaded" + ); + } + + #[test] + fn locale_root_rejects_candidates_outside_its_path_prefix() { + let locale_root = Url::parse("https://publisher.example/en").expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav("/en/news"), nav("/fr/deals")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "a locale-root crawl must not mix another locale on the same origin" + ); + } + + #[test] + fn a_section_root_does_not_treat_article_slugs_as_sections() { + let section_root = Url::parse("https://publisher.example/news").expect("should parse root"); + let plan = plan_crawl( + §ion_root, + &[nav("/news/story-one"), nav("/news/story-two")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 0, + "a generic one-segment root is a section, not necessarily a locale" + ); + assert_eq!( + segments(&plan), + ["news"], + "articles below a section root should remain one section" + ); + } + + #[test] + fn dropped_section_note_is_capped() { + let links: Vec<_> = (0..15) + .map(|index| nav(&format!("/section-{index:02}"))) + .collect(); + let plan = plan_crawl( + &root(), + &links, + &[], + CrawlBudget { + max_sections: 0, + max_pages: 1, + }, + ); + let note = plan + .notes + .iter() + .find(|note| note.contains("budget reached")) + .expect("should report dropped sections"); + + assert!(note.contains("and 5 more")); + assert!(!note.contains("section-14")); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs new file mode 100644 index 000000000..5c50e34e5 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs @@ -0,0 +1,794 @@ +//! Cross-page slot evidence: what each slot looked like on every page it was +//! observed on. +//! +//! A single page cannot distinguish a literal ad-unit path from a templated one, +//! so inference needs the *set* of observations per slot rather than one +//! snapshot. This module accumulates that set and is deliberately the only place +//! that reconciles a slot seen more than once: +//! +//! - **Formats union.** A size that appears only on article pages (a 300x600 +//! rail, say) must survive alongside the homepage's sizes. Taking the first +//! page's formats would silently narrow the slot. +//! - **Unit paths are kept, not collapsed.** Divergence across pages is the +//! signal inference reads; discarding it is what makes templating impossible. +//! - **Network ids must agree.** Two different GAM networks in one crawl means +//! the pages are not one property, and writing either one would be a guess. +//! +//! Slots are keyed on the *normalized div stem* produced by +//! [`discover_gpt_slots`](super::gpt_slots::discover_gpt_slots), because raw GPT +//! div ids carry per-render framework hashes and would otherwise look like a new +//! slot on every page. + +use std::collections::{BTreeMap, BTreeSet}; + +use super::gpt_slots::DiscoveredSlots; +use crate::error::{CliResult, cli_error}; + +/// One observation of a slot on one page. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(super) struct EvidenceRow { + /// The page path the slot was observed on, normalized (leading `/`, no + /// query or fragment). + pub(super) path: String, + /// The literal GAM ad-unit path the live page used for this slot. + pub(super) unit_path: String, +} + +/// Everything observed about one slot across the crawl. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SlotEvidence { + /// Config slot id derived from the div stem. + pub(super) id: String, + /// Normalized div stem, used as the runtime `div_id` prefix. + pub(super) div_id: String, + /// Union of every pixel size observed for this slot, smallest first. + pub(super) formats: BTreeSet<(u32, u32)>, + /// Whether any page carrying this slot showed header-bidding signals. + pub(super) has_prebid: bool, + /// Distinct `(path, unit_path)` observations, in a stable order. + pub(super) rows: BTreeSet, +} + +impl SlotEvidence { + /// The distinct literal unit paths observed for this slot. + pub(super) fn unit_paths(&self) -> BTreeSet<&str> { + self.rows.iter().map(|row| row.unit_path.as_str()).collect() + } + + /// The distinct page paths this slot was observed on. + pub(super) fn paths(&self) -> BTreeSet<&str> { + self.rows.iter().map(|row| row.path.as_str()).collect() + } +} + +/// Slots grouped by the shape that would make them one placement: an identical +/// ad-unit path and an identical format set. +type SlotsByShape<'a> = BTreeMap<(String, Vec<(u32, u32)>), Vec<&'a SlotEvidence>>; + +/// Several observed slots that are really one placement under volatile div ids. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct FragmentGroup { + /// The volatile div ids observed, in evidence order. + pub(super) div_ids: Vec, + /// The ad-unit path every fragment shared. + pub(super) unit_path: String, + /// The stable prefix the ids share, when they share a useful one. + /// + /// Offered to the operator as a starting point only. It is deliberately not + /// written as a `div_id`: the shared prefix reaches only as far as the + /// *observed* tokens happen to agree, so it would keep matching this crawl's + /// ids and stop matching the next render's. + pub(super) suggested_prefix: Option, +} + +/// Whether no two slots were ever seen on the same page. +fn pages_are_disjoint(slots: &[&SlotEvidence]) -> bool { + for (index, slot) in slots.iter().enumerate() { + let pages = slot.paths(); + if slots[index + 1..] + .iter() + .any(|other| other.paths().intersection(&pages).next().is_some()) + { + return false; + } + } + true +} + +/// The longest prefix the div ids share, trimmed back to a separator. +/// +/// Trimming matters: the raw common prefix usually ends mid-token (the leading +/// digits of a timestamp two fragments happen to share), which is worse than +/// useless as a suggestion. Cutting at the last `-` or `_` yields the part a +/// human would recognise as the placement's name. +fn shared_div_prefix(slots: &[&SlotEvidence]) -> Option { + let mut prefix: &str = slots.first()?.div_id.as_str(); + for slot in &slots[1..] { + let mut shared_end = 0; + for ((byte_index, left), right) in prefix.char_indices().zip(slot.div_id.chars()) { + if left != right { + break; + } + shared_end = byte_index + left.len_utf8(); + } + prefix = &prefix[..shared_end]; + } + let trimmed = prefix.trim_end_matches(|ch: char| ch != '-' && ch != '_'); + let candidate = trimmed.trim_end_matches(['-', '_']); + (!candidate.is_empty()).then(|| candidate.to_string()) +} + +/// Slot evidence accumulated across every collected page. +#[derive(Debug, Clone, Default)] +pub(super) struct EvidenceTable { + slots: BTreeMap, + /// Div stems in first-seen order, so generated config keeps crawl order + /// rather than alphabetical order. + order: Vec, + network_ids: BTreeSet, + /// Every page path folded in, including those that yielded no slots. + pages: BTreeSet, + /// Page paths that produced no slot evidence at all. + empty_pages: BTreeSet, + /// Page paths that produced slot evidence on at least one selected profile. + non_empty_pages: BTreeSet, + /// Div stems any page refused as ambiguous, unioned across the crawl. + /// + /// The verdict has to outlive the page that reached it. Article pages carry + /// several in-content units and refuse the shared prefix; a landing page + /// carries one and would otherwise contribute it as a usable slot, so the + /// written config would depend on which pages the crawl happened to sample. + ambiguous_stems: BTreeSet, + /// Normalized div IDs refused from generation but observed live. + refused_div_ids: BTreeSet, +} + +impl EvidenceTable { + /// Folds one page's discovered slots into the table. + /// + /// `path` is the page's normalized request path; it is what page patterns + /// and `{section}` derivation are computed from later, so it must be the + /// post-redirect path actually audited. + pub(super) fn fold_page(&mut self, path: &str, discovered: &DiscoveredSlots) { + self.pages.insert(path.to_string()); + if let Some(network_id) = &discovered.gam_network_id { + self.network_ids.insert(network_id.clone()); + } + if !discovered.had_slot_evidence { + if !self.non_empty_pages.contains(path) { + self.empty_pages.insert(path.to_string()); + } + return; + } + self.non_empty_pages.insert(path.to_string()); + self.empty_pages.remove(path); + self.ambiguous_stems + .extend(discovered.ambiguous_stems.iter().cloned()); + self.refused_div_ids + .extend(discovered.refused_div_ids.iter().cloned()); + + for slot in &discovered.slots { + let entry = self.slots.entry(slot.div_id.clone()).or_insert_with(|| { + self.order.push(slot.div_id.clone()); + SlotEvidence { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + formats: BTreeSet::new(), + has_prebid: false, + rows: BTreeSet::new(), + } + }); + // Union rather than replace: a size seen only on one page type is + // still a size this slot serves. + entry.formats.extend(slot.formats.iter().copied()); + entry.has_prebid |= slot.has_prebid; + entry.rows.insert(EvidenceRow { + path: path.to_string(), + unit_path: slot.gam_unit_path.clone(), + }); + } + } + + /// Slots in first-seen order, excluding stems any page refused as ambiguous. + pub(super) fn slots(&self) -> impl Iterator { + self.order + .iter() + .filter(|div_id| !self.ambiguous_stems.contains(*div_id)) + .filter_map(|div_id| self.slots.get(div_id)) + } + + /// Every normalized div ID observed, including all refused evidence. + pub(super) fn observed_div_ids(&self) -> impl Iterator { + self.order + .iter() + .map(String::as_str) + .chain(self.ambiguous_stems.iter().map(String::as_str)) + .chain(self.refused_div_ids.iter().map(String::as_str)) + } + + /// Normalized div IDs observed as concrete live elements. + /// + /// Unlike [`EvidenceTable::observed_div_ids`], this excludes identifiers + /// that exist only as refused ambiguity or volatility evidence. Prefix + /// routing must not treat those inferred stems as literal DOM elements. + pub(super) fn observed_literals(&self) -> impl Iterator { + self.order + .iter() + .filter(|div_id| !self.ambiguous_stems.contains(*div_id)) + .filter(|div_id| !self.refused_div_ids.contains(*div_id)) + .map(String::as_str) + } + + /// Number of usable distinct slots observed. + pub(super) fn slot_count(&self) -> usize { + self.slots().count() + } + + /// Every page path folded in, whether or not it yielded slots. + pub(super) fn pages(&self) -> &BTreeSet { + &self.pages + } + + /// Page paths that produced no slot evidence. + /// + /// A high proportion of these is the signature of a bot challenge serving + /// interstitials instead of the real site, which is worth refusing to write + /// from rather than persisting a half-empty config. + pub(super) fn empty_pages(&self) -> &BTreeSet { + &self.empty_pages + } + + /// Whether any slot was observed at all, ambiguous ones included. + /// + /// Deliberately not `slot_count() == 0`: a crawl that saw only ambiguous + /// placements did observe an ad stack, and the caller distinguishes "this + /// page has no slots" from "every slot found was refused". + pub(super) fn is_empty(&self) -> bool { + self.slots.is_empty() + } + + /// Groups of slots that are one slot wearing a different div id per page. + /// + /// Some ad stacks build div ids from a per-render token — a timestamp, a + /// framework id — so the same placement arrives under a new key on every + /// page. Written verbatim those ids never match at runtime, and the + /// fragmentation also starves template inference, which needs to see one + /// slot more than once. + /// + /// Detection is by evidence rather than by guessing at token shapes, because + /// each stack invents its own. Candidates share an identical ad-unit path and + /// identical formats; what separates a fragmented slot from two legitimate + /// siblings on the same unit is **co-occurrence**. Real siblings appear + /// together on a page; fragments of one slot never do, because each page + /// produces exactly one of them. + pub(super) fn fragmented_slots(&self) -> Vec { + let mut by_shape: SlotsByShape<'_> = BTreeMap::new(); + for slot in self.slots() { + // Only slots pinned to exactly one unit path can be compared this + // way; a slot whose unit varies is inference's problem, not this one. + let units = slot.unit_paths(); + if units.len() != 1 { + continue; + } + let unit = (*units.iter().next().expect("should have one unit path")).to_string(); + let formats: Vec<(u32, u32)> = slot.formats.iter().copied().collect(); + by_shape.entry((unit, formats)).or_default().push(slot); + } + + by_shape + .into_iter() + .filter(|(_, slots)| slots.len() > 1) + .filter(|(_, slots)| pages_are_disjoint(slots)) + .filter_map(|((unit_path, _), slots)| { + let suggested_prefix = shared_div_prefix(&slots); + (suggested_prefix.is_some() || slots.len() >= 3).then(|| FragmentGroup { + div_ids: slots.iter().map(|slot| slot.div_id.clone()).collect(), + unit_path, + suggested_prefix, + }) + }) + .collect() + } + + /// The single GAM network id observed across the crawl. + /// + /// # Errors + /// + /// Returns an error when pages disagreed. Two networks in one crawl means + /// the pages are not one property (a syndicated subdomain, a child network, + /// an off-origin redirect that slipped through), and picking either would be + /// a guess that silently bids against the wrong inventory. + pub(super) fn network_id(&self) -> CliResult> { + let mut found = self.network_ids.iter(); + let Some(first) = found.next() else { + return Ok(None); + }; + if self.network_ids.len() > 1 { + let all: Vec<&str> = self.network_ids.iter().map(String::as_str).collect(); + return cli_error(format!( + "the crawled pages reported more than one GAM network id ({}); \ + they do not appear to be one property, so no network id can be \ + chosen safely. Audit a single property, or pass explicit URLs", + all.join(", ") + )); + } + Ok(Some(first.clone())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector::CollectedGptSlot; + use crate::commands::audit::generate::gpt_slots::discover_gpt_slots; + + /// One live slot as `(unit path, div id, sizes)`. + type SlotFixture<'a> = (&'a str, &'a str, &'a [(u32, u32)]); + + fn page(slots: &[SlotFixture<'_>], has_prebid: bool) -> DiscoveredSlots { + let registry: Vec = slots + .iter() + .map(|(unit_path, div_id, sizes)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: sizes.to_vec(), + }) + .collect(); + discover_gpt_slots(®istry, &[], has_prebid) + } + + #[test] + fn formats_union_across_pages_instead_of_first_seen_winning() { + // The 300x600 rail only ever renders on article pages. Keeping the + // homepage's format list alone would silently narrow the slot. + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-rail", &[(300, 250)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-rail", &[(300, 600)])], false), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!( + slot.formats.iter().copied().collect::>(), + [(300, 250), (300, 600)], + "both pages' sizes should survive" + ); + assert_eq!(table.slot_count(), 1, "one div stem is one slot"); + } + + #[test] + fn divergent_unit_paths_are_preserved_as_separate_rows() { + // This divergence is the entire signal template inference reads. + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], false), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!( + slot.unit_paths().into_iter().collect::>(), + ["/123/site/home", "/123/site/news"], + "both observed unit paths must be retained" + ); + assert_eq!( + slot.paths().into_iter().collect::>(), + ["/", "/news/story"] + ); + } + + #[test] + fn repeated_identical_observations_collapse() { + let mut table = EvidenceTable::default(); + let observed = page(&[("/123/site/home", "ad-header", &[(728, 90)])], false); + table.fold_page("/", &observed); + table.fold_page("/", &observed); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!(slot.rows.len(), 1, "the same page twice is one observation"); + } + + #[test] + fn prebid_is_sticky_once_any_page_shows_it() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], true), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert!( + slot.has_prebid, + "a slot proven to run prebid on any page runs prebid" + ); + } + + #[test] + fn slots_keep_first_seen_order_not_alphabetical_order() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page( + &[ + ("/123/site/home", "zeta-slot", &[(728, 90)]), + ("/123/site/home", "alpha-slot", &[(300, 250)]), + ], + false, + ), + ); + + let ids: Vec<&str> = table.slots().map(|slot| slot.div_id.as_str()).collect(); + assert_eq!( + ids, + ["zeta-slot", "alpha-slot"], + "generated config should follow crawl order" + ); + } + + #[test] + fn refused_only_div_ids_are_observed_but_not_literals() { + let mut discovered = page(&[("/123/site/home", "ad-x-stable", &[(300, 250)])], false); + discovered.refused_div_ids.insert("ad-x".to_string()); + let mut table = EvidenceTable::default(); + table.fold_page("/", &discovered); + + assert_eq!( + table.observed_div_ids().collect::>(), + ["ad-x-stable", "ad-x"], + "the staleness view should retain refused evidence" + ); + assert_eq!( + table.observed_literals().collect::>(), + ["ad-x-stable"], + "prefix routing should use only concrete live-element evidence" + ); + } + + #[test] + fn later_refusal_removes_a_previously_accepted_literal() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-x", &[(300, 250)])], false), + ); + let mut refused = DiscoveredSlots { + had_slot_evidence: true, + ..DiscoveredSlots::default() + }; + refused.refused_div_ids.insert("ad-x".to_string()); + table.fold_page("/news", &refused); + + assert!( + table.observed_literals().next().is_none(), + "a site-wide refusal should remove an earlier literal-routing candidate" + ); + assert_eq!( + table.observed_div_ids().collect::>(), + BTreeSet::from(["ad-x"]), + "the refused stem should remain available to staleness accounting" + ); + } + + #[test] + fn one_placement_under_per_render_div_ids_is_detected() { + // Each page yields a new key for the same placement: same unit, same + // formats, never co-occurring. The tokens here deliberately do *not* + // match the digit-led shape `discover_gpt_slots` refuses on sight, so + // this exercises the evidence-based detector that catches the stacks + // whose token shape cannot be recognized from one observation. + let mut table = EvidenceTable::default(); + for (path, div) in [ + ("/features/a", "ex_slot_ce6Bj0uc8sL0aa_overlay_1"), + ("/news/b", "ex_slot_aoYmv4RQyN3nbb_overlay_1"), + ("/deals/c", "ex_slot_mYPDB3tz8cpBcc_overlay_1"), + ] { + table.fold_page( + path, + &page(&[("/99/site_Overlay", div, &[(300, 250)])], false), + ); + } + + let groups = table.fragmented_slots(); + + assert_eq!(groups.len(), 1, "the three fragments should form one group"); + assert_eq!(groups[0].div_ids.len(), 3); + assert_eq!(groups[0].unit_path, "/99/site_Overlay"); + assert_eq!( + groups[0].suggested_prefix.as_deref(), + Some("ex_slot"), + "the suggestion should be trimmed back off the volatile token" + ); + } + + #[test] + fn an_ambiguous_stem_stays_refused_on_every_page() { + // The article page carries two in-content units and refuses the shared + // prefix; the landing page carries one. Folding the landing page must + // not resurrect a prefix that cannot resolve to one element site-wide. + let mut table = EvidenceTable::default(); + table.fold_page( + "/news/story", + &page( + &[ + ( + "/123/site/news", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + ( + "/123/site/news", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-1", + &[(300, 250)], + ), + ], + false, + ), + ); + table.fold_page( + "/", + &page( + &[( + "/123/site/home", + "ad-in_content-1c0de08e5a2f4d6f9b3a7e5c8d1f2a4b-in_content-0", + &[(300, 250)], + )], + false, + ), + ); + + assert_eq!( + table.slots().count(), + 0, + "a stem refused on one page must stay refused, got {:?}", + table.slots().map(|slot| &slot.div_id).collect::>() + ); + assert_eq!( + table.slot_count(), + 0, + "the count should match what is written" + ); + assert!( + !table.is_empty(), + "the crawl did observe an ad stack, so this is not an empty result" + ); + assert!( + table.observed_literals().next().is_none(), + "a globally ambiguous stem must not remain a literal-routing candidate" + ); + } + + #[test] + fn genuine_siblings_on_one_unit_are_not_treated_as_fragments() { + // Two real in-content positions can share a unit path and formats. What + // distinguishes them from fragments is that they appear *together* on a + // page, so refusing to write them would lose real inventory. + let mut table = EvidenceTable::default(); + table.fold_page( + "/news/story", + &page( + &[ + ("/99/site/news", "ad-in_content-1", &[(300, 250)]), + ("/99/site/news", "ad-in_content-2", &[(300, 250)]), + ], + false, + ), + ); + + assert!( + table.fragmented_slots().is_empty(), + "co-occurring slots are siblings, not fragments" + ); + } + + #[test] + fn slots_differing_in_formats_are_not_fragments() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "slot-aaaa", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "slot-bbbb", &[(728, 90)])], false), + ); + + assert!( + table.fragmented_slots().is_empty(), + "a differing format set means these are different placements" + ); + } + + #[test] + fn a_slot_seen_alone_is_never_a_fragment() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "only-slot", &[(300, 250)])], false), + ); + + assert!(table.fragmented_slots().is_empty()); + } + + #[test] + fn fragments_with_no_shared_prefix_report_none() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "alpha-1111", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "beta-2222", &[(300, 250)])], false), + ); + + let groups = table.fragmented_slots(); + + assert!( + groups.is_empty(), + "two unrelated placements are too ambiguous to classify as fragments" + ); + } + + #[test] + fn three_disjoint_same_shape_ids_are_fragment_evidence_without_a_prefix() { + let mut table = EvidenceTable::default(); + for (path, div_id) in [("/a", "alpha"), ("/b", "bravo"), ("/c", "charlie")] { + table.fold_page(path, &page(&[("/99/site/x", div_id, &[(300, 250)])], false)); + } + + let groups = table.fragmented_slots(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].suggested_prefix, None); + } + + #[test] + fn unicode_shared_prefix_uses_a_utf8_boundary() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "ünicode-ad-a", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "ünicode-ad-b", &[(300, 250)])], false), + ); + + let groups = table.fragmented_slots(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].suggested_prefix.as_deref(), Some("ünicode-ad")); + } + + #[test] + fn a_later_non_empty_profile_clears_the_empty_page_marker() { + let mut table = EvidenceTable::default(); + table.fold_page("/news", &page(&[], false)); + table.fold_page( + "/news", + &page(&[("/99/site/news", "ad-atf", &[(300, 250)])], false), + ); + + assert!(table.empty_pages().is_empty()); + } + + #[test] + fn a_later_empty_profile_does_not_re_mark_a_non_empty_page() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/news", + &page(&[("/99/site/news", "ad-atf", &[(300, 250)])], false), + ); + table.fold_page("/news", &page(&[], false)); + + assert!( + table.empty_pages().is_empty(), + "emptiness is a page-level fact across all selected profiles" + ); + } + + #[test] + fn conflicting_network_ids_are_a_hard_error() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/111/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/222/site/news", "ad-header", &[(728, 90)])], false), + ); + + let error = table + .network_id() + .expect_err("two networks in one crawl should not resolve"); + + let rendered = format!("{error:?}"); + assert!( + rendered.contains("111") && rendered.contains("222"), + "the error should name both observed ids, got {rendered}" + ); + } + + #[test] + fn agreeing_network_ids_resolve_to_one_value() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], false), + ); + + assert_eq!( + table.network_id().expect("agreeing ids should resolve"), + Some("123".to_string()) + ); + } + + #[test] + fn pages_without_slots_are_recorded_for_challenge_detection() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page("/blocked", &page(&[], false)); + + assert_eq!( + table + .empty_pages() + .iter() + .map(String::as_str) + .collect::>(), + ["/blocked"], + "a slot-less page must be visible to the caller, not silently dropped" + ); + assert_eq!( + table.pages().len(), + 2, + "every folded page should be counted" + ); + } + + #[test] + fn collision_only_page_is_not_classified_as_empty() { + let discovered = page( + &[ + ("/123/site/home", "ad-x-aaaaaaaaaaaaaaaa-0", &[(300, 250)]), + ("/123/site/home", "ad-x-bbbbbbbbbbbbbbbb-1", &[(300, 250)]), + ], + false, + ); + let mut table = EvidenceTable::default(); + + table.fold_page("/collision-only", &discovered); + + assert!(discovered.had_slot_evidence); + assert!(discovered.slots.is_empty()); + assert!( + table.empty_pages().is_empty(), + "intentionally omitted GPT evidence must not look like a bot challenge" + ); + } + + #[test] + fn empty_table_resolves_no_network_id_rather_than_erroring() { + let table = EvidenceTable::default(); + + assert!(table.is_empty()); + assert_eq!(table.network_id().expect("empty is not a conflict"), None); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs new file mode 100644 index 000000000..24d17d3c2 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -0,0 +1,1546 @@ +//! Reconstructs `[creative_opportunities]` slots from a live page's GPT state. +//! +//! Two complementary sources feed the reconstruction: +//! +//! 1. The **live GPT registry** (`googletag.pubads().getSlots()`) is the primary +//! source. It exposes each defined slot's ad-unit path, div id, and sizes +//! directly, and is populated at `defineSlot` time — so it captures slots even +//! when the ad request never fires (consent-gated stacks, iframe-issued +//! requests). It carries no per-slot header-bidding signal, so Prebid is +//! inferred from page-level detection. +//! 2. Captured **`gampad/ads` requests** are a fallback for any div the registry +//! did not report. Each request URL encodes the ad-unit path (`iu_parts`), div +//! id (`dids`), sizes (`prev_iu_szs`), and targeting (`prev_scp`, which does +//! carry a per-slot Prebid signal). +//! +//! Neither source executes the page's ad-stack logic ourselves; both read state +//! the page's own GPT/Prebid setup produced. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::LazyLock; + +use regex::Regex; +use trusted_server_core::creative_opportunities::validate_slot_id; +use url::Url; + +use crate::commands::audit::generate::collector::{CollectedGptSlot, CollectedRequest}; + +/// A hyphen-delimited hex hash *segment* (16+ hex chars bounded by `-` or end), +/// e.g. the UUID GPT embeds in `ad-in_content--in_content-0`. Marks the +/// start of ephemeral div-id noise, like the React `_R_` hash. The trailing +/// boundary avoids truncating a legit token that merely starts with hex-like +/// characters (only `start()` of the match is used). +static HEX_HASH_SEGMENT: LazyLock = + LazyLock::new(|| Regex::new(r"-[0-9a-f]{16,}(?:-|$)").expect("should compile hex hash regex")); +static UUID_SEGMENT: LazyLock = LazyLock::new(|| { + Regex::new(r"-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?:-|$)") + .expect("should compile UUID regex") +}); + +/// Matches a React `useId` token, which changes on every render. +/// +/// React emits these in both cases — `_R_3f_` from a server render and `_r_0_` +/// from a client one — so matching only the uppercase form leaves the lowercase +/// variant in the stem. That is not merely untidy: the suffix differs per +/// render, so one logical slot fragments into a new key on every page, which +/// both breaks runtime div matching and starves template inference of the +/// repeated observations it needs. +/// +/// The uppercase form is distinctive enough to match bare, and its hash is +/// included so the match spans the whole ephemeral token — [`normalize_div_stem`] +/// only reads the match *start*, but [`ephemeral_marker_residue`] excises the +/// match, and a residue that still carried the hash would make two renders of one +/// element look like two elements. The lowercase form is anchored (`_r_`, a short +/// alphanumeric run, `_`) so an ordinary id that merely contains `_r_` keeps its +/// full stem. +static REACT_USE_ID: LazyLock = LazyLock::new(|| { + Regex::new(r"_R_[0-9a-z]*_?|_r_[0-9a-z]{1,8}_").expect("should compile react id regex") +}); + +/// Hosts that serve GPT `gampad/ads` requests. +const GAMPAD_HOSTS: &[&str] = &["securepubads.g.doubleclick.net", "pubads.g.doubleclick.net"]; + +/// Common GPT div-id prefix stripped when deriving a slot id. +const GPT_DIV_PREFIX: &str = "div-gpt-ad-"; + +/// Minimum width/height for a format to be treated as a real creative size. +/// +/// GPT encodes fluid/native aspect-ratio markers (e.g. `4x1`, `8x1`) alongside +/// pixel sizes in `prev_iu_szs`; those are not banner dimensions, so they are +/// dropped from the drafted `formats`. +const MIN_FORMAT_DIMENSION: u32 = 50; + +/// A slot reconstructed from a single GPT ad request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DiscoveredSlot { + /// Slot id derived from the div id (GPT prefix stripped). + pub(crate) id: String, + /// The HTML div id that holds the creative. + pub(crate) div_id: String, + /// The full GAM ad-unit path (e.g. `/123/desktop/homepage/leaderboard`). + pub(crate) gam_unit_path: String, + /// Candidate creative sizes as `(width, height)` pixel pairs. + pub(crate) formats: Vec<(u32, u32)>, + /// Whether the slot's targeting shows Prebid/header-bidding signals. + pub(crate) has_prebid: bool, +} + +/// The result of scanning captured requests for GPT slots. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct DiscoveredSlots { + /// GAM network id shared by the discovered slots, if any were found. + pub(crate) gam_network_id: Option, + /// Whether the page exposed any otherwise usable slot evidence, including + /// ambiguous placements that were intentionally omitted from `slots`. + pub(crate) had_slot_evidence: bool, + /// The reconstructed slots, deduplicated by div id in first-seen order. + pub(crate) slots: Vec, + /// Div stems refused because several live elements normalized onto them. + /// + /// Carried separately from `slots` because the verdict is a property of the + /// *site*, not of this page: another page that happens to render only one + /// member of the group must not resurrect the ambiguous prefix. + pub(crate) ambiguous_stems: BTreeSet, + /// Normalized div IDs refused from generation but still observed live. + pub(crate) refused_div_ids: BTreeSet, + /// Diagnostics for placements whose normalized stable stems collided. + pub(crate) warnings: Vec, +} + +/// Reconstructs GPT slots from the page's live registry and ad requests. +/// +/// The live registry (`googletag.pubads().getSlots()`) is the primary source: it +/// carries the authoritative path/div/size for every defined slot and is present +/// even when the ad request never fires. Captured `gampad/ads` requests are a +/// fallback for any div the registry did not report, and also supply per-slot +/// Prebid signals. Slots are deduplicated by div id in first-seen order. +/// +/// `page_has_prebid` marks registry slots as Prebid-enabled when the page as a +/// whole was detected running Prebid (the registry alone carries no such signal). +pub(crate) fn discover_gpt_slots( + registry: &[CollectedGptSlot], + requests: &[CollectedRequest], + page_has_prebid: bool, +) -> DiscoveredSlots { + let mut slots = Vec::new(); + let mut warnings = Vec::new(); + let mut ambiguous_stems = BTreeSet::new(); + let mut refused_div_ids = BTreeSet::new(); + let mut gam_network_id = None; + let mut had_slot_evidence = false; + let mut registry_residues: BTreeMap> = BTreeMap::new(); + // Stems refused outright, so the request fallback cannot re-add them. Kept + // apart from `registry_residues` so a later registry entry cannot read a + // refused stem as a one-member collision group. + let mut refused_stems: BTreeSet = BTreeSet::new(); + + for entry in registry { + let Some(slot) = slot_from_registry(entry, page_has_prebid) else { + continue; + }; + had_slot_evidence = true; + if gam_network_id.is_none() { + gam_network_id = network_id_from_unit_path(&entry.gam_unit_path); + } + if let Some(prefix) = volatile_prefix_before_placement(&entry.div_id) { + refused_stems.insert(slot.div_id.clone()); + refused_div_ids.insert(slot.div_id); + push_unique_warning(&mut warnings, volatile_prefix_warning(&prefix)); + continue; + } + if let Some(prefix) = + push_slot_refusing_collisions(&mut slots, &mut registry_residues, slot, &entry.div_id) + { + warnings.push(ambiguous_collision_warning(&prefix)); + ambiguous_stems.insert(prefix); + } + } + + let registry_stems: BTreeSet = registry_residues + .keys() + .cloned() + .chain(refused_stems) + .collect(); + let mut request_residues: BTreeMap> = BTreeMap::new(); + for request in requests { + let Some((network_id, slot, raw_div)) = parse_gampad_request(&request.url) else { + continue; + }; + had_slot_evidence = true; + if gam_network_id.is_none() { + gam_network_id = Some(network_id); + } + if registry_stems.contains(&slot.div_id) { + continue; + } + if let Some(prefix) = volatile_prefix_before_placement(&raw_div) { + refused_div_ids.insert(slot.div_id); + push_unique_warning(&mut warnings, volatile_prefix_warning(&prefix)); + continue; + } + if let Some(prefix) = + push_slot_refusing_collisions(&mut slots, &mut request_residues, slot, &raw_div) + { + warnings.push(ambiguous_collision_warning(&prefix)); + ambiguous_stems.insert(prefix); + } + } + make_slot_ids_unique(&mut slots); + + DiscoveredSlots { + gam_network_id, + had_slot_evidence, + slots, + ambiguous_stems, + refused_div_ids, + warnings, + } +} + +/// Adds one source-local slot unless two distinct *elements* share its stem. +/// +/// Sharing a stem is not by itself ambiguity: one element re-rendered under a +/// fresh framework token is exactly what normalization exists to absorb, and it +/// produces two raw ids that collapse onto one stem. Ambiguity is two elements, +/// which [`ephemeral_marker_residue`] separates from two renders of one. +/// +/// The first distinct residue removes the tentatively accepted slot and returns +/// its stem for one diagnostic. Repeats and later collision members stay +/// suppressed and return `None`. +fn push_slot_refusing_collisions( + slots: &mut Vec, + seen_residues: &mut BTreeMap>, + slot: DiscoveredSlot, + raw_div: &str, +) -> Option { + let normalized = slot.div_id.clone(); + let residue = ephemeral_marker_residue(raw_div); + match seen_residues.get_mut(&normalized) { + None => { + seen_residues.insert(normalized, BTreeSet::from([residue])); + slots.push(slot); + None + } + Some(residues) if residues.contains(&residue) => None, + Some(residues) => { + let became_ambiguous = residues.len() == 1; + residues.insert(residue); + if became_ambiguous { + slots.retain(|entry| entry.div_id != normalized); + Some(normalized) + } else { + None + } + } + } +} + +/// Operator-facing text for a stem several live elements normalized onto. +fn ambiguous_collision_warning(prefix: &str) -> String { + format!( + "skipped ambiguous div-id prefix `{prefix}`: multiple active elements normalized to it, \ + but the runtime can resolve a prefix to only one active element and exact div ids change \ + across renders; expose distinct stable div ids in publisher markup before configuring \ + these placements" + ) +} + +/// The stable prefix of a div id whose per-render token precedes more of the id. +/// +/// Some ad stacks build ids as `__` — a +/// millisecond timestamp plus a random suffix sitting *before* the part that +/// distinguishes one placement from the next. Such an id can be written neither +/// literally (the token changes on the next render) nor as a prefix: the only +/// stable prefix stops at the token, and that prefix reaches every placement in +/// the family, while the runtime resolves a prefix to a single element. So the +/// slot is refused from a single observation, without waiting for a second +/// placement to prove the collision. +/// +/// The shape decides, not the vendor: any segment that is a long digit run +/// followed by more alphanumerics counts, so a new stack with the same layout +/// needs no code change. A token in *trailing* position is deliberately not this +/// case — everything before it still identifies the element — and is left to +/// normalization and the same-page collision check. +fn volatile_prefix_before_placement(div_id: &str) -> Option { + let div_id = div_id.strip_suffix("-container").unwrap_or(div_id); + let mut start = 0_usize; + for (index, character) in div_id.char_indices() { + if character != '_' && character != '-' { + continue; + } + if is_per_render_token(&div_id[start..index]) { + let prefix = div_id[..start].trim_end_matches(['_', '-']); + // A delimiter is one byte, so the remainder starts just past it. + return (!prefix.is_empty() && !div_id[index + 1..].is_empty()) + .then(|| prefix.to_string()); + } + start = index + character.len_utf8(); + } + None +} + +/// Whether one div-id segment is a per-render token: a long leading digit run +/// followed by alphanumerics, or a shorter counter paired with a long random +/// suffix. +/// +/// Both halves are required. Eight-digit values need at least eight suffix +/// characters with a random-looking shape; this avoids treating calendar labels +/// followed by stable words as generated ids while still catching single-case +/// hashes and mixed alphanumeric tokens. A bare digit run is how publishers +/// write stable placement indices, and a token with a non-alphanumeric character +/// is some other structure than a generated id. +fn is_per_render_token(segment: &str) -> bool { + let leading_digits = segment.bytes().take_while(u8::is_ascii_digit).count(); + let suffix_length = segment.len().saturating_sub(leading_digits); + let suffix = &segment[leading_digits..]; + ((leading_digits >= 10 && suffix_length >= 1) + || (leading_digits >= 8 && suffix_length >= 8 && looks_random_suffix(suffix))) + && segment.bytes().all(|byte| byte.is_ascii_alphanumeric()) +} + +/// Whether a long suffix has structural signals of generated randomness. +fn looks_random_suffix(value: &str) -> bool { + let digit_count = value.bytes().filter(u8::is_ascii_digit).count(); + let letter_count = value.bytes().filter(u8::is_ascii_alphabetic).count(); + if digit_count >= 4 && letter_count >= 4 { + return true; + } + + let distinct = distinct_ascii_bytes(value); + if value.bytes().all(|byte| byte.is_ascii_hexdigit()) && distinct >= 4 { + return true; + } + if value.bytes().all(|byte| byte.is_ascii_uppercase()) && distinct >= 4 { + return true; + } + + has_random_case_alternation(value) && !has_wordlike_camel_segments(value) +} + +/// Number of distinct ASCII bytes in a candidate token. +fn distinct_ascii_bytes(value: &str) -> usize { + let mut seen = [false; 256]; + for byte in value.bytes() { + seen[usize::from(byte)] = true; + } + seen.into_iter().filter(|present| *present).count() +} + +/// Whether every CamelCase component contains a vowel-like letter. +/// +/// This distinguishes short word sequences such as `TopUsNewsAd` and +/// `MyAdUnitXy` from dense random alternation such as `AbCdEfGh`. +fn has_wordlike_camel_segments(value: &str) -> bool { + let mut segment_has_vowel = false; + for (index, byte) in value.bytes().enumerate() { + if index > 0 && byte.is_ascii_uppercase() { + if !segment_has_vowel { + return false; + } + segment_has_vowel = is_ascii_vowel(byte); + } else { + segment_has_vowel |= is_ascii_vowel(byte); + } + } + segment_has_vowel +} + +/// Whether an ASCII letter is a vowel, treating `y` as vowel-like for labels. +const fn is_ascii_vowel(byte: u8) -> bool { + matches!( + byte.to_ascii_lowercase(), + b'a' | b'e' | b'i' | b'o' | b'u' | b'y' + ) +} + +/// Whether letter case alternates densely enough to resemble a random token. +fn has_random_case_alternation(value: &str) -> bool { + let mut previous = None; + let mut comparisons = 0_usize; + let mut transitions = 0_usize; + for uppercase in value.bytes().filter_map(|byte| { + byte.is_ascii_lowercase() + .then_some(false) + .or_else(|| byte.is_ascii_uppercase().then_some(true)) + }) { + if let Some(previous) = previous { + comparisons += 1; + transitions += usize::from(previous != uppercase); + } + previous = Some(uppercase); + } + transitions >= 3 && transitions.saturating_mul(3) >= comparisons.saturating_mul(2) +} + +/// Operator-facing text for a div-id family carrying a per-render token. +fn volatile_prefix_warning(prefix: &str) -> String { + format!( + "skipped volatile div-id family `{prefix}`: a per-render token sits before the placement \ + suffix, so exact div ids change across renders and no distinct stable element prefix is \ + available; expose distinct stable div ids in publisher markup before configuring these \ + placements" + ) +} + +/// Records `warning` unless the same text was already recorded for this page. +fn push_unique_warning(warnings: &mut Vec, warning: String) { + if !warnings.contains(&warning) { + warnings.push(warning); + } +} + +/// Converts a live-registry slot into a [`DiscoveredSlot`]. +/// +/// Returns `None` when the slot has no usable pixel size or its div id is a +/// multi-slot (SRA) concatenation rather than a single element. +fn slot_from_registry(entry: &CollectedGptSlot, page_has_prebid: bool) -> Option { + if is_multi_slot_div(&entry.div_id) { + return None; + } + if !is_usable_unit_path(&entry.gam_unit_path) { + return None; + } + let formats: Vec<(u32, u32)> = entry + .sizes + .iter() + .copied() + .filter(|(width, height)| *width >= MIN_FORMAT_DIMENSION && *height >= MIN_FORMAT_DIMENSION) + .collect(); + if formats.is_empty() { + return None; + } + let div_stem = normalize_div_stem(&entry.div_id); + // Normalization truncates at the first ephemeral marker, so a div id that is + // *entirely* ephemeral (`_R_9sl…`, or exactly `-container`) reduces to the + // empty string. An empty `div_id` override fails config load outright, and + // an empty prefix would bind the slot to the first id-bearing element on the + // page, so such a slot is unusable rather than merely imprecise. + if div_stem.is_empty() { + return None; + } + Some(DiscoveredSlot { + id: slot_id_from_div(&div_stem), + div_id: div_stem, + gam_unit_path: entry.gam_unit_path.clone(), + formats, + has_prebid: page_has_prebid, + }) +} + +/// Whether a div id is a GPT single-request (SRA) concatenation of multiple +/// slots (joined with `~`) rather than one element. +fn is_multi_slot_div(div_id: &str) -> bool { + div_id.contains('~') +} + +/// Whether a scraped GAM ad-unit path can be represented in config. +/// +/// `gam_unit_path` is a template: `{` and `}` delimit placeholders and +/// [`parse_unit_template`](trusted_server_core::creative_opportunities) offers no +/// escape syntax. A live path containing a brace would either fail config load +/// or, worse, be silently reinterpreted as a placeholder-bearing template. A +/// blank path is rejected for the same reason config load rejects it. +fn is_usable_unit_path(path: &str) -> bool { + !path.trim().is_empty() && !path.contains(['{', '}']) +} + +/// Strips ephemeral GPT div-id noise so the stored id is stable across renders. +/// +/// Removes a trailing `-container` wrapper, then truncates at the first ephemeral +/// marker — a React SSR hash (`_R_`) or a hex-UUID segment — since both +/// change on every page load. Truncating (rather than excising) keeps the result +/// a valid **prefix** of the live div id, which is how verify matches slots. +/// +/// `div-gpt-ad-leaderboard-1` (stable) is unchanged; `ad-header-0-_R_9sl…-container` +/// and `ad-header-0-_r_8_` → `ad-header-0`; `ad-in_content-de66…f272-in_content-0` +/// → `ad-in_content`. +fn normalize_div_stem(div_id: &str) -> String { + let stem = div_id.strip_suffix("-container").unwrap_or(div_id); + let cut = ephemeral_marker_ranges(stem) + .first() + .map_or(stem.len(), |range| range.start); + stem[..cut].trim_end_matches('-').to_string() +} + +/// Byte ranges of every ephemeral per-render marker in `stem`, in order and +/// without overlaps. +/// +/// A hex-hash candidate must contain at least one `a`-`f`; a run of 16+ digits +/// is how publishers write stable ids, not a hash. +fn ephemeral_marker_ranges(stem: &str) -> Vec> { + let mut ranges: Vec> = REACT_USE_ID + .find_iter(stem) + .chain(UUID_SEGMENT.find_iter(stem)) + .chain(HEX_HASH_SEGMENT.find_iter(stem).filter(|matched| { + matched + .as_str() + .bytes() + .any(|byte| matches!(byte, b'a'..=b'f')) + })) + .map(|matched| matched.range()) + .collect(); + ranges.sort_by_key(|range| range.start); + let mut merged: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + match merged.last_mut() { + Some(last) if range.start < last.end => last.end = last.end.max(range.end), + _ => merged.push(range), + } + } + merged +} + +/// The parts of a raw div id that no ephemeral marker covered, NUL-joined. +/// +/// [`normalize_div_stem`] truncates at the first marker, so two ids differing +/// only *inside* a marker collapse onto one stem — the signature of one element +/// re-rendered. What the markers did not cover separates that from two elements: +/// `ad-header-0-_R_3f_` and `ad-header-0-_r_0_` leave the same residue (one +/// element, two renders), while `…-in_content-0` and `…-in_content-1` do not +/// (two siblings). A live div id cannot contain NUL, so joining on it cannot +/// make two different residues compare equal. +fn ephemeral_marker_residue(div_id: &str) -> String { + let stem = div_id.strip_suffix("-container").unwrap_or(div_id); + let mut residue = String::with_capacity(stem.len()); + let mut previous = 0_usize; + for range in ephemeral_marker_ranges(stem) { + residue.push_str(&stem[previous..range.start]); + residue.push('\0'); + previous = range.end; + } + residue.push_str(&stem[previous..]); + residue +} + +/// Extracts the leading network id from a GAM ad-unit path (`//...`). +fn network_id_from_unit_path(path: &str) -> Option { + let segment = path.trim_start_matches('/').split('/').next()?; + (!segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| segment.to_string()) +} + +/// Parses a single `gampad/ads` request URL into `(network_id, slot)`. +/// +/// Returns `None` when the URL is not a GPT ad request or is missing the fields +/// needed to describe a slot (ad-unit path, div id, and at least one size). +fn parse_gampad_request(raw_url: &str) -> Option<(String, DiscoveredSlot, String)> { + let url = Url::parse(raw_url).ok()?; + let host = url.host_str()?; + if !GAMPAD_HOSTS.contains(&host) || !url.path().ends_with("/gampad/ads") { + return None; + } + + let mut iu_parts = None; + let mut dids = None; + let mut sizes_raw = None; + let mut fallback_sizes_raw = None; + let mut scp = None; + for (key, value) in url.query_pairs() { + match key.as_ref() { + "iu_parts" => iu_parts = Some(value.into_owned()), + "dids" => dids = Some(value.into_owned()), + "prev_iu_szs" => sizes_raw = Some(value.into_owned()), + "pb_szs" => fallback_sizes_raw = Some(value.into_owned()), + "prev_scp" => scp = Some(value.into_owned()), + _ => {} + } + } + + let iu_parts = iu_parts?; + let mut parts = iu_parts.split(',').filter(|part| !part.is_empty()); + // Mirror the registry path's validation: a GAM network id is digits only. + // The percent-decoded query value is page-controlled and gets spliced into + // generated TOML, so reject anything else. + let network_id = parts + .next() + .filter(|segment| segment.bytes().all(|byte| byte.is_ascii_digit()))? + .to_string(); + let gam_unit_path = format!("/{}", iu_parts.replace(',', "/")); + if !is_usable_unit_path(&gam_unit_path) { + return None; + } + // A usable unit path needs the network id plus at least one path segment. + parts.next()?; + + let raw_div = dids?; + if raw_div.contains(',') { + return None; + } + let raw_div = raw_div.trim().to_string(); + if raw_div.is_empty() { + return None; + } + if is_multi_slot_div(&raw_div) { + return None; + } + let div_id = normalize_div_stem(&raw_div); + // See `slot_from_registry`: a fully ephemeral div id normalizes to nothing, + // which is neither a valid config value nor a usable runtime prefix. + if div_id.is_empty() { + return None; + } + + let formats = parse_sizes(sizes_raw.as_deref().or(fallback_sizes_raw.as_deref())?); + if formats.is_empty() { + return None; + } + + let id = slot_id_from_div(&div_id); + let has_prebid = scp.as_deref().is_some_and(scp_shows_prebid); + + Some(( + network_id, + DiscoveredSlot { + id, + div_id, + gam_unit_path, + formats, + has_prebid, + }, + raw_div, + )) +} + +/// Parses a GPT size list (e.g. `970x250|4x1|620x366`) into pixel pairs. +/// +/// Accepts `|` or `,` separators, ignores non-`WxH` tokens, and drops +/// fluid/native ratio markers below [`MIN_FORMAT_DIMENSION`]. +fn parse_sizes(raw: &str) -> Vec<(u32, u32)> { + let mut sizes = Vec::new(); + for token in raw.split(['|', ',']) { + let Some((width, height)) = token.trim().split_once('x') else { + continue; + }; + let (Ok(width), Ok(height)) = (width.parse::(), height.parse::()) else { + continue; + }; + if width < MIN_FORMAT_DIMENSION || height < MIN_FORMAT_DIMENSION { + continue; + } + if !sizes.contains(&(width, height)) { + sizes.push((width, height)); + } + } + sizes +} + +/// Derives a runtime-safe slot id from a div id. +/// +/// The common GPT prefix is stripped, invalid character runs become one +/// hyphen, and an all-invalid value falls back to `slot`. +fn slot_id_from_div(div_id: &str) -> String { + let candidate = div_id.strip_prefix(GPT_DIV_PREFIX).unwrap_or(div_id); + let mut id = String::with_capacity(candidate.len()); + let mut previous_was_hyphen = false; + for character in candidate.chars() { + if character.is_ascii_alphanumeric() || character == '_' { + id.push(character); + previous_was_hyphen = false; + } else if !id.is_empty() && !previous_was_hyphen { + id.push('-'); + previous_was_hyphen = true; + } + } + while id.ends_with('-') { + id.pop(); + } + if id.is_empty() { + id.push_str("slot"); + } + + if validate_slot_id(&id).is_ok() { + id + } else { + "slot".to_string() + } +} + +/// Adds deterministic numeric suffixes when sanitization produces duplicate ids. +fn make_slot_ids_unique(slots: &mut [DiscoveredSlot]) { + let mut used = BTreeSet::new(); + for slot in slots { + if used.insert(slot.id.clone()) { + continue; + } + + let base = slot.id.clone(); + let mut suffix = 2_usize; + loop { + let candidate = format!("{base}-{suffix}"); + if used.insert(candidate.clone()) { + slot.id = candidate; + break; + } + suffix += 1; + } + } +} + +/// Detects Prebid/header-bidding signals in a slot's `prev_scp` targeting. +fn scp_shows_prebid(scp: &str) -> bool { + url::form_urlencoded::parse(scp.as_bytes()).any(|(key, value)| { + let key = key.to_ascii_lowercase(); + let value = value.to_ascii_lowercase(); + (key == "test" && value == "prebid") + || (key == "tude" && value == "true") + || key.starts_with("prebid") + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A sample GPT leaderboard ad request (truncated to the fields the + /// parser reads; values are otherwise unmodified live output). + const SAMPLE_LEADERBOARD: &str = "https://securepubads.g.doubleclick.net/gampad/ads?\ + gdfp_req=1&iu_parts=123456789%2Cdesktop%2Chomepage%2Cleaderboard1\ + &prev_iu_szs=970x250%7C4x1%7C8x1%7C620x366%7C325x508%7C325x204\ + &dids=div-gpt-ad-leaderboard-1\ + &prev_scp=ad-loc%3Dleaderboard-1%26baseDivId%3Ddiv-gpt-ad-leaderboard-1%26test%3Dprebid%26tude%3Dtrue\ + &pb_szs=970x250%7C620x366"; + const SHORT_VOLATILE_DIV: &str = "vendor-tag_12345678AbCdEfGhIjKl_slot_overlay_1"; + + fn request(url: &str) -> CollectedRequest { + CollectedRequest { + url: url.to_string(), + resource_type: Some("fetch".to_string()), + } + } + + /// Discovers slots from ad requests only (no live registry). + fn from_requests(requests: &[CollectedRequest]) -> DiscoveredSlots { + discover_gpt_slots(&[], requests, false) + } + + #[test] + fn parses_leaderboard_slot() { + let discovered = from_requests(&[request(SAMPLE_LEADERBOARD)]); + + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert_eq!(discovered.slots.len(), 1, "should find one slot"); + let slot = &discovered.slots[0]; + assert_eq!(slot.id, "leaderboard-1", "should strip the GPT div prefix"); + assert_eq!(slot.div_id, "div-gpt-ad-leaderboard-1"); + assert_eq!( + slot.gam_unit_path, + "/123456789/desktop/homepage/leaderboard1" + ); + assert_eq!( + slot.formats, + vec![(970, 250), (620, 366), (325, 508), (325, 204)], + "should keep pixel sizes and drop 4x1/8x1 fluid markers" + ); + assert!(slot.has_prebid, "prev_scp test=prebid should flag prebid"); + } + + #[test] + fn prebid_detection_requires_a_targeting_key_not_a_substring() { + assert!(scp_shows_prebid("test=prebid")); + assert!(!scp_shows_prebid("noprebid=true")); + } + + #[test] + fn deduplicates_refreshed_slot_requests() { + // GPT refreshes the same slot; a second identical request must not + // produce a duplicate slot. + let discovered = from_requests(&[request(SAMPLE_LEADERBOARD), request(SAMPLE_LEADERBOARD)]); + + assert_eq!( + discovered.slots.len(), + 1, + "repeat requests for the same div should collapse" + ); + } + + #[test] + fn ignores_non_gampad_requests() { + let discovered = from_requests(&[ + request("https://securepubads.g.doubleclick.net/tag/js/gpt.js"), + request("https://cdn.example.com/app.js"), + request("https://analytics.example.com/collect?iu_parts=1%2Cfoo&dids=x"), + ]); + + assert!( + discovered.slots.is_empty(), + "only doubleclick gampad/ads requests should yield slots" + ); + assert_eq!(discovered.gam_network_id, None); + } + + #[test] + fn skips_requests_missing_sizes() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cslot&dids=div-gpt-ad-x", + )]); + + assert!( + discovered.slots.is_empty(), + "a slot with no usable size should be skipped" + ); + } + + #[test] + fn skips_requests_with_only_network_id() { + // iu_parts with just the network id yields no unit path segment. + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123&dids=div-gpt-ad-x&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a bare network id is not a usable ad-unit path" + ); + } + + #[test] + fn skips_requests_with_non_numeric_network_id() { + // A page-controlled iu_parts value must not smuggle a non-numeric + // network id (it gets spliced into generated TOML). + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%22evil%2Cslot&dids=div-gpt-ad-x&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a non-numeric network id should be rejected" + ); + assert_eq!(discovered.gam_network_id, None); + } + + #[test] + fn falls_back_to_pb_szs_when_prev_iu_szs_absent() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cslot&dids=div-gpt-ad-x&pb_szs=300x250%7C728x90", + )]); + + assert_eq!(discovered.slots.len(), 1); + assert_eq!(discovered.slots[0].formats, vec![(300, 250), (728, 90)]); + } + + fn registry_slot(path: &str, div: &str, sizes: &[(u32, u32)]) -> CollectedGptSlot { + CollectedGptSlot { + gam_unit_path: path.to_string(), + div_id: div.to_string(), + sizes: sizes.to_vec(), + } + } + + #[test] + fn lowercase_react_use_id_suffixes_collapse_to_one_slot() { + // React emits `_r_0_` client-side and `_R_3f_` server-side, and the + // token changes per render. Leaving it in the stem fragments one slot + // into a new key on every page, which starves template inference. + for volatile in [ + "ad-header-0-_r_0_", + "ad-header-0-_r_8_", + "ad-header-0-_r_a_", + "ad-header-0-_R_3f_", + ] { + let registry = vec![registry_slot("/123/site/news", volatile, &[(728, 90)])]; + let discovered = discover_gpt_slots(®istry, &[], false); + assert_eq!( + discovered.slots[0].div_id, "ad-header-0", + "`{volatile}` should normalize to a stable stem" + ); + } + } + + #[test] + fn an_ordinary_id_containing_r_is_left_alone() { + // The React shape is anchored, so a legitimate id keeps its full stem. + let registry = vec![registry_slot("/123/site/news", "ad_r_rail", &[(300, 250)])]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].div_id, "ad_r_rail"); + } + + #[test] + fn registry_slot_with_brace_in_unit_path_is_skipped() { + // `gam_unit_path` is a template and there is no escape syntax, so a + // literal brace either fails config load or is silently reinterpreted as + // a placeholder. Neither is acceptable to persist. + let registry = vec![ + registry_slot("/123/home/{section}", "div-gpt-ad-a", &[(300, 250)]), + registry_slot("/123/home/ok", "div-gpt-ad-b", &[(300, 250)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "the brace-bearing slot should be dropped, the clean one kept" + ); + assert_eq!(discovered.slots[0].gam_unit_path, "/123/home/ok"); + } + + #[test] + fn registry_slot_whose_div_id_is_entirely_ephemeral_is_skipped() { + // `_R_…` is a React SSR marker; normalizing truncates at it, leaving an + // empty stem. An empty div_id fails config load, and as a runtime prefix + // it would match the first id-bearing element on the page. + let registry = vec![registry_slot( + "/123/home/header", + "_R_9slkta7pd6", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "a slot with no stable div stem should be dropped, got {:?}", + discovered.slots + ); + } + + #[test] + fn volatile_guid_div_id_still_normalizes_to_a_usable_prefix() { + // A GUID between two copies of the placement name must still yield a + // usable stable stem; only an entirely ephemeral id is dropped. + let registry = vec![registry_slot( + "/123456789/publisher/homepage", + "ad-in_content-0949b6c5726343bf8bbec2ac47b494b4-in_content-0", + &[(300, 250)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots.len(), 1); + assert_eq!( + discovered.slots[0].div_id, "ad-in_content", + "the GUID and trailing index should be truncated to a stable prefix" + ); + } + + #[test] + fn reads_slots_from_live_registry() { + let registry = vec![registry_slot( + "/123456789/desktop/homepage/leaderboard1", + "div-gpt-ad-leaderboard-1", + &[(970, 250), (1, 1), (620, 366)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], true); + + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "network id should come from the unit path" + ); + assert_eq!(discovered.slots.len(), 1); + let slot = &discovered.slots[0]; + assert_eq!(slot.id, "leaderboard-1"); + assert_eq!( + slot.formats, + vec![(970, 250), (620, 366)], + "should drop the 1x1 out-of-page marker" + ); + assert!( + slot.has_prebid, + "page-level prebid should mark registry slots" + ); + } + + #[test] + fn registry_wins_and_requests_fill_gaps() { + // The registry reports the leaderboard; a gampad request reports a + // different div that the registry missed. Both should appear once. + let registry = vec![registry_slot( + "/123456789/desktop/homepage/leaderboard1", + "div-gpt-ad-leaderboard-1", + &[(970, 250)], + )]; + let requests = vec![ + // Same div as the registry — must not duplicate. + request(SAMPLE_LEADERBOARD), + // A div the registry did not report — must be added. + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Cdesktop%2Chomepage%2Csidebar1&dids=div-gpt-ad-sidebar-1&prev_iu_szs=300x600", + ), + ]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + let ids: Vec<&str> = discovered + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect(); + assert_eq!( + ids, + vec!["leaderboard-1", "sidebar-1"], + "registry slot kept, request fills the missing div, no duplicate" + ); + } + + #[test] + fn registry_slot_without_pixel_sizes_is_skipped() { + let registry = vec![registry_slot("/123/fluid", "div-gpt-ad-fluid", &[(1, 1)])]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "a registry slot with only fluid markers is not usable" + ); + } + + #[test] + fn normalizes_ephemeral_hash_and_container_and_dedups() { + // A framework-hashed div: the same placement appears as a hashed inner div, + // a `-container` wrapper, and re-rendered with a different hash. All must + // collapse to one stable stem. + let registry = vec![ + registry_slot( + "/987654321/homepage/header-0", + "ad-header-0-_R_9slinpflik6lb_", + &[(728, 90)], + ), + registry_slot( + "/987654321/homepage/header-0", + "ad-header-0-_R_9slinpflik6lb_-container", + &[(728, 90)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "hash + container variants collapse" + ); + assert_eq!( + discovered.slots[0].div_id, "ad-header-0", + "ephemeral React hash and -container are stripped to a stable stem" + ); + assert_eq!(discovered.slots[0].id, "ad-header-0"); + } + + #[test] + fn drops_sra_multi_slot_concatenations() { + let registry = vec![registry_slot( + "/987654321/homepage/header-0/fixed_bottom-0", + "ad-header-0-_R_9slin~ad-fixed_bottom-0-_R_ainp", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "tilde-joined SRA multi-slot divs are not real single elements" + ); + } + + #[test] + fn leaves_clean_div_ids_unchanged() { + assert_eq!( + normalize_div_stem("div-gpt-ad-leaderboard-1"), + "div-gpt-ad-leaderboard-1" + ); + } + + #[test] + fn sanitizes_page_controlled_div_ids_for_runtime_slot_ids() { + let registry = vec![registry_slot( + "/123456789/homepage/header", + "div-gpt-ad-header.main: 1", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].id, "header-main-1"); + assert_eq!( + discovered.slots[0].div_id, "div-gpt-ad-header.main: 1", + "matching should retain the original normalized div stem" + ); + trusted_server_core::creative_opportunities::validate_slot_id(&discovered.slots[0].id) + .expect("generated id should pass runtime validation"); + } + + #[test] + fn uses_fallback_for_div_id_without_safe_slot_id_characters() { + let registry = vec![registry_slot( + "/123456789/homepage/fallback", + "div-gpt-ad-...", + &[(300, 250)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].id, "slot"); + } + + #[test] + fn makes_colliding_sanitized_slot_ids_unique() { + let registry = vec![ + registry_slot( + "/123456789/homepage/dotted", + "div-gpt-ad-header.main", + &[(728, 90)], + ), + registry_slot( + "/123456789/homepage/colon", + "div-gpt-ad-header:main", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + let ids = discovered + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(); + + assert_eq!(ids, ["header-main", "header-main-2"]); + } + + #[test] + fn normalizes_react_and_hex_hashes_to_stable_prefixes() { + assert_eq!( + normalize_div_stem("ad-header-0-_R_9slinpflik6lb_-container"), + "ad-header-0" + ); + let stem = + normalize_div_stem("ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0"); + assert_eq!(stem, "ad-in_content"); + assert!( + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0".starts_with(&stem), + "stem must prefix-match any re-rendered hex variant" + ); + } + + #[test] + fn hex_hash_truncation_requires_a_segment_boundary() { + // Hex UUID bounded by `-` → truncated to the stem. + assert_eq!( + normalize_div_stem("ad-x-de669245b2ea4b05826dc96f07a36272-y"), + "ad-x" + ); + // A token that merely starts with 16 hex chars (no boundary) is left intact. + assert_eq!( + normalize_div_stem("ad-de669245b2ea4b05z"), + "ad-de669245b2ea4b05z" + ); + } + + #[test] + fn long_numeric_segments_are_stable_ids_not_hex_hashes() { + assert_eq!( + normalize_div_stem("ad-slot-1234567890123456-tail"), + "ad-slot-1234567890123456-tail" + ); + } + + #[test] + fn comma_separated_sra_dids_are_ignored() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cnews%2Catf&dids=ad-a%2Cad-b&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a comma-joined SRA did list is not one element" + ); + } + + #[test] + fn one_element_under_two_render_tokens_is_not_a_collision() { + // Both ids describe in-content placement 0; only the hash between the + // two copies of the placement name differs, which is what one element + // re-rendered looks like. Refusing here would refuse the very shape + // normalization exists to absorb. + let registry = vec![ + registry_slot( + "/987654321/site/homepage", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + registry_slot( + "/987654321/site/homepage", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-0", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "two renders of one element are one slot, got {:?}", + discovered.slots + ); + assert_eq!(discovered.slots[0].div_id, "ad-in_content"); + assert!( + discovered.warnings.is_empty(), + "a re-render is not an ambiguity to report, got {:?}", + discovered.warnings + ); + assert!(discovered.ambiguous_stems.is_empty()); + } + + #[test] + fn sibling_placements_sharing_one_stem_are_refused() { + // Same shape as above, but the trailing placement index differs: these + // are two live elements, and one prefix cannot resolve to both. + let registry = vec![ + registry_slot( + "/987654321/site/homepage", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + registry_slot( + "/987654321/site/homepage", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-1", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "neither a broad prefix nor per-render exact IDs are safe" + ); + assert_ambiguous_collision_warning(&discovered, "ad-in_content"); + assert!( + discovered.ambiguous_stems.contains("ad-in_content"), + "the verdict must travel with the evidence, got {:?}", + discovered.ambiguous_stems + ); + } + + #[test] + fn react_server_and_client_render_tokens_are_one_slot() { + // A hydrating publisher reports the SSR id and the client id for the + // same element. Both must collapse rather than refuse each other. + let registry = vec![ + registry_slot("/123456789/site/news", "ad-header-0-_R_3f_", &[(728, 90)]), + registry_slot("/123456789/site/news", "ad-header-0-_r_0_", &[(728, 90)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "SSR and client renders of one element are one slot, got {:?}", + discovered.slots + ); + assert_eq!(discovered.slots[0].div_id, "ad-header-0"); + assert!(discovered.warnings.is_empty()); + } + + #[test] + fn repeated_raw_div_after_a_normalization_collision_is_deduplicated() { + let first = "ad-x-aaaaaaaaaaaaaaaa-0"; + let second = "ad-x-bbbbbbbbbbbbbbbb-1"; + let third = "ad-x-cccccccccccccccc-2"; + let registry = vec![ + registry_slot("/123456789/site/home", first, &[(300, 250)]), + registry_slot("/123456789/site/home", second, &[(300, 250)]), + registry_slot("/123456789/site/home", first, &[(300, 250)]), + registry_slot("/123456789/site/home", second, &[(300, 250)]), + registry_slot("/123456789/site/home", third, &[(300, 250)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "no repeat or later collision member may resurrect the group" + ); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn request_normalization_collision_is_refused() { + let discovered = from_requests(&[ + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-aaaaaaaaaaaaaaaa-0&prev_iu_szs=300x250", + ), + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-bbbbbbbbbbbbbbbb-1&prev_iu_szs=300x250", + ), + ]); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "a refused placement must not be written, got {:?}", + discovered.slots + ); + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "refusing a slot must not discard the network id" + ); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn single_volatile_family_registry_slot_is_refused() { + let discovered = discover_gpt_slots( + &[registry_slot( + "/123456789/site_in-article_desktop_1", + "vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1", + &[(300, 250)], + )], + &[], + false, + ); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "one observation of a per-render family must not be written literally" + ); + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert!( + discovered + .refused_div_ids + .contains("vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1"), + "registry refusal should retain its normalized div as observed evidence" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn single_volatile_family_request_slot_is_refused() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite_in-article_desktop_1&dids=vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1&prev_iu_szs=300x250", + )]); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "a refused placement must not be written, got {:?}", + discovered.slots + ); + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "refusing a slot must not discard the network id" + ); + assert!( + discovered + .refused_div_ids + .contains("vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1"), + "request refusal should retain its normalized div as observed evidence" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn shorter_high_entropy_singleton_registry_slot_is_refused() { + let discovered = discover_gpt_slots( + &[registry_slot( + "/123456789/publisher.example_overlay_mobile", + SHORT_VOLATILE_DIV, + &[(300, 250)], + )], + &[], + false, + ); + + assert!(discovered.had_slot_evidence); + assert!( + discovered.slots.is_empty(), + "a singleton per-render ID must not be written literally" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn shorter_high_entropy_singleton_request_slot_is_refused() { + let discovered = from_requests(&[request(&format!( + "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Cpublisher.example_overlay_mobile\ + &dids={SHORT_VOLATILE_DIV}&prev_iu_szs=300x250" + ))]); + + assert!(discovered.had_slot_evidence); + assert!( + discovered.slots.is_empty(), + "request fallback must not write a singleton per-render ID" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn volatile_prefix_covers_every_placement_after_the_token() { + // The token's position is what makes the id unusable, so the placement + // that follows it is irrelevant: every one of these leaves `vendor-tag` + // as the only stable prefix, and that prefix reaches all of them. + for volatile in [ + "vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1", + "vendor-tag_12345678AbCdEfGh_slot_inarticle_1", + "vendor-tag_20260820AbCdEfGh_slot_inarticle_1", + "vendor-tag_20260820deadbeef_slot_inarticle_1", + "vendor-tag_20260820ABCDEFGH_slot_inarticle_1", + "vendor-tag_20260820ABCD1234_slot_inarticle_1", + "vendor-tag_20260820A1B2C3D4_slot_inarticle_1", + "vendor-tag_1724112345678AbCdEfGh_slot_overlay_1-container", + "vendor-tag_1724112345678AbCdEfGh_slot_sidebar_1", + "vendor-tag_1724112345678AbCdEfGh_slot_overlay_stable", + "vendor-tag_1724112345678AbCdEfGh_slot_overlay_1_extra", + ] { + assert_eq!( + volatile_prefix_before_placement(volatile).as_deref(), + Some("vendor-tag"), + "`{volatile}` should be refused as a volatile family" + ); + } + } + + #[test] + fn volatile_prefix_does_not_claim_stable_div_ids() { + for stable in [ + // No per-render token at all. + "vendor-tag_stable_slot_inarticle_1", + // A bare digit run is how stable placement indices are written. + "vendor-tag_12345678_slot_inarticle_1", + "ad-slot-1234567890123456-tail", + // Shorter counter/suffix combinations do not carry enough entropy. + "vendor-tag_1234567AbCdEfGh_slot_inarticle_1", + "vendor-tag_12345678AbCdEfG_slot_inarticle_1", + // An eight-digit calendar date plus a stable suffix is not a + // timestamp-like per-render token. + "promo-20260820a-sidebar", + "promo-20260820Football-sidebar", + "promo-20260820football-sidebar", + "promo-20260820TopStories-sidebar", + "promo-20260820TopUsNewsAd-sidebar", + "promo-20260820MyAdUnitXy-sidebar", + "promo-20260820Top10Stories-sidebar", + "ad-19700101Thumbnail-rail", + "ad-00000001AAAAAAAA-rail", + // The token is trailing, so the prefix before it still identifies + // this element and normalization/collision handling own the case. + "vendor-tag_slot_inarticle_1724112345678AbCdEfGh", + "vendor-tag-header", + ] { + assert_eq!( + volatile_prefix_before_placement(stable), + None, + "`{stable}` should stay eligible" + ); + } + } + + #[test] + fn ambiguous_registry_stem_still_suppresses_request_fallback() { + let registry = vec![ + registry_slot( + "/123456789/site/home", + "ad-x-aaaaaaaaaaaaaaaa-0", + &[(300, 250)], + ), + registry_slot( + "/123456789/site/home", + "ad-x-bbbbbbbbbbbbbbbb-1", + &[(300, 250)], + ), + ]; + let requests = vec![request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-cccccccccccccccc-2&prev_iu_szs=300x250", + )]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "request fallback must not resurrect an ambiguous registry stem" + ); + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn request_rerender_does_not_rewrite_a_stable_registry_slot() { + let registry = vec![registry_slot( + "/123456789/site/home", + "ad-x-aaaaaaaaaaaaaaaa-0", + &[(300, 250)], + )]; + let requests = vec![request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-bbbbbbbbbbbbbbbb-1&prev_iu_szs=300x250", + )]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + assert_eq!(discovered.slots.len(), 1, "registry evidence should win"); + assert_eq!( + discovered.slots[0].div_id, "ad-x", + "request fallback must not destabilize a registry-derived prefix" + ); + } + + fn assert_ambiguous_collision_warning(discovered: &DiscoveredSlots, prefix: &str) { + assert_eq!(discovered.warnings.len(), 1); + let warning = &discovered.warnings[0]; + assert!(warning.contains(prefix), "warning should name the prefix"); + assert!( + warning.contains("one active element"), + "warning should explain why the broad prefix is unsafe" + ); + assert!( + warning.contains("change across renders"), + "warning should explain why raw IDs are unsafe" + ); + assert!( + warning.contains("distinct stable div ids"), + "warning should tell the operator how to make the placements configurable" + ); + } + + fn assert_volatile_prefix_warning(discovered: &DiscoveredSlots, prefix: &str) { + assert_eq!( + discovered.warnings.len(), + 1, + "should report the family once, got {:?}", + discovered.warnings + ); + let warning = &discovered.warnings[0]; + assert!( + warning.contains(prefix), + "warning should name the family prefix, got {warning}" + ); + assert!( + warning.contains("change across renders"), + "warning should explain why the exact ids are unsafe, got {warning}" + ); + assert!( + warning.contains("distinct stable div ids"), + "warning should tell the operator how to make the placements configurable, got {warning}" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs new file mode 100644 index 000000000..953b0f034 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -0,0 +1,4207 @@ +mod analyzer; +pub(crate) mod browser_collector; +pub(crate) mod collector; +mod crawl_plan; +mod evidence; +mod gpt_slots; +mod page_patterns; +mod slot_toml; +mod unit_template; +mod validate; + +use std::collections::BTreeSet; +use std::fmt::Write as _; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use rand::RngCore as _; +use serde::Serialize; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, validate_page_pattern, +}; +use url::Url; + +use crate::commands::audit::ad_templates::{origin_changed, without_fragment}; +use crate::commands::audit::collector::GenerateBrowserOpts; +use crate::commands::audit::generate::collector::AuditCollector; +use crate::commands::audit::generate::slot_toml::{ + render_slots, replace_key_in_section, resolve_network_id, splice_creative_slots, toml_string, +}; +use crate::commands::config::init::EXAMPLE_CONFIG; +use crate::error::{CliResult, cli_error, report_error}; + +use analyzer::{analyze_collected_page, extract_gtm_container_id}; + +pub(crate) use browser_collector::DeviceProfile; +pub(crate) use crawl_plan::CrawlBudget; + +/// Writes `contents` to `path` atomically: a same-directory temp file is +/// written and fsynced, then renamed over the target, then the directory entry +/// is fsynced. +/// +/// A plain `fs::write` truncates the destination before writing, so a full disk +/// or an interrupted run would leave an operator's `trusted-server.toml` empty +/// or half-written. `rename` within a directory is atomic, so a reader sees +/// either the old file or the complete new one. +/// +/// The target's existing permissions are carried onto the replacement, since +/// the temp file is created 0600 and the config may intentionally be broader. +/// +/// # Errors +/// +/// Returns the underlying I/O error when the temp file cannot be created, +/// written, synced, or renamed over `path`. +fn write_file_atomically(path: &Path, contents: &str) -> std::io::Result<()> { + let directory = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + + let mut temp = tempfile::Builder::new() + .prefix(".ts-audit-") + .tempfile_in(directory)?; + temp.write_all(contents.as_bytes())?; + temp.as_file().sync_all()?; + if let Ok(metadata) = fs::metadata(path) { + temp.as_file().set_permissions(metadata.permissions())?; + } + temp.persist(path).map_err(|error| error.error)?; + + // Best-effort durability for the rename itself. Opening a directory handle + // is not portable (Windows rejects it), and the content is already safely + // on disk either way, so a failure here is not worth failing the command. + let _ = fs::File::open(directory).and_then(|handle| handle.sync_all()); + Ok(()) +} + +/// Arguments for `ts audit generate ` — bootstraps draft Trusted Server +/// config and JavaScript asset audit files from a live page (issue #800). +#[derive(Debug, clap::Args)] +pub(crate) struct GenerateArgs { + /// Public HTTP(S) URL to audit. + pub(crate) url: String, + /// JavaScript asset audit output path. + #[arg(long)] + pub(crate) js_assets: Option, + /// Draft Trusted Server config output path. + #[arg(long)] + pub(crate) config: Option, + /// Do not write the JavaScript asset audit file. + #[arg(long)] + pub(crate) no_js_assets: bool, + /// Do not write the draft Trusted Server config file. + #[arg(long)] + pub(crate) no_config: bool, + /// Overwrite existing output files. + #[arg(long)] + pub(crate) force: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = crate::commands::audit::parse_cookie)] + pub(crate) cookies: Vec<(String, String)>, + /// Browser and consent options shared with `ts audit ad-templates generate`. + #[command(flatten)] + pub(crate) browser: GenerateBrowserOpts, +} + +const DEFAULT_JS_ASSETS_PATH: &str = "js-assets.toml"; +const DEFAULT_CONFIG_PATH: &str = "trusted-server.toml"; + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum AssetParty { + FirstParty, + ThirdParty, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub(crate) struct AuditedAsset { + pub(crate) kind: String, + pub(crate) url: String, + pub(crate) host: String, + pub(crate) party: AssetParty, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) integration: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub(crate) struct DetectedIntegration { + pub(crate) id: String, + pub(crate) evidence: String, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub(crate) struct AuditArtifact { + pub(crate) audited_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) page_title: Option, + pub(crate) js_asset_count: usize, + pub(crate) third_party_asset_count: usize, + pub(crate) detected_integrations: Vec, + pub(crate) assets: Vec, + pub(crate) warnings: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct AuditOutputs { + pub(crate) artifact: AuditArtifact, + pub(crate) js_assets_toml: String, + pub(crate) draft_config_toml: String, + pub(crate) ad_slot_count: usize, + pub(crate) js_asset_proxy_candidate_count: usize, +} + +#[derive(Debug, Clone)] +struct DraftConfig { + toml: String, + js_asset_proxy_candidate_count: usize, +} + +#[derive(Debug, Clone)] +struct JsAssetProxySection { + toml: String, + candidate_count: usize, +} + +#[derive(Debug, Default)] +struct JsAssetProxySkipCounts { + first_party: usize, + malformed_url: usize, + non_https: usize, + duplicate_url: usize, + non_script: usize, +} + +#[derive(Debug)] +struct JsAssetProxyCandidate<'a> { + origin_url: String, + integration: Option<&'a str>, +} + +trait OpaqueAssetPathGenerator { + fn next_path(&mut self) -> String; +} + +#[derive(Debug, Default)] +struct RandomOpaqueAssetPathGenerator; + +impl OpaqueAssetPathGenerator for RandomOpaqueAssetPathGenerator { + fn next_path(&mut self) -> String { + let mut bytes = [0_u8; 12]; + rand::rngs::OsRng.fill_bytes(&mut bytes); + format!("/assets/{}.js", lowercase_hex(&bytes)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct AuditOutputPlan { + js_assets_path: Option, + config_path: Option, +} + +pub(crate) fn run_generate( + args: &GenerateArgs, + collector: &dyn AuditCollector, + out: &mut dyn Write, +) -> CliResult<()> { + let target_url = parse_audit_url(&args.url)?; + let plan = resolve_output_plan(args)?; + let collected = collector.collect_page(&target_url, &args.cookies)?; + let outputs = build_audit_outputs(&collected)?; + let wrote_config = plan.config_path.is_some(); + let written = write_audit_outputs(&outputs, &plan)?; + write_success_summary(&outputs, &written, wrote_config, out) +} + +fn parse_audit_url(value: &str) -> CliResult { + let url = Url::parse(value) + .map_err(|error| report_error(format!("invalid audit URL `{value}`: {error}")))?; + if !matches!(url.scheme(), "http" | "https") { + return cli_error(format!( + "`ts audit` only supports http/https URLs, got `{}`", + url.scheme() + )); + } + Ok(url) +} + +fn resolve_output_plan(args: &GenerateArgs) -> CliResult { + if args.no_js_assets && args.no_config { + return cli_error("nothing to do: both --no-js-assets and --no-config were set"); + } + + let js_assets_path = if args.no_js_assets { + None + } else { + Some(resolve_output_path( + args.js_assets.as_deref(), + DEFAULT_JS_ASSETS_PATH, + )?) + }; + let config_path = if args.no_config { + None + } else { + Some(resolve_output_path( + args.config.as_deref(), + DEFAULT_CONFIG_PATH, + )?) + }; + + if js_assets_path.is_some() && js_assets_path == config_path { + return cli_error("audit output paths must be distinct"); + } + + for path in [&js_assets_path, &config_path].into_iter().flatten() { + if path.exists() && !args.force { + return cli_error(format!( + "refusing to overwrite existing file `{}`; re-run with --force", + path.display() + )); + } + } + + Ok(AuditOutputPlan { + js_assets_path, + config_path, + }) +} + +fn resolve_output_path(path: Option<&Path>, default: &str) -> CliResult { + let candidate = path.unwrap_or_else(|| Path::new(default)); + if candidate.is_absolute() { + Ok(candidate.to_path_buf()) + } else { + Ok(std::env::current_dir() + .map_err(|error| report_error(format!("failed to read current directory: {error}")))? + .join(candidate)) + } +} + +fn build_audit_outputs(collected: &collector::CollectedPage) -> CliResult { + let artifact = analyze_collected_page(collected)?; + let final_url = collected + .final_url() + .map_err(|error| report_error(format!("invalid final URL: {error}")))?; + let js_assets_toml = toml::to_string_pretty(&artifact) + .map_err(|error| report_error(format!("failed to serialize audit artifact: {error}")))?; + let page_has_prebid = artifact + .detected_integrations + .iter() + .any(|integration| integration.id == "prebid"); + let slots = gpt_slots::discover_gpt_slots( + &collected.gpt_slots, + &collected.network_requests, + page_has_prebid, + ); + let ad_slot_count = slots.slots.len(); + let mut path_generator = RandomOpaqueAssetPathGenerator; + let draft_config = + build_draft_config_with_generator(&final_url, &artifact, &slots, &mut path_generator)?; + + Ok(AuditOutputs { + artifact, + js_assets_toml, + draft_config_toml: draft_config.toml, + ad_slot_count, + js_asset_proxy_candidate_count: draft_config.js_asset_proxy_candidate_count, + }) +} + +fn write_audit_outputs(outputs: &AuditOutputs, plan: &AuditOutputPlan) -> CliResult> { + let selected_paths = [&plan.js_assets_path, &plan.config_path] + .into_iter() + .flatten() + .collect::>(); + for path in &selected_paths { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent).map_err(|error| { + report_error(format!( + "failed to create parent directory {}: {error}", + parent.display() + )) + })?; + } + } + + let mut written_paths = Vec::new(); + if let Some(path) = &plan.js_assets_path { + write_file_atomically(path, &outputs.js_assets_toml).map_err(|error| { + report_error(format!( + "failed to write JS asset audit {}: {error}", + path.display() + )) + })?; + written_paths.push(path.display().to_string()); + } + if let Some(path) = &plan.config_path { + write_file_atomically(path, &outputs.draft_config_toml).map_err(|error| { + report_error(format!( + "failed to write draft config {}: {error}", + path.display() + )) + })?; + written_paths.push(path.display().to_string()); + } + + Ok(written_paths) +} + +fn write_success_summary( + outputs: &AuditOutputs, + written: &[String], + wrote_config: bool, + out: &mut dyn Write, +) -> CliResult<()> { + let integrations = outputs + .artifact + .detected_integrations + .iter() + .map(|integration| integration.id.as_str()) + .collect::>(); + let draft_note = if wrote_config { + "\nDraft config: review before validation and push" + } else { + "" + }; + let asset_proxy_note = if wrote_config && outputs.js_asset_proxy_candidate_count > 0 { + format!( + "{} disabled entries written to draft config", + outputs.js_asset_proxy_candidate_count + ) + } else if wrote_config { + "none".to_string() + } else { + "not written (--no-config)".to_string() + }; + writeln!( + out, + "Audited {}\nTitle: {}\nJS assets: {}\nThird-party assets: {}\nAd slots: {}\nDetected integrations: {}\nJS asset proxy candidates: {}\nWrote: {}{}", + outputs.artifact.audited_url, + outputs + .artifact + .page_title + .as_deref() + .unwrap_or(""), + outputs.artifact.js_asset_count, + outputs.artifact.third_party_asset_count, + outputs.ad_slot_count, + if integrations.is_empty() { + "none".to_string() + } else { + integrations.join(", ") + }, + asset_proxy_note, + if written.is_empty() { + "none".to_string() + } else { + written.join(", ") + }, + draft_note + ) + .map_err(|error| report_error(format!("failed to write command output: {error}"))) +} + +#[cfg(test)] +fn build_draft_config( + target_url: &Url, + artifact: &AuditArtifact, + slots: &gpt_slots::DiscoveredSlots, +) -> CliResult { + let mut path_generator = RandomOpaqueAssetPathGenerator; + build_draft_config_with_generator(target_url, artifact, slots, &mut path_generator) + .map(|draft| draft.toml) +} + +fn build_draft_config_with_generator( + target_url: &Url, + artifact: &AuditArtifact, + slots: &gpt_slots::DiscoveredSlots, + path_generator: &mut dyn OpaqueAssetPathGenerator, +) -> CliResult { + let host = target_url + .host_str() + .ok_or_else(|| report_error("audited URL is missing a host"))?; + let origin = target_url.origin().ascii_serialization(); + let mut draft = EXAMPLE_CONFIG.to_string(); + + draft = replace_key_in_section( + &draft, + "publisher", + "domain", + &format!("domain = \"{host}\""), + )?; + draft = replace_key_in_section( + &draft, + "publisher", + "cookie_domain", + &format!("cookie_domain = \".{host}\""), + )?; + draft = replace_key_in_section( + &draft, + "publisher", + "origin_url", + &format!("origin_url = \"{origin}\""), + )?; + + let detected = artifact + .detected_integrations + .iter() + .map(|integration| integration.id.as_str()) + .collect::>(); + + if detected.contains("gpt") { + draft = replace_key_in_section(&draft, "integrations.gpt", "enabled", "enabled = true")?; + } + if detected.contains("didomi") { + draft = replace_key_in_section(&draft, "integrations.didomi", "enabled", "enabled = true")?; + } + if detected.contains("datadome") { + draft = + replace_key_in_section(&draft, "integrations.datadome", "enabled", "enabled = true")?; + } + + let asset_proxy_section = build_js_asset_proxy_section(artifact, path_generator)?; + draft = replace_js_asset_proxy_section(&draft, &asset_proxy_section.toml)?; + + let mut manual_review = Vec::new(); + if detected.contains("google_tag_manager") { + if let Some(gtm_id) = extract_gtm_container_id(artifact) { + draft = replace_key_in_section( + &draft, + "integrations.google_tag_manager", + "enabled", + "enabled = true", + )?; + draft = replace_key_in_section( + &draft, + "integrations.google_tag_manager", + "container_id", + &format!("container_id = \"{gtm_id}\""), + )?; + } else { + manual_review.push("google_tag_manager"); + } + } + + for integration in detected { + if !matches!( + integration, + "gpt" | "didomi" | "datadome" | "google_tag_manager" + ) { + manual_review.push(integration); + } + } + + if !manual_review.is_empty() { + if !draft.ends_with('\n') { + draft.push('\n'); + } + draft.push_str("\n# Audit findings requiring manual review\n"); + for integration in manual_review { + draft.push_str(&format!( + "# - Detected {integration}; review the corresponding [integrations.{integration}] section before enabling it.\n" + )); + } + } + + if !slots.slots.is_empty() { + if let Some(network_id) = &slots.gam_network_id { + draft = replace_key_in_section( + &draft, + "creative_opportunities", + "gam_network_id", + &format!("gam_network_id = {}", toml_string(network_id)), + )?; + } + draft.push_str(&render_discovered_slots(target_url, slots)); + } + + Ok(DraftConfig { + toml: draft, + js_asset_proxy_candidate_count: asset_proxy_section.candidate_count, + }) +} + +fn build_js_asset_proxy_section( + artifact: &AuditArtifact, + path_generator: &mut dyn OpaqueAssetPathGenerator, +) -> CliResult { + let (candidates, skipped) = select_js_asset_proxy_candidates(artifact); + let mut used_paths = BTreeSet::new(); + let mut toml = String::new(); + + toml.push_str("[integrations.js_asset_proxy]\n"); + toml.push_str("enabled = false\n"); + toml.push_str("# Uncomment to override upstream cache headers for every asset below.\n"); + toml.push_str("# This replaces upstream directives, including private and no-store.\n"); + toml.push_str("# Use only when each asset's bytes are identical for every visitor.\n"); + toml.push_str("# cache_ttl_seconds = 3600\n"); + toml.push_str( + "# Asset fetches use a fixed TrustedServer/1.0 User-Agent. Do not proxy assets\n", + ); + toml.push_str( + "# that vary by browser User-Agent or use integrity hashes for UA-specific bytes.\n\n", + ); + toml.push_str("# Generated by `ts audit`; review before enabling.\n"); + toml.push_str( + "# Audit note: some discovered scripts may be runtime-injected and may not appear\n", + ); + toml.push_str( + "# in origin HTML. JS Asset Proxy rewrites only matching script src URLs present in\n", + ); + toml.push_str("# HTML processed by Trusted Server.\n"); + + if candidates.is_empty() { + toml.push_str( + "# No eligible third-party HTTPS script assets were detected by `ts audit`.\n", + ); + } + + for candidate in &candidates { + let generated_path = generate_unique_asset_path(path_generator, &mut used_paths)?; + toml.push('\n'); + toml.push_str("# Generated by `ts audit`; review before enabling.\n"); + if let Some(integration) = candidate.integration { + let integration = sanitized_comment_value(integration); + toml.push_str(&format!("# Detected integration: {integration}\n")); + toml.push_str(&format!( + "# Native integration may be preferable: [integrations.{integration}]\n" + )); + } + toml.push_str("[[integrations.js_asset_proxy.assets]]\n"); + toml.push_str(&format!("path = {}\n", toml_quoted_string(&generated_path))); + toml.push_str(&format!( + "origin_url = {}\n", + toml_quoted_string(&candidate.origin_url) + )); + if Url::parse(&candidate.origin_url).is_ok_and(|url| url.query().is_some()) { + toml.push_str( + "# This URL includes a query string and must remain stable for proxy matching.\n", + ); + } + toml.push_str("proxy = \"disabled\"\n"); + } + + append_js_asset_proxy_skip_comments(&mut toml, &skipped); + toml.push('\n'); + + Ok(JsAssetProxySection { + toml, + candidate_count: candidates.len(), + }) +} + +fn select_js_asset_proxy_candidates( + artifact: &AuditArtifact, +) -> (Vec>, JsAssetProxySkipCounts) { + let mut candidates = Vec::new(); + let mut skipped = JsAssetProxySkipCounts::default(); + let mut seen_origin_urls = BTreeSet::new(); + + for asset in &artifact.assets { + if asset.kind != "script" { + skipped.non_script += 1; + continue; + } + if asset.party != AssetParty::ThirdParty { + skipped.first_party += 1; + continue; + } + + let Ok(url) = Url::parse(&asset.url) else { + skipped.malformed_url += 1; + continue; + }; + if url.host_str().is_none() { + skipped.malformed_url += 1; + continue; + } + if url.scheme() != "https" { + skipped.non_https += 1; + continue; + } + + let origin_url = url.to_string(); + if !seen_origin_urls.insert(origin_url.clone()) { + skipped.duplicate_url += 1; + continue; + } + + candidates.push(JsAssetProxyCandidate { + origin_url, + integration: asset.integration.as_deref(), + }); + } + + (candidates, skipped) +} + +fn generate_unique_asset_path( + path_generator: &mut dyn OpaqueAssetPathGenerator, + used_paths: &mut BTreeSet, +) -> CliResult { + for _ in 0..128 { + let path = path_generator.next_path(); + if !is_valid_generated_asset_path(&path) { + return cli_error(format!( + "generated JS asset proxy path `{path}` is invalid; expected /assets/.js" + )); + } + if used_paths.insert(path.clone()) { + return Ok(path); + } + } + + cli_error("failed to generate a unique JS asset proxy path after 128 attempts") +} + +fn is_valid_generated_asset_path(path: &str) -> bool { + let Some(opaque_id) = path + .strip_prefix("/assets/") + .and_then(|value| value.strip_suffix(".js")) + else { + return false; + }; + + !opaque_id.is_empty() + && opaque_id + .chars() + .all(|ch| ch.is_ascii_hexdigit() && !ch.is_ascii_uppercase()) +} + +fn replace_js_asset_proxy_section(document: &str, replacement: &str) -> CliResult { + let lines = document.lines().collect::>(); + let start = lines + .iter() + .position(|line| line.trim() == "[integrations.js_asset_proxy]") + .ok_or_else(|| { + report_error( + "failed to update starter config because section `[integrations.js_asset_proxy]` was not found", + ) + })?; + let mut end = start + 1; + + while end < lines.len() { + let trimmed = lines[end].trim(); + if trimmed.starts_with('[') + && trimmed.ends_with(']') + && trimmed != "[[integrations.js_asset_proxy.assets]]" + { + break; + } + end += 1; + } + + // Blank lines and comments directly above the next section header document + // that section, not this one, so leave them in the draft. + while end > start + 1 { + let trimmed = lines[end - 1].trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + end -= 1; + } else { + break; + } + } + + let mut output_lines = Vec::new(); + output_lines.extend_from_slice(&lines[..start]); + output_lines.extend(replacement.trim_end_matches('\n').lines()); + if end < lines.len() && !lines[end].trim().is_empty() { + output_lines.push(""); + } + output_lines.extend_from_slice(&lines[end..]); + + let mut output = output_lines.join("\n"); + if document.ends_with('\n') { + output.push('\n'); + } + Ok(output) +} + +fn append_js_asset_proxy_skip_comments(toml: &mut String, skipped: &JsAssetProxySkipCounts) { + if skipped.first_party == 0 + && skipped.malformed_url == 0 + && skipped.non_https == 0 + && skipped.duplicate_url == 0 + && skipped.non_script == 0 + { + return; + } + + toml.push('\n'); + toml.push_str("# Skipped JS Asset Proxy audit candidates:\n"); + append_skip_count(toml, skipped.first_party, "first-party script"); + append_skip_count(toml, skipped.malformed_url, "malformed script URL"); + append_skip_count(toml, skipped.non_https, "non-HTTPS third-party script"); + append_skip_count(toml, skipped.duplicate_url, "duplicate script URL"); + append_skip_count(toml, skipped.non_script, "non-script asset"); +} + +fn append_skip_count(toml: &mut String, count: usize, label: &str) { + if count == 0 { + return; + } + + let plural = if count == 1 { "" } else { "s" }; + toml.push_str(&format!("# - {count} {label}{plural}\n")); +} + +fn sanitized_comment_value(value: &str) -> String { + value + .chars() + .map(|ch| if ch.is_control() { ' ' } else { ch }) + .collect() +} + +fn toml_quoted_string(value: &str) -> String { + let mut quoted = String::from("\""); + for ch in value.chars() { + match ch { + '\\' => quoted.push_str("\\\\"), + '"' => quoted.push_str("\\\""), + '\n' => quoted.push_str("\\n"), + '\r' => quoted.push_str("\\r"), + '\t' => quoted.push_str("\\t"), + ch if ch.is_control() => { + write!(&mut quoted, "\\u{:04X}", ch as u32).expect("should write to string"); + } + ch => quoted.push(ch), + } + } + quoted.push('"'); + quoted +} + +fn lowercase_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut encoded = String::with_capacity(bytes.len() * 2); + for byte in bytes { + encoded.push(HEX[(byte >> 4) as usize] as char); + encoded.push(HEX[(byte & 0x0f) as usize] as char); + } + encoded +} + +/// Renders discovered GPT slots as appended `[[creative_opportunities.slot]]` +/// tables. Page patterns default to the audited path and are flagged for review. +fn render_discovered_slots(target_url: &Url, slots: &gpt_slots::DiscoveredSlots) -> String { + let path = target_url.path(); + let page_pattern = if path.is_empty() { "/" } else { path }; + + let mut out = String::from( + "\n# Slots discovered from live GPT ad requests during the audit.\n\ + # Review page_patterns and formats before validating/pushing.\n", + ); + for slot in &slots.slots { + let formats = slot + .formats + .iter() + .map(|(width, height)| format!("{{ width = {width}, height = {height} }}")) + .collect::>() + .join(", "); + out.push_str(&format!( + "\n[[creative_opportunities.slot]]\n\ + id = {id}\n\ + div_id = {div_id}\n\ + gam_unit_path = {gam_unit_path}\n\ + page_patterns = [{page_pattern}]\n\ + formats = [{formats}]\n", + id = toml_string(&slot.id), + div_id = toml_string(&slot.div_id), + gam_unit_path = toml_string(&slot.gam_unit_path), + page_pattern = toml_string(page_pattern), + )); + if slot.has_prebid { + out.push_str("[creative_opportunities.slot.providers.prebid]\nbidders = {}\n"); + } + } + out +} + +/// Everything one `ts audit ad-templates generate` invocation needs. +pub(crate) struct UpdateSlotsRequest<'a> { + /// Page URL to start from; also bounds the crawl to its origin. + pub(crate) url: &'a str, + /// Operator config to rewrite in place. + pub(crate) config_path: &'a Path, + /// The config's current `[creative_opportunities]`, when it has one. + pub(crate) existing_creative: Option<&'a CreativeOpportunitiesConfig>, + /// Explicit `--page-pattern` values. When non-empty these apply to every + /// slot and pattern inference is skipped entirely. + pub(crate) page_patterns: &'a [String], + /// Replace existing slots rather than merging into them. + pub(crate) replace: bool, + /// Cookies to carry into the crawl. + pub(crate) cookies: &'a [(String, String)], + /// Print the candidate instead of writing it. + pub(crate) dry_run: bool, + /// Whether the crawl used the deterministic scroll pass. + pub(crate) scroll: bool, + /// Crawl bounds. + pub(crate) budget: crawl_plan::CrawlBudget, +} + +/// Share of crawled pages that may yield no slots before the run is refused. +/// +/// A bot-protection challenge serves an interstitial that loads fine and +/// contains no ad stack, so it looks like a page with no slots. Writing a config +/// from a crawl that was mostly challenges would silently narrow the operator's +/// slot set; refusing is the safer failure. +const MAX_EMPTY_PAGE_SHARE: f64 = 0.25; + +/// Runs `ts audit ad-templates generate`: crawl the site's sections, reconcile +/// what each slot looked like across them, infer a `{section}` ad-unit template +/// where the evidence proves one, and rewrite the config's slot array in place. +/// +/// # Errors +/// +/// Returns an error when the config cannot be read, the root page cannot be +/// collected, no slots are discovered, too many pages came back empty, the +/// pages disagree about the GAM network id, or the resulting config would not +/// load. +pub(crate) fn run_update_slots( + request: &UpdateSlotsRequest<'_>, + collectors: &[(&str, &dyn AuditCollector)], + out: &mut dyn Write, + err: &mut dyn Write, +) -> CliResult<()> { + let Some((first_label, first_collector)) = collectors.first() else { + return cli_error("no device profile was selected to audit with"); + }; + let target_url = parse_audit_url(request.url)?; + let existing = fs::read_to_string(request.config_path).map_err(|error| { + report_error(format!( + "failed to read config {}: {error}", + request.config_path.display() + )) + })?; + + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + let mut root_url = target_url.clone(); + let mut planned = None; + let mut fold_error = None; + + { + let mut progress_writer = CollectionProgressWriter { + out: err, + profile_label: first_label, + }; + let mut report_progress = + |progress: collector::CollectionProgress<'_>| progress_writer.write(progress); + first_collector.collect_site( + &target_url, + request.cookies, + &mut report_progress, + &mut |_, root| { + root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); + if origin_changed(&target_url, &root_url) { + // Origins only: the origin is what the refusal is about, and + // a full URL would echo any `user:password@` the operator + // passed into stderr. + return cli_error(format!( + "refusing cross-origin root redirect from {} to {}; the requested origin is the audit and cookie trust boundary", + target_url.origin().ascii_serialization(), + root_url.origin().ascii_serialization() + )); + } + let plan = crawl_plan::plan_crawl( + &root_url, + &root.links, + &root.sitemap_locs, + request.budget, + ); + let targets = plan.targets(); + planned = Some(plan); + Ok(targets) + }, + &mut |url, collected| { + match collected { + Ok(page) => { + let final_url = page.final_url().unwrap_or_else(|_| url.clone()); + // The requested origin is the trust boundary for every + // page, not just the root: a section page that redirects + // away would otherwise contribute foreign slots, formats + // and ad-unit paths to the generated config. + if origin_changed(&target_url, &final_url) { + notes.push(format!( + "skipped `{}` on {first_label}: it left the audited origin for {}", + url.path(), + final_url.origin().ascii_serialization() + )); + return Ok(collector::ControlFlow::Continue); + } + if let Err(error) = + fold_collected( + &mut table, + &final_url, + &page, + first_label, + &mut notes, + ) + { + fold_error = Some(error); + return Ok(collector::ControlFlow::Stop); + } + } + Err(error) => { + // Path only, like the progress lines: a planned target + // still carries the origin and any userinfo. + notes.push(format!( + "skipped `{}` on {first_label}: {error}", + url.path() + )); + } + } + Ok(collector::ControlFlow::Continue) + }, + )?; + } + if let Some(error) = fold_error { + return Err(error); + } + let plan = planned.ok_or_else(|| { + report_error(format!( + "the {first_label} browser session did not produce a root page" + )) + })?; + notes.extend(plan.notes.iter().cloned()); + // Fragments never reach the server, so only a difference the origin acted on + // counts as a redirect worth reporting. + if without_fragment(&root_url) != without_fragment(&target_url) { + notes.push(format!( + "followed a root redirect from `{}{}` to `{}{}`; slots and page patterns are derived from the final URL", + target_url.origin().ascii_serialization(), + target_url.path(), + root_url.origin().ascii_serialization(), + root_url.path() + )); + } + + // Every profile walks the same pages into the same table. When two profiles + // disagree about a slot's ad-unit path, that shows up as two observations of + // one page, which inference already refuses to represent. + for (label, collector) in collectors.iter().skip(1) { + let mut progress_writer = CollectionProgressWriter { + out: err, + profile_label: label, + }; + let successful_pages = crawl_sections( + *collector, + &root_url, + &plan, + request.cookies, + &mut table, + &mut notes, + &mut progress_writer, + )?; + if successful_pages == 0 { + return cli_error(format!( + "the selected {label} device profile did not collect any required page; refusing to generate from incomplete profile coverage" + )); + } + } + if collectors.len() > 1 { + notes.push(format!( + "audited {} device profile(s): {}", + collectors.len(), + collectors + .iter() + .map(|(label, _)| *label) + .collect::>() + .join(", ") + )); + } + + // Emit what the crawl learned before any refusal below can return early. + // The guards exist precisely for runs that went wrong, so that is when the + // per-page reasons matter most. + emit_notes(err, &mut notes)?; + + if table.is_empty() { + return cli_error(format!( + "no ad-template slots were discovered on any of the {} crawled page(s); \ + see the notes above for what each page reported", + table.pages().len() + )); + } + guard_challenge_rate(&table)?; + + let discovered_network_id = table.network_id()?; + let network_id = resolve_network_id( + request.existing_creative, + discovered_network_id.as_deref(), + request.replace, + ); + + // Templating needs a network id to bind `{network_id}` against; without one + // every path stays literal. + let inference = network_id + .as_deref() + .map(|id| unit_template::infer_unit_templates(&table, id)); + if let Some(outcome) = &inference { + notes.extend(outcome.diagnostics.iter().cloned()); + } + let policy = inference + .as_ref() + .and_then(|outcome| outcome.policy.clone()); + validate_merge_policy(request.existing_creative, policy.as_ref(), request.replace)?; + + // Slots that are one placement wearing a per-render div id cannot be + // written: the ids never match at runtime. Report them so the operator can + // add the placement once with a prefix they know is stable. + let fragmented = table.fragmented_slots(); + for group in &fragmented { + let suggestion = group.suggested_prefix.as_deref().map_or_else( + || "no stable prefix was shared".to_string(), + |prefix| format!("they share the prefix `{prefix}`"), + ); + notes.push(format!( + "skipped {} slot(s) that look like one placement under a per-render div id on \ + `{}` ({}); {suggestion}. Add it once by hand with a div_id prefix that is \ + stable across renders", + group.div_ids.len(), + group.unit_path, + group.div_ids.join(", "), + )); + } + + let slots = build_render_slots( + &table, + inference.as_ref(), + policy.as_ref(), + request, + plan.section_segment, + &fragmented, + &mut notes, + )?; + let observed_div_ids = table + .observed_div_ids() + .map(str::to_string) + .collect::>(); + let observed_literals = table + .observed_literals() + .map(str::to_string) + .collect::>(); + let (merged, merge_diagnostics) = slot_toml::merge_render_slots_with_observed_diagnostics( + request.existing_creative, + slots, + &observed_div_ids, + &observed_literals, + request.replace, + ); + notes.extend(merge_diagnostics.notes); + if !merge_diagnostics.unobserved_existing_slot_ids.is_empty() { + let slot_ids = merge_diagnostics.unobserved_existing_slot_ids.join(", "); + let follow_up = if request.scroll { + "Re-run with broader page/profile coverage; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." + } else { + "Re-run with broader coverage or --scroll; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." + }; + notes.push(format!( + "preserved {} configured slot(s) not observed during this crawl: {slot_ids}. {follow_up}", + merge_diagnostics.unobserved_existing_slot_ids.len(), + )); + } + if merged.is_empty() { + emit_notes(err, &mut notes)?; + return cli_error( + "refusing to write zero generated slots after the crawl discovered slot evidence; review the refused-slot notes and keep the existing configuration", + ); + } + let rendered_slots = render_slots(&merged); + let updated = splice_creative_slots( + &existing, + &slot_toml::CreativeSectionKeys { + network_id: network_id.as_deref(), + section_root: policy.as_ref().map(|policy| policy.section_root.as_str()), + section_segment: policy.as_ref().map(|policy| policy.section_segment), + }, + &rendered_slots, + )?; + + // Everything above is derived from a live, page-controlled ad stack, so the + // candidate has to clear the runtime's own load path before it can replace + // the operator's file. This runs on the dry-run path too — otherwise "the + // preview looked fine" would not be evidence that the config loads. + notes.extend(validate::check_candidate(&updated, &existing)?); + + emit_notes(err, &mut notes)?; + if policy.is_some() { + writeln!( + err, + "note: this config now uses a {{section}} ad-unit template. Deploy a \ + template-aware binary BEFORE pushing it, and do not roll that binary \ + back while this config is live — an older binary rejects the whole \ + config and serves an error on every route." + ) + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } + + if request.dry_run { + let old_managed = managed_creative_projection(&existing)?; + let new_managed = managed_creative_projection(&updated)?; + if old_managed == new_managed { + // Stdout is the diff surface, so an English sentence there would + // break a redirected `--dry-run`; an empty diff is the stdout answer. + writeln!(err, "No managed creative-opportunity changes.").map_err(|error| { + report_error(format!("failed to write preview output: {error}")) + })?; + return Ok(()); + } + let diff = similar::TextDiff::from_lines(&old_managed, &new_managed); + writeln!( + out, + "{}", + diff.unified_diff().context_radius(0).header( + "configured creative opportunities", + "generated creative opportunities" + ) + ) + .map_err(|error| report_error(format!("failed to write preview diff: {error}")))?; + return Ok(()); + } + let current = fs::read_to_string(request.config_path).map_err(|error| { + report_error(format!( + "failed to re-read config {} before writing: {error}", + request.config_path.display() + )) + })?; + if current != existing { + return cli_error(format!( + "refusing to overwrite {} because it changed during the browser audit; re-run against the current file", + request.config_path.display() + )); + } + // A writer could still land between this check and the rename below. That + // window is microseconds against a browser crawl's minutes, and the rename + // is atomic, so the loser of the race loses a whole write rather than half + // of one. Closing it properly would need file locking the operator's editor + // does not take part in. + write_file_atomically(request.config_path, &updated).map_err(|error| { + report_error(format!( + "failed to write config {}: {error}", + request.config_path.display() + )) + })?; + writeln!( + out, + "Wrote {} slot(s) to {} ({} slot(s) seen across {} page(s))", + merged.len(), + request.config_path.display(), + table.slot_count(), + table.pages().len(), + ) + .map_err(|error| report_error(format!("failed to write command output: {error}"))) +} + +/// Renders only fields managed by ad-template generation, excluding secrets and +/// unrelated operator configuration from dry-run output. +fn managed_creative_projection(document: &str) -> CliResult { + let value = toml::from_str::(document).map_err(|error| { + report_error(format!("failed to parse config for dry-run diff: {error}")) + })?; + let creative = value + .get("creative_opportunities") + .and_then(toml::Value::as_table); + let mut managed = toml::map::Map::new(); + if let Some(creative) = creative { + for key in ["gam_network_id", "section_root", "section_segment", "slot"] { + if let Some(value) = creative.get(key) { + managed.insert(key.to_string(), value.clone()); + } + } + } + let mut root = toml::map::Map::new(); + root.insert( + "creative_opportunities".to_string(), + toml::Value::Table(managed), + ); + toml::to_string_pretty(&toml::Value::Table(root)) + .map_err(|error| report_error(format!("failed to render dry-run projection: {error}"))) +} + +/// A page carrying fewer scripts than this is not a real publisher page. +/// +/// A production page runs dozens: the ad stack, analytics, consent, and the +/// site's own bundles. A bot-protection interstitial runs its own challenge +/// script and little else. +const INTERSTITIAL_SCRIPT_CEILING: usize = 3; + +/// Whether a page that loaded successfully is nonetheless not the real page. +/// +/// Bot protection commonly answers with **200** and a challenge document rather +/// than a 4xx, so status-code checks pass and the page simply appears to have no +/// ad stack. Left unexplained, that is indistinguishable from a publisher who +/// genuinely runs no ads on that page — and the operator's next move is entirely +/// different in each case. +fn looks_like_an_interstitial(artifact: &AuditArtifact) -> Option { + if artifact.js_asset_count > INTERSTITIAL_SCRIPT_CEILING + || !artifact.detected_integrations.is_empty() + { + return None; + } + Some(format!( + "the page returned successfully but carried only {} script(s) and no recognised \ + integrations, which is the shape of a bot-protection challenge rather than the \ + real page. Supply a current --cookie for the origin", + artifact.js_asset_count + )) +} + +/// Writes and clears the pending notes, so each is reported exactly once. +fn emit_notes(out: &mut dyn Write, notes: &mut Vec) -> CliResult<()> { + for note in notes.drain(..) { + writeln!( + out, + "note: {}", + crate::ad_templates::output::escape_terminal_text(¬e) + ) + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } + Ok(()) +} + +/// Writes one immediately visible, profile-aware crawl progress line. +fn write_collection_progress( + out: &mut dyn Write, + profile_label: &str, + progress: collector::CollectionProgress<'_>, +) -> CliResult<()> { + let line = match progress { + collector::CollectionProgress::Launching => { + format!("Auditing {profile_label}: launching browser") + } + collector::CollectionProgress::Loading { + current, + total, + url, + } => { + let path = if url.path().is_empty() { + "/" + } else { + url.path() + }; + let path = crate::ad_templates::output::escape_terminal_text(path); + let total = total.map_or_else(|| "?".to_string(), |total| total.to_string()); + format!("Auditing {profile_label} [{current}/{total}]: {path}") + } + collector::CollectionProgress::Planning => { + format!("Auditing {profile_label}: planning site crawl") + } + collector::CollectionProgress::Finalizing => { + format!("Auditing {profile_label}: finalizing browser session") + } + }; + writeln!(out, "{line}") + .map_err(|error| report_error(format!("failed to write audit progress: {error}")))?; + out.flush() + .map_err(|error| report_error(format!("failed to flush audit progress: {error}"))) +} + +struct CollectionProgressWriter<'a> { + out: &'a mut dyn Write, + profile_label: &'a str, +} + +impl CollectionProgressWriter<'_> { + fn write(&mut self, progress: collector::CollectionProgress<'_>) -> CliResult<()> { + write_collection_progress(self.out, self.profile_label, progress) + } +} + +/// Discovers a collected page's slots and folds them into `table`. +/// +/// Per-page collector warnings are appended to `notes`. They carry the reason a +/// page came back without slots — a non-2xx main document, a navigation that +/// never settled — which is the difference between "this publisher has no ad +/// stack here" and "bot protection served a challenge". Dropping them leaves +/// the operator with a refusal and no way to act on it. +fn fold_collected( + table: &mut evidence::EvidenceTable, + url: &Url, + collected: &collector::CollectedPage, + profile_label: &str, + notes: &mut Vec, +) -> CliResult<()> { + // `analyze_collected_page` already carries the collector's warnings forward, + // so this is the complete set, not a second copy. + let artifact = analyze_collected_page(collected)?; + for warning in &artifact.warnings { + // The consent stub is a property of the run, not of this page. Scoping it + // to a path and repeating it per page and profile buries the per-page + // diagnostics an operator is reading these notes for. + let note = if warning == collector::CONSENT_STUB_WARNING { + warning.clone() + } else { + format!("`{}` on {profile_label}: {warning}", url.path()) + }; + if !notes.contains(¬e) { + notes.push(note); + } + } + if let Some(reason) = looks_like_an_interstitial(&artifact) { + notes.push(format!("`{}` on {profile_label}: {reason}", url.path())); + } + let page_has_prebid = artifact + .detected_integrations + .iter() + .any(|integration| integration.id == "prebid"); + let discovered = gpt_slots::discover_gpt_slots( + &collected.gpt_slots, + &collected.network_requests, + page_has_prebid, + ); + for warning in &discovered.warnings { + if !notes.contains(warning) { + notes.push(warning.clone()); + } + } + table.fold_page(url.path(), &discovered); + Ok(()) +} + +/// Walks the planned section pages, folding each into `table`. +/// +/// A page that fails to collect is recorded as a note rather than aborting: on a +/// multi-section crawl one blocked or slow page should not discard the sections +/// that did work. The empty-page guard afterwards catches the case where enough +/// of them failed that the result is untrustworthy. +fn crawl_sections( + collector: &dyn AuditCollector, + root_url: &Url, + plan: &crawl_plan::CrawlPlan, + cookies: &[(String, String)], + table: &mut evidence::EvidenceTable, + notes: &mut Vec, + progress_writer: &mut CollectionProgressWriter<'_>, +) -> CliResult { + let additional_targets = plan.targets(); + if additional_targets.is_empty() { + notes.push( + "no additional site sections were discovered, so only the requested page was \ + audited; pass explicit --page-pattern values or more URLs to widen coverage" + .to_string(), + ); + } + // The root is deliberately part of every profile's shared batch: browser + // clearance/session state established there then carries into section pages. + let mut targets = Vec::with_capacity(additional_targets.len() + 1); + targets.push(root_url.clone()); + targets.extend(additional_targets); + + let mut fold_error = None; + let mut successful_pages = 0_usize; + { + let profile_label = progress_writer.profile_label; + let mut report_progress = + |progress: collector::CollectionProgress<'_>| progress_writer.write(progress); + collector.collect_pages( + &targets, + cookies, + &mut report_progress, + &mut |url, collected| { + match collected { + Ok(page) => { + let final_url = page.final_url().unwrap_or_else(|_| url.clone()); + // Same boundary as the first profile, and it covers this + // profile's root page too: a cross-origin redirect is not + // a page this run may learn inventory from, so it must + // not count towards profile coverage either. + if origin_changed(root_url, &final_url) { + notes.push(format!( + "skipped `{}` on {profile_label}: it left the audited origin for {}", + url.path(), + final_url.origin().ascii_serialization() + )); + return Ok(collector::ControlFlow::Continue); + } + successful_pages += 1; + if let Err(error) = + fold_collected(table, &final_url, &page, profile_label, notes) + { + fold_error = Some(error); + return Ok(collector::ControlFlow::Stop); + } + } + Err(error) => { + notes.push(format!( + "skipped `{}` on {profile_label}: {error}", + url.path() + )); + } + } + Ok(collector::ControlFlow::Continue) + }, + )?; + } + match fold_error { + Some(error) => Err(error), + None => Ok(successful_pages), + } +} + +/// Refuses a crawl where too many pages produced no slots. +fn guard_challenge_rate(table: &evidence::EvidenceTable) -> CliResult<()> { + let total = table.pages().len(); + let empty = table.empty_pages().len(); + if total == 0 || (empty as f64) <= (total as f64) * MAX_EMPTY_PAGE_SHARE { + return Ok(()); + } + let blocked: Vec<&str> = table.empty_pages().iter().map(String::as_str).collect(); + cli_error(format!( + "{empty} of {total} crawled page(s) produced no ad slots ({}), which usually means \ + bot protection served a challenge instead of the real page. Refusing to write a \ + config from partial evidence; re-run with a valid --cookie for the origin", + blocked.join(", ") + )) +} + +/// Refuses a merge that would reinterpret templated slots the config already has. +/// +/// # Errors +/// +/// Returns an error when preserved `{section}` slots were written against a +/// different section policy than this run inferred, since the merge would leave +/// them pointing at ad units nobody configured. +fn validate_merge_policy( + existing: Option<&CreativeOpportunitiesConfig>, + inferred: Option<&unit_template::SectionPolicy>, + replace: bool, +) -> CliResult<()> { + if replace { + return Ok(()); + } + let Some(existing) = existing else { + return Ok(()); + }; + let preserves_template = existing.slot.iter().any(|slot| { + slot.gam_unit_path + .as_deref() + .is_some_and(|path| path.contains("{section}")) + }); + let Some(inferred) = inferred.filter(|_| preserves_template) else { + return Ok(()); + }; + if let Some(configured_segment) = existing.section_segment + && configured_segment != inferred.section_segment + { + return cli_error(format!( + "refusing to change the section_segment used by preserved templated slots during merge: configured section_segment={configured_segment}; inferred section_segment={}. Re-run with --replace only for an intentional migration", + inferred.section_segment + )); + } + // A `{section}` slot with no `section_root` cannot load at all — + // `validate_runtime` requires one — so there is no root value to preserve. + // Adopting the inferred root makes such a config loadable, provided the + // independently configured section segment above still agrees. + let Some(configured_root) = existing + .section_root + .as_deref() + .filter(|root| !root.is_empty()) + else { + return Ok(()); + }; + let configured_segment = existing.section_segment.unwrap_or(0); + if configured_root != inferred.section_root || configured_segment != inferred.section_segment { + return cli_error(format!( + "refusing to change the section policy used by preserved templated slots during merge: configured section_root={configured_root:?}, section_segment={configured_segment}; inferred section_root={:?}, section_segment={}. Re-run with --replace only for an intentional migration", + inferred.section_root, inferred.section_segment + )); + } + Ok(()) +} + +/// Turns the evidence table into slots ready to render. +fn build_render_slots( + table: &evidence::EvidenceTable, + inference: Option<&unit_template::InferenceOutcome>, + policy: Option<&unit_template::SectionPolicy>, + request: &UpdateSlotsRequest<'_>, + fallback_section_segment: usize, + fragmented: &[evidence::FragmentGroup], + notes: &mut Vec, +) -> CliResult> { + let skip: std::collections::BTreeSet<&str> = fragmented + .iter() + .flat_map(|group| group.div_ids.iter().map(String::as_str)) + .collect(); + // Explicit `--page-pattern` values are an operator override: they apply to + // every slot and disable inference from observed paths entirely. + let explicit = !request.page_patterns.is_empty(); + if explicit { + validate_page_patterns(request.page_patterns)?; + // Not filtered against `skip`: a borrowed root implies the slot's + // ad-unit path varied across pages, and `fragmented_slots` only groups + // slots pinned to exactly one unit path, so the two sets are disjoint. + if let Some(outcome) = inference + && !outcome.borrowed_section_root.is_empty() + { + let affected = outcome + .borrowed_section_root + .iter() + .map(|stem| format!("`{stem}`")) + .collect::>() + .join(", "); + return cli_error(format!( + "cannot apply --page-pattern to slot(s) with div id(s) {affected} because their \ + {{section}} templates borrow section_root; remove --page-pattern so patterns \ + can be derived from the paths where each slot was observed" + )); + } + } + let section_segment = policy.map_or(fallback_section_segment, |policy| policy.section_segment); + + let mut slots = Vec::with_capacity(table.slot_count()); + for slot in table.slots() { + if skip.contains(slot.div_id.as_str()) { + continue; + } + let patterns = if explicit { + request.page_patterns.to_vec() + } else { + let derived = page_patterns::patterns_for_paths(slot.paths(), section_segment); + validate_page_patterns(&derived)?; + derived + }; + let unit_path = match inference.and_then(|outcome| outcome.decision(&slot.div_id)) { + Some(unit_template::SlotDecision::Template(template)) => Some(template.clone()), + Some(unit_template::SlotDecision::Literal(path)) => Some(path.clone()), + Some(unit_template::SlotDecision::Refuse { reasons }) => { + notes.push(format!( + "skipped refused slot `{}` (`{}`): {}", + slot.id, + slot.div_id, + reasons.join("; ") + )); + continue; + } + None => None, + }; + slots.push(slot_toml::RenderSlot::from_evidence( + &slot.id, + &slot.div_id, + unit_path, + slot.formats.iter().copied(), + patterns, + slot.has_prebid, + )); + } + Ok(slots) +} +/// Rejects any page pattern the runtime's glob compiler would not accept. +/// +/// Uses [`validate_page_pattern`] so the accepted set is exactly what +/// `CreativeOpportunitySlot::compile_patterns` accepts at startup, including the +/// `**`→`*` normalisation. All patterns are reported at once so an operator +/// passing several `--page-pattern` values fixes them in one pass. +/// +/// # Errors +/// +/// Returns a user-facing error listing every pattern that does not compile. +fn validate_page_patterns(patterns: &[String]) -> CliResult<()> { + let invalid: Vec = patterns + .iter() + .filter_map(|pattern| validate_page_pattern(pattern).err()) + .collect(); + if invalid.is_empty() { + return Ok(()); + } + cli_error(format!( + "refusing to write invalid page pattern(s): {}", + invalid.join("; ") + )) +} + +#[cfg(test)] +mod tests { + use std::cell::{Cell, RefCell}; + use std::collections::VecDeque; + use std::io; + use std::rc::Rc; + + use tempfile::TempDir; + + use super::*; + use crate::app_config::AppConfigArgs; + use crate::commands::audit::generate::collector::{ + CollectedPage, CollectedRequest, CollectedScriptTag, + }; + use crate::commands::config::init::EXAMPLE_CONFIG; + + struct FakeCollector { + collected: CollectedPage, + calls: Cell, + } + + struct FixedPathGenerator { + paths: VecDeque, + } + + impl FixedPathGenerator { + fn new(paths: &[&str]) -> Self { + Self { + paths: paths.iter().map(|path| (*path).to_string()).collect(), + } + } + } + + impl OpaqueAssetPathGenerator for FixedPathGenerator { + fn next_path(&mut self) -> String { + self.paths + .pop_front() + .expect("should have a fixed generated asset path") + } + } + + struct MutatingCollector { + collected: CollectedPage, + config_path: std::path::PathBuf, + replacement: String, + } + + impl AuditCollector for MutatingCollector { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + fs::write(&self.config_path, &self.replacement) + .map_err(|error| report_error(format!("failed to mutate test config: {error}")))?; + Ok(self.collected.clone()) + } + } + + impl FakeCollector { + fn new(collected: CollectedPage) -> Self { + Self { + collected, + calls: Cell::new(0), + } + } + } + + impl AuditCollector for FakeCollector { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + self.calls.set(self.calls.get() + 1); + Ok(self.collected.clone()) + } + } + + /// A collector serving a distinct page per URL, recording the crawl order. + struct SiteCollector { + pages: std::collections::HashMap, + visited: std::cell::RefCell>, + } + + struct FailingCollector; + + #[derive(Clone, Default)] + struct SharedProgressState { + bytes: Rc>>, + flushes: Rc>, + } + + struct SharedProgressWriter { + state: SharedProgressState, + } + + impl Write for SharedProgressWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + self.state.bytes.borrow_mut().extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.state.flushes.set(self.state.flushes.get() + 1); + Ok(()) + } + } + + struct ObservingProgressCollector { + collected: CollectedPage, + state: SharedProgressState, + saw_flushed_progress: Cell, + } + + impl AuditCollector for ObservingProgressCollector { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + Ok(self.collected.clone()) + } + + fn collect_site( + &self, + root: &Url, + _cookies: &[(String, String)], + on_progress: collector::ProgressSink<'_>, + planner: collector::RootPlanner<'_>, + on_page: collector::PageSink<'_>, + ) -> CliResult<()> { + on_progress(collector::CollectionProgress::Loading { + current: 1, + total: None, + url: root, + })?; + self.saw_flushed_progress + .set(!self.state.bytes.borrow().is_empty() && self.state.flushes.get() > 0); + on_progress(collector::CollectionProgress::Planning)?; + let _ = planner(root, &self.collected)?; + let _ = on_page(root, Ok(self.collected.clone()))?; + Ok(()) + } + } + + #[derive(Default)] + struct ProgressWriter { + bytes: Vec, + flushes: usize, + fail_write: bool, + fail_flush: bool, + } + + impl Write for ProgressWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + if self.fail_write { + return Err(io::Error::other("simulated progress write failure")); + } + self.bytes.extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.flushes += 1; + if self.fail_flush { + return Err(io::Error::other("simulated progress flush failure")); + } + Ok(()) + } + } + + #[test] + fn progress_lines_are_profile_aware_and_flush_immediately() { + let url = + Url::parse("https://user:pass@publisher.example/news\u{1b}[31m?token=secret#fragment") + .expect("should parse progress URL"); + let mut writer = ProgressWriter::default(); + + for progress in [ + collector::CollectionProgress::Launching, + collector::CollectionProgress::Loading { + current: 1, + total: None, + url: &url, + }, + collector::CollectionProgress::Planning, + collector::CollectionProgress::Loading { + current: 2, + total: Some(17), + url: &url, + }, + collector::CollectionProgress::Finalizing, + ] { + write_collection_progress(&mut writer, "desktop", progress) + .expect("should write progress"); + } + + let rendered = String::from_utf8(writer.bytes).expect("should render UTF-8 progress"); + assert_eq!( + rendered, + "Auditing desktop: launching browser\n\ + Auditing desktop [1/?]: /news%1B[31m\n\ + Auditing desktop: planning site crawl\n\ + Auditing desktop [2/17]: /news%1B[31m\n\ + Auditing desktop: finalizing browser session\n" + ); + assert_eq!(writer.flushes, 5, "should flush every progress line"); + assert!(!rendered.contains("user"), "should omit URL userinfo"); + assert!(!rendered.contains("secret"), "should omit URL query values"); + assert!(!rendered.contains("fragment"), "should omit URL fragments"); + assert!( + !rendered.contains('\u{1b}'), + "should not emit terminal escapes" + ); + } + + #[test] + fn progress_write_and_flush_failures_are_reported() { + let mut write_failure = ProgressWriter { + fail_write: true, + ..ProgressWriter::default() + }; + let write_error = write_collection_progress( + &mut write_failure, + "desktop", + collector::CollectionProgress::Launching, + ) + .expect_err("should report progress write failure"); + assert!(format!("{write_error:?}").contains("failed to write audit progress")); + + let mut flush_failure = ProgressWriter { + fail_flush: true, + ..ProgressWriter::default() + }; + let flush_error = write_collection_progress( + &mut flush_failure, + "desktop", + collector::CollectionProgress::Finalizing, + ) + .expect_err("should report progress flush failure"); + assert!(format!("{flush_error:?}").contains("failed to flush audit progress")); + } + + #[test] + fn update_slots_flushes_progress_before_collection_returns() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let state = SharedProgressState::default(); + let collector = ObservingProgressCollector { + collected: collected_page_with_header_slot(), + state: state.clone(), + saw_flushed_progress: Cell::new(false), + }; + let mut progress_writer = SharedProgressWriter { state }; + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut progress_writer, + ) + .expect("should generate slots"); + + assert!( + collector.saw_flushed_progress.get(), + "collector should observe flushed progress before returning" + ); + assert!( + !String::from_utf8(out) + .expect("should write UTF-8 output") + .contains("Auditing "), + "stdout should not contain progress" + ); + } + + impl AuditCollector for FailingCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + cli_error(format!("simulated navigation failure for {target_url}")) + } + } + + impl SiteCollector { + fn new(pages: Vec<(&str, CollectedPage)>) -> Self { + Self { + pages: pages + .into_iter() + .map(|(url, page)| (url.to_string(), page)) + .collect(), + visited: std::cell::RefCell::new(Vec::new()), + } + } + } + + impl AuditCollector for SiteCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + self.visited.borrow_mut().push(target_url.to_string()); + self.pages + .get(target_url.as_str()) + .cloned() + .ok_or_else(|| report_error(format!("no fake page for {target_url}"))) + } + } + + /// Builds a page carrying one GPT slot plus same-origin nav links. + fn site_page(url: &str, unit_path: &str, nav_paths: &[&str]) -> CollectedPage { + let mut page = collected_page(); + page.requested_url = url.to_string(); + page.final_url = url.to_string(); + page.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: unit_path.to_string(), + div_id: "ad-header-0".to_string(), + sizes: vec![(728, 90)], + }]; + page.links = nav_paths + .iter() + .map(|path| collector::CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: true, + }) + .collect(); + page + } + + fn collected_page() -> CollectedPage { + CollectedPage { + requested_url: "https://publisher.example/page".to_string(), + final_url: "https://publisher.example/page".to_string(), + page_title: Some("Example Publisher".to_string()), + html: r#"Example Publisher"#.to_string(), + script_tags: vec![ + CollectedScriptTag { + src: Some("https://www.googletagmanager.com/gtm.js?id=GTM-ABC123".to_string()), + inline_text: None, + }, + CollectedScriptTag { + src: Some("https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string()), + inline_text: None, + }, + ], + network_requests: vec![CollectedRequest { + url: "https://cdn.publisher.example/app.js".to_string(), + resource_type: Some("script".to_string()), + }], + gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), + warnings: Vec::new(), + } + } + + /// A collected page carrying one discoverable GPT slot, for `run_update_slots`. + fn collected_page_with_header_slot() -> CollectedPage { + let mut collected = collected_page(); + collected.requested_url = "https://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/".to_string(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + collected + } + + fn collected_page_with_ambiguous_slots(url: &str) -> CollectedPage { + let mut collected = collected_page(); + collected.requested_url = url.to_string(); + collected.final_url = url.to_string(); + collected.gpt_slots = vec![ + collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/in-content".to_string(), + div_id: "ad-x-aaaaaaaaaaaaaaaa-0".to_string(), + sizes: vec![(300, 250)], + }, + collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/in-content".to_string(), + div_id: "ad-x-bbbbbbbbbbbbbbbb-1".to_string(), + sizes: vec![(300, 250)], + }, + ]; + collected + } + + fn audited_asset(url: &str, party: AssetParty, integration: Option<&str>) -> AuditedAsset { + AuditedAsset { + kind: "script".to_string(), + url: url.to_string(), + host: Url::parse(url) + .ok() + .and_then(|parsed| parsed.host_str().map(str::to_string)) + .unwrap_or_default(), + party, + integration: integration.map(str::to_string), + } + } + + fn audit_args(url: &str) -> GenerateArgs { + GenerateArgs { + url: url.to_string(), + js_assets: None, + config: None, + no_js_assets: false, + no_config: false, + force: false, + cookies: Vec::new(), + browser: GenerateBrowserOpts::default(), + } + } + + #[test] + fn parse_audit_url_accepts_http_and_https() { + assert!(parse_audit_url("http://publisher.example").is_ok()); + assert!(parse_audit_url("https://publisher.example").is_ok()); + } + + #[test] + fn parse_audit_url_rejects_non_http_schemes() { + for url in [ + "file:///etc/passwd", + "data:text/html,hello", + "chrome://version", + ] { + let error = parse_audit_url(url).expect_err("should reject non-http URL"); + assert!( + format!("{error:?}").contains("only supports http/https"), + "should explain scheme restriction" + ); + } + } + + #[test] + fn repeated_ambiguous_collision_note_is_emitted_once() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + for url in [ + "https://publisher.example/", + "https://publisher.example/news", + ] { + fold_collected( + &mut table, + &Url::parse(url).expect("should parse fixture URL"), + &collected_page_with_ambiguous_slots(url), + "desktop", + &mut notes, + ) + .expect("should fold ambiguous page evidence"); + } + + assert_eq!( + notes.len(), + 1, + "the same site-wide collision guidance should not repeat per page" + ); + } + + #[test] + fn merge_refuses_to_change_policy_used_by_preserved_templates() { + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\nsection_root = \"home\"\nsection_segment = 0\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let inferred = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + + let error = validate_merge_policy(Some(&existing), Some(&inferred), false) + .expect_err("merge must preserve the existing template policy"); + + assert!(format!("{error:?}").contains("--replace")); + validate_merge_policy(Some(&existing), Some(&inferred), true) + .expect("replace is an explicit policy migration"); + } + + #[test] + fn the_consent_stub_note_is_reported_once_and_unscoped() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + for url in [ + "https://publisher.example/", + "https://publisher.example/news", + ] { + let mut page = collected_page(); + page.requested_url = url.to_string(); + page.final_url = url.to_string(); + page.warnings + .push(collector::CONSENT_STUB_WARNING.to_string()); + fold_collected( + &mut table, + &Url::parse(url).expect("should parse fixture URL"), + &page, + "desktop", + &mut notes, + ) + .expect("should fold page evidence"); + } + + assert_eq!( + notes, + [collector::CONSENT_STUB_WARNING.to_string()], + "a run-wide fact should appear once, without a page path" + ); + } + + #[test] + fn page_warnings_remain_distinct_across_profiles() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + let mut page = collected_page(); + page.requested_url = "https://publisher.example/news".to_string(); + page.final_url = page.requested_url.clone(); + page.warnings.push("navigation did not settle".to_string()); + let url = Url::parse(&page.final_url).expect("should parse fixture URL"); + + fold_collected(&mut table, &url, &page, "desktop", &mut notes) + .expect("should fold desktop evidence"); + fold_collected(&mut table, &url, &page, "mobile", &mut notes) + .expect("should fold mobile evidence"); + + assert_eq!( + notes.len(), + 2, + "profile-specific warnings must not collapse" + ); + assert!(notes.iter().any(|note| note.contains("on desktop"))); + assert!(notes.iter().any(|note| note.contains("on mobile"))); + } + + #[test] + fn merge_adopts_the_inferred_policy_when_none_is_configured() { + // A hand-written `{section}` slot with no `section_root` describes a + // config the runtime refuses to load, so the first merge should repair it + // rather than demand `--replace` (which would discard the hand-tuned + // slots it is preserving). + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let inferred = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + + validate_merge_policy(Some(&existing), Some(&inferred), false) + .expect("should have no policy to preserve when section_root is unset"); + } + + #[test] + fn merge_preserves_an_explicit_segment_when_section_root_is_unset() { + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\nsection_segment = 1\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let mismatched = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }; + + let error = validate_merge_policy(Some(&existing), Some(&mismatched), false) + .expect_err("should preserve an explicitly configured segment"); + + assert!(format!("{error:?}").contains("section_segment=1")); + + let matching = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + validate_merge_policy(Some(&existing), Some(&matching), false) + .expect("should adopt a root without changing the configured segment"); + } + + #[test] + fn resolve_output_plan_rejects_no_outputs() { + let mut args = audit_args("https://publisher.example"); + args.no_js_assets = true; + args.no_config = true; + + let error = resolve_output_plan(&args).expect_err("should reject empty output set"); + + assert!( + format!("{error:?}").contains("nothing to do"), + "should explain no-output error" + ); + } + + #[test] + fn resolve_output_plan_rejects_existing_files_without_force() { + let temp = TempDir::new().expect("should create temp dir"); + let path = temp.path().join("js-assets.toml"); + fs::write(&path, "existing").expect("should write existing file"); + let mut args = audit_args("https://publisher.example"); + args.js_assets = Some(path); + args.no_config = true; + + let error = resolve_output_plan(&args).expect_err("should reject overwrite"); + + assert!( + format!("{error:?}").contains("refusing to overwrite"), + "should explain overwrite refusal" + ); + } + + #[test] + fn resolve_output_plan_allows_existing_files_with_force() { + let temp = TempDir::new().expect("should create temp dir"); + let path = temp.path().join("js-assets.toml"); + fs::write(&path, "existing").expect("should write existing file"); + let mut args = audit_args("https://publisher.example"); + args.js_assets = Some(path.clone()); + args.no_config = true; + args.force = true; + + let plan = resolve_output_plan(&args).expect("should allow forced overwrite"); + + assert_eq!(plan.js_assets_path.as_deref(), Some(path.as_path())); + } + + #[test] + fn run_generate_writes_selected_outputs_and_summary() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("audit/js-assets.toml"); + let config = temp.path().join("audit/trusted-server.toml"); + let args = GenerateArgs { + url: "https://publisher.example/page".to_string(), + js_assets: Some(js_assets.clone()), + config: Some(config.clone()), + no_js_assets: false, + no_config: false, + force: false, + cookies: Vec::new(), + browser: GenerateBrowserOpts::default(), + }; + let collector = FakeCollector::new(collected_page()); + let mut out = Vec::new(); + + run_generate(&args, &collector, &mut out).expect("should run audit"); + + assert_eq!(collector.calls.get(), 1, "should collect page once"); + assert!(js_assets.exists(), "should write JS assets"); + assert!(config.exists(), "should write draft config"); + let summary = String::from_utf8(out).expect("summary should be UTF-8"); + assert!(summary.contains("Audited https://publisher.example/page")); + assert!(summary.contains("Detected integrations: google_tag_manager, gpt")); + assert!(summary.contains("Draft config: review before validation and push")); + } + + #[test] + fn run_generate_respects_no_config() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("js-assets.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.js_assets = Some(js_assets.clone()); + args.no_config = true; + let collector = FakeCollector::new(collected_page()); + + run_generate(&args, &collector, &mut Vec::new()).expect("should run audit"); + + assert!(js_assets.exists(), "should write assets"); + assert!( + !temp.path().join("trusted-server.toml").exists(), + "should not write config" + ); + } + + #[test] + fn run_generate_respects_no_js_assets() { + let temp = TempDir::new().expect("should create temp dir"); + let config = temp.path().join("trusted-server.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.config = Some(config.clone()); + args.no_js_assets = true; + let collector = FakeCollector::new(collected_page()); + let mut out = Vec::new(); + + run_generate(&args, &collector, &mut out).expect("should run audit"); + + assert!(config.exists(), "should write config"); + assert!( + !temp.path().join("js-assets.toml").exists(), + "should not write JS assets" + ); + let summary = String::from_utf8(out).expect("summary should be UTF-8"); + assert!(summary.contains("Draft config: review before validation and push")); + } + + #[test] + fn run_generate_writes_collector_warnings_to_asset_artifact() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("js-assets.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.js_assets = Some(js_assets.clone()); + args.no_config = true; + let mut collected = collected_page(); + collected.warnings.push( + "browser audit timed out while waiting for the page to settle; results may be partial" + .to_string(), + ); + let collector = FakeCollector::new(collected); + + run_generate(&args, &collector, &mut Vec::new()).expect("should run audit"); + + let artifact = fs::read_to_string(js_assets).expect("should read artifact"); + assert!( + artifact.contains("results may be partial"), + "should persist collector warning" + ); + } + + #[test] + fn run_generate_conflict_prevents_collection() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("js-assets.toml"); + fs::write(&js_assets, "existing").expect("should write existing file"); + let mut args = audit_args("https://publisher.example/page"); + args.js_assets = Some(js_assets); + args.no_config = true; + let collector = FakeCollector::new(collected_page()); + + let error = run_generate(&args, &collector, &mut Vec::new()) + .expect_err("should reject existing output"); + + assert_eq!(collector.calls.get(), 0, "should not collect page"); + assert!( + format!("{error:?}").contains("refusing to overwrite"), + "should report overwrite conflict" + ); + } + + #[test] + fn build_draft_config_writes_disabled_js_asset_proxy_candidates() { + let url = Url::parse("https://publisher.example/page").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: Some("Example".to_string()), + js_asset_count: 2, + third_party_asset_count: 2, + detected_integrations: vec![DetectedIntegration { + id: "gpt".to_string(), + evidence: "https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string(), + }], + assets: vec![ + audited_asset( + "https://cdn.vendor.example/sdk.js", + AssetParty::ThirdParty, + None, + ), + audited_asset( + "https://securepubads.g.doubleclick.net/tag/js/gpt.js", + AssetParty::ThirdParty, + Some("gpt"), + ), + ], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&[ + "/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js", + "/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js", + ]); + + let draft = build_draft_config_with_generator( + &url, + &artifact, + &gpt_slots::DiscoveredSlots::default(), + &mut generator, + ) + .expect("should build draft config"); + + assert_eq!(draft.js_asset_proxy_candidate_count, 2); + assert!( + draft + .toml + .contains("[integrations.js_asset_proxy]\nenabled = false") + ); + assert!(draft.toml.contains("/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js")); + assert!(draft.toml.contains("/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js")); + assert!( + draft + .toml + .contains("origin_url = \"https://cdn.vendor.example/sdk.js\"") + ); + assert!(draft.toml.contains("proxy = \"disabled\"")); + assert!(draft.toml.contains("Detected integration: gpt")); + assert!( + !draft.toml.contains("example-vendor-loader"), + "should remove the starter placeholder asset" + ); + assert!( + draft.toml.contains( + "# Proxy behavior and first-party asset routing. Kept active with defaults.\n[proxy]" + ), + "should preserve documentation for the section following the replaced block" + ); + toml::from_str::(&draft.toml).expect("draft should parse as TOML"); + } + + #[test] + fn asset_proxy_generation_deduplicates_and_summarizes_skips() { + let artifact = AuditArtifact { + audited_url: "https://publisher.example/page".to_string(), + page_title: None, + js_asset_count: 4, + third_party_asset_count: 3, + detected_integrations: Vec::new(), + assets: vec![ + audited_asset( + "https://cdn.vendor.example/sdk.js", + AssetParty::ThirdParty, + None, + ), + audited_asset( + "https://cdn.vendor.example/sdk.js", + AssetParty::ThirdParty, + None, + ), + audited_asset( + "https://publisher.example/app.js", + AssetParty::FirstParty, + None, + ), + audited_asset( + "http://cdn.vendor.example/insecure.js", + AssetParty::ThirdParty, + None, + ), + ], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&["/assets/111111111111111111111111.js"]); + + let section = + build_js_asset_proxy_section(&artifact, &mut generator).expect("should build section"); + + assert_eq!(section.candidate_count, 1); + assert_eq!( + section + .toml + .matches("[[integrations.js_asset_proxy.assets]]") + .count(), + 1 + ); + assert!(section.toml.contains("# - 1 first-party script")); + assert!(section.toml.contains("# - 1 non-HTTPS third-party script")); + assert!(section.toml.contains("# - 1 duplicate script URL")); + } + + #[test] + fn asset_proxy_generation_with_no_candidates_removes_placeholder_asset() { + let url = Url::parse("https://publisher.example/page").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: None, + js_asset_count: 1, + third_party_asset_count: 0, + detected_integrations: Vec::new(), + assets: vec![audited_asset( + "https://publisher.example/app.js", + AssetParty::FirstParty, + None, + )], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&[]); + + let draft = build_draft_config_with_generator( + &url, + &artifact, + &gpt_slots::DiscoveredSlots::default(), + &mut generator, + ) + .expect("should build draft config"); + + assert_eq!(draft.js_asset_proxy_candidate_count, 0); + assert!( + draft + .toml + .contains("No eligible third-party HTTPS script assets") + ); + assert!( + !draft + .toml + .contains("[[integrations.js_asset_proxy.assets]]") + ); + assert!(!draft.toml.contains("example-vendor-loader")); + } + + #[test] + fn asset_proxy_generation_warns_about_query_string_candidates() { + let artifact = AuditArtifact { + audited_url: "https://publisher.example/page".to_string(), + page_title: None, + js_asset_count: 1, + third_party_asset_count: 1, + detected_integrations: Vec::new(), + assets: vec![audited_asset( + "https://cdn.vendor.example/sdk.js?v=one", + AssetParty::ThirdParty, + None, + )], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&["/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js"]); + + let section = + build_js_asset_proxy_section(&artifact, &mut generator).expect("should build section"); + + assert!( + section + .toml + .contains("query string and must remain stable for proxy matching") + ); + } + + #[test] + fn run_generate_summary_reports_written_asset_proxy_candidates() { + let temp = TempDir::new().expect("should create temp dir"); + let config = temp.path().join("trusted-server.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.config = Some(config); + args.no_js_assets = true; + let collector = FakeCollector::new(collected_page()); + let mut out = Vec::new(); + + run_generate(&args, &collector, &mut out).expect("should run audit"); + + let summary = String::from_utf8(out).expect("summary should be UTF-8"); + assert!(summary.contains("JS asset proxy candidates:")); + assert!(summary.contains("disabled entries written to draft config")); + } + + #[test] + fn build_draft_config_uses_final_url_and_detected_integrations() { + let url = Url::parse("https://www.publisher.example:8443/path").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: Some("Example".to_string()), + js_asset_count: 2, + third_party_asset_count: 2, + detected_integrations: vec![ + DetectedIntegration { + id: "google_tag_manager".to_string(), + evidence: "GTM-ABC123".to_string(), + }, + DetectedIntegration { + id: "gpt".to_string(), + evidence: "https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string(), + }, + DetectedIntegration { + id: "prebid".to_string(), + evidence: "inline script matched `prebid`".to_string(), + }, + ], + assets: Vec::new(), + warnings: Vec::new(), + }; + + let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) + .expect("should build draft config"); + + assert!(draft.contains("domain = \"www.publisher.example\"")); + assert!(draft.contains("cookie_domain = \".www.publisher.example\"")); + assert!(draft.contains("origin_url = \"https://www.publisher.example:8443\"")); + assert!(draft.contains("[integrations.gpt]\nenabled = true")); + assert!(draft.contains("[integrations.google_tag_manager]\nenabled = true")); + assert!(draft.contains("container_id = \"GTM-ABC123\"")); + assert!(draft.contains("Detected prebid")); + toml::from_str::(&draft).expect("draft should parse as TOML"); + } + + #[test] + fn build_draft_config_does_not_enable_gtm_without_container_id() { + let url = Url::parse("https://publisher.example/path").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: None, + js_asset_count: 1, + third_party_asset_count: 1, + detected_integrations: vec![DetectedIntegration { + id: "google_tag_manager".to_string(), + evidence: "https://www.googletagmanager.com/gtm.js".to_string(), + }], + assets: Vec::new(), + warnings: Vec::new(), + }; + + let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) + .expect("should build draft config"); + + assert!(draft.contains("[integrations.google_tag_manager]\nenabled = false")); + assert!(draft.contains("Detected google_tag_manager")); + } + + #[test] + fn build_audit_outputs_reconstructs_creative_opportunity_slots() { + let collected = CollectedPage { + requested_url: "https://example.com/".to_string(), + final_url: "https://example.com/".to_string(), + page_title: Some("Example Publisher".to_string()), + html: "".to_string(), + script_tags: Vec::new(), + network_requests: vec![CollectedRequest { + url: "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Cdesktop%2Chomepage%2Cleaderboard1\ + &prev_iu_szs=970x250%7C4x1%7C620x366\ + &dids=div-gpt-ad-leaderboard-1\ + &prev_scp=baseDivId%3Ddiv-gpt-ad-leaderboard-1%26test%3Dprebid" + .to_string(), + resource_type: Some("fetch".to_string()), + }], + gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), + warnings: Vec::new(), + }; + + let outputs = build_audit_outputs(&collected).expect("should build outputs"); + assert_eq!(outputs.ad_slot_count, 1, "should discover one slot"); + + // The drafted config must be valid TOML with the reconstructed slot. + let value = toml::from_str::(&outputs.draft_config_toml) + .expect("should parse draft config"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("123456789")); + let slot = &creative["slot"][0]; + assert_eq!(slot["id"].as_str(), Some("leaderboard-1")); + assert_eq!( + slot["gam_unit_path"].as_str(), + Some("/123456789/desktop/homepage/leaderboard1") + ); + assert_eq!( + slot["formats"][0]["width"].as_integer(), + Some(970), + "should keep the 970x250 pixel size" + ); + assert!( + slot["providers"]["prebid"].is_table(), + "prev_scp test=prebid should emit a prebid provider" + ); + } + + #[test] + fn render_discovered_slots_escapes_page_controlled_strings() { + // Slot fields scraped from the live page must be escaped so a quote + // cannot inject TOML into the drafted config. + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/head\"er".to_string(), + div_id: "div-gpt-ad-head\"er".to_string(), + sizes: vec![(728, 90)], + }]; + let slots = gpt_slots::discover_gpt_slots(®istry, &[], false); + let url = Url::parse("https://publisher.example/").expect("should parse URL"); + + let rendered = render_discovered_slots(&url, &slots); + + let value = toml::from_str::(&rendered) + .expect("should render valid TOML despite embedded quotes"); + let slot = &value["creative_opportunities"]["slot"][0]; + assert_eq!( + slot["div_id"].as_str(), + Some("div-gpt-ad-head\"er"), + "should keep the quote as data, not TOML syntax" + ); + } + + #[test] + fn update_slots_defaults_pattern_to_final_url_after_redirect() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + // The requested URL redirects; slots are scraped from the final page. + let mut collected = collected_page(); + collected.requested_url = "https://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/news/story".to_string(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + let collector = FakeCollector::new(collected); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse valid TOML"); + let patterns: Vec<&str> = value["creative_opportunities"]["slot"][0]["page_patterns"] + .as_array() + .expect("should have page_patterns array") + .iter() + .map(|entry| entry.as_str().expect("should have pattern string")) + .collect(); + // Patterns come from the post-redirect path: had the requested `/` been + // used, this would be `["/"]`. They now cover the whole section rather + // than only the one article that happened to be scraped. + assert_eq!( + patterns, + ["/news", "/news/*"], + "should derive section patterns from the post-redirect path" + ); + } + + #[test] + fn update_slots_reports_preserved_unobserved_slots_contextually() { + for (scroll, expected_follow_up, unexpected_follow_up) in [ + (false, "or --scroll", "page/profile coverage"), + (true, "page/profile coverage", "or --scroll"), + ] { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = loadable_config() + .replace("gam_network_id = \"123456789\"", "gam_network_id = \"222\""); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"header\"\n\ + div_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"sidebar\"\n\ + div_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\n\ + page_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut notes, + ) + .expect("should preserve unobserved slot"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains( + "preserved 1 configured slot(s) not observed during this crawl: sidebar" + ), + "should name the preserved slot, got {notes:?}" + ); + assert!( + notes.contains(expected_follow_up), + "should suggest the follow-up matching the scroll setting, got {notes:?}" + ); + assert!( + !notes.contains(unexpected_follow_up), + "should omit the follow-up that does not apply, got {notes:?}" + ); + assert!( + notes.contains("discards every hand-written field"), + "should explain the full cost of --replace, got {notes:?}" + ); + assert!(out.is_empty(), "unchanged dry-run stdout should stay empty"); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "dry-run should preserve the original config" + ); + } + } + + #[test] + fn observed_but_refused_slot_is_not_reported_as_unobserved() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = loadable_config(); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"stable\"\n\ + div_id = \"ad-stable\"\n\ + gam_unit_path = \"/123456789/site/header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"ad-refused\"\n\ + gam_unit_path = \"/123456789/desktop/homepage\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + + let page = |profile: &str| { + let mut page = collected_page(); + page.requested_url = "https://publisher.example/".to_string(); + page.final_url = page.requested_url.clone(); + page.gpt_slots = vec![ + collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/header".to_string(), + div_id: "ad-stable".to_string(), + sizes: vec![(728, 90)], + }, + collector::CollectedGptSlot { + gam_unit_path: format!("/123456789/{profile}/homepage"), + div_id: "ad-refused".to_string(), + sizes: vec![(300, 250)], + }, + ]; + page + }; + let desktop = FakeCollector::new(page("desktop")); + let mobile = FakeCollector::new(page("mobile")); + let mut out = Vec::new(); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + &mut notes, + ) + .expect("the accepted slot should let generation complete"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains("skipped refused slot `ad-refused` (`ad-refused`)"), + "should retain the refusal diagnostic, got {notes:?}" + ); + assert!( + !notes.contains("not observed during this crawl: ad-refused"), + "a crawl-observed refused slot must not be labeled unobserved, got {notes:?}" + ); + } + + #[test] + fn ambiguous_configured_stem_is_not_reported_as_unobserved() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = + loadable_config().replace("gam_network_id = \"123456789\"", "gam_network_id = \"222\""); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"in-content\"\n\ + div_id = \"ad-x\"\n\ + gam_unit_path = \"/222/homepage/in-content\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + let mut page = collected_page_with_ambiguous_slots("https://publisher.example/"); + page.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/222/site/header".to_string(), + div_id: "ad-stable".to_string(), + sizes: vec![(728, 90)], + }); + let collector = FakeCollector::new(page); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: true, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut notes, + ) + .expect("should preserve the configured ambiguous placement"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains("skipped ambiguous div-id prefix `ad-x`"), + "should retain the ambiguity diagnostic, got {notes:?}" + ); + assert!( + !notes.contains("not observed during this crawl: in-content"), + "an ambiguity-refused placement must not be labeled unobserved, got {notes:?}" + ); + } + + #[test] + fn volatile_refusals_from_registry_and_requests_keep_prefix_observed() { + for source in ["registry", "request"] { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = loadable_config(); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"stable\"\n\ + div_id = \"ad-stable\"\n\ + gam_unit_path = \"/123456789/site/header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"volatile-family\"\n\ + div_id = \"vendor-tag\"\n\ + gam_unit_path = \"/123456789/site/overlay\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + let mut page = collected_page(); + page.requested_url = "https://publisher.example/".to_string(); + page.final_url = page.requested_url.clone(); + page.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/header".to_string(), + div_id: "ad-stable".to_string(), + sizes: vec![(728, 90)], + }); + let volatile_div = "vendor-tag_1724112345678AbCdEfGh_slot_overlay_1"; + if source == "registry" { + page.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/overlay".to_string(), + div_id: volatile_div.to_string(), + sizes: vec![(300, 250)], + }); + } else { + page.network_requests.push(CollectedRequest { + url: format!( + "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Csite%2Coverlay&dids={volatile_div}\ + &prev_iu_szs=300x250" + ), + resource_type: Some("fetch".to_string()), + }); + } + let collector = FakeCollector::new(page); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: true, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut notes, + ) + .expect("the stable slot should let generation complete"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains("skipped volatile div-id family `vendor-tag`"), + "should retain the {source} volatile refusal, got {notes:?}" + ); + assert!( + !notes.contains("not observed during this crawl: volatile-family"), + "a live configured prefix refused from {source} evidence must stay observed, got {notes:?}" + ); + } + } + + #[test] + fn static_locale_root_slot_uses_the_planned_section_depth_for_patterns() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let nav = ["/en/news"]; + let mut root_page = site_page("https://publisher.example/en", "/123456789/site/root", &nav); + root_page.gpt_slots[0].div_id = "ad-root-only".to_string(); + let collector = SiteCollector::new(vec![ + ("https://publisher.example/en", root_page), + ( + "https://publisher.example/en/news", + site_page( + "https://publisher.example/en/news", + "/123456789/site/static", + &nav, + ), + ), + ]); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/en", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect("should write static locale-root slot"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse config"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("should have slots"); + let target = slots + .iter() + .find(|slot| slot["div_id"].as_str() == Some("ad-header-0")) + .expect("should have the section slot"); + let patterns = target["page_patterns"] + .as_array() + .expect("should have patterns") + .iter() + .map(|pattern| pattern.as_str().expect("should be string")) + .collect::>(); + assert_eq!(patterns, ["/en/news", "/en/news/*"]); + } + + #[test] + fn update_slots_rejects_a_cross_origin_root_redirect() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.final_url = "https://foreign.example/news".to_string(); + let collector = FakeCollector::new(collected); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("cross-origin redirect must leave the requested trust boundary"); + + assert!(format!("{error:?}").contains("cross-origin")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "foreign evidence must not rewrite the config" + ); + } + + #[test] + fn update_slots_skips_a_section_page_that_redirects_off_origin() { + // Only the root navigation was origin-checked before planning. A section + // page that redirects away must not contribute its slots, unit paths or + // page patterns to the generated config either. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let nav = ["/news"]; + let mut root_page = site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ); + root_page.gpt_slots[0].div_id = "ad-root".to_string(); + let mut redirected = site_page( + "https://publisher.example/news", + "/999888777/foreign/news", + &nav, + ); + redirected.final_url = "https://foreign.example/news".to_string(); + redirected.gpt_slots[0].div_id = "ad-foreign".to_string(); + let collector = SiteCollector::new(vec![ + ("https://publisher.example/", root_page), + ("https://publisher.example/news", redirected), + ]); + let mut err = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + scroll: false, + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut err, + ) + .expect("should generate from the same-origin evidence alone"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + assert!( + written.contains("ad-root"), + "same-origin evidence should still be written, got:\n{written}" + ); + assert!( + !written.contains("ad-foreign") && !written.contains("999888777"), + "the redirect destination must not reach the config, got:\n{written}" + ); + let progress = String::from_utf8_lossy(&err); + assert!( + progress.contains( + "skipped `/news` on desktop: it left the audited origin for https://foreign.example" + ), + "the skipped section page should be reported, got:\n{progress}" + ); + } + + #[test] + fn update_slots_skips_a_later_profile_root_that_redirects_off_origin() { + // The later profiles re-walk the plan without a fresh root origin check. + // A mobile root that redirects away carries a foreign ad unit for the + // same div the desktop profile saw; folding it would both write foreign + // inventory and fake a device disagreement on the real slot. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let nav = ["/news"]; + let section_page = |unit_path: &str| { + let mut page = site_page("https://publisher.example/news", unit_path, &nav); + page.gpt_slots[0].div_id = "ad-news".to_string(); + page + }; + let mut desktop_root = site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ); + desktop_root.gpt_slots[0].div_id = "ad-root".to_string(); + let mut mobile_root = site_page( + "https://publisher.example/", + "/999888777/foreign/homepage", + &nav, + ); + mobile_root.gpt_slots[0].div_id = "ad-root".to_string(); + mobile_root.final_url = "https://foreign.example/".to_string(); + let desktop = SiteCollector::new(vec![ + ("https://publisher.example/", desktop_root), + ( + "https://publisher.example/news", + section_page("/123456789/site/news"), + ), + ]); + let mobile = SiteCollector::new(vec![ + ("https://publisher.example/", mobile_root), + ( + "https://publisher.example/news", + section_page("/123456789/site/news"), + ), + ]); + let mut err = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + scroll: false, + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut std::io::sink(), + &mut err, + ) + .expect("the same-origin pages of both profiles agree"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + assert!( + written.contains("/123456789/site/homepage"), + "the same-origin root unit path should be written, got:\n{written}" + ); + assert!( + !written.contains("999888777"), + "the redirect destination must not reach the config, got:\n{written}" + ); + let progress = String::from_utf8_lossy(&err); + assert!( + progress.contains( + "skipped `/` on mobile: it left the audited origin for https://foreign.example" + ), + "the skipped profile root should be reported, got:\n{progress}" + ); + } + + #[test] + fn update_slots_accepts_a_same_host_https_upgrade() { + // The ordinary canonical redirect: an operator types the bare http URL + // and the site upgrades it. The host is unchanged, so the cookie and + // audit trust boundary is unchanged, and generation must not stall on it. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.requested_url = "http://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/".to_string(); + let collector = FakeCollector::new(collected); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "http://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut notes, + ) + .expect("a same-host HTTPS upgrade should not be treated as cross-origin"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse config"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["div_id"].as_str(), + Some("div-gpt-ad-header"), + "evidence from the upgraded root should be written" + ); + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains( + "followed a root redirect from `http://publisher.example/` to \ + `https://publisher.example/`" + ), + "an accepted redirect should say the run switched URLs, got {notes:?}" + ); + } + + #[test] + fn update_slots_rejects_an_https_downgrade_root_redirect() { + // The mirror image of the accepted upgrade: same host, but dropping TLS + // leaves the requested trust boundary and must still be refused. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.final_url = "http://publisher.example/".to_string(); + let collector = FakeCollector::new(collected); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("an HTTPS downgrade must leave the requested trust boundary"); + + assert!(format!("{error:?}").contains("cross-origin")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "downgraded evidence must not rewrite the config" + ); + } + + #[test] + fn update_slots_requires_evidence_from_every_selected_profile() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + let desktop = FakeCollector::new(collected_page_with_header_slot()); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &FailingCollector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("a selected profile with no usable page must refuse generation"); + + assert!(format!("{error:?}").contains("mobile")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "incomplete profile coverage must not rewrite the config" + ); + } + + #[test] + fn update_slots_rejects_invalid_page_pattern_without_touching_config() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["[".to_string()], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect_err("should reject an invalid glob"); + + assert!( + format!("{error:?}").contains("page pattern '['"), + "error should name the offending pattern, got {error:?}" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "a rejected pattern must leave the operator config untouched" + ); + } + + #[test] + fn explicit_page_patterns_refuse_a_template_that_borrows_section_root() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let root = site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ); + let mut news = site_page( + "https://publisher.example/news", + "/123456789/site/news", + &nav, + ); + news.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/news".to_string(), + div_id: "ad-sidebar".to_string(), + sizes: vec![(300, 250)], + }); + let mut deals = site_page( + "https://publisher.example/deals", + "/123456789/site/deals", + &nav, + ); + deals.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/deals".to_string(), + div_id: "ad-sidebar".to_string(), + sizes: vec![(300, 250)], + }); + let collector = SiteCollector::new(vec![ + ("https://publisher.example/", root), + ("https://publisher.example/news", news), + ("https://publisher.example/deals", deals), + ]); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["/".to_string(), "/*".to_string()], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("explicit patterns cannot preserve borrowed-root safety"); + + let message = format!("{error:?}"); + assert!(message.contains("--page-pattern"), "got {message}"); + assert!(message.contains("ad-sidebar"), "got {message}"); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "a refused override must leave the config unchanged" + ); + } + + #[test] + fn update_slots_accepts_double_star_pattern_like_the_runtime() { + // `/20**` does not compile directly but the runtime normalises it to + // `/20*`; validation must accept exactly what the runtime accepts. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["/20**".to_string()], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should accept a runtime-normalisable pattern"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["page_patterns"][0].as_str(), + Some("/20**") + ); + } + + #[test] + fn update_slots_write_replaces_the_config_without_leaving_temp_files() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update slots"); + + let entries: Vec = fs::read_dir(temp.path()) + .expect("should read temp dir") + .map(|entry| { + entry + .expect("should read entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + assert_eq!( + entries, + ["trusted-server.toml"], + "the atomic write should leave no stray temp file behind" + ); + let written = fs::read_to_string(&config_path).expect("should read config"); + toml::from_str::(&written).expect("rewritten config is valid TOML"); + } + + /// A full, loadable config with real secrets substituted, so the write-side + /// validation gate is live rather than downgraded by a broken baseline. + fn loadable_config() -> String { + EXAMPLE_CONFIG + .replace( + "password = \"handler_password\"", + "password = \"test-admin-password-32-bytes-minimum\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"test-proxy-secret-32-bytes-minimum\"", + ) + } + + #[test] + fn a_crawl_writes_a_section_template_and_per_section_patterns() { + // The end-to-end payoff: crawl sections, reconcile the slot across them, + // infer `{section}`, and write a config the runtime loads. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let collector = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/site/news", + &nav, + ), + ), + ( + "https://publisher.example/deals", + site_page( + "https://publisher.example/deals", + "/123456789/site/deals", + &nav, + ), + ), + ]); + let mut out = Vec::new(); + let mut err = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut err, + ) + .expect("should crawl and update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + + assert_eq!( + creative["section_root"].as_str(), + Some("homepage"), + "the unvisited-section fallback should come from the root page" + ); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + let slot = &creative["slot"][0]; + assert_eq!( + slot["gam_unit_path"].as_str(), + Some("/{network_id}/site/{section}"), + "the varying segment should become a template" + ); + let patterns: Vec<&str> = slot["page_patterns"] + .as_array() + .expect("patterns array") + .iter() + .map(|entry| entry.as_str().expect("pattern")) + .collect(); + assert_eq!( + patterns, + ["/", "/deals", "/deals/*", "/news", "/news/*"], + "each witnessed section should contribute both halves of its pair" + ); + + // The whole point of the gate: what was written must actually load. + trusted_server_core::settings::Settings::from_toml(&written) + .expect("generated config must load through the runtime path"); + + let report = String::from_utf8(err).expect("should produce UTF-8 output"); + assert!( + report.contains("Deploy a template-aware binary BEFORE pushing"), + "a templated config must warn about the rollback contract, got:\n{report}" + ); + } + + #[test] + fn disagreeing_device_profiles_refuse_to_write_a_unit_path() { + // Two profiles serving different ad units for the same page is exactly + // the failure a single-profile crawl cannot see. Writing either path + // would be correct for one device and silently wrong for the other. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news"]; + let desktop = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/desktop/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/desktop/news", + &nav, + ), + ), + ]); + let mobile = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/mobile/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/mobile/news", + &nav, + ), + ), + ]); + let mut out = Vec::new(); + let mut err = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + &mut err, + ) + .expect_err("an all-refused crawl must not write an empty slot array"); + + assert!(format!("{error:?}").contains("zero generated slots")); + let progress = String::from_utf8_lossy(&err); + for expected in [ + "Auditing desktop [1/?]: /", + "Auditing desktop: planning site crawl", + "Auditing desktop [2/2]: /news", + "Auditing mobile [1/2]: /", + "Auditing mobile [2/2]: /news", + ] { + assert!( + progress.contains(expected), + "should report `{expected}` while crawling, got:\n{progress}" + ); + } + assert!( + !String::from_utf8_lossy(&out).contains("Auditing "), + "progress must remain on stderr" + ); + assert!( + progress.contains("skipped refused slot"), + "the refusal reason should be reported" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a refused crawl must preserve the operator config" + ); + } + + #[test] + fn a_root_only_site_is_still_collected_on_every_device_profile() { + // A site whose root offers no crawl targets is audited on the root page + // alone. If the later profiles never load it, a device split there is + // invisible and the first profile's literal path gets written as if + // every device agreed with it. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let desktop = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/desktop/homepage", + &[], + ), + )]); + let mobile = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/mobile/homepage", + &[], + ), + )]); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + &mut std::io::sink(), + ) + .expect_err("an all-refused crawl must not write an empty slot array"); + + assert_eq!( + mobile.visited.borrow().as_slice(), + ["https://publisher.example/"], + "the mobile profile must load the root even when there is nothing else to crawl" + ); + assert!(format!("{error:?}").contains("zero generated slots")); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a root-only refusal must preserve the operator config" + ); + } + + #[test] + fn a_crawl_refuses_when_most_pages_are_challenged() { + // Bot protection serves an interstitial that loads fine and has no ad + // stack, so it looks like a page with no slots. Writing from that would + // silently narrow the operator's slot set. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let mut blocked_news = site_page("https://publisher.example/news", "/123456789/x", &nav); + blocked_news.gpt_slots.clear(); + let mut blocked_deals = site_page("https://publisher.example/deals", "/123456789/x", &nav); + blocked_deals.gpt_slots.clear(); + let collector = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + ), + ("https://publisher.example/news", blocked_news), + ("https://publisher.example/deals", blocked_deals), + ]); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect_err("a mostly-challenged crawl should refuse"); + + assert!( + format!("{error:?}").contains("bot protection"), + "the error should name the likely cause, got {error:?}" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a refused run must leave the config untouched" + ); + } + + #[test] + fn max_pages_one_restores_single_page_behavior() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&config_path, loadable_config()).expect("should write config"); + + let nav = ["/news", "/deals"]; + let collector = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + )]); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget { + max_sections: 8, + max_pages: 1, + }, + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update from the single page"); + + assert_eq!( + collector.visited.borrow().len(), + 1, + "max_pages = 1 must not crawl beyond the requested page" + ); + let written = fs::read_to_string(&config_path).expect("read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert!( + value["creative_opportunities"] + .get("section_root") + .is_none(), + "one page cannot witness a section, so no rollback-fatal key may be written" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["gam_unit_path"].as_str(), + Some("/123456789/site/homepage"), + "a single page keeps the literal path" + ); + } + + #[test] + fn generated_config_loads_through_the_runtime_settings_path() { + // The end-to-end contract: whatever `generate` writes must survive the + // same load path the adapter runs at startup. An unloadable config is a + // full-site outage once pushed, not a degraded ad stack. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let baseline = loadable_config(); + trusted_server_core::settings::Settings::from_toml(&baseline) + .expect("test baseline must itself be loadable or the gate is not exercised"); + fs::write(&config_path, &baseline).expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let settings = trusted_server_core::settings::Settings::from_toml(&written) + .expect("generated config must load through the runtime path"); + let creative = settings + .creative_opportunities + .expect("generated config should carry creative opportunities"); + assert_eq!( + creative.slot.len(), + 1, + "the discovered slot should be present after a real load" + ); + assert_eq!( + creative.slot[0].div_id.as_deref(), + Some("div-gpt-ad-header") + ); + } + + #[test] + fn update_slots_dry_run_does_not_persist_environment_overlay_config() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let config = EXAMPLE_CONFIG + .replace( + "password = \"handler_password\"", + "password = \"test-admin-password-32-bytes-minimum\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"test-proxy-secret-32-bytes-minimum\"", + ); + let config = format!( + "{config}\n\ + [[creative_opportunities.slot]]\n\ + id = \"file-only\"\n\ + div_id = \"div-gpt-ad-file\"\n\ + gam_unit_path = \"/123456789/homepage/file\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{{ width = 728, height = 90 }}]\n" + ); + fs::write(&config_path, &config).expect("should write config"); + let args = AppConfigArgs { + app_config: Some(config_path.clone()), + manifest: manifest_path, + no_env: false, + }; + + temp_env::with_var( + "TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__GAM_NETWORK_ID", + Some("987654321"), + || { + let effective = crate::app_config::load_settings(&args) + .expect("should load effective settings"); + assert_eq!( + effective + .settings + .creative_opportunities + .as_ref() + .expect("should have creative config") + .gam_network_id, + "987654321", + "test environment should override the network id" + ); + let loaded = crate::app_config::load_file_settings(&args) + .expect("should load file-only settings"); + let mut collected = collected_page(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/123456789/homepage/file".to_string(), + div_id: "div-gpt-ad-file".to_string(), + sizes: vec![(728, 90)], + }]; + let collector = FakeCollector::new(collected); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &loaded.app_config_path, + existing_creative: loaded.settings.creative_opportunities.as_ref(), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should render dry-run update"); + + let output = String::from_utf8(out).expect("output should be UTF-8"); + assert!(output.starts_with("--- configured creative opportunities\n")); + assert!(output.contains("+++ generated creative opportunities\n")); + assert!( + !output.contains("test-admin-password-32-bytes-minimum"), + "dry run must not expose unrelated secrets" + ); + assert!( + !output.contains("987654321"), + "dry run must not persist environment-only config" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("should re-read config"), + config, + "dry run must not modify the config file" + ); + }, + ); + } + + #[test] + fn update_slots_refuses_to_overwrite_a_config_changed_during_collection() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + let replacement = format!("{original}\n# edited while the browser was running\n"); + fs::write(&config_path, &original).expect("should write config"); + let collector = MutatingCollector { + collected: collected_page_with_header_slot(), + config_path: config_path.clone(), + replacement: replacement.clone(), + }; + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("a stale update should be refused"); + + assert!(format!("{error:?}").contains("changed during the browser audit")); + assert_eq!( + fs::read_to_string(&config_path).expect("should re-read config"), + replacement, + "the concurrent edit must not be overwritten" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs new file mode 100644 index 000000000..740acc86f --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs @@ -0,0 +1,150 @@ +//! Derives `page_patterns` globs from the paths a slot was actually observed on. +//! +//! A slot seen on `/news/story-abc` should serve every article in that section, +//! not just that one URL — but nothing here extrapolates beyond a *witnessed* +//! section. Each observed path contributes the section prefix it belongs to and +//! nothing else, so a crawl that never visited `/reviews` never claims it. +//! +//! Each section yields a pair, because one glob cannot cover both halves: +//! `*` crosses `/` in this glob dialect, so `/news/*` matches `/news/a/b` but +//! **not** the bare `/news` landing page. Emitting only the star form silently +//! drops the landing page from the slot. + +use std::collections::BTreeSet; + +/// The root pattern, matching only the site root. +const ROOT_PATTERN: &str = "/"; + +/// Expands observed page paths into the glob set a slot should carry. +/// +/// `section_segment` is the index the section is taken from, matching the +/// config key of the same name: a path is reduced to its first +/// `section_segment + 1` segments, which is the prefix every page of that +/// section shares. A shorter observed landing path is emitted literally; only +/// the actual site root contributes `/`. +/// +/// Results are deduplicated and ordered with `/` first, then alphabetically, so +/// re-running against unchanged evidence produces an unchanged file. +pub(super) fn patterns_for_paths<'a>( + paths: impl IntoIterator, + section_segment: usize, +) -> Vec { + let mut patterns: BTreeSet = BTreeSet::new(); + let mut has_root = false; + + for path in paths { + let segments: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + if segments.len() <= section_segment { + if segments.is_empty() { + has_root = true; + } else { + patterns.insert(glob::Pattern::escape(path)); + } + continue; + } + let prefix = glob::Pattern::escape(&format!("/{}", segments[..=section_segment].join("/"))); + // The landing page and everything beneath it. + patterns.insert(prefix.clone()); + patterns.insert(format!("{prefix}/*")); + } + + let mut out = Vec::with_capacity(patterns.len() + usize::from(has_root)); + if has_root { + out.push(ROOT_PATTERN.to_string()); + } + out.extend(patterns); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_section_article_yields_both_halves_of_the_pair() { + // `/news/*` alone would not match the bare `/news` landing page, because + // `*` crosses `/` but does not match the empty remainder. + let patterns = patterns_for_paths(["/news/story-abc"], 0); + + assert_eq!(patterns, ["/news", "/news/*"]); + } + + #[test] + fn the_root_path_contributes_the_root_pattern_first() { + let patterns = patterns_for_paths(["/deals/x", "/", "/news/y"], 0); + + assert_eq!( + patterns, + ["/", "/deals", "/deals/*", "/news", "/news/*"], + "root first, then sections alphabetically" + ); + } + + #[test] + fn a_landing_page_and_its_article_collapse_to_one_pair() { + let patterns = patterns_for_paths(["/news", "/news/story-abc"], 0); + + assert_eq!(patterns, ["/news", "/news/*"], "no duplicate entries"); + } + + #[test] + fn a_locale_prefixed_site_keeps_the_locale_in_the_prefix() { + // section_segment = 1 means the section is the second segment, so the + // shared prefix every page of that section carries includes the locale. + let patterns = patterns_for_paths(["/en/news/story", "/en/deals/x", "/en"], 1); + + assert_eq!( + patterns, + ["/en", "/en/deals", "/en/deals/*", "/en/news", "/en/news/*"] + ); + } + + #[test] + fn literal_glob_metacharacters_are_escaped_and_match_the_source() { + let source = "/news[local]/story"; + let patterns = patterns_for_paths([source], 0); + + assert_eq!(patterns, ["/news[[]local[]]", "/news[[]local[]]/*"]); + assert!(patterns.iter().any(|pattern| { + glob::Pattern::new(pattern) + .expect("should compile emitted glob") + .matches(source) + })); + } + + #[test] + fn unwitnessed_sections_are_never_invented() { + let patterns = patterns_for_paths(["/news/story"], 0); + + assert_eq!( + patterns, + ["/news", "/news/*"], + "only the crawled section may appear" + ); + } + + #[test] + fn output_is_stable_regardless_of_input_order() { + let one = patterns_for_paths(["/news/a", "/deals/b", "/"], 0); + let two = patterns_for_paths(["/", "/deals/b", "/news/a"], 0); + + assert_eq!(one, two, "re-running should not reorder the written file"); + } + + #[test] + fn every_emitted_pattern_compiles_as_a_runtime_glob() { + let patterns = patterns_for_paths(["/", "/news/story", "/site-news/x"], 0); + + for pattern in &patterns { + trusted_server_core::creative_opportunities::validate_page_pattern(pattern) + .unwrap_or_else(|error| { + panic!("emitted pattern `{pattern}` must compile: {error}") + }); + } + } + + #[test] + fn no_paths_yield_no_patterns() { + assert!(patterns_for_paths([], 0).is_empty()); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs new file mode 100644 index 000000000..a7b4f1c58 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -0,0 +1,2341 @@ +//! TOML-side slot config: the [`RenderSlot`] model, run merging, rendering, +//! and in-place `[creative_opportunities]` splicing for `ts audit ad-templates +//! generate`. + +use std::collections::{BTreeMap, BTreeSet}; + +use toml_edit::{DocumentMut, Item, Table}; +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, CreativeOpportunitySlot, +}; + +#[cfg(test)] +use crate::commands::audit::generate::gpt_slots; +use crate::error::{CliResult, cli_error, report_error}; + +/// A slot ready to render — the union of discovered and existing fields, without +/// the core type's `pub(crate)` compiled-pattern cache. +#[derive(Debug, Clone)] +pub(super) struct RenderSlot { + id: String, + div_id: Option, + gam_unit_path: Option, + page_patterns: Vec, + /// `(width, height, non-banner media type)`. + formats: Vec<(u32, u32, Option<&'static str>)>, + floor_price: Option, + targeting: BTreeMap, + aps_slot_id: Option, + /// `Some` when the slot runs Prebid; the map is per-bidder params (often empty). + prebid_bidders: Option>, +} + +impl RenderSlot { + /// The stable exact identity fallback used when no configured div prefix + /// matches a discovered slot. + fn key(&self) -> String { + self.div_id + .as_deref() + .unwrap_or(&self.id) + .trim_end_matches('-') + .to_string() + } + + /// Whether this configured slot carries fields that discovery cannot infer. + fn has_tuned_fields(&self) -> bool { + self.floor_price.is_some() + || !self.targeting.is_empty() + || self.aps_slot_id.is_some() + || self.prebid_bidders.is_some() + } + + /// Builds a slot from one page's discovery. + /// + /// Superseded in production by [`RenderSlot::from_evidence`], which reads + /// cross-page evidence; retained as test scaffolding for the merge cases. + #[cfg(test)] + fn from_discovered(slot: &gpt_slots::DiscoveredSlot, patterns: &[String]) -> Self { + Self { + id: slot.id.clone(), + div_id: Some(slot.div_id.clone()), + gam_unit_path: Some(slot.gam_unit_path.clone()), + page_patterns: patterns.to_vec(), + formats: slot + .formats + .iter() + .map(|&(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: slot.has_prebid.then(BTreeMap::new), + } + } + + /// Builds a slot from cross-page evidence and the inferred unit path. + /// + /// Refused inference decisions are filtered before this constructor. A + /// `None` path therefore means inference was unavailable and deliberately + /// leaves the runtime's configured default-path behavior in effect. + pub(super) fn from_evidence( + id: &str, + div_id: &str, + gam_unit_path: Option, + formats: impl IntoIterator, + page_patterns: Vec, + has_prebid: bool, + ) -> Self { + Self { + id: id.to_string(), + div_id: Some(div_id.to_string()), + gam_unit_path, + page_patterns, + formats: formats + .into_iter() + .map(|(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: has_prebid.then(BTreeMap::new), + } + } + + fn from_existing(slot: &CreativeOpportunitySlot) -> Self { + Self { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + gam_unit_path: slot.gam_unit_path.clone(), + page_patterns: slot.page_patterns.clone(), + formats: slot + .formats + .iter() + .map(|format| { + ( + format.width, + format.height, + media_type_label(&format.media_type), + ) + }) + .collect(), + floor_price: slot.floor_price, + targeting: slot + .targeting + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + aps_slot_id: slot.providers.aps.as_ref().map(|aps| aps.slot_id.clone()), + prebid_bidders: slot.providers.prebid.as_ref().map(|prebid| { + prebid + .bidders + .iter() + .map(|(name, params)| (name.clone(), params.clone())) + .collect() + }), + } + } +} + +/// The non-default (non-banner) media-type label to emit, or `None` for banner. +fn media_type_label(media_type: &MediaType) -> Option<&'static str> { + match media_type { + MediaType::Banner => None, + MediaType::Video => Some("video"), + MediaType::Native => Some("native"), + } +} + +/// Merges discovered slots into the existing slot set, keyed by [`RenderSlot::key`]. +/// +/// - `--replace` (or no existing slots): the result is exactly the discovered set. +/// - Otherwise existing slots are preserved (covering other pages / hand-tuned +/// fields); a slot re-seen this run has its page patterns and formats unioned; +/// slots seen only this run are appended. +/// - Format identity includes media type, so equal dimensions observed for two +/// media types remain two intentional entries. +#[cfg(test)] +pub(super) fn merge_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered: &gpt_slots::DiscoveredSlots, + run_patterns: &[String], + replace: bool, +) -> Vec { + let discovered_slots: Vec = discovered + .slots + .iter() + .map(|slot| RenderSlot::from_discovered(slot, run_patterns)) + .collect(); + merge_render_slots(existing, discovered_slots, replace) +} + +/// Merges already-built slots into the existing set. +/// +/// Same reconciliation as the single-page test helper, but the caller supplies the slots — +/// the crawl path builds them from cross-page evidence rather than from one +/// page's discoveries. A slot re-seen this run keeps its configured fields and +/// gains this run's patterns; a genuinely new slot is appended with a +/// non-colliding id. +#[cfg(test)] +pub(super) fn merge_render_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + replace: bool, +) -> Vec { + merge_render_slots_with_diagnostics(existing, discovered_slots, replace).0 +} + +/// Diagnostics produced while merging discovered and configured slots. +#[derive(Debug, Default, PartialEq, Eq)] +pub(super) struct MergeDiagnostics { + /// Operator-facing reconciliation notes. + pub(super) notes: Vec, + /// Configured slots preserved without matching any normalized evidence div. + pub(super) unobserved_existing_slot_ids: Vec, +} + +/// Merges slots and reports prefix collisions and unobserved preserved slots. +#[cfg(test)] +pub(super) fn merge_render_slots_with_diagnostics( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + replace: bool, +) -> (Vec, MergeDiagnostics) { + let observed_div_ids = discovered_slots + .iter() + .filter_map(|slot| slot.div_id.clone()) + .collect::>(); + merge_render_slots_with_observed_diagnostics( + existing, + discovered_slots, + &observed_div_ids, + &observed_div_ids, + replace, + ) +} + +/// Merges renderable slots using normalized evidence div IDs for observation. +/// +/// `observed_div_ids` must be the full normalized evidence set, including divs +/// refused by template inference, skipped as fragments, or refused as +/// ambiguous. `observed_literal_div_ids` contains only concrete live elements; +/// it controls whether a configured div ID remains eligible as a runtime prefix. +/// Passing only the rendered subset for observation can falsely report a live +/// configured slot as unobserved, while treating refused stems as literals can +/// incorrectly disqualify a configured prefix from merge routing. +pub(super) fn merge_render_slots_with_observed_diagnostics( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + observed_div_ids: &[String], + observed_literal_div_ids: &[String], + replace: bool, +) -> (Vec, MergeDiagnostics) { + let existing_slots = existing.map(|config| config.slot.as_slice()).unwrap_or(&[]); + if replace || existing_slots.is_empty() { + return (discovered_slots, MergeDiagnostics::default()); + } + + let observed_literals = observed_literal_div_ids + .iter() + .map(String::as_str) + .collect::>(); + + let mut merged: Vec = existing_slots + .iter() + .map(RenderSlot::from_existing) + .collect(); + let existing_count = merged.len(); + let mut prefix_claims: BTreeMap> = BTreeMap::new(); + let mut split_warnings = BTreeSet::new(); + let mut observed_existing = observed_div_ids + .iter() + .flat_map(|div_id| matching_observed_div_indexes(&merged, div_id, &observed_literals)) + .collect::>(); + for mut slot in discovered_slots { + // Prefix reconciliation is a property of the operator's config, so only + // the slots that were already configured may claim a discovered div. + // Slots this run appended match by exact identity instead, otherwise + // discovery order decides whether `ad-top` swallows a later + // `ad-top-sidebar` and discards its unit path and provider state. + let matched = matching_slot_index(&merged[..existing_count], &slot, &observed_literals) + .or_else(|| { + let key = slot.key(); + merged[existing_count..] + .iter() + .position(|added| added.key() == key) + .map(|offset| offset + existing_count) + }); + if let Some(index) = matched { + if index < existing_count { + observed_existing.insert(index); + } + if index < existing_count + && let (Some(prefix), Some(discovered_div)) = + (merged[index].div_id.as_deref(), slot.div_id.as_deref()) + && discovered_div.starts_with(prefix) + { + prefix_claims + .entry(index) + .or_default() + .insert(discovered_div.to_string()); + } + let present = &mut merged[index]; + for pattern in &slot.page_patterns { + if !present.page_patterns.contains(pattern) { + present.page_patterns.push(pattern.clone()); + } + } + for format in &slot.formats { + if !present.formats.contains(format) { + present.formats.push(*format); + } + } + } else { + if let Some(discovered_div) = slot.div_id.as_deref() { + for parent in merged[..existing_count].iter().filter(|configured| { + configured.div_id.as_deref().is_some_and(|prefix| { + !prefix.is_empty() + && observed_literals.contains(prefix) + && discovered_div != prefix + && discovered_div.starts_with(prefix) + }) && configured.has_tuned_fields() + }) { + split_warnings.insert(format!( + "discovered div `{discovered_div}` was split from configured div_id prefix \ + `{}`; the new slot does not inherit that configured slot's floor price, \ + targeting, or provider settings", + parent.div_id.as_deref().unwrap_or_default(), + )); + } + } + slot.id = unique_slot_id(&slot.id, &merged); + merged.push(slot); + } + } + let notes = split_warnings + .into_iter() + .chain( + prefix_claims + .into_iter() + .filter(|(_, divs)| divs.len() > 1) + .map(|(index, divs)| { + let slot = &merged[index]; + let sample = divs.iter().take(5).cloned().collect::>().join(", "); + let remainder = divs.len().saturating_sub(5); + let suffix = if remainder == 0 { + String::new() + } else { + format!(", and {remainder} more") + }; + format!( + "configured slot `{}` with div_id prefix `{}` matched {} discovered divs \ + ({sample}{suffix}); runtime can resolve this configured slot to at most one \ + active element, so review whether they are distinct placements", + slot.id, + slot.div_id.as_deref().unwrap_or_default(), + divs.len(), + ) + }), + ) + .collect(); + let unobserved_existing_slot_ids = existing_slots + .iter() + .enumerate() + .filter(|(index, _)| !observed_existing.contains(index)) + .map(|(_, slot)| slot.id.clone()) + .collect(); + ( + merged, + MergeDiagnostics { + notes, + unobserved_existing_slot_ids, + }, + ) +} + +fn unique_slot_id(candidate: &str, existing: &[RenderSlot]) -> String { + if existing.iter().all(|slot| slot.id != candidate) { + return candidate.to_string(); + } + + let mut suffix = 2_usize; + loop { + let unique = format!("{candidate}-{suffix}"); + if existing.iter().all(|slot| slot.id != unique) { + return unique; + } + suffix += 1; + } +} + +/// Finds the configured slot matching a discovered normalized slot. +/// +/// Stable-key equality wins first. Otherwise, configured `div_id` values are +/// eligible runtime prefixes unless that value was itself observed as a +/// distinct literal. Equal-length prefix ties retain configuration order. +fn matching_slot_index( + existing: &[RenderSlot], + discovered: &RenderSlot, + observed_literals: &BTreeSet<&str>, +) -> Option { + let key = discovered.key(); + if let Some(index) = existing.iter().position(|slot| slot.key() == key) { + return Some(index); + } + + discovered + .div_id + .as_deref() + .and_then(|div_id| matching_div_id_index(existing, div_id, observed_literals)) +} + +fn matching_div_id_index( + existing: &[RenderSlot], + discovered_div: &str, + observed_literals: &BTreeSet<&str>, +) -> Option { + let mut best = None; + let mut best_length = 0; + for (index, slot) in existing.iter().enumerate() { + let Some(prefix) = slot.div_id.as_deref().filter(|prefix| !prefix.is_empty()) else { + continue; + }; + if observed_literals.contains(prefix) { + continue; + } + if discovered_div.starts_with(prefix) && prefix.len() > best_length { + best = Some(index); + best_length = prefix.len(); + } + } + best +} + +/// Finds every configured slot that can resolve to one normalized evidence div. +/// +/// Merge routing remains exact-then-longest-prefix through +/// [`matching_slot_index`], but observation is deliberately multi-match: an +/// exact configured slot and every eligible broad prefix are all live when the +/// element exists. +fn matching_observed_div_indexes<'a>( + existing: &'a [RenderSlot], + discovered_div: &'a str, + observed_literals: &'a BTreeSet<&'a str>, +) -> impl Iterator + 'a { + let discovered_key = discovered_div.trim_end_matches('-'); + existing + .iter() + .enumerate() + .filter_map(move |(index, slot)| { + if slot.key() == discovered_key { + return Some(index); + } + let prefix = slot.div_id.as_deref().filter(|prefix| !prefix.is_empty())?; + (!observed_literals.contains(prefix) && discovered_div.starts_with(prefix)) + .then_some(index) + }) +} + +/// Header comment emitted above the structurally replaced managed slot array. +const MANAGED_SLOTS_COMMENT: &str = "# Slots managed by `ts audit ad-templates generate`."; +/// Second line of the managed-slot header comment. +const MANAGED_SLOTS_REVIEW_COMMENT: &str = + "# Review page_patterns and formats before validating/pushing."; + +/// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. +pub(super) fn render_slots(slots: &[RenderSlot]) -> String { + let mut out = format!("\n{MANAGED_SLOTS_COMMENT}\n{MANAGED_SLOTS_REVIEW_COMMENT}\n"); + for slot in slots { + out.push_str("\n[[creative_opportunities.slot]]\n"); + out.push_str(&format!("id = {}\n", toml_string(&slot.id))); + if let Some(div_id) = &slot.div_id { + out.push_str(&format!("div_id = {}\n", toml_string(div_id))); + } + if let Some(path) = &slot.gam_unit_path { + out.push_str(&format!("gam_unit_path = {}\n", toml_string(path))); + } + out.push_str("page_patterns = [\n"); + for pattern in &slot.page_patterns { + out.push_str(&format!(" {},\n", toml_string(pattern))); + } + out.push_str("]\n"); + out.push_str("formats = [\n"); + for (width, height, media_type) in &slot.formats { + let rendered = match media_type { + Some(kind) => { + format!("{{ width = {width}, height = {height}, media_type = \"{kind}\" }}") + } + None => format!("{{ width = {width}, height = {height} }}"), + }; + out.push_str(&format!(" {rendered},\n")); + } + out.push_str("]\n"); + if let Some(floor) = slot.floor_price { + // `f64` Display prints `NaN`, which is not valid TOML (`nan` is); + // normalize non-finite values so the spliced config stays parseable. + if floor.is_finite() { + out.push_str(&format!("floor_price = {floor}\n")); + } else if floor.is_nan() { + out.push_str("floor_price = nan\n"); + } else if floor.is_sign_positive() { + out.push_str("floor_price = inf\n"); + } else { + out.push_str("floor_price = -inf\n"); + } + } + if !slot.targeting.is_empty() { + let pairs = slot + .targeting + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_string(value))) + .collect::>() + .join(", "); + out.push_str(&format!("targeting = {{ {pairs} }}\n")); + } + if let Some(slot_id) = &slot.aps_slot_id { + out.push_str("[creative_opportunities.slot.providers.aps]\n"); + out.push_str(&format!("slot_id = {}\n", toml_string(slot_id))); + } + if let Some(bidders) = &slot.prebid_bidders { + out.push_str("[creative_opportunities.slot.providers.prebid]\n"); + let rendered = bidders + .iter() + .map(|(name, params)| format!("{} = {}", toml_key(name), toml_inline_value(params))) + .collect::>() + .join(", "); + if rendered.is_empty() { + out.push_str("bidders = {}\n"); + } else { + out.push_str(&format!("bidders = {{ {rendered} }}\n")); + } + } + } + out +} + +/// Quotes and escapes a string as a TOML basic string, including control chars. +pub(super) fn toml_string(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for ch in value.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + // TOML basic strings reject U+0000..U+001F and DEL (U+007F). + control if (control as u32) < 0x20 || control == '\u{7f}' => { + out.push_str(&format!("\\u{:04X}", control as u32)); + } + other => out.push(other), + } + } + out.push('"'); + out +} + +/// Renders a TOML table key: bare when it is a valid bare key, else a quoted key. +fn toml_key(key: &str) -> String { + let is_bare = !key.is_empty() + && key + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-'); + if is_bare { + key.to_string() + } else { + toml_string(key) + } +} + +/// Renders a JSON value as a compact inline TOML value (for prebid bidder params). +fn toml_inline_value(value: &serde_json::Value) -> String { + match value { + serde_json::Value::Null => "{}".to_string(), + serde_json::Value::Bool(bool) => bool.to_string(), + serde_json::Value::Number(number) => number.to_string(), + serde_json::Value::String(string) => toml_string(string), + serde_json::Value::Array(items) => { + let rendered = items + .iter() + .map(toml_inline_value) + .collect::>() + .join(", "); + format!("[{rendered}]") + } + serde_json::Value::Object(map) => { + let rendered = map + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_inline_value(value))) + .collect::>() + .join(", "); + format!("{{ {rendered} }}") + } + } +} + +/// The config-level values a splice writes alongside the slot array. +#[derive(Debug, Clone, Default)] +pub(super) struct CreativeSectionKeys<'a> { + /// GAM network id, when one was resolved. + pub(super) network_id: Option<&'a str>, + /// `section_root`, written only when a slot uses a `{section}` template. + pub(super) section_root: Option<&'a str>, + /// `section_segment`, written only alongside `section_root`. + pub(super) section_segment: Option, +} + +fn max_table_position(table: &Table) -> Option { + table.iter().fold(table.position(), |maximum, (_, item)| { + let child_maximum = match item { + Item::Table(child) => max_table_position(child), + Item::ArrayOfTables(array) => array.iter().filter_map(max_table_position).max(), + Item::None | Item::Value(_) => None, + }; + maximum.max(child_maximum) + }) +} + +fn set_table_position_recursive(table: &mut Table, position: isize) { + table.set_position(position); + for (_, item) in table.iter_mut() { + match item { + Item::Table(child) => set_table_position_recursive(child, position), + Item::ArrayOfTables(array) => { + for child in array.iter_mut() { + set_table_position_recursive(child, position); + } + } + Item::None | Item::Value(_) => {} + } + } +} + +/// Structurally replaces the generator-managed creative-opportunities fields. +/// +/// All unrelated TOML items and their decorations remain in the parsed +/// document. Missing inferred scalar values preserve their existing values; a +/// fresh section is created only when a network id is available. +pub(super) fn splice_creative_slots( + existing: &str, + keys: &CreativeSectionKeys<'_>, + rendered_slots: &str, +) -> CliResult { + let mut document = existing.parse::().map_err(|error| { + report_error(format!( + "failed to parse target config before updating slots: {error}" + )) + })?; + let had_section = document.get("creative_opportunities").is_some(); + let existing_section_position = document + .get("creative_opportunities") + .and_then(Item::as_table) + .and_then(Table::position); + let section_position = existing_section_position + .unwrap_or_else(|| max_table_position(document.as_table()).unwrap_or(0) + 1); + if !had_section && keys.network_id.is_none() { + return cli_error( + "refusing to create a `[creative_opportunities]` section without a \ + GAM network id: none could be determined from the audited page, and \ + the key is required. Add `[creative_opportunities]` with a \ + `gam_network_id` to the config and re-run", + ); + } + + let generated = format!( + "[creative_opportunities]\n{}\n", + rendered_slots.trim_matches('\n') + ); + let mut generated = generated + .parse::() + .map_err(|error| report_error(format!("failed to parse generated slot tables: {error}")))?; + let mut generated_slots = generated["creative_opportunities"] + .as_table_mut() + .and_then(|table| table.remove("slot")) + .unwrap_or_else(|| Item::ArrayOfTables(toml_edit::ArrayOfTables::new())); + if let Item::ArrayOfTables(array) = &mut generated_slots { + for table in array.iter_mut() { + set_table_position_recursive(table, section_position); + } + } + + if !had_section { + document["creative_opportunities"] = Item::Table(toml_edit::Table::new()); + } + let creative = document["creative_opportunities"] + .as_table_mut() + .ok_or_else(|| { + report_error( + "target config's `creative_opportunities` value is not an editable table; \ + rewrite it as a `[creative_opportunities]` table and re-run", + ) + })?; + // `toml_edit` stably sorts tables by document position. Imported tables + // retain positions from their source document, so anchor the whole subtree + // here to keep the parent, slots, and provider tables together. + creative.set_position(section_position); + if let Some(network_id) = keys.network_id { + creative["gam_network_id"] = toml_edit::value(network_id); + } + if let Some(section_root) = keys.section_root { + creative["section_root"] = toml_edit::value(section_root); + if let Some(section_segment) = keys.section_segment { + creative["section_segment"] = toml_edit::value(section_segment as i64); + } + } + creative.insert("slot", generated_slots); + + let mut result = document.to_string(); + if uses_crlf(existing) { + result = convert_document_lf_to_crlf(&result); + } + ensure_only_managed_fields_changed(existing, &result)?; + Ok(result) +} + +/// Verifies that the structural update changed only generator-managed fields. +fn ensure_only_managed_fields_changed(before: &str, after: &str) -> CliResult<()> { + fn unmanaged(document: &str) -> CliResult { + let mut value = toml::from_str::(document) + .map_err(|error| report_error(format!("failed to validate updated config: {error}")))?; + if let Some(root) = value.as_table_mut() { + let remove_empty = if let Some(creative) = root + .get_mut("creative_opportunities") + .and_then(toml::Value::as_table_mut) + { + for key in ["slot", "gam_network_id", "section_root", "section_segment"] { + creative.remove(key); + } + creative.is_empty() + } else { + false + }; + if remove_empty { + root.remove("creative_opportunities"); + } + } + Ok(value) + } + + if unmanaged(before)? != unmanaged(after)? { + return cli_error( + "refusing to update config because fields outside the managed \ + creative-opportunities keys would change", + ); + } + Ok(()) +} + +/// Byte offsets of the `\n` bytes that terminate a document line. +/// +/// Only newlines outside comments and string values delimit lines, so the scan +/// skips a `#` comment to end of line, skips single-line basic and literal +/// strings, and tracks multiline `"""` / `'''` bodies. Without the comment and +/// single-line-string cases a stray triple quote desynchronizes the scan and the +/// document's line endings are flipped or left mixed — a rewrite +/// [`ensure_only_managed_fields_changed`] cannot catch, because it compares +/// parsed values. +fn document_newlines(document: &str) -> Vec { + let bytes = document.as_bytes(); + let mut newlines = Vec::new(); + let mut index = 0_usize; + while index < bytes.len() { + match bytes[index] { + b'#' => { + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + } + b'\n' => { + newlines.push(index); + index += 1; + } + quote @ (b'"' | b'\'') => { + if bytes[index..].starts_with(&[quote, quote, quote]) { + index += 3; + while index < bytes.len() && !bytes[index..].starts_with(&[quote, quote, quote]) + { + index += 1; + } + index = index.saturating_add(3).min(bytes.len()); + } else { + index += 1; + while index < bytes.len() && bytes[index] != quote && bytes[index] != b'\n' { + index += if quote == b'"' && bytes[index] == b'\\' { + 2 + } else { + 1 + }; + } + if index < bytes.len() && bytes[index] == quote { + index += 1; + } + } + } + _ => index += 1, + } + } + newlines +} + +/// Whether `document` uses CRLF line endings (so edits preserve them). +fn uses_crlf(document: &str) -> bool { + let bytes = document.as_bytes(); + document_newlines(document) + .first() + .is_some_and(|&index| index > 0 && bytes[index - 1] == b'\r') +} + +/// Converts document line terminators while leaving string content intact. +fn convert_document_lf_to_crlf(document: &str) -> String { + let bytes = document.as_bytes(); + let mut output = String::with_capacity(document.len()); + let mut previous = 0_usize; + for index in document_newlines(document) { + output.push_str(&document[previous..index]); + if index == 0 || bytes[index - 1] != b'\r' { + output.push('\r'); + } + output.push('\n'); + previous = index + 1; + } + output.push_str(&document[previous..]); + output +} + +/// Strips a trailing inline `# comment` from a candidate table-header line. +/// +/// Only valid on header candidates: header lines cannot contain `#` before the +/// closing bracket unless it is inside a quoted key, which the configs this +/// updater manages never use. +fn strip_inline_comment(line: &str) -> &str { + match line.find('#') { + Some(position) => line[..position].trim_end(), + None => line, + } +} + +pub(super) fn replace_key_in_section( + document: &str, + section: &str, + key: &str, + replacement_line: &str, +) -> CliResult { + let section_header = format!("[{section}]"); + let mut in_section = false; + let mut replaced = false; + let mut saw_section = false; + let mut lines = Vec::new(); + + for line in document.lines() { + let trimmed = line.trim(); + let header_candidate = strip_inline_comment(trimmed); + if header_candidate.starts_with('[') && header_candidate.ends_with(']') { + in_section = header_candidate == section_header; + saw_section |= in_section; + } + + if in_section && !replaced && is_key_line(trimmed, key) { + lines.push(replacement_line.to_string()); + replaced = true; + } else { + lines.push(line.to_string()); + } + } + + if !saw_section { + return cli_error(format!( + "failed to update starter config because section `{section_header}` was not found" + )); + } + if !replaced { + return cli_error(format!( + "failed to update starter config because key `{key}` was not found in `{section_header}`" + )); + } + + let mut output = lines.join("\n"); + if document.ends_with('\n') { + output.push('\n'); + } + if uses_crlf(document) { + // `lines()` stripped the `\r`s; restore the document's CRLF endings. + output = output.replace("\r\n", "\n").replace('\n', "\r\n"); + } + Ok(output) +} + +fn is_key_line(trimmed_line: &str, key: &str) -> bool { + trimmed_line + .strip_prefix(key) + .and_then(|remaining| remaining.trim_start().strip_prefix('=')) + .is_some() +} + +/// Chooses the `gam_network_id` to write. +/// +/// The existing id is kept only when a real merge preserves existing slots. +/// On `--replace`, or when the config had no slots (e.g. a placeholder +/// `[creative_opportunities]` section), the discovered id wins — mirroring +/// the slot merge, which returns discovered-only in those cases. +pub(super) fn resolve_network_id( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_network_id: Option<&str>, + replace: bool, +) -> Option { + let existing_network_id = existing.map(|config| config.gam_network_id.clone()); + let preserving_existing = !replace && existing.is_some_and(|config| !config.slot.is_empty()); + if preserving_existing { + existing_network_id.or_else(|| discovered_network_id.map(str::to_string)) + } else { + discovered_network_id + .map(str::to_string) + .or(existing_network_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector; + + fn discovered_header_slot() -> gpt_slots::DiscoveredSlots { + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + gpt_slots::discover_gpt_slots(®istry, &[], false) + } + + /// Rendered slot text for the discovered header slot, patterns = `/`. + fn header_rendered() -> String { + let merged = merge_slots(None, &discovered_header_slot(), &["/".to_string()], true); + render_slots(&merged) + } + + fn two_provider_slots_rendered() -> &'static str { + r#" +# Slots managed by `ts audit ad-templates generate`. +# Review page_patterns and formats before validating/pushing. + +[[creative_opportunities.slot]] +id = "header" +div_id = "header" +gam_unit_path = "/222/{section}/header" +page_patterns = ["/"] +formats = [{ width = 728, height = 90 }] +[creative_opportunities.slot.providers.prebid] +bidders = {} + +[[creative_opportunities.slot]] +id = "sidebar" +div_id = "sidebar" +gam_unit_path = "/222/{section}/sidebar" +page_patterns = ["/"] +formats = [{ width = 300, height = 250 }] +[creative_opportunities.slot.providers.aps] +slot_id = "sidebar" +"# + } + + fn table_headers(document: &str) -> Vec<&str> { + document + .lines() + .map(str::trim) + .filter(|line| line.starts_with('[')) + .collect() + } + + /// Section keys carrying only a network id, the common test case. + fn network_keys(network_id: &str) -> CreativeSectionKeys<'_> { + CreativeSectionKeys { + network_id: Some(network_id), + ..CreativeSectionKeys::default() + } + } + + fn existing_config(toml_str: &str) -> CreativeOpportunitiesConfig { + toml::from_str::(toml_str).expect("valid creative config") + } + + #[test] + fn splice_replaces_slots_and_preserves_other_sections() { + let existing = "[publisher]\ndomain = \"x\"\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\nprice_granularity = \"dense\"\n\n\ + [[creative_opportunities.slot]]\nid = \"old\"\ndiv_id = \"old\"\n\ + gam_unit_path = \"/111/old\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + out.contains("gam_network_id = \"222\""), + "network id updated" + ); + assert!(!out.contains("id = \"old\""), "old slot removed"); + assert!( + out.contains("gam_unit_path = \"/222/homepage/header\""), + "new slot written" + ); + assert!( + out.contains("[publisher]") && out.contains("domain = \"x\""), + "publisher section preserved" + ); + assert!( + out.contains("[auction]") && out.contains("enabled = true"), + "trailing auction section preserved" + ); + toml::from_str::(&out).expect("spliced config is valid TOML"); + } + + #[test] + fn splice_updates_a_quoted_section_header_structurally() { + let existing = "[\"creative_opportunities\"]\ngam_network_id = \"111\"\n"; + + let updated = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should update quoted table structurally"); + + assert_eq!(updated.matches("creative_opportunities").count(), 2); + assert!(updated.contains("gam_network_id = \"222\"")); + toml::from_str::(&updated).expect("should remain valid TOML"); + } + + #[test] + fn splice_preserves_multiline_values_comments_and_noncontiguous_tables() { + let existing = "title = \"publisher\" # keep this comment\n\ + description = \"\"\"a line that looks like [creative_opportunities]\n\ + and another [[creative_opportunities.slot]] line\"\"\"\n\ + dimensions = [\n 300,\n 250,\n]\n\n\ + [creative_opportunities] # managed section\n\ + gam_network_id = \"111\" # old network\n\n\ + [[creative_opportunities.slot]]\nid = \"old-a\"\ndiv_id = \"old-a\"\n\ + gam_unit_path = \"/111/a\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [auction]\nenabled = true # keep auction comment\n\n\ + [[creative_opportunities.slot]]\nid = \"old-b\"\ndiv_id = \"old-b\"\n\ + gam_unit_path = \"/111/b\"\npage_patterns = [\"/b\"]\n\ + formats = [{ width = 320, height = 50 }]\n"; + + let updated = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should update structurally"); + + assert!(updated.contains("looks like [creative_opportunities]")); + assert!(updated.contains("dimensions = [\n 300,\n 250,\n]")); + assert!(updated.contains("enabled = true # keep auction comment")); + assert!(!updated.contains("id = \"old-a\"")); + assert!(!updated.contains("id = \"old-b\"")); + let value = toml::from_str::(&updated).expect("should remain valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"] + .as_array() + .map(Vec::len), + Some(1) + ); + } + + #[test] + fn splice_keeps_generated_slots_and_providers_contiguous() { + let existing = "[publisher]\ndomain = \"example.com\"\n\n\ + [tester_cookie]\nenabled = true\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\n\n\ + [debug]\nauction_html_comment = true\n"; + + let updated = splice_creative_slots( + existing, + &network_keys("222"), + two_provider_slots_rendered(), + ) + .expect("should splice slots"); + + assert_eq!( + table_headers(&updated), + vec![ + "[publisher]", + "[tester_cookie]", + "[creative_opportunities]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.prebid]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.aps]", + "[debug]", + ] + ); + } + + #[test] + fn splice_groups_a_new_creative_section_with_its_slots() { + let existing = "[publisher]\ndomain = \"example.com\"\n\n\ + [debug]\nauction_html_comment = true\n\n\ + [auction]\nenabled = true\n"; + + let updated = splice_creative_slots( + existing, + &network_keys("222"), + two_provider_slots_rendered(), + ) + .expect("should create creative section and splice slots"); + + assert_eq!( + table_headers(&updated), + vec![ + "[publisher]", + "[debug]", + "[auction]", + "[creative_opportunities]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.prebid]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.aps]", + ] + ); + } + + #[test] + fn splice_rejects_top_level_inline_creative_opportunities_table() { + let existing = "creative_opportunities = { gam_network_id = \"111\" }\n"; + + let error = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect_err("should refuse a top-level inline table"); + + assert!( + format!("{error:?}").contains("rewrite it as"), + "error should tell the operator to rewrite the section, got {error:?}" + ); + } + + /// Section keys for a templated run: network id plus the section policy. + fn template_keys<'a>( + network_id: &'a str, + root: &'a str, + segment: usize, + ) -> CreativeSectionKeys<'a> { + CreativeSectionKeys { + network_id: Some(network_id), + section_root: Some(root), + section_segment: Some(segment), + } + } + + #[test] + fn splice_inserts_section_policy_keys_a_config_does_not_have_yet() { + // The whole point of `upsert`: every config predating templating lacks + // these keys, so a replace-only writer could never add them. + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots( + existing, + &template_keys("222", "homepage", 0), + &header_rendered(), + ) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("222")); + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "inserting must not disturb later sections" + ); + } + + #[test] + fn splice_replaces_section_policy_keys_that_are_already_present() { + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\ + section_root = \"old\"\nsection_segment = 2\n"; + + let out = splice_creative_slots( + existing, + &template_keys("111", "homepage", 1), + &header_rendered(), + ) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(1)); + assert_eq!( + out.matches("section_root").count(), + 1, + "the key must be replaced, not duplicated" + ); + } + + #[test] + fn splice_omits_section_policy_when_no_slot_needs_it() { + // `section_root`/`section_segment` are `deny_unknown_fields` additions: + // writing them into a config that does not need them would make it + // unloadable by an older binary for no benefit. + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + !out.contains("section_root") && !out.contains("section_segment"), + "an untemplated run must not add rollback-fatal keys, got:\n{out}" + ); + } + + #[test] + fn splice_writes_section_policy_into_a_freshly_created_section() { + let existing = "[publisher]\ndomain = \"x\"\n"; + + let out = splice_creative_slots( + existing, + &template_keys("222", "homepage", 0), + &header_rendered(), + ) + .expect("should append a fresh section"); + + let value = toml::from_str::(&out).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("222")); + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + } + + #[test] + fn splice_refuses_fresh_section_without_a_network_id() { + // Reachable whenever the scraped unit path has no all-digit leading + // segment (MCM/child-network paths). Writing the section anyway produces + // a config missing a required field, which fails load and takes every + // route to the startup error router once pushed. + let existing = "[publisher]\ndomain = \"x\"\n"; + + let error = splice_creative_slots( + existing, + &CreativeSectionKeys::default(), + &header_rendered(), + ) + .expect_err("should refuse to create a section with no network id"); + + assert!( + format!("{error:?}").contains("without a GAM network id"), + "error should name the missing network id, got {error:?}" + ); + } + + #[test] + fn splice_appends_section_when_config_has_none() { + let existing = "[publisher]\ndomain = \"x\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should append a fresh section"); + + let value = toml::from_str::(&out).expect("appended config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222") + ); + } + + #[test] + fn splice_preserves_section_scalars_and_provider_subtables() { + // Mirrors the templated operator shape: section policy scalars in the + // head block and a per-slot prebid provider subtable. + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + auction_timeout_ms = 2000\n\ + section_root = \"homepage\"\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"ad-header-0\"\n\ + div_id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\ + [creative_opportunities.slot.providers.prebid]\n\ + bidders = {}\n\n\ + [auction]\nenabled = true\n"; + let existing_config = existing_config( + &existing + .replace("[creative_opportunities]\n", "") + .replace("[[creative_opportunities.slot]]", "[[slot]]") + .replace("[creative_opportunities.slot.", "[slot.") + .replace("\n[auction]\nenabled = true\n", ""), + ); + let discovered = discovered_header_slot(); + let merged = merge_slots( + Some(&existing_config), + &discovered, + &["/news/*".to_string()], + false, + ); + + let out = splice_creative_slots(existing, &network_keys("111"), &render_slots(&merged)) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!( + creative["section_root"].as_str(), + Some("homepage"), + "section policy scalars must survive the splice" + ); + assert_eq!(creative["auction_timeout_ms"].as_integer(), Some(2000)); + assert_eq!( + creative["slot"][0]["gam_unit_path"].as_str(), + Some("/{network_id}/example/{section}"), + "an existing templated unit path must not be rewritten to a literal" + ); + assert!( + creative["slot"][0]["providers"]["prebid"]["bidders"].is_table(), + "the prebid provider subtable must be re-emitted" + ); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "trailing sections must be preserved" + ); + } + + #[test] + fn splice_preserves_crlf_line_endings() { + let existing = "[creative_opportunities]\r\ngam_network_id = \"111\"\r\n\r\n\ + [auction]\r\nenabled = true\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "every line ending should stay CRLF" + ); + let value = toml::from_str::(&out).expect("spliced CRLF config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "network id updated in CRLF config" + ); + } + + #[test] + fn splice_does_not_infer_document_endings_from_multiline_string_content() { + let existing = "[publisher]\nother = \"\"\"a\r\nb\"\"\"\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice LF document"); + + assert!( + out.contains("[publisher]\nother"), + "an embedded CRLF must not convert document line endings" + ); + assert!( + out.contains("a\r\nb"), + "an unrelated multiline string value must remain byte-identical" + ); + } + + #[test] + fn a_triple_quote_in_a_comment_does_not_desynchronize_the_line_scan() { + // A `"""` inside a comment is not a multiline string. Treating it as one + // makes the rest of the document read as string content, so a CRLF file + // is detected as LF and gets rewritten wholesale. + let existing = "# see \"\"\" docs\r\n[creative_opportunities]\r\n\ + gam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "the document's CRLF endings must survive a triple quote in a comment, got {out:?}" + ); + } + + #[test] + fn a_triple_quote_in_a_single_line_string_does_not_desynchronize_the_line_scan() { + let existing = "[publisher]\r\nlabel = 'a \"\"\" b'\r\n\r\n\ + [creative_opportunities]\r\ngam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "the document's CRLF endings must survive a triple quote in a value, got {out:?}" + ); + } + + #[test] + fn splice_does_not_rewrite_bare_lf_inside_crlf_multiline_string() { + let existing = "[publisher]\r\nother = \"\"\"a\nb\"\"\"\r\n\r\n\ + [creative_opportunities]\r\ngam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + out.contains("a\nb"), + "a bare LF inside an unrelated multiline value must remain unchanged" + ); + } + + #[test] + fn render_slots_writes_non_finite_floor_price_as_valid_toml() { + let slot = RenderSlot { + id: "header".to_string(), + div_id: Some("div-gpt-ad-header".to_string()), + gam_unit_path: Some("/222/homepage/header".to_string()), + page_patterns: vec!["/".to_string()], + formats: vec![(728, 90, None)], + floor_price: Some(f64::NAN), + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: None, + }; + + let rendered = render_slots(&[slot]); + + assert!( + rendered.contains("floor_price = nan"), + "NaN should render as TOML `nan`, not Rust `NaN`" + ); + toml::from_str::(&rendered).expect("rendered slots are valid TOML"); + } + + #[test] + fn render_slots_formats_long_arrays_across_indented_lines() { + let slot = RenderSlot { + id: "header".to_string(), + div_id: Some("div-gpt-ad-header".to_string()), + gam_unit_path: Some("/222/homepage/header".to_string()), + page_patterns: vec!["/".to_string(), "/news".to_string(), "/news/*".to_string()], + formats: vec![(728, 90, None), (970, 250, None), (300, 250, None)], + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: None, + }; + + let rendered = render_slots(&[slot]); + + assert!( + rendered.contains("page_patterns = [\n \"/\",\n \"/news\",\n \"/news/*\",\n]\n"), + "page patterns should be readable one-per-line" + ); + assert!( + rendered.contains( + "formats = [\n { width = 728, height = 90 },\n \ + { width = 970, height = 250 },\n \ + { width = 300, height = 250 },\n]\n" + ), + "formats should be readable one-per-line" + ); + toml::from_str::(&rendered).expect("formatted slots are valid TOML"); + } + + #[test] + fn splice_creates_section_when_absent() { + // Config with no [creative_opportunities] at all — generate should append it. + let existing = "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "appended section carries the discovered network id" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert!( + value["publisher"]["domain"].as_str() == Some("x") + && value["auction"]["enabled"].as_bool() == Some(true), + "existing sections preserved when appending" + ); + } + + #[test] + fn resplice_does_not_accumulate_managed_comment() { + // A re-run splices into a config that already carries the managed + // header comment; it must keep exactly one copy, not append another. + let first = splice_creative_slots( + "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n", + &network_keys("222"), + &header_rendered(), + ) + .expect("first splice"); + let second = splice_creative_slots(&first, &network_keys("222"), &header_rendered()) + .expect("second splice"); + let third = splice_creative_slots(&second, &network_keys("222"), &header_rendered()) + .expect("third splice"); + + assert_eq!( + third + .lines() + .filter(|line| line.trim() == MANAGED_SLOTS_COMMENT) + .count(), + 1, + "managed header comment must not accumulate across re-splices" + ); + toml::from_str::(&third).expect("re-spliced config stays valid TOML"); + } + + #[test] + fn splice_recognizes_inline_commented_section_header() { + // `[creative_opportunities] # comment` is valid TOML; the splice must + // update it in place instead of appending a duplicate section. + let existing = "[creative_opportunities] # ad templates\ngam_network_id = \"111\"\n\n\ + [auction] # flags\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert_eq!( + out.lines() + .filter(|line| { strip_inline_comment(line.trim()) == "[creative_opportunities]" }) + .count(), + 1, + "commented header must not be duplicated" + ); + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "network id updated under a commented header" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "commented trailing section preserved" + ); + } + + #[test] + fn splice_inserts_when_no_existing_slots() { + let existing = + "[creative_opportunities]\ngam_network_id = \"111\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header"), + "inserted slot id strips the div-gpt-ad- prefix" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["div_id"].as_str(), + Some("div-gpt-ad-header"), + "div_id keeps the stable stem" + ); + assert!( + value["auction"]["enabled"].as_bool() == Some(true), + "auction section preserved after inserted slots" + ); + } + + #[test] + fn splice_replaces_inline_slot_array() { + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + slot = [{ id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should replace inline slot array"); + + let value = toml::from_str::(&out).expect("spliced config should be valid"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("slots should be an array"); + assert_eq!(slots.len(), 1, "old inline slot should be removed"); + assert_eq!(slots[0]["id"].as_str(), Some("header")); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "unrelated tables should be preserved" + ); + } + + #[test] + fn splice_replaces_inline_slot_map() { + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + slot = { \"0\" = { id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] } }\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should replace inline slot map"); + + let value = toml::from_str::(&out).expect("spliced config should be valid"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("slots should be an array"); + assert_eq!(slots.len(), 1, "old inline slot should be removed"); + assert_eq!(slots[0]["id"].as_str(), Some("header")); + } + + #[test] + fn merge_second_run_unions_page_patterns() { + // Existing slot on "/"; re-discovered this run with "/news/*". + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/news/*".to_string()], + false, + ); + + assert_eq!(merged.len(), 1, "same slot is not duplicated"); + assert_eq!( + merged[0].page_patterns, + vec!["/".to_string(), "/news/*".to_string()], + "this run's pattern is unioned into the existing slot" + ); + } + + #[test] + fn merge_second_run_unions_formats() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90), (970, 250)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots(Some(&existing), &discovered, &["/".to_string()], false); + + assert_eq!( + merged[0].formats, + [(728, 90, None), (970, 250, None)], + "a later audit must retain newly observed formats" + ); + } + + #[test] + fn merge_uses_longest_existing_div_prefix() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/broad/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"atf\"\ndiv_id = \"ad-atf-\"\n\ + gam_unit_path = \"/222/atf\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![(728, 90)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots( + Some(&existing), + &discovered, + &["/news/*".to_string()], + false, + ); + + assert_eq!( + merged.len(), + 2, + "prefix match should not append a duplicate" + ); + let broad = merged + .iter() + .find(|slot| slot.id == "broad") + .expect("should keep broad slot"); + assert_eq!( + broad.page_patterns, + ["/broad/*"], + "shorter prefix should not claim the discovered div" + ); + let atf = merged + .iter() + .find(|slot| slot.id == "atf") + .expect("should keep specific slot"); + assert_eq!( + atf.page_patterns, + ["/", "/news/*"], + "longest matching prefix should receive this run's pattern" + ); + } + + #[test] + fn observed_literal_does_not_claim_numeric_siblings() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-sidebar-1\"\ndiv_id = \"ad-sidebar-1\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10", "ad-sidebar-11"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!(merged.len(), 3); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-10")); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-11")); + assert!(diagnostics.notes.is_empty()); + assert!(diagnostics.unobserved_existing_slot_ids.is_empty()); + } + + #[test] + fn split_sibling_warns_when_tuned_parent_fields_are_not_inherited() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-sidebar-1\"\ndiv_id = \"ad-sidebar-1\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 1.5\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + let sibling = merged + .iter() + .find(|slot| slot.id == "ad-sidebar-10") + .expect("should append the distinct sibling"); + assert_eq!( + sibling.floor_price, None, + "a distinct placement must not inherit the configured parent's floor" + ); + assert_eq!(diagnostics.notes.len(), 1, "should emit one split warning"); + assert!( + diagnostics.notes[0].contains("discovered div `ad-sidebar-10`"), + "should name the split sibling, got {:?}", + diagnostics.notes + ); + assert!( + diagnostics.notes[0].contains("configured div_id prefix `ad-sidebar-1`"), + "should name the disqualified parent prefix, got {:?}", + diagnostics.notes + ); + assert!( + diagnostics.notes[0].contains("does not inherit"), + "should explain the tuned-field consequence, got {:?}", + diagnostics.notes + ); + } + + #[test] + fn refused_stem_observes_prefix_without_disqualifying_prefix_routing() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-x\"\ndiv_id = \"ad-x\"\n\ + gam_unit_path = \"/222/ad-x\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 1.5\n", + ); + let discovered = vec![RenderSlot::from_evidence( + "ad-x-stable", + "ad-x-stable", + Some("/222/ad-x".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + )]; + + let (merged, diagnostics) = merge_render_slots_with_observed_diagnostics( + Some(&existing), + discovered, + &["ad-x".to_string(), "ad-x-stable".to_string()], + &["ad-x-stable".to_string()], + false, + ); + + assert_eq!( + merged.len(), + 1, + "the configured prefix should absorb its sibling" + ); + assert_eq!(merged[0].page_patterns, ["/", "/news/*"]); + assert!( + diagnostics.notes.is_empty(), + "a refused stem is not a literal split boundary" + ); + assert!( + diagnostics.unobserved_existing_slot_ids.is_empty(), + "the refused stem should still prove the configured prefix was observed" + ); + } + + #[test] + fn split_sibling_warns_for_every_tuned_parent_prefix() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 1.0\n\n\ + [[slot]]\nid = \"side\"\ndiv_id = \"ad-side\"\n\ + gam_unit_path = \"/222/side\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 2.0\n", + ); + let discovered = ["ad", "ad-side", "ad-sidebar"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/new".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (_, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!( + diagnostics.notes.len(), + 2, + "both tuned ancestors should be named" + ); + assert!( + diagnostics + .notes + .iter() + .any(|note| note.contains("prefix `ad`")) + ); + assert!( + diagnostics + .notes + .iter() + .any(|note| note.contains("prefix `ad-side`")) + ); + } + + #[test] + fn newly_appended_literal_does_not_claim_numeric_sibling() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"legacy\"\ndiv_id = \"legacy-slot\"\n\ + gam_unit_path = \"/222/legacy\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!(merged.len(), 3); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-1")); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-10")); + assert!(diagnostics.notes.is_empty()); + } + + #[test] + fn normalized_stem_is_the_literal_merge_boundary() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-header-0\"\ndiv_id = \"ad-header-0\"\n\ + gam_unit_path = \"/222/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![ + collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "ad-header-0-_R_3f_".to_string(), + sizes: vec![(728, 90)], + }, + collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "ad-header-01".to_string(), + sizes: vec![(728, 90)], + }, + ]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots(Some(&existing), &discovered, &["/".to_string()], false); + + assert_eq!(merged.len(), 2); + assert!(merged.iter().any(|slot| slot.id == "ad-header-0")); + assert!(merged.iter().any(|slot| slot.id == "ad-header-01")); + } + + #[test] + fn merge_reports_when_a_broad_prefix_claims_multiple_discovered_divs() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = vec![ + RenderSlot::from_evidence( + "header", + "ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + RenderSlot::from_evidence( + "footer", + "ad-footer", + Some("/222/footer".to_string()), + [(300, 250)], + vec!["/".to_string()], + false, + ), + ]; + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!( + merged.len(), + 1, + "the configured prefix still controls merging" + ); + assert_eq!(diagnostics.notes.len(), 1); + assert!(diagnostics.notes[0].contains("matched 2 discovered divs")); + assert!(diagnostics.notes[0].contains("ad-footer")); + assert!( + diagnostics.notes[0] + .contains("runtime can resolve this configured slot to at most one"), + "diagnostic should explain the runtime consequence" + ); + assert!(diagnostics.notes[0].contains("ad-header")); + } + + #[test] + fn a_slot_appended_this_run_never_absorbs_a_later_discovery() { + // Prefix reconciliation belongs to the operator's config. If a slot + // appended during this run could act as a prefix, `ad-top` would swallow + // `ad-top-sidebar` whenever discovery happened to see it first, dropping + // the absorbed slot's unit path and provider state, and no broad-prefix + // diagnostic would report it. + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"sidebar-ad\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 600 }]\n", + ); + let candidates = [ + RenderSlot::from_evidence( + "ad-top", + "ad-top", + Some("/222/top".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + RenderSlot::from_evidence( + "ad-top-sidebar", + "ad-top-sidebar", + Some("/222/top-sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + true, + ), + ]; + + for order in [[0_usize, 1], [1, 0]] { + let discovered: Vec = order + .iter() + .map(|index| candidates[*index].clone()) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert!( + diagnostics.notes.is_empty(), + "no configured prefix claimed a discovered div in order {order:?}, got {diagnostics:?}" + ); + assert_eq!( + merged.len(), + 3, + "both discovered slots must survive in order {order:?}" + ); + let sidebar_ad = merged + .iter() + .find(|slot| slot.div_id.as_deref() == Some("ad-top-sidebar")) + .unwrap_or_else(|| { + panic!("the longer div must stay its own slot in order {order:?}") + }); + assert_eq!( + sidebar_ad.gam_unit_path.as_deref(), + Some("/222/top-sidebar"), + "the absorbed slot's unit path must survive in order {order:?}" + ); + assert_eq!( + sidebar_ad.page_patterns, + ["/news/*"], + "patterns must not be pooled in order {order:?}" + ); + assert!( + sidebar_ad.prebid_bidders.is_some(), + "provider state must survive in order {order:?}" + ); + let top = merged + .iter() + .find(|slot| slot.div_id.as_deref() == Some("ad-top")) + .unwrap_or_else(|| panic!("the shorter div must stay in order {order:?}")); + assert_eq!( + top.page_patterns, + ["/"], + "the longer slot's pattern must not leak into the shorter one in order {order:?}" + ); + } + } + + #[test] + fn merge_renames_new_slot_id_that_collides_with_existing_config() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header-main\"\ndiv_id = \"legacy-header\"\n\ + gam_unit_path = \"/222/legacy\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "div-gpt-ad-header.main".to_string(), + sizes: vec![(728, 90)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots( + Some(&existing), + &discovered, + &["/news/*".to_string()], + false, + ); + let ids = merged + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(); + + assert_eq!(ids, ["header-main", "header-main-2"]); + } + + #[test] + fn merge_keeps_existing_only_slots() { + // Existing has header + sidebar; this run re-sees only header. + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 0.5\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header", "sidebar"], "sidebar preserved"); + let sidebar = merged + .iter() + .find(|slot| slot.id == "sidebar") + .expect("sidebar"); + assert_eq!( + sidebar.floor_price, + Some(0.5), + "hand-tuned fields preserved" + ); + } + + #[test] + fn merge_reports_preserved_unobserved_slots_in_config_order() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"footer\"\ndiv_id = \"ad-footer\"\n\ + gam_unit_path = \"/222/footer\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let discovered = vec![RenderSlot::from_evidence( + "header", + "div-gpt-ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + )]; + + let (_, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!( + diagnostics.unobserved_existing_slot_ids, + ["sidebar", "footer"], + "unobserved slots should retain configuration order" + ); + + let all_discovered = vec![ + RenderSlot::from_evidence( + "header", + "div-gpt-ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + RenderSlot::from_evidence( + "sidebar", + "ad-sidebar", + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ), + RenderSlot::from_evidence( + "footer", + "ad-footer", + Some("/222/footer".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + ]; + let (_, fully_observed) = + merge_render_slots_with_diagnostics(Some(&existing), all_discovered.clone(), false); + let (_, replaced) = + merge_render_slots_with_diagnostics(Some(&existing), all_discovered.clone(), true); + let (_, no_existing) = merge_render_slots_with_diagnostics(None, all_discovered, false); + + assert!( + fully_observed.unobserved_existing_slot_ids.is_empty(), + "fully observed slots should not be reported as stale" + ); + assert!( + replaced.unobserved_existing_slot_ids.is_empty(), + "--replace should not report discarded existing slots as stale" + ); + assert!( + no_existing.unobserved_existing_slot_ids.is_empty(), + "a config without existing slots should not report stale slots" + ); + } + + #[test] + fn observed_div_marks_exact_slot_and_live_broad_prefix() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/222/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let discovered = vec![RenderSlot::from_evidence( + "header", + "ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/news/*".to_string()], + false, + )]; + + let (_, diagnostics) = merge_render_slots_with_observed_diagnostics( + Some(&existing), + discovered, + &["ad-header".to_string()], + &["ad-header".to_string()], + false, + ); + + assert!( + diagnostics.unobserved_existing_slot_ids.is_empty(), + "the exact slot and every live configured prefix should be observed, got {diagnostics:?}" + ); + } + + #[test] + fn merge_replace_wipes_existing() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + true, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header"], "--replace keeps only discovered slots"); + } + + #[test] + fn resolve_network_id_prefers_discovered_unless_preserving_existing() { + let with_slots = existing_config( + "gam_network_id = \"111\"\n\n[[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\n\ + gam_unit_path = \"/111/s\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let empty = existing_config("gam_network_id = \"111\"\n"); + + // Real merge → keep existing. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), false).as_deref(), + Some("111") + ); + // Placeholder section with no slots → discovered wins. + assert_eq!( + resolve_network_id(Some(&empty), Some("222"), false).as_deref(), + Some("222") + ); + // --replace → discovered wins. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), true).as_deref(), + Some("222") + ); + // No existing config → discovered. + assert_eq!( + resolve_network_id(None, Some("222"), false).as_deref(), + Some("222") + ); + } + + #[test] + fn toml_key_quotes_only_non_bare_keys() { + assert_eq!(toml_key("zone"), "zone"); + assert_eq!(toml_key("ad-loc"), "ad-loc"); + assert_eq!(toml_key("a.b"), "\"a.b\""); + assert_eq!(toml_key("with space"), "\"with space\""); + assert_eq!(toml_key(""), "\"\""); + } + + #[test] + fn toml_string_escapes_quotes_backslashes_and_controls() { + assert_eq!(toml_string("a\"b\\c"), "\"a\\\"b\\\\c\""); + assert_eq!(toml_string("line\nbreak\t!"), "\"line\\nbreak\\t!\""); + } + + #[test] + fn toml_string_escapes_del_control_char() { + assert_eq!(toml_string("a\u{7f}b"), "\"a\\u007Fb\""); + let doc = format!("value = {}", toml_string("a\u{7f}b")); + let value = toml::from_str::(&doc).expect("DEL escapes to valid TOML"); + assert_eq!( + value["value"].as_str(), + Some("a\u{7f}b"), + "escaped DEL round-trips as data" + ); + } + + #[test] + fn replace_key_handles_inline_commented_headers() { + let document = "[creative_opportunities] # managed\ngam_network_id = \"111\"\n\n\ + [auction] # flags\nenabled = true\n"; + + let updated = replace_key_in_section( + document, + "creative_opportunities", + "gam_network_id", + "gam_network_id = \"222\"", + ) + .expect("should find the commented section header"); + + assert!( + updated.contains("gam_network_id = \"222\""), + "key replaced under a commented header" + ); + assert!( + updated.contains("enabled = true"), + "later commented section left untouched" + ); + } + + #[test] + fn render_quotes_exotic_targeting_keys_to_valid_toml() { + let existing = existing_config( + "gam_network_id = \"1\"\n\n\ + [[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\ngam_unit_path = \"/1/s\"\n\ + page_patterns = [\"/\"]\nformats = [{ width = 300, height = 250 }]\n\ + targeting = { \"a.b\" = \"x\" }\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + let doc = format!( + "[creative_opportunities]\ngam_network_id = \"1\"\n{}", + render_slots(&merged) + ); + + toml::from_str::(&doc).expect("exotic targeting key renders as valid TOML"); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs new file mode 100644 index 000000000..06b7c955d --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs @@ -0,0 +1,1028 @@ +//! Infers a `{network_id}`/`{section}` ad-unit template from observed evidence. +//! +//! The generator otherwise writes the literal path each page happened to +//! request, which pins a slot to the one section it was scraped from. A template +//! generalizes across sections — but a *wrong* template makes the publisher bid +//! against inventory that does not exist, which is worse than a narrow literal. +//! So this module is built to refuse rather than guess. +//! +//! The inference applies three evidence rules: +//! +//! 1. **Positional binding.** `{network_id}` is bound to unit segment 0 and only +//! if that segment is the resolved network id. Substring replacement would +//! corrupt `/123/sports123/home` into `/{network_id}/sports{network_id}/home`. +//! 2. **Exactly one varying segment.** Zero means nothing was proven and the +//! path stays literal; two means the unit varies along a dimension the +//! request path cannot supply (device, geo, experiment), so it is refused. +//! 3. **Cross-page variation.** Two pages must show *different* derived sections +//! and different unit segments. A single-page crawl is +//! indistinguishable from a static path — literal, `{network_id}`-only and +//! `{section}` all reproduce one observation equally well, and round-trip +//! verification cannot tell them apart. Only variation can. +//! +//! Every accepted template is then replayed through the runtime's own +//! [`render_gam_unit_path`](CreativeOpportunitySlot::render_gam_unit_path) and +//! [`derive_section`] against every observation. A template that does not +//! reproduce what the live page actually requested is downgraded, not written. + +use std::collections::{BTreeMap, BTreeSet}; + +use trusted_server_core::creative_opportunities::{CreativeOpportunitySlot, derive_section}; + +use super::evidence::{EvidenceTable, SlotEvidence}; +use super::slot_toml::toml_string; + +/// Candidate `section_segment` values considered, `0..=MAX_SECTION_SEGMENT`. +/// +/// A locale-prefixed site (`/en/news/story`) needs 1. Beyond 2 the "section" is +/// no longer a taxonomy the operator would recognise, and every extra candidate +/// is another chance for two indices to both fit and force a refusal. +const MAX_SECTION_SEGMENT: usize = 2; + +/// The config-level section policy an inferred template depends on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SectionPolicy { + /// Value substituted for `{section}` on paths with no section segment. + pub(super) section_root: String, + /// Index of the path segment `{section}` is taken from. + pub(super) section_segment: usize, +} + +/// What to write for one slot's `gam_unit_path`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum SlotDecision { + /// Write this templated path; it reproduced every observation. + Template(String), + /// Write this literal path; nothing generalizable was proven. + Literal(String), + /// Write no path at all — the observations cannot be represented. + Refuse { + /// Operator-facing explanations, one per reason. + reasons: Vec, + }, +} + +/// The outcome of inference across the whole evidence table. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct InferenceOutcome { + /// Section policy to write, present only when some slot templated. + pub(super) policy: Option, + /// Per-slot decision, keyed by div stem, in evidence order. + pub(super) decisions: Vec<(String, SlotDecision)>, + /// Operator-facing notes about why inference went the way it did. + pub(super) diagnostics: Vec, + /// Div stems whose templates rely on a root witnessed by another slot. + pub(super) borrowed_section_root: Vec, +} + +impl InferenceOutcome { + /// The decision for a slot, by div stem. + pub(super) fn decision(&self, div_id: &str) -> Option<&SlotDecision> { + self.decisions + .iter() + .find(|(key, _)| key == div_id) + .map(|(_, decision)| decision) + } +} + +/// Per-slot analysis under one candidate `section_segment`. +#[derive(Debug, Clone, PartialEq, Eq)] +enum SlotAnalysis { + /// Templatable: unit segment `varying` tracks the derived section, and root + /// pages agreed on `section_root`. + Templatable { + varying: usize, + section_root: String, + }, + /// The unit path never varied, so nothing about `{section}` was proven. + Static, + /// Cannot be represented; carries the operator-facing reason. + Refuse(String), + /// Unit segment `varying` tracks the derived section on every page this slot + /// was seen on, but none of those pages lacked the section segment, so the + /// slot witnessed no `section_root` of its own. + /// + /// Carries `varying` because such a slot is still templatable *when another + /// slot witnessed the config-level `section_root`*: a placement that only + /// exists on section pages (a sidebar, an in-article unit) never renders on a + /// path where `{section}` would fall back to the root. + RootUnwitnessed { varying: usize }, +} + +/// Infers unit-path templates for every slot in `table`. +/// +/// `network_id` is the resolved GAM network id; `{network_id}` is only ever +/// bound to a unit segment that already equals it. +pub(super) fn infer_unit_templates(table: &EvidenceTable, network_id: &str) -> InferenceOutcome { + let slots: Vec<&SlotEvidence> = table.slots().collect(); + let mut diagnostics = Vec::new(); + + // Evaluate every candidate index independently; ambiguity between two that + // both fit is a refusal, not a preference for the smaller one. + let mut qualifying: Vec<(usize, String, BTreeMap)> = Vec::new(); + let mut root_witness_missing = false; + let mut root_unwitnessed_stems = BTreeSet::new(); + for segment in 0..=MAX_SECTION_SEGMENT { + let analyses: BTreeMap = slots + .iter() + .map(|slot| (slot.div_id.clone(), analyse_slot(slot, network_id, segment))) + .collect(); + + let roots: BTreeSet<&str> = analyses + .values() + .filter_map(|analysis| match analysis { + SlotAnalysis::Templatable { section_root, .. } => Some(section_root.as_str()), + _ => None, + }) + .collect(); + // Slots must agree: `section_root` is one config-level value, so two + // slots claiming different roots means this index is not the real one. + let Some(root) = roots.iter().next().copied() else { + // Distinguish "nothing tracks the section" from "everything does but + // no crawled page lacked the section segment": the second is a crawl + // gap the operator can close, and the generic literal-path refusal + // below does not say so. + root_witness_missing |= analyses + .values() + .any(|analysis| matches!(analysis, SlotAnalysis::RootUnwitnessed { .. })); + root_unwitnessed_stems.extend( + analyses + .iter() + .filter(|(_, analysis)| { + matches!(analysis, SlotAnalysis::RootUnwitnessed { .. }) + }) + .map(|(stem, _)| stem.clone()), + ); + continue; + }; + if roots.len() > 1 { + continue; + } + qualifying.push((segment, root.to_string(), analyses)); + } + + let chosen = match qualifying.len() { + 0 => None, + 1 => qualifying.into_iter().next(), + _ => { + let indices: Vec = qualifying + .iter() + .map(|(segment, _, _)| segment.to_string()) + .collect(); + diagnostics.push(format!( + "more than one section_segment ({}) explains the observed ad-unit paths \ + equally well, so no template can be chosen safely; slots without one safe literal path are omitted", + indices.join(", ") + )); + None + } + }; + + let Some((section_segment, section_root, analyses)) = chosen else { + // Only a crawl gap justifies rewriting the per-slot reasons. When + // inference stopped on segment ambiguity instead, that pushed its own + // diagnostic, and blaming the crawl here would send the operator to + // widen it when the remedy is pinning `section_segment`. + let root_gap = diagnostics.is_empty() && root_witness_missing; + if diagnostics.is_empty() { + diagnostics.push(if root_witness_missing { + "the ad-unit paths do track the page section, but no crawled page lacked a \ + section segment, so `section_root` could not be witnessed and no {section} \ + template can be written; include the site root in the crawl (or set \ + section_root by hand) to template these slots" + .to_string() + } else { + "no ad-unit path varied by page section across the crawl, so paths were kept \ + literal; crawl more sections to enable a {section} template" + .to_string() + }); + } + let mut decisions = literal_decisions(&slots); + if root_gap { + for (stem, decision) in &mut decisions { + if root_unwitnessed_stems.contains(stem) + && let SlotDecision::Refuse { reasons } = decision + { + *reasons = vec![ + "the paths tracked the page section, but no crawled page lacked a \ + section segment, so `section_root` could not be witnessed" + .to_string(), + ]; + } + } + } + return InferenceOutcome { + policy: None, + decisions, + diagnostics, + borrowed_section_root: Vec::new(), + }; + }; + + let mut decisions = Vec::with_capacity(slots.len()); + let mut borrowed_section_root = Vec::new(); + let mut templated = 0_usize; + for slot in &slots { + let analysis = analyses + .get(&slot.div_id) + .cloned() + .unwrap_or(SlotAnalysis::Static); + let templatable = match analysis { + SlotAnalysis::Templatable { varying, .. } => Some((varying, true)), + // The config-level `section_root` is witnessed by another slot on the + // same property, and this slot's page patterns are derived from the + // paths it was seen on — all of which carry a section segment — so + // `{section}` never falls back to the root for it. Refusing here cost + // real inventory: a sidebar or in-article unit that simply does not + // exist on the site root was omitted from the config entirely. + SlotAnalysis::RootUnwitnessed { varying } => Some((varying, false)), + SlotAnalysis::Static | SlotAnalysis::Refuse(_) => None, + }; + let decision = match (templatable, analysis) { + (Some((varying, witnessed_root)), _) => { + let template = build_template(slot, varying); + match verify_round_trip(&template, slot, network_id, §ion_root, section_segment) + { + Ok(()) => { + templated += 1; + if !witnessed_root { + borrowed_section_root.push(slot.div_id.clone()); + diagnostics.push(format!( + "slot `{}` was never observed on a page without a section \ + segment, so its `{{section}}` template relies on the \ + config-level section_root `{section_root}` witnessed by other \ + slots; it is only rendered for the paths this slot was seen on", + slot.id + )); + } + SlotDecision::Template(template) + } + Err(reason) => { + diagnostics.push(format!( + "slot `{}` template `{template}` did not reproduce the observed \ + ad-unit paths ({reason}); refusing any unsafe fallback", + slot.id + )); + literal_decision(slot) + } + } + } + (None, SlotAnalysis::Refuse(reason)) => SlotDecision::Refuse { + reasons: vec![reason], + }, + (None, _) => literal_decision(slot), + }; + decisions.push((slot.div_id.clone(), decision)); + } + + if templated == 0 { + return InferenceOutcome { + policy: None, + decisions, + diagnostics, + borrowed_section_root: Vec::new(), + }; + } + + diagnostics.push(format!( + "inferred section_segment = {section_segment} and section_root = \"{section_root}\" \ + from {} page(s); {templated} slot(s) templated", + table.pages().len() + )); + InferenceOutcome { + policy: Some(SectionPolicy { + section_root, + section_segment, + }), + decisions, + diagnostics, + borrowed_section_root, + } +} + +/// Checks the properties of a slot's observations that do not depend on which +/// `section_segment` is being considered. +/// +/// Kept separate because these refusals are final: no candidate index can +/// rescue a slot whose observations are not one template with a single hole in +/// them, and the operator needs the specific reason rather than a generic one. +/// +/// Returns the single varying unit segment, `None` when nothing varied, or the +/// reason the observations cannot be represented at all. +fn structural_check(slot: &SlotEvidence) -> Result, String> { + // One page reporting two different ad-unit paths for the same slot means the + // unit varies along something the request path cannot express — a device or + // geo split, or two profiles disagreeing. Nothing here can represent that. + let mut per_path: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for row in &slot.rows { + per_path + .entry(row.path.as_str()) + .or_default() + .insert(row.unit_path.as_str()); + } + if let Some((path, units)) = per_path.iter().find(|(_, units)| units.len() > 1) { + let observed: Vec<&str> = units.iter().copied().collect(); + return Err(format!( + "page `{path}` requested more than one ad-unit path for this slot ({}); \ + the unit varies by something the request path cannot derive", + observed.join(", ") + )); + } + + let split: Vec> = slot + .rows + .iter() + .map(|row| segments(&row.unit_path)) + .collect(); + let Some(first) = split.first() else { + return Ok(None); + }; + // Differing shapes are not one template with a hole in it. + if split.iter().any(|parts| parts.len() != first.len()) { + return Err( + "the observed ad-unit paths have different segment counts, so they are not \ + one template" + .to_string(), + ); + } + + let varying: Vec = (0..first.len()) + .filter(|index| { + split + .iter() + .map(|parts| parts[*index]) + .collect::>() + .len() + > 1 + }) + .collect(); + match varying.len() { + 0 => Ok(None), + 1 if varying[0] == 0 => { + Err("the network-id segment of the ad-unit path varied across pages".to_string()) + } + 1 => Ok(Some(varying[0])), + count => Err(format!( + "{count} ad-unit segments vary across pages, so the path does not track the \ + page section alone" + )), + } +} + +/// Analyses one slot under a candidate `section_segment`. +/// +/// [`structural_check`] has already established that a templatable candidate +/// contains more than one observed unit path. Therefore a successful derived +/// section match here is itself the required variation witness; a second +/// witness predicate would only restate that invariant. +fn analyse_slot(slot: &SlotEvidence, network_id: &str, section_segment: usize) -> SlotAnalysis { + let varying = match structural_check(slot) { + Err(reason) => return SlotAnalysis::Refuse(reason), + Ok(None) => return SlotAnalysis::Static, + Ok(Some(varying)) => varying, + }; + + let split: Vec> = slot + .rows + .iter() + .map(|row| segments(&row.unit_path)) + .collect(); + // `{network_id}` binds positionally and only to the resolved id. Substring + // replacement would rewrite an unrelated segment that merely contains it. + if split.first().and_then(|parts| parts.first()) != Some(&network_id) { + return SlotAnalysis::Static; + } + + // Partition observations into pages that have a section segment and pages + // that do not; the latter are what determine `section_root`. + let mut root_values = BTreeSet::new(); + for (row, parts) in slot.rows.iter().zip(split.iter()) { + let observed = parts[varying]; + if path_segments(&row.path).len() > section_segment { + // The empty root is unused here: the path has this segment. + if derive_section(&row.path, "", section_segment) != observed { + return SlotAnalysis::Static; + } + } else { + root_values.insert(observed); + } + } + + let mut roots = root_values.into_iter(); + let Some(section_root) = roots.next() else { + // Without a root observation, `section_root` would be a guess that + // silently mis-renders every short path. + return SlotAnalysis::RootUnwitnessed { varying }; + }; + if roots.next().is_some() { + return SlotAnalysis::Static; + } + // A root that is not `[A-Za-z0-9_-]+` makes any `{section}` template fail + // config load; catch it here rather than at push time. + if section_root.is_empty() + || !section_root + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-') + { + return SlotAnalysis::Static; + } + + SlotAnalysis::Templatable { + varying, + section_root: section_root.to_string(), + } +} + +/// Builds the template text by substituting the two proven placeholders. +fn build_template(slot: &SlotEvidence, varying: usize) -> String { + let first = slot + .rows + .iter() + .next() + .map(|row| row.unit_path.as_str()) + .unwrap_or_default(); + let rendered: Vec = segments(first) + .into_iter() + .enumerate() + .map(|(index, value)| { + if index == 0 { + "{network_id}".to_string() + } else if index == varying { + "{section}".to_string() + } else { + value.to_string() + } + }) + .collect(); + format!("/{}", rendered.join("/")) +} + +/// Replays `template` through the runtime renderer against every observation. +/// +/// Defense in depth rather than the primary gate: [`analyse_slot`] already +/// refuses to call a slot templatable when the derived section and the observed +/// segment disagree — a publisher whose `/site-news` pages request +/// `.../sitenews`, say — so a mismatch reaching here would mean inference and +/// the runtime renderer disagree. The template is then dropped instead of +/// written, and the diagnostic names the paths that did not reproduce. +fn verify_round_trip( + template: &str, + slot: &SlotEvidence, + network_id: &str, + section_root: &str, + section_segment: usize, +) -> Result<(), String> { + let probe = probe_slot(template)?; + for row in &slot.rows { + let section = derive_section(&row.path, section_root, section_segment); + match probe.render_gam_unit_path(network_id, §ion) { + Some(rendered) if rendered == row.unit_path => {} + Some(rendered) => { + return Err(format!( + "on `{}` it renders `{rendered}` but the page requested `{}`", + row.path, row.unit_path + )); + } + None => { + return Err(format!( + "on `{}` it renders past the GAM ad-unit path byte limit", + row.path + )); + } + } + } + Ok(()) +} + +/// Builds a throwaway slot carrying `template`, for rendering only. +/// +/// Deserializing is how the runtime itself builds slots, so this exercises the +/// same template parsing rather than a parallel implementation. +fn probe_slot(template: &str) -> Result { + let document = format!( + "id = \"probe\"\ngam_unit_path = {}\npage_patterns = [\"/\"]\n\ + formats = [{{ width = 1, height = 1 }}]\n", + toml_string(template) + ); + toml::from_str::(&document) + .map_err(|error| format!("template is not representable in config: {error}")) +} + +/// The decision for a slot no template was proven for. +/// +/// A structural refusal wins over the generic "several paths" message, so the +/// operator sees *why* the slot could not be represented (a device split, an +/// extra varying dimension) rather than only that it could not. +fn literal_decision(slot: &SlotEvidence) -> SlotDecision { + if let Err(reason) = structural_check(slot) { + return SlotDecision::Refuse { + reasons: vec![reason], + }; + } + let units = slot.unit_paths(); + let mut found = units.iter(); + match (found.next(), found.next()) { + (Some(only), None) => SlotDecision::Literal((*only).to_string()), + (Some(_), Some(_)) => SlotDecision::Refuse { + reasons: vec![format!( + "the slot used several ad-unit paths ({}) and none generalized, so no \ + single literal path is correct", + units.into_iter().collect::>().join(", ") + )], + }, + _ => SlotDecision::Refuse { + reasons: vec!["no ad-unit path was observed for this slot".to_string()], + }, + } +} + +fn literal_decisions(slots: &[&SlotEvidence]) -> Vec<(String, SlotDecision)> { + slots + .iter() + .map(|slot| (slot.div_id.clone(), literal_decision(slot))) + .collect() +} + +/// Non-empty path segments of an ad-unit path. +fn segments(unit_path: &str) -> Vec<&str> { + unit_path + .split('/') + .filter(|part| !part.is_empty()) + .collect() +} + +/// Non-empty path segments of a request path. +fn path_segments(path: &str) -> Vec<&str> { + path.split('/').filter(|part| !part.is_empty()).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector::CollectedGptSlot; + use crate::commands::audit::generate::gpt_slots::discover_gpt_slots; + + /// Folds `(path, unit_path)` observations for one div into a table. + fn table_for(div_id: &str, observations: &[(&str, &str)]) -> EvidenceTable { + let mut table = EvidenceTable::default(); + for (path, unit_path) in observations { + let registry = vec![CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: div_id.to_string(), + sizes: vec![(728, 90)], + }]; + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + table + } + + /// Folds pages carrying different slot sets into one table. + /// + /// Each entry is `(request path, [(div id, ad-unit path)])`. + fn table_for_pages(pages: &[(&str, &[(&str, &str)])]) -> EvidenceTable { + let mut table = EvidenceTable::default(); + for (path, slots) in pages { + let registry: Vec = slots + .iter() + .map(|(div_id, unit_path)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: vec![(728, 90)], + }) + .collect(); + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + table + } + + fn only_decision(outcome: &InferenceOutcome) -> &SlotDecision { + assert_eq!(outcome.decisions.len(), 1, "fixture should have one slot"); + &outcome.decisions[0].1 + } + + #[test] + fn templates_a_section_varying_unit_path() { + // The shape the operator writes by hand today. + let table = table_for( + "ad-header", + &[ + ("/", "/123456789/publisher/homepage"), + ("/news/story-abc", "/123456789/publisher/news"), + ("/deals/thing", "/123456789/publisher/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123456789"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }) + ); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Template("/{network_id}/publisher/{section}".to_string()) + ); + } + + #[test] + fn a_single_page_never_templates() { + // Literal, {network_id}-only and {section} all reproduce one observation, + // so only variation can distinguish them. This is the witness rule. + let table = table_for("ad-header", &[("/news/story", "/123/site/news")]); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Literal("/123/site/news".to_string()) + ); + } + + #[test] + fn a_static_unit_path_across_sections_stays_literal() { + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/fixed"), + ("/news/story", "/123/site/fixed"), + ("/deals/x", "/123/site/fixed"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None, "nothing varied, so nothing is proven"); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Literal("/123/site/fixed".to_string()) + ); + } + + #[test] + fn a_device_split_is_refused_rather_than_guessed() { + // Two units for the SAME path: the desktop/mobile cross-check surfaces + // here, and the request path cannot express the difference. + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/desktop/news"), + ("/news/story", "/123/mobile/news"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!( + "a device split must refuse, got {:?}", + only_decision(&outcome) + ); + }; + assert!( + reasons[0].contains("more than one ad-unit path"), + "reason should name the conflict, got {reasons:?}" + ); + } + + #[test] + fn two_varying_segments_are_refused() { + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/desktop/news"), + ("/deals/x", "/123/mobile/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("two varying dimensions must refuse"); + }; + assert!( + reasons[0].contains("segments vary"), + "reason should name the extra dimension, got {reasons:?}" + ); + } + + #[test] + fn a_slug_the_path_cannot_reproduce_is_refused() { + // `/site-news` requests `.../sitenews`: the derived section and + // the observed segment differ, so the template would render the wrong + // unit. Candidate analysis rejects the inconsistent section mapping. + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/homepage"), + ("/news/story", "/123/site/news"), + ("/site-news/x", "/123/site/sitenews"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, None, + "a section whose slug is not derivable must not template" + ); + assert!(matches!( + only_decision(&outcome), + SlotDecision::Refuse { .. } + )); + } + + #[test] + fn an_unwitnessed_root_is_refused() { + // Every crawled page had a section, so `section_root` would be a guess + // that silently mis-renders the homepage. + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/site/news"), + ("/deals/x", "/123/site/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None); + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("two literal paths and no template is not representable as one literal"); + }; + assert!( + reasons + .iter() + .any(|reason| reason.contains("section_root") && reason.contains("witnessed")), + "the per-slot reason should name the crawl gap; got {reasons:?}" + ); + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("section_root` could not be witnessed")), + "the crawl gap, not \"nothing generalized\", is the reason; got {:?}", + outcome.diagnostics + ); + } + + #[test] + fn a_slot_absent_from_the_root_templates_from_the_witnessed_policy() { + // The live shape behind the `ad-atf_sidebar-0` refusal: a header on the + // root and every section witnesses `section_root`, while a sidebar exists + // only on section pages. The sidebar's unit path tracks the section just + // as well, and its page patterns never cover the root, so refusing it + // dropped real inventory from the config. + let mut table = EvidenceTable::default(); + let pages: &[(&str, &[(&str, &str)])] = &[ + ("/", &[("ad-header", "/123/site/homepage")]), + ( + "/news/story", + &[ + ("ad-header", "/123/site/news"), + ("ad-sidebar", "/123/site/news"), + ], + ), + ( + "/deals/x", + &[ + ("ad-header", "/123/site/deals"), + ("ad-sidebar", "/123/site/deals"), + ], + ), + ]; + for (path, slots) in pages { + let registry: Vec = slots + .iter() + .map(|(div_id, unit_path)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: vec![(728, 90)], + }) + .collect(); + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }), + "the header witnesses the config-level policy" + ); + assert_eq!( + outcome.decision("ad-sidebar"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )), + "a slot that only exists on section pages is still templatable" + ); + assert_eq!( + outcome.decision("ad-header"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )) + ); + assert_eq!( + outcome.borrowed_section_root, + ["ad-sidebar".to_string()], + "the outcome should identify templates whose safety depends on derived patterns" + ); + assert!( + outcome.diagnostics.iter().any(|note| note + .contains("`ad-sidebar` was never observed on a page without a section segment")), + "the borrowed section_root should be stated; got {:?}", + outcome.diagnostics + ); + } + + #[test] + fn segment_ambiguity_does_not_blame_the_crawl_for_an_unwitnessed_root() { + // `ad-header` fits section_segment 0 and `ad-locale` fits 1, so + // inference stops on ambiguity. `ad-deep` is separately + // `RootUnwitnessed` at segment 2. Its refusal must not tell the + // operator to widen the crawl when the remedy is pinning + // `section_segment`. + let table = table_for_pages(&[ + ("/", &[("ad-header", "/99/site/home")]), + ("/news", &[("ad-header", "/99/site/news")]), + ("/en", &[("ad-locale", "/99/site/en-root")]), + ("/en/news", &[("ad-locale", "/99/site/news")]), + ("/a/b/news", &[("ad-deep", "/99/site/news")]), + ("/a/b/deals", &[("ad-deep", "/99/site/deals")]), + ]); + + let outcome = infer_unit_templates(&table, "99"); + + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("more than one section_segment")), + "the fixture should stop on ambiguity; got {:?}", + outcome.diagnostics + ); + assert!( + !outcome + .diagnostics + .iter() + .any(|note| note.contains("include the site root in the crawl")), + "an ambiguous run must not also blame the crawl; got {:?}", + outcome.diagnostics + ); + let Some(SlotDecision::Refuse { reasons }) = outcome.decision("ad-deep") else { + panic!("expected a refusal, got {:?}", outcome.decision("ad-deep")); + }; + assert!( + reasons + .iter() + .all(|reason| !reason.contains("no crawled page lacked a section segment")), + "the crawl-gap reason belongs only to a run that stopped on the crawl gap; got {reasons:?}" + ); + } + + #[test] + fn a_locale_prefixed_site_infers_the_deeper_segment() { + let table = table_for( + "ad-header", + &[ + ("/en", "/123/site/homepage"), + ("/en/news/story", "/123/site/news"), + ("/en/deals/x", "/123/site/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }), + "the locale prefix should push the section one segment deeper" + ); + } + + #[test] + fn network_id_is_bound_positionally_not_by_substring() { + // `sports123` merely contains the network id; substring replacement + // would corrupt it into `sports{network_id}`. + let table = table_for( + "ad-header", + &[ + ("/", "/123/sports123/homepage"), + ("/news/story", "/123/sports123/news"), + ("/deals/x", "/123/sports123/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + only_decision(&outcome), + &SlotDecision::Template("/{network_id}/sports123/{section}".to_string()), + "only segment 0 may become {{network_id}}" + ); + } + + #[test] + fn a_unit_path_not_starting_with_the_network_id_stays_literal() { + let table = table_for( + "ad-header", + &[ + ("/", "/999/site/homepage"), + ("/news/story", "/999/site/news"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, None, + "segment 0 must equal the resolved network id" + ); + } + + #[test] + fn differing_segment_counts_are_refused() { + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/homepage"), + ("/news/story", "/123/site/news/extra"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("differing shapes are not one template"); + }; + assert!( + reasons[0].contains("segment counts"), + "reason should name the shape mismatch, got {reasons:?}" + ); + } + + #[test] + fn a_static_slot_stays_literal_alongside_a_templated_one() { + let mut table = EvidenceTable::default(); + for (path, section_unit) in [ + ("/", "homepage"), + ("/news/story", "news"), + ("/deals/x", "deals"), + ] { + let registry = vec![ + CollectedGptSlot { + gam_unit_path: format!("/123/site/{section_unit}"), + div_id: "ad-header".to_string(), + sizes: vec![(728, 90)], + }, + CollectedGptSlot { + gam_unit_path: "/123/site/sticky".to_string(), + div_id: "ad-sticky".to_string(), + sizes: vec![(300, 250)], + }, + ]; + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + + let outcome = infer_unit_templates(&table, "123"); + + assert!(outcome.policy.is_some(), "the varying slot should template"); + assert_eq!( + outcome.decision("ad-header"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )) + ); + assert_eq!( + outcome.decision("ad-sticky"), + Some(&SlotDecision::Literal("/123/site/sticky".to_string())), + "a genuinely static slot must not be dragged into the template" + ); + } + + #[test] + fn diagnostics_explain_why_nothing_templated() { + let table = table_for("ad-header", &[("/news/story", "/123/site/news")]); + + let outcome = infer_unit_templates(&table, "123"); + + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("crawl more sections")), + "the operator should learn why, got {:?}", + outcome.diagnostics + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/validate.rs b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs new file mode 100644 index 000000000..2178a1d5d --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs @@ -0,0 +1,119 @@ +//! Write-side validation for generated ad-template config. +//! +//! Everything the generator writes is derived from a live, page-controlled ad +//! stack, so the candidate document has to clear the same bar the runtime +//! applies at startup *before* it replaces the operator's file. A config the +//! runtime rejects is not a degraded ad stack — `build_state` fails and the +//! adapter answers every route from the startup error router, so an unloadable +//! `trusted-server.toml` is a full-site outage once pushed. + +use trusted_server_core::settings::Settings; + +use crate::error::{CliResult, cli_error}; + +/// Validates the candidate config text the generator is about to persist. +/// +/// Runs [`Settings::from_toml`], which drives the identical +/// `finalize_deserialized` chain the runtime uses — serde (`deny_unknown_fields` +/// plus required fields), then `compile_slots` → `compile_unit_templates` → +/// `validate_runtime`, then the validator pass — with no I/O. +/// +/// `baseline` is the config as it was read from disk. When the baseline is +/// *already* unloadable, this run cannot be blamed for it: the candidate is +/// accepted and the pre-existing error is returned as a warning instead. Without +/// that escape hatch a freshly bootstrapped config carrying placeholder secrets +/// could never be updated by `generate`. +/// +/// # Errors +/// +/// Returns a user-facing error when the candidate fails to load and the baseline +/// loaded cleanly — that is, when this run introduced the failure. +pub(super) fn check_candidate(candidate: &str, baseline: &str) -> CliResult> { + let Err(candidate_error) = Settings::from_toml(candidate) else { + return Ok(Vec::new()); + }; + + if let Err(baseline_error) = Settings::from_toml(baseline) { + return Ok(vec![format!( + "target config was already invalid before this run, so the generated \ + result could not be verified: {baseline_error}" + )]); + } + + cli_error(format!( + "refusing to write: the generated config would fail to load, which would \ + take the service down once pushed: {candidate_error}" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A minimal config that loads cleanly, used as the valid baseline. + fn baseline() -> String { + crate::commands::config::init::EXAMPLE_CONFIG + .replace( + "password = \"handler_password\"", + "password = \"test-admin-password-32-bytes-minimum\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"test-proxy-secret-32-bytes-minimum\"", + ) + } + + #[test] + fn valid_candidate_passes_without_warnings() { + let config = baseline(); + + let warnings = check_candidate(&config, &config).expect("should accept valid candidate"); + + assert!( + warnings.is_empty(), + "a clean candidate should not warn, got {warnings:?}" + ); + } + + #[test] + fn candidate_this_run_broke_is_refused() { + let good = baseline(); + // An empty div_id override is exactly what a div id normalized down to + // nothing would produce, and `validate_runtime` rejects it. + let broken = format!( + "{good}\n[[creative_opportunities.slot]]\n\ + id = \"broken\"\ndiv_id = \"\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{{ width = 300, height = 250 }}]\n" + ); + + let error = check_candidate(&broken, &good).expect_err("should refuse a broken candidate"); + + assert!( + format!("{error:?}").contains("refusing to write"), + "error should name the refusal, got {error:?}" + ); + } + + #[test] + fn pre_existing_breakage_downgrades_to_a_warning() { + // The operator's file was already unloadable; `generate` must still be + // able to update it rather than blaming this run for the old error. + let broken_baseline = "[creative_opportunities]\n"; + let broken_candidate = "[creative_opportunities]\n"; + + let warnings = check_candidate(broken_candidate, broken_baseline) + .expect("a pre-existing failure should not block the write"); + + assert_eq!(warnings.len(), 1, "should surface exactly one warning"); + assert!( + warnings[0].contains("already invalid"), + "warning should name the pre-existing failure, got {:?}", + warnings[0] + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index 8f37e36fc..211060662 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -1,1343 +1,611 @@ -mod analyzer; -pub(crate) mod browser_collector; -pub(crate) mod collector; - -use std::collections::BTreeSet; -use std::fmt::Write as _; -use std::fs; -use std::io::Write; -use std::path::{Path, PathBuf}; - -use rand::RngCore as _; - -use serde::Serialize; -use url::Url; - -use crate::commands::audit::collector::AuditCollector; -use crate::commands::config::init::EXAMPLE_CONFIG; -use crate::error::{CliResult, cli_error, report_error}; - -use analyzer::{analyze_collected_page, extract_gtm_container_id}; - -/// Arguments for the `ts audit` command. -#[derive(Debug, clap::Args)] +//! Browser-backed `ts audit` command namespace. +//! +//! `ts audit page ` is the generic page audit; `ts audit ad-templates verify +//! ...` is the ad-template verifier; `ts audit generate ` bootstraps a +//! draft config from a live page (issue #800). `ts audit ` is a hidden +//! compatibility alias for `ts audit generate `. + +pub mod ad_templates; +pub mod browser; +mod browser_scroll; +pub mod collector; +pub mod generate; +pub mod page; + +use clap::{Args, Subcommand}; + +use crate::app_config::AppConfigArgs; +use crate::commands::audit::collector::{BrowserOpts, GenerateBrowserOpts}; +use crate::commands::audit::page::PageAuditArgs; +use crate::error::{CliResult, cli_error}; +use crate::run::RunOutcome; + +/// Parses and validates an `http`/`https` URL, rejecting all other schemes. +/// +/// # Errors +/// +/// Returns a user-facing string when the input is not a valid `http`/`https` URL. +pub(crate) fn parse_http_url(raw: &str) -> Result { + let url = url::Url::parse(raw).map_err(|error| format!("invalid URL `{raw}`: {error}"))?; + match url.scheme() { + "http" | "https" => Ok(url), + other => Err(format!( + "unsupported URL scheme `{other}` (expected http or https)" + )), + } +} + +/// Parses a `name=value` cookie argument into its `(name, value)` parts. +/// +/// Splits on the first `=` so cookie values may themselves contain `=`. The name +/// must be non-empty; the value may be empty. +/// +/// # Errors +/// +/// Returns a user-facing string when the input has no `=` or an empty name. +pub(crate) fn parse_cookie(raw: &str) -> Result<(String, String), String> { + let (name, value) = raw + .split_once('=') + .ok_or_else(|| format!("invalid cookie `{raw}` (expected NAME=VALUE)"))?; + if name.is_empty() { + return Err(format!("invalid cookie `{raw}` (empty name)")); + } + Ok((name.to_string(), value.to_string())) +} + +/// `ts audit` arguments: an optional subcommand plus a hidden legacy URL positional. +#[derive(Debug, Args)] +#[command(arg_required_else_help = true)] pub(crate) struct AuditArgs { - /// Public HTTP(S) URL to audit. - pub(crate) url: String, + #[command(subcommand)] + pub(crate) command: Option, + /// Hidden compatibility alias: `ts audit ` behaves like `ts audit generate `. + /// + /// The hidden flags below all `requires` this positional, so putting one + /// before a subcommand (`ts audit --chrome X generate `) is rejected + /// rather than silently dropped. `value_name` keeps that rejection from + /// naming the field: an operator told to supply `` cannot find + /// it in `--help`, because the alias is deliberately undocumented. + #[arg(value_parser = parse_http_url, hide = true, value_name = "URL")] + pub(crate) legacy_url: Option, + #[command(flatten)] + pub(crate) legacy_generate: LegacyGenerateArgs, +} + +/// Hidden generation flags retained for the legacy `ts audit ` form. +#[derive(Debug, Default, Args)] +pub(crate) struct LegacyGenerateArgs { /// JavaScript asset audit output path. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) js_assets: Option, /// Draft Trusted Server config output path. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) config: Option, /// Do not write the JavaScript asset audit file. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) no_js_assets: bool, /// Do not write the draft Trusted Server config file. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) no_config: bool, /// Overwrite existing output files. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) force: bool, -} - -const DEFAULT_JS_ASSETS_PATH: &str = "js-assets.toml"; -const DEFAULT_CONFIG_PATH: &str = "trusted-server.toml"; - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum AssetParty { - FirstParty, - ThirdParty, -} - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub(crate) struct AuditedAsset { - pub(crate) kind: String, - pub(crate) url: String, - pub(crate) host: String, - pub(crate) party: AssetParty, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) integration: Option, -} - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub(crate) struct DetectedIntegration { - pub(crate) id: String, - pub(crate) evidence: String, -} - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub(crate) struct AuditArtifact { - pub(crate) audited_url: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) page_title: Option, - pub(crate) js_asset_count: usize, - pub(crate) third_party_asset_count: usize, - pub(crate) detected_integrations: Vec, - pub(crate) assets: Vec, - pub(crate) warnings: Vec, -} - -#[derive(Debug, Clone)] -pub(crate) struct AuditOutputs { - pub(crate) artifact: AuditArtifact, - pub(crate) js_assets_toml: String, - pub(crate) draft_config_toml: String, - pub(crate) js_asset_proxy_candidate_count: usize, -} - -#[derive(Debug, Clone)] -struct DraftConfig { - toml: String, - js_asset_proxy_candidate_count: usize, -} - -#[derive(Debug, Clone)] -struct JsAssetProxySection { - toml: String, - candidate_count: usize, -} - -#[derive(Debug, Default)] -struct JsAssetProxySkipCounts { - first_party: usize, - malformed_url: usize, - non_https: usize, - duplicate_url: usize, - non_script: usize, -} - -#[derive(Debug)] -struct JsAssetProxyCandidate<'a> { - origin_url: String, - integration: Option<&'a str>, -} - -trait OpaqueAssetPathGenerator { - fn next_path(&mut self) -> String; -} - -#[derive(Debug, Default)] -struct RandomOpaqueAssetPathGenerator; - -impl OpaqueAssetPathGenerator for RandomOpaqueAssetPathGenerator { - fn next_path(&mut self) -> String { - let mut bytes = [0_u8; 12]; - rand::rngs::OsRng.fill_bytes(&mut bytes); - format!("/assets/{}.js", lowercase_hex(&bytes)) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct AuditOutputPlan { - js_assets_path: Option, - config_path: Option, -} - -pub(crate) fn run_audit( - args: &AuditArgs, - collector: &dyn AuditCollector, - out: &mut dyn Write, -) -> CliResult<()> { - let target_url = parse_audit_url(&args.url)?; - let plan = resolve_output_plan(args)?; - let collected = collector.collect_page(&target_url)?; - let outputs = build_audit_outputs(&collected)?; - let wrote_config = plan.config_path.is_some(); - let written = write_audit_outputs(&outputs, &plan)?; - write_success_summary(&outputs, &written, wrote_config, out) -} - -fn parse_audit_url(value: &str) -> CliResult { - let url = Url::parse(value) - .map_err(|error| report_error(format!("invalid audit URL `{value}`: {error}")))?; - if !matches!(url.scheme(), "http" | "https") { - return cli_error(format!( - "`ts audit` only supports http/https URLs, got `{}`", - url.scheme() - )); - } - Ok(url) -} - -fn resolve_output_plan(args: &AuditArgs) -> CliResult { - if args.no_js_assets && args.no_config { - return cli_error("nothing to do: both --no-js-assets and --no-config were set"); - } - - let js_assets_path = if args.no_js_assets { - None - } else { - Some(resolve_output_path( - args.js_assets.as_deref(), - DEFAULT_JS_ASSETS_PATH, - )?) - }; - let config_path = if args.no_config { - None - } else { - Some(resolve_output_path( - args.config.as_deref(), - DEFAULT_CONFIG_PATH, - )?) - }; - - if js_assets_path.is_some() && js_assets_path == config_path { - return cli_error("audit output paths must be distinct"); - } - - for path in [&js_assets_path, &config_path].into_iter().flatten() { - if path.exists() && !args.force { - return cli_error(format!( - "refusing to overwrite existing file `{}`; re-run with --force", - path.display() - )); + /// Cookie to send with the page request, as `name=value`. Repeatable. + #[arg( + long = "cookie", + value_name = "NAME=VALUE", + value_parser = parse_cookie, + hide = true, + requires = "legacy_url" + )] + pub(crate) cookies: Vec<(String, String)>, + #[command(flatten)] + pub(crate) browser: LegacyBrowserOpts, +} + +/// Hidden browser flags retained for the legacy `ts audit ` form. +#[derive(Debug, Args)] +pub(crate) struct LegacyBrowserOpts { + /// Path to the Chrome/Chromium executable. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) chrome: Option, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) no_assume_consent: bool, + /// Route the browser through this proxy. + #[arg(long, value_name = "HOST:PORT", hide = true, requires = "legacy_url")] + pub(crate) browser_proxy: Option, + /// Quiet window in milliseconds that marks the page settled. + #[arg( + long, + default_value_t = crate::commands::audit::collector::GENERATE_SETTLE_QUIET_MS, + hide = true, + requires = "legacy_url" + )] + pub(crate) settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg( + long, + default_value_t = crate::commands::audit::collector::GENERATE_SETTLE_MAX_MS, + hide = true, + requires = "legacy_url" + )] + pub(crate) settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) danger_accept_invalid_certs: bool, +} + +impl Default for LegacyBrowserOpts { + fn default() -> Self { + Self { + chrome: None, + headful: false, + no_assume_consent: false, + browser_proxy: None, + settle_quiet_ms: crate::commands::audit::collector::GENERATE_SETTLE_QUIET_MS, + settle_max_ms: crate::commands::audit::collector::GENERATE_SETTLE_MAX_MS, + danger_accept_invalid_certs: false, } } - - Ok(AuditOutputPlan { - js_assets_path, - config_path, - }) -} - -fn resolve_output_path(path: Option<&Path>, default: &str) -> CliResult { - let candidate = path.unwrap_or_else(|| Path::new(default)); - if candidate.is_absolute() { - Ok(candidate.to_path_buf()) - } else { - Ok(std::env::current_dir() - .map_err(|error| report_error(format!("failed to read current directory: {error}")))? - .join(candidate)) - } } -fn build_audit_outputs(collected: &collector::CollectedPage) -> CliResult { - let artifact = analyze_collected_page(collected)?; - let final_url = collected - .final_url() - .map_err(|error| report_error(format!("invalid final URL: {error}")))?; - let js_assets_toml = toml::to_string_pretty(&artifact) - .map_err(|error| report_error(format!("failed to serialize audit artifact: {error}")))?; - let mut path_generator = RandomOpaqueAssetPathGenerator; - let draft_config = - build_draft_config_with_generator(&final_url, &artifact, &mut path_generator)?; - - Ok(AuditOutputs { - artifact, - js_assets_toml, - draft_config_toml: draft_config.toml, - js_asset_proxy_candidate_count: draft_config.js_asset_proxy_candidate_count, - }) -} - -fn write_audit_outputs(outputs: &AuditOutputs, plan: &AuditOutputPlan) -> CliResult> { - let selected_paths = [&plan.js_assets_path, &plan.config_path] - .into_iter() - .flatten() - .collect::>(); - for path in &selected_paths { - if let Some(parent) = path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - { - fs::create_dir_all(parent).map_err(|error| { - report_error(format!( - "failed to create parent directory {}: {error}", - parent.display() - )) - })?; +impl From<&LegacyBrowserOpts> for GenerateBrowserOpts { + fn from(options: &LegacyBrowserOpts) -> Self { + Self { + chrome: options.chrome.clone(), + headful: options.headful, + no_assume_consent: options.no_assume_consent, + browser_proxy: options.browser_proxy.clone(), + settle_quiet_ms: options.settle_quiet_ms, + settle_max_ms: options.settle_max_ms, + danger_accept_invalid_certs: options.danger_accept_invalid_certs, } } - - let mut written_paths = Vec::new(); - if let Some(path) = &plan.js_assets_path { - fs::write(path, &outputs.js_assets_toml).map_err(|error| { - report_error(format!( - "failed to write JS asset audit {}: {error}", - path.display() - )) - })?; - written_paths.push(path.display().to_string()); - } - if let Some(path) = &plan.config_path { - fs::write(path, &outputs.draft_config_toml).map_err(|error| { - report_error(format!( - "failed to write draft config {}: {error}", - path.display() - )) - })?; - written_paths.push(path.display().to_string()); - } - - Ok(written_paths) -} - -fn write_success_summary( - outputs: &AuditOutputs, - written: &[String], - wrote_config: bool, - out: &mut dyn Write, -) -> CliResult<()> { - let integrations = outputs - .artifact - .detected_integrations - .iter() - .map(|integration| integration.id.as_str()) - .collect::>(); - let draft_note = if wrote_config { - "\nDraft config: review before validation and push" - } else { - "" - }; - let asset_proxy_note = if wrote_config && outputs.js_asset_proxy_candidate_count > 0 { - format!( - "{} disabled entries written to draft config", - outputs.js_asset_proxy_candidate_count - ) - } else if wrote_config { - "none".to_string() - } else { - "not written (--no-config)".to_string() - }; - writeln!( - out, - "Audited {}\nTitle: {}\nJS assets: {}\nThird-party assets: {}\nDetected integrations: {}\nJS asset proxy candidates: {}\nWrote: {}{}", - outputs.artifact.audited_url, - outputs - .artifact - .page_title - .as_deref() - .unwrap_or(""), - outputs.artifact.js_asset_count, - outputs.artifact.third_party_asset_count, - if integrations.is_empty() { - "none".to_string() - } else { - integrations.join(", ") - }, - asset_proxy_note, - if written.is_empty() { - "none".to_string() - } else { - written.join(", ") - }, - draft_note - ) - .map_err(|error| report_error(format!("failed to write command output: {error}"))) } -fn build_draft_config_with_generator( - target_url: &Url, - artifact: &AuditArtifact, - path_generator: &mut dyn OpaqueAssetPathGenerator, -) -> CliResult { - let host = target_url - .host_str() - .ok_or_else(|| report_error("audited URL is missing a host"))?; - let origin = target_url.origin().ascii_serialization(); - let mut draft = EXAMPLE_CONFIG.to_string(); - - draft = replace_key_in_section( - &draft, - "publisher", - "domain", - &format!("domain = \"{host}\""), - )?; - draft = replace_key_in_section( - &draft, - "publisher", - "cookie_domain", - &format!("cookie_domain = \".{host}\""), - )?; - draft = replace_key_in_section( - &draft, - "publisher", - "origin_url", - &format!("origin_url = \"{origin}\""), - )?; - - let detected = artifact - .detected_integrations - .iter() - .map(|integration| integration.id.as_str()) - .collect::>(); - - if detected.contains("gpt") { - draft = replace_key_in_section(&draft, "integrations.gpt", "enabled", "enabled = true")?; - } - if detected.contains("didomi") { - draft = replace_key_in_section(&draft, "integrations.didomi", "enabled", "enabled = true")?; - } - if detected.contains("datadome") { - draft = - replace_key_in_section(&draft, "integrations.datadome", "enabled", "enabled = true")?; - } - - let asset_proxy_section = build_js_asset_proxy_section(artifact, path_generator)?; - draft = replace_js_asset_proxy_section(&draft, &asset_proxy_section.toml)?; - - let mut manual_review = Vec::new(); - if detected.contains("google_tag_manager") { - if let Some(gtm_id) = extract_gtm_container_id(artifact) { - draft = replace_key_in_section( - &draft, - "integrations.google_tag_manager", - "enabled", - "enabled = true", - )?; - draft = replace_key_in_section( - &draft, - "integrations.google_tag_manager", - "container_id", - &format!("container_id = \"{gtm_id}\""), - )?; - } else { - manual_review.push("google_tag_manager"); - } - } - - for integration in detected { - if !matches!( - integration, - "gpt" | "didomi" | "datadome" | "google_tag_manager" - ) { - manual_review.push(integration); - } - } - - if !manual_review.is_empty() { - if !draft.ends_with('\n') { - draft.push('\n'); - } - draft.push_str("\n# Audit findings requiring manual review\n"); - for integration in manual_review { - draft.push_str(&format!( - "# - Detected {integration}; review the corresponding [integrations.{integration}] section before enabling it.\n" - )); +/// `ts audit` subcommands. +#[derive(Debug, Subcommand)] +pub(crate) enum AuditSubcommand { + /// Audit a single page and print a read-only summary. + Page(PageAuditArgs), + /// Verify configured ad-template slots against live page evidence. + #[command(name = "ad-templates", subcommand)] + AdTemplates(AuditAdTemplatesCommand), + /// Bootstrap a draft Trusted Server config + JS asset audit from a live page. + Generate(generate::GenerateArgs), +} + +/// `ts audit ad-templates` subcommands. +#[derive(Debug, Subcommand)] +pub(crate) enum AuditAdTemplatesCommand { + /// Scrape a live page's GPT slots and update the config's + /// `[creative_opportunities]` slots in place. + Generate(AuditAdTemplatesGenerateArgs), + /// Verify ad-template slots for one or more live URLs. + Verify(AuditAdTemplatesVerifyArgs), +} + +/// Arguments for `ts audit ad-templates generate `. +#[derive(Debug, Args)] +pub(crate) struct AuditAdTemplatesGenerateArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page URL to scrape for GPT slots (http or https). + #[arg(value_parser = parse_http_url)] + pub url: url::Url, + /// Glob applied to every slot discovered this run (e.g. `/`, `/news/*`). + /// Repeatable. Defaults to the scraped URL's path. Re-running with a + /// different pattern unions it into slots already in the config. + #[arg(long = "page-pattern", value_name = "GLOB")] + pub page_patterns: Vec, + /// Replace all existing slots instead of merging this run into them. + #[arg(long)] + pub replace: bool, + /// Preview the updated config on stdout instead of writing it. + #[arg(long)] + pub dry_run: bool, + /// Perform a deterministic scroll pass after each page initially settles. + #[arg(long)] + pub scroll: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] + pub cookies: Vec<(String, String)>, + /// Maximum site sections to sample. Each contributes a landing page and an + /// article, so this bounds how much of the publisher's taxonomy is covered. + #[arg(long, default_value_t = 8)] + pub max_sections: usize, + /// Maximum pages to load in total, including the requested page. + /// + /// Set to 1 to restore single-page behavior: no crawl, no section + /// discovery, and the audited path as the only page pattern. + #[arg(long, default_value_t = 17)] + pub max_pages: usize, + /// Device profiles to audit, comma-separated: `desktop`, `mobile`. + /// + /// Defaults to `desktop`. Publishers often serve different GAM ad units per + /// device, which a single-profile crawl cannot see — it would infer a + /// template correct for the profile it used and silently wrong elsewhere. + /// Passing both crawls each page twice and refuses to write an ad-unit path + /// for any slot where the profiles disagree. + #[arg(long, value_delimiter = ',', default_value = "desktop")] + pub profiles: Vec, + /// Pause in milliseconds between page loads during the crawl. + /// + /// A crawl issues a dozen navigations in a row. Firing them back to back is + /// discourteous to the origin, and request pacing is one of the signals bot + /// protection scores, so an unpaced crawl can trigger the challenge that + /// empties the rest of the run. + #[arg(long, default_value_t = 750)] + pub page_delay_ms: u64, + /// Browser and consent options shared with `ts audit generate`. + #[command(flatten)] + pub browser: GenerateBrowserOpts, +} + +impl AuditAdTemplatesGenerateArgs { + /// The crawl bounds these arguments describe. + pub(crate) fn budget(&self) -> generate::CrawlBudget { + generate::CrawlBudget { + max_sections: self.max_sections, + max_pages: self.max_pages, } } - Ok(DraftConfig { - toml: draft, - js_asset_proxy_candidate_count: asset_proxy_section.candidate_count, - }) -} - -fn build_js_asset_proxy_section( - artifact: &AuditArtifact, - path_generator: &mut dyn OpaqueAssetPathGenerator, -) -> CliResult { - let (candidates, skipped) = select_js_asset_proxy_candidates(artifact); - let mut used_paths = BTreeSet::new(); - let mut toml = String::new(); - - toml.push_str("[integrations.js_asset_proxy]\n"); - toml.push_str("enabled = false\n"); - toml.push_str("# Uncomment to override upstream cache headers for every asset below.\n"); - toml.push_str("# This replaces upstream directives, including private and no-store.\n"); - toml.push_str("# Use only when each asset's bytes are identical for every visitor.\n"); - toml.push_str("# cache_ttl_seconds = 3600\n"); - toml.push_str( - "# Asset fetches use a fixed TrustedServer/1.0 User-Agent. Do not proxy assets\n", - ); - toml.push_str( - "# that vary by browser User-Agent or use integrity hashes for UA-specific bytes.\n\n", - ); - toml.push_str("# Generated by `ts audit`; review before enabling.\n"); - toml.push_str( - "# Audit note: some discovered scripts may be runtime-injected and may not appear\n", - ); - toml.push_str( - "# in origin HTML. JS Asset Proxy rewrites only matching script src URLs present in\n", - ); - toml.push_str("# HTML processed by Trusted Server.\n"); - - if candidates.is_empty() { - toml.push_str( - "# No eligible third-party HTTPS script assets were detected by `ts audit`.\n", - ); - } - - for candidate in &candidates { - let generated_path = generate_unique_asset_path(path_generator, &mut used_paths)?; - toml.push('\n'); - toml.push_str("# Generated by `ts audit`; review before enabling.\n"); - if let Some(integration) = candidate.integration { - let integration = sanitized_comment_value(integration); - toml.push_str(&format!("# Detected integration: {integration}\n")); - toml.push_str(&format!( - "# Native integration may be preferable: [integrations.{integration}]\n" - )); + /// The device profiles to audit, deduplicated in the order given. + /// + /// # Errors + /// + /// Returns an error when a name is not a known profile, or when none were + /// given. + pub(crate) fn profiles(&self) -> Result, String> { + let mut profiles: Vec = Vec::new(); + for raw in &self.profiles { + let profile = generate::DeviceProfile::parse(raw)?; + if !profiles.contains(&profile) { + profiles.push(profile); + } } - toml.push_str("[[integrations.js_asset_proxy.assets]]\n"); - toml.push_str(&format!("path = {}\n", toml_quoted_string(&generated_path))); - toml.push_str(&format!( - "origin_url = {}\n", - toml_quoted_string(&candidate.origin_url) - )); - if Url::parse(&candidate.origin_url).is_ok_and(|url| url.query().is_some()) { - toml.push_str( - "# This URL includes a query string and must remain stable for proxy matching.\n", - ); + if profiles.is_empty() { + return Err("--profiles needs at least one of: desktop, mobile".to_string()); } - toml.push_str("proxy = \"disabled\"\n"); + Ok(profiles) } - - append_js_asset_proxy_skip_comments(&mut toml, &skipped); - toml.push('\n'); - - Ok(JsAssetProxySection { - toml, - candidate_count: candidates.len(), - }) } -fn select_js_asset_proxy_candidates( - artifact: &AuditArtifact, -) -> (Vec>, JsAssetProxySkipCounts) { - let mut candidates = Vec::new(); - let mut skipped = JsAssetProxySkipCounts::default(); - let mut seen_origin_urls = BTreeSet::new(); - - for asset in &artifact.assets { - if asset.kind != "script" { - skipped.non_script += 1; - continue; - } - if asset.party != AssetParty::ThirdParty { - skipped.first_party += 1; - continue; - } - - let Ok(url) = Url::parse(&asset.url) else { - skipped.malformed_url += 1; - continue; - }; - if url.host_str().is_none() { - skipped.malformed_url += 1; - continue; - } - if url.scheme() != "https" { - skipped.non_https += 1; - continue; - } - - let origin_url = url.to_string(); - if !seen_origin_urls.insert(origin_url.clone()) { - skipped.duplicate_url += 1; - continue; - } - - candidates.push(JsAssetProxyCandidate { - origin_url, - integration: asset.integration.as_deref(), - }); - } - - (candidates, skipped) -} - -fn generate_unique_asset_path( - path_generator: &mut dyn OpaqueAssetPathGenerator, - used_paths: &mut BTreeSet, -) -> CliResult { - for _ in 0..128 { - let path = path_generator.next_path(); - if !is_valid_generated_asset_path(&path) { - return cli_error(format!( - "generated JS asset proxy path `{path}` is invalid; expected /assets/.js" - )); - } - if used_paths.insert(path.clone()) { - return Ok(path); +/// Arguments for `ts audit ad-templates verify ...`. +#[derive(Debug, Args)] +pub(crate) struct AuditAdTemplatesVerifyArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// One or more page URLs to verify (http or https). + #[arg(required = true, value_parser = parse_http_url)] + pub urls: Vec, + /// Exit non-zero when a matched slot is missing or only partially confirmed. + #[arg(long)] + pub strict: bool, + /// Emit machine-readable JSON instead of human output. + #[arg(long)] + pub json: bool, + /// Perform a deterministic scroll pass after the initial settle. + #[arg(long)] + pub scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + /// + /// Off by default: slots are matched on the post-redirect path, so an + /// off-origin page could otherwise satisfy `--strict`. Enable only for a + /// known redirect between your own properties (e.g. apex to `www`). + #[arg(long)] + pub allow_cross_origin_redirect: bool, + /// Cookie to send with each page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] + pub cookies: Vec<(String, String)>, + #[command(flatten)] + pub browser: BrowserOpts, +} + +/// Dispatches a `ts audit` invocation. +/// +/// `legacy_url` (if present) routes to artifact generation, while the `page` +/// subcommand routes to the generic read-only page audit. +/// +/// # Errors +/// +/// Returns a user-facing string when no URL or subcommand is provided, or when +/// the underlying command fails. +pub(crate) fn run_audit(args: &AuditArgs) -> Result { + match &args.command { + Some(AuditSubcommand::Page(page_args)) => { + page::run_page(page_args).map(|()| RunOutcome::Success) } - } - - cli_error("failed to generate a unique JS asset proxy path after 128 attempts") -} - -fn is_valid_generated_asset_path(path: &str) -> bool { - let Some(opaque_id) = path - .strip_prefix("/assets/") - .and_then(|value| value.strip_suffix(".js")) - else { - return false; - }; - - !opaque_id.is_empty() - && opaque_id - .chars() - .all(|ch| ch.is_ascii_hexdigit() && !ch.is_ascii_uppercase()) -} - -fn replace_js_asset_proxy_section(document: &str, replacement: &str) -> CliResult { - let lines = document.lines().collect::>(); - let start = lines - .iter() - .position(|line| line.trim() == "[integrations.js_asset_proxy]") - .ok_or_else(|| { - report_error( - "failed to update starter config because section `[integrations.js_asset_proxy]` was not found", + Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Generate(gen_args))) => { + gen_args.browser.validate()?; + let app_config_path = crate::app_config::resolve_app_config_file(&gen_args.config)?; + let raw_config = std::fs::read_to_string(&app_config_path).map_err(|error| { + format!("failed to read {}: {error}", app_config_path.display()) + })?; + let existing_creative = creative_config(&raw_config, &app_config_path)?; + let profiles = gen_args.profiles()?; + let collectors: Vec = profiles + .iter() + .map(|profile| { + generate::browser_collector::BrowserAuditCollector::with_profile(*profile) + .with_page_delay(std::time::Duration::from_millis(gen_args.page_delay_ms)) + .with_browser_options(&gen_args.browser) + .with_scroll(gen_args.scroll) + }) + .collect(); + let selected: Vec<(&str, &dyn generate::collector::AuditCollector)> = profiles + .iter() + .zip(collectors.iter()) + .map(|(profile, collector)| { + ( + profile.label(), + collector as &dyn generate::collector::AuditCollector, + ) + }) + .collect(); + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let stderr = std::io::stderr(); + let mut err = stderr.lock(); + generate::run_update_slots( + &generate::UpdateSlotsRequest { + url: gen_args.url.as_str(), + config_path: &app_config_path, + existing_creative: existing_creative.as_ref(), + page_patterns: &gen_args.page_patterns, + replace: gen_args.replace, + cookies: &gen_args.cookies, + dry_run: gen_args.dry_run, + scroll: gen_args.scroll, + budget: gen_args.budget(), + }, + &selected, + &mut out, + &mut err, ) - })?; - let mut end = start + 1; - - while end < lines.len() { - let trimmed = lines[end].trim(); - if trimmed.starts_with('[') - && trimmed.ends_with(']') - && trimmed != "[[integrations.js_asset_proxy.assets]]" - { - break; + .map(|()| RunOutcome::Success) } - end += 1; - } - - // Blank lines and comments directly above the next section header document - // that section, not this one, so leave them in the draft. - while end > start + 1 { - let trimmed = lines[end - 1].trim(); - if trimmed.is_empty() || trimmed.starts_with('#') { - end -= 1; - } else { - break; + Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Verify(verify_args))) => { + ad_templates::run_verify(verify_args) } - } - - let mut output_lines = Vec::new(); - output_lines.extend_from_slice(&lines[..start]); - output_lines.extend(replacement.trim_end_matches('\n').lines()); - if end < lines.len() && !lines[end].trim().is_empty() { - output_lines.push(""); - } - output_lines.extend_from_slice(&lines[end..]); - - let mut output = output_lines.join("\n"); - if document.ends_with('\n') { - output.push('\n'); - } - Ok(output) -} - -fn append_js_asset_proxy_skip_comments(toml: &mut String, skipped: &JsAssetProxySkipCounts) { - if skipped.first_party == 0 - && skipped.malformed_url == 0 - && skipped.non_https == 0 - && skipped.duplicate_url == 0 - && skipped.non_script == 0 - { - return; - } - - toml.push('\n'); - toml.push_str("# Skipped JS Asset Proxy audit candidates:\n"); - append_skip_count(toml, skipped.first_party, "first-party script"); - append_skip_count(toml, skipped.malformed_url, "malformed script URL"); - append_skip_count(toml, skipped.non_https, "non-HTTPS third-party script"); - append_skip_count(toml, skipped.duplicate_url, "duplicate script URL"); - append_skip_count(toml, skipped.non_script, "non-script asset"); -} - -fn append_skip_count(toml: &mut String, count: usize, label: &str) { - if count == 0 { - return; - } - - let plural = if count == 1 { "" } else { "s" }; - toml.push_str(&format!("# - {count} {label}{plural}\n")); -} - -fn sanitized_comment_value(value: &str) -> String { - value - .chars() - .map(|ch| if ch.is_control() { ' ' } else { ch }) - .collect() -} - -fn toml_quoted_string(value: &str) -> String { - let mut quoted = String::from("\""); - for ch in value.chars() { - match ch { - '\\' => quoted.push_str("\\\\"), - '"' => quoted.push_str("\\\""), - '\n' => quoted.push_str("\\n"), - '\r' => quoted.push_str("\\r"), - '\t' => quoted.push_str("\\t"), - ch if ch.is_control() => { - write!(&mut quoted, "\\u{:04X}", ch as u32).expect("should write to string"); - } - ch => quoted.push(ch), + Some(AuditSubcommand::Generate(generate_args)) => { + generate_args.browser.validate()?; + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let collector = generate::browser_collector::BrowserAuditCollector::default() + .with_browser_options(&generate_args.browser); + generate::run_generate(generate_args, &collector, &mut out) + .map(|()| RunOutcome::Success) } + None => match args.legacy_url.as_ref() { + Some(url) => { + let generate_args = legacy_generate_args(args, url); + generate_args.browser.validate()?; + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let collector = generate::browser_collector::BrowserAuditCollector::default() + .with_browser_options(&generate_args.browser); + generate::run_generate(&generate_args, &collector, &mut out) + .map(|()| RunOutcome::Success) + } + None => Err( + "provide a URL or a subcommand (`generate`, `page`, `ad-templates`)".to_string(), + ), + }, } - quoted.push('"'); - quoted -} - -fn lowercase_hex(bytes: &[u8]) -> String { - const HEX: &[u8; 16] = b"0123456789abcdef"; - let mut encoded = String::with_capacity(bytes.len() * 2); - for byte in bytes { - encoded.push(HEX[(byte >> 4) as usize] as char); - encoded.push(HEX[(byte & 0x0f) as usize] as char); - } - encoded } -fn replace_key_in_section( +/// Reads the config's `[creative_opportunities]` section, when it has one. +/// +/// An unrelated invalid setting elsewhere in the document must not hide the +/// section — the runtime rejects such a file, but the operator still has to be +/// able to update slots in it — so the document is read as plain TOML rather +/// than through [`Settings`](trusted_server_core::settings::Settings). +/// +/// A section that is present but unreadable is *not* treated as absent. +/// `CreativeOpportunitiesConfig` uses `deny_unknown_fields`, so one mistyped key +/// would otherwise leave the merge with nothing to merge into and replace the +/// operator's entire slot array. +/// +/// # Errors +/// +/// Returns a user-facing error when the document is malformed or the section is +/// present but cannot be deserialized. +fn creative_config( document: &str, - section: &str, - key: &str, - replacement_line: &str, -) -> CliResult { - let section_header = format!("[{section}]"); - let mut in_section = false; - let mut replaced = false; - let mut saw_section = false; - let mut lines = Vec::new(); - - for line in document.lines() { - let trimmed = line.trim(); - if trimmed.starts_with('[') && trimmed.ends_with(']') { - in_section = trimmed == section_header; - saw_section |= in_section; - } - - if in_section && !replaced && is_key_line(trimmed, key) { - lines.push(replacement_line.to_string()); - replaced = true; - } else { - lines.push(line.to_string()); - } - } - - if !saw_section { - return cli_error(format!( - "failed to update starter config because section `{section_header}` was not found" - )); - } - if !replaced { - return cli_error(format!( - "failed to update starter config because key `{key}` was not found in `{section_header}`" - )); - } - - let mut output = lines.join("\n"); - if document.ends_with('\n') { - output.push('\n'); + path: &std::path::Path, +) -> CliResult> { + // Plain `format!`, not `report_error`: the top-level `[ts]` printer already + // logs whatever is returned here, and this message embeds a multi-line + // `toml::de::Error`, so logging it here too would print the whole block + // twice. The guidance leads so the parse error can trail unbroken. + let value = toml::from_str::(document).map_err(|error| { + format!( + "failed to parse {} before generating slots; fix the TOML syntax and re-run:\n{error}", + path.display() + ) + })?; + let Some(section) = value.get("creative_opportunities").cloned() else { + return Ok(None); + }; + match section.try_into() { + Ok(config) => Ok(Some(config)), + Err(error) => cli_error(format!( + "failed to read the existing `[creative_opportunities]` section, so generating \ + slots would discard the configured ones: {error}. Fix the section (or delete it) \ + and re-run" + )), } - Ok(output) } -fn is_key_line(trimmed_line: &str, key: &str) -> bool { - trimmed_line - .strip_prefix(key) - .and_then(|remaining| remaining.trim_start().strip_prefix('=')) - .is_some() +fn legacy_generate_args(args: &AuditArgs, url: &url::Url) -> generate::GenerateArgs { + generate::GenerateArgs { + url: url.to_string(), + js_assets: args.legacy_generate.js_assets.clone(), + config: args.legacy_generate.config.clone(), + no_js_assets: args.legacy_generate.no_js_assets, + no_config: args.legacy_generate.no_config, + force: args.legacy_generate.force, + cookies: args.legacy_generate.cookies.clone(), + browser: GenerateBrowserOpts::from(&args.legacy_generate.browser), + } } #[cfg(test)] mod tests { - use std::cell::Cell; - use std::collections::VecDeque; - - use tempfile::TempDir; - use super::*; - use crate::commands::audit::collector::{CollectedPage, CollectedRequest, CollectedScriptTag}; - - struct FakeCollector { - collected: CollectedPage, - calls: Cell, - } - - struct FixedPathGenerator { - paths: VecDeque, - } - - impl FixedPathGenerator { - fn new(paths: &[&str]) -> Self { - Self { - paths: paths.iter().map(|path| (*path).to_string()).collect(), - } - } - } - - impl OpaqueAssetPathGenerator for FixedPathGenerator { - fn next_path(&mut self) -> String { - self.paths - .pop_front() - .expect("should have a fixed generated asset path") - } - } - - impl FakeCollector { - fn new(collected: CollectedPage) -> Self { - Self { - collected, - calls: Cell::new(0), - } - } - } - - impl AuditCollector for FakeCollector { - fn collect_page(&self, _target_url: &Url) -> CliResult { - self.calls.set(self.calls.get() + 1); - Ok(self.collected.clone()) - } - } - - fn collected_page() -> CollectedPage { - CollectedPage { - requested_url: "https://publisher.example/page".to_string(), - final_url: "https://publisher.example/page".to_string(), - page_title: Some("Example Publisher".to_string()), - html: r#"Example Publisher"#.to_string(), - script_tags: vec![ - CollectedScriptTag { - src: Some("https://www.googletagmanager.com/gtm.js?id=GTM-ABC123".to_string()), - inline_text: None, - }, - CollectedScriptTag { - src: Some("https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string()), - inline_text: None, - }, - ], - network_requests: vec![CollectedRequest { - url: "https://cdn.publisher.example/app.js".to_string(), - resource_type: Some("script".to_string()), - }], - warnings: Vec::new(), - } - } - - fn audit_args(url: &str) -> AuditArgs { - AuditArgs { - url: url.to_string(), - js_assets: None, - config: None, - no_js_assets: false, - no_config: false, - force: false, - } - } - - fn audited_asset(url: &str, party: AssetParty, integration: Option<&str>) -> AuditedAsset { - AuditedAsset { - kind: "script".to_string(), - url: url.to_string(), - host: Url::parse(url) - .ok() - .and_then(|parsed| parsed.host_str().map(str::to_string)) - .unwrap_or_default(), - party, - integration: integration.map(str::to_string), - } - } - - #[test] - fn parse_audit_url_accepts_http_and_https() { - assert!(parse_audit_url("http://publisher.example").is_ok()); - assert!(parse_audit_url("https://publisher.example").is_ok()); - } #[test] - fn parse_audit_url_rejects_non_http_schemes() { - for url in [ - "file:///etc/passwd", - "data:text/html,hello", - "chrome://version", - ] { - let error = parse_audit_url(url).expect_err("should reject non-http URL"); - assert!( - format!("{error:?}").contains("only supports http/https"), - "should explain scheme restriction" - ); - } - } - - #[test] - fn resolve_output_plan_rejects_no_outputs() { - let mut args = audit_args("https://publisher.example"); - args.no_js_assets = true; - args.no_config = true; - - let error = resolve_output_plan(&args).expect_err("should reject empty output set"); - - assert!( - format!("{error:?}").contains("nothing to do"), - "should explain no-output error" + fn parse_cookie_splits_on_first_equals() { + let (name, value) = parse_cookie("datadome=abc=def~ghi").expect("should parse cookie"); + assert_eq!(name, "datadome", "name should be the pre-`=` portion"); + assert_eq!( + value, "abc=def~ghi", + "value should keep later `=` characters" ); } #[test] - fn resolve_output_plan_rejects_existing_files_without_force() { - let temp = TempDir::new().expect("should create temp dir"); - let path = temp.path().join("js-assets.toml"); - fs::write(&path, "existing").expect("should write existing file"); - let mut args = audit_args("https://publisher.example"); - args.js_assets = Some(path); - args.no_config = true; - - let error = resolve_output_plan(&args).expect_err("should reject overwrite"); - - assert!( - format!("{error:?}").contains("refusing to overwrite"), - "should explain overwrite refusal" - ); + fn parse_cookie_allows_empty_value() { + let (name, value) = parse_cookie("session=").expect("should parse empty value"); + assert_eq!(name, "session"); + assert!(value.is_empty(), "empty value should be allowed"); } #[test] - fn resolve_output_plan_allows_existing_files_with_force() { - let temp = TempDir::new().expect("should create temp dir"); - let path = temp.path().join("js-assets.toml"); - fs::write(&path, "existing").expect("should write existing file"); - let mut args = audit_args("https://publisher.example"); - args.js_assets = Some(path.clone()); - args.no_config = true; - args.force = true; + fn invalid_setting_outside_the_section_still_yields_creative_config() { + let document = "unknown_runtime_key = true\n\ + [creative_opportunities]\ngam_network_id = \"123\"\n"; - let plan = resolve_output_plan(&args).expect("should allow forced overwrite"); + let creative = creative_config(document, std::path::Path::new("trusted-server.toml")) + .expect("an unrelated invalid setting must not hide creative config") + .expect("the section is present"); - assert_eq!(plan.js_assets_path.as_deref(), Some(path.as_path())); + assert_eq!(creative.gam_network_id, "123"); } #[test] - fn run_audit_writes_selected_outputs_and_summary() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("audit/js-assets.toml"); - let config = temp.path().join("audit/trusted-server.toml"); - let args = AuditArgs { - url: "https://publisher.example/page".to_string(), - js_assets: Some(js_assets.clone()), - config: Some(config.clone()), - no_js_assets: false, - no_config: false, - force: false, - }; - let collector = FakeCollector::new(collected_page()); - let mut out = Vec::new(); - - run_audit(&args, &collector, &mut out).expect("should run audit"); - - assert_eq!(collector.calls.get(), 1, "should collect page once"); - assert!(js_assets.exists(), "should write JS assets"); - assert!(config.exists(), "should write draft config"); - let summary = String::from_utf8(out).expect("summary should be UTF-8"); - assert!(summary.contains("Audited https://publisher.example/page")); - assert!(summary.contains("Detected integrations: google_tag_manager, gpt")); - assert!(summary.contains("Draft config: review before validation and push")); - } - - #[test] - fn run_audit_respects_no_config() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("js-assets.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.js_assets = Some(js_assets.clone()); - args.no_config = true; - let collector = FakeCollector::new(collected_page()); - - run_audit(&args, &collector, &mut Vec::new()).expect("should run audit"); + fn absent_section_reads_as_absent() { + let creative = creative_config( + "[auction]\nenabled = true\n", + std::path::Path::new("trusted-server.toml"), + ) + .expect("should read the document"); - assert!(js_assets.exists(), "should write assets"); assert!( - !temp.path().join("trusted-server.toml").exists(), - "should not write config" + creative.is_none(), + "a document with no `[creative_opportunities]` has no configured slots" ); } #[test] - fn run_audit_respects_no_js_assets() { - let temp = TempDir::new().expect("should create temp dir"); - let config = temp.path().join("trusted-server.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.config = Some(config.clone()); - args.no_js_assets = true; - let collector = FakeCollector::new(collected_page()); - let mut out = Vec::new(); - - run_audit(&args, &collector, &mut out).expect("should run audit"); + fn malformed_document_is_rejected_before_creative_config_extraction() { + let error = creative_config( + "[creative_opportunities\ngam_network_id = \"123\"\n", + std::path::Path::new("/tmp/example/trusted-server.toml"), + ) + .expect_err("should reject malformed TOML"); - assert!(config.exists(), "should write config"); assert!( - !temp.path().join("js-assets.toml").exists(), - "should not write JS assets" + error.contains("failed to parse /tmp/example/trusted-server.toml"), + "error should name the config file it could not parse, got {error}" ); - let summary = String::from_utf8(out).expect("summary should be UTF-8"); - assert!(summary.contains("Draft config: review before validation and push")); - } - - #[test] - fn run_audit_writes_collector_warnings_to_asset_artifact() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("js-assets.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.js_assets = Some(js_assets.clone()); - args.no_config = true; - let mut collected = collected_page(); - collected.warnings.push( - "browser audit timed out while waiting for the page to settle; results may be partial" - .to_string(), - ); - let collector = FakeCollector::new(collected); - - run_audit(&args, &collector, &mut Vec::new()).expect("should run audit"); - - let artifact = fs::read_to_string(js_assets).expect("should read artifact"); assert!( - artifact.contains("results may be partial"), - "should persist collector warning" + error.contains("fix the TOML syntax and re-run:\n"), + "the guidance should lead so the multi-line parse error trails it, got {error}" ); } #[test] - fn run_audit_conflict_prevents_collection() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("js-assets.toml"); - fs::write(&js_assets, "existing").expect("should write existing file"); - let mut args = audit_args("https://publisher.example/page"); - args.js_assets = Some(js_assets); - args.no_config = true; - let collector = FakeCollector::new(collected_page()); - - let error = run_audit(&args, &collector, &mut Vec::new()) - .expect_err("should reject existing output"); + fn unreadable_section_is_refused_rather_than_read_as_absent() { + // `deny_unknown_fields` makes one mistyped key inside the section fail + // to deserialize. Reading that as "no slots configured" would let a + // merge replace the operator's entire slot array. + let document = "[creative_opportunities]\n\ + gam_network_id = \"123\"\n\ + gam_netwrok_id = \"123\"\n\ + [[creative_opportunities.slot]]\n\ + id = \"header\"\n\ + div_id = \"ad-header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + + let error = creative_config(document, std::path::Path::new("trusted-server.toml")) + .expect_err("should refuse an unreadable section"); - assert_eq!(collector.calls.get(), 0, "should not collect page"); assert!( - format!("{error:?}").contains("refusing to overwrite"), - "should report overwrite conflict" + error.contains("would discard the configured ones"), + "error should say what merging would cost, got {error}" ); } #[test] - fn build_draft_config_writes_disabled_js_asset_proxy_candidates() { - let url = Url::parse("https://publisher.example/page").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: Some("Example".to_string()), - js_asset_count: 2, - third_party_asset_count: 2, - detected_integrations: vec![DetectedIntegration { - id: "gpt".to_string(), - evidence: "https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string(), - }], - assets: vec![ - audited_asset( - "https://cdn.vendor.example/sdk.js", - AssetParty::ThirdParty, - None, - ), - audited_asset( - "https://securepubads.g.doubleclick.net/tag/js/gpt.js", - AssetParty::ThirdParty, - Some("gpt"), - ), - ], - warnings: Vec::new(), - }; - let mut generator = FixedPathGenerator::new(&[ - "/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js", - "/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js", - ]); - - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config"); - - assert_eq!( - draft.js_asset_proxy_candidate_count, 2, - "should report generated disabled entries" - ); - assert!( - draft - .toml - .contains("[integrations.js_asset_proxy]\nenabled = false") - ); - assert!(draft.toml.contains("/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js")); - assert!(draft.toml.contains("/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js")); + fn parse_cookie_rejects_missing_equals() { + let err = parse_cookie("datadome").expect_err("should reject missing `=`"); assert!( - draft - .toml - .contains("origin_url = \"https://cdn.vendor.example/sdk.js\"") - ); - assert!(draft.toml.contains("proxy = \"disabled\"")); - assert!(draft.toml.contains("Detected integration: gpt")); - assert!( - draft - .toml - .contains("Native integration may be preferable: [integrations.gpt]") - ); - assert!( - !draft.toml.contains("example-vendor-loader"), - "should remove starter-template placeholder asset" - ); - assert!( - draft.toml.contains( - "# Proxy behavior and first-party asset routing. Kept active with defaults.\n[proxy]" - ), - "should preserve documentation for the section following the replaced block" - ); - let parsed = - toml::from_str::(&draft.toml).expect("draft should parse as TOML"); - assert!( - parsed["integrations"]["js_asset_proxy"] - .get("cache_ttl_seconds") - .is_none(), - "generated config should inherit upstream cache headers by default" + err.contains("NAME=VALUE"), + "error should show expected form" ); } #[test] - fn generated_asset_proxy_paths_are_opaque() { - let url = Url::parse("https://publisher.example/page").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: None, - js_asset_count: 1, - third_party_asset_count: 1, - detected_integrations: Vec::new(), - assets: vec![audited_asset( - "https://cdn.vendor.example/vendor-loader.js", - AssetParty::ThirdParty, - None, - )], - warnings: Vec::new(), - }; - let mut generator = FixedPathGenerator::new(&["/assets/0123456789abcdef01234567.js"]); - - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config"); - let path_line = draft - .toml - .lines() - .find(|line| line.starts_with("path = ") && line.contains("0123456789abcdef")) - .expect("should include generated path"); - - assert!(path_line.contains("/assets/0123456789abcdef01234567.js")); - assert!( - !path_line.contains("vendor") - && !path_line.contains("cdn") - && !path_line.contains("loader"), - "generated path should not include vendor, domain, or filename semantics" - ); + fn parse_cookie_rejects_empty_name() { + let err = parse_cookie("=value").expect_err("should reject empty name"); + assert!(err.contains("empty name"), "error should name the problem"); } #[test] - fn asset_proxy_generation_deduplicates_and_summarizes_skips() { - let url = Url::parse("https://publisher.example/page").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: None, - js_asset_count: 4, - third_party_asset_count: 3, - detected_integrations: Vec::new(), - assets: vec![ - audited_asset( - "https://cdn.vendor.example/sdk.js", - AssetParty::ThirdParty, - None, - ), - audited_asset( - "https://cdn.vendor.example/sdk.js", - AssetParty::ThirdParty, - None, - ), - audited_asset( - "https://publisher.example/app.js", - AssetParty::FirstParty, - None, - ), - audited_asset( - "http://cdn.vendor.example/insecure.js", - AssetParty::ThirdParty, - None, - ), - ], - warnings: Vec::new(), + fn legacy_url_builds_artifact_generation_args() { + let args = AuditArgs { + command: None, + legacy_url: Some( + url::Url::parse("https://www.example.com/").expect("should parse URL"), + ), + legacy_generate: LegacyGenerateArgs { + js_assets: Some("audit/assets.toml".into()), + config: Some("audit/config.toml".into()), + no_js_assets: false, + no_config: false, + force: true, + cookies: vec![("session".to_string(), "example".to_string())], + browser: LegacyBrowserOpts { + headful: true, + ..LegacyBrowserOpts::default() + }, + }, }; - let mut generator = FixedPathGenerator::new(&["/assets/111111111111111111111111.js"]); - - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config"); - assert_eq!(draft.js_asset_proxy_candidate_count, 1); - assert_eq!( - draft - .toml - .matches("[[integrations.js_asset_proxy.assets]]") - .count(), - 1, - "should only emit one candidate entry" + let generate = legacy_generate_args( + &args, + args.legacy_url.as_ref().expect("should have legacy URL"), ); - assert!(draft.toml.contains("# - 1 first-party script")); - assert!(draft.toml.contains("# - 1 non-HTTPS third-party script")); - assert!(draft.toml.contains("# - 1 duplicate script URL")); - } - - #[test] - fn asset_proxy_generation_warns_about_query_string_candidates() { - let url = Url::parse("https://publisher.example/page").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: None, - js_asset_count: 2, - third_party_asset_count: 2, - detected_integrations: Vec::new(), - assets: vec![ - audited_asset( - "https://cdn.vendor.example/sdk.js?v=one", - AssetParty::ThirdParty, - None, - ), - audited_asset( - "https://cdn.vendor.example/sdk.js?v=two", - AssetParty::ThirdParty, - None, - ), - ], - warnings: Vec::new(), - }; - let mut generator = FixedPathGenerator::new(&[ - "/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js", - "/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js", - ]); - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config"); - - assert_eq!(draft.js_asset_proxy_candidate_count, 2); + assert_eq!(generate.url, "https://www.example.com/"); assert_eq!( - draft - .toml - .matches("This URL includes a query string and must remain stable") - .count(), - 2, - "each query-string candidate should explain exact-match behavior" + generate.js_assets.as_deref(), + Some(std::path::Path::new("audit/assets.toml")) ); - } - - #[test] - fn asset_proxy_generation_with_no_candidates_removes_placeholder_asset() { - let url = Url::parse("https://publisher.example/page").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: None, - js_asset_count: 1, - third_party_asset_count: 0, - detected_integrations: Vec::new(), - assets: vec![audited_asset( - "https://publisher.example/app.js", - AssetParty::FirstParty, - None, - )], - warnings: Vec::new(), - }; - let mut generator = FixedPathGenerator::new(&[]); - - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config"); - - assert_eq!(draft.js_asset_proxy_candidate_count, 0); - assert!( - draft - .toml - .contains("No eligible third-party HTTPS script assets") + assert_eq!( + generate.config.as_deref(), + Some(std::path::Path::new("audit/config.toml")) ); - assert!( - !draft - .toml - .contains("[[integrations.js_asset_proxy.assets]]"), - "should not emit asset array entries without candidates" + assert!(generate.force); + assert_eq!( + generate.cookies, + [("session".to_string(), "example".to_string())] ); assert!( - !draft.toml.contains("example-vendor-loader"), - "should remove starter-template placeholder asset" + generate.browser.headful, + "browser flags passed to the legacy form should reach generation" ); - toml::from_str::(&draft.toml).expect("draft should parse as TOML"); - } - - #[test] - fn run_audit_summary_reports_written_asset_proxy_candidates() { - let temp = TempDir::new().expect("should create temp dir"); - let config = temp.path().join("trusted-server.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.config = Some(config); - args.no_js_assets = true; - let collector = FakeCollector::new(collected_page()); - let mut out = Vec::new(); - - run_audit(&args, &collector, &mut out).expect("should run audit"); - - let summary = String::from_utf8(out).expect("summary should be UTF-8"); - assert!(summary.contains("JS asset proxy candidates:")); - assert!(summary.contains("disabled entries written to draft config")); - } - - #[test] - fn build_draft_config_uses_final_url_and_detected_integrations() { - let url = Url::parse("https://www.publisher.example:8443/path").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: Some("Example".to_string()), - js_asset_count: 2, - third_party_asset_count: 2, - detected_integrations: vec![ - DetectedIntegration { - id: "google_tag_manager".to_string(), - evidence: "GTM-ABC123".to_string(), - }, - DetectedIntegration { - id: "gpt".to_string(), - evidence: "https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string(), - }, - DetectedIntegration { - id: "prebid".to_string(), - evidence: "inline script matched `prebid`".to_string(), - }, - ], - assets: Vec::new(), - warnings: Vec::new(), - }; - - let mut generator = FixedPathGenerator::new(&[]); - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config") - .toml; - - assert!(draft.contains("domain = \"www.publisher.example\"")); - assert!(draft.contains("cookie_domain = \".www.publisher.example\"")); - assert!(draft.contains("origin_url = \"https://www.publisher.example:8443\"")); - assert!(draft.contains("[integrations.gpt]\nenabled = true")); - assert!(draft.contains("[integrations.google_tag_manager]\nenabled = true")); - assert!(draft.contains("container_id = \"GTM-ABC123\"")); - assert!(draft.contains("Detected prebid")); - toml::from_str::(&draft).expect("draft should parse as TOML"); - } - - #[test] - fn build_draft_config_does_not_enable_gtm_without_container_id() { - let url = Url::parse("https://publisher.example/path").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: None, - js_asset_count: 1, - third_party_asset_count: 1, - detected_integrations: vec![DetectedIntegration { - id: "google_tag_manager".to_string(), - evidence: "https://www.googletagmanager.com/gtm.js".to_string(), - }], - assets: Vec::new(), - warnings: Vec::new(), - }; - - let mut generator = FixedPathGenerator::new(&[]); - let draft = build_draft_config_with_generator(&url, &artifact, &mut generator) - .expect("should build draft config") - .toml; - - assert!(draft.contains("[integrations.google_tag_manager]\nenabled = false")); - assert!(draft.contains("Detected google_tag_manager")); } } diff --git a/crates/trusted-server-cli/src/commands/audit/page.rs b/crates/trusted-server-cli/src/commands/audit/page.rs new file mode 100644 index 000000000..31cbf4b1b --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/page.rs @@ -0,0 +1,157 @@ +//! Generic `ts audit page ` command: a read-only page summary. + +use std::io::{self, Write}; + +use clap::Args; + +use crate::ad_templates::output::escape_terminal_text; +use crate::commands::audit::browser::BrowserCollector; +use crate::commands::audit::collector::{ + AuditCollector, BrowserCollectRequest, BrowserOpts, CollectedPage, +}; + +/// Arguments for `ts audit page `. +#[derive(Debug, Args)] +pub(crate) struct PageAuditArgs { + /// The page URL to audit (http or https). + #[arg(value_parser = crate::commands::audit::parse_http_url)] + pub url: url::Url, + /// Perform a deterministic scroll pass after the initial settle. + #[arg(long)] + pub scroll: bool, + #[command(flatten)] + pub browser: BrowserOpts, +} + +/// Runs the generic page audit for the `page` subcommand. +/// +/// # Errors +/// +/// Returns a user-facing string when the browser cannot collect the page. +pub(crate) fn run_page(args: &PageAuditArgs) -> Result<(), String> { + args.browser.validate()?; + run_with_collector( + &BrowserCollector::from_opts(&args.browser), + &args.url, + args.scroll, + ) +} + +fn run_with_collector( + collector: &dyn AuditCollector, + url: &url::Url, + scroll: bool, +) -> Result<(), String> { + let page = collector.collect_page(BrowserCollectRequest { + url: url.clone(), + init_scripts: Vec::new(), + scroll, + collect_ad_evidence: false, + cookies: Vec::new(), + })?; + + let stdout = io::stdout(); + let mut out = stdout.lock(); + write_summary(&mut out, url, &page) +} + +fn write_summary(out: &mut dyn Write, url: &url::Url, page: &CollectedPage) -> Result<(), String> { + let to_err = |error: io::Error| format!("failed to write command output: {error}"); + writeln!(out, "url: {url}").map_err(to_err)?; + // The final URL, title, and collector warning messages are page-controlled, + // so escape control characters before they reach the operator's terminal. + writeln!( + out, + "final url: {}", + escape_terminal_text(page.final_url.as_str()) + ) + .map_err(to_err)?; + writeln!(out, "title: {}", escape_terminal_text(&page.title)).map_err(to_err)?; + writeln!(out, "scripts: {}", page.script_count).map_err(to_err)?; + writeln!(out, "resources: {}", page.resource_count).map_err(to_err)?; + for warning in &page.warnings { + writeln!( + out, + "warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(to_err)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ad_templates::output::Warning; + + fn collected(final_url: &str, title: &str, warnings: Vec) -> CollectedPage { + CollectedPage { + final_url: url::Url::parse(final_url).expect("should parse fixture URL"), + title: title.to_string(), + script_count: 3, + resource_count: 42, + warnings, + ad_evidence: None, + } + } + + fn summary(page: &CollectedPage, requested: &str) -> String { + let url = url::Url::parse(requested).expect("should parse requested URL"); + let mut out = Vec::new(); + write_summary(&mut out, &url, page).expect("should write summary"); + String::from_utf8(out).expect("summary should be UTF-8") + } + + #[test] + fn summary_reports_the_requested_and_final_urls_with_counts() { + let page = collected( + "https://publisher.example/news/story", + "Example Publisher", + Vec::new(), + ); + + let out = summary(&page, "https://publisher.example/news"); + + assert!( + out.contains("url: https://publisher.example/news\n"), + "should echo the requested URL, got {out:?}" + ); + assert!( + out.contains("final url: https://publisher.example/news/story\n"), + "should report the post-redirect URL, got {out:?}" + ); + assert!(out.contains("scripts: 3"), "got {out:?}"); + assert!(out.contains("resources: 42"), "got {out:?}"); + } + + #[test] + fn page_controlled_text_is_escaped_before_it_reaches_the_terminal() { + // Title and warning text are page-controlled and can contain raw + // terminal controls. URL percent-encoding is asserted separately. + let page = collected( + "https://publisher.example/a%1B%5B2Jb", + "Example\u{1b}[2J", + vec![Warning { + code: "page_\u{1b}[31m".to_string(), + message: "message\u{1b}[0m".to_string(), + }], + ); + + let out = summary(&page, "https://publisher.example/"); + + assert!( + !out.contains('\u{1b}'), + "no escape sequence may reach the terminal, got {out:?}" + ); + assert!( + out.contains("final url: https://publisher.example/a%1B%5B2Jb\n"), + "the final URL should retain URL's percent encoding, got {out:?}" + ); + assert!( + out.contains("warning [page_"), + "warnings should still be reported, got {out:?}" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/config/ad_templates.rs b/crates/trusted-server-cli/src/commands/config/ad_templates.rs new file mode 100644 index 000000000..b8d9f538e --- /dev/null +++ b/crates/trusted-server-cli/src/commands/config/ad_templates.rs @@ -0,0 +1,832 @@ +use std::collections::BTreeSet; +use std::io::{self, Write}; + +use crate::ad_templates::expected::normalize_path_or_url; +use crate::ad_templates::output::escape_terminal_text; +use crate::app_config::{AppConfigArgs, load_settings}; +use clap::{ArgGroup, Args, Subcommand}; +use http::Method; +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + AdStackGateInput, AdStackGateName, CreativeOpportunityFormat, CreativeOpportunitySlot, + RuntimeAdStackExpected, evaluate_ad_stack_gate, match_slots, validate_page_pattern, +}; + +use crate::run::RunOutcome; + +enum CheckFailure { + Tool(String), + Assertion(String), +} + +#[derive(Debug, Subcommand)] +pub enum AdTemplatesCommand { + /// Validate ad-template config and summarize deploy-time implications. + Lint(AdTemplatesLintArgs), + /// Show creative opportunity slots matching a page path or URL. + Match(AdTemplatesMatchArgs), + /// Assert that a page path or URL matches the expected slot set. + Check(AdTemplatesCheckArgs), + /// Explain why a page path or URL would or would not run the ad stack. + Explain(AdTemplatesExplainArgs), +} + +#[derive(Debug, Args)] +pub struct AdTemplatesLintArgs { + #[command(flatten)] + pub config: AppConfigArgs, +} + +#[derive(Debug, Args)] +pub struct AdTemplatesMatchArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page path or full URL to evaluate. + pub path_or_url: String, + /// Include slot div, GAM path, formats, and providers. + #[arg(long)] + pub details: bool, +} + +#[derive(Debug, Args)] +#[command(group( + ArgGroup::new("expectation") + .required(true) + .args(["expected_slots", "expect_no_slots"]) +))] +pub struct AdTemplatesCheckArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page path or full URL to evaluate. + pub path_or_url: String, + /// Expected slot id. Repeat for multiple slots. + #[arg(long = "expected-slot", value_name = "ID")] + pub expected_slots: Vec, + /// Assert that no slots match the page path or URL. + #[arg(long)] + pub expect_no_slots: bool, + /// Allow additional matched slots beyond --expected-slot values. + #[arg(long, conflicts_with = "expect_no_slots")] + pub allow_extra_slots: bool, +} + +#[derive(Debug, Args)] +pub struct AdTemplatesExplainArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page path or full URL to evaluate. + pub path_or_url: String, + /// HTTP method to model. + #[arg(long, default_value = "GET", value_parser = parse_http_method)] + pub method: Method, + /// Model a non-navigation request. + #[arg(long)] + pub non_navigation: bool, + /// Model a prefetch request. + #[arg(long)] + pub prefetch: bool, + /// Model a known crawler user agent. + #[arg(long)] + pub bot: bool, + /// Model consent denying server-side auction. + #[arg(long)] + pub consent_denied: bool, +} + +fn parse_http_method(raw: &str) -> Result { + let normalized = raw.to_ascii_uppercase(); + Method::from_bytes(normalized.as_bytes()) + .map_err(|error| format!("invalid HTTP method `{raw}`: {error}")) +} + +/// Run an ad-template CLI command. +/// +/// # Errors +/// +/// Returns a user-facing string when config loading, matching, or assertion +/// checks fail. +pub fn run_ad_templates(args: &AdTemplatesCommand) -> Result { + let stdout = io::stdout(); + let mut out = stdout.lock(); + if let AdTemplatesCommand::Check(args) = args { + return match run_check_classified(args, &mut out) { + Ok(()) => Ok(RunOutcome::Success), + Err(CheckFailure::Tool(error)) => Err(error), + Err(CheckFailure::Assertion(message)) => { + let stderr = io::stderr(); + let mut err = stderr.lock(); + writeln!(err, "{message}").map_err(output_error)?; + Ok(RunOutcome::AssertionFailed) + } + }; + } + run_ad_templates_with_writer(args, &mut out).map(|()| RunOutcome::Success) +} + +fn run_ad_templates_with_writer( + args: &AdTemplatesCommand, + out: &mut dyn Write, +) -> Result<(), String> { + match args { + AdTemplatesCommand::Lint(args) => run_lint(args, out), + AdTemplatesCommand::Match(args) => run_match(args, out), + AdTemplatesCommand::Check(args) => run_check(args, out), + AdTemplatesCommand::Explain(args) => run_explain(args, out), + } +} + +fn run_lint(args: &AdTemplatesLintArgs, out: &mut dyn Write) -> Result<(), String> { + let loaded = load_settings(&args.config)?; + writeln!(out, "app config: {}", loaded.app_config_path.display()).map_err(output_error)?; + + let Some(config) = &loaded.settings.creative_opportunities else { + writeln!(out, "server-side ad templates: not configured").map_err(output_error)?; + return Ok(()); + }; + + writeln!( + out, + "server-side ad templates: configured ({} slot{})", + config.slot.len(), + plural(config.slot.len()) + ) + .map_err(output_error)?; + writeln!( + out, + "gam_network_id: {}", + escape_terminal_text(&config.gam_network_id) + ) + .map_err(output_error)?; + writeln!( + out, + "auction_timeout_ms: {}", + config + .auction_timeout_ms + .unwrap_or(loaded.settings.auction.timeout_ms) + ) + .map_err(output_error)?; + writeln!( + out, + "creative_opportunities.enabled: {}", + if config.enabled { "true" } else { "false" } + ) + .map_err(output_error)?; + writeln!( + out, + "auction.enabled: {}", + if loaded.settings.auction.enabled { + "true" + } else { + "false" + } + ) + .map_err(output_error)?; + writeln!( + out, + "auction.providers: {}", + if loaded.settings.auction.providers.is_empty() { + "(none)".to_string() + } else { + let providers = loaded + .settings + .auction + .providers + .keys() + .map(trusted_server_core::auction::ProviderId::as_str) + .collect::>() + .join(", "); + escape_terminal_text(&providers).into_owned() + } + ) + .map_err(output_error)?; + + if config.slot.is_empty() { + writeln!(out, "status: disabled because no slots are configured").map_err(output_error)?; + } else if !config.enabled { + writeln!( + out, + "status: slots are configured, but [creative_opportunities].enabled is false" + ) + .map_err(output_error)?; + } else if !loaded.settings.auction.enabled { + writeln!( + out, + "status: slots are configured, but [auction].enabled is false" + ) + .map_err(output_error)?; + } else if loaded.settings.auction.providers.is_empty() { + writeln!( + out, + "status: slots are configured, but [auction].providers is empty" + ) + .map_err(output_error)?; + } else { + writeln!(out, "status: eligible for legacy-path server-side auctions") + .map_err(output_error)?; + } + + for slot in &config.slot { + for pattern in &slot.page_patterns { + if let Err(error) = validate_page_pattern(pattern) { + writeln!( + out, + "invalid page pattern for slot `{}`: {}", + escape_terminal_text(&slot.id), + escape_terminal_text(&error), + ) + .map_err(output_error)?; + } + } + } + + Ok(()) +} + +fn run_match(args: &AdTemplatesMatchArgs, out: &mut dyn Write) -> Result<(), String> { + let loaded = load_settings(&args.config)?; + let path = normalize_path_or_url(&args.path_or_url)?; + let Some(config) = &loaded.settings.creative_opportunities else { + writeln!( + out, + "{path}: no slots matched (creative_opportunities not configured)" + ) + .map_err(output_error)?; + return Ok(()); + }; + let matched = match_slots(&config.slot, &path); + + write_match_result( + out, + &path, + &matched, + &config.gam_network_id, + &config.section_for_path(&path), + args.details, + ) +} + +fn run_check(args: &AdTemplatesCheckArgs, out: &mut dyn Write) -> Result<(), String> { + run_check_classified(args, out).map_err(|failure| match failure { + CheckFailure::Tool(error) | CheckFailure::Assertion(error) => error, + }) +} + +fn run_check_classified( + args: &AdTemplatesCheckArgs, + out: &mut dyn Write, +) -> Result<(), CheckFailure> { + let loaded = load_settings(&args.config).map_err(CheckFailure::Tool)?; + let path = normalize_path_or_url(&args.path_or_url).map_err(CheckFailure::Tool)?; + let matched = loaded + .settings + .creative_opportunities + .as_ref() + .map(|config| match_slots(&config.slot, &path)) + .unwrap_or_default(); + let actual: BTreeSet<&str> = matched.iter().map(|slot| slot.id.as_str()).collect(); + + if args.expect_no_slots { + if actual.is_empty() { + writeln!(out, "{path}: OK, no slots matched") + .map_err(output_error) + .map_err(CheckFailure::Tool)?; + return Ok(()); + } + return Err(CheckFailure::Assertion(format!( + "{path}: expected no slots, matched {}", + join_set(&actual) + ))); + } + + let expected: BTreeSet<&str> = args.expected_slots.iter().map(String::as_str).collect(); + let missing: BTreeSet<&str> = expected.difference(&actual).copied().collect(); + let extra: BTreeSet<&str> = actual.difference(&expected).copied().collect(); + + if missing.is_empty() && (args.allow_extra_slots || extra.is_empty()) { + writeln!(out, "{path}: OK, matched {}", join_set(&actual)) + .map_err(output_error) + .map_err(CheckFailure::Tool)?; + return Ok(()); + } + + let mut problems = Vec::new(); + if !missing.is_empty() { + problems.push(format!("missing {}", join_set(&missing))); + } + if !args.allow_extra_slots && !extra.is_empty() { + problems.push(format!("unexpected {}", join_set(&extra))); + } + Err(CheckFailure::Assertion(format!( + "{path}: {}", + problems.join("; ") + ))) +} + +fn run_explain(args: &AdTemplatesExplainArgs, out: &mut dyn Write) -> Result<(), String> { + let loaded = load_settings(&args.config)?; + let path = normalize_path_or_url(&args.path_or_url)?; + writeln!(out, "path: {path}").map_err(output_error)?; + + let has_matches = if let Some(config) = &loaded.settings.creative_opportunities { + let matched = match_slots(&config.slot, &path); + write_match_result( + out, + &path, + &matched, + &config.gam_network_id, + &config.section_for_path(&path), + true, + )?; + !matched.is_empty() + } else { + writeln!(out, "creative_opportunities: not configured").map_err(output_error)?; + false + }; + + let method_pass = args.method == Method::GET; + let navigation_pass = !args.non_navigation; + let consent_pass = !args.consent_denied; + let auction_enabled = loaded.settings.auction.enabled; + let ad_templates_enabled = loaded + .settings + .creative_opportunities + .as_ref() + .is_some_and(|config| config.enabled); + let providers_configured = !loaded.settings.auction.providers.is_empty(); + + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get: method_pass, + navigation: navigation_pass, + prefetch: args.prefetch, + bot: args.bot, + matched_slots: has_matches, + consent_allows_auction: Some(consent_pass), + auction_enabled, + ad_templates_enabled, + }); + let blocked: Vec = gate.blocking_gates().collect(); + write_gate( + out, + "method GET", + !blocked.contains(&AdStackGateName::MethodGet), + )?; + write_gate( + out, + "navigation", + !blocked.contains(&AdStackGateName::Navigation), + )?; + write_gate( + out, + "not prefetch", + !blocked.contains(&AdStackGateName::NotPrefetch), + )?; + write_gate(out, "not bot", !blocked.contains(&AdStackGateName::NotBot))?; + write_gate( + out, + "consent allows auction", + !blocked.contains(&AdStackGateName::ConsentAllowsAuction), + )?; + write_gate( + out, + "auction.enabled", + !blocked.contains(&AdStackGateName::AuctionEnabled), + )?; + write_gate( + out, + "creative_opportunities.enabled", + !blocked.contains(&AdStackGateName::AdTemplatesEnabled), + )?; + write_gate( + out, + "matched slots", + !blocked.contains(&AdStackGateName::MatchedSlots), + )?; + writeln!( + out, + "advisory auction providers configured: {}", + if providers_configured { "yes" } else { "no" } + ) + .map_err(output_error)?; + writeln!( + out, + "server-side ad stack: {}", + match gate.expected { + RuntimeAdStackExpected::Yes => "yes", + RuntimeAdStackExpected::No => "no", + // `explain` always supplies a consent decision, which is the only + // input that yields `Unknown`; the arm is here for exhaustiveness. + RuntimeAdStackExpected::Unknown => "unknown", + } + ) + .map_err(output_error)?; + + Ok(()) +} + +fn write_match_result( + out: &mut dyn Write, + path: &str, + matched: &[&CreativeOpportunitySlot], + gam_network_id: &str, + section: &str, + details: bool, +) -> Result<(), String> { + if matched.is_empty() { + writeln!(out, "{}: no slots matched", escape_terminal_text(path)).map_err(output_error)?; + return Ok(()); + } + + let ids = matched + .iter() + .map(|slot| escape_terminal_text(&slot.id).into_owned()) + .collect::>() + .join(", "); + writeln!(out, "{}: matched {ids}", escape_terminal_text(path)).map_err(output_error)?; + + if details { + for slot in matched { + writeln!(out, "- {}", format_slot(slot, gam_network_id, section)) + .map_err(output_error)?; + } + } + + Ok(()) +} + +fn write_gate(out: &mut dyn Write, label: &str, pass: bool) -> Result<(), String> { + writeln!(out, "gate {label}: {}", if pass { "pass" } else { "block" }).map_err(output_error) +} + +/// Formats one matched slot for `--details` output. +/// +/// `section` is the value the runtime derives from the evaluated path, so a +/// `{section}` template renders the same unit path the live request would use. +fn format_slot(slot: &CreativeOpportunitySlot, gam_network_id: &str, section: &str) -> String { + let formats = slot + .formats + .iter() + .map(format_format) + .collect::>() + .join(", "); + let providers = format_providers(slot); + // `None` means a dynamic template renders past GAM's unit-path byte limit — + // a config the runtime rejects, so surface it rather than printing a path. + let gam_unit_path = slot + .render_gam_unit_path(gam_network_id, section) + .unwrap_or_else(|| "".to_string()); + format!( + "{} div={} gam={} patterns=[{}] formats=[{}] providers=[{}]", + escape_terminal_text(&slot.id), + escape_terminal_text(slot.resolved_div_id()), + escape_terminal_text(&gam_unit_path), + escape_terminal_text(&slot.page_patterns.join(", ")), + formats, + providers, + ) +} + +fn format_format(format: &CreativeOpportunityFormat) -> String { + let media_type = match format.media_type { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + }; + format!("{}x{} {media_type}", format.width, format.height) +} + +fn format_providers(slot: &CreativeOpportunitySlot) -> String { + let mut providers = Vec::new(); + if slot.providers.aps.is_some() { + providers.push("aps"); + } + if slot.providers.prebid.is_some() { + providers.push("prebid"); + } + if providers.is_empty() { + return "none".to_string(); + } + providers.join(", ") +} + +/// Renders a set of config-derived slot ids for the terminal. +/// +/// Config can arrive from a pushed blob or the env overlay, not only from a file +/// the operator read, so the ids are escaped before they reach a terminal — the +/// assertion-failure path prints them too. +fn join_set(set: &BTreeSet<&str>) -> String { + if set.is_empty() { + return "(none)".to_string(); + } + set.iter() + .map(|id| escape_terminal_text(id).into_owned()) + .collect::>() + .join(", ") +} + +fn plural(count: usize) -> &'static str { + if count == 1 { "" } else { "s" } +} + +#[allow( + clippy::needless_pass_by_value, + reason = "used as a map_err fn that receives io::Error by value" +)] +fn output_error(err: io::Error) -> String { + format!("failed to write command output: {err}") +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + + const EXAMPLE_CONFIG: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../trusted-server.example.toml" + )); + + fn project_with_config(config: &str) -> (TempDir, AppConfigArgs) { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + fs::write(&config_path, config).expect("should write app config"); + ( + temp, + AppConfigArgs { + app_config: Some(config_path), + manifest: manifest_path, + no_env: true, + }, + ) + } + + fn config_with_slots() -> String { + let base_config = EXAMPLE_CONFIG + .replace( + "password = \"handler_password\"", + "password = \"test-admin-password-32-bytes-minimum\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"test-proxy-secret-32-bytes-minimum\"", + ); + format!( + "{base_config}\n\ + [[creative_opportunities.slot]]\n\ + id = \"atf\"\n\ + page_patterns = [\"/news/*\", \"/\"]\n\ + formats = [{{ width = 300, height = 250 }}]\n\ + targeting = {{ zone = \"atf\" }}\n\ + [creative_opportunities.slot.providers.prebid]\n\ + bidders = {{}}\n\ + \n\ + [[creative_opportunities.slot]]\n\ + id = \"sports-sidebar\"\n\ + div_id = \"sports-ad\"\n\ + page_patterns = [\"/sports/*\"]\n\ + formats = [{{ width = 300, height = 600 }}]\n" + ) + } + + #[test] + fn match_reports_slots_for_path() { + let (_temp, config) = project_with_config(&config_with_slots()); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Match(AdTemplatesMatchArgs { + config, + path_or_url: "https://example.com/news/story?utm=1".to_string(), + details: true, + }), + &mut out, + ) + .expect("should match slots"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("/news/story: matched atf"), + "should report matched slot" + ); + assert!( + output.contains("formats=[300x250 banner]"), + "should include details" + ); + } + + #[test] + fn check_rejects_unexpected_extra_slots_by_default() { + let (_temp, config) = project_with_config(&config_with_slots()); + + let err = run_ad_templates_with_writer( + &AdTemplatesCommand::Check(AdTemplatesCheckArgs { + config, + path_or_url: "/sports/game".to_string(), + expected_slots: vec!["atf".to_string()], + expect_no_slots: false, + allow_extra_slots: false, + }), + &mut Vec::new(), + ) + .expect_err("should reject mismatch"); + + assert!( + err.contains("missing atf") && err.contains("unexpected sports-sidebar"), + "should describe missing and unexpected slots" + ); + } + + #[test] + fn check_accepts_no_slots() { + let (_temp, config) = project_with_config(&config_with_slots()); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Check(AdTemplatesCheckArgs { + config, + path_or_url: "/weather/today".to_string(), + expected_slots: Vec::new(), + expect_no_slots: true, + allow_extra_slots: false, + }), + &mut out, + ) + .expect("should accept no slots"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("/weather/today: OK, no slots matched"), + "should report no-slot assertion" + ); + } + + #[test] + fn explain_keeps_provider_state_separate_from_runtime_verdict() { + let config_text = config_with_slots().replacen( + "\nenabled = false\n# Rewrite", + "\nenabled = true\n# Rewrite", + 1, + ); + let (_temp, config) = project_with_config(&config_text); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Explain(AdTemplatesExplainArgs { + config, + path_or_url: "/news/story".to_string(), + method: Method::GET, + non_navigation: false, + prefetch: false, + bot: false, + consent_denied: false, + }), + &mut out, + ) + .expect("should explain path"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("server-side ad stack: yes"), + "runtime verdict should not include provider configuration: {output}" + ); + assert!( + output.contains("advisory auction providers configured: yes"), + "provider state should be a separate advisory: {output}" + ); + } + + #[test] + fn lint_reports_configured_slot_count_and_auction_state() { + let (_temp, config) = project_with_config(&config_with_slots()); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Lint(AdTemplatesLintArgs { config }), + &mut out, + ) + .expect("should lint configured slots"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("server-side ad templates: configured (2 slots)"), + "should report the configured slot count" + ); + assert!( + output.contains("auction.enabled:"), + "should report the auction kill-switch state" + ); + assert!(!output.contains("legacy fallback")); + } + + #[test] + fn lint_and_explain_report_the_disabled_template_switch() { + // `[creative_opportunities].enabled = false` is a runtime kill switch: + // the publisher path matches no slots at all while it is off, so the + // diagnostics must not claim the ad stack would run. + let config_text = config_with_slots().replace("enabled = true", "enabled = false"); + let (_temp, config) = project_with_config(&config_text); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Lint(AdTemplatesLintArgs { + config: config.clone(), + }), + &mut out, + ) + .expect("should lint a disabled template switch"); + let lint_output = String::from_utf8(out).expect("should be utf8"); + + assert!( + lint_output.contains("creative_opportunities.enabled: false"), + "lint should report the template switch state: {lint_output}" + ); + assert!( + lint_output.contains( + "status: slots are configured, but [creative_opportunities].enabled is false" + ), + "lint status should name the template switch: {lint_output}" + ); + + let mut out = Vec::new(); + run_ad_templates_with_writer( + &AdTemplatesCommand::Explain(AdTemplatesExplainArgs { + config, + path_or_url: "/news/story".to_string(), + method: Method::GET, + non_navigation: false, + prefetch: false, + bot: false, + consent_denied: false, + }), + &mut out, + ) + .expect("should explain a disabled template switch"); + let explain_output = String::from_utf8(out).expect("should be utf8"); + + assert!( + explain_output.contains("gate creative_opportunities.enabled: block"), + "explain should fail the template-switch gate: {explain_output}" + ); + assert!( + explain_output.contains("server-side ad stack: no"), + "explain verdict should follow the switch: {explain_output}" + ); + } + + #[test] + fn lint_reports_page_patterns_the_runtime_drops() { + let config_text = config_with_slots().replace( + "page_patterns = [\"/news/*\", \"/\"]", + "page_patterns = [\"/news/*\", \"[\"]", + ); + let (_temp, config) = project_with_config(&config_text); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Lint(AdTemplatesLintArgs { config }), + &mut out, + ) + .expect("should lint mixed valid and invalid patterns"); + let output = String::from_utf8(out).expect("should be utf8"); + + assert!( + output.contains("invalid page pattern for slot `atf`") + && output.contains("page pattern '[' is not a valid glob"), + "lint should surface the runtime-dropped pattern: {output}" + ); + } + + #[test] + fn public_check_reports_drift_as_assertion_outcome() { + let (_temp, config) = project_with_config(&config_with_slots()); + + let outcome = run_ad_templates(&AdTemplatesCommand::Check(AdTemplatesCheckArgs { + config, + path_or_url: "/sports/game".to_string(), + expected_slots: vec!["atf".to_string()], + expect_no_slots: false, + allow_extra_slots: false, + })) + .expect("assertion drift should not be a tool error"); + + assert_eq!(outcome, RunOutcome::AssertionFailed); + } + + #[test] + fn http_method_parser_normalizes_standard_methods() { + assert_eq!( + parse_http_method("get").expect("should parse lowercase GET"), + Method::GET, + "lowercase GET must evaluate the same runtime gate as uppercase GET" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/config/mod.rs b/crates/trusted-server-cli/src/commands/config/mod.rs index 43763f10a..77af7de85 100644 --- a/crates/trusted-server-cli/src/commands/config/mod.rs +++ b/crates/trusted-server-cli/src/commands/config/mod.rs @@ -1 +1,2 @@ +pub mod ad_templates; pub mod init; diff --git a/crates/trusted-server-cli/src/lib.rs b/crates/trusted-server-cli/src/lib.rs index 405bc6187..a3352ba93 100644 --- a/crates/trusted-server-cli/src/lib.rs +++ b/crates/trusted-server-cli/src/lib.rs @@ -1,4 +1,8 @@ #[cfg(not(target_arch = "wasm32"))] +mod ad_templates; +#[cfg(not(target_arch = "wasm32"))] +mod app_config; +#[cfg(not(target_arch = "wasm32"))] mod error; #[cfg(not(target_arch = "wasm32"))] mod prebid_bundle; @@ -6,7 +10,7 @@ mod prebid_bundle; mod run; #[cfg(not(target_arch = "wasm32"))] -pub use run::run_from_env; +pub use run::{RunOutcome, run_from_env}; // Every `ts` subcommand's implementation lives under `commands/`. The // `ts dev` group is available on every host target; its only subcommand, diff --git a/crates/trusted-server-cli/src/main.rs b/crates/trusted-server-cli/src/main.rs index 7cee5b1ca..0a325bd55 100644 --- a/crates/trusted-server-cli/src/main.rs +++ b/crates/trusted-server-cli/src/main.rs @@ -2,10 +2,21 @@ fn main() { use std::process; + // Dependencies such as chromiumoxide instrument their internals with + // `tracing`. Without a subscriber, tracing's log-compatibility fallback + // forwards tolerated CDP decode warnings into the CLI's user-facing logger. + // Trusted Server uses `log` for intentional operator output, so install a + // no-op tracing subscriber to keep dependency diagnostics out of stdout and + // stderr without changing the process-wide `log` level. + let _ = tracing::subscriber::set_global_default(tracing::subscriber::NoSubscriber::default()); edgezero_cli::init_cli_logger(); - if let Err(err) = trusted_server_cli::run_from_env() { - log::error!("[ts] {err}"); - process::exit(2); + match trusted_server_cli::run_from_env() { + Ok(outcome) if outcome.exit_code() != 0 => process::exit(outcome.exit_code()), + Ok(_) => {} + Err(err) => { + log::error!("[ts] {err}"); + process::exit(2); + } } } diff --git a/crates/trusted-server-cli/src/prebid_bundle.rs b/crates/trusted-server-cli/src/prebid_bundle.rs index abc545926..7a9eb4c66 100644 --- a/crates/trusted-server-cli/src/prebid_bundle.rs +++ b/crates/trusted-server-cli/src/prebid_bundle.rs @@ -10,6 +10,7 @@ use toml_edit::{DocumentMut, Item, table, value}; pub(crate) type CliResult = Result; const NODE_MODULES_MISSING_HELP: &str = "Prebid bundling dependencies are missing. Run `cd crates/trusted-server-js/lib && npm ci`, then retry `ts prebid bundle`."; +const USER_ID_REGISTRY_RELATIVE_PATH: &str = "src/integrations/prebid/user_id_modules.json"; #[derive(Debug, clap::Args)] pub(crate) struct PrebidBundleArgs { @@ -33,6 +34,7 @@ fn cli_error(message: impl Into) -> CliResult { pub(crate) struct PrebidBundleConfig { pub adapters: Vec, pub user_id_modules: Option>, + pub managed_user_id_names: Vec, pub external_bundle_url: Option, } @@ -120,11 +122,85 @@ fn npm_prebid_bundle_args(request: &PrebidBundleGenerateRequest) -> Vec #[derive(Debug, Deserialize)] struct PrebidBundleManifest { + #[serde(rename = "userIdModules")] + user_id_modules: Vec, sha256: String, sri: String, filename: String, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PrebidUserIdModuleRegistry { + modules: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PrebidUserIdModuleRegistryEntry { + module_name: String, + config_names: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct RequiredPrebidUserIdModule { + config_name: String, + module_name: String, +} + +fn load_user_id_registry(js_lib_dir: &Path) -> CliResult<(PathBuf, PrebidUserIdModuleRegistry)> { + let path = js_lib_dir.join(USER_ID_REGISTRY_RELATIVE_PATH); + let contents = fs::read_to_string(&path).map_err(|error| { + report_error(format!( + "failed to read Prebid User ID registry {}: {error}", + path.display() + )) + })?; + let registry = serde_json::from_str(&contents).map_err(|error| { + report_error(format!( + "failed to parse Prebid User ID registry {}: {error}", + path.display() + )) + })?; + Ok((path, registry)) +} + +fn resolve_managed_user_id_modules( + managed_names: &[String], + registry: &PrebidUserIdModuleRegistry, + registry_path: &Path, +) -> CliResult> { + managed_names + .iter() + .map(|config_name| { + let mut candidates = registry + .modules + .iter() + .filter(|entry| entry.config_names.iter().any(|name| name == config_name)) + .map(|entry| entry.module_name.clone()) + .collect::>(); + candidates.sort(); + candidates.dedup(); + + match candidates.as_slice() { + [] => cli_error(format!( + "managed User ID name {config_name:?} is not registered in {}", + registry_path.display() + )), + [module_name] => Ok(RequiredPrebidUserIdModule { + config_name: config_name.clone(), + module_name: module_name.clone(), + }), + _ => cli_error(format!( + "managed User ID name {config_name:?} is ambiguous in {}; candidate modules: {}", + registry_path.display(), + candidates.join(", ") + )), + } + }) + .collect() +} + pub(crate) fn run_bundle( args: &PrebidBundleArgs, generator: &mut dyn PrebidBundleGenerator, @@ -135,11 +211,51 @@ pub(crate) fn run_bundle( let current_dir = env::current_dir() .map_err(|error| report_error(format!("failed to read current directory: {error}")))?; let js_lib_dir = find_js_lib_dir(¤t_dir)?; - let out_dir = resolve_output_dir(¤t_dir, &args.out); + let (registry_path, registry) = load_user_id_registry(&js_lib_dir)?; + + run_bundle_with_context( + args, + config, + PrebidBundleRunContext { + current_dir: ¤t_dir, + js_lib_dir, + registry_path: ®istry_path, + registry: ®istry, + }, + generator, + out, + err, + ) +} + +struct PrebidBundleRunContext<'a> { + current_dir: &'a Path, + js_lib_dir: PathBuf, + registry_path: &'a Path, + registry: &'a PrebidUserIdModuleRegistry, +} + +fn run_bundle_with_context( + args: &PrebidBundleArgs, + config: PrebidBundleConfig, + context: PrebidBundleRunContext<'_>, + generator: &mut dyn PrebidBundleGenerator, + out: &mut dyn Write, + err: &mut dyn Write, +) -> CliResult<()> { + let requirements = resolve_managed_user_id_modules( + &config.managed_user_id_names, + context.registry, + context.registry_path, + )?; + let out_dir = resolve_output_dir(context.current_dir, &args.out); ensure_output_dir_writable(&out_dir)?; + let manifest_path = out_dir.join("manifest.json"); + invalidate_manifest(&manifest_path)?; + let request = PrebidBundleGenerateRequest { - js_lib_dir, + js_lib_dir: context.js_lib_dir, out_dir: out_dir.clone(), adapters: config.adapters, user_id_modules: config.user_id_modules, @@ -147,8 +263,8 @@ pub(crate) fn run_bundle( generator.generate(&request, out, err)?; - let manifest_path = out_dir.join("manifest.json"); let manifest = load_manifest(&manifest_path)?; + validate_managed_user_id_modules(&requirements, &manifest, &args.config)?; patch_config_metadata(&args.config, &manifest.sha256, &manifest.sri)?; writeln!( @@ -179,6 +295,40 @@ pub(crate) fn run_bundle( Ok(()) } +fn validate_managed_user_id_modules( + requirements: &[RequiredPrebidUserIdModule], + manifest: &PrebidBundleManifest, + config_path: &Path, +) -> CliResult<()> { + for requirement in requirements { + if !manifest + .user_id_modules + .iter() + .any(|module| module == &requirement.module_name) + { + return cli_error(format!( + "{} configures managed User ID {:?}, which requires Prebid module {:?}, but the generated manifest omits it; add {:?} to integrations.prebid.bundle.user_id_modules and rerun `ts prebid bundle`", + config_path.display(), + requirement.config_name, + requirement.module_name, + requirement.module_name, + )); + } + } + Ok(()) +} + +fn invalidate_manifest(path: &Path) -> CliResult<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => cli_error(format!( + "failed to remove stale Prebid manifest {}: {error}", + path.display() + )), + } +} + pub(crate) fn load_bundle_config(config_path: &Path) -> CliResult { let contents = fs::read_to_string(config_path).map_err(|error| { report_error(format!( @@ -235,6 +385,8 @@ pub(crate) fn load_bundle_config(config_path: &Path) -> CliResult CliResult CliResult> { + let Some(value) = prebid.get("managed_user_ids") else { + return Ok(Vec::new()); + }; + let entries = value.as_array().ok_or_else(|| { + report_error(format!( + "{} integrations.prebid.managed_user_ids must be an array of tables", + config_path.display() + )) + })?; + + entries + .iter() + .enumerate() + .map(|(index, entry)| { + let table = entry.as_table().ok_or_else(|| { + report_error(format!( + "{} integrations.prebid.managed_user_ids[{index}] must be a table", + config_path.display() + )) + })?; + let field = format!("integrations.prebid.managed_user_ids[{index}].name"); + let name = table + .get("name") + .and_then(toml::Value::as_str) + .ok_or_else(|| { + report_error(format!( + "{} {field} must be a non-empty string", + config_path.display() + )) + })?; + if name.trim().is_empty() { + return cli_error(format!( + "{} {field} must be a non-empty string", + config_path.display() + )); + } + Ok(name.to_string()) + }) + .collect() +} + fn read_required_string_array( table: &toml::Value, key: &str, @@ -568,6 +763,65 @@ user_id_modules = ["sharedIdSystem", "uid2IdSystem"] .to_string() } + fn managed_identity_link_config() -> String { + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-old.js" +external_bundle_sha256 = "old-sha256" +external_bundle_sri = "sha384-old" + +[[integrations.prebid.managed_user_ids]] +name = "identityLink" + +[integrations.prebid.bundle] +adapters = ["rubicon"] +user_id_modules = ["identityLinkIdSystem"] +"# + .to_string() + } + + fn two_managed_ids_config() -> String { + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +external_bundle_sha256 = "old-sha256" +external_bundle_sri = "sha384-old" + +[[integrations.prebid.managed_user_ids]] +name = "identityLink" + +[[integrations.prebid.managed_user_ids]] +name = "uid2" + +[integrations.prebid.bundle] +adapters = ["rubicon"] +user_id_modules = ["identityLinkIdSystem", "uid2IdSystem"] +"# + .to_string() + } + + fn shared_aliases_config() -> String { + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "sharedId" + +[[integrations.prebid.managed_user_ids]] +name = "pubCommonId" + +[integrations.prebid.bundle] +adapters = ["rubicon"] +user_id_modules = ["sharedIdSystem"] +"# + .to_string() + } + #[test] fn bundle_config_loader_accepts_valid_settings() { let (_temp, path) = write_config(&valid_config()); @@ -586,6 +840,7 @@ user_id_modules = ["sharedIdSystem", "uid2IdSystem"] config.external_bundle_url.as_deref(), Some("https://assets.example.com/prebid/trusted-prebid-old.js") ); + assert!(config.managed_user_id_names.is_empty()); } #[test] @@ -604,6 +859,238 @@ adapters = ["rubicon"] assert_eq!(config.adapters, ["rubicon"]); assert_eq!(config.user_id_modules, None); + assert!(config.managed_user_id_names.is_empty()); + } + + #[test] + fn bundle_config_loader_reads_managed_user_id_names_in_order() { + let (_temp, path) = write_config( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "identityLink" + +[[integrations.prebid.managed_user_ids]] +name = "pubCommonId" + +[integrations.prebid.bundle] +adapters = ["rubicon"] +"#, + ); + + let config = load_bundle_config(&path).expect("should load managed names"); + + assert_eq!( + config.managed_user_id_names, + ["identityLink", "pubCommonId"], + "should preserve managed entry order" + ); + } + + #[test] + fn bundle_config_loader_rejects_non_array_managed_user_ids() { + for managed_user_ids in ["\"identityLink\"", "{ name = \"identityLink\" }"] { + let (_temp, path) = write_config(&format!( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +managed_user_ids = {managed_user_ids} + +[integrations.prebid.bundle] +adapters = ["rubicon"] +"# + )); + + let error = load_bundle_config(&path).expect_err("should require an array"); + + assert!( + error.contains("integrations.prebid.managed_user_ids must be an array of tables"), + "should identify the malformed managed list: {error}" + ); + } + } + + #[test] + fn bundle_config_loader_rejects_non_table_managed_entry() { + let (_temp, path) = write_config( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +managed_user_ids = ["identityLink"] + +[integrations.prebid.bundle] +adapters = ["rubicon"] +"#, + ); + + let error = load_bundle_config(&path).expect_err("should require managed tables"); + + assert!( + error.contains("integrations.prebid.managed_user_ids[0] must be a table"), + "should identify the malformed managed entry: {error}" + ); + } + + #[test] + fn bundle_config_loader_rejects_managed_entry_without_string_name() { + for entry in [ + "{ params = { pid = \"999\" } }", + "{ name = 123 }", + "{ name = \"\" }", + "{ name = \" \" }", + ] { + let (_temp, path) = write_config(&format!( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +managed_user_ids = [{entry}] + +[integrations.prebid.bundle] +adapters = ["rubicon"] +"# + )); + + let error = load_bundle_config(&path).expect_err("should reject malformed name"); + + assert!( + error.contains("integrations.prebid.managed_user_ids[0].name"), + "should identify the malformed managed name: {error}" + ); + } + } + + #[test] + fn managed_names_resolve_aliases_to_registered_modules() { + let registry = PrebidUserIdModuleRegistry { + modules: vec![PrebidUserIdModuleRegistryEntry { + module_name: "sharedIdSystem".to_string(), + config_names: vec!["sharedId".to_string(), "pubCommonId".to_string()], + }], + }; + let registry_path = Path::new("user_id_modules.json"); + + let required = resolve_managed_user_id_modules( + &["pubCommonId".to_string(), "sharedId".to_string()], + ®istry, + registry_path, + ) + .expect("should resolve aliases"); + + assert_eq!( + required, + [ + RequiredPrebidUserIdModule { + config_name: "pubCommonId".to_string(), + module_name: "sharedIdSystem".to_string(), + }, + RequiredPrebidUserIdModule { + config_name: "sharedId".to_string(), + module_name: "sharedIdSystem".to_string(), + }, + ], + "should retain each managed name while allowing a shared module" + ); + } + + #[test] + fn checked_in_registry_resolves_identity_link() { + let current_dir = env::current_dir().expect("should read current directory"); + let js_lib_dir = find_js_lib_dir(¤t_dir).expect("should locate JS library"); + let (registry_path, registry) = + load_user_id_registry(&js_lib_dir).expect("should load checked-in registry"); + + let required = resolve_managed_user_id_modules( + &["identityLink".to_string()], + ®istry, + ®istry_path, + ) + .expect("should resolve checked-in identityLink entry"); + + assert_eq!( + required, + [RequiredPrebidUserIdModule { + config_name: "identityLink".to_string(), + module_name: "identityLinkIdSystem".to_string(), + }] + ); + } + + #[test] + fn unknown_managed_name_identifies_name_and_registry() { + let registry = PrebidUserIdModuleRegistry { + modules: Vec::new(), + }; + let registry_path = Path::new("registry/user_id_modules.json"); + + let error = + resolve_managed_user_id_modules(&["unknownId".to_string()], ®istry, registry_path) + .expect_err("should reject unknown name"); + + assert!( + error.contains("unknownId"), + "should identify the name: {error}" + ); + assert!( + error.contains(®istry_path.display().to_string()), + "should identify the registry: {error}" + ); + } + + #[test] + fn ambiguous_managed_name_lists_sorted_candidate_modules() { + let registry = PrebidUserIdModuleRegistry { + modules: vec![ + PrebidUserIdModuleRegistryEntry { + module_name: "zetaIdSystem".to_string(), + config_names: vec!["ambiguousId".to_string()], + }, + PrebidUserIdModuleRegistryEntry { + module_name: "alphaIdSystem".to_string(), + config_names: vec!["ambiguousId".to_string()], + }, + PrebidUserIdModuleRegistryEntry { + module_name: "zetaIdSystem".to_string(), + config_names: vec!["ambiguousId".to_string()], + }, + ], + }; + let registry_path = Path::new("registry/user_id_modules.json"); + + let error = + resolve_managed_user_id_modules(&["ambiguousId".to_string()], ®istry, registry_path) + .expect_err("should reject ambiguous name"); + + assert!( + error.contains("ambiguousId"), + "should identify the name: {error}" + ); + assert!( + error.contains("alphaIdSystem, zetaIdSystem"), + "should list sorted unique candidates: {error}" + ); + assert!( + error.contains(®istry_path.display().to_string()), + "should identify the registry: {error}" + ); + } + + #[test] + fn empty_managed_names_require_no_modules() { + let registry = PrebidUserIdModuleRegistry { + modules: Vec::new(), + }; + + let required = + resolve_managed_user_id_modules(&[], ®istry, Path::new("user_id_modules.json")) + .expect("should accept no managed names"); + + assert!(required.is_empty()); } #[test] @@ -792,7 +1279,18 @@ adapters = ["rubicon", 123] struct FakeGenerator { generate_error: Option, generate_calls: Vec, - write_manifest: bool, + manifest: Option, + } + + fn fake_manifest(user_id_modules: &serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "prebidVersion": "10.26.0", + "adapters": ["rubicon"], + "userIdModules": user_id_modules, + "sha256": "b".repeat(64), + "sri": "sha384-test", + "filename": format!("trusted-prebid-{}.js", "b".repeat(64)) + }) } impl PrebidBundleGenerator for FakeGenerator { @@ -809,21 +1307,10 @@ adapters = ["rubicon", 123] err.write_all(b"generator stderr\n") .expect("should capture generator stderr"); - if self.write_manifest { + if let Some(manifest) = &self.manifest { fs::create_dir_all(&request.out_dir).expect("should create output dir"); - fs::write( - request.out_dir.join("manifest.json"), - serde_json::json!({ - "prebidVersion": "10.26.0", - "adapters": request.adapters, - "userIdModules": request.user_id_modules.clone().unwrap_or_default(), - "sha256": "b".repeat(64), - "sri": "sha384-test", - "filename": format!("trusted-prebid-{}.js", "b".repeat(64)) - }) - .to_string(), - ) - .expect("should write fake manifest"); + fs::write(request.out_dir.join("manifest.json"), manifest.to_string()) + .expect("should write fake manifest"); } if let Some(error) = &self.generate_error { @@ -843,7 +1330,10 @@ adapters = ["rubicon", 123] let mut generator = FakeGenerator { generate_error: None, generate_calls: Vec::new(), - write_manifest: true, + manifest: Some(fake_manifest(&serde_json::json!([ + "sharedIdSystem", + "uid2IdSystem" + ]))), }; let mut out = Vec::new(); let mut err = Vec::new(); @@ -885,7 +1375,7 @@ adapters = ["rubicon", 123] let mut generator = FakeGenerator { generate_error: Some("builder failed".to_string()), generate_calls: Vec::new(), - write_manifest: false, + manifest: None, }; let mut out = Vec::new(); let mut err = Vec::new(); @@ -901,6 +1391,332 @@ adapters = ["rubicon", 123] assert!(fs::read_to_string(&args.config).expect("should read config") == original_config); } + #[test] + fn run_bundle_rejects_managed_name_when_manifest_omits_required_module() { + let (_temp, config_path) = write_config(&managed_identity_link_config()); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!(["sharedIdSystem"]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should reject missing managed module"); + + assert!( + error.contains("identityLink"), + "should name managed config: {error}" + ); + assert!( + error.contains("identityLinkIdSystem"), + "should name required module: {error}" + ); + assert!( + error.contains("integrations.prebid.bundle.user_id_modules"), + "should identify corrective field: {error}" + ); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original, + "should not patch metadata after consistency failure" + ); + } + + #[test] + fn run_bundle_accepts_manifest_with_required_managed_module() { + let (_temp, config_path) = write_config(&managed_identity_link_config()); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!(["identityLinkIdSystem"]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect("should accept required managed module"); + + assert_eq!(generator.generate_calls.len(), 1); + } + + #[test] + fn run_bundle_requires_every_managed_module() { + let (_temp, config_path) = write_config(&two_managed_ids_config()); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!(["identityLinkIdSystem"]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should require every managed module"); + + assert!( + error.contains("uid2"), + "should identify omitted config: {error}" + ); + assert!( + error.contains("uid2IdSystem"), + "should identify omitted module: {error}" + ); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original + ); + } + + #[test] + fn run_bundle_accepts_two_aliases_backed_by_one_module() { + let (_temp, config_path) = write_config(&shared_aliases_config()); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!(["sharedIdSystem"]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect("should accept aliases backed by one module"); + } + + #[test] + fn run_bundle_accepts_default_manifest_module_when_module_list_is_omitted() { + let config = managed_identity_link_config() + .replace("user_id_modules = [\"identityLinkIdSystem\"]\n", ""); + let (_temp, config_path) = write_config(&config); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!(["identityLinkIdSystem"]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect("should validate the generated default module set"); + + assert_eq!(generator.generate_calls[0].user_id_modules, None); + } + + #[test] + fn run_bundle_rejects_unknown_managed_name_before_generation() { + let config = managed_identity_link_config().replace("identityLink", "unknownId"); + let (_temp, config_path) = write_config(&config); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!([]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should reject unknown managed name"); + + assert!(error.contains("unknownId")); + assert!(generator.generate_calls.is_empty()); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original + ); + } + + #[test] + fn run_bundle_rejects_ambiguous_managed_name_before_generation() { + let config = managed_identity_link_config().replace("identityLink", "ambiguousId"); + let (_temp, config_path) = write_config(&config); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let registry = PrebidUserIdModuleRegistry { + modules: vec![ + PrebidUserIdModuleRegistryEntry { + module_name: "zetaIdSystem".to_string(), + config_names: vec!["ambiguousId".to_string()], + }, + PrebidUserIdModuleRegistryEntry { + module_name: "alphaIdSystem".to_string(), + config_names: vec!["ambiguousId".to_string()], + }, + ], + }; + let registry_path = Path::new("synthetic/user_id_modules.json"); + let args = PrebidBundleArgs { + config: config_path.clone(), + out: output_root.path().join("prebid"), + }; + let loaded = load_bundle_config(&config_path).expect("should load focused config"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!([]))), + }; + + let error = run_bundle_with_context( + &args, + loaded, + PrebidBundleRunContext { + current_dir: output_root.path(), + js_lib_dir: PathBuf::from("unused-js-lib"), + registry_path, + registry: ®istry, + }, + &mut generator, + &mut Vec::new(), + &mut Vec::new(), + ) + .expect_err("should reject ambiguous managed name"); + + assert!(error.contains("ambiguousId")); + assert!(error.contains("alphaIdSystem, zetaIdSystem")); + assert!(generator.generate_calls.is_empty()); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original + ); + } + + #[test] + fn run_bundle_rejects_malformed_managed_name_before_generation() { + let config = managed_identity_link_config() + .replace("name = \"identityLink\"", "params = { pid = \"999\" }"); + let (_temp, config_path) = write_config(&config); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!([]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should reject malformed managed name"); + + assert!(error.contains("managed_user_ids[0].name")); + assert!(generator.generate_calls.is_empty()); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original + ); + } + + #[test] + fn run_bundle_rejects_manifest_without_user_id_modules() { + let (_temp, config_path) = write_config(&managed_identity_link_config()); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut manifest = fake_manifest(&serde_json::json!([])); + manifest + .as_object_mut() + .expect("should be an object") + .remove("userIdModules"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(manifest), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should require manifest userIdModules"); + + assert!( + error.contains("userIdModules"), + "should identify missing field: {error}" + ); + } + + #[test] + fn run_bundle_rejects_non_array_manifest_user_id_modules() { + let (_temp, config_path) = write_config(&managed_identity_link_config()); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(&serde_json::json!("identityLinkIdSystem"))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should require array manifest userIdModules"); + + assert!( + error.contains("failed to parse generated Prebid manifest"), + "should identify manifest parsing: {error}" + ); + } + + #[test] + fn run_bundle_cannot_reuse_stale_manifest() { + let (_temp, config_path) = write_config(&managed_identity_link_config()); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let out_dir = output_root.path().join("prebid"); + fs::create_dir_all(&out_dir).expect("should create output directory"); + let manifest_path = out_dir.join("manifest.json"); + fs::write( + &manifest_path, + fake_manifest(&serde_json::json!(["identityLinkIdSystem"])).to_string(), + ) + .expect("should write stale manifest"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: None, + }; + let args = PrebidBundleArgs { + config: config_path, + out: out_dir, + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should reject missing fresh manifest"); + + assert!( + error.contains("manifest"), + "should identify missing manifest: {error}" + ); + assert!(!manifest_path.exists(), "should remove stale manifest"); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original + ); + } + #[test] fn missing_node_modules_fails_with_npm_ci_instruction() { let temp = tempfile::TempDir::new().expect("should create temp dir"); diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 13009d448..ec56238c1 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -7,8 +7,8 @@ use edgezero_cli::args::{ }; use trusted_server_core::config::TrustedServerAppConfig; -use crate::commands::audit::AuditArgs; -use crate::commands::audit::browser_collector::BrowserAuditCollector; +use crate::commands::audit::{AuditArgs, run_audit}; +use crate::commands::config::ad_templates::{AdTemplatesCommand, run_ad_templates}; use crate::commands::config::init::{ConfigInitArgs, run_config_init}; use crate::prebid_bundle::{NpmPrebidBundleGenerator, PrebidBundleArgs, run_bundle}; @@ -23,8 +23,8 @@ struct Args { enum Command { /// Print the currently active deployment version for a target adapter. ActiveVersion(ActiveVersionArgs), - /// Audit a public page and write draft Trusted Server artifacts. - Audit(AuditArgs), + /// Browser-backed page and ad-template audits. + Audit(Box), /// Sign in / out / status against an `EdgeZero` adapter. Auth(AuthArgs), /// Build the project for a target adapter. @@ -51,6 +51,9 @@ enum Command { #[derive(Debug, Subcommand)] enum ConfigCommand { + /// Diagnose server-side ad-template configuration and path matching. + #[command(name = "ad-templates", subcommand)] + AdTemplates(AdTemplatesCommand), /// Initialize a Trusted Server config file from the example template. Init(ConfigInitArgs), /// Diff `trusted-server.toml` against the live `EdgeZero` config. @@ -75,44 +78,71 @@ enum PrebidCommand { Bundle(PrebidBundleArgs), } +/// Process-level outcome for commands that distinguish drift from tool errors. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RunOutcome { + /// Command completed without drift. + Success, + /// Command ran successfully and found assertion drift. + AssertionFailed, +} + +impl RunOutcome { + /// Stable process exit code for this outcome. + #[must_use] + pub const fn exit_code(self) -> i32 { + match self { + Self::Success => 0, + Self::AssertionFailed => 1, + } + } +} + /// Run the CLI using process arguments. /// /// # Errors /// /// Returns an error when command parsing, config validation, `EdgeZero` /// delegation, audit collection, config initialization, or Prebid bundle generation fails. -pub fn run_from_env() -> Result<(), String> { +pub fn run_from_env() -> Result { dispatch(Args::parse()) } -fn dispatch(args: Args) -> Result<(), String> { +fn dispatch(args: Args) -> Result { match args.command { - Command::ActiveVersion(args) => edgezero_cli::run_active_version(&args), - Command::Audit(args) => { - let stdout = std::io::stdout(); - let mut out = stdout.lock(); - let collector = BrowserAuditCollector; - crate::commands::audit::run_audit(&args, &collector, &mut out) + Command::ActiveVersion(args) => { + edgezero_cli::run_active_version(&args).map(|()| RunOutcome::Success) + } + Command::Auth(args) => edgezero_cli::run_auth(&args).map(|()| RunOutcome::Success), + Command::Audit(args) => run_audit(&args), + Command::Build(args) => edgezero_cli::run_build(&args).map(|()| RunOutcome::Success), + Command::Config(ConfigCommand::AdTemplates(args)) => run_ad_templates(&args), + Command::Config(ConfigCommand::Init(args)) => { + run_config_init(&args).map(|()| RunOutcome::Success) } - Command::Auth(args) => edgezero_cli::run_auth(&args), - Command::Build(args) => edgezero_cli::run_build(&args), - Command::Config(ConfigCommand::Init(args)) => run_config_init(&args), Command::Config(ConfigCommand::Diff(args)) => { match edgezero_cli::run_config_diff_typed::(&args) { - Ok(edgezero_cli::DiffExit { code: 0 }) => Ok(()), + Ok(edgezero_cli::DiffExit { code: 0 }) => Ok(RunOutcome::Success), + Ok(edgezero_cli::DiffExit { code: 1 }) => Ok(RunOutcome::AssertionFailed), Ok(edgezero_cli::DiffExit { code }) => process::exit(code), Err(err) => Err(err), } } - Command::Config(ConfigCommand::Gc(args)) => edgezero_cli::run_config_gc(&args), + Command::Config(ConfigCommand::Gc(args)) => { + edgezero_cli::run_config_gc(&args).map(|()| RunOutcome::Success) + } Command::Config(ConfigCommand::Push(args)) => { edgezero_cli::run_config_push_typed::(&args) + .map(|()| RunOutcome::Success) } Command::Config(ConfigCommand::Validate(args)) => { edgezero_cli::run_config_validate_typed::(&args) + .map(|()| RunOutcome::Success) + } + Command::Deploy(args) => edgezero_cli::run_deploy(&args).map(|()| RunOutcome::Success), + Command::Healthcheck(args) => { + edgezero_cli::run_healthcheck(&args).map(|()| RunOutcome::Success) } - Command::Deploy(args) => edgezero_cli::run_deploy(&args), - Command::Healthcheck(args) => edgezero_cli::run_healthcheck(&args), Command::Prebid(prebid) => { let mut generator = NpmPrebidBundleGenerator; let mut stdout = std::io::stdout(); @@ -120,13 +150,16 @@ fn dispatch(args: Args) -> Result<(), String> { match prebid.command { PrebidCommand::Bundle(args) => { run_bundle(&args, &mut generator, &mut stdout, &mut stderr) + .map(|()| RunOutcome::Success) } } } - Command::Provision(args) => edgezero_cli::run_provision(&args), - Command::Rollback(args) => edgezero_cli::run_rollback(&args), - Command::Serve(args) => edgezero_cli::run_serve(&args), - Command::Dev(command) => crate::commands::dev::run(command), + Command::Provision(args) => { + edgezero_cli::run_provision(&args).map(|()| RunOutcome::Success) + } + Command::Rollback(args) => edgezero_cli::run_rollback(&args).map(|()| RunOutcome::Success), + Command::Serve(args) => edgezero_cli::run_serve(&args).map(|()| RunOutcome::Success), + Command::Dev(command) => crate::commands::dev::run(command).map(|()| RunOutcome::Success), } } @@ -143,6 +176,12 @@ mod tests { Args::try_parse_from(args).expect("should parse args") } + #[test] + fn run_outcomes_use_documented_exit_codes() { + assert_eq!(RunOutcome::Success.exit_code(), 0); + assert_eq!(RunOutcome::AssertionFailed.exit_code(), 1); + } + #[test] fn top_level_version_flag_is_available() { let err = Args::try_parse_from(["ts", "--version"]) @@ -353,64 +392,6 @@ mod tests { ); } - #[test] - fn parses_audit_with_default_outputs() { - let args = parse(&["ts", "audit", "https://publisher.example"]); - let Command::Audit(audit) = args.command else { - panic!("expected audit command"); - }; - assert_eq!(audit.url, "https://publisher.example"); - assert_eq!(audit.js_assets, None); - assert_eq!(audit.config, None); - assert!(!audit.no_js_assets); - assert!(!audit.no_config); - assert!(!audit.force); - } - - #[test] - fn parses_audit_with_custom_outputs() { - let args = parse(&[ - "ts", - "audit", - "https://publisher.example", - "--js-assets", - "audit/js-assets.toml", - "--config", - "audit/trusted-server.toml", - "--no-js-assets", - "--no-config", - "--force", - ]); - let Command::Audit(audit) = args.command else { - panic!("expected audit command"); - }; - assert_eq!(audit.js_assets, Some(PathBuf::from("audit/js-assets.toml"))); - assert_eq!( - audit.config, - Some(PathBuf::from("audit/trusted-server.toml")) - ); - assert!(audit.no_js_assets); - assert!(audit.no_config); - assert!(audit.force); - } - - #[test] - fn audit_does_not_accept_adapter_option() { - let error = Args::try_parse_from([ - "ts", - "audit", - "https://publisher.example", - "--adapter", - "fastly", - ]) - .expect_err("should reject audit adapter option"); - assert!( - error.to_string().contains("unexpected argument") - || error.to_string().contains("Found argument"), - "error should explain unsupported option" - ); - } - #[test] fn parses_build_with_adapter_args() { let args = parse(&[ @@ -638,6 +619,420 @@ mod tests { assert_eq!(validate.manifest, default_validate.manifest); } + #[test] + fn config_ad_templates_match_parses_app_config_flags() { + let args = parse(&[ + "ts", + "config", + "ad-templates", + "match", + "--app-config", + "publisher-a.toml", + "--no-env", + "--details", + "/news/story", + ]); + let Command::Config(ConfigCommand::AdTemplates(AdTemplatesCommand::Match(match_args))) = + args.command + else { + panic!("expected ad-templates match command"); + }; + assert_eq!( + match_args.config.app_config, + Some(PathBuf::from("publisher-a.toml")) + ); + assert!(match_args.config.no_env); + assert!(match_args.details); + assert_eq!(match_args.path_or_url, "/news/story"); + } + + #[test] + fn config_ad_templates_check_parses_expected_slots() { + let args = parse(&[ + "ts", + "config", + "ad-templates", + "check", + "/sports/game", + "--expected-slot", + "atf", + "--expected-slot", + "sports-sidebar", + "--allow-extra-slots", + ]); + let Command::Config(ConfigCommand::AdTemplates(AdTemplatesCommand::Check(check_args))) = + args.command + else { + panic!("expected ad-templates check command"); + }; + assert_eq!(check_args.path_or_url, "/sports/game"); + assert_eq!(check_args.expected_slots, ["atf", "sports-sidebar"]); + assert!(check_args.allow_extra_slots); + assert!(!check_args.expect_no_slots); + } + + #[test] + fn config_ad_templates_check_requires_an_expectation_mode() { + assert!(Args::try_parse_from(["ts", "config", "ad-templates", "check", "/news"]).is_err()); + } + + #[test] + fn config_ad_templates_check_rejects_extra_slots_with_no_slots_mode() { + assert!( + Args::try_parse_from([ + "ts", + "config", + "ad-templates", + "check", + "/news", + "--expect-no-slots", + "--allow-extra-slots", + ]) + .is_err() + ); + } + + #[test] + fn config_ad_templates_explain_rejects_removed_edgezero_model() { + assert!( + Args::try_parse_from([ + "ts", + "config", + "ad-templates", + "explain", + "/news", + "--edgezero-enabled", + ]) + .is_err() + ); + } + + #[test] + fn cli_definition_is_valid() { + // clap validates `requires` / `conflicts_with` argument-id references + // only from an explicit `debug_assert`. Without this, renaming or + // typoing an id compiles and ships. + ::command().debug_assert(); + } + + #[test] + fn bare_audit_namespace_displays_help_as_an_error() { + let error = Args::try_parse_from(["ts", "audit"]).expect_err("should require audit mode"); + + assert_eq!( + error.kind(), + clap::error::ErrorKind::DisplayHelpOnMissingArgumentOrSubcommand + ); + } + + #[test] + fn audit_legacy_url_parses_with_artifact_generation_flags() { + let args = parse(&[ + "ts", + "audit", + "https://www.example.com/", + "--js-assets", + "audit/assets.toml", + "--config", + "audit/config.toml", + "--force", + "--cookie", + "session=example", + "--chrome", + "/tmp/test-chrome", + "--headful", + "--no-assume-consent", + "--browser-proxy", + "127.0.0.1:8080", + "--settle-quiet-ms", + "900", + "--settle-max-ms", + "13000", + "--danger-accept-invalid-certs", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + assert_eq!( + audit.legacy_generate.js_assets, + Some(PathBuf::from("audit/assets.toml")) + ); + assert_eq!( + audit.legacy_generate.config, + Some(PathBuf::from("audit/config.toml")) + ); + assert!(audit.legacy_generate.force); + assert_eq!( + audit.legacy_generate.cookies, + [("session".to_string(), "example".to_string())] + ); + assert_eq!( + audit.legacy_generate.browser.chrome, + Some(PathBuf::from("/tmp/test-chrome")) + ); + assert!(audit.legacy_generate.browser.headful); + assert!(audit.legacy_generate.browser.no_assume_consent); + assert_eq!( + audit.legacy_generate.browser.browser_proxy.as_deref(), + Some("127.0.0.1:8080") + ); + assert_eq!(audit.legacy_generate.browser.settle_quiet_ms, 900); + assert_eq!(audit.legacy_generate.browser.settle_max_ms, 13_000); + assert!(audit.legacy_generate.browser.danger_accept_invalid_certs); + } + + #[test] + fn audit_help_does_not_advertise_hidden_legacy_browser_flags() { + let error = + Args::try_parse_from(["ts", "audit", "--help"]).expect_err("should render audit help"); + let help = error.to_string(); + + for flag in [ + "--chrome", + "--headful", + "--no-assume-consent", + "--browser-proxy", + "--settle-quiet-ms", + "--settle-max-ms", + "--danger-accept-invalid-certs", + ] { + assert!( + !help.contains(flag), + "`{flag}` is a legacy-only alias flag and must stay hidden; got {help}" + ); + } + } + + #[test] + fn audit_rejects_parent_browser_flags_before_a_subcommand() { + // `is_err()` alone would also pass if `--chrome` were deleted from + // `LegacyBrowserOpts` (an `UnknownArgument`), which is the opposite of + // the invariant this pins: the flag exists but requires the legacy URL. + let error = Args::try_parse_from([ + "ts", + "audit", + "--chrome", + "/tmp/test-chrome", + "generate", + "https://www.example.com/", + ]) + .expect_err("a parent-level browser flag must not be silently ignored"); + + assert_eq!( + error.kind(), + clap::error::ErrorKind::MissingRequiredArgument, + "should reject the flag for lacking the legacy URL it requires" + ); + } + + #[test] + fn audit_page_subcommand_parses_with_page_settle_defaults() { + let args = parse(&["ts", "audit", "page", "https://www.example.com/"]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::Page(page)) = audit.command else { + panic!("expected audit page command"); + }; + assert_eq!(page.browser.settle_quiet_ms, 750); + assert_eq!(page.browser.settle_max_ms, 10_000); + } + + #[test] + fn audit_generate_subcommands_use_generation_settle_defaults() { + let args = parse(&["ts", "audit", "generate", "https://www.example.com/"]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::Generate(generate)) = audit.command + else { + panic!("expected audit generate command"); + }; + assert_eq!(generate.browser.settle_quiet_ms, 750); + assert_eq!(generate.browser.settle_max_ms, 12_000); + + let args = parse(&[ + "ts", + "audit", + "ad-templates", + "generate", + "https://www.example.com/", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(generate), + )) = audit.command + else { + panic!("expected audit ad-templates generate command"); + }; + assert_eq!(generate.browser.settle_quiet_ms, 750); + assert_eq!(generate.browser.settle_max_ms, 12_000); + assert!(!generate.scroll, "generation should not scroll by default"); + } + + #[test] + fn audit_ad_templates_generate_parses_scroll() { + let args = parse(&[ + "ts", + "audit", + "ad-templates", + "generate", + "https://www.example.com/", + "--scroll", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(generate), + )) = audit.command + else { + panic!("expected audit ad-templates generate command"); + }; + + assert!( + generate.scroll, + "--scroll should enable generation scrolling" + ); + } + + #[test] + fn audit_ad_templates_verify_parses() { + let args = parse(&[ + "ts", + "audit", + "ad-templates", + "verify", + "https://www.example.com/", + ]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_browser_options_are_shared_by_generate_and_verify() { + for mode in ["generate", "verify"] { + let args = parse(&[ + "ts", + "audit", + "ad-templates", + mode, + "https://www.example.com/", + "--chrome", + "/tmp/test-chrome", + "--headful", + "--browser-proxy", + "127.0.0.1:8080", + "--no-assume-consent", + "--settle-quiet-ms", + "100", + "--settle-max-ms", + "200", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let (chrome, headful, no_assume_consent, browser_proxy, validation) = + match audit.command.expect("should parse audit subcommand") { + crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(args), + ) => { + let validation = args.browser.validate(); + ( + args.browser.chrome, + args.browser.headful, + args.browser.no_assume_consent, + args.browser.browser_proxy, + validation, + ) + } + crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Verify(args), + ) => { + let validation = args.browser.validate(); + ( + args.browser.chrome, + args.browser.headful, + args.browser.no_assume_consent, + args.browser.browser_proxy, + validation, + ) + } + _ => panic!("expected ad-template mode"), + }; + assert_eq!(chrome, Some(PathBuf::from("/tmp/test-chrome"))); + assert!(headful); + assert!(no_assume_consent); + assert_eq!(browser_proxy.as_deref(), Some("127.0.0.1:8080")); + validation.expect("should validate settle bounds"); + } + } + + #[test] + fn audit_generate_does_not_expose_the_ignored_browser_profile_flag() { + assert!( + Args::try_parse_from([ + "ts", + "audit", + "ad-templates", + "generate", + "https://www.example.com/", + "--browser-profile", + "mobile", + ]) + .is_err(), + "generation device selection must use --profiles" + ); + } + + #[test] + fn browser_settle_quiet_cannot_exceed_maximum() { + let args = parse(&[ + "ts", + "audit", + "page", + "https://www.example.com/", + "--settle-quiet-ms", + "201", + "--settle-max-ms", + "200", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let crate::commands::audit::AuditSubcommand::Page(page) = + audit.command.expect("should parse page subcommand") + else { + panic!("expected page audit"); + }; + assert!(page.browser.validate().is_err()); + } + + #[test] + fn audit_ad_templates_without_verify_is_error() { + assert!(Args::try_parse_from(["ts", "audit", "ad-templates"]).is_err()); + } + + #[test] + fn audit_rejects_non_http_url() { + assert!(Args::try_parse_from(["ts", "audit", "ftp://www.example.com/"]).is_err()); + } + + #[test] + fn audit_does_not_accept_adapter_option() { + let error = Args::try_parse_from([ + "ts", + "audit", + "page", + "https://www.example.com/", + "--adapter", + "fastly", + ]) + .expect_err("should reject audit adapter option"); + assert!(error.to_string().contains("unexpected argument")); + } + #[test] fn prebid_bundle_defaults_match_spec() { let args = parse(&["ts", "prebid", "bundle"]); diff --git a/crates/trusted-server-core/examples/local_dev_config.rs b/crates/trusted-server-core/examples/local_dev_config.rs new file mode 100644 index 000000000..acbb61b6c --- /dev/null +++ b/crates/trusted-server-core/examples/local_dev_config.rs @@ -0,0 +1,129 @@ +//! Generate a ready-to-use local dev config envelope for the Axum adapter. +//! +//! Reads `trusted-server.example.toml`, replaces the placeholder secrets with +//! random values, flips the flags a local smoke test needs, validates the +//! result through [`trusted_server_core::settings::Settings::from_toml`], and +//! prints the blob envelope JSON that the Axum adapter's +//! `TRUSTED_SERVER_CONFIG_{STORE}_{KEY}` environment variable expects. With +//! the default store and key both named `trusted_server_config`, the +//! concrete variable resolves (not a typo) to +//! `TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG`. +//! +//! The random values are time-and-pid seeded, not cryptographic. This tool +//! exists for throwaway local test instances only; never use its output for a +//! deployed service. +//! +//! Usage: +//! +//! ```text +//! cargo run -p trusted-server-core --example local_dev_config \ +//! --target -- [origin-url] [--realistic] +//! ``` +//! +//! `origin-url` defaults to `https://www.example.com`. By default every +//! response is forced `Cache-Control: private, no-store` so the Server-Timing +//! header is visible on all routes; pass `--realistic` to keep the origin's +//! own cache policy instead. + +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Deliberately non-cryptographic generator for local placeholder secrets. +struct WeakRandom(u64); + +impl WeakRandom { + fn from_environment() -> Self { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should compute epoch time") + .subsec_nanos() as u64; + let secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should compute epoch time") + .as_secs(); + let pid = std::process::id() as u64; + Self(nanos ^ (secs << 20) ^ (pid << 40) ^ 0x9e37_79b9_7f4a_7c15) + } + + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + fn hex(&mut self, chars: usize) -> String { + let mut out = String::with_capacity(chars); + while out.len() < chars { + out.push_str(&format!("{:016x}", self.next())); + } + out.truncate(chars); + out + } +} + +#[allow(clippy::print_stdout, clippy::print_stderr)] +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let realistic = args.iter().any(|a| a == "--realistic"); + let origin = args + .iter() + .find(|a| !a.starts_with("--")) + .cloned() + .unwrap_or_else(|| "https://www.example.com".to_string()); + + let template = std::fs::read_to_string("trusted-server.example.toml") + .expect("should read trusted-server.example.toml from the repo root"); + + let mut random = WeakRandom::from_environment(); + let mut config = template + .replace( + "password = \"replace-with-admin-password-32-bytes\"", + &format!("password = \"{}\"", random.hex(48)), + ) + .replace( + "proxy_secret = \"change-me-proxy-secret\"", + &format!("proxy_secret = \"{}\"", random.hex(48)), + ) + .replace( + "passphrase = \"trusted-server-placeholder-secret\"", + &format!("passphrase = \"{}\"", random.hex(48)), + ) + .replace( + "server_timing_enabled = false", + "server_timing_enabled = true", + ); + + let origin_line = config + .lines() + .find(|line| line.starts_with("origin_url = ")) + .expect("should find the origin_url line in the template") + .to_string(); + config = config.replace(&origin_line, &format!("origin_url = \"{origin}\"")); + + if !realistic { + config = config.replace( + "# [response_headers]", + "[response_headers]\n\"Cache-Control\" = \"private, no-store\"", + ); + } + + let settings = trusted_server_core::settings::Settings::from_toml(&config) + .expect("should validate the generated local config"); + let data = serde_json::to_value(&settings).expect("should serialize settings"); + let generated_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should compute epoch time") + .as_secs() + .to_string(); + let envelope = edgezero_core::blob_envelope::BlobEnvelope::new(data, generated_at); + println!( + "{}", + serde_json::to_string(&envelope).expect("should serialize the envelope") + ); + eprintln!( + "local dev envelope generated: origin={origin} force_private={}", + !realistic + ); +} diff --git a/crates/trusted-server-core/src/access_telemetry.rs b/crates/trusted-server-core/src/access_telemetry.rs new file mode 100644 index 000000000..ba18c869a --- /dev/null +++ b/crates/trusted-server-core/src/access_telemetry.rs @@ -0,0 +1,627 @@ +//! Access telemetry: route classification and the per-request access log row. +//! +//! Extends the reserved `access_logs_raw` Tinybird datasource with bounded, +//! content-free route identity (see [`RouteClass`] and +//! [`publisher_route_template`]) instead of the raw request path, which would +//! otherwise carry identifiers, search terms, and other user-generated +//! content into a 30-day dataset. See the design spec +//! `docs/superpowers/specs/2026-08-24-request-phase-timing-design.md` +//! section 9. + +use serde_json::json; + +use crate::request_timing::{AuctionWaitPlacement, TimingSnapshot}; + +/// Maximum length of a publisher path's first segment before +/// [`publisher_route_template`] rejects it to `/other/*`. Longer segments +/// are opaque-identifier or slug shaped (a UUID is 36 characters), and a +/// truncated prefix of either would still be identifying, so the segment +/// is rejected whole rather than truncated. +const MAX_SEGMENT_LEN: usize = 32; + +/// Maximum number of ASCII digits in a publisher path's first segment +/// before [`publisher_route_template`] rejects it to `/other/*`. Hex ids, +/// base36 ids, and reset tokens are digit-heavy; real section names carry +/// at most a year (`2026`) or a small version number, so a segment with +/// more digits than this is treated as an identifier, not a name. +const MAX_SEGMENT_DIGITS: usize = 7; + +/// Normalizes an HTTP method token into the bounded set of values stored in +/// the `method` `LowCardinality` column. +/// +/// HTTP permits arbitrary extension-method tokens (`PROPFIND`, `MKCOL`, or +/// any client-supplied garbage), and the token on an inbound request is +/// entirely client controlled. Capturing one verbatim into a 30-day +/// `LowCardinality(String)` column would let a single caller inflate that +/// column's cardinality without bound and would violate this dataset's +/// bounded-dimension privacy rule (see the module doc). Every standard +/// method maps to its uppercase form; anything else maps to `"other"`. Runs +/// inside [`access_event_row`] rather than at each capture site, so every +/// row-building path is covered regardless of how `method` was populated. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::access_telemetry::normalize_method; +/// +/// assert_eq!(normalize_method("get"), "GET"); +/// assert_eq!(normalize_method("PROPFIND"), "other"); +/// assert_eq!(normalize_method(""), + "other", + "an unbounded client-controlled token must not reach the row verbatim" + ); + } + + #[test] + fn row_normalizes_method_even_when_snapshot_carries_a_raw_token() { + // The normalizer runs inside `access_event_row` so every row-building + // path is covered, regardless of what the snapshot's `method` field + // holds — a caller-controlled extension method must never leak into + // the row unnormalized. + let mut snapshot = unknown_snapshot(RouteClass::Other, "/other/*"); + snapshot.method = "PROPFIND".to_owned(); + let row = access_event_row(&snapshot, &TimingSnapshot::default(), 0); + let parsed: serde_json::Value = + serde_json::from_str(&row).expect("should serialize valid JSON"); + + assert_eq!(parsed["method"], "other"); + } +} diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 954869e38..4f913f235 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -4171,7 +4171,8 @@ mod tests { .expect("current adapters should accept completed late mediator responses"); assert!( current_mediator.response_time_ms >= 50, - "mediator timing should preserve actual elapsed duration" + "mediator timing should preserve actual elapsed duration, got {} ms", + current_mediator.response_time_ms ); assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 6be0c9cfa..80d3f989e 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -182,6 +182,10 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { vec![optional_object("tinybird"), object("auction_token_secret")], true, ), + field( + vec![optional_object("tinybird"), object("access_token_secret")], + true, + ), field( vec![ optional_object("integrations"), @@ -421,7 +425,7 @@ fn validate_secret_key_references(settings: &Settings) -> Result<(), Report Result<(), Report("datadome")? { if datadome.enable_protection { @@ -765,6 +777,7 @@ formats = [{ width = 300, height = 250 }] ("handlers[*].password".to_owned(), false), ("trusted_client_ip.shared_secret".to_owned(), false), ("tinybird.auction_token_secret".to_owned(), true), + ("tinybird.access_token_secret".to_owned(), true), ( "integrations.datadome.server_side_key_secret_name".to_owned(), true, @@ -1187,10 +1200,14 @@ password = "production-admin-password-32-bytes" ); } - /// Integrations that default to disabled do not validate inactive fields. + /// `enabled` defaults to `false`, so a section that omits the flag resolves + /// to disabled and must not have its fields validated. #[test] fn deploy_validation_skips_field_validation_for_integrations_with_omitted_enabled() { let mut settings = valid_settings(); + // `endpoint` parses as a plain string but would fail the `url` + // validator, so this section only survives if validation is skipped for + // integrations that resolve to disabled. settings .integrations .insert_config( diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index b604aeb65..b1e5fed24 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -61,16 +61,30 @@ pub fn settings_from_config_blob( } fn remove_inactive_secret_references(data: &mut serde_json::Value) { - if data - .pointer("/tinybird/enabled") - .and_then(serde_json::Value::as_bool) - != Some(true) - && let Some(tinybird) = data - .get_mut("tinybird") - .and_then(serde_json::Value::as_object_mut) + if let Some(tinybird) = data + .get_mut("tinybird") + .and_then(serde_json::Value::as_object_mut) { - tinybird.remove("auction_token_secret"); - tinybird.remove("access_token_secret"); + let enabled = tinybird.get("enabled").and_then(serde_json::Value::as_bool) == Some(true); + if !enabled { + tinybird.remove("auction_token_secret"); + tinybird.remove("access_token_secret"); + } else { + if tinybird + .get("auction_enabled") + .and_then(serde_json::Value::as_bool) + == Some(false) + { + tinybird.remove("auction_token_secret"); + } + if tinybird + .get("access_enabled") + .and_then(serde_json::Value::as_bool) + != Some(true) + { + tinybird.remove("access_token_secret"); + } + } } if let Some(partners) = data @@ -127,7 +141,10 @@ fn json_bool_or_string_is_true(value: Option<&serde_json::Value>) -> bool { #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; + use crate::integrations::IntegrationRegistry; use crate::platform::{PlatformError, StoreId}; use crate::redacted::Redacted; use crate::settings::{ @@ -803,9 +820,15 @@ mod tests { #[test] fn runtime_blob_accepts_disabled_browser_bidder_ownership_overlap() { let original = settings_with_browser_bidder_overlap(false); + let reconstructed = load_settings(&envelope_json(&original)) + .expect("should decode dormant conflicting runtime blob"); + let plan = Arc::new( + crate::auction::compile_auction_plan(&reconstructed) + .expect("should compile decoded disabled auction plan"), + ); - load_settings(&envelope_json(&original)) - .expect("runtime should accept disabled browser bidder ownership overlap"); + IntegrationRegistry::with_plan(&reconstructed, plan) + .expect("runtime registry should accept disabled ownership overlap"); } #[test] diff --git a/crates/trusted-server-core/src/consent/mod.rs b/crates/trusted-server-core/src/consent/mod.rs index f205a8363..e149c76ee 100644 --- a/crates/trusted-server-core/src/consent/mod.rs +++ b/crates/trusted-server-core/src/consent/mod.rs @@ -703,7 +703,7 @@ mod tests { use super::{ ConsentPipelineInput, allows_ec_creation, apply_expiration_check, apply_tcf_conflict_resolution, build_consent_context, build_context_from_signals, - consent_allows_server_side_auction, has_explicit_ec_withdrawal, + consent_allows_server_side_auction, gate_eids_by_consent, has_explicit_ec_withdrawal, }; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ @@ -1080,6 +1080,36 @@ mod tests { TcfBuilder::new().with_storage(has_storage).build() } + #[test] + fn gate_eids_by_consent_strips_every_eid_when_personalization_is_denied() { + let context = ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + tcf: Some( + TcfBuilder::new() + .with_storage(true) + .with_personalized_ads(false) + .build(), + ), + ..ConsentContext::default() + }; + + // Gating is all-or-nothing across sources; LiveRamp is included here as + // the case that motivated this coverage, not as a special case. + let gated = gate_eids_by_consent( + Some(vec![ + ("liveramp.com", "opaque-test-envelope"), + ("sharedid.org", "shared-test-id"), + ]), + Some(&context), + ); + + assert!( + gated.is_none(), + "should remove every EID when personalization consent is denied" + ); + } + #[test] fn ec_allowed_gdpr_with_storage_consent() { let ctx = ConsentContext { diff --git a/crates/trusted-server-core/src/constants.rs b/crates/trusted-server-core/src/constants.rs index d444238e8..74e88e56b 100644 --- a/crates/trusted-server-core/src/constants.rs +++ b/crates/trusted-server-core/src/constants.rs @@ -36,6 +36,8 @@ pub const HEADER_X_TS_ENV: HeaderName = HeaderName::from_static("x-ts-env"); // Fastly environment variables pub const ENV_FASTLY_SERVICE_VERSION: &str = "FASTLY_SERVICE_VERSION"; pub const ENV_FASTLY_IS_STAGING: &str = "FASTLY_IS_STAGING"; +pub const ENV_FASTLY_SERVICE_ID: &str = "FASTLY_SERVICE_ID"; +pub const ENV_FASTLY_POP: &str = "FASTLY_POP"; // Common standard header names used across modules pub const HEADER_USER_AGENT: HeaderName = HeaderName::from_static("user-agent"); diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 2254c27d5..4b1405359 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -173,8 +173,12 @@ fn sanitize_section(segment: &str) -> String { /// The path is used **raw** (not percent-decoded) so this stays consistent with /// how [`page_patterns`](CreativeOpportunitySlot::page_patterns) glob-match the /// same path — e.g. `/new%20s` yields `new_20s`, never the decoded `new_s`. +/// +/// Public so operator tooling that *infers* a `{section}` template from observed +/// ad-unit paths can check its inference against the exact derivation the +/// runtime will perform, rather than reimplementing the sanitization rules. #[must_use] -fn derive_section(path: &str, section_root: &str, section_segment: usize) -> String { +pub fn derive_section(path: &str, section_root: &str, section_segment: usize) -> String { match path .split('/') .filter(|segment| !segment.is_empty()) @@ -701,15 +705,7 @@ impl CreativeOpportunitySlot { // skip `compile_patterns`). Re-compiles on every call. self.page_patterns .iter() - .any(|pattern| match Pattern::new(pattern) { - Ok(p) => p.matches(path), - Err(_) => { - let normalised = pattern.replace("**", "*"); - Pattern::new(&normalised) - .map(|p| p.matches(path)) - .unwrap_or(false) - } - }) + .any(|pattern| compile_page_pattern(pattern).is_ok_and(|p| p.matches(path))) } /// Compile [`page_patterns`](Self::page_patterns) into the @@ -726,22 +722,20 @@ impl CreativeOpportunitySlot { self.compiled_patterns = self .page_patterns .iter() - .filter_map(|pattern| { - match Pattern::new(pattern).or_else(|_| Pattern::new(&pattern.replace("**", "*"))) { - Ok(compiled) => Some(compiled), - Err(_) => { - // Build-time validation only requires *one* valid pattern - // per slot, so a mixed valid/invalid set passes the build - // with the bad pattern silently dropped here. Warn so the - // operator can see the slot matches fewer pages than - // configured. - log::warn!( - "slot `{}`: dropping page pattern '{}' — it does not compile as a glob", - self.id, - pattern - ); - None - } + .filter_map(|pattern| match compile_page_pattern(pattern) { + Ok(compiled) => Some(compiled), + Err(error) => { + // Build-time validation only requires *one* valid pattern + // per slot, so a mixed valid/invalid set passes the build + // with the bad pattern silently dropped here. Warn so the + // operator can see the slot matches fewer pages than + // configured. + log::warn!( + "slot `{}`: dropping page pattern '{}': {error}", + self.id, + pattern + ); + None } }) .collect(); @@ -994,6 +988,48 @@ pub struct PrebidSlotParams { pub bidders: HashMap, } +/// Compiles a [`page_patterns`](CreativeOpportunitySlot::page_patterns) entry +/// using the runtime's normalisation. +/// +/// This is the single definition of what the runtime accepts as a page glob: +/// a direct [`Pattern::new`], falling back to the `**`→`*` rewrite that +/// [`CreativeOpportunitySlot::compile_patterns`] and +/// [`matches_path`](CreativeOpportunitySlot::matches_path) apply. +/// +/// # Errors +/// +/// Returns an error string when the pattern compiles neither directly nor after +/// normalisation. +pub(crate) fn compile_page_pattern(pattern: &str) -> Result { + Pattern::new(pattern) + .or_else(|_| Pattern::new(&pattern.replace("**", "*"))) + .map_err(|error| format!("page pattern '{pattern}' is not a valid glob: {error}")) +} + +/// Validates a [`page_patterns`](CreativeOpportunitySlot::page_patterns) entry +/// using the runtime's normalisation. +/// +/// This exposes validation without leaking the runtime's `glob::Pattern` type +/// into the public API. +/// +/// # Errors +/// +/// Returns an error string when the pattern compiles neither directly nor after +/// the runtime's `**` to `*` normalisation. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::creative_opportunities::validate_page_pattern; +/// +/// assert!(validate_page_pattern("/news/*").is_ok()); +/// assert!(validate_page_pattern("/20**").is_ok()); +/// assert!(validate_page_pattern("[").is_err()); +/// ``` +pub fn validate_page_pattern(pattern: &str) -> Result<(), String> { + compile_page_pattern(pattern).map(|_| ()) +} + /// Validates that a slot ID contains only safe characters. /// /// Allowed characters: ASCII alphanumerics, underscores (`_`), and hyphens (`-`). @@ -1026,6 +1062,151 @@ pub fn match_slots<'a>( slots.iter().filter(|s| s.matches_path(path)).collect() } +/// Three-state outcome of the server-side ad-stack gate. +/// +/// [`Yes`](RuntimeAdStackExpected::Yes) and [`No`](RuntimeAdStackExpected::No) +/// are decided purely from known inputs; [`Unknown`](RuntimeAdStackExpected::Unknown) +/// is reserved for callers (such as the operator CLI) that cannot prove the live +/// consent state and pass `None` for `consent_allows_auction`. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RuntimeAdStackExpected { + /// All known gates pass and consent is known to allow the auction. + Yes, + /// At least one known gate blocks the server-side ad stack. + No, + /// All known gates pass but consent is unproven. + Unknown, +} + +/// Identifies a single gate evaluated by [`evaluate_ad_stack_gate`]. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum AdStackGateName { + /// Request method is `GET`. + MethodGet, + /// Request is a top-level navigation. + Navigation, + /// Request is not a prefetch. + NotPrefetch, + /// Request is not from a known bot. + NotBot, + /// At least one configured slot matches the request path. + MatchedSlots, + /// Consent is known to allow the auction. + ConsentAllowsAuction, + /// The global `[auction].enabled` kill switch is on. + AuctionEnabled, + /// The `[creative_opportunities].enabled` template switch is on. + AdTemplatesEnabled, +} + +impl AdStackGateName { + const ALL: [Self; 8] = [ + Self::MethodGet, + Self::Navigation, + Self::NotPrefetch, + Self::NotBot, + Self::MatchedSlots, + Self::ConsentAllowsAuction, + Self::AuctionEnabled, + Self::AdTemplatesEnabled, + ]; + + fn blocks(self, input: AdStackGateInput) -> bool { + match self { + Self::MethodGet => !input.method_get, + Self::Navigation => !input.navigation, + Self::NotPrefetch => input.prefetch, + Self::NotBot => input.bot, + Self::MatchedSlots => !input.matched_slots, + Self::ConsentAllowsAuction => input.consent_allows_auction == Some(false), + Self::AuctionEnabled => !input.auction_enabled, + Self::AdTemplatesEnabled => !input.ad_templates_enabled, + } + } +} + +/// Inputs to [`evaluate_ad_stack_gate`]. +/// +/// `consent_allows_auction` is tri-state: `Some(true)` allows, `Some(false)` +/// blocks, and `None` means the caller cannot prove the consent state. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct AdStackGateInput { + /// Request method is `GET`. + pub method_get: bool, + /// Request is a top-level navigation. + pub navigation: bool, + /// Request advertises itself as a prefetch. + pub prefetch: bool, + /// Request is from a known bot. + pub bot: bool, + /// At least one configured slot matches the request path. + pub matched_slots: bool, + /// Whether consent allows the auction. + /// + /// `Some(true)` allows the auction, `Some(false)` blocks it, and `None` + /// means the caller cannot prove either state. Unknown consent is not a + /// denial: it produces [`RuntimeAdStackExpected::Unknown`] when every known + /// boolean gate passes. + pub consent_allows_auction: Option, + /// The global `[auction].enabled` kill switch. + pub auction_enabled: bool, + /// The `[creative_opportunities].enabled` template switch. + /// + /// `false` whenever creative opportunities are absent from the + /// configuration, so an unconfigured publisher blocks here as well. + pub ad_templates_enabled: bool, +} + +/// Result of [`evaluate_ad_stack_gate`]: the three-state expectation plus the +/// original inputs used to derive per-gate diagnostics on demand. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct AdStackGateResult { + /// The three-state ad-stack expectation. + pub expected: RuntimeAdStackExpected, + input: AdStackGateInput, +} + +impl AdStackGateResult { + /// Returns the gates that blocked the server-side ad stack. + pub fn blocking_gates(&self) -> impl Iterator + '_ { + AdStackGateName::ALL + .into_iter() + .filter(|gate| gate.blocks(self.input)) + } +} + +/// Evaluates whether the server-side ad stack should run for a request. +/// +/// Any known gate that fails sets [`No`](RuntimeAdStackExpected::No) and is +/// recorded in [`AdStackGateResult::blocking_gates`]. When no known gate blocks, +/// the result is [`Yes`](RuntimeAdStackExpected::Yes) if consent is known to +/// allow the auction, or [`Unknown`](RuntimeAdStackExpected::Unknown) when +/// `consent_allows_auction` is `None`. +/// +/// Gate polarity mirrors the runtime publisher path: `method_get`, `navigation`, +/// `matched_slots`, `auction_enabled`, and `ad_templates_enabled` block when +/// `false`; `prefetch` and `bot` block when `true`. +#[must_use] +pub fn evaluate_ad_stack_gate(input: AdStackGateInput) -> AdStackGateResult { + let known_gate_blocks = !input.method_get + || !input.navigation + || input.prefetch + || input.bot + || !input.matched_slots + || input.consent_allows_auction == Some(false) + || !input.auction_enabled + || !input.ad_templates_enabled; + let expected = if known_gate_blocks { + RuntimeAdStackExpected::No + } else if input.consent_allows_auction.is_none() { + RuntimeAdStackExpected::Unknown + } else { + RuntimeAdStackExpected::Yes + }; + + AdStackGateResult { expected, input } +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; @@ -1041,6 +1222,159 @@ mod tests { use crate::auction::routing::route_auction; use crate::auction::types::{AuctionRequest, PublisherInfo, UserInfo}; + #[test] + fn ad_stack_gate_passes_for_eligible_navigation() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: true, + ad_templates_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Yes); + assert_eq!(result.blocking_gates().count(), 0); + } + + #[test] + fn ad_stack_gate_blocks_known_kill_switch() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: false, + ad_templates_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::No); + assert!( + result + .blocking_gates() + .any(|gate| gate == AdStackGateName::AuctionEnabled) + ); + } + + #[test] + fn ad_stack_gate_blocks_disabled_ad_templates() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: true, + ad_templates_enabled: false, + }); + + assert_eq!( + result.expected, + RuntimeAdStackExpected::No, + "a disabled [creative_opportunities].enabled switch should block the ad stack" + ); + assert!( + result + .blocking_gates() + .any(|gate| gate == AdStackGateName::AdTemplatesEnabled), + "the template switch should be named as the blocking gate" + ); + } + + #[test] + fn ad_stack_gate_is_unknown_when_consent_is_unknown() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: None, + auction_enabled: true, + ad_templates_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Unknown); + } + + // Locks the spec §5.2 mirror invariant: with Some(consent) supplied for every + // input combination, `expected == Yes` must equal the legacy all-AND boolean. + #[test] + fn ad_stack_gate_with_known_consent_matches_legacy_boolean() { + for bits in 0u16..256 { + let input = AdStackGateInput { + method_get: bits & 1 != 0, + navigation: bits & 2 != 0, + prefetch: bits & 4 != 0, + bot: bits & 8 != 0, + matched_slots: bits & 16 != 0, + consent_allows_auction: Some(bits & 32 != 0), + auction_enabled: bits & 64 != 0, + ad_templates_enabled: bits & 128 != 0, + }; + // Legacy semantics: all positive gates true, both negative gates false. + let legacy = input.method_get + && input.navigation + && !input.prefetch + && !input.bot + && input.matched_slots + && input.consent_allows_auction == Some(true) + && input.auction_enabled + && input.ad_templates_enabled; + let got = evaluate_ad_stack_gate(input).expected == RuntimeAdStackExpected::Yes; + assert_eq!(got, legacy, "gate mismatch for bits={bits}"); + } + } + + #[test] + fn ad_stack_gate_with_unknown_consent_matches_known_boolean_gates() { + for bits in 0u8..128 { + let input = AdStackGateInput { + method_get: bits & 1 != 0, + navigation: bits & 2 != 0, + prefetch: bits & 4 != 0, + bot: bits & 8 != 0, + matched_slots: bits & 16 != 0, + consent_allows_auction: None, + auction_enabled: bits & 32 != 0, + ad_templates_enabled: bits & 64 != 0, + }; + let known_gates_pass = input.method_get + && input.navigation + && !input.prefetch + && !input.bot + && input.matched_slots + && input.auction_enabled + && input.ad_templates_enabled; + let expected = if known_gates_pass { + RuntimeAdStackExpected::Unknown + } else { + RuntimeAdStackExpected::No + }; + + assert_eq!( + evaluate_ad_stack_gate(input).expected, + expected, + "should match unknown-consent gate semantics for bits={bits}" + ); + } + } + + #[test] + fn validate_page_pattern_preserves_specific_compile_error() { + let error = validate_page_pattern("[").expect_err("should reject invalid glob"); + + assert!( + error.contains("page pattern '[' is not a valid glob"), + "should retain the invalid pattern in the error: {error}" + ); + } + fn make_slot(id: &str, patterns: Vec<&str>) -> CreativeOpportunitySlot { CreativeOpportunitySlot { id: id.to_string(), diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index fd9aa5773..9d559221b 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -1165,6 +1165,28 @@ mod tests { assert!(ts > 0, "should return a nonzero timestamp"); } + #[test] + fn kv_span_accumulates_across_graph_operations() { + let timings = crate::request_timing::RequestTimings::new(); + let graph = KvIdentityGraph::new(crate::platform::TimedKvStore::new( + crate::ec::kv_backend::test_support::InMemoryEcKv::new("test-store"), + timings.clone(), + )); + + graph + .create("ec-1", &live_entry()) + .expect("should create entry through the timed store"); + graph + .get("ec-1") + .expect("should read the entry back through the timed store"); + + timings.mark_headers_ready(); + assert!( + timings.snapshot().kv_ms.is_some(), + "should accumulate Phase::EcKv across both graph operations, not just the last write" + ); + } + #[test] fn serialize_entry_produces_valid_json() { let entry = KvEntry::tombstone(1000); diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 3072473dd..114de99df 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -800,6 +800,29 @@ mod tests { ); } + #[test] + fn ingest_liveramp_eid_cookie_preserves_the_opaque_envelope() { + let registry = make_registry(vec![("liveramp", "liveramp.com")]); + let cookie = encode_json(&json!([ + { + "source": "liveramp.com", + "uids": [{"id": "opaque-test-envelope", "atype": 3}] + } + ])); + let writer = RecordingWriter::default(); + + ingest_eid_cookies_with_writer(Some(&cookie), None, "ec-id", &writer, ®istry); + + let calls = writer.calls.borrow(); + assert_eq!(calls.len(), 1, "should perform one bulk writer call"); + assert_eq!(calls[0].len(), 1, "should write one LiveRamp partner ID"); + assert_eq!( + calls[0][0], + PartnerIdUpdate::new("liveramp.com", "opaque-test-envelope"), + "should preserve the opaque envelope without decoding it" + ); + } + #[test] fn ingest_eid_cookies_sharedid_cookie_overrides_prebid_sharedid_update() { let registry = make_registry(vec![("sharedid", "sharedid.org")]); diff --git a/crates/trusted-server-core/src/geo.rs b/crates/trusted-server-core/src/geo.rs index 63f7907f5..fe5785d26 100644 --- a/crates/trusted-server-core/src/geo.rs +++ b/crates/trusted-server-core/src/geo.rs @@ -48,6 +48,24 @@ impl GeoInfo { } } +/// Carries the outcome of a request-phase geo lookup across to +/// response-phase finalization, so a finalize consumer can reuse it instead +/// of performing a second lookup for the same request. +/// +/// Attached as a response extension on every exit path that attempted a +/// lookup, including the asset-route fallback (which does not carry an EC +/// finalize state). +#[derive(Debug, Clone)] +pub enum GeoLookupState { + /// No lookup has been attempted for this request. + NotAttempted, + /// A lookup ran and failed (or returned no result). This must not be + /// retried: finalize treats it the same as no geo info being available. + Attempted, + /// A lookup ran and resolved geo info. + Resolved(GeoInfo), +} + fn insert_geo_header(headers: &mut http::HeaderMap, name: http::header::HeaderName, value: &str) { match HeaderValue::from_str(value) { Ok(header_value) => { diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 4f45fbeb9..53b30ed6f 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -78,42 +78,54 @@ const APS_RENDERER_DOCUMENT: &str = r#" var match=/^#tsaps=([A-Za-z0-9_-]{22,128})$/.exec(location.hash); var expected=match&&match[1]; try{history.replaceState(null,'',location.pathname+location.search);}catch(_error){} -if(!expected)return; +var reported=false; +function report(reason,nonce){ + if(reported)return; + reported=true; + try{parent.postMessage({message:'trusted-server/aps/renderer-failed',nonce:nonce,reason:reason},'*');}catch(_error){} +} +if(!expected){report('bad_hash');return;} function keys(value,expectedKeys){ if(!value||typeof value!=='object'||Array.isArray(value))return false; var actual=Object.keys(value).sort(); return actual.length===expectedKeys.length&&actual.every(function(key,index){return key===expectedKeys[index];}); } -function validRenderer(renderer){ +function rendererProblem(renderer){ if(!keys(renderer,['aaxResponse','accountId','bidId','creativeId','creativeUrl','height','tagType','type','version','width'])&& - !keys(renderer,['aaxResponse','accountId','bidId','creativeUrl','height','tagType','type','version','width']))return false; - if(renderer.type!=='aps'||renderer.version!==1||typeof renderer.accountId!=='string'||!renderer.accountId||new TextEncoder().encode(renderer.accountId).length>1024)return false; - if(typeof renderer.bidId!=='string'||!renderer.bidId||!Number.isInteger(renderer.width)||renderer.width<=0||!Number.isInteger(renderer.height)||renderer.height<=0)return false; - if(Object.prototype.hasOwnProperty.call(renderer,'creativeId')&&(typeof renderer.creativeId!=='string'||!renderer.creativeId||new TextEncoder().encode(renderer.creativeId).length>1024))return false; - if(renderer.tagType!=='iframe'&&renderer.tagType!=='script')return false; - if(typeof renderer.creativeUrl!=='string'||new TextEncoder().encode(renderer.creativeUrl).length>4096)return false; - if(typeof renderer.aaxResponse!=='string'||!renderer.aaxResponse||renderer.aaxResponse.length>349528)return false; + !keys(renderer,['aaxResponse','accountId','bidId','creativeUrl','height','tagType','type','version','width']))return 'descriptor_keys'; + if(renderer.type!=='aps'||renderer.version!==1||typeof renderer.accountId!=='string'||!renderer.accountId||new TextEncoder().encode(renderer.accountId).length>1024)return 'descriptor_fields'; + if(typeof renderer.bidId!=='string'||!renderer.bidId||!Number.isInteger(renderer.width)||renderer.width<=0||!Number.isInteger(renderer.height)||renderer.height<=0)return 'descriptor_fields'; + if(Object.prototype.hasOwnProperty.call(renderer,'creativeId')&&(typeof renderer.creativeId!=='string'||!renderer.creativeId||new TextEncoder().encode(renderer.creativeId).length>1024))return 'descriptor_fields'; + if(renderer.tagType!=='iframe'&&renderer.tagType!=='script')return 'descriptor_fields'; + if(typeof renderer.creativeUrl!=='string'||new TextEncoder().encode(renderer.creativeUrl).length>4096)return 'descriptor_fields'; + if(typeof renderer.aaxResponse!=='string'||!renderer.aaxResponse||renderer.aaxResponse.length>349528)return 'descriptor_fields'; try{ var url=new URL(renderer.creativeUrl); - if(url.protocol!=='https:'||url.username||url.password)return false; + if(url.protocol!=='https:'||url.username||url.password)return 'descriptor_envelope'; var binary=atob(renderer.aaxResponse); - if(binary.length>262144||btoa(binary)!==renderer.aaxResponse)return false; + if(binary.length>262144||btoa(binary)!==renderer.aaxResponse)return 'descriptor_envelope'; var bytes=Uint8Array.from(binary,function(character){return character.charCodeAt(0);}); var decoded=JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes)); - if(!keys(decoded,['seatbid'])||!Array.isArray(decoded.seatbid)||decoded.seatbid.length!==1)return false; + if(!keys(decoded,['seatbid'])||!Array.isArray(decoded.seatbid)||decoded.seatbid.length!==1)return 'descriptor_envelope'; var seat=decoded.seatbid[0]; - if(!keys(seat,['bid'])||!Array.isArray(seat.bid)||seat.bid.length!==1)return false; + if(!keys(seat,['bid'])||!Array.isArray(seat.bid)||seat.bid.length!==1)return 'descriptor_envelope'; var bid=seat.bid[0]; - if(!keys(bid,['ext','h','id','price','w'])||!keys(bid.ext,['creativeurl','tagtype']))return false; - return bid.id===renderer.bidId&&bid.w===renderer.width&&bid.h===renderer.height&& + if(!keys(bid,['ext','h','id','price','w'])||!keys(bid.ext,['creativeurl','tagtype']))return 'descriptor_envelope'; + if(bid.id===renderer.bidId&&bid.w===renderer.width&&bid.h===renderer.height&& bid.ext.creativeurl===renderer.creativeUrl&&bid.ext.tagtype===renderer.tagType&& - typeof bid.price==='number'&&Number.isFinite(bid.price)&&bid.price>=0; - }catch(_error){return false;} + typeof bid.price==='number'&&Number.isFinite(bid.price)&&bid.price>=0)return undefined; + return 'descriptor_envelope'; + }catch(_error){return 'descriptor_envelope';} } function receive(event){ - if(event.source!==parent)return; var message=event.data; - if(!keys(message,['nonce','renderer'])||message.nonce!==expected||!validRenderer(message.renderer))return; + // Stay silent for traffic that is not shaped like the render handshake, so an + // unrelated sender cannot consume this frame's single report. + if(!keys(message,['nonce','renderer']))return; + if(event.source!==parent){report('source_mismatch');return;} + if(message.nonce!==expected){report('nonce_mismatch');return;} + var problem=rendererProblem(message.renderer); + if(problem){report(problem,message.nonce);return;} removeEventListener('message',receive); var acceptedNonce=expected; expected=''; @@ -128,7 +140,7 @@ function receive(event){ var script=document.createElement('script'); script.src='https://client.aps.amazon-adsystem.com/prebid-creative.js'; script.onload=function(){parent.postMessage({message:'trusted-server/aps/renderer-ready',nonce:acceptedNonce},'*');}; - script.onerror=function(){parent.postMessage({message:'trusted-server/aps/renderer-failed',nonce:acceptedNonce},'*');}; + script.onerror=function(){report('amazon_script_error',acceptedNonce);}; document.head.appendChild(script); } addEventListener('message',receive); @@ -3308,4 +3320,41 @@ mod tests { assert!(APS_RENDERER_CSP.contains("sandbox allow-forms")); assert!(!APS_RENDERER_CSP.contains("allow-same-origin")); } + + #[test] + fn renderer_document_reports_a_reason_for_every_silent_guard() { + for reason in [ + "bad_hash", + "source_mismatch", + "nonce_mismatch", + "descriptor_keys", + "descriptor_fields", + "descriptor_envelope", + "amazon_script_error", + ] { + assert!( + APS_RENDERER_DOCUMENT.contains(reason), + "renderer document should report a `{reason}` reason instead of returning silently" + ); + } + + // Reasons travel on the existing failure message rather than a new channel. + assert!( + APS_RENDERER_DOCUMENT.contains("reason:reason"), + "should attach the reason to the failure message" + ); + + // A reason is a fixed category, never a copy of the rejected descriptor. + assert!(!APS_RENDERER_DOCUMENT.contains("JSON.stringify(renderer)")); + assert!(!APS_RENDERER_DOCUMENT.contains("reason:message")); + + // Reporting is one-shot so a hostile sender cannot flood the parent. + assert!( + APS_RENDERER_DOCUMENT.contains("if(reported)return"), + "should report at most one reason per frame" + ); + + // A foreign sender is answered through the parent, never the sender. + assert!(!APS_RENDERER_DOCUMENT.contains("event.source.postMessage")); + } } diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 67f004a1f..3292cc5b3 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -1247,12 +1247,16 @@ mod tests { "should set ts_initial sentinel" ); assert!( - !combined.contains("addEventListener(\"slotRenderEnded\""), - "inline bootstrap cannot prove TS creative rendering from GPT slotRenderEnded" + combined.contains("addEventListener(\"slotRequested\""), + "should observe publisher GPT requests before delayed adInit" + ); + assert!( + combined.contains("addEventListener(\"slotRenderEnded\""), + "should observe publisher GPT renders before delayed adInit" ); assert!( !combined.contains("sendBeacon"), - "inline bootstrap must not fire win/billing beacons from GPT slotRenderEnded" + "inline bootstrap lifecycle ownership must not fire win/billing beacons" ); assert!( !combined.contains("getTargeting(\"hb_adid\")"), diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 2475c5082..425ff90f7 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -102,6 +102,179 @@ pubads.__tsInitialLoadHooked = true; }); + var FIRST_IMPRESSION_LEASE_MS = 5000; + var MAX_FIRST_IMPRESSION_SLOTS = 256; + + function firstImpressionState(now) { + var generation = ts.navGeneration || 0; + if ( + !ts.firstImpression || + ts.firstImpression.generation !== generation + ) { + ts.firstImpression = { + generation: generation, + nextToken: 0, + slots: {}, + fallbackSlots: {}, + }; + } + var state = ts.firstImpression; + state.slots = state.slots || {}; + state.fallbackSlots = state.fallbackSlots || {}; + Object.keys(state.slots).forEach(function (elementId) { + var claim = state.slots[elementId]; + if ( + claim.generation !== generation || + claim.slotElementId !== elementId || + claim.element.ownerDocument !== document || + claim.element !== document.getElementById(elementId) || + !claim.element.isConnected + ) { + delete state.slots[elementId]; + return; + } + var hasReservedFallback = + claim.owner === "publisher" && + (claim.phase === "auctioning" || claim.phase === "delivery_pending") && + state.fallbackSlots[elementId] === claim.element; + Object.keys(claim.publisherAuctions || {}).forEach(function (token) { + var auction = claim.publisherAuctions[token]; + if ( + auction.expiresAt <= now && + !hasReservedFallback && + !(claim.owner === "trusted_server" && auction.suppressDelivery) + ) { + delete claim.publisherAuctions[token]; + } + }); + if ( + claim.owner === "publisher" && + (claim.phase === "auctioning" || claim.phase === "delivery_pending") && + Object.keys(claim.publisherAuctions || {}).length === 0 && + claim.expiresAt <= now && + !hasReservedFallback + ) { + delete state.slots[elementId]; + } + }); + Object.keys(state.fallbackSlots).forEach(function (elementId) { + var element = state.fallbackSlots[elementId]; + if ( + !element.isConnected || + element.id !== elementId || + document.getElementById(elementId) !== element + ) { + delete state.fallbackSlots[elementId]; + } + }); + return state; + } + + function firstImpressionClaim(element) { + return firstImpressionState(Date.now()).slots[element.id]; + } + + function storeFirstImpressionClaim(state, claim) { + if ( + !state.slots[claim.slotElementId] && + Object.keys(state.slots).length >= MAX_FIRST_IMPRESSION_SLOTS + ) { + return false; + } + state.slots[claim.slotElementId] = claim; + return true; + } + + function claimFirstImpressionForTrustedServer(element) { + var now = Date.now(); + var state = firstImpressionState(now); + var existing = state.slots[element.id]; + if (existing) { + var canTransitionPublisherFallback = + existing.owner === "publisher" && + existing.phase !== "requested" && + existing.phase !== "rendered" && + existing.expiresAt <= now && + state.fallbackSlots[element.id] === element; + if (!canTransitionPublisherFallback) return null; + existing.owner = "trusted_server"; + existing.phase = "delivery_pending"; + existing.expiresAt = now + FIRST_IMPRESSION_LEASE_MS; + Object.keys(existing.publisherAuctions || {}).forEach(function (token) { + existing.publisherAuctions[token].suppressDelivery = true; + }); + return existing; + } + var claim = { + generation: state.generation, + slotElementId: element.id, + element: element, + owner: "trusted_server", + phase: "delivery_pending", + expiresAt: now + FIRST_IMPRESSION_LEASE_MS, + publisherAuctions: {}, + }; + return storeFirstImpressionClaim(state, claim) ? claim : null; + } + + function releaseTrustedServerFirstImpressionClaim(element, claim) { + var state = firstImpressionState(Date.now()); + if ( + state.slots[element.id] === claim && + claim.owner === "trusted_server" && + claim.phase === "delivery_pending" + ) { + delete state.slots[element.id]; + if (state.fallbackSlots[element.id] === element) { + delete state.fallbackSlots[element.id]; + } + } + } + + function installFirstImpressionListeners() { + if (ts.firstImpressionListenersInstalled) return; + tag.cmd.push(function () { + if (ts.firstImpressionListenersInstalled) return; + var pubads = window.googletag.pubads(); + if (!pubads || typeof pubads.addEventListener !== "function") return; + var observe = function (phase) { + return function (event) { + var elementId = + event.slot && event.slot.getSlotElementId + ? event.slot.getSlotElementId() + : ""; + var element = elementId && document.getElementById(elementId); + if (!element) return; + var state = firstImpressionState(Date.now()); + var claim = state.slots[elementId]; + if (!claim) { + storeFirstImpressionClaim(state, { + generation: state.generation, + slotElementId: elementId, + element: element, + owner: "publisher", + phase: phase, + expiresAt: Number.POSITIVE_INFINITY, + publisherAuctions: {}, + }); + return; + } + claim.phase = phase; + if (claim.owner === "publisher") { + claim.expiresAt = Number.POSITIVE_INFINITY; + } else { + claim.publisherRegistrationClosed = true; + } + }; + }; + pubads.addEventListener("slotRequested", observe("requested")); + pubads.addEventListener("slotRenderEnded", observe("rendered")); + ts.firstImpressionListenersInstalled = true; + }); + } + + installFirstImpressionListeners(); + // Minimal fallback for tsjs.scheduleInitialAdInit, mirroring the bundle's // hydration-safe scheduler in // crates/trusted-server-js/lib/src/integrations/gpt/index.ts: the @@ -117,7 +290,7 @@ // and deliberately identical to the bundle scheduler — the impression is // spent on a viewed tab, and the post-hydration guarantee holds whenever // the request is actually issued. - ts.scheduleInitialAdInit = function (initialBids, initialSlots) { + ts.scheduleInitialAdInit = function (initialBids, initialSlots, initialAuctionDiagnostics) { // The bundle may replace this scheduler after the fallback claims the initial // pass. Keep the latch on the shared document API so replacement cannot reset it. if ((ts.navGeneration || 0) !== 0 || ts.initialAdInitScheduled) return; @@ -127,6 +300,9 @@ // would overwrite a committed SPA navigation's slots. if (initialSlots !== undefined) ts.adSlots = initialSlots; if (initialBids !== undefined) ts.bids = initialBids; + if (initialAuctionDiagnostics !== undefined) { + ts.auctionDiagnostics = initialAuctionDiagnostics; + } var fire = function () { if ((ts.navGeneration || 0) !== 0) return; if (typeof ts.adInit === "function") ts.adInit(); @@ -412,10 +588,138 @@ installSlotHandoff(); + function bootstrapTargeting(slot, bid) { + var targeting = Object.assign({}, slot.targeting || {}); + ["hb_pb", "hb_bidder", "hb_adid", "hb_cache_host", "hb_cache_path"].forEach( + function (key) { + if (bid[key]) targeting[key] = String(bid[key]); + }, + ); + targeting.ts_initial = "1"; + return targeting; + } + + function scheduleFirstImpressionFallback(slot, bid, element, generation) { + var state = firstImpressionState(Date.now()); + if (state.fallbackSlots[element.id]) return; + state.fallbackSlots[element.id] = element; + + var retry = function () { + if ( + (ts.navGeneration || 0) !== generation || + !element.isConnected || + document.getElementById(element.id) !== element + ) { + return; + } + var claim = firstImpressionClaim(element); + if (claim) { + if ( + claim.owner !== "publisher" || + claim.phase === "requested" || + claim.phase === "rendered" + ) { + return; + } + var delay = Math.max(0, claim.expiresAt - Date.now()); + if (delay > 0) { + window.setTimeout(retry, delay + 1); + return; + } + } + + tag.cmd.push(function () { + if ( + (ts.navGeneration || 0) !== generation || + !element.isConnected || + document.getElementById(element.id) !== element + ) { + return; + } + var fallbackClaim = claimFirstImpressionForTrustedServer(element); + if (!fallbackClaim) return; + var pubads = window.googletag.pubads(); + var existingSlots = pubads.getSlots ? pubads.getSlots() : []; + var gptSlot = + existingSlots.find(function (candidate) { + return candidate.getSlotElementId() === element.id; + }) || null; + var tsOwned = false; + if (!gptSlot) { + gptSlot = runHandoffInternal(function () { + return window.googletag.defineSlot( + slot.gam_unit_path, + slot.formats, + element.id, + ); + }); + if (!gptSlot) { + releaseTrustedServerFirstImpressionClaim(element, fallbackClaim); + return; + } + gptSlot.addService(pubads); + tsOwned = true; + ts.gptSlotHandoffs = ts.gptSlotHandoffs || {}; + ts.gptSlotHandoffs[element.id] = { + gamUnitPath: slot.gam_unit_path, + formats: slot.formats, + divIdPrefix: slot.div_id, + slotElementId: element.id, + publisherClaimed: false, + suppressPublisherDisplay: false, + suppressPublisherRefresh: false, + }; + } + + var targeting = bootstrapTargeting(slot, bid); + Object.entries(targeting).forEach(function (entry) { + gptSlot.setTargeting(entry[0], entry[1]); + }); + fallbackClaim.targeting = targeting; + var slotElementId = gptSlot.getSlotElementId() || element.id; + ts.divToSlotId = ts.divToSlotId || {}; + ts.divToSlotId[element.id] = slot.id; + ts.divToSlotId[slotElementId] = slot.id; + ts.prevSlotTargetingKeys = ts.prevSlotTargetingKeys || {}; + var targetingKeys = Object.keys(slot.targeting || {}); + ts.prevSlotTargetingKeys[element.id] = targetingKeys; + ts.prevSlotTargetingKeys[slotElementId] = targetingKeys; + if (tsOwned) { + ts.prevGptSlots = ts.prevGptSlots || []; + ts.prevGptSlots.push(gptSlot); + } + if (!ts.servicesEnabled) { + pubads.enableSingleRequest(); + window.googletag.enableServices(); + ts.servicesEnabled = true; + } + if (tsOwned) { + runHandoffInternal(function () { + window.googletag.display(slotElementId); + }); + } + syncInitialLoadDisabled(window.googletag); + if (!tsOwned || ts.gptInitialLoadDisabled) { + ts.adInitRefreshInProgress = true; + try { + runHandoffInternal(function () { + pubads.refresh([gptSlot]); + }); + } finally { + ts.adInitRefreshInProgress = false; + } + } + }); + }; + + retry(); + } + ts.adInit = function () { var slots = ts.adSlots || []; var bids = ts.bids || {}; var divToSlotId = {}; + var nextSlotTargetingKeys = {}; // Generation this invocation belongs to. The slot work below is queued on // googletag.cmd, which drains only when GPT loads; recheck first inside // the queued callback so a navigation committed in the gap cancels the @@ -476,6 +780,14 @@ } var actualDivId = el.id; var b = bids[slot.id] || {}; + var tsClaim = claimFirstImpressionForTrustedServer(el); + if (!tsClaim) { + var currentClaim = firstImpressionClaim(el); + if (currentClaim && currentClaim.owner === "publisher") { + scheduleFirstImpressionFallback(slot, b, el, generation); + } + return; + } var existingSlots = googletag.pubads().getSlots(); var s = @@ -493,7 +805,10 @@ actualDivId, ); }); - if (!s) return; + if (!s) { + releaseTrustedServerFirstImpressionClaim(el, tsClaim); + return; + } s.addService(googletag.pubads()); tsOwned = true; ts.gptSlotHandoffs = ts.gptSlotHandoffs || {}; @@ -508,27 +823,21 @@ }; } - Object.entries(slot.targeting || {}).forEach(function (e) { - s.setTargeting(e[0], e[1]); - }); - [ - "hb_pb", - "hb_bidder", - "hb_adid", - "hb_cache_host", - "hb_cache_path", - ].forEach(function (k) { - if (b[k]) s.setTargeting(k, b[k]); + var targeting = bootstrapTargeting(slot, b); + Object.entries(targeting).forEach(function (entry) { + s.setTargeting(entry[0], entry[1]); }); - // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts - s.setTargeting("ts_initial", "1"); + tsClaim.targeting = targeting; // Map the resolved inner div to the slot ID. This bootstrap fires no // beacons and registers no slotRenderEnded listener; the map is consumed // by the bundle's render bridge (index.ts) once it loads. divToSlotId[actualDivId] = slot.id; var slotElementId = s.getSlotElementId(); + var targetingKeys = Object.keys(slot.targeting || {}); + nextSlotTargetingKeys[actualDivId] = targetingKeys; if (slotElementId && slotElementId !== actualDivId) { divToSlotId[slotElementId] = slot.id; + nextSlotTargetingKeys[slotElementId] = targetingKeys; } if (tsOwned) { newSlots.push(s); @@ -540,7 +849,10 @@ }); ts.prevGptSlots = newSlots; ts.divToSlotId = divToSlotId; - if (!ts.servicesEnabled) { + ts.prevSlotTargetingKeys = nextSlotTargetingKeys; + var hasRenderableWork = + slotsToDisplay.length > 0 || slotsToRefresh.length > 0; + if (!ts.servicesEnabled && hasRenderableWork) { googletag.pubads().enableSingleRequest(); googletag.enableServices(); ts.servicesEnabled = true; diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index 1447a8358..de1aedd36 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -62,6 +62,7 @@ pub enum GptDiagnosticsCookieAction { #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct GptDiagnosticsRequestDecision { active: bool, + browser_session_active: bool, clean_browser_path_and_query: Option, cookie_action: GptDiagnosticsCookieAction, } @@ -73,6 +74,16 @@ impl GptDiagnosticsRequestDecision { self.active } + /// Whether this request came from an activated diagnostics browser session. + /// + /// Unlike [`Self::active`], this remains true for non-document requests such + /// as the SPA page-bids fetch. It is captured before the private activation + /// cookie is stripped from the request. + #[must_use] + pub(crate) fn browser_session_active(&self) -> bool { + self.browser_session_active + } + /// Whether the response must be private and non-storeable. #[must_use] pub fn requires_private_no_store(&self) -> bool { @@ -121,6 +132,7 @@ impl GptDiagnosticsRequestDecision { pub(crate) fn active_for_tests() -> Self { Self { active: true, + browser_session_active: true, clean_browser_path_and_query: None, cookie_action: GptDiagnosticsCookieAction::None, } @@ -143,6 +155,7 @@ mod head_seam_invariant_tests { ] { out.push(GptDiagnosticsRequestDecision { active, + browser_session_active: active, clean_browser_path_and_query: clean.clone(), cookie_action, }); @@ -279,12 +292,19 @@ pub fn prepare_request( replace_path_and_query(request, &clean_path)?; } - let mut decision = GptDiagnosticsRequestDecision::default(); + let mut decision = GptDiagnosticsRequestDecision { + browser_session_active: integration_enabled + && directive == QueryDirective::Absent + && cookie_state.occurrences == 1 + && cookie_state.canonical, + ..GptDiagnosticsRequestDecision::default() + }; if integration_enabled && eligible_navigation && had_reserved_query { decision.clean_browser_path_and_query = Some(clean_path); match directive { QueryDirective::Enable => { decision.active = true; + decision.browser_session_active = true; decision.cookie_action = GptDiagnosticsCookieAction::SetSession; } QueryDirective::Disable => { @@ -547,6 +567,22 @@ mod tests { assert_eq!(duplicate.headers()[header::COOKIE], "other=value"); } + #[test] + fn active_cookie_marks_non_document_requests_without_activating_document_behavior() { + let mut request = Request::builder() + .method(Method::GET) + .uri("https://publisher.example/_ts/page-bids?path=/article") + .header(header::COOKIE, "__Host-ts-console=1; other=value") + .body(EdgeBody::empty()) + .expect("should build page-bids request"); + + let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + + assert!(!decision.active()); + assert!(decision.browser_session_active()); + assert_eq!(request.headers()[header::COOKIE], "other=value"); + } + #[test] fn invalid_duplicate_and_disable_directives_fail_closed() { for query in [ diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 228cee1cd..f896762b1 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1,6 +1,4 @@ -use std::collections::HashMap; -#[cfg(test)] -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::{Arc, LazyLock}; #[cfg(test)] use std::time::Duration; @@ -222,6 +220,113 @@ fn extract_prebid_error_message( #[cfg(test)] const GPC_US_PRIVACY: &str = "1YYN"; +/// Rejects a Prebid User ID identifier that Prebid.js could not address. +/// +/// Applies only the constraints Prebid itself imposes on a `userSync.userIds` +/// entry name and on a storage key: a non-empty, untrimmed-free ASCII token. +/// Anything narrower would encode one vendor's rules into core. +fn validate_prebid_user_id_token(value: &str) -> Result<(), ValidationError> { + let is_valid = !value.is_empty() + && value.trim() == value + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.')); + if is_valid { + return Ok(()); + } + + let mut error = ValidationError::new("invalid_prebid_user_id_token"); + error.message = Some( + "must be a non-empty ASCII token of letters, digits, `_`, `-`, or `.` without surrounding whitespace" + .into(), + ); + Err(error) +} + +/// Browser storage mechanism for an operator-managed Prebid User ID module. +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum PrebidUserIdStorageType { + /// Store the module's value in a browser cookie. + #[default] + Cookie, + /// Store the module's value in browser local storage. + Html5, +} + +/// Browser storage settings forwarded verbatim to a Prebid User ID module. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct PrebidManagedUserIdStorage { + /// Browser storage mechanism. + #[serde(default, rename = "type")] + pub storage_type: PrebidUserIdStorageType, + /// Cookie or local-storage key the module reads and writes. + #[validate(custom(function = "validate_prebid_user_id_token"))] + pub name: String, + /// Number of days the browser retains the stored value. + /// + /// Omitted leaves Prebid's own default in place. Core applies no upper + /// bound: the ceiling is a property of the selected module, not of Trusted + /// Server. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub expires: Option, + /// Number of seconds before the module may refresh the stored value. + /// + /// Omitted leaves Prebid's own default in place. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub refresh_in_seconds: Option, +} + +/// Rejects a managed User ID list that names the same module twice. +/// +/// Prebid keys `userSync.userIds` by entry name, so two entries sharing a name +/// give one submodule two conflicting configurations with no defined winner. +fn validate_unique_managed_user_id_names( + entries: &[PrebidManagedUserIdConfig], +) -> Result<(), ValidationError> { + let mut seen = HashSet::with_capacity(entries.len()); + let Some(duplicate) = entries + .iter() + .find(|entry| !seen.insert(entry.name.as_str())) + else { + return Ok(()); + }; + + let mut error = ValidationError::new("duplicate_managed_user_id_name"); + error.message = Some( + format!( + "managed Prebid User ID module `{}` is configured more than once", + duplicate.name + ) + .into(), + ); + Err(error) +} + +/// Operator-owned Prebid User ID module entry that Trusted Server manages. +/// +/// Core treats every entry as opaque: it validates only what Prebid.js needs to +/// address the module, then forwards the entry to the browser unchanged. Which +/// identity vendor an entry selects is an operator configuration choice, not a +/// property of core. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct PrebidManagedUserIdConfig { + /// Prebid `userSync.userIds` entry name, for example `sharedId`. + #[validate(custom(function = "validate_prebid_user_id_token"))] + pub name: String, + /// Module-specific parameters, forwarded to Prebid without inspection. + #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] + pub params: serde_json::Map, + /// Optional browser storage settings for the module. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(nested)] + pub storage: Option, +} + #[cfg(test)] #[derive(Debug, Clone, Deserialize, Serialize, Validate)] pub struct LegacyPrebidServerConfig { @@ -234,6 +339,13 @@ pub struct LegacyPrebidServerConfig { /// it in JavaScript. #[serde(default)] pub account_id: Option, + /// Prebid User ID modules that Trusted Server installs and keeps installed. + /// + /// Each entry is forwarded to Prebid.js verbatim; publisher-configured + /// entries with other names are preserved. Names must be unique. + #[serde(default)] + #[validate(nested, custom(function = "validate_unique_managed_user_id_names"))] + pub managed_user_ids: Vec, #[serde(default = "default_timeout_ms")] #[validate(range(min = 1, max = 60000))] pub timeout_ms: u32, @@ -393,6 +505,13 @@ pub struct PrebidIntegrationConfig { pub enabled: bool, #[serde(default)] pub account_id: Option, + /// Prebid User ID modules that Trusted Server installs and keeps installed. + /// + /// Each entry is forwarded to Prebid.js verbatim; publisher-configured + /// entries with other names are preserved. Names must be unique. + #[serde(default)] + #[validate(nested, custom(function = "validate_unique_managed_user_id_names"))] + pub managed_user_ids: Vec, #[serde(default = "default_timeout_ms")] pub timeout_ms: u32, #[serde(default)] @@ -429,6 +548,7 @@ impl Default for PrebidIntegrationConfig { Self { enabled: default_enabled(), account_id: None, + managed_user_ids: Vec::new(), timeout_ms: default_timeout_ms(), debug: false, script_patterns: default_script_patterns(), @@ -454,6 +574,7 @@ impl From<&LegacyPrebidServerConfig> for PrebidIntegrationConfig { Self { enabled: config.enabled, account_id: config.account_id.clone(), + managed_user_ids: config.managed_user_ids.clone(), timeout_ms: config.timeout_ms, debug: config.debug, script_patterns: config.script_patterns.clone(), @@ -1369,10 +1490,58 @@ impl IntegrationHeadInjector for PrebidIntegration { if let Some(inserts) = &self.planned_head_inserts { return inserts.clone(); } + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct InjectedManagedUserIdStorage<'a> { + #[serde(rename = "type")] + storage_type: PrebidUserIdStorageType, + name: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + expires: Option, + #[serde(skip_serializing_if = "Option::is_none")] + refresh_in_seconds: Option, + } + + impl<'a> From<&'a PrebidManagedUserIdStorage> for InjectedManagedUserIdStorage<'a> { + fn from(storage: &'a PrebidManagedUserIdStorage) -> Self { + Self { + storage_type: storage.storage_type, + name: &storage.name, + expires: storage.expires, + refresh_in_seconds: storage.refresh_in_seconds, + } + } + } + + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct InjectedManagedUserId<'a> { + name: &'a str, + #[serde(skip_serializing_if = "serde_json::Map::is_empty")] + params: &'a serde_json::Map, + #[serde(skip_serializing_if = "Option::is_none")] + storage: Option>, + } + + impl<'a> From<&'a PrebidManagedUserIdConfig> for InjectedManagedUserId<'a> { + fn from(config: &'a PrebidManagedUserIdConfig) -> Self { + Self { + name: &config.name, + params: &config.params, + storage: config + .storage + .as_ref() + .map(InjectedManagedUserIdStorage::from), + } + } + } + #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct InjectedPrebidClientConfig<'a> { account_id: &'a str, + #[serde(skip_serializing_if = "Vec::is_empty")] + managed_user_ids: Vec>, timeout: u32, debug: bool, bidders: &'a [String], @@ -1384,6 +1553,12 @@ impl IntegrationHeadInjector for PrebidIntegration { let payload = InjectedPrebidClientConfig { account_id: self.config.account_id.as_deref().unwrap_or_default(), + managed_user_ids: self + .config + .managed_user_ids + .iter() + .map(InjectedManagedUserId::from) + .collect(), timeout: self.config.timeout_ms, debug: self.config.debug, bidders: { @@ -3487,6 +3662,7 @@ mod tests { enabled: true, server_url: "https://prebid.example".to_string(), account_id: Some("test-account".to_string()), + managed_user_ids: Vec::new(), timeout_ms: 1000, bidders: vec!["exampleBidder".to_string()], debug: false, @@ -3509,6 +3685,19 @@ mod tests { } } + fn valid_managed_user_id() -> PrebidManagedUserIdConfig { + PrebidManagedUserIdConfig { + name: "exampleId".to_string(), + params: serde_json::Map::from_iter([("pid".to_string(), json!("999"))]), + storage: Some(PrebidManagedUserIdStorage { + storage_type: PrebidUserIdStorageType::Cookie, + name: "example_env".to_string(), + expires: Some(15), + refresh_in_seconds: Some(1800), + }), + } + } + struct PredictOnlyBackend; impl PlatformBackend for PredictOnlyBackend { @@ -3879,6 +4068,219 @@ server_url = "https://prebid.example/openrtb2/auction" ); } #[test] + fn managed_user_ids_parse_with_opaque_params() { + let config = parse_prebid_toml( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "exampleId" +params = { pid = "999", notUse3P = false, nested = { depth = 2 } } + +[integrations.prebid.managed_user_ids.storage] +type = "html5" +name = "example_env" +expires = 30 +refresh_in_seconds = 3600 +"#, + ); + + let [entry] = config.managed_user_ids.as_slice() else { + panic!("should parse exactly one managed User ID entry"); + }; + assert_eq!(entry.name, "exampleId", "should preserve the module name"); + assert_eq!( + Json::Object(entry.params.clone()), + json!({"pid": "999", "notUse3P": false, "nested": {"depth": 2}}), + "should carry module parameters through without inspecting them" + ); + + let storage = entry.storage.as_ref().expect("should parse storage"); + assert_eq!( + storage.storage_type, + PrebidUserIdStorageType::Html5, + "should preserve the configured storage mechanism" + ); + assert_eq!(storage.name, "example_env", "should preserve storage key"); + assert_eq!(storage.expires, Some(30), "should preserve expiry"); + assert_eq!( + storage.refresh_in_seconds, + Some(3600), + "should preserve refresh interval" + ); + } + + #[test] + fn managed_user_ids_leave_prebid_defaults_in_place_when_unset() { + let config = parse_prebid_toml( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "exampleId" + +[integrations.prebid.managed_user_ids.storage] +name = "example_env" +"#, + ); + + let [entry] = config.managed_user_ids.as_slice() else { + panic!("should parse exactly one managed User ID entry"); + }; + assert!( + entry.params.is_empty(), + "should treat parameters as optional" + ); + + let storage = entry.storage.as_ref().expect("should parse storage"); + assert_eq!( + storage.storage_type, + PrebidUserIdStorageType::Cookie, + "should default to cookie storage" + ); + assert_eq!( + storage.expires, None, + "should leave Prebid's own expiry default in place" + ); + assert_eq!( + storage.refresh_in_seconds, None, + "should leave Prebid's own refresh default in place" + ); + } + + #[test] + fn managed_user_ids_allow_an_entry_without_storage() { + let config = parse_prebid_toml( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "exampleId" +"#, + ); + + let [entry] = config.managed_user_ids.as_slice() else { + panic!("should parse exactly one managed User ID entry"); + }; + assert!( + entry.storage.is_none(), + "should treat storage as optional for modules that need none" + ); + } + + #[test] + fn managed_user_ids_reject_invalid_values() { + for (name, entry_section) in [ + ("missing name", "params = { pid = \"999\" }"), + ("empty name", "name = \"\""), + ("padded name", "name = \" exampleId \""), + ("name with a space", "name = \"example id\""), + ( + "unknown entry field", + "name = \"exampleId\"\nunsupported = true", + ), + ( + "empty storage name", + "name = \"exampleId\"\n\n[integrations.prebid.managed_user_ids.storage]\nname = \"\"", + ), + ( + "missing storage name", + "name = \"exampleId\"\n\n[integrations.prebid.managed_user_ids.storage]\ntype = \"cookie\"", + ), + ( + "zero expiry", + "name = \"exampleId\"\n\n[integrations.prebid.managed_user_ids.storage]\nname = \"example_env\"\nexpires = 0", + ), + ( + "zero refresh", + "name = \"exampleId\"\n\n[integrations.prebid.managed_user_ids.storage]\nname = \"example_env\"\nrefresh_in_seconds = 0", + ), + ( + "unknown storage mechanism", + "name = \"exampleId\"\n\n[integrations.prebid.managed_user_ids.storage]\nname = \"example_env\"\ntype = \"session\"", + ), + ( + "unknown storage field", + "name = \"exampleId\"\n\n[integrations.prebid.managed_user_ids.storage]\nname = \"example_env\"\nunsupported = true", + ), + ] { + let result = parse_prebid_toml_result(&format!( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +{entry_section} +"# + )); + + assert!(result.is_err(), "should reject {name}"); + } + } + + #[test] + fn managed_user_ids_reject_a_repeated_module_name() { + let result = parse_prebid_toml_result( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "exampleId" +params = { pid = "1" } + +[[integrations.prebid.managed_user_ids]] +name = "exampleId" +params = { pid = "2" } +"#, + ); + + assert!( + result.is_err(), + "should reject the same module configured twice" + ); + } + + #[test] + fn managed_user_ids_accept_distinct_module_names() { + let config = parse_prebid_toml( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "exampleId" + +[[integrations.prebid.managed_user_ids]] +name = "otherExampleId" +"#, + ); + + assert_eq!( + config.managed_user_ids.len(), + 2, + "should keep every distinctly named module" + ); + } + + #[test] + fn managed_user_ids_default_to_none_configured() { + let config = parse_prebid_toml( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" +"#, + ); + + assert!( + config.managed_user_ids.is_empty(), + "should manage no User ID modules by default" + ); + } + #[test] fn excluded_gam_ad_unit_path_suffixes_reject_invalid_values() { for (suffix, expected_message) in [ ("", "must not be empty"), @@ -4903,6 +5305,121 @@ external_bundle_sri = "sha384-AAAA" assert!(!config.debug); } + #[test] + fn head_injector_includes_managed_user_ids() { + let mut config = base_config(); + config.managed_user_ids = vec![PrebidManagedUserIdConfig { + name: "exampleId".to_string(), + params: serde_json::Map::from_iter([ + ("pid".to_string(), json!("999")), + ("notUse3P".to_string(), json!(true)), + ]), + storage: Some(PrebidManagedUserIdStorage { + storage_type: PrebidUserIdStorageType::Html5, + name: "example_env".to_string(), + expires: Some(30), + refresh_in_seconds: Some(3600), + }), + }]; + let integration = PrebidIntegration::new(config); + let document_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "pub.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + + let inserts = integration.head_inserts(&ctx); + let script = &inserts[0]; + + assert!( + script.contains( + r#""managedUserIds":[{"name":"exampleId","params":{"notUse3P":true,"pid":"999"},"storage":{"type":"html5","name":"example_env","expires":30,"refreshInSeconds":3600}}]"# + ), + "should inject the managed User ID entry verbatim: {script}" + ); + } + + #[test] + fn head_injector_omits_optional_managed_user_id_fields_when_unset() { + let mut config = base_config(); + config.managed_user_ids = vec![PrebidManagedUserIdConfig { + name: "exampleId".to_string(), + params: serde_json::Map::new(), + storage: None, + }]; + let integration = PrebidIntegration::new(config); + let document_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "pub.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + + let inserts = integration.head_inserts(&ctx); + let script = &inserts[0]; + + assert!( + script.contains(r#""managedUserIds":[{"name":"exampleId"}]"#), + "should omit empty parameters and absent storage: {script}" + ); + } + + #[test] + fn head_injector_omits_managed_user_ids_when_none_configured() { + let integration = PrebidIntegration::new(base_config()); + let document_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "pub.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + + let inserts = integration.head_inserts(&ctx); + let script = &inserts[0]; + + assert!( + !script.contains("managedUserIds"), + "should omit managed User IDs when none are configured: {script}" + ); + } + + #[test] + fn head_injector_escapes_script_breakout_in_managed_user_ids() { + let mut config = base_config(); + config.managed_user_ids = vec![PrebidManagedUserIdConfig { + params: serde_json::Map::from_iter([( + "pid".to_string(), + json!("1"), + )]), + ..valid_managed_user_id() + }]; + let integration = PrebidIntegration::new(config); + let document_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "pub.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + + let inserts = integration.head_inserts(&ctx); + let script = &inserts[0]; + + assert!( + script.contains(r#""pid":"1\u003c/script>\u003cscript>alert(1)\u003c/script>""#), + "should retain the escaped module parameter: {script}" + ); + assert_eq!( + script.matches("").count(), + 1, + "should contain only the legitimate outer closing script tag" + ); + } + #[test] fn head_injector_includes_excluded_gam_ad_unit_path_suffixes() { let mut config = base_config(); diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index d858e5a12..855ad339d 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -932,6 +932,17 @@ impl IntegrationRegistry { self.find_route(method, path).is_some() } + /// Return true when at least one integration request filter is + /// registered. + /// + /// Adapters use this to decide whether to record a request-filter phase + /// timing span, so unconfigured deployments (no request filters) omit + /// that entry from observability output entirely. + #[must_use] + pub fn has_request_filters(&self) -> bool { + !self.inner.request_filters.is_empty() + } + /// Run pre-routing request filters. /// /// Request header mutations are applied immediately so later filters and diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 76621baf7..1db7b318f 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -31,6 +31,7 @@ ) )] +pub mod access_telemetry; pub(crate) mod asset_image_optimizer; pub mod auction; pub mod auction_config_types; @@ -61,6 +62,7 @@ pub mod proxy; pub mod publisher; pub mod redacted; pub mod request_signing; +pub mod request_timing; pub mod response_privacy; pub mod rsc_flight; pub(crate) mod s3_sigv4; diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 2553229a4..4454d25cc 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -43,6 +43,7 @@ mod template_assembly; mod template_cache; #[cfg(test)] pub(crate) mod test_support; +mod timed_kv; mod traits; mod types; @@ -72,6 +73,7 @@ pub use template_cache::{ TemplateEntry, TemplateMetadata, TemplateMetadataEncodeError, UnavailableTemplateCache, VaryHeaderValues, VarySpec, }; +pub use timed_kv::TimedKvStore; pub use traits::{PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore}; pub use types::{ ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 70eb55a9a..0e1b9f8e7 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -593,6 +593,7 @@ impl PlatformHttpClient for StubHttpClient { .pop_front() .ok_or_else(|| Report::new(PlatformError::HttpClient))?; + let stream_response = stream_response || response.stream_body; let edge_response = build_stub_pending_response( StubPendingResponse { backend_name: request.backend_name, @@ -600,7 +601,7 @@ impl PlatformHttpClient for StubHttpClient { body: response.body, headers: response.headers, }, - stream_response || response.stream_body, + stream_response, request_is_head, )?; diff --git a/crates/trusted-server-core/src/platform/timed_kv.rs b/crates/trusted-server-core/src/platform/timed_kv.rs new file mode 100644 index 000000000..2a81ac0f7 --- /dev/null +++ b/crates/trusted-server-core/src/platform/timed_kv.rs @@ -0,0 +1,185 @@ +//! Latency-only timing decorator for KV store handles. +//! +//! [`TimedKvStore`] wraps an inner store plus a [`RequestTimings`] handle and +//! records [`Phase::EcKv`] around every call. It implements both +//! [`PlatformKvStore`] (for consent-store access obtained through +//! [`RuntimeServices`](super::RuntimeServices)) and [`EcKvStore`] (for +//! [`KvIdentityGraph`](crate::ec::kv::KvIdentityGraph) construction sites), +//! because no single existing abstraction covers the whole `ts-kv` taxonomy: +//! EC graph operations go through [`EcKvStore`] while consent persistence +//! uses [`PlatformKvStore`] directly. +//! +//! The decorator measures store-call latency only: it never reads, parses, +//! or logs any value passing through it. + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use bytes::Bytes; +use edgezero_core::key_value_store::{KvError, KvPage, KvStore as PlatformKvStore}; +use error_stack::Report; + +use crate::ec::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteOutcome}; +use crate::error::TrustedServerError; +use crate::request_timing::{Phase, RequestTimings}; + +/// Wraps `inner` plus a [`RequestTimings`] handle, recording [`Phase::EcKv`] +/// around every store call made through it. +pub struct TimedKvStore { + /// The wrapped store handle. + inner: S, + /// The request's phase-timing collector. + timings: RequestTimings, +} + +impl TimedKvStore { + /// Creates a decorator around `inner` that records into `timings`. + #[must_use] + pub fn new(inner: S, timings: RequestTimings) -> Self { + Self { inner, timings } + } +} + +#[async_trait(?Send)] +impl PlatformKvStore for TimedKvStore> { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + let _span = self.timings.span(Phase::EcKv); + self.inner.get_bytes(key).await + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + let _span = self.timings.span(Phase::EcKv); + self.inner.put_bytes(key, value).await + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: Bytes, + ttl: Duration, + ) -> Result<(), KvError> { + let _span = self.timings.span(Phase::EcKv); + self.inner.put_bytes_with_ttl(key, value, ttl).await + } + + async fn delete(&self, key: &str) -> Result<(), KvError> { + let _span = self.timings.span(Phase::EcKv); + self.inner.delete(key).await + } + + async fn list_keys_page( + &self, + prefix: &str, + cursor: Option<&str>, + limit: usize, + ) -> Result { + let _span = self.timings.span(Phase::EcKv); + self.inner.list_keys_page(prefix, cursor, limit).await + } +} + +impl EcKvStore for TimedKvStore { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + let _span = self.timings.span(Phase::EcKv); + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + let _span = self.timings.span(Phase::EcKv); + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + let _span = self.timings.span(Phase::EcKv); + self.inner.insert(key, write) + } + + fn count_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result> { + let _span = self.timings.span(Phase::EcKv); + self.inner.count_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + let _span = self.timings.span(Phase::EcKv); + self.inner.delete(key) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration as StdDuration; + + use super::*; + use crate::ec::kv_backend::test_support::InMemoryEcKv; + + #[test] + fn ec_kv_store_operations_accumulate_into_ec_kv_phase() { + let timings = RequestTimings::new(); + let store = TimedKvStore::new(InMemoryEcKv::new("test-store"), timings.clone()); + + store + .insert( + "key-a", + EcKvWrite { + body: "{}", + metadata: "{}", + ttl: StdDuration::from_secs(60), + mode: crate::ec::kv_backend::EcKvWriteMode::Add, + }, + ) + .expect("should insert into the in-memory store"); + store.lookup("key-a").expect("should read back the entry"); + + timings.mark_headers_ready(); + assert!( + timings.snapshot().kv_ms.is_some(), + "should record Phase::EcKv across both store calls" + ); + } + + #[test] + fn store_name_is_not_timed() { + let timings = RequestTimings::new(); + let store = TimedKvStore::new(InMemoryEcKv::new("test-store"), timings.clone()); + + assert_eq!(store.store_name(), "test-store"); + timings.mark_headers_ready(); + assert!( + timings.snapshot().kv_ms.is_none(), + "store_name is a metadata accessor, not a store operation" + ); + } + + #[test] + fn platform_kv_store_operations_accumulate_into_ec_kv_phase() { + let timings = RequestTimings::new(); + let inner: Arc = Arc::new(crate::platform::UnavailableKvStore); + let store = TimedKvStore::new(inner, timings.clone()); + + // UnavailableKvStore errors on every call; the decorator still times + // the attempt regardless of outcome. + futures::executor::block_on(async { + let _ = store.get_bytes("key").await; + let _ = store.put_bytes("key", Bytes::from_static(b"value")).await; + }); + + timings.mark_headers_ready(); + assert!( + timings.snapshot().kv_ms.is_some(), + "should record Phase::EcKv even when the inner store errors" + ); + } +} diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 05daf0b4e..e4a03f610 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -57,7 +57,10 @@ use crate::cache_policy::{ use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; -use crate::creative_opportunities::{AssemblyMode, CreativeOpportunitiesConfig}; +use crate::creative_opportunities::{ + AdStackGateInput, AssemblyMode, CreativeOpportunitiesConfig, RuntimeAdStackExpected, + evaluate_ad_stack_gate, +}; use crate::ec::EcContext; use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; @@ -70,6 +73,7 @@ use crate::platform::{ contains_publisher_esi_directive, }; use crate::price_bucket::{PriceGranularity, price_bucket}; +use crate::request_timing::{AuctionWaitPlacement, Phase, RequestTimings}; use crate::response_privacy::{ apply_inactive_ad_stack_browser_cache_policy, cache_control_forbids_shared_storage, enforce_synthesized_html_cache_privacy, enforce_terminal_private_cache_privacy, @@ -90,21 +94,44 @@ const DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); const HEADER_X_TS_TEMPLATE_CACHE: &str = "x-ts-template-cache"; const HEADER_X_TS_ASSEMBLY: &str = "x-ts-assembly"; -#[derive(Clone, Copy, PartialEq, Eq)] -enum TemplateCacheResponseState { +/// Outcome of a template-cache lookup/store attempt for one response. +/// +/// Set on every response that passes through the assembly pipeline via +/// [`set_template_cache_response_state`], which writes both the +/// `x-ts-template-cache` response header and this same value as a typed +/// response extension, so the two can never drift. Access telemetry reads +/// the extension rather than the header, since operator-configured response +/// headers can override a managed header but cannot touch extensions. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TemplateCacheResponseState { + /// The cached template was found and reused. Hit, + /// No cached template existed; the cache store is reserved for this + /// content type. MissReserved, + /// No cached template existed; one was stored after assembly. MissStored, + /// No cached template existed; storing the freshly assembled template + /// failed. MissStoreError, + /// The request bypassed the cache lookup. BypassRequest, + /// The response bypassed the cache store. BypassResponse, + /// The response's content type is not supported by the template cache. Unsupported, + /// The cached template entry was invalid and could not be reused. Invalid, + /// A backend error prevented the cache lookup or store. BackendError, } impl TemplateCacheResponseState { - const fn as_str(self) -> &'static str { + /// Renders this variant as the string written to the + /// `x-ts-template-cache` header and the `template_cache_state` access + /// telemetry column. + #[must_use] + pub const fn as_str(self) -> &'static str { match self { Self::Hit => "hit", Self::MissReserved => "miss-reserved", @@ -127,6 +154,7 @@ fn set_template_cache_response_state( HEADER_X_TS_TEMPLATE_CACHE, HeaderValue::from_static(state.as_str()), ); + response.extensions_mut().insert(state); } #[derive(Clone, Copy, PartialEq, Eq)] @@ -1616,6 +1644,12 @@ pub struct OwnedProcessResponseParams { /// rescanned from the output, which cannot tell a `nonce` attribute from the same /// word inside a script. pub(crate) csp_nonce_observed: Option>, + /// Per-request phase-timing handle, carried into the streaming/buffered + /// finalizers so the `` seam wait can be recorded with the right + /// [`AuctionWaitPlacement`]. Cheap to clone (an `Arc` handle); a request that + /// never attached one to its extensions gets a fresh, unattached collector + /// that nothing ever renders. + pub(crate) timings: RequestTimings, } /// Response-authorized template cache insert inputs. The key is built before origin lookup; the @@ -1859,6 +1893,8 @@ pub async fn buffer_publisher_response_async( ¶ms.request_scheme, ¶ms.request_host, ), + timings: params.timings.clone(), + placement: AuctionWaitPlacement::PreHeader, }, ) .await; @@ -2018,6 +2054,7 @@ fn build_template_assembly_params( request_scheme: &str, price_granularity: PriceGranularity, ad_bids_state: AdBidsState, + timings: RequestTimings, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { csp_nonce_observed: None, @@ -2040,6 +2077,7 @@ fn build_template_assembly_params( price_granularity, gpt_diagnostics: None, suppress_datadome_client_side_tag: false, + timings, } } @@ -2382,6 +2420,8 @@ pub async fn publisher_response_into_streaming_response( ¶ms.request_scheme, ¶ms.request_host, ), + timings: params.timings.clone(), + placement: AuctionWaitPlacement::InStream, }, ) .await; @@ -2500,6 +2540,7 @@ pub async fn publisher_response_into_streaming_response( &orchestrator, &services, &settings, + AuctionWaitPlacement::InStream, ) .await; // Collection reached a terminal result; disarm only now @@ -2521,6 +2562,8 @@ pub async fn publisher_response_into_streaming_response( ¶ms.request_scheme, ¶ms.request_host, ), + timings: params.timings.clone(), + placement: AuctionWaitPlacement::InStream, }; while let Some(step) = hold_step_next_chunk( @@ -2856,6 +2899,7 @@ pub async fn stream_publisher_body_async( orchestrator, services, settings, + AuctionWaitPlacement::PreHeader, ) .await; if body.is_stream() { @@ -2922,6 +2966,8 @@ pub async fn stream_publisher_body_async( services, settings, request_origin: request_origin(¶ms.request_scheme, ¶ms.request_host), + timings: params.timings.clone(), + placement: AuctionWaitPlacement::PreHeader, }, }, ) @@ -3052,15 +3098,12 @@ pub(crate) fn is_prefetch_request(req: &Request) -> bool { header("sec-purpose") || header("purpose") } -#[derive(Debug, Clone, Copy)] -struct ServerSideAdStackConfig { - /// Dedicated `[creative_opportunities].enabled` switch. - ad_templates_enabled: bool, - /// Global `[auction].enabled` gate used by publisher/page-bids flows. - auction_enabled: bool, -} - /// Returns whether request-scoped signals permit an ad-eligible navigation. +/// +/// This is the request half of the shared ad-stack gate: the configuration +/// halves (`matched_slots`, the kill switches) are deliberately absent, because +/// the cache policy for a structurally inactive template must distinguish a +/// page that no request could activate from one this particular request skipped. fn is_server_side_ad_eligible_navigation( is_get: bool, is_navigation: bool, @@ -3071,29 +3114,6 @@ fn is_server_side_ad_eligible_navigation( is_get && is_navigation && !is_prefetch && !is_bot && consent_allows_auction } -/// Returns true only when the publisher should inject and run server-side ad templates. -/// -/// This includes auction dispatch plus initial ad-slot injection. -fn should_run_server_side_ad_stack( - is_get: bool, - is_navigation: bool, - is_prefetch: bool, - is_bot: bool, - has_matched_slots: bool, - consent_allows_auction: bool, - config: ServerSideAdStackConfig, -) -> bool { - is_server_side_ad_eligible_navigation( - is_get, - is_navigation, - is_prefetch, - is_bot, - consent_allows_auction, - ) && config.ad_templates_enabled - && has_matched_slots - && config.auction_enabled -} - /// Write winning bids from an auction result into the shared `ad_bids_state` lock. /// Build the request origin (`scheme://host`, where `host` includes any port) /// used to emit absolute first-party URLs in inline creatives. Returns an empty @@ -3121,6 +3141,40 @@ fn request_origin(scheme: &str, host: &str) -> String { /// JSON for every non-empty map; `serde_json::from_str` failed and `unwrap_or_default()` /// turned the failure into `{}`. Shared modes therefore served **zero bids**, silently, /// on every request that had any. Every fixture had empty bids, so nothing caught it. +#[derive(Clone, Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +struct BrowserAuctionDiagnostics { + #[serde(skip_serializing_if = "Option::is_none")] + auction_dispatched_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auction_resolved_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auction_committed_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auction_wait_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auction_wait_placement: Option<&'static str>, +} + +impl BrowserAuctionDiagnostics { + fn from_request_timings(timings: &RequestTimings) -> Option { + let snapshot = timings.snapshot(); + snapshot.auction_dispatched_ms?; + Some(Self { + auction_dispatched_ms: snapshot.auction_dispatched_ms, + auction_resolved_ms: snapshot.auction_resolved_ms, + auction_committed_ms: snapshot.auction_committed_ms, + auction_wait_ms: snapshot.auction_wait_ms, + auction_wait_placement: snapshot.auction_wait_placement.map( + |placement| match placement { + AuctionWaitPlacement::PreHeader => "pre_header", + AuctionWaitPlacement::InStream => "in_stream", + }, + ), + }) + } +} + #[derive(Clone, Default)] pub(crate) struct AdBidsState { /// Rendered bids `` sequences inside the string. pub(crate) fn build_bids_script(bid_map: &serde_json::Map) -> String { + build_bids_script_with_diagnostics(bid_map, None) +} + +fn build_bids_script_with_diagnostics( + bid_map: &serde_json::Map, + auction_diagnostics: Option<&BrowserAuctionDiagnostics>, +) -> String { let json = serde_json::to_string(bid_map) .expect("serde_json::to_string of Map should be infallible"); let escaped = html_escape_for_script(&json); @@ -5512,6 +5677,23 @@ pub(crate) fn build_bids_script(bid_map: &serde_json::Map(function(){{\ +var t=window.tsjs=window.tsjs||{{}};\ +var b=JSON.parse(\"{}\");\ +var d=JSON.parse(\"{}\");\ +var s=t.scheduleInitialAdInit;\ +if(typeof s===\"function\")s(b,void 0,d);\ +else{{t.bids=b;t.auctionDiagnostics=d;}}\ +}})();", + escaped, + html_escape_for_script(&diagnostics) + ); + } + format!( "", + html_escape_for_script(slots_json), + html_escape_for_script(&bids), + html_escape_for_script(&diagnostics) + ); + } + format!( "`, + }), + ); + await page.route(IFRAME_CREATIVE_URL, (route) => { + creativeRequests += 1; + return route.fulfill({ + status: 200, + contentType: "text/html", + body: IFRAME_CREATIVE, + }); + }); + + await page.goto(runtimeUrl("/aps-puc-topology-test")); + await page.addScriptTag({ path: clientAuctionBundlePaths().gpt }); + await page.evaluate( + ({ creativeUrl, outerUrl, selectedAdId }) => { + const typedWindow = window as unknown as { + tsjs: Record; + pucEvents: Array>; + }; + typedWindow.tsjs = { + bids: { + "aps-slot": { + hb_adid: selectedAdId, + hb_bidder: "fictional", + hb_pb: "1.23", + adm: ``, + w: 300, + h: 250, + }, + }, + adSlots: [ + { + id: "aps-slot", + div_id: "div-aps", + gam_unit_path: "/fictional/aps", + formats: [[300, 250]], + }, + ], + }; + typedWindow.pucEvents = []; + const locator = document.createElement("iframe"); + locator.name = "__pb_locator__"; + document.body.appendChild(locator); + window.addEventListener("message", (event) => { + try { + const message = JSON.parse( + String(event.data), + ) as Record; + if (message.message === "Prebid Event") { + typedWindow.pucEvents.push(message); + } + } catch { + // Ignore unrelated publisher messages. + } + }); + + const slot = document.getElementById("div-aps")!; + slot.style.width = "1px"; + slot.style.height = "1px"; + const outerShell = document.createElement("div"); + outerShell.id = "aps-outer-shell"; + outerShell.style.width = "1px"; + outerShell.style.height = "1px"; + const innerShell = document.createElement("div"); + innerShell.id = "aps-inner-shell"; + innerShell.style.width = "1px"; + innerShell.style.height = "1px"; + const frame = document.createElement("iframe"); + frame.id = "google_ads_iframe_fictional_0"; + frame.width = "1"; + frame.height = "1"; + frame.style.width = "1px"; + frame.style.height = "1px"; + frame.src = outerUrl; + innerShell.appendChild(frame); + outerShell.appendChild(innerShell); + slot.appendChild(outerShell); + + const other = document.getElementById("div-other")!; + const otherFrame = document.createElement("iframe"); + otherFrame.width = "1"; + otherFrame.height = "1"; + otherFrame.style.width = "1px"; + otherFrame.style.height = "1px"; + other.appendChild(otherFrame); + }, + { + creativeUrl: IFRAME_CREATIVE_URL, + outerUrl: outerCreativeUrl, + selectedAdId: adId, + }, + ); + + await expect.poll(() => creativeRequests).toBe(1); + await expect + .poll(() => + page.evaluate(() => + ( + window as unknown as { + pucEvents: Array>; + } + ).pucEvents.some( + (event) => event.event === "adRenderSucceeded", + ), + ), + ) + .toBe(true); + await expect(page.locator("#google_ads_iframe_fictional_0")).toHaveCSS( + "width", + "300px", + ); + await expect(page.locator("#google_ads_iframe_fictional_0")).toHaveCSS( + "height", + "250px", + ); + await expect(page.locator("#div-aps")).toHaveCSS("width", "300px"); + await expect(page.locator("#div-aps")).toHaveCSS("height", "250px"); + await expect(page.locator("#aps-outer-shell")).toHaveCSS( + "width", + "300px", + ); + await expect(page.locator("#aps-outer-shell")).toHaveCSS( + "height", + "250px", + ); + await expect(page.locator("#aps-inner-shell")).toHaveCSS( + "width", + "300px", + ); + await expect(page.locator("#aps-inner-shell")).toHaveCSS( + "height", + "250px", + ); + await expect(page.locator("#div-other iframe")).toHaveCSS( + "width", + "1px", + ); + await expect(page.locator("#div-other iframe")).toHaveCSS( + "height", + "1px", + ); + }); + test("renders a trustedServer adapter bid using Prebid's generated GAM ad ID", async ({ page, }) => { diff --git a/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml b/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml index 816dcbfcf..1e403e2d7 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml @@ -86,6 +86,8 @@ [local_server.config_stores.edgezero_runtime_env] format = "inline-toml" [local_server.config_stores.edgezero_runtime_env.contents] + # Viceroy reports this fixed synthetic service id. EdgeZero scopes + # Fastly runtime mappings by service id. EDGEZERO__SERVICES__0000000000000000000000__STORES__SECRETS__TRUSTED_SERVER_SECRETS__NAME = "ts_secrets" # Generated integration configs inject the trusted_server_config blob diff --git a/crates/trusted-server-js/lib/build-prebid-external.mjs b/crates/trusted-server-js/lib/build-prebid-external.mjs index eb6e42826..24176fef8 100644 --- a/crates/trusted-server-js/lib/build-prebid-external.mjs +++ b/crates/trusted-server-js/lib/build-prebid-external.mjs @@ -241,6 +241,13 @@ function generateExternalEntry(entryFile, adapters, bidderCodes) { "import 'prebid.js/modules/consentManagementTcf.js';", "import 'prebid.js/modules/consentManagementGpp.js';", "import 'prebid.js/modules/consentManagementUsp.js';", + // consentManagement* only retrieves the consent signal. tcfControl is what + // registers the activity controls (accessDevice, syncUser, enrichEids, + // transmitEids, fetchBids) that act on it, so without it a TC string that + // denies a purpose changes nothing: User ID submodules still write storage + // and still call their vendor endpoints. Keep it bundled whenever + // consentManagementTcf is bundled. + "import 'prebid.js/modules/tcfControl.js';", "import 'prebid.js/modules/userId.js';", "import './_adapters.generated';", "import { INCLUDED_PREBID_USER_ID_MODULES } from './_user_ids.generated';", diff --git a/crates/trusted-server-js/lib/src/core/first_impression.ts b/crates/trusted-server-js/lib/src/core/first_impression.ts new file mode 100644 index 000000000..cf976e0e3 --- /dev/null +++ b/crates/trusted-server-js/lib/src/core/first_impression.ts @@ -0,0 +1,370 @@ +import { resolveSlotElementByDivId } from './slot_element'; +import type { + FirstImpressionPhase, + FirstImpressionPublisherAuction, + FirstImpressionSlotClaim, + FirstImpressionState, + TsjsApi, +} from './types'; + +/** Time allowed for one navigation's losing first-impression delivery. */ +export const FIRST_IMPRESSION_LEASE_MS = 5000; + +const MAX_FIRST_IMPRESSION_SLOTS = 256; +const MAX_PUBLISHER_AUCTIONS_PER_SLOT = 16; + +function currentGeneration(ts: TsjsApi): number { + return ts.navGeneration ?? 0; +} + +function claimMatchesElement( + claim: FirstImpressionSlotClaim, + element: HTMLElement, + generation: number +): boolean { + return ( + claim.generation === generation && + claim.slotElementId === element.id && + claim.element === element && + element.ownerDocument === document && + element.isConnected && + document.getElementById(element.id) === element + ); +} + +function removePublisherAuction( + state: FirstImpressionState, + claim: FirstImpressionSlotClaim, + token: string, + now: number +): void { + delete claim.publisherAuctions[token]; + if ( + claim.owner === 'publisher' && + (claim.phase === 'auctioning' || claim.phase === 'delivery_pending') && + Object.keys(claim.publisherAuctions).length === 0 && + claim.expiresAt <= now + ) { + delete state.slots[claim.slotElementId]; + } +} + +function pruneFirstImpressionState(ts: TsjsApi, now = Date.now()): FirstImpressionState { + const generation = currentGeneration(ts); + if (ts.firstImpression?.generation !== generation) { + ts.firstImpression = { generation, nextToken: 0, slots: {}, fallbackSlots: {} }; + } + + const state = ts.firstImpression; + state.slots ??= {}; + state.fallbackSlots ??= {}; + for (const [elementId, claim] of Object.entries(state.slots)) { + if ( + claim.slotElementId !== elementId || + !claimMatchesElement(claim, claim.element, generation) + ) { + delete state.slots[elementId]; + continue; + } + const hasReservedFallback = + claim.owner === 'publisher' && + (claim.phase === 'auctioning' || claim.phase === 'delivery_pending') && + state.fallbackSlots[elementId] === claim.element; + for (const [token, auction] of Object.entries(claim.publisherAuctions)) { + // A TS-owned losing publisher auction remains a fail-closed tombstone for + // this physical element and navigation. Publisher registrations also stay + // intact while an expired claim is waiting to transition to its reserved + // TS fallback, so an overlapping late callback cannot escape suppression. + if ( + auction.expiresAt <= now && + !hasReservedFallback && + !(claim.owner === 'trusted_server' && auction.suppressDelivery) + ) { + removePublisherAuction(state, claim, token, now); + } + } + if ( + claim.owner === 'publisher' && + (claim.phase === 'auctioning' || claim.phase === 'delivery_pending') && + Object.keys(claim.publisherAuctions).length === 0 && + claim.expiresAt <= now && + !hasReservedFallback + ) { + delete state.slots[elementId]; + } + } + for (const [elementId, element] of Object.entries(state.fallbackSlots)) { + if ( + !element.isConnected || + element.id !== elementId || + document.getElementById(elementId) !== element + ) { + delete state.fallbackSlots[elementId]; + } + } + return state; +} + +/** Resolve a publisher ad-unit code with the same contract GPT uses. */ +export function resolveFirstImpressionElement(adUnitCode: string): HTMLElement | undefined { + return resolveSlotElementByDivId(adUnitCode).element ?? undefined; +} + +/** Return the live ownership claim for an exact slot element. */ +export function firstImpressionClaim( + ts: TsjsApi, + element: HTMLElement +): FirstImpressionSlotClaim | undefined { + const state = pruneFirstImpressionState(ts); + const claim = state.slots[element.id]; + return claim && claimMatchesElement(claim, element, state.generation) ? claim : undefined; +} + +function storeClaim(state: FirstImpressionState, claim: FirstImpressionSlotClaim): boolean { + if ( + !state.slots[claim.slotElementId] && + Object.keys(state.slots).length >= MAX_FIRST_IMPRESSION_SLOTS + ) { + return false; + } + state.slots[claim.slotElementId] = claim; + return true; +} + +/** Atomically claim an untouched slot for Trusted Server. */ +export function claimFirstImpressionForTrustedServer( + ts: TsjsApi, + element: HTMLElement, + now = Date.now() +): FirstImpressionSlotClaim | undefined { + const state = pruneFirstImpressionState(ts, now); + const existing = state.slots[element.id]; + if (existing && claimMatchesElement(existing, element, state.generation)) { + const canTransitionPublisherFallback = + existing.owner === 'publisher' && + existing.phase !== 'requested' && + existing.phase !== 'rendered' && + existing.expiresAt <= now && + state.fallbackSlots[element.id] === element; + if (!canTransitionPublisherFallback) return undefined; + + existing.owner = 'trusted_server'; + existing.phase = 'delivery_pending'; + existing.expiresAt = now + FIRST_IMPRESSION_LEASE_MS; + for (const auction of Object.values(existing.publisherAuctions)) { + auction.suppressDelivery = true; + } + return existing; + } + + const claim: FirstImpressionSlotClaim = { + generation: state.generation, + slotElementId: element.id, + element, + owner: 'trusted_server', + phase: 'delivery_pending', + expiresAt: now + FIRST_IMPRESSION_LEASE_MS, + publisherAuctions: {}, + }; + return storeClaim(state, claim) ? claim : undefined; +} + +function schedulePublisherAuctionExpiry(ts: TsjsApi, token: string): void { + window.setTimeout(() => { + // Pruning releases ordinary publisher claims. TS-owned suppression tokens + // deliberately survive as bounded tombstones until navigation/element change. + findPublisherAuction(ts, token); + }, FIRST_IMPRESSION_LEASE_MS); +} + +/** Release a TS claim when slot setup failed before any request could start. */ +export function releaseTrustedServerFirstImpressionClaim( + ts: TsjsApi, + element: HTMLElement, + claim: FirstImpressionSlotClaim +): void { + const state = pruneFirstImpressionState(ts); + if ( + state.slots[element.id] === claim && + claim.owner === 'trusted_server' && + claim.phase === 'delivery_pending' + ) { + delete state.slots[element.id]; + if (state.fallbackSlots[element.id] === element) { + delete state.fallbackSlots[element.id]; + } + } +} + +/** Register real publisher auctions before native `requestBids()` starts. */ +export function registerPublisherFirstImpressionAuctions( + ts: TsjsApi, + adUnitCodes: Iterable, + now = Date.now() +): Map { + const state = pruneFirstImpressionState(ts, now); + const registrations = new Map(); + + for (const adUnitCode of adUnitCodes) { + const element = resolveFirstImpressionElement(adUnitCode); + if (!element) continue; + + let claim = state.slots[element.id]; + if (!claim || !claimMatchesElement(claim, element, state.generation)) { + claim = { + generation: state.generation, + slotElementId: element.id, + element, + owner: 'publisher', + phase: 'auctioning', + expiresAt: now + FIRST_IMPRESSION_LEASE_MS, + publisherAuctions: {}, + }; + if (!storeClaim(state, claim)) continue; + } + + if ( + claim.owner === 'publisher' && + (claim.phase === 'requested' || claim.phase === 'rendered') + ) { + continue; + } + if ( + claim.owner === 'trusted_server' && + (claim.publisherRegistrationClosed || claim.expiresAt <= now) + ) { + continue; + } + if (Object.keys(claim.publisherAuctions).length >= MAX_PUBLISHER_AUCTIONS_PER_SLOT) continue; + + const token = `${state.generation}:${++state.nextToken}`; + const auction: FirstImpressionPublisherAuction = { + token, + adUnitCode, + expiresAt: now + FIRST_IMPRESSION_LEASE_MS, + suppressDelivery: claim.owner === 'trusted_server', + }; + claim.publisherAuctions[token] = auction; + if (claim.owner === 'publisher') claim.expiresAt = Math.max(claim.expiresAt, auction.expiresAt); + registrations.set(adUnitCode, token); + schedulePublisherAuctionExpiry(ts, token); + } + + return registrations; +} + +function findPublisherAuction( + ts: TsjsApi, + token: string, + now = Date.now() +): + | { + state: FirstImpressionState; + claim: FirstImpressionSlotClaim; + auction: FirstImpressionPublisherAuction; + } + | undefined { + const state = pruneFirstImpressionState(ts, now); + for (const claim of Object.values(state.slots)) { + const auction = claim.publisherAuctions[token]; + if (auction) return { state, claim, auction }; + } + return undefined; +} + +/** Release exactly one publisher auction token after failure, timeout, or removal. */ +export function releasePublisherFirstImpressionAuction( + ts: TsjsApi, + token: string, + now = Date.now() +): void { + const found = findPublisherAuction(ts, token, now); + if (!found) return; + if (found.claim.owner === 'trusted_server' && found.auction.suppressDelivery) { + found.claim.publisherRegistrationClosed = true; + return; + } + found.auction.expiresAt = Math.min(found.auction.expiresAt, now); + if ( + found.claim.owner === 'publisher' && + Object.keys(found.claim.publisherAuctions).length === 1 + ) { + found.claim.expiresAt = now; + } + removePublisherAuction(found.state, found.claim, token, now); +} + +/** Consume one correlated publisher delivery and report whether TS owns it. */ +export function consumePublisherFirstImpressionDelivery( + ts: TsjsApi, + token: string | undefined, + now = Date.now() +): boolean { + if (!token) return false; + const found = findPublisherAuction(ts, token, now); + if (!found) return false; + + const suppress = found.claim.owner === 'trusted_server' && found.auction.suppressDelivery; + delete found.claim.publisherAuctions[token]; + if (suppress) found.claim.publisherRegistrationClosed = true; + return suppress; +} + +/** Record a GPT request or render, using publisher ownership when no claimant exists. */ +export function observeFirstImpressionGptLifecycle( + ts: TsjsApi, + element: HTMLElement, + phase: Extract, + now = Date.now() +): void { + const state = pruneFirstImpressionState(ts, now); + let claim = state.slots[element.id]; + if (!claim || !claimMatchesElement(claim, element, state.generation)) { + claim = { + generation: state.generation, + slotElementId: element.id, + element, + owner: 'publisher', + phase, + expiresAt: Number.POSITIVE_INFINITY, + publisherAuctions: {}, + }; + storeClaim(state, claim); + return; + } + + claim.phase = phase; + if (claim.owner === 'publisher') { + claim.expiresAt = Number.POSITIVE_INFINITY; + } else { + // Once TS has committed a GPT request, only publisher auctions that were + // already registered can still represent an overlapping first impression. + // New publisher refreshes are ordinary later impressions and must proceed. + claim.publisherRegistrationClosed = true; + } +} + +/** Reserve the only Trusted Server fallback allowed for this physical slot and generation. */ +export function reservePublisherFirstImpressionFallback( + ts: TsjsApi, + element: HTMLElement +): boolean { + const state = pruneFirstImpressionState(ts); + const reservedElement = state.fallbackSlots[element.id]; + if (reservedElement) return false; + state.fallbackSlots[element.id] = element; + return true; +} + +/** Delay before an abandoned publisher claim can receive one per-slot TS fallback. */ +export function publisherFirstImpressionRetryDelay( + ts: TsjsApi, + element: HTMLElement, + now = Date.now() +): number | undefined { + const claim = firstImpressionClaim(ts, element); + if (!claim) return 0; + if (claim.owner !== 'publisher') return undefined; + if (claim.phase === 'requested' || claim.phase === 'rendered') return undefined; + return Math.max(0, claim.expiresAt - now); +} diff --git a/crates/trusted-server-js/lib/src/core/slot_element.ts b/crates/trusted-server-js/lib/src/core/slot_element.ts new file mode 100644 index 000000000..25d58bf1b --- /dev/null +++ b/crates/trusted-server-js/lib/src/core/slot_element.ts @@ -0,0 +1,87 @@ +/** Result of resolving one configured slot div ID against the live DOM. */ +export interface SlotElementResolution { + element: HTMLElement | null; + prefixMatchCount: number; + activeMatchCount: number; +} + +function isElementVisible(element: HTMLElement): boolean { + const elementWithVisibilityCheck = element as HTMLElement & { + checkVisibility?: (options?: { + checkVisibilityCSS?: boolean; + visibilityProperty?: boolean; + }) => boolean; + }; + if (typeof elementWithVisibilityCheck.checkVisibility === 'function') { + return elementWithVisibilityCheck.checkVisibility({ + checkVisibilityCSS: true, + visibilityProperty: true, + }); + } + + for (let current: HTMLElement | null = element; current; current = current.parentElement) { + const style = window.getComputedStyle(current); + if ( + style.display === 'none' || + style.visibility === 'hidden' || + style.visibility === 'collapse' + ) { + return false; + } + } + return true; +} + +function slotElementHasLayout(element: HTMLElement): boolean { + if (!isElementVisible(element)) return false; + const elementRect = element.getBoundingClientRect(); + if (elementRect.width > 0 && elementRect.height > 0) return true; + + const container = document.getElementById(`${element.id}-container`); + if (!container || !isElementVisible(container)) return false; + const containerRect = container.getBoundingClientRect(); + return containerRect.width > 0; +} + +/** Resolve an exact ID or one unambiguous visible/layout prefix match. */ +export function resolveSlotElementByDivId(divId: string): SlotElementResolution { + if (!divId) { + return { element: null, prefixMatchCount: 0, activeMatchCount: 0 }; + } + + // An exact ID is already unambiguous, so hidden exact elements still resolve. + // Prefixes may match several elements and require visibility and layout tiers + // before one candidate can be trusted. + const exact = document.getElementById(divId); + if (exact) { + return { element: exact, prefixMatchCount: 1, activeMatchCount: 1 }; + } + + const prefixMatches = Array.from(document.querySelectorAll('[id]')).filter( + (element) => element.id.startsWith(divId) && !element.id.endsWith('-container') + ); + // A unique lazy slot may not have layout yet, but its ancestors must be visible. + if (prefixMatches.length === 1 && isElementVisible(prefixMatches[0]!)) { + return { + element: prefixMatches[0]!, + prefixMatchCount: 1, + activeMatchCount: 1, + }; + } + + const visibleMatches = prefixMatches.filter(isElementVisible); + if (visibleMatches.length === 1) { + return { + element: visibleMatches[0]!, + prefixMatchCount: prefixMatches.length, + activeMatchCount: 1, + }; + } + + const activeMatches = visibleMatches.filter(slotElementHasLayout); + return { + element: activeMatches.length === 1 ? activeMatches[0]! : null, + prefixMatchCount: prefixMatches.length, + activeMatchCount: activeMatches.length, + }; +} diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 03ff0aca2..5602ab39e 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -87,10 +87,6 @@ export interface AuctionBidData { hb_cache_path?: string; /** Opaque server-auction correlation ID used only by GPT diagnostics. */ hb_auction_id?: string; - /** Winning creative width; the bridge sizes the inline render from this. */ - w?: number; - /** Winning creative height; the bridge sizes the inline render from this. */ - h?: number; nurl?: string; burl?: string; /** Typed winning-bid renderer capability. */ @@ -108,6 +104,43 @@ export interface AuctionBidData { debug_bid?: AuctionDebugBidData; } +/** Server-measured auction timings relative to their documented server-side origin. */ +export interface AuctionDiagnosticsData { + auctionDispatchedMs?: number; + auctionResolvedMs?: number; + auctionCommittedMs?: number; + auctionWaitMs?: number; + auctionWaitPlacement?: 'pre_header' | 'in_stream'; +} + +/** Auction path presented by GPT diagnostics. */ +export type GptDiagnosticsAuctionType = 'ssat' | 'trusted_server' | 'client_side' | 'competing'; + +/** Clock origin for server auction timings, independent of aggregate auction classification. */ +export type GptDiagnosticsServerAuctionTimingOrigin = 'navigation' | 'spa_auction'; + +/** Sanitized bid facts already exposed as bucketed ad-server targeting. */ +export interface GptDiagnosticsAuctionWinner { + bidder: string; + priceBucket: string; + /** ISO currency supplied by the evidence source; absent means not supplied. */ + currency?: string; +} + +/** A completed, exactly correlated client-side Prebid auction. */ +export interface GptDiagnosticsPrebidAuctionEvidence { + auctionId: string; + targetingCandidate?: GptDiagnosticsAuctionWinner; + win?: GptDiagnosticsAuctionWinner; +} + +/** Internal Trusted Server auction evidence attached to the next GPT request. */ +export interface GptDiagnosticsAuctionFacts { + auctionType?: Extract; + winner?: GptDiagnosticsAuctionWinner; + serverTimings?: AuctionDiagnosticsData; +} + export type GptDiagnosticsCallbackKind = | 'slotRequested' | 'slotResponseReceived' @@ -181,12 +214,37 @@ export type GptDiagnosticsTrustedServerOpportunity = | 'unrenderable_candidate' | 'no_candidate'; -/** A safe failure category observed while obtaining or posting creative markup. */ +/** + * A safe failure category observed while obtaining or posting creative markup. + * + * The `aps_` members cover the APS Universal Creative render path, where a + * blank slot is otherwise indistinguishable from a filled one: Ad Manager + * reports a non-empty 1x1 render whether or not the creative ever drew. Each + * member names the exact guard that stopped the render. + */ export type GptDiagnosticsCreativeFailure = | 'missing_render_source' | 'cache_fetch_failed' | 'invalid_cache_payload' - | 'response_post_failed'; + | 'response_post_failed' + // Reported by the sandboxed renderer document and relayed by the creative. + | 'aps_bad_hash' + | 'aps_nonce_mismatch' + | 'aps_source_mismatch' + | 'aps_descriptor_keys' + | 'aps_descriptor_fields' + | 'aps_descriptor_envelope' + | 'aps_runner_script_error' + // Observed by the Universal Creative source around its renderer frame. + | 'aps_frame_timeout' + | 'aps_frame_load_error' + | 'aps_frame_reported_failure' + | 'aps_unknown' + // Observed on the Trusted Server side of the capability handshake. + | 'aps_consumed_tombstone' + | 'aps_source_not_in_ad_unit' + | 'aps_missing_renderer_url' + | 'aps_tombstone_capacity'; /** Delivery evidence derived for a GPT request cycle. */ export type GptDiagnosticsDelivery = @@ -224,6 +282,14 @@ export interface GptDiagnosticsRequestCycle { requestPath?: GptDiagnosticsRequestPath; requestIntentId?: number; trustedServerAuctionId?: string; + auctionType?: GptDiagnosticsAuctionType; + /** Compatibility field: winner of the observed server auction, not necessarily the served creative. */ + auctionWinner?: GptDiagnosticsAuctionWinner; + /** Completed Prebid facts, correlated to this exact slot, request, and auction attempt. */ + prebidAuction?: GptDiagnosticsPrebidAuctionEvidence; + serverAuctionTimings?: AuctionDiagnosticsData; + /** Retained separately because `auctionType` can become `competing`. */ + serverAuctionTimingOrigin?: GptDiagnosticsServerAuctionTimingOrigin; opportunityToRequestMs?: number; replacedRequestNumber?: number; previousRenderToRequestMs?: number; @@ -332,10 +398,23 @@ export interface GptDiagnosticsRecorder { auctionSlotId: string, opportunity: GptDiagnosticsTrustedServerOpportunity, trustedServerAuctionId?: string, - requestedSlotSizes?: ReadonlyArray + requestedSlotSizes?: ReadonlyArray, + auctionFacts?: GptDiagnosticsAuctionFacts ): void; /** Mark slots whose next observed GPT request follows the Prebid refresh path. */ recordPrebidRefresh(slots: GptDiagnosticsSlotHandle[]): void; + /** Record a completed Prebid attempt at its targeting boundary for one exact GPT slot. */ + recordPrebidAuction( + slot: GptDiagnosticsSlotHandle, + auctionId: string, + targetingCandidate?: GptDiagnosticsAuctionWinner + ): void; + /** Record Prebid's documented `bidWon` observation for that exact attempt. */ + recordPrebidWin( + slot: GptDiagnosticsSlotHandle, + auctionId: string, + winner: GptDiagnosticsAuctionWinner + ): void; /** Record a creative markup request and return its opaque attempt ID. */ recordTrustedServerCreativeRequest(auctionSlotId: string): number | undefined; /** Record that a creative attempt successfully posted markup. */ @@ -365,6 +444,39 @@ export interface GptSlotHandoff { suppressPublisherRefresh: boolean; } +export type FirstImpressionOwner = 'publisher' | 'trusted_server'; +export type FirstImpressionPhase = 'auctioning' | 'delivery_pending' | 'requested' | 'rendered'; + +/** One publisher auction participating in the current navigation's first impression. */ +export interface FirstImpressionPublisherAuction { + token: string; + adUnitCode: string; + expiresAt: number; + suppressDelivery: boolean; +} + +/** First-impression ownership for one exact physical slot element. */ +export interface FirstImpressionSlotClaim { + generation: number; + slotElementId: string; + element: HTMLElement; + owner: FirstImpressionOwner; + phase: FirstImpressionPhase; + expiresAt: number; + publisherAuctions: Record; + /** No later publisher auction may join this TS-owned first impression. */ + publisherRegistrationClosed?: boolean; + targeting?: Record; +} + +/** Bounded first-impression state shared by the GPT bootstrap, GPT, and Prebid bundles. */ +export interface FirstImpressionState { + generation: number; + nextToken: number; + slots: Record; + fallbackSlots: Record; +} + export interface TsjsApi { version: string; que: Array<() => void>; @@ -392,6 +504,8 @@ export interface TsjsApi { adSlots?: AuctionSlot[]; /** Winning bid targeting data injected before . */ bids?: Record; + /** Server-measured timing evidence for the auction that populated `bids`. */ + auctionDiagnostics?: AuctionDiagnosticsData; /** * Bounded client-side Prebid APS renderer capabilities keyed by Prebid's generated * `hb_adid`. The Universal Creative bridge consumes each entry at most once. @@ -436,6 +550,10 @@ export interface TsjsApi { gptSlotHandoffs?: Record; /** True only while TS calls a GPT function that the handoff wrappers observe. */ gptSlotHandoffInternal?: boolean; + /** Per-navigation first-impression ownership shared by GPT and Prebid. */ + firstImpression?: FirstImpressionState; + /** Guards the shared production GPT lifecycle listener installation. */ + firstImpressionListenersInstalled?: boolean; /** Guards SPA pushState hook installation. */ spaHookInstalled?: boolean; /** Internal one-shot state shared by bootstrap and bundle scheduler installs. */ @@ -473,7 +591,8 @@ export interface TsjsApi { */ scheduleInitialAdInit?: ( initialBids?: Record, - initialSlots?: AuctionSlot[] + initialSlots?: AuctionSlot[], + initialAuctionDiagnostics?: AuctionDiagnosticsData ) => void; /** Read-only GPT lifecycle diagnostics API, present only in an activated tab. */ gptDiagnostics?: GptDiagnosticsApi; diff --git a/crates/trusted-server-js/lib/src/integrations/aps/render.ts b/crates/trusted-server-js/lib/src/integrations/aps/render.ts index adec0b036..cd1b88bd1 100644 --- a/crates/trusted-server-js/lib/src/integrations/aps/render.ts +++ b/crates/trusted-server-js/lib/src/integrations/aps/render.ts @@ -1,6 +1,11 @@ import { log } from '../../core/log'; import { findSlot } from '../../core/render'; -import type { ApsPrebidRendererEntry, ApsRendererV1, TsjsApi } from '../../core/types'; +import type { + ApsPrebidRendererEntry, + ApsRendererV1, + GptDiagnosticsCreativeFailure, + TsjsApi, +} from '../../core/types'; export const APS_RENDERER_PATH = '/integrations/aps/renderer'; export const APS_RENDERING_MODE_ATTRIBUTE_NAME = 'data-ts-aps-rendering-mode'; @@ -32,6 +37,58 @@ const activeFrames = new WeakMap(); const pendingFrameCancels = new WeakMap void>(); const RENDERER_READY_MESSAGE = 'trusted-server/aps/renderer-ready'; const RENDERER_FAILED_MESSAGE = 'trusted-server/aps/renderer-failed'; +/** + * Message the Universal Creative frame relays to the top window when an APS + * render never completes. + * + * The creative frame is cross-origin, so the top-window listener treats every + * field as untrusted and validates the reason against + * [`APS_RENDER_FAILURE_REASONS`] before recording it. The relay is + * diagnostics-only and never influences creative delivery. + */ +export const APS_RENDER_FAILED_MESSAGE = 'trusted-server/aps/render-failed'; + +/** + * Wire reasons the render path can emit, mapped onto safe diagnostic categories. + * + * Built on a null prototype so a hostile `__proto__`, `constructor`, or + * `toString` relayed by the cross-origin creative frame resolves to `undefined` + * rather than an inherited member. + */ +const APS_RENDER_FAILURE_REASONS: Readonly> = + Object.freeze( + Object.assign( + Object.create(null) as Record, + { + bad_hash: 'aps_bad_hash', + nonce_mismatch: 'aps_nonce_mismatch', + source_mismatch: 'aps_source_mismatch', + descriptor_keys: 'aps_descriptor_keys', + descriptor_fields: 'aps_descriptor_fields', + descriptor_envelope: 'aps_descriptor_envelope', + amazon_script_error: 'aps_runner_script_error', + frame_timeout: 'aps_frame_timeout', + frame_load_error: 'aps_frame_load_error', + frame_reported_failure: 'aps_frame_reported_failure', + unknown: 'aps_unknown', + } as const + ) + ); + +/** + * Resolve a relayed render failure reason to a safe diagnostic category. + * + * Returns `undefined` for anything not on the allowlist, so an unrecognized or + * hostile value from the cross-origin creative frame is dropped instead of + * being recorded. + * + * @example + * apsRenderFailureReason('frame_timeout'); // 'aps_frame_timeout' + * apsRenderFailureReason('__proto__'); // undefined + */ +export function apsRenderFailureReason(value: unknown): GptDiagnosticsCreativeFailure | undefined { + return typeof value === 'string' ? APS_RENDER_FAILURE_REASONS[value] : undefined; +} const RENDERER_READY_TIMEOUT_MS = 10_000; const MAX_PREBID_RENDERER_ENTRIES = 256; const DEFAULT_PREBID_RENDERER_TTL_SECONDS = 300; @@ -70,8 +127,7 @@ function sourceMatchedCandidates( source?: MessageEventSource | null ): HTMLElement[] { if (!source) return candidates; - const sourceMatches = candidates.filter((element) => sourceBelongsToElement(source, element)); - return sourceMatches.length > 0 ? sourceMatches : candidates; + return candidates.filter((element) => sourceBelongsToElement(source, element)); } function dynamicSlotCandidates( @@ -103,23 +159,26 @@ function findApsContainer(slotId: string, source?: MessageEventSource | null): H if (slotId.endsWith('-container')) { const inner = findSlot(slotId.slice(0, -'-container'.length)); - if (inner) return inner; + if (inner) return source && !sourceBelongsToElement(source, inner) ? null : inner; } const direct = findSlot(slotId); - if (direct && !direct.id.endsWith('-container')) return direct; + if (direct && !direct.id.endsWith('-container')) { + return source && !sourceBelongsToElement(source, direct) ? null : direct; + } const configuredDivId = window.tsjs?.adSlots?.find((slot) => slot.id === slotId)?.div_id; if (configuredDivId) { const configured = findSlot(configuredDivId); - if (configured) return configured; + if (configured) { + return source && !sourceBelongsToElement(source, configured) ? null : configured; + } const dynamic = uniqueSlotCandidate(dynamicSlotCandidates(configuredDivId, source)); if (dynamic) return dynamic; } - const dynamic = uniqueSlotCandidate(dynamicSlotCandidates(slotId, source)); - return dynamic ?? direct; + return uniqueSlotCandidate(dynamicSlotCandidates(slotId, source)); } catch { return null; } @@ -711,12 +770,13 @@ var b=new Uint8Array(16);c.getRandomValues(b);var s="";for(var i=0;i 0; + const winner = + isNonEmptyString(bid.hb_bidder) && isNonEmptyString(bid.hb_pb) + ? { bidder: bid.hb_bidder, priceBucket: bid.hb_pb } + : undefined; + if (!winner && auctionDiagnostics === undefined) return undefined; + + return { + auctionType: isSpaAuction ? 'trusted_server' : 'ssat', + ...(winner ? { winner } : {}), + ...(auctionDiagnostics ? { serverTimings: auctionDiagnostics } : {}), + }; +} + // ------------------------------------------------------------------ // googletag type stubs (minimal surface needed by the shim) // ------------------------------------------------------------------ @@ -83,156 +115,221 @@ interface SlotRenderEndedEvent { slot: GoogleTagSlot; } -interface SlotElementResolution { - element: HTMLElement | null; - prefixMatchCount: number; - activeMatchCount: number; +function findSlotElementByDivId(divId: string): HTMLElement | null { + return resolveSlotElementByDivId(divId).element; } -function isElementVisible(element: HTMLElement): boolean { - const elementWithVisibilityCheck = element as HTMLElement & { - checkVisibility?: (options?: { - checkVisibilityCSS?: boolean; - visibilityProperty?: boolean; - }) => boolean; - }; - if (typeof elementWithVisibilityCheck.checkVisibility === 'function') { - return elementWithVisibilityCheck.checkVisibility({ - checkVisibilityCSS: true, - visibilityProperty: true, - }); +function candidateSlotRoots(elementId: string): HTMLElement[] { + const roots: HTMLElement[] = []; + const slotEl = document.getElementById(elementId); + if (slotEl) { + roots.push(slotEl); } - for (let current: HTMLElement | null = element; current; current = current.parentElement) { - const style = window.getComputedStyle(current); - if ( - style.display === 'none' || - style.visibility === 'hidden' || - style.visibility === 'collapse' - ) { - return false; - } + const container = document.getElementById(`${elementId}-container`); + if (container && !roots.includes(container)) { + roots.push(container); } - return true; -} -function slotElementHasLayout(element: HTMLElement): boolean { - if (!isElementVisible(element)) return false; - const elementRect = element.getBoundingClientRect(); - if (elementRect.width > 0 && elementRect.height > 0) return true; + return roots; +} - const container = document.getElementById(`${element.id}-container`); - if (!container || !isElementVisible(container)) return false; - const containerRect = container.getBoundingClientRect(); - return containerRect.width > 0; +interface MessageSourceFrame { + iframe: HTMLIFrameElement; + root: HTMLElement; } -function resolveSlotElementByDivId(divId: string): SlotElementResolution { - if (!divId) { - return { element: null, prefixMatchCount: 0, activeMatchCount: 0 }; - } - // Exact-id matches intentionally skip the visibility tiers below: a - // configured literal id is unambiguous, so a hidden match is still the - // right element (adInit defines the slot; GPT simply renders nothing while - // it is hidden). Prefix matches go through the tiers because a prefix can - // match several candidates and only visibility/layout disambiguates them — - // so a hidden exact-id match resolves while a hidden prefix match does not. - const exact = document.getElementById(divId); - if (exact) { - return { element: exact, prefixMatchCount: 1, activeMatchCount: 1 }; +function sourceFrameInRoots( + source: MessageEventSource | null, + roots: readonly HTMLElement[] +): MessageSourceFrame | undefined { + if (!source) return undefined; + const matches = new Map(); + for (const root of roots) { + for (const iframe of root.querySelectorAll('iframe')) { + if (iframe.contentWindow === source && !matches.has(iframe)) matches.set(iframe, root); + } } + if (matches.size !== 1) return undefined; + const [iframe, root] = matches.entries().next().value as [HTMLIFrameElement, HTMLElement]; + return { iframe, root }; +} - const prefixMatches = Array.from(document.querySelectorAll('[id]')).filter( - (element) => element.id.startsWith(divId) && !element.id.endsWith('-container') - ); - // A unique prefix match may be a lazy slot that has not been sized yet, but - // it must still be visible through its ancestor containers. - if (prefixMatches.length === 1 && isElementVisible(prefixMatches[0]!)) { - return { - element: prefixMatches[0]!, - prefixMatchCount: 1, - activeMatchCount: 1, - }; - } +function sourceFrameForConfiguredDivId( + source: MessageEventSource | null, + divId: string +): MessageSourceFrame | undefined { + const exact = document.getElementById(divId); + const candidates = exact + ? [exact] + : Array.from(document.querySelectorAll('[id]')).filter( + (element) => element.id.startsWith(divId) && !element.id.endsWith('-container') + ); + const matches = candidates + .map((element) => sourceFrameInRoots(source, candidateSlotRoots(element.id))) + .filter((frame): frame is MessageSourceFrame => frame !== undefined); + return matches.length === 1 ? matches[0] : undefined; +} - const visibleMatches = prefixMatches.filter(isElementVisible); - if (visibleMatches.length === 1) { - return { - element: visibleMatches[0]!, - prefixMatchCount: prefixMatches.length, - activeMatchCount: 1, - }; +function uniqueSourceFrame( + frames: Array +): MessageSourceFrame | undefined { + const matches = new Map(); + for (const frame of frames) { + if (frame) matches.set(frame.iframe, frame); } + return matches.size === 1 ? matches.values().next().value : undefined; +} - const activeMatches = visibleMatches.filter(slotElementHasLayout); - return { - element: activeMatches.length === 1 ? activeMatches[0]! : null, - prefixMatchCount: prefixMatches.length, - activeMatchCount: activeMatches.length, - }; +function sourceFrameForSlotId( + source: MessageEventSource | null, + slotId: string +): MessageSourceFrame | undefined { + const mappedFrames = Object.entries(window.tsjs?.divToSlotId ?? {}) + .filter(([, mappedSlotId]) => mappedSlotId === slotId) + .map(([elementId]) => sourceFrameInRoots(source, candidateSlotRoots(elementId))); + const configuredFrames = (window.tsjs?.adSlots ?? []) + .filter((slot) => slot.id === slotId) + .map((slot) => sourceFrameForConfiguredDivId(source, slot.div_id)); + return uniqueSourceFrame([...mappedFrames, ...configuredFrames]); } -function findSlotElementByDivId(divId: string): HTMLElement | null { - return resolveSlotElementByDivId(divId).element; +interface MessageSourceSlotFrame extends MessageSourceFrame { + slotId: string; } -function candidateSlotRoots(elementId: string): HTMLElement[] { - const roots: HTMLElement[] = []; - const slotEl = document.getElementById(elementId); - if (slotEl) { - roots.push(slotEl); +function slotFrameForMessageSource( + source: MessageEventSource | null +): MessageSourceSlotFrame | undefined { + const slotIds = new Set(); + for (const [elementId, slotId] of Object.entries(window.tsjs?.divToSlotId ?? {})) { + if (sourceFrameInRoots(source, candidateSlotRoots(elementId))) slotIds.add(slotId); } - - const container = document.getElementById(`${elementId}-container`); - if (container && !roots.includes(container)) { - roots.push(container); + for (const slot of window.tsjs?.adSlots ?? []) { + if (sourceFrameForConfiguredDivId(source, slot.div_id)) slotIds.add(slot.id); } + if (slotIds.size !== 1) return undefined; + const slotId = slotIds.values().next().value as string; + const frame = sourceFrameForSlotId(source, slotId); + return frame ? { ...frame, slotId } : undefined; +} - return roots; +function sourceFrameForAdUnit( + source: MessageEventSource | null, + adUnitCode: string +): MessageSourceFrame | undefined { + return sourceFrameForConfiguredDivId(source, adUnitCode); } -function candidateSlotRootsForConfiguredDivId(divId: string): HTMLElement[] { - const roots = candidateSlotRoots(divId); - const dynamicElements = Array.from(document.querySelectorAll('[id]')).filter( - (element) => element.id.startsWith(divId) && !element.id.endsWith('-container') - ); - for (const element of dynamicElements) { - if (!roots.includes(element)) roots.push(element); - const container = document.getElementById(`${element.id}-container`); - if (container && !roots.includes(container)) roots.push(container); - } - return roots; +function hasCollapsedDimension(element: HTMLElement, dimension: 'width' | 'height'): boolean { + const value = window.getComputedStyle(element)[dimension]; + const match = /^(\d+(?:\.\d+)?)px$/.exec(value); + return match !== null && Number(match[1]) <= 1; } -function sourceIsInSlotRoots(source: MessageEventSource, roots: HTMLElement[]): boolean { - return roots.some((root) => - Array.from(root.querySelectorAll('iframe')).some((iframe) => iframe.contentWindow === source) - ); +function usesFixedPositioning(element: HTMLElement): boolean { + const position = window.getComputedStyle(element).position; + return position === 'fixed' || position === 'sticky'; } -function slotIdForMessageSource(source: MessageEventSource | null): string | undefined { - if (!source) return undefined; +const MAX_CREATIVE_SHELL_DIMENSION = 10_000; + +function creativeFrameIsCurrent( + source: MessageEventSource | null, + frame: MessageSourceFrame, + generation: number, + stillOwnsCreative: () => boolean +): boolean { + return ( + (window.tsjs?.navGeneration ?? 0) === generation && + stillOwnsCreative() && + frame.iframe.isConnected && + frame.root.isConnected && + frame.root.contains(frame.iframe) && + frame.iframe.contentWindow === source + ); +} - const divToSlotId = window.tsjs?.divToSlotId ?? {}; - const resolvedSlotId = Object.entries(divToSlotId).find(([elementId]) => - sourceIsInSlotRoots(source, candidateSlotRoots(elementId)) - )?.[1]; - if (resolvedSlotId) return resolvedSlotId; +/** Resize the authenticated source iframe and collapsed ancestors through its slot root. */ +function resizeCollapsedCreativeFrame( + source: MessageEventSource | null, + frame: MessageSourceFrame, + width: number, + height: number, + generation: number, + stillOwnsCreative: () => boolean +): void { + if ( + !creativeFrameIsCurrent(source, frame, generation, stillOwnsCreative) || + !Number.isFinite(width) || + !Number.isFinite(height) || + width <= 0 || + height <= 0 || + width > MAX_CREATIVE_SHELL_DIMENSION || + height > MAX_CREATIVE_SHELL_DIMENSION || + frame.iframe.getAttribute('width') !== '1' || + frame.iframe.getAttribute('height') !== '1' || + !hasCollapsedDimension(frame.iframe, 'width') || + !hasCollapsedDimension(frame.iframe, 'height') || + usesFixedPositioning(frame.iframe) || + frame.iframe.closest( + 'ins[data-anchor-status], [data-google-interstitial], [data-vignette-loaded]' + ) + ) { + return; + } - const slots = window.tsjs?.adSlots ?? []; - return [...slots] - .sort((left, right) => right.div_id.length - left.div_id.length) - .find((slot) => sourceIsInSlotRoots(source, candidateSlotRootsForConfiguredDivId(slot.div_id))) - ?.id; + const collapsedAncestors: HTMLElement[] = []; + let reachedRoot = false; + for (let ancestor = frame.iframe.parentElement; ancestor; ancestor = ancestor.parentElement) { + if ( + ancestor === document.body || + ancestor === document.documentElement || + !ancestor.isConnected || + usesFixedPositioning(ancestor) || + ancestor.matches( + 'ins[data-anchor-status], [data-google-interstitial], [data-vignette-loaded]' + ) + ) { + return; + } + if (hasCollapsedDimension(ancestor, 'width') || hasCollapsedDimension(ancestor, 'height')) { + collapsedAncestors.push(ancestor); + } + if (ancestor === frame.root) { + reachedRoot = true; + break; + } + } + if (!reachedRoot) return; + + frame.iframe.width = String(width); + frame.iframe.height = String(height); + frame.iframe.style.width = `${width}px`; + frame.iframe.style.height = `${height}px`; + for (const ancestor of collapsedAncestors) { + if (hasCollapsedDimension(ancestor, 'width')) ancestor.style.width = `${width}px`; + if (hasCollapsedDimension(ancestor, 'height')) ancestor.style.height = `${height}px`; + } } -function messageSourceBelongsToAdUnit( +function safelyResizeCollapsedCreativeFrame( source: MessageEventSource | null, - adUnitCode: string -): boolean { - return source - ? sourceIsInSlotRoots(source, candidateSlotRootsForConfiguredDivId(adUnitCode)) - : false; + frame: MessageSourceFrame, + width: number, + height: number, + generation: number, + stillOwnsCreative: () => boolean +): void { + try { + resizeCollapsedCreativeFrame(source, frame, width, height, generation, stillOwnsCreative); + } catch (err) { + try { + log.warn(`[tsjs-gpt] creative shell resize failed for '${frame.root.id}'`, err); + } catch { + // Resize and logging failures must not replace successful delivery evidence. + } + } } function clearTargetingKeys(slot: GoogleTagSlot, keys: Iterable): void { @@ -712,12 +809,16 @@ function installInitialLoadDetector(ts: TsjsApi): void { function installScheduleInitialAdInit(ts: TsjsApi): void { ts.scheduleInitialAdInit = function ( initialBids?: Record, - initialSlots?: AuctionSlot[] + initialSlots?: AuctionSlot[], + initialAuctionDiagnostics?: AuctionDiagnosticsData ) { if ((ts.navGeneration ?? 0) !== 0 || ts.initialAdInitScheduled) return; ts.initialAdInitScheduled = true; if (initialSlots !== undefined) ts.adSlots = initialSlots; if (initialBids !== undefined) ts.bids = initialBids; + if (initialAuctionDiagnostics !== undefined) { + ts.auctionDiagnostics = initialAuctionDiagnostics; + } const runUnlessNavigated = (): void => { if ((ts.navGeneration ?? 0) !== 0) return; ts.adInit?.(); @@ -930,11 +1031,196 @@ function installLatePublisherSlotHandoff(ts: TsjsApi): void { }); } +function installFirstImpressionLifecycleObservers(ts: TsjsApi, g: Partial): void { + if (ts.firstImpressionListenersInstalled) return; + g.cmd?.push(() => { + if (ts.firstImpressionListenersInstalled) return; + const pubads = g.pubads?.(); + if (!pubads?.addEventListener) return; + + const observe = + (phase: 'requested' | 'rendered') => + (event: SlotRenderEndedEvent): void => { + const elementId = event.slot?.getSlotElementId?.(); + const element = elementId ? document.getElementById(elementId) : null; + if (element) observeFirstImpressionGptLifecycle(ts, element, phase); + }; + pubads.addEventListener('slotRequested', observe('requested')); + pubads.addEventListener('slotRenderEnded', observe('rendered')); + ts.firstImpressionListenersInstalled = true; + }); +} + +function trustedServerTargeting( + slot: AuctionSlot, + bid: AuctionBidData +): Record { + const targeting: Record = { ...(slot.targeting ?? {}) }; + for (const key of TS_BID_TARGETING_KEYS) { + if (bid[key]) targeting[key] = String(bid[key]); + } + targeting[TS_INITIAL_TARGETING_KEY] = '1'; + return targeting; +} + +function applyTrustedServerTargeting( + ts: TsjsApi, + gptSlot: GoogleTagSlot, + slot: AuctionSlot, + bid: AuctionBidData, + elementIds: readonly string[] +): string[] { + const previousKeys = ts.prevSlotTargetingKeys ?? {}; + clearTargetingKeys(gptSlot, [ + ...TS_BASE_TARGETING_KEYS, + ...elementIds.flatMap((elementId) => previousKeys[elementId] ?? []), + ]); + const targeting = trustedServerTargeting(slot, bid); + for (const [key, value] of Object.entries(targeting)) gptSlot.setTargeting(key, value); + const element = document.getElementById(elementIds[0]!); + const claim = element ? firstImpressionClaim(ts, element) : undefined; + if (claim?.owner === 'trusted_server') claim.targeting = targeting; + return Object.keys(slot.targeting ?? {}); +} + +function clearPreviousNavigationTargeting(ts: TsjsApi, g: Partial): void { + const previousKeys = ts.prevSlotTargetingKeys ?? {}; + const touchedElementIds = new Set([ + ...Object.keys(previousKeys), + ...Object.keys(ts.divToSlotId ?? {}), + ]); + + const pubads = g.pubads?.(); + if (pubads && touchedElementIds.size > 0) { + for (const slot of pubads.getSlots?.() ?? []) { + const elementId = slot.getSlotElementId(); + if (!touchedElementIds.has(elementId)) continue; + clearTargetingKeys(slot, [...TS_BASE_TARGETING_KEYS, ...(previousKeys[elementId] ?? [])]); + } + } + + ts.prevSlotTargetingKeys = {}; + ts.divToSlotId = {}; +} + +function schedulePublisherFirstImpressionFallback( + ts: TsjsApi, + g: Partial, + slot: AuctionSlot, + bid: AuctionBidData, + element: HTMLElement, + generation: number +): void { + if (!reservePublisherFirstImpressionFallback(ts, element)) return; + + const retry = (): void => { + if ( + (ts.navGeneration ?? 0) !== generation || + !element.isConnected || + document.getElementById(element.id) !== element + ) { + return; + } + const delay = publisherFirstImpressionRetryDelay(ts, element); + if (delay === undefined) return; + if (delay > 0) { + window.setTimeout(retry, delay + 1); + return; + } + + g.cmd?.push(() => { + if ( + (ts.navGeneration ?? 0) !== generation || + !element.isConnected || + document.getElementById(element.id) !== element + ) { + return; + } + const claim = claimFirstImpressionForTrustedServer(ts, element); + if (!claim) return; + + const pubads = g.pubads?.(); + if (!pubads) { + releaseTrustedServerFirstImpressionClaim(ts, element, claim); + return; + } + let gptSlot = pubads + .getSlots?.() + .find((candidate) => candidate.getSlotElementId() === element.id); + let tsOwned = false; + if (!gptSlot) { + gptSlot = + withGptSlotHandoffInternal(ts, () => + g.defineSlot?.(slot.gam_unit_path, slot.formats, element.id) + ) ?? undefined; + if (!gptSlot) { + releaseTrustedServerFirstImpressionClaim(ts, element, claim); + return; + } + gptSlot.addService(pubads); + tsOwned = true; + (ts.gptSlotHandoffs ??= {})[element.id] = { + gamUnitPath: slot.gam_unit_path, + formats: slot.formats, + divIdPrefix: slot.div_id, + slotElementId: element.id, + publisherClaimed: false, + suppressPublisherDisplay: false, + suppressPublisherRefresh: false, + }; + } + + const slotElementId = gptSlot.getSlotElementId?.() ?? element.id; + const targetingKeys = applyTrustedServerTargeting(ts, gptSlot, slot, bid, [ + element.id, + slotElementId, + ]); + (ts.divToSlotId ??= {})[element.id] = slot.id; + if (slotElementId !== element.id) ts.divToSlotId[slotElementId] = slot.id; + (ts.prevSlotTargetingKeys ??= {})[element.id] = targetingKeys; + if (slotElementId !== element.id) ts.prevSlotTargetingKeys[slotElementId] = targetingKeys; + if (tsOwned) (ts.prevGptSlots ??= []).push(gptSlot); + + try { + ts.gptDiagnosticsRecorder?.recordTrustedServerOpportunity( + gptSlot, + slot.id, + trustedServerOpportunity(bid), + bid.hb_auction_id, + slot.formats + ); + } catch { + // Diagnostics must not alter fallback delivery. + } + + if (!ts.servicesEnabled) { + pubads.enableSingleRequest(); + g.enableServices?.(); + ts.servicesEnabled = true; + } + if (tsOwned) withGptSlotHandoffInternal(ts, () => g.display?.(slotElementId)); + syncInitialLoadDisabled(g, ts); + if (!tsOwned || ts.gptInitialLoadDisabled) { + ts.adInitRefreshInProgress = true; + try { + withGptSlotHandoffInternal(ts, () => pubads.refresh([gptSlot!])); + } finally { + ts.adInitRefreshInProgress = false; + } + } + }); + }; + + retry(); +} + export function installTsAdInit(): void { const ts = (window.tsjs ??= {} as TsjsApi); installInitialLoadDetector(ts); installScheduleInitialAdInit(ts); + const g = (window as GptWindow).googletag; + if (g) installFirstImpressionLifecycleObservers(ts, g); installLatePublisherSlotHandoff(ts); ts.adInit = function () { const slots = ts.adSlots ?? []; @@ -949,8 +1235,10 @@ export function installTsAdInit(): void { // first act and stands down rather than applying this invocation's // slots/bids to the newer route's DOM and double-requesting it. const generation = ts.navGeneration ?? 0; + const auctionDiagnostics = ts.auctionDiagnostics ? { ...ts.auctionDiagnostics } : undefined; const g = (window as GptWindow).googletag; if (!g) return; + installFirstImpressionLifecycleObservers(ts, g); const warnedResolutionFailures = new Set(); g.cmd?.push(() => { @@ -1000,6 +1288,8 @@ export function installTsAdInit(): void { (g.pubads!().getSlots?.() ?? []).forEach((gptSlot: GoogleTagSlot) => { const elementId = gptSlot.getSlotElementId(); if (!prevTouchedDivIds.has(elementId)) return; + const element = document.getElementById(elementId); + if (element && firstImpressionClaim(ts, element)) return; clearTargetingKeys(gptSlot, [ ...TS_BASE_TARGETING_KEYS, ...(prevSlotTargetingKeys[elementId] ?? []), @@ -1037,6 +1327,14 @@ export function installTsAdInit(): void { } const actualDivId = el.id; const bid = bids[slot.id] ?? {}; + const firstImpression = claimFirstImpressionForTrustedServer(ts, el); + if (!firstImpression) { + const claim = firstImpressionClaim(ts, el); + if (claim?.owner === 'publisher') { + schedulePublisherFirstImpressionFallback(ts, g, slot, bid, el, generation); + } + return; + } const existingSlot = g.pubads!() .getSlots?.() @@ -1052,7 +1350,10 @@ export function installTsAdInit(): void { const defined = withGptSlotHandoffInternal(ts, () => g.defineSlot?.(slot.gam_unit_path, slot.formats, actualDivId) ); - if (!defined) return; + if (!defined) { + releaseTrustedServerFirstImpressionClaim(ts, el, firstImpression); + return; + } defined.addService(g.pubads!()); gptSlot = defined; tsOwned = true; @@ -1068,29 +1369,35 @@ export function installTsAdInit(): void { } const slotDivId2 = gptSlot.getSlotElementId?.() ?? actualDivId; - clearTargetingKeys(gptSlot, [ - ...TS_BASE_TARGETING_KEYS, - ...(prevSlotTargetingKeys[actualDivId] ?? []), - ...(prevSlotTargetingKeys[slotDivId2] ?? []), + const slotTargetingKeys = applyTrustedServerTargeting(ts, gptSlot, slot, bid, [ + actualDivId, + slotDivId2, ]); - - Object.entries(slot.targeting ?? {}).forEach(([k, v]) => gptSlot.setTargeting(k, v)); - TS_BID_TARGETING_KEYS.forEach((key) => { - if (bid[key]) gptSlot.setTargeting(key, String(bid[key]!)); - }); - gptSlot.setTargeting(TS_INITIAL_TARGETING_KEY, '1'); // Diagnostics are observational only. A missing or malformed debug // implementation must never interrupt slot mapping or delivery. try { const requestedSlotSizes = ts.gptSlotHandoffs?.[slotDivId2]?.formats; const opportunity = trustedServerOpportunity(bid); - ts.gptDiagnosticsRecorder?.recordTrustedServerOpportunity( - gptSlot, - slot.id, - opportunity, - bid.hb_auction_id, - requestedSlotSizes - ); + const auctionFacts = diagnosticsAuctionFacts(generation, auctionDiagnostics, bid); + const recorder = ts.gptDiagnosticsRecorder; + if (auctionFacts) { + recorder?.recordTrustedServerOpportunity( + gptSlot, + slot.id, + opportunity, + bid.hb_auction_id, + requestedSlotSizes, + auctionFacts + ); + } else { + recorder?.recordTrustedServerOpportunity( + gptSlot, + slot.id, + opportunity, + bid.hb_auction_id, + requestedSlotSizes + ); + } } catch { // Diagnostics must not alter ad delivery. } @@ -1098,7 +1405,6 @@ export function installTsAdInit(): void { // injection address the same, single GPT slot. divToSlotId[actualDivId] = slot.id; if (slotDivId2 !== actualDivId) divToSlotId[slotDivId2] = slot.id; - const slotTargetingKeys = Object.keys(slot.targeting ?? {}); nextSlotTargetingKeys[actualDivId] = slotTargetingKeys; if (slotDivId2 !== actualDivId) nextSlotTargetingKeys[slotDivId2] = slotTargetingKeys; if (tsOwned) { @@ -1193,6 +1499,7 @@ export function installTsAdInit(): void { interface PageBidsResponse { slots: AuctionSlot[]; bids: Record; + auctionDiagnostics?: AuctionDiagnosticsData; } /** Canonical SPA re-auction endpoint. Mirrors `PAGE_BIDS_PATH` in Rust. */ @@ -1397,7 +1704,10 @@ export function installSpaAuctionHook(): void { async function onNavigate(path: string): Promise { if (path === currentPath) return; currentPath = path; + const g = (window as GptWindow).googletag; + if (g) clearPreviousNavigationTargeting(ts, g); ts.navGeneration = (ts.navGeneration ?? 0) + 1; + delete ts.firstImpression; // A route change invalidates hydration aliases before the new route's // publisher can define a same-prefix slot while page-bids is in flight. for (const [elementId, handoff] of Object.entries(ts.gptSlotHandoffs ?? {})) { @@ -1424,6 +1734,7 @@ export function installSpaAuctionHook(): void { if (inflight !== controller) return; ts.adSlots = data.slots; ts.bids = data.bids; + ts.auctionDiagnostics = data.auctionDiagnostics; // This route is now the committed, loaded state — a later failed // navigation rolls back here, and a return trip no-ops correctly. lastAppliedPath = path; @@ -1459,9 +1770,13 @@ export function installSpaAuctionHook(): void { patchHistoryMethod('pushState'); patchHistoryMethod('replaceState'); - window.addEventListener('popstate', () => { - void onNavigate(location.pathname); - }); + window.addEventListener( + 'popstate', + () => { + void onNavigate(location.pathname); + }, + true + ); } /** @@ -1597,11 +1912,49 @@ function safelyRecordCreativeFailure( } } +/** + * Open a diagnostics attempt for an APS capability handshake. + * + * The APS path runs on the publisher's own Prebid ad units, which never pass + * through Trusted Server slot mapping, so no creative opportunity has been + * recorded for them. Without one the store rejects the attempt as + * `creative_request_without_slot` and the request cycle stays `unknown`, + * leaving a blank APS render indistinguishable from a delivered one. + */ +function beginApsCreativeAttempt(adUnitCode: string): number | undefined { + try { + const pubads = window.googletag?.pubads?.(); + const slot = pubads ? findGptSlotByElementId(pubads, adUnitCode) : undefined; + if (slot) { + window.tsjs?.gptDiagnosticsRecorder?.recordTrustedServerOpportunity( + slot, + adUnitCode, + 'renderable_candidate' + ); + } + } catch { + // Diagnostics must not alter creative delivery. + } + return safelyRecordCreativeRequest(adUnitCode); +} + +/** + * A consumed APS ad ID, retained as a security tombstone. + * + * `attemptId` carries the diagnostics attempt the capability was served under so + * a later replay, or a failure relayed by the creative frame, is attributed to + * the render it belongs to. + */ +interface ApsConsumedTombstone { + expiresAt: number; + attemptId?: number; +} + /** Maximum number of consumed APS Prebid IDs retained as security tombstones. */ const MAX_CONSUMED_PREBID_APS_IDS = 256; function pruneConsumedPrebidApsIds( - consumedIds: Map, + consumedIds: Map, now: number ): void { for (const [adId, consumed] of consumedIds) { @@ -1610,7 +1963,7 @@ function pruneConsumedPrebidApsIds( } function hasConsumedPrebidApsIdCapacity( - consumedIds: Map, + consumedIds: Map, adId: string ): boolean { if (consumedIds.has(adId) || consumedIds.size < MAX_CONSUMED_PREBID_APS_IDS) return true; @@ -1620,11 +1973,12 @@ function hasConsumedPrebidApsIdCapacity( } function recordConsumedPrebidApsId( - consumedIds: Map, + consumedIds: Map, adId: string, - expiresAt: number + expiresAt: number, + attemptId: number | undefined ): void { - consumedIds.set(adId, { expiresAt }); + consumedIds.set(adId, { expiresAt, attemptId }); } /** @@ -1658,7 +2012,7 @@ export function installTsRenderBridge(): void { // is scoped to the slot, not the bare adId: hb_adid is not unique per bid, so // keying on it alone would let one slot block a distinct slot's render. const renderingKeys = new Set(); - const consumedPrebidApsIds = new Map(); + const consumedPrebidApsIds = new Map(); // One consumed APS ad ID per slot is sufficient: a newer bid replaces the // slot's old ad ID in `window.tsjs.bids`, so the ownership guard rejects it. const consumedServerApsBySlot = new Map(); @@ -1674,6 +2028,20 @@ export function installTsRenderBridge(): void { return; } + // Diagnostics relayed by the APS Universal Creative frame. The creative is + // cross-origin, so every field is untrusted: the reason must resolve through + // the allowlist and the attempt comes from our own tombstone, never the + // message. Recording only, and it never answers the sender. + if (data['message'] === APS_RENDER_FAILED_MESSAGE) { + const failedAdId = data['adId']; + const reason = apsRenderFailureReason(data['reason']); + if (typeof failedAdId === 'string' && reason !== undefined) { + pruneConsumedPrebidApsIds(consumedPrebidApsIds, Date.now()); + safelyRecordCreativeFailure(consumedPrebidApsIds.get(failedAdId)?.attemptId, reason); + } + return; + } + if (data['message'] !== 'Prebid Request') return; const adId = data['adId'] as string | undefined; if (!adId) return; @@ -1682,6 +2050,7 @@ export function installTsRenderBridge(): void { if (!port) return; const now = Date.now(); + const generation = window.tsjs?.navGeneration ?? 0; pruneConsumedPrebidApsIds(consumedPrebidApsIds, now); const consumedPrebidAps = consumedPrebidApsIds.get(adId); if (consumedPrebidAps) { @@ -1689,6 +2058,7 @@ export function installTsRenderBridge(): void { // other iframe. Letting Prebid's global handler answer a foreign source // would expose the creative despite the slot-bound capability check. e.stopImmediatePropagation(); + safelyRecordCreativeFailure(consumedPrebidAps.attemptId, 'aps_consumed_tombstone'); return; } @@ -1698,11 +2068,28 @@ export function installTsRenderBridge(): void { // Prebid handles ad IDs globally and would otherwise answer a request from // an unrelated iframe when this slot-bound capability rejects it. e.stopImmediatePropagation(); - if (!messageSourceBelongsToAdUnit(e.source, prebidRendererEntry.adUnitCode)) return; + const attemptId = beginApsCreativeAttempt(prebidRendererEntry.adUnitCode); + const sourceFrame = sourceFrameForAdUnit(e.source, prebidRendererEntry.adUnitCode); + if (!sourceFrame) { + safelyRecordCreativeFailure(attemptId, 'aps_source_not_in_ad_unit'); + return; + } const renderer = validateApsRenderer(prebidRendererEntry.renderer); - if (!renderer || !hasConsumedPrebidApsIdCapacity(consumedPrebidApsIds, adId)) return; + if (!renderer) { + safelyRecordCreativeFailure(attemptId, 'aps_descriptor_fields'); + return; + } + if (!hasConsumedPrebidApsIdCapacity(consumedPrebidApsIds, adId)) { + safelyRecordCreativeFailure(attemptId, 'aps_tombstone_capacity'); + return; + } if (!consumeApsPrebidRenderer(adId, prebidRendererEntry)) return; - recordConsumedPrebidApsId(consumedPrebidApsIds, adId, prebidRendererEntry.expiresAt); + recordConsumedPrebidApsId( + consumedPrebidApsIds, + adId, + prebidRendererEntry.expiresAt, + attemptId + ); const markUsed = (): void => { try { @@ -1717,7 +2104,17 @@ export function installTsRenderBridge(): void { source: e.source, trustedServer: (validatedRenderer) => { const rendererUrl = apsRendererUrl(); - if (!rendererUrl) return false; + if (!rendererUrl) { + safelyRecordCreativeFailure(attemptId, 'aps_missing_renderer_url'); + return false; + } + const stillOwnsCreative = () => + sourceFrameForAdUnit(e.source, prebidRendererEntry.adUnitCode)?.iframe === + sourceFrame.iframe; + if (!creativeFrameIsCurrent(e.source, sourceFrame, generation, stillOwnsCreative)) { + safelyRecordCreativeFailure(attemptId, 'aps_source_not_in_ad_unit'); + return false; + } try { port.postMessage( JSON.stringify({ @@ -1731,11 +2128,21 @@ export function installTsRenderBridge(): void { height: validatedRenderer.height, }) ); - return true; + safelyRecordCreativeResponse(attemptId); } catch (err) { log.warn(`[tsjs-gpt] APS Prebid response post failed for '${adId}'`, err); + safelyRecordCreativeFailure(attemptId, 'response_post_failed'); return false; } + safelyResizeCollapsedCreativeFrame( + e.source, + sourceFrame, + validatedRenderer.width, + validatedRenderer.height, + generation, + stillOwnsCreative + ); + return creativeFrameIsCurrent(e.source, sourceFrame, generation, stillOwnsCreative); }, }); if (typeof dispatched === 'boolean') { @@ -1748,8 +2155,8 @@ export function installTsRenderBridge(): void { return; } - const sourceSlotId = slotIdForMessageSource(e.source); - if (!sourceSlotId) return; + const sourceSlotFrame = slotFrameForMessageSource(e.source); + if (!sourceSlotFrame) return; // Resolve the bid by the requesting slot, not by the first bid whose hb_adid // matches. hb_adid is not unique per bid: absent PBS Cache it falls back to a @@ -1758,7 +2165,7 @@ export function installTsRenderBridge(): void { // first-match-by-adId lookup would resolve every duplicate to one slot, so all // but that slot render blank. const bids = window.tsjs?.bids ?? {}; - const slotId = sourceSlotId; + const slotId = sourceSlotFrame.slotId; const matchedBid = bids[slotId]; // Not a TS bid, or the requesting slot's bid does not own this adId — let @@ -1782,6 +2189,13 @@ export function installTsRenderBridge(): void { trustedServer: (validatedRenderer) => { const rendererUrl = apsRendererUrl(); if (!rendererUrl) return false; + const stillOwnsCreative = () => + window.tsjs?.bids?.[slotId] === matchedBid && + matchedBid.hb_adid === adId && + sourceFrameForSlotId(e.source, slotId)?.iframe === sourceSlotFrame.iframe; + if (!creativeFrameIsCurrent(e.source, sourceSlotFrame, generation, stillOwnsCreative)) { + return false; + } try { port.postMessage( JSON.stringify({ @@ -1795,11 +2209,19 @@ export function installTsRenderBridge(): void { height: validatedRenderer.height, }) ); - return true; } catch (err) { log.warn(`[tsjs-gpt] APS server response post failed for '${slotId}'`, err); return false; } + safelyResizeCollapsedCreativeFrame( + e.source, + sourceSlotFrame, + validatedRenderer.width, + validatedRenderer.height, + generation, + stillOwnsCreative + ); + return creativeFrameIsCurrent(e.source, sourceSlotFrame, generation, stillOwnsCreative); }, }) ); @@ -1825,6 +2247,15 @@ export function installTsRenderBridge(): void { if (inlineAdm) { e.stopImmediatePropagation(); + const stillOwnsCreative = () => + Boolean( + window.tsjs?.bids?.[slotId] === matchedBid && + matchedBid.hb_adid === adId && + sourceFrameForSlotId(e.source, slotId)?.iframe === sourceSlotFrame.iframe + ); + if (!creativeFrameIsCurrent(e.source, sourceSlotFrame, generation, stillOwnsCreative)) { + return; + } try { port.postMessage( JSON.stringify({ @@ -1841,6 +2272,15 @@ export function installTsRenderBridge(): void { log.warn(`[tsjs-gpt] pbRender bridge: response post failed for '${slotId}'`, err); return; } + safelyResizeCollapsedCreativeFrame( + e.source, + sourceSlotFrame, + width, + height, + generation, + stillOwnsCreative + ); + if (!creativeFrameIsCurrent(e.source, sourceSlotFrame, generation, stillOwnsCreative)) return; safelyRecordCreativeResponse(attemptId); fireWinBillingBeacons(slotId, matchedBid); log.debug(`[tsjs-gpt] pbRender bridge served '${slotId}' from inline adm`); @@ -1890,6 +2330,15 @@ export function installTsRenderBridge(): void { cached.price !== undefined ? expandAuctionPriceMacro(cached.adm, cached.price) : cached.adm; + const cachedWidth = cached.width ?? width; + const cachedHeight = cached.height ?? height; + const stillOwnsCreative = () => + window.tsjs?.bids?.[slotId] === matchedBid && + matchedBid.hb_adid === adId && + sourceFrameForSlotId(e.source, slotId)?.iframe === sourceSlotFrame.iframe; + if (!creativeFrameIsCurrent(e.source, sourceSlotFrame, generation, stillOwnsCreative)) { + return; + } try { port.postMessage( JSON.stringify({ @@ -1897,8 +2346,8 @@ export function installTsRenderBridge(): void { adId, ad, renderer: TS_DISPLAY_RENDERER, - width: cached.width ?? width, - height: cached.height ?? height, + width: cachedWidth, + height: cachedHeight, }) ); } catch (err) { @@ -1906,6 +2355,17 @@ export function installTsRenderBridge(): void { log.warn(`[tsjs-gpt] pbRender bridge: response post failed for '${slotId}'`, err); return; } + safelyResizeCollapsedCreativeFrame( + e.source, + sourceSlotFrame, + cachedWidth, + cachedHeight, + generation, + stillOwnsCreative + ); + if (!creativeFrameIsCurrent(e.source, sourceSlotFrame, generation, stillOwnsCreative)) { + return; + } safelyRecordCreativeResponse(attemptId); // Beacons carry the server-expanded ${AUCTION_PRICE} from the auction's // clearing price, not `cached.price` — the auction result is the diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts index 475bc7f93..956824f84 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/api.ts @@ -1,5 +1,7 @@ import type { GptDiagnosticsApi, + GptDiagnosticsAuctionFacts, + GptDiagnosticsAuctionWinner, GptDiagnosticsCreativeFailure, GptDiagnosticsExportV1, GptDiagnosticsRecorder, @@ -18,9 +20,20 @@ interface ApiStore { auctionSlotId: string, opportunity: GptDiagnosticsTrustedServerOpportunity, trustedServerAuctionId?: string, - requestedSlotSizes?: ReadonlyArray + requestedSlotSizes?: ReadonlyArray, + auctionFacts?: GptDiagnosticsAuctionFacts ): void; recordPrebidRefresh(slots: GptDiagnosticsSlotHandle[]): void; + recordPrebidAuction( + slot: GptDiagnosticsSlotHandle, + auctionId: string, + targetingCandidate?: GptDiagnosticsAuctionWinner + ): void; + recordPrebidWin( + slot: GptDiagnosticsSlotHandle, + auctionId: string, + winner: GptDiagnosticsAuctionWinner + ): void; recordTrustedServerCreativeRequest(auctionSlotId: string): number | undefined; recordTrustedServerCreativeResponse(attemptId: number): void; recordTrustedServerCreativeFailure( @@ -67,6 +80,21 @@ function cloneExportSnapshot(snapshot: GptDiagnosticsExportV1): GptDiagnosticsEx requestedSlotSizes: cycle.requestedSlotSizes?.map((size) => [...size]), size: cycle.size ? [...cycle.size] : undefined, observedSlotSize: cycle.observedSlotSize ? [...cycle.observedSlotSize] : undefined, + ...(cycle.auctionWinner ? { auctionWinner: { ...cycle.auctionWinner } } : {}), + ...(cycle.prebidAuction + ? { + prebidAuction: { + ...cycle.prebidAuction, + ...(cycle.prebidAuction.targetingCandidate + ? { targetingCandidate: { ...cycle.prebidAuction.targetingCandidate } } + : {}), + ...(cycle.prebidAuction.win ? { win: { ...cycle.prebidAuction.win } } : {}), + }, + } + : {}), + ...(cycle.serverAuctionTimings + ? { serverAuctionTimings: { ...cycle.serverAuctionTimings } } + : {}), adManager: cycle.adManager ? { ...cycle.adManager, @@ -157,7 +185,8 @@ export class GptDiagnosticsApiController { auctionSlotId, opportunity, trustedServerAuctionId, - requestedSlotSizes + requestedSlotSizes, + auctionFacts ) => safelyRecord(() => { this.store.recordTrustedServerOpportunity( @@ -165,10 +194,15 @@ export class GptDiagnosticsApiController { auctionSlotId, opportunity, trustedServerAuctionId, - requestedSlotSizes + requestedSlotSizes, + auctionFacts ); }), recordPrebidRefresh: (slots) => safelyRecord(() => this.store.recordPrebidRefresh(slots)), + recordPrebidAuction: (slot, auctionId, targetingCandidate) => + safelyRecord(() => this.store.recordPrebidAuction(slot, auctionId, targetingCandidate)), + recordPrebidWin: (slot, auctionId, winner) => + safelyRecord(() => this.store.recordPrebidWin(slot, auctionId, winner)), recordTrustedServerCreativeRequest: (auctionSlotId) => safelyCreateAttempt(() => this.store.recordTrustedServerCreativeRequest(auctionSlotId)), recordTrustedServerCreativeResponse: (attemptId) => @@ -200,6 +234,21 @@ export class GptDiagnosticsApiController { requestedSlotSizes: cycle.requestedSlotSizes?.map((size) => [...size]), size: cycle.size ? [...cycle.size] : undefined, observedSlotSize: cycle.observedSlotSize ? [...cycle.observedSlotSize] : undefined, + ...(cycle.auctionWinner ? { auctionWinner: { ...cycle.auctionWinner } } : {}), + ...(cycle.prebidAuction + ? { + prebidAuction: { + ...cycle.prebidAuction, + ...(cycle.prebidAuction.targetingCandidate + ? { targetingCandidate: { ...cycle.prebidAuction.targetingCandidate } } + : {}), + ...(cycle.prebidAuction.win ? { win: { ...cycle.prebidAuction.win } } : {}), + }, + } + : {}), + ...(cycle.serverAuctionTimings + ? { serverAuctionTimings: { ...cycle.serverAuctionTimings } } + : {}), adManager: cycle.adManager ? { ...cycle.adManager, diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts index 643dc355a..18f080695 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/badges.ts @@ -2,7 +2,12 @@ import type { GptDiagnosticsRequestCycle } from '../../core/types'; import type { GptDiagnosticsBindingManager } from './binding'; import { unhandledCase } from './exhaustive'; -import { formatSizes, scheduleFrame } from './presentation_helpers'; +import { + auctionTypeBadgeLabel, + displayableGptFillSize, + formatSizes, + scheduleFrame, +} from './presentation_helpers'; import type { GptDiagnosticsBindingInput, GptDiagnosticsStoreSlotSnapshot, @@ -33,6 +38,7 @@ interface BadgeOptions { window?: BadgeWindow; document?: Document; scheduleFrame?: (callback: () => void) => void; + onActivate?: (runtimeSlotNumber: number, requestNumber: number) => void; } function intersectsViewport(rectangle: DOMRect, window: Window): boolean { @@ -106,9 +112,12 @@ function badgeText(cycle: GptDiagnosticsRequestCycle): string { else if (cycle.isEmpty === false) firstLine.push('Filled'); else if (cycle.renderAtMs !== undefined) firstLine.push('Rendered (fill unknown)'); else firstLine.push('Pending'); + if (cycle.auctionType) firstLine.push(auctionTypeBadgeLabel(cycle.auctionType)); const delivery = deliveryLabel(cycle); if (delivery) firstLine.push(delivery); - if (cycle.requestPath === 'competing') firstLine.push('Competing paths'); + if (cycle.requestPath === 'competing' && cycle.auctionType !== 'competing') { + firstLine.push('Competing paths'); + } if (cycle.requestedSlotSizes) { const displayedSizes = cycle.requestedSlotSizes.slice(0, MAX_BADGE_REQUESTED_SLOT_SIZES); const remainingSizeCount = cycle.requestedSlotSizes.length - displayedSizes.length; @@ -116,16 +125,17 @@ function badgeText(cycle: GptDiagnosticsRequestCycle): string { `Req ${formatSizes(displayedSizes)}${remainingSizeCount > 0 ? ` +${remainingSizeCount}` : ''}` ); } - if (cycle.size) firstLine.push(`Fill ${cycle.size[0]}×${cycle.size[1]}`); + const fillSize = displayableGptFillSize(cycle.size); + if (fillSize) firstLine.push(`Fill ${fillSize[0]}×${fillSize[1]}`); if (cycle.observedSlotSize) { - firstLine.push(`Box ${cycle.observedSlotSize[0]}×${cycle.observedSlotSize[1]}`); + firstLine.push(`Size filled ${cycle.observedSlotSize[0]}×${cycle.observedSlotSize[1]}`); } const timingLine: string[] = []; const response = formatMilliseconds(cycle.durations.requestToResponseMs); const render = formatMilliseconds(cycle.durations.responseToRenderMs); - if (response) timingLine.push(`Response ${response}`); - if (render) timingLine.push(`Render ${render}`); + if (response) timingLine.push(`GAM request → response ${response}`); + if (render) timingLine.push(`GAM response → render ${render}`); const lines = [firstLine.join(' · ')]; if (timingLine.length > 0) lines.push(timingLine.join(' · ')); @@ -148,6 +158,7 @@ export class GptDiagnosticsBadgeManager { private readonly window: BadgeWindow; private readonly document: Document; private readonly scheduleFrame: (callback: () => void) => void; + private readonly onActivate: (runtimeSlotNumber: number, requestNumber: number) => void; private readonly unsubscribeStore: () => void; private readonly unsubscribeBindings: () => void; private readonly slotElementIds = new Set(); @@ -165,6 +176,7 @@ export class GptDiagnosticsBadgeManager { this.document = options.document ?? document; this.scheduleFrame = options.scheduleFrame ?? ((callback) => scheduleFrame(this.window, callback)); + this.onActivate = options.onActivate ?? (() => undefined); this.refreshSlotElementIds(); this.unsubscribeStore = this.store.subscribe(() => { this.refreshSlotElementIds(); @@ -201,10 +213,19 @@ export class GptDiagnosticsBadgeManager { if (!intersectsViewport(rectangle, this.window)) continue; observedElements.push(element); - const badge = this.document.createElement('div'); + const badge = this.document.createElement('button'); + badge.type = 'button'; badge.className = 'tsgd-badge'; badge.dataset.runtimeSlot = String(slot.runtimeSlotNumber); - badge.textContent = badgeText(cycle); + badge.dataset.requestNumber = String(cycle.requestNumber); + badge.textContent = `Ad #${slot.runtimeSlotNumber} · Request #${cycle.requestNumber} · ${badgeText(cycle)}`; + badge.setAttribute( + 'aria-label', + `Open diagnostics for Ad #${slot.runtimeSlotNumber}, Request #${cycle.requestNumber}` + ); + badge.addEventListener('click', () => + this.onActivate(slot.runtimeSlotNumber, cycle.requestNumber) + ); badge.style.maxWidth = `${BADGE_MAX_WIDTH_PX}px`; badge.style.left = `${Math.max( BADGE_EDGE_GUTTER_PX, @@ -222,7 +243,8 @@ export class GptDiagnosticsBadgeManager { badges.push(badge); } - this.layer.replaceChildren(...badges); + for (const badge of this.layer.querySelectorAll('.tsgd-badge')) badge.remove(); + this.layer.append(...badges); this.resizeObserver?.disconnect(); for (const element of observedElements) this.resizeObserver?.observe(element); } diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts index d7271710c..635ca51b9 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/index.ts @@ -60,6 +60,8 @@ export function installGptDiagnosticsRuntime( badges = new GptDiagnosticsBadgeManager(store, bindings, { window: target, document: target.document, + onActivate: (runtimeSlotNumber, requestNumber) => + overlay?.selectRequest(runtimeSlotNumber, requestNumber), }); slotSizeObserver = new GptDiagnosticsSlotSizeObserver(store, bindings, { window: target }); overlay = new GptDiagnosticsOverlay(store, bindings, { diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts index 63c0b6fb3..045c062c8 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/overlay.ts @@ -3,7 +3,12 @@ import type { GptDiagnosticsRequestCycle } from '../../core/types'; import type { GptDiagnosticsBindingManager } from './binding'; import { unhandledCase } from './exhaustive'; -import { formatSizes, scheduleFrame } from './presentation_helpers'; +import { + auctionTypeLabel, + displayableGptFillSize, + formatSizes, + scheduleFrame, +} from './presentation_helpers'; import type { GptDiagnosticsStoreSlotSnapshot, GptDiagnosticsStoreSnapshot } from './store'; export const GPT_DIAGNOSTICS_HOST_ID = 'trusted-server-gpt-diagnostics'; @@ -69,7 +74,7 @@ const PANEL_STYLES = ` font: inherit; } button { cursor: pointer; } - button:focus-visible, select:focus-visible, summary:focus-visible { outline: 2px solid #60a5fa; outline-offset: 2px; } + button:focus-visible, select:focus-visible, summary:focus-visible, a:focus-visible { outline: 2px solid #60a5fa; outline-offset: 2px; } .tsgd-toolbar { display: flex; gap: 8px; align-items: center; border-bottom: 1px solid #334155; } .tsgd-toolbar label { color: #cbd5e1; } .tsgd-summary { color: #cbd5e1; border-bottom: 1px solid #334155; } @@ -78,6 +83,13 @@ const PANEL_STYLES = ` .tsgd-empty { padding: 18px 12px; color: #94a3b8; } .tsgd-slot { border-bottom: 1px solid #334155; } .tsgd-slot:last-child { border-bottom: 0; } + .tsgd-slot[aria-current="true"], .tsgd-cycle[aria-current="true"] { outline: 2px solid #60a5fa; outline-offset: -2px; } + .tsgd-group { margin-top: 8px; } + .tsgd-group h3 { margin: 0; color: #e2e8f0; font-size: 12px; } + .tsgd-help { padding: 0 12px 8px; color: #cbd5e1; } + .tsgd-help p { margin: 6px 0 0; } + .tsgd-locate { margin-top: 8px; } + .tsgd-selection-note { padding: 8px 12px; color: #fde68a; border-bottom: 1px solid #334155; } .tsgd-slot-title { display: flex; gap: 8px; align-items: baseline; } .tsgd-slot-title strong { overflow-wrap: anywhere; } .tsgd-state { margin-left: auto; color: #fde68a; white-space: nowrap; } @@ -89,6 +101,7 @@ const PANEL_STYLES = ` .tsgd-badge-layer { position: fixed; z-index: 2147483646; inset: 0; pointer-events: none; } .tsgd-badge { position: fixed; + pointer-events: auto; padding: 5px 7px; color: #fff; background: rgb(15 23 42 / 94%); @@ -97,6 +110,15 @@ const PANEL_STYLES = ` box-shadow: 0 2px 8px rgb(0 0 0 / 35%); font: 11px/1.35 ui-sans-serif, system-ui, sans-serif; white-space: pre-line; + text-align: left; + cursor: pointer; + } + .tsgd-highlight { + position: fixed; + border: 3px solid #fbbf24; + background: rgb(251 191 36 / 18%); + box-shadow: 0 0 0 3px rgb(15 23 42 / 75%); + pointer-events: none; } `; @@ -120,28 +142,33 @@ function formatMilliseconds(value: number | undefined): string | undefined { return `${Math.round(value * 10) / 10} ms`; } -function deliveryFact(cycle: GptDiagnosticsRequestCycle): string | undefined { +function deliveryFact(cycle: GptDiagnosticsRequestCycle): string { switch (cycle.delivery) { case 'trusted_server_response_sent': - return 'Trusted Server selected; markup response sent to PUC'; + return 'Creative markup sent; execution not confirmed'; case 'trusted_server_selected': - return 'Trusted Server selected; no markup response confirmed'; + return 'Server bid selected by the creative bridge; response not confirmed'; case 'candidate_unconfirmed': - return 'Trusted Server candidate unconfirmed — another GAM result or a creative/bridge failure is possible'; + return 'Server bid available; selection not confirmed'; case 'no_candidate': - return 'adInit observed no direct Trusted Server candidate for this request'; + return 'No direct Trusted Server candidate'; case 'unknown': - return 'Delivery status unknown — required GPT or direct-candidate evidence was not observed'; + return 'Delivery status unknown — required evidence was not observed'; case 'pending': return 'Waiting for Trusted Server creative evidence'; case 'not_applicable': + return 'Delivery evidence: Not applicable'; case undefined: - return undefined; + return 'Delivery evidence: Not observed'; default: return unhandledCase(cycle.delivery); } } +function servedBidderFact(cycle: GptDiagnosticsRequestCycle): string | undefined { + return cycle.isEmpty === false ? 'Served bidder not confirmed' : undefined; +} + function requestPathFact(cycle: GptDiagnosticsRequestCycle): string { switch (cycle.requestPath) { case 'trusted_server_direct': @@ -151,24 +178,24 @@ function requestPathFact(cycle: GptDiagnosticsRequestCycle): string { case 'publisher_refresh': return 'Request path: Publisher refresh'; case 'competing': - return 'Request path: Competing paths'; + return 'Request path: Multiple paths observed'; case 'unattributed': - return 'Request path: Unattributed'; + return 'Request path: Not observed'; case undefined: - return 'Request path: Unknown (not observed)'; + return 'Request path: Not observed'; } } function trustedServerOpportunityFact(cycle: GptDiagnosticsRequestCycle): string { switch (cycle.trustedServerOpportunity) { case 'renderable_candidate': - return 'Direct opportunity: Renderable candidate'; + return 'Server bid available; creative source present'; case 'unrenderable_candidate': - return 'Direct opportunity: Unrenderable candidate'; + return 'Server bid available; creative source incomplete'; case 'no_candidate': return 'Direct opportunity: No candidate'; case undefined: - return 'Direct opportunity: Unknown (not observed)'; + return 'Direct opportunity: Not observed'; } } @@ -221,16 +248,122 @@ function responseClassFact(cycle: GptDiagnosticsRequestCycle): string | undefine } } -function cycleFacts(cycle: GptDiagnosticsRequestCycle): string[] { - const facts: string[] = [requestPathFact(cycle), trustedServerOpportunityFact(cycle)]; +function auctionFacts(cycle: GptDiagnosticsRequestCycle): string[] { + const facts = [ + requestPathFact(cycle), + `Auction evidence: ${cycle.auctionType ? auctionTypeLabel(cycle.auctionType) : 'Auction not observed'}`, + ]; + if (cycle.auctionWinner) { + facts.push(`Server auction winner: ${cycle.auctionWinner.bidder}`); + facts.push( + `Server bid price bucket: ${cycle.auctionWinner.priceBucket} ${cycle.auctionWinner.currency ?? '(currency not supplied)'}` + ); + } + if (cycle.prebidAuction?.targetingCandidate) { + const candidate = cycle.prebidAuction.targetingCandidate; + facts.push(`Prebid targeting candidate: ${candidate.bidder}`); + facts.push( + `Prebid candidate price bucket: ${candidate.priceBucket} ${candidate.currency ?? '(currency not supplied)'}` + ); + } + if (cycle.prebidAuction?.win) { + const win = cycle.prebidAuction.win; + facts.push(`Prebid bidWon observation: ${win.bidder}`); + facts.push( + `Prebid win price bucket: ${win.priceBucket} ${win.currency ?? '(currency not supplied)'}` + ); + } + const servedBidder = servedBidderFact(cycle); + if (servedBidder) facts.push(servedBidder); + return facts; +} + +function timingFacts(cycle: GptDiagnosticsRequestCycle): string[] { + const serverTimingApplies = + cycle.auctionType === 'ssat' || + cycle.auctionType === 'trusted_server' || + cycle.auctionType === 'competing'; + const missingServerTiming = serverTimingApplies ? 'Unavailable' : 'Not applicable'; + const serverTimings = [ + ['Server request start → auction dispatched', cycle.serverAuctionTimings?.auctionDispatchedMs], + ['Server request start → auction collected', cycle.serverAuctionTimings?.auctionResolvedMs], + ['Server request start → bids ready', cycle.serverAuctionTimings?.auctionCommittedMs], + ] as const; + const facts = serverTimings.map( + ([label, timing]) => `${label} ${formatMilliseconds(timing) ?? missingServerTiming}` + ); + const wait = formatMilliseconds(cycle.serverAuctionTimings?.auctionWaitMs); + if (wait) { + const placement = + cycle.serverAuctionTimings?.auctionWaitPlacement === 'pre_header' + ? 'pre-header' + : cycle.serverAuctionTimings?.auctionWaitPlacement === 'in_stream' + ? 'in stream' + : 'placement unknown'; + facts.push(`Auction collection wait (${placement}) ${wait}`); + } else { + facts.push(`Auction collection wait ${missingServerTiming}`); + } + facts.push( + `Opportunity → request ${formatMilliseconds(cycle.opportunityToRequestMs) ?? 'Unavailable'}` + ); + const durations = [ + ['GAM request → response', cycle.durations.requestToResponseMs], + ['GAM response → render', cycle.durations.responseToRenderMs], + ['GAM request → render', cycle.durations.requestToRenderMs], + ['Render → load', cycle.durations.renderToLoadMs], + ['Render → viewable', cycle.durations.renderToViewableMs], + ] as const; + for (const [label, duration] of durations) { + facts.push(`${label} ${formatMilliseconds(duration) ?? 'Unavailable'}`); + } + return facts; +} + +function deliveryFacts(cycle: GptDiagnosticsRequestCycle): string[] { + const facts = [ + deliveryFact(cycle), + responseClassFact(cycle) ?? 'Ad Manager response class: Not observed', + adManagerFact(cycle) ?? 'Ad Manager fields: Not observed', + ]; + if (cycle.loadAtMs !== undefined) facts.push('GPT slot onload observed'); + if (cycle.viewableAtMs !== undefined) facts.push('GPT impressionViewable observed'); + if (cycle.incompleteSequence) facts.push('Incomplete sequence'); + if (cycle.isBackfill !== undefined) facts.push(`Backfill ${cycle.isBackfill ? 'yes' : 'no'}`); + if (cycle.slotContentChanged !== undefined) { + facts.push(`Slot content changed ${cycle.slotContentChanged ? 'yes' : 'no'}`); + } + return facts; +} + +function sizeFacts(cycle: GptDiagnosticsRequestCycle): string[] { + const fillSize = displayableGptFillSize(cycle.size); + return [ + cycle.requestedSlotSizes + ? `Requested sizes ${formatSizes(cycle.requestedSlotSizes)}` + : 'Requested sizes: Not observed', + fillSize + ? `GPT-reported size ${formatSizes([fillSize])}` + : cycle.size?.[0] === 1 && cycle.size[1] === 1 + ? 'GPT-reported size: 1×1 placeholder hidden' + : 'GPT-reported size: Not observed', + cycle.observedSlotSize + ? `Size filled ${formatSizes([cycle.observedSlotSize])} · Measured outer slot size` + : 'Size filled: Not observed · Measured outer slot size', + ]; +} + +function technicalCycleFacts(cycle: GptDiagnosticsRequestCycle): string[] { + const facts = [trustedServerOpportunityFact(cycle)]; if (Number.isSafeInteger(cycle.requestIntentId) && cycle.requestIntentId! > 0) { facts.push(`Request intent: ${cycle.requestIntentId}`); } if (typeof cycle.trustedServerAuctionId === 'string' && cycle.trustedServerAuctionId.length > 0) { facts.push(`Trusted Server auction: ${cycle.trustedServerAuctionId}`); } - const opportunityToRequest = formatMilliseconds(cycle.opportunityToRequestMs); - if (opportunityToRequest) facts.push(`Opportunity → request ${opportunityToRequest}`); + if (cycle.prebidAuction?.auctionId) { + facts.push(`Prebid auction: ${cycle.prebidAuction.auctionId}`); + } const previousRenderToRequest = formatMilliseconds(cycle.previousRenderToRequestMs); if (cycle.replacedRequestNumber !== undefined && previousRenderToRequest) { facts.push( @@ -261,41 +394,19 @@ function cycleFacts(cycle: GptDiagnosticsRequestCycle): string[] { for (const failure of new Set(cycle.trustedServerCreativeFailures ?? [])) { facts.push(creativeFailureFact(failure)); } - const deliveryLine = deliveryFact(cycle); - if (deliveryLine) facts.push(deliveryLine); - const responseClassLine = responseClassFact(cycle); - if (responseClassLine) facts.push(responseClassLine); - const adManagerLine = adManagerFact(cycle); - if (adManagerLine) facts.push(adManagerLine); - if (cycle.loadAtMs !== undefined) facts.push('GPT slot onload observed'); - if (cycle.viewableAtMs !== undefined) facts.push('GPT impressionViewable observed'); - if (cycle.incompleteSequence) facts.push('Incomplete sequence'); - if (cycle.requestedSlotSizes) { - facts.push(`Requested slot sizes ${formatSizes(cycle.requestedSlotSizes)}`); - } - if (cycle.size) facts.push(`GPT-reported fill size ${cycle.size[0]}×${cycle.size[1]}`); - if (cycle.observedSlotSize) { - facts.push(`Observed outer slot box ${cycle.observedSlotSize[0]}×${cycle.observedSlotSize[1]}`); - } - if (cycle.isBackfill !== undefined) facts.push(`Backfill ${cycle.isBackfill ? 'yes' : 'no'}`); - if (cycle.slotContentChanged !== undefined) { - facts.push(`Slot content changed ${cycle.slotContentChanged ? 'yes' : 'no'}`); - } - - const durations = [ - ['Request → response', cycle.durations.requestToResponseMs], - ['Response → render', cycle.durations.responseToRenderMs], - ['Request → render', cycle.durations.requestToRenderMs], - ['Render → load', cycle.durations.renderToLoadMs], - ['Render → viewable', cycle.durations.renderToViewableMs], - ] as const; - for (const [label, duration] of durations) { - const formatted = formatMilliseconds(duration); - if (formatted) facts.push(`${label} ${formatted}`); - } return facts; } +function cycleFacts(cycle: GptDiagnosticsRequestCycle): string[] { + return [ + ...auctionFacts(cycle), + ...deliveryFacts(cycle), + ...timingFacts(cycle), + ...sizeFacts(cycle), + ...technicalCycleFacts(cycle), + ]; +} + function cycleLabel(cycle: GptDiagnosticsRequestCycle): string { return cycle.requestNumber === 1 ? 'Initial request' : `Refresh ${cycle.requestNumber - 1}`; } @@ -327,6 +438,21 @@ function appendFacts(document: Document, parent: HTMLElement, facts: string[]): parent.append(list); } +function appendGroup( + document: Document, + parent: HTMLElement, + heading: string, + facts: string[] +): void { + const section = document.createElement('section'); + section.className = 'tsgd-group'; + const title = document.createElement('h3'); + title.textContent = heading; + section.append(title); + appendFacts(document, section, facts); + parent.append(section); +} + /** Owns hydration-safe mounting and the closed-shadow diagnostics panel. */ export class GptDiagnosticsOverlay { private readonly store: OverlayStore; @@ -341,6 +467,7 @@ export class GptDiagnosticsOverlay { private readonly unsubscribeBindings: () => void; private host?: HTMLElement; private panel?: HTMLElement; + private badgeLayer?: HTMLElement; private lifecycleObserver?: MutationObserver; private visualReady = false; private mountWaitStarted = false; @@ -351,6 +478,8 @@ export class GptDiagnosticsOverlay { private dismissed = false; private destroyed = false; private filter: GptDiagnosticsFilter = 'all'; + private selectedRequest?: { runtimeSlotNumber: number; requestNumber: number }; + private selectedRequestHasFocus = false; constructor(store: OverlayStore, bindings: OverlayBindings, options: OverlayOptions = {}) { this.store = store; @@ -381,6 +510,23 @@ export class GptDiagnosticsOverlay { this.removeHost(); } + /** Open and focus the exact retained request selected from an on-page badge. */ + selectRequest(runtimeSlotNumber: number, requestNumber: number): void { + if (this.destroyed) return; + this.selectedRequest = { runtimeSlotNumber, requestNumber }; + this.filter = 'all'; + this.collapsed = false; + this.show(); + this.render(); + this.scheduleFrame(() => { + const selected = this.panel?.querySelector( + `[data-runtime-slot="${runtimeSlotNumber}"][data-request-number="${requestNumber}"]` + ); + selected?.focus(); + selected?.scrollIntoView?.({ block: 'nearest' }); + }); + } + destroy(): void { if (this.destroyed) return; this.destroyed = true; @@ -439,11 +585,11 @@ export class GptDiagnosticsOverlay { panel.setAttribute('aria-label', 'GPT runtime diagnostics'); const badgeLayer = this.document.createElement('div'); badgeLayer.className = 'tsgd-badge-layer'; - badgeLayer.setAttribute('aria-hidden', 'true'); root.append(style, badgeLayer, panel); this.host = host; this.panel = panel; + this.badgeLayer = badgeLayer; (this.document.body ?? this.document.documentElement).append(host); this.onShadowRoot?.(root); this.onBadgeLayerChange?.(badgeLayer); @@ -455,6 +601,7 @@ export class GptDiagnosticsOverlay { const host = this.host; this.host = undefined; this.panel = undefined; + this.badgeLayer = undefined; host?.remove(); } @@ -502,8 +649,9 @@ export class GptDiagnosticsOverlay { const panel = this.panel; const previousContent = panel.querySelector('.tsgd-content'); const previousScrollTop = previousContent?.scrollTop ?? 0; + const selectedRequestWasFocused = this.selectedRequestHasFocus; const openHistorySlots = new Set( - Array.from(panel.querySelectorAll('.tsgd-slot details[open]')) + Array.from(panel.querySelectorAll('.tsgd-history[open]')) .map((details) => details.closest('.tsgd-slot')?.dataset.runtimeSlot) .filter((runtimeSlot): runtimeSlot is string => runtimeSlot !== undefined) ); @@ -554,10 +702,27 @@ export class GptDiagnosticsOverlay { this.render(); }); const exportButton = this.button('Export JSON', () => this.onExport()); + const dictionaryLink = this.document.createElement('a'); + dictionaryLink.href = + 'https://iabtechlab.github.io/trusted-server/guide/integrations/gpt-diagnostics-dictionary'; + dictionaryLink.target = '_blank'; + dictionaryLink.rel = 'noopener'; + dictionaryLink.textContent = 'Label dictionary'; + dictionaryLink.setAttribute('aria-label', 'Open GPT diagnostics label dictionary'); filterLabel.append(select); - toolbar.append(filterLabel, exportButton); + toolbar.append(filterLabel, exportButton, dictionaryLink); panel.append(toolbar); + const help = this.document.createElement('details'); + help.className = 'tsgd-help'; + const helpSummary = this.document.createElement('summary'); + helpSummary.textContent = 'How to read this evidence'; + const helpText = this.document.createElement('p'); + helpText.textContent = + 'Auction winners, Prebid candidates, and GPT render results are separate observations. “Filled” does not identify the served bidder. Browser and server timings use separate clocks.'; + help.append(helpSummary, helpText); + panel.append(help); + const summary = this.document.createElement('div'); summary.className = 'tsgd-summary'; summary.textContent = `${snapshot.slots.length} slots · ${snapshot.callbackIssues.length} callback issues · ${snapshot.attributionIssues?.length ?? 0} attribution issues`; @@ -572,6 +737,21 @@ export class GptDiagnosticsOverlay { summary.append(coverage); panel.append(summary); + if ( + this.selectedRequest && + !snapshot.slots.some( + (slot) => + slot.runtimeSlotNumber === this.selectedRequest?.runtimeSlotNumber && + slot.requests.some((cycle) => cycle.requestNumber === this.selectedRequest?.requestNumber) + ) + ) { + const note = this.document.createElement('div'); + note.className = 'tsgd-selection-note'; + note.setAttribute('role', 'status'); + note.textContent = `Ad #${this.selectedRequest.runtimeSlotNumber}, Request #${this.selectedRequest.requestNumber} is no longer retained.`; + panel.append(note); + } + const content = this.document.createElement('div'); content.className = 'tsgd-content'; const filteredSlots = snapshot.slots.filter((slot) => @@ -585,22 +765,44 @@ export class GptDiagnosticsOverlay { content.append(empty); } else { for (const slot of filteredSlots) { - content.append(this.renderSlot(slot, openHistorySlots.has(String(slot.runtimeSlotNumber)))); + const selectedPreviousRequest = + this.selectedRequest?.runtimeSlotNumber === slot.runtimeSlotNumber && + this.selectedRequest.requestNumber !== latestCycle(slot)?.requestNumber; + content.append( + this.renderSlot( + slot, + openHistorySlots.has(String(slot.runtimeSlotNumber)) || selectedPreviousRequest + ) + ); } } panel.append(content); content.scrollTop = previousScrollTop; + if (selectedRequestWasFocused) { + panel.querySelector('[aria-current="true"]')?.focus({ preventScroll: true }); + } } private renderSlot(slot: GptDiagnosticsStoreSlotSnapshot, historyOpen: boolean): HTMLElement { const container = this.document.createElement('article'); container.className = 'tsgd-slot'; + const latest = latestCycle(slot); container.dataset.runtimeSlot = String(slot.runtimeSlotNumber); + if (latest) container.dataset.requestNumber = String(latest.requestNumber); + const latestSelected = + latest !== undefined && + this.selectedRequest?.runtimeSlotNumber === slot.runtimeSlotNumber && + this.selectedRequest.requestNumber === latest.requestNumber; + if (latestSelected) { + container.tabIndex = -1; + container.setAttribute('aria-current', 'true'); + this.trackSelectedRequestFocus(container); + } + const title = this.document.createElement('div'); title.className = 'tsgd-slot-title'; const name = this.document.createElement('strong'); - name.textContent = slot.slotElementId ?? `Unbound GPT slot ${slot.runtimeSlotNumber}`; - const latest = latestCycle(slot); + name.textContent = `Ad #${slot.runtimeSlotNumber}${latest ? ` · Request #${latest.requestNumber}` : ''} · ${slot.slotElementId ?? 'Unbound GPT slot'}`; const state = this.document.createElement('span'); state.className = 'tsgd-state'; state.textContent = primaryState(latest); @@ -608,41 +810,111 @@ export class GptDiagnosticsOverlay { container.append(title); const binding = this.bindings.get(slot.runtimeSlotNumber); - const facts = [ - slot.adUnitPath ? `Ad unit ${slot.adUnitPath}` : undefined, - binding.binding.status === 'bound' - ? `Bound · ${binding.visible ? 'Visible' : 'Outside viewport'}` - : binding.binding.status === 'ambiguous' - ? `Ambiguous binding · ${binding.binding.reason ?? 'unknown'}` - : `Unbound · ${binding.binding.reason ?? 'unknown'}`, - slot.currentVisibilityPercentage !== undefined - ? `GPT visibility ${slot.currentVisibilityPercentage}% (maximum ${slot.maximumVisibilityPercentage ?? slot.currentVisibilityPercentage}%)` - : undefined, - latest ? cycleLabel(latest) : undefined, - ...(latest ? cycleFacts(latest) : []), - ].filter((fact): fact is string => fact !== undefined); - appendFacts(this.document, container, facts); + if (binding.binding.status === 'bound' && binding.element?.isConnected) { + const locate = this.button('Locate on page', () => this.locateOnPage(slot.runtimeSlotNumber)); + locate.className = 'tsgd-locate'; + container.append(locate); + } + + if (latest) { + const summaryFacts = [ + `Ad #${slot.runtimeSlotNumber} · Request #${latest.requestNumber}`, + `GPT result: ${primaryState(latest)}`, + `Observed auction path: ${latest.auctionType ? auctionTypeLabel(latest.auctionType) : 'Auction not observed'}`, + deliveryFact(latest), + ]; + const servedBidder = servedBidderFact(latest); + if (servedBidder) summaryFacts.push(servedBidder); + appendGroup(this.document, container, 'Summary', summaryFacts); + appendGroup(this.document, container, 'Auction evidence', auctionFacts(latest)); + appendGroup(this.document, container, 'Delivery evidence', deliveryFacts(latest)); + appendGroup(this.document, container, 'Timing', timingFacts(latest)); + appendGroup(this.document, container, 'Size and visibility', [ + ...sizeFacts(latest), + binding.binding.status === 'bound' + ? `Binding: Bound · ${binding.visible ? 'Visible' : 'Outside viewport'}` + : `Binding: ${binding.binding.status} · ${binding.binding.reason ?? 'reason unavailable'}`, + slot.currentVisibilityPercentage !== undefined + ? `GPT visibility ${slot.currentVisibilityPercentage}% (maximum ${slot.maximumVisibilityPercentage ?? slot.currentVisibilityPercentage}%)` + : 'GPT visibility: Not observed', + ]); + } if (slot.requests.length > 1) { const history = this.document.createElement('details'); + history.className = 'tsgd-history'; history.open = historyOpen; const summary = this.document.createElement('summary'); - summary.textContent = `Previous requests (${slot.requests.length - 1})`; + summary.textContent = `Request history (${slot.requests.length - 1} previous)`; history.append(summary); for (const cycle of slot.requests.slice(0, -1).reverse()) { const previous = this.document.createElement('div'); previous.className = 'tsgd-cycle'; + previous.dataset.runtimeSlot = String(slot.runtimeSlotNumber); + previous.dataset.requestNumber = String(cycle.requestNumber); + const selected = + this.selectedRequest?.runtimeSlotNumber === slot.runtimeSlotNumber && + this.selectedRequest.requestNumber === cycle.requestNumber; + if (selected) { + previous.tabIndex = -1; + previous.setAttribute('aria-current', 'true'); + this.trackSelectedRequestFocus(previous); + } const heading = this.document.createElement('strong'); - heading.textContent = `${cycleLabel(cycle)} · ${primaryState(cycle)}`; + heading.textContent = `Request #${cycle.requestNumber} · ${cycleLabel(cycle)} · ${primaryState(cycle)}`; previous.append(heading); appendFacts(this.document, previous, cycleFacts(cycle)); history.append(previous); } container.append(history); } + + const technical = this.document.createElement('details'); + const technicalSummary = this.document.createElement('summary'); + technicalSummary.textContent = 'Technical details'; + technical.append(technicalSummary); + appendFacts(this.document, technical, [ + slot.adUnitPath ? `Ad unit ${slot.adUnitPath}` : 'Ad unit: Unavailable', + binding.binding.status === 'bound' + ? `Bound · ${binding.visible ? 'Visible' : 'Outside viewport'}` + : binding.binding.status === 'ambiguous' + ? `Ambiguous binding · ${binding.binding.reason ?? 'reason unavailable'}` + : `Unbound · ${binding.binding.reason ?? 'reason unavailable'}`, + ...(latest ? technicalCycleFacts(latest) : []), + ]); + container.append(technical); return container; } + private trackSelectedRequestFocus(element: HTMLElement): void { + element.addEventListener('focus', () => { + this.selectedRequestHasFocus = true; + }); + element.addEventListener('blur', () => { + this.selectedRequestHasFocus = false; + }); + } + + private locateOnPage(runtimeSlotNumber: number): void { + const binding = this.bindings.get(runtimeSlotNumber); + const element = binding.element; + if (binding.binding.status !== 'bound' || !element?.isConnected) return; + element.scrollIntoView({ behavior: 'auto', block: 'center', inline: 'nearest' }); + this.scheduleFrame(() => { + if (!this.badgeLayer?.isConnected || !element.isConnected) return; + const rectangle = element.getBoundingClientRect(); + const highlight = this.document.createElement('div'); + highlight.className = 'tsgd-highlight'; + highlight.setAttribute('aria-hidden', 'true'); + highlight.style.left = `${rectangle.left}px`; + highlight.style.top = `${rectangle.top}px`; + highlight.style.width = `${rectangle.width}px`; + highlight.style.height = `${rectangle.height}px`; + this.badgeLayer.append(highlight); + this.window.setTimeout(() => highlight.remove(), 1500); + }); + } + private button(label: string, action: () => void): HTMLButtonElement { const button = this.document.createElement('button'); button.type = 'button'; diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/presentation_helpers.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/presentation_helpers.ts index b601833b8..42ce496c6 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/presentation_helpers.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/presentation_helpers.ts @@ -1,10 +1,43 @@ -import type { Size } from '../../core/types'; +import type { GptDiagnosticsAuctionType, Size } from '../../core/types'; /** Formats CSS sizes consistently across diagnostics presentation surfaces. */ export function formatSizes(sizes: ReadonlyArray): string { return sizes.map((size) => `${size[0]}×${size[1]}`).join(', '); } +/** Hide GPT's ubiquitous 1×1 placeholder from presentation while retaining it in exports. */ +export function displayableGptFillSize(size: Size | undefined): Size | undefined { + return size?.[0] === 1 && size[1] === 1 ? undefined : size; +} + +/** Human-readable auction classification shared by the badge and side panel. */ +export function auctionTypeLabel(type: GptDiagnosticsAuctionType): string { + switch (type) { + case 'ssat': + return 'SSAT: initial-page server auction'; + case 'trusted_server': + return 'TS auction: SPA server auction'; + case 'client_side': + return 'Client-side Prebid auction'; + case 'competing': + return 'Multiple auction paths observed'; + } +} + +/** Compact auction classification shared by on-page badges. */ +export function auctionTypeBadgeLabel(type: GptDiagnosticsAuctionType): string { + switch (type) { + case 'ssat': + return 'SSAT'; + case 'trusted_server': + return 'TS auction'; + case 'client_side': + return 'Prebid auction'; + case 'competing': + return 'Multiple paths'; + } +} + /** Schedules presentation work in the target window's next animation frame. */ export function scheduleFrame( window: Pick, diff --git a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts index ca3348ae9..6fcbacfb5 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt_diagnostics/store.ts @@ -1,4 +1,5 @@ import type { + AuctionDiagnosticsData, GptDiagnosticsAdManagerIdentity, GptDiagnosticsAttributionIssue, GptDiagnosticsAttributionIssueReason, @@ -8,10 +9,15 @@ import type { GptDiagnosticsCoverageCounters, GptDiagnosticsCreativeFailure, GptDiagnosticsDelivery, + GptDiagnosticsAuctionFacts, + GptDiagnosticsAuctionType, + GptDiagnosticsAuctionWinner, + GptDiagnosticsPrebidAuctionEvidence, GptDiagnosticsDurations, GptDiagnosticsRequestCycle, GptDiagnosticsRequestPath, GptDiagnosticsResponseClass, + GptDiagnosticsServerAuctionTimingOrigin, GptDiagnosticsSlotHandle, GptDiagnosticsTrustedServerOpportunity, Size, @@ -108,6 +114,10 @@ interface PendingSourceEvidence { trustedServerOpportunity?: GptDiagnosticsTrustedServerOpportunity; trustedServerAuctionId?: string; requestedSlotSizes?: ReadonlyArray; + auctionType?: Extract; + auctionWinner?: GptDiagnosticsAuctionWinner; + prebidAuction?: GptDiagnosticsPrebidAuctionEvidence; + serverAuctionTimings?: AuctionDiagnosticsData; } interface PendingRequestIntent { @@ -230,6 +240,78 @@ function normalizedRequestedSlotSizes(value: unknown): ReadonlyArray | und return requestedSlotSizes.length > 0 ? Object.freeze(requestedSlotSizes) : undefined; } +function normalizedBoundedString(value: unknown, maxBytes: number): string | undefined { + if (typeof value !== 'string') return undefined; + const trimmed = value.trim(); + if (trimmed.length === 0 || new TextEncoder().encode(trimmed).length > maxBytes) return undefined; + return trimmed; +} + +function normalizedAuctionWinner(value: unknown): GptDiagnosticsAuctionWinner | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const candidate = value as Partial; + const bidder = normalizedBoundedString(candidate.bidder, 128); + const priceBucket = normalizedBoundedString(candidate.priceBucket, 64); + if (!bidder || !priceBucket || !/^\d+(?:\.\d+)?$/.test(priceBucket)) return undefined; + const currency = normalizedBoundedString(candidate.currency, 3)?.toUpperCase(); + return Object.freeze({ + bidder, + priceBucket, + ...(currency && /^[A-Z]{3}$/.test(currency) ? { currency } : {}), + }); +} + +const MAX_SERVER_AUCTION_TIMING_MS = 0xffffffff; + +function normalizedTiming(value: unknown): number | undefined { + return typeof value === 'number' && + Number.isFinite(value) && + value >= 0 && + value <= MAX_SERVER_AUCTION_TIMING_MS + ? value + : undefined; +} + +function normalizedServerAuctionTimings(value: unknown): AuctionDiagnosticsData | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const candidate = value as AuctionDiagnosticsData; + const timings: AuctionDiagnosticsData = { + auctionDispatchedMs: normalizedTiming(candidate.auctionDispatchedMs), + auctionResolvedMs: normalizedTiming(candidate.auctionResolvedMs), + auctionCommittedMs: normalizedTiming(candidate.auctionCommittedMs), + auctionWaitMs: normalizedTiming(candidate.auctionWaitMs), + auctionWaitPlacement: + candidate.auctionWaitPlacement === 'pre_header' || + candidate.auctionWaitPlacement === 'in_stream' + ? candidate.auctionWaitPlacement + : undefined, + }; + return Object.values(timings).some((entry) => entry !== undefined) + ? Object.freeze(timings) + : undefined; +} + +function normalizedAuctionFacts( + value: unknown +): Omit< + PendingSourceEvidence, + 'observedAtMs' | 'trustedServerOpportunity' | 'trustedServerAuctionId' | 'requestedSlotSizes' +> { + if (typeof value !== 'object' || value === null) return {}; + const candidate = value as GptDiagnosticsAuctionFacts; + const auctionType = + candidate.auctionType === 'ssat' || candidate.auctionType === 'trusted_server' + ? candidate.auctionType + : undefined; + const auctionWinner = normalizedAuctionWinner(candidate.winner); + const serverAuctionTimings = normalizedServerAuctionTimings(candidate.serverTimings); + return { + ...(auctionType ? { auctionType } : {}), + ...(auctionWinner ? { auctionWinner } : {}), + ...(serverAuctionTimings ? { serverAuctionTimings } : {}), + }; +} + function responseClass(cycle: MutableRequestCycle): GptDiagnosticsResponseClass | undefined { if (cycle.renderAtMs === undefined) return undefined; if (cycle.isEmpty === true) return 'empty'; @@ -285,6 +367,21 @@ function copyCycle(cycle: MutableRequestCycle, nowMs: number): GptDiagnosticsReq companyIds: cycle.adManager.companyIds ? [...cycle.adManager.companyIds] : undefined, } : undefined, + ...(cycle.auctionWinner ? { auctionWinner: { ...cycle.auctionWinner } } : {}), + ...(cycle.prebidAuction + ? { + prebidAuction: { + ...cycle.prebidAuction, + ...(cycle.prebidAuction.targetingCandidate + ? { targetingCandidate: { ...cycle.prebidAuction.targetingCandidate } } + : {}), + ...(cycle.prebidAuction.win ? { win: { ...cycle.prebidAuction.win } } : {}), + }, + } + : {}), + ...(cycle.serverAuctionTimings + ? { serverAuctionTimings: { ...cycle.serverAuctionTimings } } + : {}), trustedServerCreativeFailures: cycle.trustedServerCreativeFailures ? [...cycle.trustedServerCreativeFailures] : undefined, @@ -337,7 +434,8 @@ export class GptDiagnosticsStore { auctionSlotId: string, opportunity: GptDiagnosticsTrustedServerOpportunity, trustedServerAuctionId?: string, - requestedSlotSizes?: ReadonlyArray + requestedSlotSizes?: ReadonlyArray, + auctionFacts?: GptDiagnosticsAuctionFacts ): void { if ( !isSlotObject(slot) || @@ -360,6 +458,7 @@ export class GptDiagnosticsStore { trustedServerOpportunity: opportunity, trustedServerAuctionId: normalizedAuctionId(trustedServerAuctionId), requestedSlotSizes: normalizedRequestedSlotSizes(requestedSlotSizes), + ...normalizedAuctionFacts(auctionFacts), }); } @@ -373,6 +472,47 @@ export class GptDiagnosticsStore { } } + /** Record a completed Prebid attempt for one exact slot's next request. */ + recordPrebidAuction( + slot: GptDiagnosticsSlotLike, + auctionId: string, + targetingCandidate?: GptDiagnosticsAuctionWinner + ): void { + if (!isSlotObject(slot)) return; + const normalizedId = normalizedAuctionId(auctionId); + if (!normalizedId) return; + const candidate = normalizedAuctionWinner(targetingCandidate); + this.recordRequestIntentSource(slot, 'prebid_refresh', { + prebidAuction: Object.freeze({ + auctionId: normalizedId, + ...(candidate ? { targetingCandidate: candidate } : {}), + }), + }); + } + + /** Attach a documented Prebid win only to its retained exact slot and attempt. */ + recordPrebidWin( + slot: GptDiagnosticsSlotLike, + auctionId: string, + winner: GptDiagnosticsAuctionWinner + ): void { + if (!isSlotObject(slot)) return; + const normalizedId = normalizedAuctionId(auctionId); + const normalizedWinner = normalizedAuctionWinner(winner); + if (!normalizedId || !normalizedWinner) return; + const runtimeSlotNumber = this.slotNumbers.get(slot); + const record = runtimeSlotNumber === undefined ? undefined : this.slots.get(runtimeSlotNumber); + const matches = record?.requests.filter( + (cycle) => cycle.prebidAuction?.auctionId === normalizedId + ); + if (!matches || matches.length !== 1 || !matches[0]?.prebidAuction) return; + matches[0].prebidAuction = Object.freeze({ + ...matches[0].prebidAuction, + win: normalizedWinner, + }); + this.notify(); + } + /** Record publisher refresh observation from the private GPT diagnostics observer. */ recordPublisherRefresh(slots: GptDiagnosticsSlotLike[]): void { if (!Array.isArray(slots)) return; @@ -594,6 +734,13 @@ export class GptDiagnosticsStore { const intent = this.consumeRequestIntent(slot, timestampMs); const trustedServerEvidence = intent?.sources.get('trusted_server_direct'); const requestPath = this.requestPath(intent); + const auctionType = this.auctionType(intent, trustedServerEvidence); + const serverAuctionTimingOrigin: GptDiagnosticsServerAuctionTimingOrigin | undefined = + trustedServerEvidence?.serverAuctionTimings === undefined + ? undefined + : trustedServerEvidence.auctionType === 'trusted_server' + ? 'spa_auction' + : 'navigation'; record.requests.push({ requestNumber, requestedAtMs: timestampMs, @@ -607,6 +754,17 @@ export class GptDiagnosticsStore { ...(trustedServerEvidence?.trustedServerAuctionId !== undefined ? { trustedServerAuctionId: trustedServerEvidence.trustedServerAuctionId } : {}), + ...(auctionType !== undefined ? { auctionType } : {}), + ...(trustedServerEvidence?.auctionWinner !== undefined + ? { auctionWinner: trustedServerEvidence.auctionWinner } + : {}), + ...(intent?.sources.get('prebid_refresh')?.prebidAuction !== undefined + ? { prebidAuction: intent.sources.get('prebid_refresh')?.prebidAuction } + : {}), + ...(trustedServerEvidence?.serverAuctionTimings !== undefined + ? { serverAuctionTimings: trustedServerEvidence.serverAuctionTimings } + : {}), + ...(serverAuctionTimingOrigin !== undefined ? { serverAuctionTimingOrigin } : {}), ...(trustedServerEvidence?.requestedSlotSizes !== undefined ? { requestedSlotSizes: trustedServerEvidence.requestedSlotSizes } : {}), @@ -934,10 +1092,7 @@ export class GptDiagnosticsStore { private recordRequestIntentSource( slot: object, source: RequestIntentSource, - facts: Pick< - PendingSourceEvidence, - 'trustedServerOpportunity' | 'trustedServerAuctionId' | 'requestedSlotSizes' - > = {} + facts: Omit = {} ): void { const observedAtMs = this.now(); let intent = this.pendingRequestIntents.get(slot); @@ -1025,6 +1180,17 @@ export class GptDiagnosticsStore { return source ?? 'unattributed'; } + private auctionType( + intent: PendingRequestIntent | undefined, + trustedServerEvidence: PendingSourceEvidence | undefined + ): GptDiagnosticsAuctionType | undefined { + const trustedServerAuction = trustedServerEvidence?.auctionType; + const hasClientSideAuction = intent?.sources.get('prebid_refresh')?.prebidAuction !== undefined; + if (trustedServerAuction && hasClientSideAuction) return 'competing'; + if (hasClientSideAuction) return 'client_side'; + return trustedServerAuction; + } + private recordReplacement(record: MutableSlotRecord, cycle: MutableRequestCycle): void { const currentIndex = record.requests.indexOf(cycle); if (currentIndex <= 0) return; @@ -1093,13 +1259,15 @@ export class GptDiagnosticsStore { } } - const runtimeSlotNumber = this.nextRuntimeSlotNumber; - this.nextRuntimeSlotNumber += 1; + const runtimeSlotNumber = existingNumber ?? this.nextRuntimeSlotNumber; + if (existingNumber === undefined) { + this.nextRuntimeSlotNumber += 1; + this.slotNumbers.set(slot, runtimeSlotNumber); + } const record: MutableSlotRecord = { runtimeSlotNumber, requests: [], }; - this.slotNumbers.set(slot, runtimeSlotNumber); this.refreshSlotMetadata(record, slot); this.slots.set(runtimeSlotNumber, record); this.slotOrder.push(runtimeSlotNumber); diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 0cca444f3..db74b1b03 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -13,11 +13,19 @@ import type _pbjsDefault from 'prebid.js'; +import { + consumePublisherFirstImpressionDelivery, + FIRST_IMPRESSION_LEASE_MS, + firstImpressionClaim, + registerPublisherFirstImpressionAuctions, + releasePublisherFirstImpressionAuction, + resolveFirstImpressionElement, +} from '../../core/first_impression'; import { log } from '../../core/log'; import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; import { registerApsPrebidRenderer, validateApsRenderer } from '../aps/render'; import type { AuctionBid, AuctionEid } from '../../core/auction'; -import type { AuctionSlot, TsjsApi } from '../../core/types'; +import type { AuctionSlot, GptDiagnosticsAuctionWinner, TsjsApi } from '../../core/types'; import { PREBID_USER_ID_MODULE_REGISTRY } from './user_id_modules'; @@ -130,7 +138,8 @@ const TS_REFRESH_TARGETING_KEYS = [ ] as const; const MAX_PUBLISHER_AD_UNIT_SNAPSHOTS = 256; const MAX_PENDING_PUBLISHER_BIDS = 2048; -const PENDING_PUBLISHER_DELIVERY_TTL_MS = 5000; +const PENDING_PUBLISHER_DELIVERY_TTL_MS = FIRST_IMPRESSION_LEASE_MS; +const MANAGED_USER_IDS_SET_CONFIG_SENTINEL = '__tsManagedUserIdsSetConfigInstalled'; /** Configuration options for the Prebid integration. */ export interface PrebidNpmConfig { @@ -156,8 +165,32 @@ interface InjectedPrebidConfig { clientSideBidders?: string[]; /** GAM ad-unit-path suffixes excluded from refresh auctions. */ excludedGamAdUnitPathSuffixes?: string[]; + /** Operator-owned Prebid User ID module entries, forwarded verbatim. */ + managedUserIds?: InjectedManagedUserId[]; } +/** + * One operator-owned Prebid `userSync.userIds` entry. + * + * The server does not interpret these: `name`, `params`, and `storage` are + * whatever the operator configured, passed straight to Prebid.js. Which + * identity vendor an entry selects is a configuration choice. + */ +interface InjectedManagedUserId { + name: string; + params?: Record; + storage?: InjectedManagedUserIdStorage; +} + +interface InjectedManagedUserIdStorage { + type: 'cookie' | 'html5'; + name: string; + expires?: number; + refreshInSeconds?: number; +} + +type PrebidUserIdConfigEntry = Record & { name: string }; + interface PrebidUserIdDiagnostics { includedModules: string[]; configuredUserIdNames: string[]; @@ -192,29 +225,89 @@ export function collectBidders(adUnits: Array<{ bids?: Array<{ bidder?: string } return [...bidders]; } +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function configuredUserIdEntries(config: unknown): PrebidUserIdConfigEntry[] { + let userIds: unknown; + if (Array.isArray(config)) { + userIds = config; + } else if (isRecord(config)) { + userIds = isRecord(config.userSync) ? config.userSync.userIds : undefined; + if (!Array.isArray(userIds)) { + userIds = config.userIds; + } + } + + if (!Array.isArray(userIds)) return []; + + return userIds.filter( + (entry): entry is PrebidUserIdConfigEntry => + isRecord(entry) && typeof entry.name === 'string' && entry.name.length > 0 + ); +} + +function hasUserIdsPath(config: unknown): config is Record & { + userSync: Record & { userIds: unknown }; +} { + return ( + isRecord(config) && + isRecord(config.userSync) && + Object.prototype.hasOwnProperty.call(config.userSync, 'userIds') + ); +} + function configuredUserIdNamesFromConfig(config: unknown): string[] { - const userIds = Array.isArray(config) - ? config - : config && typeof config === 'object' - ? (( - config as { - userSync?: { userIds?: Array<{ name?: unknown }> }; - userIds?: Array<{ name?: unknown }>; - } - ).userSync?.userIds ?? (config as { userIds?: Array<{ name?: unknown }> }).userIds) - : undefined; + const userIds = configuredUserIdEntries(config); - if (!Array.isArray(userIds)) { - return []; + return [...new Set(userIds.map((entry) => entry.name))].sort(); +} + +/** + * Deep-copies a value the server injected as JSON. + * + * A spread would copy only the top level, leaving nested objects shared with + * `window.__tsjs_prebid` and with every entry built from it. `params` accepts + * arbitrary operator-authored tables, so nesting is expected. The injected + * config is serialized JSON by construction, which makes a round-trip total + * here and avoids depending on `structuredClone` availability. + */ +function cloneInjectedJson(value: T): T { + return JSON.parse(JSON.stringify(value)) as T; +} + +function managedUserIdEntry(managed: InjectedManagedUserId): PrebidUserIdConfigEntry { + // Rebuild the entry per call rather than sharing one object. Prebid retains + // whatever it receives as `submodule.config` for the life of the page, so a + // shared instance would let any mutation there leak into later builds. + const entry: PrebidUserIdConfigEntry = { name: managed.name }; + if (managed.params) { + entry.params = cloneInjectedJson(managed.params); + } + if (managed.storage) { + entry.storage = cloneInjectedJson(managed.storage); } + return entry; +} + +function withManagedUserIds( + config: PbjsConfig, + managedUserIds: InjectedManagedUserId[] +): PbjsConfig { + if (!hasUserIdsPath(config)) return config; - return [ - ...new Set( - userIds - .map((entry) => entry?.name) - .filter((name): name is string => typeof name === 'string' && name.length > 0) - ), - ].sort(); + const managedNames = new Set(managedUserIds.map((managed) => managed.name)); + const retained = configuredUserIdEntries(config.userSync.userIds).filter( + (entry) => !managedNames.has(entry.name) + ); + return { + ...config, + userSync: { + ...config.userSync, + userIds: [...retained, ...managedUserIds.map(managedUserIdEntry)], + }, + } as PbjsConfig; } function readConfiguredUserIdNames(): string[] { @@ -223,9 +316,14 @@ function readConfiguredUserIdNames(): string[] { return []; } - return configuredUserIdNamesFromConfig(getConfig('userSync.userIds')).concat( - configuredUserIdNamesFromConfig(getConfig()) - ); + try { + return configuredUserIdNamesFromConfig(getConfig('userSync.userIds')).concat( + configuredUserIdNamesFromConfig(getConfig()) + ); + } catch (error) { + log.error('[tsjs-prebid] effective User ID configuration could not be read', error); + return []; + } } /** Warn-once flag for an unstamped User ID manifest; reset by installPrebidNpm. */ @@ -361,6 +459,144 @@ export function auctionBidsToPrebidBids( // --------------------------------------------------------------------------- type PbjsConfig = Parameters[0]; +type PrebidGetConfig = (key?: string) => unknown; +type ManagedTcfConsentActivation = { acceptCmpEvents: boolean }; +type TcfApi = ( + command: string, + version: number, + callback: ((result: unknown, success: boolean) => void) | undefined, + parameter?: unknown +) => unknown; + +function activateManagedUserIdTcfConsent( + managedUserIds: InjectedManagedUserId[] | undefined, + setConfig: typeof pbjs.setConfig, + getConfig: PrebidGetConfig | undefined +): ManagedTcfConsentActivation | undefined { + const tcfApi = + typeof window === 'undefined' ? undefined : (window as { __tcfapi?: unknown }).__tcfapi; + if ( + !managedUserIds?.length || + typeof window === 'undefined' || + typeof tcfApi !== 'function' || + typeof getConfig !== 'function' + ) { + return undefined; + } + + let effectiveConsentManagement: unknown; + try { + effectiveConsentManagement = getConfig.call(pbjs, 'consentManagement'); + } catch (error) { + log.error('[tsjs-prebid] effective consentManagement configuration could not be read', error); + return undefined; + } + + if (effectiveConsentManagement !== undefined && !isRecord(effectiveConsentManagement)) { + log.error('[tsjs-prebid] effective consentManagement configuration is not mergeable'); + return undefined; + } + + const activation: ManagedTcfConsentActivation = { acceptCmpEvents: true }; + try { + const effectiveConsent = effectiveConsentManagement ?? {}; + if (Object.prototype.hasOwnProperty.call(effectiveConsent, 'gdpr')) { + return undefined; + } + + const originalTcfApi = tcfApi as TcfApi; + // Prebid owns the callback once it subscribes. Guard only the subscription + // created by this automatic activation so a delayed first CMP response + // cannot overwrite consent after publisher ownership transfers. + const guardedTcfApi: TcfApi = function (command, version, callback, parameter) { + if (command !== 'addEventListener' || typeof callback !== 'function') { + return originalTcfApi.call(window, command, version, callback, parameter); + } + + const guardedCallback = (result: unknown, success: boolean) => { + if (activation.acceptCmpEvents) { + callback(result, success); + return; + } + + try { + const listenerId = isRecord(result) ? result.listenerId : undefined; + if (listenerId !== undefined && listenerId !== null) { + // CMP bootstrap stubs are commonly replaced before callbacks drain. + // Prefer the current live API so removal does not enter a stale queue. + const currentTcfApi = (window as { __tcfapi?: unknown }).__tcfapi; + const removalTcfApi = + typeof currentTcfApi === 'function' ? (currentTcfApi as TcfApi) : originalTcfApi; + removalTcfApi.call(window, 'removeEventListener', version, () => {}, listenerId); + } + } catch (error) { + log.error( + '[tsjs-prebid] stale automatic IAB consent listener could not be removed', + error + ); + } + }; + + return originalTcfApi.call(window, command, version, guardedCallback, parameter); + }; + + const tcfWindow = window as typeof window & { __tcfapi: TcfApi }; + tcfWindow.__tcfapi = guardedTcfApi; + try { + setConfig({ + consentManagement: { + ...effectiveConsent, + gdpr: { cmpApi: 'iab' }, + }, + } as PbjsConfig); + } finally { + if (tcfWindow.__tcfapi === guardedTcfApi) tcfWindow.__tcfapi = originalTcfApi; + } + } catch (error) { + activation.acceptCmpEvents = false; + log.error( + '[tsjs-prebid] effective consentManagement configuration could not be inspected', + error + ); + return undefined; + } + + return activation; +} + +function publisherClaimsGdprOwnership(publisherConfig: PbjsConfig): boolean { + if ( + !isRecord(publisherConfig) || + !Object.prototype.hasOwnProperty.call(publisherConfig, 'consentManagement') + ) { + return false; + } + + const consentManagement = publisherConfig.consentManagement; + return ( + !isRecord(consentManagement) || Object.prototype.hasOwnProperty.call(consentManagement, 'gdpr') + ); +} + +function enableMergedPublisherGdpr(publisherConfig: PbjsConfig): PbjsConfig { + if (!isRecord(publisherConfig)) return publisherConfig; + + const consentManagement = publisherConfig.consentManagement; + if (!isRecord(consentManagement)) return publisherConfig; + + const gdpr = consentManagement.gdpr; + if (!isRecord(gdpr) || gdpr.enabled !== undefined) { + return publisherConfig; + } + + return { + ...publisherConfig, + consentManagement: { + ...consentManagement, + gdpr: { ...gdpr, enabled: true }, + }, + } as PbjsConfig; +} type TrustedServerBid = { bidder?: string; params?: Record }; type BannerSize = [number, number]; @@ -380,10 +616,19 @@ type PendingPublisherBid = { adUnitCode: string; expiresAt: number; registrationId: number; + generation: number; + element: HTMLElement; + retainUntilContextChange: boolean; + firstImpressionToken?: string; }; type PendingPublisherCode = { + adUnitCode: string; expiresAt: number; registrationId: number; + generation: number; + element: HTMLElement; + retainUntilContextChange: boolean; + firstImpressionToken?: string; }; type RemoveAdUnit = (adUnitCode?: string | string[]) => unknown; type PrebidWithRemoveAdUnit = { @@ -393,8 +638,9 @@ type PrebidWithRemoveAdUnit = { let publisherAdUnitSnapshots = new Map(); let pendingPublisherBids = new Map(); -let pendingPublisherCodes = new Map(); +let pendingPublisherCodes = new Map>(); let pendingPublisherRegistrationId = 0; +let publisherFirstImpressionTokens = new Map>(); let syntheticRefreshAdUnits = new WeakSet(); type TrustedServerBidRequest = { adUnitCode?: string; @@ -419,6 +665,7 @@ type RefreshGptSlot = { getSlotElementId?: () => string; getAdUnitPath?: () => string; getTargeting?: (key: string) => string[]; + setTargeting?: (key: string, value: string | string[]) => RefreshGptSlot; clearTargeting?: (key?: string) => RefreshGptSlot; getSizes?: () => unknown[]; }; @@ -431,6 +678,135 @@ function recordPrebidRefreshForDiagnostics(slots: RefreshGptSlot[]): void { } } +const MAX_PREBID_DIAGNOSTIC_ATTEMPTS = 128; +const PREBID_DIAGNOSTIC_WINDOW_MS = 30_000; + +interface PrebidDiagnosticAttempt { + slot: RefreshGptSlot; + generation: number; + expiresAtMs: number; +} + +const prebidDiagnosticAttempts = new Map(); + +function prebidDiagnosticKey(auctionId: string, adUnitCode: string): string { + return `${auctionId}\u0000${adUnitCode}`; +} + +function boundedTargetingValue( + slot: RefreshGptSlot, + key: string, + maxBytes: number +): string | undefined { + let values: string[] | undefined; + try { + values = slot.getTargeting?.(key); + } catch { + return undefined; + } + if (!Array.isArray(values) || values.length !== 1) return undefined; + const value = values[0]?.trim(); + if (!value || new TextEncoder().encode(value).length > maxBytes) return undefined; + return value; +} + +function targetingCandidate(slot: RefreshGptSlot): GptDiagnosticsAuctionWinner | undefined { + const bidder = boundedTargetingValue(slot, 'hb_bidder', 128); + const priceBucket = boundedTargetingValue(slot, 'hb_pb', 64); + if (!bidder || !priceBucket || !/^\d+(?:\.\d+)?$/.test(priceBucket)) return undefined; + const suppliedCurrency = boundedTargetingValue(slot, 'hb_cur', 3)?.toUpperCase(); + return { + bidder, + priceBucket, + ...(suppliedCurrency && /^[A-Z]{3}$/.test(suppliedCurrency) + ? { currency: suppliedCurrency } + : {}), + }; +} + +function recordCompletedPrebidAuction( + rawAuctionId: unknown, + auctionSlots: RefreshGptSlot[], + adUnitCodes: string[] +): void { + const recorder = window.tsjs?.gptDiagnosticsRecorder; + if (!recorder || typeof rawAuctionId !== 'string') return; + const auctionId = rawAuctionId; + if ( + !auctionId || + auctionId !== auctionId.trim() || + new TextEncoder().encode(auctionId).length > 256 + ) + return; + if (auctionSlots.length !== adUnitCodes.length) return; + installPrebidWinDiagnostics(); + const counts = new Map(); + for (const code of adUnitCodes) counts.set(code, (counts.get(code) ?? 0) + 1); + const nowMs = performance.now(); + const generation = window.tsjs?.navGeneration ?? 0; + for (let index = 0; index < auctionSlots.length; index += 1) { + const slot = auctionSlots[index]; + const code = adUnitCodes[index]; + if (!slot || !code || counts.get(code) !== 1) continue; + try { + recorder.recordPrebidAuction(slot, auctionId, targetingCandidate(slot)); + } catch { + // Diagnostics must not suppress the GAM request. + } + const key = prebidDiagnosticKey(auctionId, code); + prebidDiagnosticAttempts.delete(key); + prebidDiagnosticAttempts.set(key, { + slot, + generation, + expiresAtMs: nowMs + PREBID_DIAGNOSTIC_WINDOW_MS, + }); + while (prebidDiagnosticAttempts.size > MAX_PREBID_DIAGNOSTIC_ATTEMPTS) { + const oldest = prebidDiagnosticAttempts.keys().next().value; + if (oldest === undefined) break; + prebidDiagnosticAttempts.delete(oldest); + } + } +} + +function installPrebidWinDiagnostics(): void { + const diagnosticPbjs = pbjs as PbjsGlobal & { __tsDiagnosticsBidWonInstalled?: boolean }; + if (diagnosticPbjs.__tsDiagnosticsBidWonInstalled || typeof pbjs.onEvent !== 'function') return; + diagnosticPbjs.__tsDiagnosticsBidWonInstalled = true; + pbjs.onEvent('bidWon', (rawBid: unknown) => { + if (typeof rawBid !== 'object' || rawBid === null) return; + const bid = rawBid as Record; + const auctionId = typeof bid.auctionId === 'string' ? bid.auctionId : undefined; + const adUnitCode = typeof bid.adUnitCode === 'string' ? bid.adUnitCode : undefined; + if (!auctionId || !adUnitCode) return; + const key = prebidDiagnosticKey(auctionId, adUnitCode); + const attempt = prebidDiagnosticAttempts.get(key); + prebidDiagnosticAttempts.delete(key); + if ( + !attempt || + performance.now() > attempt.expiresAtMs || + (window.tsjs?.navGeneration ?? 0) !== attempt.generation + ) { + return; + } + const adserverTargeting = + typeof bid.adserverTargeting === 'object' && bid.adserverTargeting !== null + ? (bid.adserverTargeting as Record) + : {}; + const winner = targetingCandidate({ + getTargeting: (targetingKey) => { + const value = adserverTargeting[targetingKey]; + return typeof value === 'string' ? [value] : []; + }, + }); + if (!winner) return; + try { + window.tsjs?.gptDiagnosticsRecorder?.recordPrebidWin(attempt.slot, auctionId, winner); + } catch { + // Diagnostics must not alter delivery. + } + }); +} + function dispatchPrebidRefresh( refresh: (slots?: unknown[], opts?: unknown) => T, slots: unknown[] | undefined, @@ -827,41 +1203,179 @@ function clearRefreshTargeting(slot: RefreshGptSlot): void { } } +function restoreTrustedServerFirstImpressionTargeting(slot: RefreshGptSlot): void { + const ts = window.tsjs; + const injectedSlot = findInjectedSlotForRefresh(slot); + const element = [refreshSlotElementId(slot), injectedSlot?.div_id] + .filter((elementId): elementId is string => Boolean(elementId)) + .map((elementId) => document.getElementById(elementId)) + .find((candidate): candidate is HTMLElement => + Boolean(candidate && ts && firstImpressionClaim(ts, candidate)?.owner === 'trusted_server') + ); + const claim = ts && element ? firstImpressionClaim(ts, element) : undefined; + if (claim?.owner !== 'trusted_server' || !claim.targeting || !slot.setTargeting) return; + clearRefreshTargeting(slot); + for (const [key, value] of Object.entries(claim.targeting)) slot.setTargeting(key, value); +} + +/** Track a first-impression token until its exact auction is consumed or abandoned. */ +function trackPublisherFirstImpressionToken(adUnitCode: string, token: string): void { + const tokens = publisherFirstImpressionTokens.get(adUnitCode) ?? new Set(); + tokens.add(token); + publisherFirstImpressionTokens.set(adUnitCode, tokens); +} + +function forgetPublisherFirstImpressionToken(adUnitCode: string, token?: string): void { + const tokens = publisherFirstImpressionTokens.get(adUnitCode); + if (!tokens) return; + if (token === undefined) { + if (window.tsjs) { + for (const current of tokens) releasePublisherFirstImpressionAuction(window.tsjs, current); + } + publisherFirstImpressionTokens.delete(adUnitCode); + return; + } + tokens.delete(token); + if (tokens.size === 0) publisherFirstImpressionTokens.delete(adUnitCode); +} + /** Remove pending delivery state for an ad unit, optionally from one registration only. */ function removePendingPublisherBidsForCode(adUnitCode: string, registrationId?: number): void { - const pendingCode = pendingPublisherCodes.get(adUnitCode); - if (registrationId !== undefined && pendingCode?.registrationId !== registrationId) return; + const registrations = pendingPublisherCodes.get(adUnitCode); + if (registrations) { + if (registrationId === undefined) { + pendingPublisherCodes.delete(adUnitCode); + } else { + const pending = registrations.get(registrationId); + if (!pending?.retainUntilContextChange) registrations.delete(registrationId); + if (registrations.size === 0) pendingPublisherCodes.delete(adUnitCode); + } + } - pendingPublisherCodes.delete(adUnitCode); for (const [adId, pendingBid] of pendingPublisherBids) { if ( pendingBid.adUnitCode === adUnitCode && - (registrationId === undefined || pendingBid.registrationId === registrationId) + (registrationId === undefined || pendingBid.registrationId === registrationId) && + (registrationId === undefined || !pendingBid.retainUntilContextChange) ) { pendingPublisherBids.delete(adId); + if (pendingBid.firstImpressionToken) { + forgetPublisherFirstImpressionToken(adUnitCode, pendingBid.firstImpressionToken); + } + } + } +} + +function removeConsumedPublisherRegistration(adUnitCode: string, registrationId: number): void { + const registrations = pendingPublisherCodes.get(adUnitCode); + const pendingCode = registrations?.get(registrationId); + registrations?.delete(registrationId); + if (registrations?.size === 0) pendingPublisherCodes.delete(adUnitCode); + + const tokens = new Set(); + if (pendingCode?.firstImpressionToken) tokens.add(pendingCode.firstImpressionToken); + for (const [adId, pendingBid] of pendingPublisherBids) { + if (pendingBid.adUnitCode !== adUnitCode || pendingBid.registrationId !== registrationId) { + continue; } + pendingPublisherBids.delete(adId); + if (pendingBid.firstImpressionToken) tokens.add(pendingBid.firstImpressionToken); } + for (const token of tokens) forgetPublisherFirstImpressionToken(adUnitCode, token); +} + +function pendingPublisherContextIsCurrent( + pending: PendingPublisherBid | PendingPublisherCode +): boolean { + return ( + pending.generation === (window.tsjs?.navGeneration ?? 0) && + pending.element.isConnected && + document.getElementById(pending.element.id) === pending.element && + resolvePublisherDeliveryElement(pending.adUnitCode) === pending.element + ); +} + +function resolvePublisherDeliveryElement(adUnitCode: string): HTMLElement | undefined { + const direct = resolveFirstImpressionElement(adUnitCode); + if (direct) return direct; + + const gpt = ( + window as unknown as { + googletag?: { pubads?(): { getSlots?(): RefreshGptSlot[] } }; + } + ).googletag; + const matches = (gpt?.pubads?.().getSlots?.() ?? []) + .filter((slot) => { + const injectedSlot = findInjectedSlotForRefresh(slot); + return refreshSlotElementId(slot) === adUnitCode || injectedSlot?.div_id === adUnitCode; + }) + .map((slot) => { + const elementId = refreshSlotElementId(slot); + return elementId ? document.getElementById(elementId) : null; + }) + .filter((element): element is HTMLElement => Boolean(element?.isConnected)); + return matches.length === 1 ? matches[0] : undefined; +} + +function pendingPublisherContextMatchesSlot( + pending: PendingPublisherBid | PendingPublisherCode, + slot: RefreshGptSlot +): boolean { + if (!pendingPublisherContextIsCurrent(pending)) return false; + const injectedSlot = findInjectedSlotForRefresh(slot); + return [refreshSlotElementId(slot), injectedSlot?.div_id] + .filter((code): code is string => typeof code === 'string' && code.length > 0) + .some((code) => { + const exact = document.getElementById(code); + return ( + exact === pending.element || + Boolean(exact && (pending.element.contains(exact) || exact.contains(pending.element))) || + resolvePublisherDeliveryElement(code) === pending.element + ); + }); } /** Discard delivery state that outlived the publisher auction which created it. */ function prunePendingPublisherBids(now = Date.now()): void { - for (const [adUnitCode, pendingCode] of pendingPublisherCodes) { - if (pendingCode.expiresAt <= now) pendingPublisherCodes.delete(adUnitCode); + for (const [adUnitCode, registrations] of pendingPublisherCodes) { + for (const [registrationId, pendingCode] of registrations) { + if ( + !pendingPublisherContextIsCurrent(pendingCode) || + (pendingCode.expiresAt <= now && !pendingCode.retainUntilContextChange) + ) { + registrations.delete(registrationId); + } + } + if (registrations.size === 0) pendingPublisherCodes.delete(adUnitCode); } for (const [adId, pendingBid] of pendingPublisherBids) { - if (pendingBid.expiresAt <= now) pendingPublisherBids.delete(adId); + if ( + !pendingPublisherContextIsCurrent(pendingBid) || + (pendingBid.expiresAt <= now && !pendingBid.retainUntilContextChange) + ) { + pendingPublisherBids.delete(adId); + } } } -/** Store a short-lived pending publisher ad-unit code for delivery correlation. */ -function storePendingPublisherCode(adUnitCode: string, pendingCode: PendingPublisherCode): void { - pendingPublisherCodes.delete(adUnitCode); - pendingPublisherCodes.set(adUnitCode, pendingCode); - - if (pendingPublisherCodes.size > MAX_PENDING_PUBLISHER_BIDS) { - const oldestCode = pendingPublisherCodes.keys().next().value; - if (oldestCode !== undefined) removePendingPublisherBidsForCode(oldestCode); +/** Store a short-lived pending publisher ad-unit code without erasing overlaps. */ +function storePendingPublisherCode(pendingCode: PendingPublisherCode): void { + const registrations = pendingPublisherCodes.get(pendingCode.adUnitCode) ?? new Map(); + registrations.set(pendingCode.registrationId, pendingCode); + pendingPublisherCodes.set(pendingCode.adUnitCode, registrations); + + let registrationCount = 0; + for (const pending of pendingPublisherCodes.values()) registrationCount += pending.size; + if (registrationCount > MAX_PENDING_PUBLISHER_BIDS) { + for (const [adUnitCode, pendingRegistrations] of pendingPublisherCodes) { + const evictable = [...pendingRegistrations.values()].find( + (pending) => !pending.retainUntilContextChange + ); + if (!evictable) continue; + removePendingPublisherBidsForCode(adUnitCode, evictable.registrationId); + break; + } } } @@ -876,29 +1390,18 @@ function storePendingPublisherBid(adId: string, pendingBid: PendingPublisherBid) } } -/** Register every requested publisher code and any bid IDs returned for that auction. */ -function registerPendingPublisherBids( +function publisherResponseAdIds( publisherAdUnitCodes: Set, bidResponses: unknown -): number { - prunePendingPublisherBids(); - const registrationId = ++pendingPublisherRegistrationId; - const expiresAt = Date.now() + PENDING_PUBLISHER_DELIVERY_TTL_MS; - - for (const adUnitCode of publisherAdUnitCodes) { - removePendingPublisherBidsForCode(adUnitCode); - storePendingPublisherCode(adUnitCode, { expiresAt, registrationId }); - } - - if (!bidResponses || typeof bidResponses !== 'object' || Array.isArray(bidResponses)) { - return registrationId; - } +): Map { + const adIds = new Map(); + if (!bidResponses || typeof bidResponses !== 'object' || Array.isArray(bidResponses)) + return adIds; for (const [responseCode, responseGroup] of Object.entries(bidResponses)) { if (!responseGroup || typeof responseGroup !== 'object') continue; const bids = (responseGroup as { bids?: unknown }).bids; if (!Array.isArray(bids)) continue; - for (const bid of bids) { if (!bid || typeof bid !== 'object') continue; const response = bid as { adId?: unknown; adUnitCode?: unknown }; @@ -906,29 +1409,91 @@ function registerPendingPublisherBids( const adUnitCode = typeof response.adUnitCode === 'string' ? response.adUnitCode : responseCode; if (!adId || !adUnitCode || !publisherAdUnitCodes.has(adUnitCode)) continue; + adIds.set(adUnitCode, [...(adIds.get(adUnitCode) ?? []), adId]); + } + } + return adIds; +} - storePendingPublisherBid(adId, { adUnitCode, expiresAt, registrationId }); +/** Register every requested publisher code and any bid IDs returned for that auction. */ +function registerPendingPublisherBids( + publisherAdUnitCodes: Set, + bidResponses: unknown, + firstImpressionTokens: Map +): number { + prunePendingPublisherBids(); + const registrationId = ++pendingPublisherRegistrationId; + const expiresAt = Date.now() + PENDING_PUBLISHER_DELIVERY_TTL_MS; + const responseAdIds = publisherResponseAdIds(publisherAdUnitCodes, bidResponses); + + for (const adUnitCode of publisherAdUnitCodes) { + const element = resolvePublisherDeliveryElement(adUnitCode); + if (!element) continue; + const firstImpressionToken = firstImpressionTokens.get(adUnitCode); + const retainUntilContextChange = Boolean( + firstImpressionToken && + window.tsjs && + firstImpressionClaim(window.tsjs, element)?.owner === 'trusted_server' + ); + storePendingPublisherCode({ + adUnitCode, + expiresAt, + registrationId, + generation: window.tsjs?.navGeneration ?? 0, + element, + retainUntilContextChange, + firstImpressionToken, + }); + } + + for (const [adUnitCode, adIds] of responseAdIds) { + const element = resolvePublisherDeliveryElement(adUnitCode); + if (!element) continue; + const firstImpressionToken = firstImpressionTokens.get(adUnitCode); + const retainUntilContextChange = Boolean( + firstImpressionToken && + window.tsjs && + firstImpressionClaim(window.tsjs, element)?.owner === 'trusted_server' + ); + for (const adId of adIds) { + storePendingPublisherBid(adId, { + adUnitCode, + expiresAt, + registrationId, + generation: window.tsjs?.navGeneration ?? 0, + element, + retainUntilContextChange, + firstImpressionToken, + }); } } return registrationId; } -/** - * Partition slots by whether they belong to a pending publisher auction. - * - * A current `hb_adid` is the precise signal. When publishers intentionally - * omit that targeting, a short-lived requested-code match preserves delivery - * for no-bid and custom-targeting auctions. Without an ID, that fallback cannot - * distinguish a delayed delivery from the first independent refresh, so it may - * conservatively suppress one auction before its one-shot state is consumed. - * A non-empty unmatched ID remains independent so stale targeting cannot - * suppress a fresh auction. Every match is consumed once. - */ -function publisherDeliverySlots(targetSlots: RefreshGptSlot[]): Set { +interface PublisherDeliveryPartition { + deliverySlots: Set; + suppressedSlots: Set; +} + +/** Consume the equivalent one-shot suppression owned by the inner GPT wrapper. */ +function consumeGptPublisherRefreshSuppression(slot: RefreshGptSlot): void { + const elementId = refreshSlotElementId(slot); + const handoff = elementId ? window.tsjs?.gptSlotHandoffs?.[elementId] : undefined; + if (handoff?.suppressPublisherRefresh) handoff.suppressPublisherRefresh = false; +} + +/** Restore TS targeting and consume any equivalent GPT-wrapper handoff. */ +function prepareSuppressedPublisherSlot(slot: RefreshGptSlot): void { + restoreTrustedServerFirstImpressionTargeting(slot); + consumeGptPublisherRefreshSuppression(slot); +} + +/** Partition correlated publisher deliveries from one losing first-impression delivery. */ +function publisherDeliverySlots(targetSlots: RefreshGptSlot[]): PublisherDeliveryPartition { prunePendingPublisherBids(); const deliverySlots = new Set(); - const deliveredCodes = new Set(); + const suppressedSlots = new Set(); for (const slot of targetSlots) { const adIds = slot.getTargeting?.('hb_adid'); @@ -936,25 +1501,45 @@ function publisherDeliverySlots(targetSlots: RefreshGptSlot[]): Set typeof adId === 'string' && adId.length > 0) .map((adId) => pendingPublisherBids.get(adId)) - .find((bid): bid is PendingPublisherBid => bid !== undefined) + .find( + (bid): bid is PendingPublisherBid => + bid !== undefined && pendingPublisherContextMatchesSlot(bid, slot) + ) : undefined; const hasAdId = Array.isArray(adIds) && adIds.some((adId) => typeof adId === 'string' && adId.length > 0); const injectedSlot = findInjectedSlotForRefresh(slot); - const pendingCode = hasAdId - ? undefined - : [refreshSlotElementId(slot), injectedSlot?.div_id] + const pendingCodeCandidates = [ + ...new Map( + [refreshSlotElementId(slot), injectedSlot?.div_id] .filter((code): code is string => typeof code === 'string' && code.length > 0) - .find((code) => pendingPublisherCodes.has(code)); - const adUnitCode = pendingBid?.adUnitCode ?? pendingCode; - if (!adUnitCode) continue; + .flatMap((code) => [...(pendingPublisherCodes.get(code)?.values() ?? [])]) + .filter( + (pending) => + pendingPublisherContextMatchesSlot(pending, slot) && + (!hasAdId || pending.retainUntilContextChange) + ) + .map((pending) => [pending.registrationId, pending] as const) + ).values(), + ]; + const pendingCode = pendingCodeCandidates.length === 1 ? pendingCodeCandidates[0] : undefined; + const pending = pendingBid ?? pendingCode; + if (!pending) { + if (pendingCodeCandidates.some((candidate) => candidate.retainUntilContextChange)) { + suppressedSlots.add(slot); + } + continue; + } - deliverySlots.add(slot); - deliveredCodes.add(adUnitCode); + const suppress = + pending.firstImpressionToken && window.tsjs + ? consumePublisherFirstImpressionDelivery(window.tsjs, pending.firstImpressionToken) + : false; + removeConsumedPublisherRegistration(pending.adUnitCode, pending.registrationId); + (suppress ? suppressedSlots : deliverySlots).add(slot); } - deliveredCodes.forEach((adUnitCode) => removePendingPublisherBidsForCode(adUnitCode)); - return deliverySlots; + return { deliverySlots, suppressedSlots }; } /** Evict publisher state after Prebid removes one or more ad units. */ @@ -963,6 +1548,9 @@ function removePublisherState(adUnitCode?: string | string[]): void { publisherAdUnitSnapshots.clear(); pendingPublisherBids.clear(); pendingPublisherCodes.clear(); + for (const code of publisherFirstImpressionTokens.keys()) { + forgetPublisherFirstImpressionToken(code); + } return; } @@ -970,6 +1558,7 @@ function removePublisherState(adUnitCode?: string | string[]): void { for (const code of adUnitCodes) { publisherAdUnitSnapshots.delete(code); removePendingPublisherBidsForCode(code); + forgetPublisherFirstImpressionToken(code); } } @@ -1098,6 +1687,7 @@ export function installPrebidNpm(config?: Partial): typeof pbjs pendingPublisherBids = new Map(); pendingPublisherCodes = new Map(); pendingPublisherRegistrationId = 0; + publisherFirstImpressionTokens = new Map(); syntheticRefreshAdUnits = new WeakSet(); const prebidWithRemoveAdUnit = pbjs as unknown as PrebidWithRemoveAdUnit; @@ -1120,6 +1710,163 @@ export function installPrebidNpm(config?: Partial): typeof pbjs debug: config?.debug ?? injected?.debug, }; + const managedPbjs = pbjs as typeof pbjs & Record; + const managedUserIds = injected?.managedUserIds; + if ( + managedUserIds && + managedUserIds.length > 0 && + managedPbjs[MANAGED_USER_IDS_SET_CONFIG_SENTINEL] !== true + ) { + const originalSetConfig = pbjs.setConfig.bind(pbjs); + const prebidConfigApi = pbjs as typeof pbjs & { + mergeConfig?: typeof pbjs.setConfig; + }; + const originalMergeConfig = prebidConfigApi.mergeConfig?.bind(pbjs); + const getConfig = (pbjs as unknown as { getConfig?: PrebidGetConfig }).getConfig; + + let automaticTcfConsentActivation = activateManagedUserIdTcfConsent( + managedUserIds, + originalSetConfig, + getConfig + ); + + const retireAutomaticTcfConsent = ( + publisherConfig: PbjsConfig, + cleanupAllowed = true + ): boolean => { + if (!automaticTcfConsentActivation) return false; + + let claimsOwnership: boolean; + try { + claimsOwnership = publisherClaimsGdprOwnership(publisherConfig); + } catch (error) { + log.error( + '[tsjs-prebid] publisher consentManagement configuration could not be inspected', + error + ); + return false; + } + if (!claimsOwnership) return false; + + automaticTcfConsentActivation.acceptCmpEvents = false; + if (!cleanupAllowed) { + automaticTcfConsentActivation = undefined; + return false; + } + + let effectiveConsentManagement: unknown; + try { + effectiveConsentManagement = getConfig?.call(pbjs, 'consentManagement'); + } catch (error) { + log.error( + '[tsjs-prebid] effective consentManagement configuration could not be read', + error + ); + automaticTcfConsentActivation = undefined; + return false; + } + + if (effectiveConsentManagement !== undefined && !isRecord(effectiveConsentManagement)) { + log.error('[tsjs-prebid] effective consentManagement configuration is not mergeable'); + automaticTcfConsentActivation = undefined; + return false; + } + + let disabledConsentManagement: Record; + try { + disabledConsentManagement = { + ...(effectiveConsentManagement ?? {}), + gdpr: { enabled: false }, + }; + } catch (error) { + log.error( + '[tsjs-prebid] effective consentManagement configuration could not be inspected', + error + ); + automaticTcfConsentActivation = undefined; + return false; + } + + try { + originalSetConfig({ + consentManagement: disabledConsentManagement, + } as PbjsConfig); + automaticTcfConsentActivation = undefined; + return true; + } catch (error) { + // Prebid writes topical config before synchronously notifying + // subscribers, so a throw here may still mean cleanup took effect. + // Complete the one-way ownership transfer and use the publisher merge + // that was prepared before this cleanup attempt. + automaticTcfConsentActivation = undefined; + log.error('[tsjs-prebid] automatic IAB consent listener could not be retired', error); + return true; + } + }; + + const normalizePublisherConfig = (publisherConfig: PbjsConfig): PbjsConfig => { + try { + return withManagedUserIds(publisherConfig, managedUserIds); + } catch (error) { + // Publisher configuration is arbitrary page data: a throwing accessor + // must not break the publisher's own setConfig call. + log.error('[tsjs-prebid] managed User ID entries could not be normalized', error); + return publisherConfig; + } + }; + + pbjs.setConfig = ((publisherConfig: PbjsConfig) => { + retireAutomaticTcfConsent(publisherConfig); + return originalSetConfig(normalizePublisherConfig(publisherConfig)); + }) as typeof pbjs.setConfig; + if (originalMergeConfig) { + prebidConfigApi.mergeConfig = ((publisherConfig: PbjsConfig) => { + const normalizedConfig = normalizePublisherConfig(publisherConfig); + let mergedConfig = normalizedConfig; + let cleanupAllowed = true; + if (automaticTcfConsentActivation) { + try { + if (publisherClaimsGdprOwnership(normalizedConfig)) { + mergedConfig = enableMergedPublisherGdpr(normalizedConfig); + } + } catch (error) { + cleanupAllowed = false; + log.error( + '[tsjs-prebid] publisher consentManagement merge could not be normalized', + error + ); + } + } + const retiredAutomaticConsent = retireAutomaticTcfConsent(publisherConfig, cleanupAllowed); + return originalMergeConfig(retiredAutomaticConsent ? mergedConfig : normalizedConfig); + }) as typeof pbjs.setConfig; + } + managedPbjs[MANAGED_USER_IDS_SET_CONFIG_SENTINEL] = true; + + if (typeof getConfig === 'function') { + let effectiveUserIds: PrebidUserIdConfigEntry[] | undefined; + try { + effectiveUserIds = configuredUserIdEntries(getConfig.call(pbjs, 'userSync.userIds')); + } catch (error) { + log.error( + '[tsjs-prebid] effective User ID entries could not be read; managed User ID entries not seeded', + error + ); + } + if (effectiveUserIds) { + pbjs.setConfig({ userSync: { userIds: effectiveUserIds } } as PbjsConfig); + } + } else { + // Without getConfig the effective User ID entries cannot be read, and + // seeding the managed entries alone would silently drop every publisher + // module already configured. Leave the wrappers installed so the next + // publisher userIds call still gets the managed entries. + log.error( + '[tsjs-prebid] window.pbjs.getConfig is unavailable; managed User ID entries not seeded' + ); + } + } + auctionEndpoint = merged.endpoint ?? '/auction'; const apsRendererSupported = hasApsRendererApi(); if (apsRendererSupported) { @@ -1194,7 +1941,22 @@ export function installPrebidNpm(config?: Partial): typeof pbjs const opts = { ...(requestObj ?? {}) }; // eslint-disable-next-line @typescript-eslint/no-explicit-any - const adUnits = ((opts as any).adUnits || pbjs.adUnits || []) as TrustedServerAdUnit[]; + const explicitAdUnits = (opts as any).adUnits as TrustedServerAdUnit[] | undefined; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const requestedAdUnitCodes = Array.isArray((opts as any).adUnitCodes) + ? new Set( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ((opts as any).adUnitCodes as unknown[]).filter( + (code): code is string => typeof code === 'string' + ) + ) + : undefined; + const adUnits = (explicitAdUnits ?? (pbjs.adUnits as TrustedServerAdUnit[]) ?? []).filter( + (unit) => + explicitAdUnits !== undefined || + requestedAdUnitCodes === undefined || + requestedAdUnitCodes.has(unit.code ?? '') + ); const isSyntheticRefresh = adUnits.length > 0 && adUnits.every((unit) => syntheticRefreshAdUnits.has(unit)); const publisherAdUnitCodes = new Set( @@ -1203,6 +1965,20 @@ export function installPrebidNpm(config?: Partial): typeof pbjs .map((unit) => unit.code) .filter((code): code is string => typeof code === 'string' && code.length > 0) ); + const firstImpressionTokens = + !isSyntheticRefresh && !window.tsjs?.adInitRefreshInProgress + ? registerPublisherFirstImpressionAuctions( + (window.tsjs ??= {} as TsjsApi), + publisherAdUnitCodes + ) + : new Map(); + for (const [adUnitCode, token] of firstImpressionTokens) { + trackPublisherFirstImpressionToken(adUnitCode, token); + window.setTimeout( + () => forgetPublisherFirstImpressionToken(adUnitCode, token), + PENDING_PUBLISHER_DELIVERY_TTL_MS + ); + } // Ensure every ad unit has a trustedServer bid entry for (const unit of adUnits) { @@ -1292,7 +2068,7 @@ export function installPrebidNpm(config?: Partial): typeof pbjs syncPrebidEidsCookie(); const registrationId = isSyntheticRefresh ? undefined - : registerPendingPublisherBids(publisherAdUnitCodes, args[0]); + : registerPendingPublisherBids(publisherAdUnitCodes, args[0], firstImpressionTokens); if (typeof originalBidsBack !== 'function') return; try { @@ -1303,11 +2079,23 @@ export function installPrebidNpm(config?: Partial): typeof pbjs removePendingPublisherBidsForCode(code, registrationId) ); } + for (const [adUnitCode, token] of firstImpressionTokens) { + releasePublisherFirstImpressionAuction(window.tsjs!, token); + forgetPublisherFirstImpressionToken(adUnitCode, token); + } throw error; } }; - return originalRequestBids(opts); + try { + return originalRequestBids(opts); + } catch (error) { + for (const [adUnitCode, token] of firstImpressionTokens) { + releasePublisherFirstImpressionAuction(window.tsjs!, token); + forgetPublisherFirstImpressionToken(adUnitCode, token); + } + throw error; + } }; // Apply initial configuration @@ -1415,11 +2203,16 @@ export function installRefreshHandler(timeoutMs = 1500): void { return originalRefresh(slots, opts); } - const deliverySlots = publisherDeliverySlots(targetSlots); - const independentSlots = targetSlots.filter((slot) => !deliverySlots.has(slot)); + const { deliverySlots, suppressedSlots } = publisherDeliverySlots(targetSlots); + suppressedSlots.forEach(prepareSuppressedPublisherSlot); + const remainingSlots = targetSlots.filter((slot) => !suppressedSlots.has(slot)); + if (remainingSlots.length === 0) return; + const forwardedSlots = suppressedSlots.size > 0 ? remainingSlots : slots; + const independentSlots = remainingSlots.filter((slot) => !deliverySlots.has(slot)); if (independentSlots.length === 0) { - recordPrebidRefreshForDiagnostics(targetSlots); - return dispatchPrebidRefresh(originalRefresh, slots, opts); + remainingSlots.forEach(consumeGptPublisherRefreshSuppression); + recordPrebidRefreshForDiagnostics(remainingSlots); + return dispatchPrebidRefresh(originalRefresh, forwardedSlots, opts); } // Clear stale Trusted Server/Prebid targeting from independent slots before @@ -1433,7 +2226,29 @@ export function installRefreshHandler(timeoutMs = 1500): void { (slot) => !isExcludedFromRefreshAuction(slot, excludedGamAdUnitPathSuffixes) ); if (!auctionSlots.length) { - return originalRefresh(slots, opts); + const immediateSlotCodes = new Map(); + remainingSlots.forEach((slot) => { + const elementId = refreshSlotElementId(slot); + if (elementId) immediateSlotCodes.set(slot, elementId); + }); + const immediateTokens = registerPublisherFirstImpressionAuctions( + (window.tsjs ??= {} as TsjsApi), + immediateSlotCodes.values() + ); + const immediateSuppressedSlots = new Set(); + for (const [slot, elementId] of immediateSlotCodes) { + const token = immediateTokens.get(elementId); + if (token && window.tsjs && consumePublisherFirstImpressionDelivery(window.tsjs, token)) { + immediateSuppressedSlots.add(slot); + } + } + immediateSuppressedSlots.forEach(prepareSuppressedPublisherSlot); + const immediateSlots = remainingSlots.filter((slot) => !immediateSuppressedSlots.has(slot)); + if (immediateSlots.length === 0) return; + immediateSlots.forEach(consumeGptPublisherRefreshSuppression); + const immediateForwardedSlots = + immediateSuppressedSlots.size > 0 ? immediateSlots : forwardedSlots; + return originalRefresh(immediateForwardedSlots, opts); } const adUnits = auctionSlots.map((slot) => { @@ -1476,6 +2291,20 @@ export function installRefreshHandler(timeoutMs = 1500): void { // unrelated GPT slots whose targeting this wrapper only cleared for // `targetSlots` — leaving their next request dependent on stale state. const refreshAdUnitCodes = adUnits.map((unit) => unit.code); + const refreshTs = (window.tsjs ??= {} as TsjsApi); + const refreshGeneration = refreshTs.navGeneration ?? 0; + const delayedRefreshCodes = new Map(); + const delayedRefreshElements = new Map(); + remainingSlots.forEach((slot) => { + const elementId = refreshSlotElementId(slot); + if (elementId) delayedRefreshCodes.set(slot, elementId); + const element = elementId ? resolveFirstImpressionElement(elementId) : undefined; + if (element) delayedRefreshElements.set(slot, element); + }); + const refreshFirstImpressionTokens = registerPublisherFirstImpressionAuctions( + refreshTs, + delayedRefreshCodes.values() + ); adUnits.forEach((unit) => syntheticRefreshAdUnits.add(unit)); // Preserve GPT Single Request Architecture: when a publisher refresh @@ -1485,29 +2314,73 @@ export function installRefreshHandler(timeoutMs = 1500): void { // slots, and a late callback cannot issue a second GAM request. let completed = false; let fallbackTimer: ReturnType | undefined; - function completeRefresh(applyTargeting: boolean): void { + function completeRefresh(applyTargeting: boolean, completedAuctionId?: string): void { if (completed) return; completed = true; if (fallbackTimer !== undefined) clearTimeout(fallbackTimer); - if (applyTargeting) { + // The publisher refresh itself started before this asynchronous auction. + // Reconcile its per-slot token only when the callback is ready to issue + // GPT: TS may have won an already-overlapping first impression while the + // auction was pending, while a publisher-first token prevents TS from + // claiming the slot midway through the same refresh. + const callbackFilteredSlots = new Set(); + const callbackSuppressedSlots = new Set(); + for (const slot of remainingSlots) { + const elementId = delayedRefreshCodes.get(slot); + const token = elementId ? refreshFirstImpressionTokens.get(elementId) : undefined; + const element = delayedRefreshElements.get(slot); + const contextIsStale = Boolean( + element && + ((window.tsjs?.navGeneration ?? 0) !== refreshGeneration || + !element.isConnected || + document.getElementById(element.id) !== element) + ); + const suppress = Boolean( + token && window.tsjs && consumePublisherFirstImpressionDelivery(window.tsjs, token) + ); + if (contextIsStale) { + callbackFilteredSlots.add(slot); + } else if (suppress) { + callbackFilteredSlots.add(slot); + callbackSuppressedSlots.add(slot); + } + } + callbackSuppressedSlots.forEach(prepareSuppressedPublisherSlot); + + const completedSlots = remainingSlots.filter((slot) => !callbackFilteredSlots.has(slot)); + if (completedSlots.length === 0) return; + const completedAdUnitCodes = refreshAdUnitCodes.filter( + (_code, index) => !callbackFilteredSlots.has(auctionSlots[index]) + ); + const completedAuctionSlots = auctionSlots.filter( + (slot) => !callbackFilteredSlots.has(slot) + ); + let targetingApplied = false; + if (applyTargeting && typeof pbjs.setTargetingForGPTAsync === 'function') { try { - pbjs.setTargetingForGPTAsync?.(refreshAdUnitCodes); + pbjs.setTargetingForGPTAsync(completedAdUnitCodes); + targetingApplied = true; } catch (error) { log.error('[tsjs-prebid] refresh targeting failed', error); } } - recordPrebidRefreshForDiagnostics(targetSlots); - // Preserve the publisher's original refresh form. In particular, a bare - // GPT refresh remains bare so GPT resolves its registered slot set when - // the auction completes; the dispatch wrapper only scopes the shared - // diagnostics context around the delegated call. - dispatchPrebidRefresh(originalRefresh, slots, opts); + completedSlots.forEach(consumeGptPublisherRefreshSuppression); + recordPrebidRefreshForDiagnostics(completedSlots); + if (targetingApplied) { + recordCompletedPrebidAuction(completedAuctionId, completedAuctionSlots, completedAdUnitCodes); + } + // Preserve the publisher's original refresh form unless one losing + // first-impression slot was filtered. A delayed bare call must also + // become explicit so slots added after the auction snapshot cannot join. + const completedForwardedSlots = + slots === undefined || callbackFilteredSlots.size > 0 ? completedSlots : forwardedSlots; + dispatchPrebidRefresh(originalRefresh, completedForwardedSlots, opts); } try { pbjs.requestBids({ adUnits, - bidsBackHandler: () => completeRefresh(true), + bidsBackHandler: (_bids, _timedOut, auctionId) => completeRefresh(true, auctionId), timeout: timeoutMs, }); // A one-shot watchdog completes the GAM request even if Prebid never diff --git a/crates/trusted-server-js/lib/test/build-prebid-external.test.mjs b/crates/trusted-server-js/lib/test/build-prebid-external.test.mjs index f22717f79..1997cac95 100644 --- a/crates/trusted-server-js/lib/test/build-prebid-external.test.mjs +++ b/crates/trusted-server-js/lib/test/build-prebid-external.test.mjs @@ -79,6 +79,33 @@ describe('build-prebid-external metadata', () => { } }, 120_000); + it('builds and stamps identityLinkIdSystem when explicitly selected', async () => { + const outputDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), 'trusted-server-liveramp-prebid-build-test-') + ); + + try { + await main([ + '--adapters', + 'rubicon', + '--user-id-modules', + 'identityLinkIdSystem', + '--out', + outputDirectory, + ]); + + const manifest = JSON.parse( + fs.readFileSync(path.join(outputDirectory, 'manifest.json'), 'utf8') + ); + const bundle = fs.readFileSync(path.join(outputDirectory, manifest.filename), 'utf8'); + + expect(manifest.userIdModules).toEqual(['identityLinkIdSystem']); + expect(bundle).toContain('identityLinkIdSystem'); + } finally { + fs.rmSync(outputDirectory, { recursive: true, force: true }); + } + }, 120_000); + it('resolves relative output paths against the current working directory', () => { const parsed = parseArgs(['--adapters', 'rubicon', '--out', 'dist/prebid']); diff --git a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts index d9a92742b..c62544bac 100644 --- a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts @@ -9,8 +9,10 @@ import { APS_RENDERER_PATH, APS_RENDERER_SANDBOX, APS_RENDERING_MODE_ATTRIBUTE_NAME, + APS_RENDER_FAILED_MESSAGE, APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, + apsRenderFailureReason, apsRendererUrl, dispatchApsRendering as dispatchDefaultApsRendering, getApsPrebidRenderer, @@ -734,6 +736,26 @@ describe('direct APS rendering', () => { }); }); +describe('APS render failure reason allowlist', () => { + it('maps every reason the renderer frame and creative source can emit', () => { + expect(apsRenderFailureReason('descriptor_envelope')).toBe('aps_descriptor_envelope'); + expect(apsRenderFailureReason('source_mismatch')).toBe('aps_source_mismatch'); + expect(apsRenderFailureReason('bad_hash')).toBe('aps_bad_hash'); + expect(apsRenderFailureReason('frame_timeout')).toBe('aps_frame_timeout'); + expect(apsRenderFailureReason('amazon_script_error')).toBe('aps_runner_script_error'); + }); + + it('rejects unlisted, inherited, and non-string reasons from the cross-origin frame', () => { + expect(apsRenderFailureReason('not_a_real_reason')).toBeUndefined(); + expect(apsRenderFailureReason('__proto__')).toBeUndefined(); + expect(apsRenderFailureReason('constructor')).toBeUndefined(); + expect(apsRenderFailureReason('toString')).toBeUndefined(); + expect(apsRenderFailureReason(42)).toBeUndefined(); + expect(apsRenderFailureReason(undefined)).toBeUndefined(); + expect(apsRenderFailureReason({ toString: () => 'frame_timeout' })).toBeUndefined(); + }); +}); + describe('Universal Creative APS source', () => { it('uses the deployed dynamic renderer protocol and only creates the opaque route frame', () => { expect(APS_UNIVERSAL_CREATIVE_RENDERER_VERSION).toBeGreaterThanOrEqual(4); @@ -803,4 +825,91 @@ describe('Universal Creative APS source', () => { document.body.innerHTML = ''; } }); + + it('relays the frame failure reason to the top window for diagnostics', async () => { + const dynamicWindow = window as unknown as { + render?: (data: Record, helper: unknown, target: Window) => Promise; + }; + const relayed: unknown[] = []; + const capture = (event: MessageEvent): void => { + const data = event.data as { message?: unknown } | undefined; + if (data && data.message === APS_RENDER_FAILED_MESSAGE) relayed.push(data); + }; + window.addEventListener('message', capture); + window.eval(APS_UNIVERSAL_CREATIVE_RENDERER); + + try { + const rendered = dynamicWindow.render!( + { adId: 'example-ad-id', apsRenderer: descriptor(), rendererUrl: apsRendererUrl() }, + undefined, + window + ); + const iframe = document.body.querySelector('iframe')!; + const postMessage = vi.spyOn(iframe.contentWindow!, 'postMessage'); + iframe.dispatchEvent(new Event('load')); + const sent = postMessage.mock.calls[0][0] as { nonce: string }; + + window.dispatchEvent( + new MessageEvent('message', { + data: { + message: 'trusted-server/aps/renderer-failed', + nonce: sent.nonce, + reason: 'descriptor_envelope', + }, + source: iframe.contentWindow, + }) + ); + + await expect(rendered).rejects.toThrow(); + // `postMessage` is delivered on a later task than the rejection microtask. + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(relayed).toEqual([ + { + message: APS_RENDER_FAILED_MESSAGE, + adId: 'example-ad-id', + reason: 'descriptor_envelope', + }, + ]); + } finally { + window.removeEventListener('message', capture); + delete dynamicWindow.render; + document.body.innerHTML = ''; + } + }); + + it('reports a frame timeout when the renderer frame never acknowledges', async () => { + const dynamicWindow = window as unknown as { + render?: (data: Record, helper: unknown, target: Window) => Promise; + }; + const relayed: unknown[] = []; + const capture = (event: MessageEvent): void => { + const data = event.data as { message?: unknown } | undefined; + if (data && data.message === APS_RENDER_FAILED_MESSAGE) relayed.push(data); + }; + window.addEventListener('message', capture); + vi.useFakeTimers(); + window.eval(APS_UNIVERSAL_CREATIVE_RENDERER); + + try { + const rendered = dynamicWindow.render!( + { adId: 'timeout-ad-id', apsRenderer: descriptor(), rendererUrl: apsRendererUrl() }, + undefined, + window + ); + rendered.catch(() => {}); + document.body.querySelector('iframe')!.dispatchEvent(new Event('load')); + + // Async advance so the queued `postMessage` dispatch task also runs. + await vi.advanceTimersByTimeAsync(10_001); + + expect(relayed).toEqual([ + { message: APS_RENDER_FAILED_MESSAGE, adId: 'timeout-ad-id', reason: 'frame_timeout' }, + ]); + } finally { + vi.useRealTimers(); + window.removeEventListener('message', capture); + delete dynamicWindow.render; + document.body.innerHTML = ''; + } + }); }); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index b7186518b..113ce6f9b 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -6,6 +6,10 @@ import { resolve } from 'node:path'; import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vitest'; import envelope from '../../fixtures/aps-renderer-v1.json'; +import { + registerPublisherFirstImpressionAuctions, + resolveFirstImpressionElement, +} from '../../../src/core/first_impression'; import type { AuctionBidData, TsjsApi } from '../../../src/core/types'; import { APS_PREBID_CREATIVE_RUNNER_URL, @@ -248,6 +252,7 @@ describe('installTsAdInit', () => { const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), + clearTargeting: vi.fn().mockReturnThis(), getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), getTargeting: vi.fn().mockReturnValue([]), }; @@ -282,6 +287,105 @@ describe('installTsAdInit', () => { return { mockPubads, mockSlot }; } + it('leaves a publisher-auctioned slot untouched when delayed adInit receives no candidate', async () => { + const recordTrustedServerOpportunity = vi.fn(); + const { mockPubads, mockSlot } = configureOpportunityDiagnostics( + undefined, + recordTrustedServerOpportunity + ); + const ts = (window as TestWindow).tsjs as TsjsApi; + registerPublisherFirstImpressionAuctions(ts, ['div-atf-sidebar']); + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + ts.adInit!(); + + expect(mockSlot.setTargeting).not.toHaveBeenCalled(); + expect(mockPubads.refresh).not.toHaveBeenCalled(); + expect(recordTrustedServerOpportunity).not.toHaveBeenCalled(); + expect(ts.divToSlotId).toEqual({}); + expect(ts.prevSlotTargetingKeys).toEqual({}); + }); + + it('falls back once when a publisher auction abandons its first-impression claim', async () => { + vi.useFakeTimers(); + try { + const recordTrustedServerOpportunity = vi.fn(); + const { mockPubads, mockSlot } = configureOpportunityDiagnostics( + { hb_pb: '1.10', hb_adid: 'example-fallback-ad', adm: '
Fallback
' }, + recordTrustedServerOpportunity + ); + const ts = (window as TestWindow).tsjs as TsjsApi; + registerPublisherFirstImpressionAuctions(ts, ['div-atf-sidebar']); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + ts.adInit!(); + + expect(mockPubads.refresh).not.toHaveBeenCalled(); + vi.advanceTimersByTime(5001); + + expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); + expect(mockPubads.refresh).toHaveBeenCalledOnce(); + expect(mockPubads.refresh).toHaveBeenCalledWith([mockSlot]); + expect(recordTrustedServerOpportunity).toHaveBeenCalledOnce(); + + vi.advanceTimersByTime(10_000); + expect(mockPubads.refresh).toHaveBeenCalledOnce(); + } finally { + vi.useRealTimers(); + } + }); + + it('does not clear targeting or request again after TS claims an existing slot', async () => { + const recordTrustedServerOpportunity = vi.fn(); + const { mockPubads, mockSlot } = configureOpportunityDiagnostics( + undefined, + recordTrustedServerOpportunity + ); + const ts = (window as TestWindow).tsjs as TsjsApi; + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + + ts.adInit!(); + const clearCalls = mockSlot.clearTargeting.mock.calls.length; + const targetingCalls = mockSlot.setTargeting.mock.calls.length; + ts.adInit!(); + + expect(mockSlot.clearTargeting).toHaveBeenCalledTimes(clearCalls); + expect(mockSlot.setTargeting).toHaveBeenCalledTimes(targetingCalls); + expect(mockPubads.refresh).toHaveBeenCalledOnce(); + expect(recordTrustedServerOpportunity).toHaveBeenCalledOnce(); + }); + + it.each(['slotRequested', 'slotRenderEnded'] as const)( + 'leaves a publisher slot untouched after an earlier %s event', + async (eventName) => { + const recordTrustedServerOpportunity = vi.fn(); + const { mockPubads, mockSlot } = configureOpportunityDiagnostics( + { hb_pb: '2.00', hb_adid: 'late-page-bid', adm: '
Late
' }, + recordTrustedServerOpportunity + ); + const ts = (window as TestWindow).tsjs as TsjsApi; + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + const lifecycleListener = mockPubads.addEventListener.mock.calls.find( + ([registeredEvent]) => registeredEvent === eventName + )?.[1] as ((event: SlotRenderEvent) => void) | undefined; + expect(lifecycleListener).toBeDefined(); + lifecycleListener!({ isEmpty: false, slot: mockSlot }); + + ts.adInit!(); + + expect(mockSlot.setTargeting).not.toHaveBeenCalled(); + expect(mockPubads.refresh).not.toHaveBeenCalled(); + expect(recordTrustedServerOpportunity).not.toHaveBeenCalled(); + expect(ts.firstImpression?.slots['div-atf-sidebar']?.owner).toBe('publisher'); + expect(ts.firstImpression?.slots['div-atf-sidebar']?.phase).toBe( + eventName === 'slotRequested' ? 'requested' : 'rendered' + ); + } + ); + it.each([ [ 'inline markup', @@ -358,6 +462,65 @@ describe('installTsAdInit', () => { 'atf_sidebar_ad', 'unrenderable_candidate', 'auction-123', + undefined, + { + auctionType: 'ssat', + winner: { bidder: 'example', priceBucket: '1.00' }, + } + ); + }); + + it('snapshots auction timing with the bids before queued GPT work runs', async () => { + const recordTrustedServerOpportunity = vi.fn(); + const { mockSlot } = configureOpportunityDiagnostics( + { hb_pb: '1.00', hb_bidder: 'example', hb_adid: 'creative-1' }, + recordTrustedServerOpportunity + ); + const ts = (window as TestWindow).tsjs!; + ts.auctionDiagnostics = { auctionResolvedMs: 84 }; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + let queued: (() => void) | undefined; + const googletag = (window as TestWindow).googletag as { + cmd: { push(callback: () => void): void }; + }; + googletag.cmd.push = (callback) => { + queued = callback; + }; + + ts.adInit!(); + ts.auctionDiagnostics.auctionResolvedMs = 99; + queued?.(); + + expect(recordTrustedServerOpportunity).toHaveBeenCalledWith( + mockSlot, + 'atf_sidebar_ad', + 'unrenderable_candidate', + undefined, + undefined, + { + auctionType: 'ssat', + winner: { bidder: 'example', priceBucket: '1.00' }, + serverTimings: { auctionResolvedMs: 84 }, + } + ); + }); + + it('does not infer an SPA auction from navigation generation alone', async () => { + const recordTrustedServerOpportunity = vi.fn(); + const { mockSlot } = configureOpportunityDiagnostics(undefined, recordTrustedServerOpportunity); + (window as TestWindow).tsjs!.navGeneration = 1; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + expect(recordTrustedServerOpportunity).toHaveBeenCalledWith( + mockSlot, + 'atf_sidebar_ad', + 'no_candidate', + undefined, undefined ); }); @@ -1858,7 +2021,9 @@ describe('installTsAdInit', () => { (window as TestWindow).tsjs!.adInit!(); - expect(nativeRefresh).toHaveBeenCalledWith([mockSlot]); + // The slot already spent its first impression above. Changing GPT's + // initial-load mode must not make a repeated adInit request it again. + expect(nativeRefresh).not.toHaveBeenCalled(); nativeRefresh.mockClear(); gpt.setConfig({ disableInitialLoad: false }); @@ -1877,7 +2042,7 @@ describe('installTsAdInit', () => { (window as TestWindow).tsjs!.adInit!(); - expect(nativeRefresh).toHaveBeenCalledWith([mockSlot]); + expect(nativeRefresh).not.toHaveBeenCalled(); // A later modern call can re-enable initial load after the legacy API. nativeRefresh.mockClear(); @@ -2688,6 +2853,7 @@ describe('installTsAdInit', () => { ) ); const selectedElement = selectedIndex === null ? undefined : elements[selectedIndex]; + expect(resolveFirstImpressionElement(divId)).toBe(selectedElement); const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), @@ -3034,6 +3200,30 @@ describe('installTsRenderBridge', () => { return iframe.contentWindow!; } + function createCollapsedTrustedSlotIframe(divId = 'div-header') { + const slot = document.createElement('div'); + slot.id = divId; + const wrapper = document.createElement('div'); + wrapper.style.width = '1px'; + wrapper.style.height = '1px'; + const iframe = document.createElement('iframe'); + iframe.width = '1'; + iframe.height = '1'; + iframe.style.width = '1px'; + iframe.style.height = '1px'; + wrapper.appendChild(iframe); + slot.appendChild(wrapper); + document.body.appendChild(slot); + return { iframe, slot, source: iframe.contentWindow!, wrapper }; + } + + // The APS capability path runs on publisher Prebid ad units, so diagnostics + // resolve the GPT slot by element ID rather than through TS slot mapping. + function stubGoogletagSlot(elementId: string): void { + const slot = { getSlotElementId: () => elementId }; + vi.stubGlobal('googletag', { pubads: () => ({ getSlots: () => [slot] }) }); + } + async function captureBridgeListener(): Promise<(e: MessageEvent) => unknown> { let bridgeListener: ((e: MessageEvent) => unknown) | undefined; const origAdd = window.addEventListener.bind(window); @@ -3095,6 +3285,133 @@ describe('installTsRenderBridge', () => { expect(recordTrustedServerCreativeFailure).not.toHaveBeenCalled(); }); + it('expands an authenticated collapsed inline creative shell after response delivery', async () => { + const tsjs = (window as TestWindow).tsjs!; + tsjs.bids.homepage_header.adm = '
Fictional creative
'; + tsjs.bids.homepage_header.w = 728; + tsjs.bids.homepage_header.h = 90; + delete tsjs.bids.homepage_header.nurl; + delete tsjs.bids.homepage_header.burl; + const bridgeListener = await captureBridgeListener(); + const collapsed = createCollapsedTrustedSlotIframe(); + const postMessage = vi.fn(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage }], + source: collapsed.iframe.contentWindow!, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + + expect(postMessage).toHaveBeenCalledOnce(); + expect(collapsed.iframe.width).toBe('728'); + expect(collapsed.iframe.height).toBe('90'); + expect(collapsed.wrapper.style.width).toBe('728px'); + expect(collapsed.wrapper.style.height).toBe('90px'); + }); + + it('expands every collapsed ancestor through the authenticated slot root', async () => { + const tsjs = (window as TestWindow).tsjs!; + tsjs.bids.homepage_header.adm = '
Fictional creative
'; + tsjs.bids.homepage_header.w = 728; + tsjs.bids.homepage_header.h = 90; + delete tsjs.bids.homepage_header.nurl; + delete tsjs.bids.homepage_header.burl; + const bridgeListener = await captureBridgeListener(); + const collapsed = createCollapsedTrustedSlotIframe(); + const outerWrapper = document.createElement('div'); + outerWrapper.style.width = '1px'; + outerWrapper.style.height = '1px'; + collapsed.slot.insertBefore(outerWrapper, collapsed.wrapper); + outerWrapper.appendChild(collapsed.wrapper); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage: vi.fn() }], + source: collapsed.iframe.contentWindow!, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + + expect(collapsed.wrapper.style.width).toBe('728px'); + expect(collapsed.wrapper.style.height).toBe('90px'); + expect(outerWrapper.style.width).toBe('728px'); + expect(outerWrapper.style.height).toBe('90px'); + }); + + it.each([ + ['width', '1px', '120px', '728px', '120px'], + ['height', '640px', '1px', '640px', '90px'], + ] as const)( + 'changes only a collapsed %s on an ancestor', + async (_dimension, initialWidth, initialHeight, expectedWidth, expectedHeight) => { + const tsjs = (window as TestWindow).tsjs!; + tsjs.bids.homepage_header.adm = '
Fictional creative
'; + tsjs.bids.homepage_header.w = 728; + tsjs.bids.homepage_header.h = 90; + delete tsjs.bids.homepage_header.nurl; + delete tsjs.bids.homepage_header.burl; + const bridgeListener = await captureBridgeListener(); + const collapsed = createCollapsedTrustedSlotIframe(); + const outerWrapper = document.createElement('div'); + outerWrapper.style.width = initialWidth; + outerWrapper.style.height = initialHeight; + collapsed.slot.insertBefore(outerWrapper, collapsed.wrapper); + outerWrapper.appendChild(collapsed.wrapper); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage: vi.fn() }], + source: collapsed.iframe.contentWindow!, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + + expect(outerWrapper.style.width).toBe(expectedWidth); + expect(outerWrapper.style.height).toBe(expectedHeight); + } + ); + + it.each(['fixed', 'anchor', 'expanded', 'oversized'] as const)( + 'does not resize a %s Universal Creative shell', + async (guard) => { + const tsjs = (window as TestWindow).tsjs!; + tsjs.bids.homepage_header.adm = '
Fictional creative
'; + tsjs.bids.homepage_header.w = guard === 'oversized' ? 10_001 : 300; + tsjs.bids.homepage_header.h = 250; + delete tsjs.bids.homepage_header.nurl; + delete tsjs.bids.homepage_header.burl; + const bridgeListener = await captureBridgeListener(); + const collapsed = createCollapsedTrustedSlotIframe(); + if (guard === 'fixed') collapsed.iframe.style.position = 'fixed'; + if (guard === 'expanded') collapsed.iframe.style.width = '300px'; + if (guard === 'anchor') { + const anchor = document.createElement('ins'); + anchor.dataset.anchorStatus = 'displayed'; + collapsed.slot.insertBefore(anchor, collapsed.wrapper); + anchor.appendChild(collapsed.wrapper); + } + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage: vi.fn() }], + source: collapsed.source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + + expect(collapsed.iframe.width).toBe('1'); + expect(collapsed.iframe.height).toBe('1'); + expect(collapsed.wrapper.style.width).toBe('1px'); + expect(collapsed.wrapper.style.height).toBe('1px'); + } + ); + it('records no creative evidence for an ad ID the requesting slot does not own', async () => { const recordTrustedServerCreativeRequest = vi.fn().mockReturnValue(42); const recordTrustedServerCreativeResponse = vi.fn(); @@ -3195,7 +3512,7 @@ describe('installTsRenderBridge', () => { expect(fetchStub).not.toHaveBeenCalled(); }); - it('records response_post_failed when posting inline markup throws', async () => { + it('records response_post_failed without resizing when posting inline markup throws', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); const recordTrustedServerCreativeRequest = vi.fn().mockReturnValue(46); const recordTrustedServerCreativeResponse = vi.fn(); @@ -3209,7 +3526,7 @@ describe('installTsRenderBridge', () => { tsjs.bids.homepage_header.adm = '
Creative
'; const bridgeListener = await captureBridgeListener(); - const source = createTrustedSlotIframe(); + const collapsed = createCollapsedTrustedSlotIframe(); const stopImmediatePropagation = vi.fn(); expect(() => bridgeListener( @@ -3222,7 +3539,7 @@ describe('installTsRenderBridge', () => { }), }, ], - source, + source: collapsed.source, stopImmediatePropagation, }) as unknown as MessageEvent ) @@ -3232,6 +3549,8 @@ describe('installTsRenderBridge', () => { expect(recordTrustedServerCreativeFailure).toHaveBeenCalledTimes(1); expect(recordTrustedServerCreativeFailure).toHaveBeenCalledWith(46, 'response_post_failed'); expect(recordTrustedServerCreativeResponse).not.toHaveBeenCalled(); + expect(collapsed.iframe.width).toBe('1'); + expect(collapsed.iframe.height).toBe('1'); expect(beaconSpy).not.toHaveBeenCalled(); beaconSpy.mockRestore(); }); @@ -3252,7 +3571,8 @@ describe('installTsRenderBridge', () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); const bridgeListener = await captureBridgeListener(); - const source = createTrustedSlotIframe(); + const collapsed = createCollapsedTrustedSlotIframe(); + const source = collapsed.source; const stopSpy = vi.fn(); const portMessages: string[] = []; const fakePort = { postMessage: (message: string) => portMessages.push(message) }; @@ -3297,6 +3617,10 @@ describe('installTsRenderBridge', () => { }); expect(String(response.renderer)).not.toContain(renderer.accountId); expect(String(response.renderer)).not.toContain(renderer.aaxResponse); + expect(collapsed.iframe.width).toBe('300'); + expect(collapsed.iframe.height).toBe('250'); + expect(collapsed.wrapper.style.width).toBe('300px'); + expect(collapsed.wrapper.style.height).toBe('250px'); // Universal Creative's dynamic-renderer path evaluates the returned static // source and calls window.render(response, helper, targetWindow). Consume @@ -3339,6 +3663,7 @@ describe('installTsRenderBridge', () => { const marker = enablePublisherNativeMode(); try { + stubGoogletagSlot('div-header'); const bridgeListener = await captureBridgeListener(); const source = createTrustedSlotIframe(); const portMessages: string[] = []; @@ -3378,6 +3703,7 @@ describe('installTsRenderBridge', () => { const marker = enablePublisherNativeMode(); try { + stubGoogletagSlot('div-header'); const bridgeListener = await captureBridgeListener(); const source = createTrustedSlotIframe(); const portMessages: string[] = []; @@ -3417,7 +3743,8 @@ describe('installTsRenderBridge', () => { }; const bridgeListener = await captureBridgeListener(); - const source = createTrustedSlotIframe(); + const collapsed = createCollapsedTrustedSlotIframe(); + const source = collapsed.source; const stopSpy = vi.fn(); const portMessages: string[] = []; const event = Object.assign(new Event('message'), { @@ -3453,10 +3780,105 @@ describe('installTsRenderBridge', () => { ); expect(renderer.bidId).not.toBe(prebidAdId); expect((window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId]).toBeUndefined(); + expect(collapsed.iframe.width).toBe('300'); + expect(collapsed.iframe.height).toBe('250'); + expect(collapsed.wrapper.style.width).toBe('300px'); + expect(collapsed.wrapper.style.height).toBe('250px'); expect(fetchStub).not.toHaveBeenCalled(); foreignIframe.remove(); }); + it('records a creative attempt for a registered APS renderer so delivery is attributable', async () => { + const renderer = apsRenderer(); + const prebidAdId = 'prebid-diagnostics-ad-id'; + const recordTrustedServerOpportunity = vi.fn(); + const recordTrustedServerCreativeRequest = vi.fn(() => 7); + const recordTrustedServerCreativeResponse = vi.fn(); + const recordTrustedServerCreativeFailure = vi.fn(); + (window as TestWindow).tsjs.gptDiagnosticsRecorder = { + recordTrustedServerOpportunity, + recordTrustedServerCreativeRequest, + recordTrustedServerCreativeResponse, + recordTrustedServerCreativeFailure, + } as never; + (window as TestWindow).tsjs.apsPrebidRenderers = { + [prebidAdId]: { + adUnitCode: 'div-header', + renderer, + registeredAt: Date.now(), + expiresAt: Date.now() + 60_000, + markUsed: vi.fn(), + }, + }; + + try { + stubGoogletagSlot('div-header'); + const bridgeListener = await captureBridgeListener(); + const portMessages: string[] = []; + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: prebidAdId }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source: createTrustedSlotIframe(), + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + + expect(portMessages).toHaveLength(1); + expect(recordTrustedServerOpportunity.mock.calls[0]?.slice(1, 3)).toEqual([ + 'div-header', + 'renderable_candidate', + ]); + expect(recordTrustedServerCreativeRequest).toHaveBeenCalledWith('div-header'); + expect(recordTrustedServerCreativeResponse).toHaveBeenCalledWith(7); + expect(recordTrustedServerCreativeFailure).not.toHaveBeenCalled(); + } finally { + delete (window as TestWindow).tsjs.gptDiagnosticsRecorder; + } + }); + + it('records the tombstone reason when a consumed APS ad ID is replayed', async () => { + const renderer = apsRenderer(); + const prebidAdId = 'prebid-replayed-ad-id'; + const recordTrustedServerCreativeFailure = vi.fn(); + (window as TestWindow).tsjs.gptDiagnosticsRecorder = { + recordTrustedServerOpportunity: vi.fn(), + recordTrustedServerCreativeRequest: vi.fn(() => 11), + recordTrustedServerCreativeResponse: vi.fn(), + recordTrustedServerCreativeFailure, + } as never; + (window as TestWindow).tsjs.apsPrebidRenderers = { + [prebidAdId]: { + adUnitCode: 'div-header', + renderer, + registeredAt: Date.now(), + expiresAt: Date.now() + 60_000, + markUsed: vi.fn(), + }, + }; + + try { + stubGoogletagSlot('div-header'); + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const request = (): MessageEvent => + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: prebidAdId }), + ports: [{ postMessage: () => {} }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent; + + bridgeListener(request()); + recordTrustedServerCreativeFailure.mockClear(); + bridgeListener(request()); + + expect(recordTrustedServerCreativeFailure).toHaveBeenCalledWith(11, 'aps_consumed_tombstone'); + } finally { + delete (window as TestWindow).tsjs.gptDiagnosticsRecorder; + } + }); + it('contract test: fails a registered APS runner without a Universal Creative response or markUsed', async () => { const renderer = apsRenderer(); const prebidAdId = 'native-prebid-decline-ad-id'; @@ -3484,6 +3906,10 @@ describe('installTsRenderBridge', () => { }) as unknown as MessageEvent; bridgeListener(request); + expect( + (window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId], + 'the registered capability should be consumed before native rendering' + ).toBeUndefined(); nativeRunnerIn('div-header').runner.dispatchEvent(new Event('error')); await Promise.resolve(); await Promise.resolve(); @@ -3526,6 +3952,10 @@ describe('installTsRenderBridge', () => { }) as unknown as MessageEvent; bridgeListener(request); + expect( + (window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId], + 'the registered capability should be consumed before native rendering' + ).toBeUndefined(); expect(markUsed).not.toHaveBeenCalled(); const native = nativeRunnerIn('div-header'); native.runner.dispatchEvent(new Event('load')); @@ -3542,7 +3972,7 @@ describe('installTsRenderBridge', () => { } }); - it('uses the requesting frame to resolve a registered APS dynamic slot prefix', async () => { + it('uses the requesting frame to disambiguate a registered APS slot prefix', async () => { const renderer = apsRenderer(); const prebidAdId = 'native-dynamic-prebid-ad-id'; const markUsed = vi.fn(); @@ -3556,7 +3986,7 @@ describe('installTsRenderBridge', () => { }, }; const marker = enablePublisherNativeMode(); - const firstSource = createTrustedSlotIframe('div-native-first'); + createTrustedSlotIframe('div-native-first'); const source = createTrustedSlotIframe('div-native-second'); try { @@ -3569,6 +3999,7 @@ describe('installTsRenderBridge', () => { stopImmediatePropagation: vi.fn(), }) as unknown as MessageEvent ); + const native = nativeRunnerIn('div-native-second'); native.runner.dispatchEvent(new Event('load')); await Promise.resolve(); @@ -3576,11 +4007,7 @@ describe('installTsRenderBridge', () => { expect(native.frame.style.display).toBe(''); expect(markUsed).toHaveBeenCalledOnce(); - expect( - Array.from(document.querySelectorAll('#div-native-first iframe')).some( - (frame) => frame.contentWindow === firstSource - ) - ).toBe(true); + expect((window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId]).toBeUndefined(); } finally { marker.remove(); document.getElementById('div-native-first')?.remove(); @@ -4033,6 +4460,7 @@ describe('installTsRenderBridge', () => { }); try { + stubGoogletagSlot('div-header'); const bridgeListener = await captureBridgeListener(); const source = createTrustedSlotIframe(); const postMessage = vi.fn(); @@ -4116,6 +4544,48 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('keeps cache response and billing evidence when shell resizing throws', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + const recordTrustedServerCreativeResponse = vi.fn(); + const recordTrustedServerCreativeFailure = vi.fn(); + (window as TestWindow).tsjs!.gptDiagnosticsRecorder = { + recordTrustedServerCreativeRequest: vi.fn().mockReturnValue(55), + recordTrustedServerCreativeResponse, + recordTrustedServerCreativeFailure, + } as unknown as TsjsApi['gptDiagnosticsRecorder']; + fetchStub.mockResolvedValue({ + ok: true, + text: () => Promise.resolve(JSON.stringify({ adm: '
Creative
' })), + } as Response); + + const bridgeListener = await captureBridgeListener(); + const collapsed = createCollapsedTrustedSlotIframe(); + const postMessage = vi.fn(); + const computedStyleSpy = vi.spyOn(window, 'getComputedStyle').mockImplementation(() => { + throw new Error('style unavailable'); + }); + + try { + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage }], + source: collapsed.source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(postMessage).toHaveBeenCalledOnce(); + expect(recordTrustedServerCreativeResponse).toHaveBeenCalledWith(55); + expect(recordTrustedServerCreativeFailure).not.toHaveBeenCalled(); + expect(beaconSpy).toHaveBeenCalledTimes(2); + } finally { + computedStyleSpy.mockRestore(); + beaconSpy.mockRestore(); + } + }); + it('records only response_post_failed when posting cached markup throws', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); const recordTrustedServerCreativeRequest = vi.fn().mockReturnValue(48); @@ -4274,7 +4744,86 @@ describe('installTsRenderBridge', () => { expect(fetchStub).not.toHaveBeenCalled(); }); - it('uses the adInit-resolved div when a responsive prefix becomes ambiguous', async () => { + it('uses the requesting frame to resolve inline adm under an ambiguous prefix', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + const tsjs = (window as TestWindow).tsjs!; + tsjs.bids.homepage_header.adm = '
Prefix inline creative
'; + delete tsjs.bids.homepage_header.hb_cache_host; + delete tsjs.bids.homepage_header.hb_cache_path; + tsjs.adSlots = [ + { + id: 'homepage_header', + formats: [[728, 90]], + gam_unit_path: '/a/b/c', + div_id: 'div-inline-prefix-', + targeting: {}, + }, + ]; + tsjs.divToSlotId = {}; + createTrustedSlotIframe('div-inline-prefix-first'); + const source = createTrustedSlotIframe('div-inline-prefix-second'); + const bridgeListener = await captureBridgeListener(); + const postMessage = vi.fn(); + const stopImmediatePropagation = vi.fn(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage }], + source, + stopImmediatePropagation, + }) as unknown as MessageEvent + ); + + expect(postMessage).toHaveBeenCalledOnce(); + expect(JSON.parse(postMessage.mock.calls[0]![0])).toEqual( + expect.objectContaining({ ad: '
Prefix inline creative
' }) + ); + expect(stopImmediatePropagation).toHaveBeenCalledOnce(); + expect(fetchStub).not.toHaveBeenCalled(); + beaconSpy.mockRestore(); + }); + + it('rejects a requesting frame owned by multiple prefix candidates', async () => { + const tsjs = (window as TestWindow).tsjs!; + tsjs.bids.homepage_header.adm = '
Ambiguous inline creative
'; + tsjs.adSlots = [ + { + id: 'homepage_header', + formats: [[728, 90]], + gam_unit_path: '/a/b/c', + div_id: 'div-nested-prefix-', + targeting: {}, + }, + ]; + tsjs.divToSlotId = {}; + const outer = document.createElement('div'); + outer.id = 'div-nested-prefix-outer'; + const inner = document.createElement('div'); + inner.id = 'div-nested-prefix-inner'; + const iframe = document.createElement('iframe'); + inner.appendChild(iframe); + outer.appendChild(inner); + document.body.appendChild(outer); + const bridgeListener = await captureBridgeListener(); + const postMessage = vi.fn(); + const stopImmediatePropagation = vi.fn(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage }], + source: iframe.contentWindow, + stopImmediatePropagation, + }) as unknown as MessageEvent + ); + + expect(postMessage).not.toHaveBeenCalled(); + expect(stopImmediatePropagation).not.toHaveBeenCalled(); + expect(fetchStub).not.toHaveBeenCalled(); + }); + + it('uses the requesting frame when a responsive prefix is ambiguous', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); fetchStub.mockResolvedValue({ ok: true, @@ -4299,9 +4848,7 @@ describe('installTsRenderBridge', () => { targeting: {}, }, ]; - (window as TestWindow).tsjs!.divToSlotId = { - 'div-responsive-a': 'homepage_header', - }; + (window as TestWindow).tsjs!.divToSlotId = {}; const bridgeListener = await captureBridgeListener(); const portMessages: string[] = []; @@ -4409,7 +4956,7 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); - it('sizes a PBS Cache render from the cached bid dimensions', async () => { + it('sizes a PBS Cache render and its collapsed shell from cached bid dimensions', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); // Cached bid is 300x250 while the slot's first format is 728x90 (from the // default setup). The response must use the cached dimensions. @@ -4421,13 +4968,13 @@ describe('installTsRenderBridge', () => { const bridgeListener = await captureBridgeListener(); const portMessages: string[] = []; const fakePort = { postMessage: (s: string) => portMessages.push(s) }; - const source = createTrustedSlotIframe(); + const collapsed = createCollapsedTrustedSlotIframe(); bridgeListener( Object.assign(new Event('message'), { data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), ports: [fakePort], - source, + source: collapsed.source, stopImmediatePropagation: vi.fn(), }) as unknown as MessageEvent ); @@ -4438,6 +4985,52 @@ describe('installTsRenderBridge', () => { const parsed = JSON.parse(portMessages[0]) as PrebidResponseMessage; expect(parsed.width).toBe(300); expect(parsed.height).toBe(250); + expect(collapsed.iframe.width).toBe('300'); + expect(collapsed.iframe.height).toBe('250'); + expect(collapsed.wrapper.style.width).toBe('300px'); + expect(collapsed.wrapper.style.height).toBe('250px'); + beaconSpy.mockRestore(); + }); + + it('does not resize a stale cache response after navigation', async () => { + const recordTrustedServerCreativeResponse = vi.fn(); + (window as TestWindow).tsjs!.gptDiagnosticsRecorder = { + recordTrustedServerCreativeRequest: vi.fn().mockReturnValue(91), + recordTrustedServerCreativeResponse, + recordTrustedServerCreativeFailure: vi.fn(), + } as unknown as TsjsApi['gptDiagnosticsRecorder']; + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + let resolveText: ((body: string) => void) | undefined; + fetchStub.mockResolvedValue({ + ok: true, + text: () => + new Promise((resolve) => { + resolveText = resolve; + }), + } as Response); + const bridgeListener = await captureBridgeListener(); + const collapsed = createCollapsedTrustedSlotIframe(); + const postMessage = vi.fn(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage }], + source: collapsed.source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + await Promise.resolve(); + expect(resolveText).toBeDefined(); + (window as TestWindow).tsjs!.navGeneration = 1; + resolveText?.(JSON.stringify({ adm: '
cached
', w: 300, h: 250 })); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(postMessage).not.toHaveBeenCalled(); + expect(recordTrustedServerCreativeResponse).not.toHaveBeenCalled(); + expect(beaconSpy).not.toHaveBeenCalled(); + expect(collapsed.iframe.width).toBe('1'); + expect(collapsed.iframe.height).toBe('1'); beaconSpy.mockRestore(); }); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index ab6d646f2..57c75dba9 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -3,7 +3,8 @@ import path from 'node:path'; import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import type { TsjsApi } from '../../../src/core/types'; +import { FIRST_IMPRESSION_LEASE_MS } from '../../../src/core/first_impression'; +import type { FirstImpressionSlotClaim, TsjsApi } from '../../../src/core/types'; /** * Executable coverage for the edge-injected `gpt_bootstrap.js` — the @@ -233,6 +234,272 @@ describe('gpt_bootstrap.js fallback', () => { expect((window as TestWindow).tsjs!.gptInitialLoadDisabled).toBe(true); }); + it('keeps the bootstrap lease synchronized with the bundle contract', () => { + const bootstrapLease = /var FIRST_IMPRESSION_LEASE_MS = (\d+);/.exec(BOOTSTRAP_SOURCE); + + expect(Number(bootstrapLease?.[1])).toBe(FIRST_IMPRESSION_LEASE_MS); + }); + + it('clears the bootstrap fallback reservation when transitioned slot setup fails', () => { + vi.useFakeTimers(); + vi.setSystemTime(100); + try { + const pubads = { + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = makeGoogleTag({ + cmd: { push: (command) => command() }, + defineSlot: vi.fn(() => null), + pubads: vi.fn(() => pubads), + }); + document.body.innerHTML = '
'; + + runBootstrap(); + const ts = (window as TestWindow).tsjs!; + const element = document.getElementById('failed-bootstrap-fallback')!; + const publisherClaim: FirstImpressionSlotClaim = { + generation: 0, + slotElementId: element.id, + element, + owner: 'publisher', + phase: 'auctioning', + expiresAt: 5_100, + publisherAuctions: { + original: { + token: 'original', + adUnitCode: element.id, + expiresAt: 5_100, + suppressDelivery: false, + }, + }, + }; + ts.firstImpression = { + generation: 0, + nextToken: 1, + slots: { [element.id]: publisherClaim }, + fallbackSlots: {}, + }; + ts.adSlots = [ + { + id: 'failed-bootstrap-fallback-ad', + gam_unit_path: '/123/failed-bootstrap-fallback', + div_id: element.id, + formats: [[300, 250]], + targeting: {}, + }, + ]; + ts.bids = { 'failed-bootstrap-fallback-ad': { hb_pb: '1.00' } }; + + ts.adInit!(); + expect(ts.firstImpression.fallbackSlots[element.id]).toBe(element); + + vi.advanceTimersByTime(5_001); + + expect(ts.firstImpression.slots[element.id]).toBeUndefined(); + expect(ts.firstImpression.fallbackSlots[element.id]).toBeUndefined(); + } finally { + vi.clearAllTimers(); + vi.useRealTimers(); + } + }); + + it('retains an expired TS suppression tombstone in the persistent bootstrap listener', () => { + const queue: Array<() => void> = []; + const listeners = new Map void>(); + const pubads = { + addEventListener: vi.fn((name: string, listener: (event: never) => void) => { + listeners.set(name, listener as (event: { slot: { getSlotElementId(): string } }) => void); + }), + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = makeGoogleTag({ + cmd: queue, + pubads: vi.fn(() => pubads), + }); + document.body.innerHTML = '
'; + + runBootstrap(); + [...queue].forEach((command) => command()); + const element = document.getElementById('persistent-slot')!; + const claim: FirstImpressionSlotClaim = { + generation: 0, + slotElementId: element.id, + element, + owner: 'trusted_server', + phase: 'delivery_pending', + expiresAt: 0, + publisherAuctions: { + late: { + token: 'late', + adUnitCode: element.id, + expiresAt: 0, + suppressDelivery: true, + }, + }, + }; + (window as TestWindow).tsjs!.firstImpression = { + generation: 0, + nextToken: 1, + slots: { [element.id]: claim }, + fallbackSlots: {}, + }; + + listeners.get('slotRequested')!({ slot: { getSlotElementId: () => element.id } }); + + expect(claim.publisherAuctions.late).toBeDefined(); + expect(claim.publisherRegistrationClosed).toBe(true); + }); + + it('prunes a malformed bootstrap registry key before recording the main-document slot', () => { + const queue: Array<() => void> = []; + const listeners = new Map void>(); + const pubads = { + addEventListener: vi.fn((name: string, listener: (event: never) => void) => { + listeners.set(name, listener as (event: { slot: { getSlotElementId(): string } }) => void); + }), + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = makeGoogleTag({ + cmd: queue, + pubads: vi.fn(() => pubads), + }); + document.body.innerHTML = '
'; + + runBootstrap(); + [...queue].forEach((command) => command()); + const element = document.getElementById('malformed-bootstrap-slot')!; + const malformedClaim: FirstImpressionSlotClaim = { + generation: 0, + slotElementId: element.id, + element, + owner: 'trusted_server', + phase: 'delivery_pending', + expiresAt: Number.POSITIVE_INFINITY, + publisherAuctions: {}, + }; + (window as TestWindow).tsjs!.firstImpression = { + generation: 0, + nextToken: 0, + slots: { 'wrong-registry-key': malformedClaim }, + fallbackSlots: {}, + }; + + listeners.get('slotRequested')!({ slot: { getSlotElementId: () => element.id } }); + + const slots = (window as TestWindow).tsjs!.firstImpression!.slots; + expect(slots['wrong-registry-key']).toBeUndefined(); + expect(slots[element.id]).toEqual( + expect.objectContaining({ element, owner: 'publisher', phase: 'requested' }) + ); + }); + + it('rejects a connected same-ID bootstrap claim from a foreign document', () => { + const queue: Array<() => void> = []; + const listeners = new Map void>(); + const pubads = { + addEventListener: vi.fn((name: string, listener: (event: never) => void) => { + listeners.set(name, listener as (event: { slot: { getSlotElementId(): string } }) => void); + }), + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = makeGoogleTag({ + cmd: queue, + pubads: vi.fn(() => pubads), + }); + document.body.innerHTML = '
'; + + runBootstrap(); + [...queue].forEach((command) => command()); + const element = document.getElementById('foreign-bootstrap-slot')!; + const foreignDocument = document.implementation.createHTMLDocument('foreign'); + const foreignElement = foreignDocument.createElement('div'); + foreignElement.id = element.id; + foreignDocument.body.appendChild(foreignElement); + const foreignClaim: FirstImpressionSlotClaim = { + generation: 0, + slotElementId: element.id, + element: foreignElement, + owner: 'trusted_server', + phase: 'delivery_pending', + expiresAt: 0, + publisherAuctions: { + foreign: { + token: 'foreign', + adUnitCode: element.id, + expiresAt: 0, + suppressDelivery: true, + }, + }, + }; + (window as TestWindow).tsjs!.firstImpression = { + generation: 0, + nextToken: 1, + slots: { [element.id]: foreignClaim }, + fallbackSlots: {}, + }; + + expect(foreignElement.isConnected).toBe(true); + listeners.get('slotRequested')!({ slot: { getSlotElementId: () => element.id } }); + + const currentClaim = (window as TestWindow).tsjs!.firstImpression!.slots[element.id]; + expect(currentClaim).toEqual( + expect.objectContaining({ element, owner: 'publisher', phase: 'requested' }) + ); + expect(currentClaim!.publisherAuctions).toEqual({}); + }); + + it('refuses a 257th bootstrap lifecycle claim without evicting live claims', () => { + const queue: Array<() => void> = []; + const listeners = new Map void>(); + const pubads = { + addEventListener: vi.fn((name: string, listener: (event: never) => void) => { + listeners.set(name, listener as (event: { slot: { getSlotElementId(): string } }) => void); + }), + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = makeGoogleTag({ + cmd: queue, + pubads: vi.fn(() => pubads), + }); + + runBootstrap(); + [...queue].forEach((command) => command()); + const slots: Record = {}; + for (let index = 0; index < 256; index += 1) { + const element = document.createElement('div'); + element.id = `bounded-slot-${index}`; + document.body.appendChild(element); + slots[element.id] = { + generation: 0, + slotElementId: element.id, + element, + owner: 'publisher', + phase: 'rendered', + expiresAt: Number.POSITIVE_INFINITY, + publisherAuctions: {}, + }; + } + (window as TestWindow).tsjs!.firstImpression = { + generation: 0, + nextToken: 0, + slots, + fallbackSlots: {}, + }; + const overflow = document.createElement('div'); + overflow.id = 'bounded-slot-overflow'; + document.body.appendChild(overflow); + + listeners.get('slotRequested')!({ slot: { getSlotElementId: () => overflow.id } }); + + expect(Object.keys(slots)).toHaveLength(256); + expect(slots[overflow.id]).toBeUndefined(); + }); + it('installs fallback adInit and scheduleInitialAdInit when the bundle is absent', () => { runBootstrap(); const ts = (window as TestWindow).tsjs!; @@ -348,10 +615,14 @@ describe('gpt_bootstrap.js fallback', () => { const adInit = vi.fn(); ts.adInit = adInit; ts.bids = { live_slot: { hb_pb: '2.50' } }; + ts.auctionDiagnostics = { auctionResolvedMs: 10 }; ts.navGeneration = 1; - ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }); + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, undefined, { + auctionResolvedMs: 99, + }); expect(ts.bids).toEqual({ live_slot: { hb_pb: '2.50' } }); + expect(ts.auctionDiagnostics).toEqual({ auctionResolvedMs: 10 }); window.dispatchEvent(new Event('load')); flushFrame(); @@ -380,8 +651,10 @@ describe('gpt_bootstrap.js fallback', () => { formats: [[728, 90]] as Array<[number, number]>, }; - ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + const auctionDiagnostics = { auctionResolvedMs: 84 }; + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot], auctionDiagnostics); expect(ts.adSlots).toEqual([ssrSlot]); + expect(ts.auctionDiagnostics).toEqual(auctionDiagnostics); ts.adSlots = [liveSlot]; ts.navGeneration = 1; @@ -419,6 +692,7 @@ describe('gpt_bootstrap.js fallback', () => { gam_unit_path: '/123/atf', div_id: 'div-atf-sidebar', formats: [[300, 250]], + targeting: { ts_route: 'home' }, }, ]; ts.bids = { atf_sidebar_ad: { hb_pb: '1.00' } }; @@ -428,10 +702,71 @@ describe('gpt_bootstrap.js fallback', () => { expect(defineSlot).toHaveBeenCalledWith('/123/atf', [[300, 250]], 'div-atf-sidebar'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); + expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_route', 'home'); + expect(ts.prevSlotTargetingKeys).toEqual({ 'div-atf-sidebar': ['ts_route'] }); expect(display).toHaveBeenCalledWith('div-atf-sidebar'); expect(ts.servicesEnabled).toBe(true); }); + it('fallback adInit leaves a publisher-rendered slot untouched', () => { + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + }; + const mockPubads = { + addEventListener: vi.fn(), + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([mockSlot]), + refresh: vi.fn(), + }; + const nativeRefresh = mockPubads.refresh; + const defineSlot = vi.fn(); + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot, + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + display: vi.fn(), + }; + document.body.innerHTML = '
'; + runBootstrap(); + const ts = (window as TestWindow).tsjs!; + const element = document.getElementById('div-atf-sidebar')!; + ts.firstImpression = { + generation: 0, + nextToken: 0, + fallbackSlots: {}, + slots: { + 'div-atf-sidebar': { + generation: 0, + slotElementId: 'div-atf-sidebar', + element, + owner: 'publisher', + phase: 'rendered', + expiresAt: Number.POSITIVE_INFINITY, + publisherAuctions: {}, + }, + }, + }; + ts.adSlots = [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + }, + ]; + ts.bids = { atf_sidebar_ad: { hb_pb: '1.00' } }; + + ts.adInit!(); + + expect(mockSlot.setTargeting).not.toHaveBeenCalled(); + expect(nativeRefresh).not.toHaveBeenCalled(); + expect(defineSlot).not.toHaveBeenCalled(); + expect(ts.servicesEnabled).not.toBe(true); + }); + it('fallback adInit cancels queued work when the generation advances before the queue drains', () => { const commandQueue: Array<() => void> = []; const nativeRefresh = vi.fn(); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts index f684d7188..1b6488f73 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts @@ -612,7 +612,7 @@ describe('GPT GAM attribution bundle fallback', () => { expect(typeof win.tsjs?.adInit).toBe('function'); expect(typeof win.tsjs?.scheduleInitialAdInit).toBe('function'); expect(win.tsjs?.spaHookInstalled).toBe(true); - expect(addEventListenerSpy).toHaveBeenCalledWith('popstate', expect.any(Function)); + expect(addEventListenerSpy).toHaveBeenCalledWith('popstate', expect.any(Function), true); expect(addEventListenerSpy).toHaveBeenCalledWith('load', expect.any(Function)); expect(addEventListenerSpy).toHaveBeenCalledWith('message', expect.any(Function)); if (setConfig) { diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts index 727300aa1..03835a3c3 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/schedule_initial_ad_init.test.ts @@ -325,9 +325,13 @@ describe('scheduleInitialAdInit', () => { await flushAsync(); expect(ts.navGeneration).toBe(1); ts.bids = { live_slot: { hb_pb: '2.50' } }; + ts.auctionDiagnostics = { auctionResolvedMs: 10 }; - ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }); + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, undefined, { + auctionResolvedMs: 99, + }); expect(ts.bids).toEqual({ live_slot: { hb_pb: '2.50' } }); + expect(ts.auctionDiagnostics).toEqual({ auctionResolvedMs: 10 }); window.dispatchEvent(new Event('load')); flushFrame(); @@ -346,6 +350,7 @@ describe('scheduleInitialAdInit', () => { json: async () => ({ slots: [{ id: 's1', div_id: 'div-s1' }], bids: { s1: { hb_pb: '3.00' } }, + auctionDiagnostics: { auctionResolvedMs: 84 }, }), }); await importGptModule(); @@ -356,6 +361,7 @@ describe('scheduleInitialAdInit', () => { history.pushState({}, '', '/b'); await flushAsync(); expect(ts.bids).toEqual({ s1: { hb_pb: '3.00' } }); + expect(ts.auctionDiagnostics).toEqual({ auctionResolvedMs: 84 }); expect(adInit).toHaveBeenCalledTimes(1); ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }); @@ -364,6 +370,7 @@ describe('scheduleInitialAdInit', () => { flushFrame(); expect(ts.bids).toEqual({ s1: { hb_pb: '3.00' } }); + expect(ts.auctionDiagnostics).toEqual({ auctionResolvedMs: 84 }); expect(adInit).toHaveBeenCalledTimes(1); }); @@ -381,10 +388,18 @@ describe('scheduleInitialAdInit', () => { formats: [[728, 90]] as Array<[number, number]>, }; - ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot]); + const auctionDiagnostics = { + auctionDispatchedMs: 4, + auctionResolvedMs: 84, + auctionCommittedMs: 85, + auctionWaitMs: 80, + auctionWaitPlacement: 'pre_header' as const, + }; + ts.scheduleInitialAdInit!({ ssr_slot: { hb_pb: '1.00' } }, [ssrSlot], auctionDiagnostics); expect(ts.adSlots).toEqual([ssrSlot]); expect(ts.bids).toEqual({ ssr_slot: { hb_pb: '1.00' } }); + expect(ts.auctionDiagnostics).toEqual(auctionDiagnostics); }); it('preserves head-injected slots when initialSlots is omitted', async () => { diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 314348fa8..979b5b0c7 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -61,7 +61,7 @@ describe('installSpaAuctionHook', () => { // Drop any ad containers inserted by a test so DOM state does not leak. document.body.innerHTML = ''; // Remove this test's popstate listener(s) so they do not fire in later tests. - popstateHandlers.forEach((handler) => window.removeEventListener('popstate', handler)); + popstateHandlers.forEach((handler) => window.removeEventListener('popstate', handler, true)); popstateHandlers = []; vi.restoreAllMocks(); vi.unstubAllGlobals(); @@ -237,9 +237,9 @@ describe('installSpaAuctionHook', () => { expect(adInit).not.toHaveBeenCalled(); }); - it('runs adInit on an empty page-bids response when prior TS state exists', async () => { - // When TS touched slots on a previous navigation, an empty response still - // needs adInit() to sweep the stale TS targeting from those slots. + it('does not defer cleanup to adInit when an empty response has only prior targeting', async () => { + // Navigation clears prior targeting synchronously, so an empty response + // does not need adInit when TS owns no slots that still require destruction. fetchStub.mockResolvedValue({ ok: true, json: async () => ({ slots: [], bids: {} }), @@ -255,7 +255,103 @@ describe('installSpaAuctionHook', () => { await flushAsync(); expect(ts.adSlots).toEqual([]); - expect(adInit).toHaveBeenCalledTimes(1); + expect(adInit).not.toHaveBeenCalled(); + }); + + it('clears prior targeting before page-bids resolves without touching new publisher targeting', async () => { + let resolveFetch: ((response: Response) => void) | undefined; + fetchStub.mockImplementation( + () => + new Promise((resolve) => { + resolveFetch = resolve; + }) + ); + const element = document.createElement('div'); + element.id = 'div-route-slot'; + document.body.appendChild(element); + const clearTargeting = vi.fn(); + const gptSlot = { + addService: vi.fn().mockReturnThis(), + clearTargeting, + getSlotElementId: vi.fn().mockReturnValue(element.id), + getTargeting: vi.fn().mockReturnValue([]), + setTargeting: vi.fn().mockReturnThis(), + }; + const pubads = { + addEventListener: vi.fn(), + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([gptSlot]), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(gptSlot), + destroySlots: vi.fn(), + display: vi.fn(), + enableServices: vi.fn(), + pubads: vi.fn().mockReturnValue(pubads), + }; + + const { installSpaAuctionHook, installTsAdInit } = await importGptModule(); + installTsAdInit(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + ts.prevSlotTargetingKeys = { [element.id]: ['ts_route'] }; + ts.divToSlotId = { [element.id]: 'route_slot' }; + + history.pushState({}, '', '/publisher-route'); + + expect(clearTargeting.mock.calls.map(([key]) => key)).toEqual([ + 'hb_pb', + 'hb_bidder', + 'hb_adid', + 'hb_cache_host', + 'hb_cache_path', + 'ts_initial', + 'ts_route', + ]); + expect(ts.prevSlotTargetingKeys).toEqual({}); + expect(ts.divToSlotId).toEqual({}); + const cleanupCallCount = clearTargeting.mock.calls.length; + + ts.firstImpression = { + generation: 1, + nextToken: 0, + fallbackSlots: {}, + slots: { + [element.id]: { + generation: 1, + slotElementId: element.id, + element, + owner: 'publisher', + phase: 'auctioning', + expiresAt: Date.now() + 5000, + publisherAuctions: {}, + }, + }, + }; + gptSlot.setTargeting('hb_adid', 'publisher-current'); + resolveFetch!( + new Response( + JSON.stringify({ + slots: [ + { + id: 'route_slot', + gam_unit_path: '/123/route', + div_id: element.id, + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } } + ) + ); + await flushAsync(); + + expect(clearTargeting).toHaveBeenCalledTimes(cleanupCallCount); + expect(gptSlot.setTargeting).toHaveBeenCalledWith('hb_adid', 'publisher-current'); }); it('defers applying bids until the route ad container is inserted', async () => { diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts index abbdeca69..71e71ddb8 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/api.test.ts @@ -92,7 +92,9 @@ describe('GptDiagnosticsApiController', () => { 'subscribe', ]); expect(Object.keys(controller.recorder).sort()).toEqual([ + 'recordPrebidAuction', 'recordPrebidRefresh', + 'recordPrebidWin', 'recordTrustedServerCreativeFailure', 'recordTrustedServerCreativeRequest', 'recordTrustedServerCreativeResponse', @@ -112,7 +114,10 @@ describe('GptDiagnosticsApiController', () => { controller.recorder.recordTrustedServerOpportunity( slot, 'auction-slot-example', - 'renderable_candidate' + 'renderable_candidate', + undefined, + undefined, + { auctionType: 'ssat' } ); controller.recorder.recordPrebidRefresh(slots); const attemptId = @@ -126,7 +131,8 @@ describe('GptDiagnosticsApiController', () => { 'auction-slot-example', 'renderable_candidate', undefined, - undefined + undefined, + { auctionType: 'ssat' } ); expect(store.recordPrebidRefresh).toHaveBeenCalledTimes(1); expect(store.recordPrebidRefresh).toHaveBeenCalledWith(slots); @@ -159,6 +165,7 @@ describe('GptDiagnosticsApiController', () => { 'auction-slot-example', 'renderable_candidate', 'auction-123', + undefined, undefined ); }); @@ -225,6 +232,9 @@ describe('GptDiagnosticsApiController', () => { yieldGroupIds: [10], companyIds: [20], }, + auctionWinner: { bidder: 'example', priceBucket: '1.20' }, + serverAuctionTimings: { auctionResolvedMs: 84 }, + serverAuctionTimingOrigin: 'spa_auction' as const, trustedServerCreativeFailures: ['cache_fetch_failed' as const], }, ], @@ -280,6 +290,13 @@ describe('GptDiagnosticsApiController', () => { expect(cycle?.adManager?.companyIds).not.toBe( source.slots[0]?.requests[0]?.adManager.companyIds ); + expect(cycle?.auctionWinner).toEqual({ bidder: 'example', priceBucket: '1.20' }); + expect(cycle?.auctionWinner).not.toBe(source.slots[0]?.requests[0]?.auctionWinner); + expect(cycle?.serverAuctionTimings).toEqual({ auctionResolvedMs: 84 }); + expect(cycle?.serverAuctionTimings).not.toBe( + source.slots[0]?.requests[0]?.serverAuctionTimings + ); + expect(cycle?.serverAuctionTimingOrigin).toBe('spa_auction'); expect(snapshot.metadata).not.toBe(source.metadata); expect(snapshot.metadata.droppedAttributionIssues).toBe(2); }); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts index 329c4d299..fd635fd8b 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/badges.test.ts @@ -112,6 +112,43 @@ describe('GptDiagnosticsBadgeManager', () => { expect(layer.querySelectorAll('.tsgd-badge')).toHaveLength(1); expect(layer.querySelector('.tsgd-badge')?.dataset.runtimeSlot).toBe('1'); + expect(layer.querySelector('.tsgd-badge')?.textContent).toContain('Ad #1'); + manager.destroy(); + }); + + it('renders an accessible request-scoped control and activates its exact request', () => { + const frames: Array<() => void> = []; + const store = new GptDiagnosticsStore({ schedule: (callback) => callback() }); + const bindings = new FakeBindings(); + const element = document.createElement('div'); + document.body.append(element); + vi.spyOn(element, 'getBoundingClientRect').mockReturnValue(rectangle(10, 100, 300, 250)); + const observedSlot = slot('accessible'); + store.recordSlotRequested(observedSlot); + store.recordSlotRequested(observedSlot); + bindings.set(1, { status: 'bound' }, element, true); + const activate = vi.fn(); + const layer = document.createElement('div'); + document.body.append(layer); + const manager = new GptDiagnosticsBadgeManager(store, bindings, { + scheduleFrame: (callback) => frames.push(callback), + onActivate: activate, + }); + manager.setLayer(layer); + runFrame(frames); + + const badge = layer.querySelector('.tsgd-badge'); + expect(badge).toBeInstanceOf(HTMLButtonElement); + expect(badge?.textContent).toContain('Ad #1 · Request #2'); + expect(badge?.getAttribute('aria-label')).toContain('Ad #1, Request #2'); + badge?.click(); + expect(activate).toHaveBeenCalledWith(1, 2); + + const highlight = document.createElement('div'); + highlight.className = 'tsgd-highlight'; + layer.append(highlight); + manager.update(); + expect(layer.querySelector('.tsgd-highlight')).toBe(highlight); manager.destroy(); }); @@ -266,7 +303,7 @@ describe('GptDiagnosticsBadgeManager', () => { }, }) ).toBe( - 'Filled · Req 728×90, 970×250 · Fill 728×90 · Box 980×270\nResponse 276 ms · Render 42 ms\nViewable after 1 s' + 'Filled · Req 728×90, 970×250 · Fill 728×90 · Size filled 980×270\nGAM request → response 276 ms · GAM response → render 42 ms\nViewable after 1 s' ); expect( gptDiagnosticsBadgeTextForTest({ @@ -282,6 +319,17 @@ describe('GptDiagnosticsBadgeManager', () => { durations: {}, }) ).toBe('Filled · Req 300×250, 320×50, 728×90 +1'); + expect( + gptDiagnosticsBadgeTextForTest({ + requestNumber: 1, + isEmpty: false, + size: [1, 1], + observedSlotSize: [728, 90], + auctionType: 'ssat', + incompleteSequence: false, + durations: {}, + }) + ).toBe('Filled · SSAT · Size filled 728×90'); expect( gptDiagnosticsBadgeTextForTest({ requestNumber: 1, diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/index.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/index.test.ts index fa50d6366..6c5461815 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/index.test.ts @@ -124,7 +124,9 @@ describe('GPT diagnostics integration composition', () => { // Evidence writers live on their own channel; the operator API stays read-only. expect(Object.keys(first!).sort()).toEqual(['export', 'hide', 'show', 'snapshot', 'subscribe']); expect(Object.keys(target.tsjs!.gptDiagnosticsRecorder!).sort()).toEqual([ + 'recordPrebidAuction', 'recordPrebidRefresh', + 'recordPrebidWin', 'recordTrustedServerCreativeFailure', 'recordTrustedServerCreativeRequest', 'recordTrustedServerCreativeResponse', diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts index 41cad667a..7e59a0ec6 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/overlay.test.ts @@ -110,7 +110,7 @@ describe('GptDiagnosticsOverlay', () => { overlay.destroy(); }); - it('presents the request path and Trusted Server delivery evidence without winner claims', () => { + it('presents request-path, auction, winner, timing, and delivery evidence', () => { const frames: Array<() => void> = []; let now = 10; const store = new GptDiagnosticsStore({ @@ -125,7 +125,19 @@ describe('GptDiagnosticsOverlay', () => { responseSentSlot, 'auction-response-sent', 'renderable_candidate', - 'auction-123' + 'auction-123', + undefined, + { + auctionType: 'ssat', + winner: { bidder: 'example-bidder', priceBucket: '1.20' }, + serverTimings: { + auctionDispatchedMs: 4, + auctionResolvedMs: 84, + auctionCommittedMs: 85, + auctionWaitMs: 80, + auctionWaitPlacement: 'in_stream', + }, + } ); store.recordSlotRequested(responseSentSlot); now = 11; @@ -162,7 +174,19 @@ describe('GptDiagnosticsOverlay', () => { store.recordTrustedServerOpportunity( selectedSlot, 'auction-selected', - 'unrenderable_candidate' + 'unrenderable_candidate', + undefined, + undefined, + { + auctionType: 'trusted_server', + serverTimings: { + auctionDispatchedMs: 0, + auctionResolvedMs: 40, + auctionCommittedMs: 41, + auctionWaitMs: 40, + auctionWaitPlacement: 'pre_header', + }, + } ); store.recordPrebidRefresh([selectedSlot]); store.recordSlotRequested(selectedSlot); @@ -248,8 +272,15 @@ describe('GptDiagnosticsOverlay', () => { expect(responseSentArticle).toContain('Request path: Trusted Server direct'); expect(responseSentArticle).toContain('Request intent: 1'); expect(responseSentArticle).toContain('Trusted Server auction: auction-123'); + expect(responseSentArticle).toContain('Auction evidence: SSAT: initial-page server auction'); + expect(responseSentArticle).toContain('Server auction winner: example-bidder'); + expect(responseSentArticle).toContain('Server bid price bucket: 1.20'); + expect(responseSentArticle).toContain('Server request start → auction dispatched 4 ms'); + expect(responseSentArticle).toContain('Server request start → auction collected 84 ms'); + expect(responseSentArticle).toContain('Server request start → bids ready 85 ms'); + expect(responseSentArticle).toContain('Auction collection wait (in stream) 80 ms'); expect(responseSentArticle).toContain('Opportunity → request 0 ms'); - expect(responseSentArticle).toContain('Direct opportunity: Renderable candidate'); + expect(responseSentArticle).toContain('Server bid available; creative source present'); expect(responseSentArticle).toContain('Trusted Server creative request observed at 13 ms'); expect(responseSentArticle).toContain('Trusted Server markup response sent at 14 ms'); expect( @@ -258,7 +289,7 @@ describe('GptDiagnosticsOverlay', () => { expect(responseSentArticle).toContain('Creative bridge failure: cache fetch failed'); expect(responseSentArticle).toContain('Creative bridge failure: invalid cache payload'); expect(responseSentArticle).toContain('Creative bridge failure: response post failed'); - expect(responseSentArticle).toContain('Trusted Server selected; markup response sent to PUC'); + expect(responseSentArticle).toContain('Creative markup sent; execution not confirmed'); expect(responseSentArticle).toContain( 'Ad Manager reported line item 6543210987 · order 2345678901' ); @@ -270,48 +301,47 @@ describe('GptDiagnosticsOverlay', () => { expect(responseSentArticle).not.toMatch(/creative rendered|ad visible|pixels confirmed/i); const selectedArticle = slotArticle(root!, 'selected-slot').textContent ?? ''; - expect(selectedArticle).toContain('Request path: Competing paths'); - expect(selectedArticle).toContain('Direct opportunity: Unrenderable candidate'); + expect(selectedArticle).toContain('Request path: Multiple paths observed'); + expect(selectedArticle).toContain('Auction evidence: TS auction: SPA server auction'); + expect(selectedArticle).toContain('Server request start → auction dispatched 0 ms'); + expect(selectedArticle).toContain('Server request start → auction collected 40 ms'); + expect(selectedArticle).toContain('Server bid available; creative source incomplete'); expect(selectedArticle).toContain('Trusted Server creative request observed at 23 ms'); expect(selectedArticle).not.toContain('Trusted Server markup response sent'); - expect(selectedArticle).toContain('Trusted Server selected; no markup response confirmed'); + expect(selectedArticle).toContain( + 'Server bid selected by the creative bridge; response not confirmed' + ); const noCandidateArticle = slotArticle(root!, 'no-candidate-slot').textContent ?? ''; expect(noCandidateArticle).toContain('Request path: Trusted Server direct'); expect(noCandidateArticle).toContain('Direct opportunity: No candidate'); - expect(noCandidateArticle).toContain( - 'adInit observed no direct Trusted Server candidate for this request' - ); + expect(noCandidateArticle).toContain('No direct Trusted Server candidate'); const unattributedArticle = slotArticle(root!, 'unattributed-slot').textContent ?? ''; - expect(unattributedArticle).toContain('Request path: Unattributed'); - expect(unattributedArticle).toContain('Direct opportunity: Unknown (not observed)'); + expect(unattributedArticle).toContain('Request path: Not observed'); + expect(unattributedArticle).toContain('Direct opportunity: Not observed'); expect(unattributedArticle).toContain( - 'Delivery status unknown — required GPT or direct-candidate evidence was not observed' + 'Delivery status unknown — required evidence was not observed' ); const prebidArticle = slotArticle(root!, 'prebid-slot').textContent ?? ''; expect(prebidArticle).toContain('Request path: Prebid refresh'); - expect(prebidArticle).toContain('Direct opportunity: Unknown (not observed)'); - expect(prebidArticle).toContain( - 'Delivery status unknown — required GPT or direct-candidate evidence was not observed' - ); + expect(prebidArticle).toContain('Direct opportunity: Not observed'); + expect(prebidArticle).toContain('Delivery status unknown — required evidence was not observed'); const unconfirmedArticle = slotArticle(root!, 'unconfirmed-slot').textContent ?? ''; expect(unconfirmedArticle).toContain('Request path: Trusted Server direct'); - expect(unconfirmedArticle).toContain('Direct opportunity: Renderable candidate'); - expect(unconfirmedArticle).toContain( - 'Trusted Server candidate unconfirmed — another GAM result or a creative/bridge failure is possible' - ); + expect(unconfirmedArticle).toContain('Server bid available; creative source present'); + expect(unconfirmedArticle).toContain('Server bid available; selection not confirmed'); const pendingArticle = slotArticle(root!, 'candidate-pending-slot').textContent ?? ''; expect(pendingArticle).toContain('Request path: Trusted Server direct'); - expect(pendingArticle).toContain('Direct opportunity: Renderable candidate'); + expect(pendingArticle).toContain('Server bid available; creative source present'); expect(pendingArticle).toContain('Waiting for Trusted Server creative evidence'); const notApplicableArticle = slotArticle(root!, 'not-applicable-slot').textContent ?? ''; expect(notApplicableArticle).toContain('Request path: Trusted Server direct'); - expect(notApplicableArticle).toContain('Direct opportunity: Renderable candidate'); + expect(notApplicableArticle).toContain('Server bid available; creative source present'); expect(notApplicableArticle).not.toMatch( /Trusted Server selected|candidate unconfirmed|no direct Trusted Server candidate|Delivery status unknown|Waiting for Trusted Server creative evidence/ ); @@ -466,22 +496,35 @@ describe('GptDiagnosticsOverlay', () => { expect(root!.textContent).toContain('GPT observed'); expect(root!.textContent).toContain('callback issues'); expect(root!.textContent).toContain('attribution issues'); - expect(root!.textContent).toContain('filled-slot'); + expect(root!.textContent).toContain('Ad #1 · Request #2 · filled-slot'); expect(root!.textContent).toContain('/example/site/filled-slot'); expect(root!.textContent).toContain('Empty'); - expect(root!.textContent).toContain('Previous requests (1)'); - expect(root!.textContent).toContain('Requested slot sizes 300×250, 728×90, 320×50, 970×250'); - expect(root!.textContent).toContain('GPT-reported fill size 300×250'); - expect(root!.textContent).toContain('Observed outer slot box 320×270'); + expect(root!.textContent).toContain('Request history (1 previous)'); + expect(root!.textContent).toContain('Requested sizes 300×250, 728×90, 320×50, 970×250'); + expect(root!.textContent).toContain('GPT-reported size 300×250'); + expect(root!.textContent).toContain('Size filled 320×270'); expect(root!.textContent).toContain('Backfill yes'); expect(root!.textContent).toContain('GPT slot onload observed'); expect(root!.textContent).toContain('GPT impressionViewable observed'); - expect(root!.textContent).toContain('Request → response 10 ms'); + expect(root!.textContent).toContain('GAM request → response 10 ms'); expect(root!.textContent).toContain('GPT visibility 60%'); expect(root!.textContent).toContain('Requesting'); expect(root!.textContent).toContain('Ambiguous binding'); expect(root!.textContent).toContain('Incomplete sequence'); + expect(slotArticle(root!, 'pending-slot').textContent).toContain( + 'Delivery evidence: Not applicable' + ); + expect(slotArticle(root!, 'pending-slot').textContent).not.toContain( + 'Served bidder not confirmed' + ); + const emptySummary = slotArticle(root!, 'filled-slot').querySelector('.tsgd-group'); + expect(emptySummary?.textContent).toContain('Delivery evidence: Not applicable'); + expect(emptySummary?.textContent).not.toContain('Served bidder not confirmed'); + expect(root!.textContent).toContain('How to read this evidence'); + expect(root!.querySelector('a')?.href).toBe( + 'https://iabtechlab.github.io/trusted-server/guide/integrations/gpt-diagnostics-dictionary' + ); button(root!, 'Export JSON').click(); expect(exportSnapshot).toHaveBeenCalledTimes(1); @@ -522,6 +565,64 @@ describe('GptDiagnosticsOverlay', () => { overlay.destroy(); }); + it('reveals an exact request and locates it without mutating publisher markup', () => { + const frames: Array<() => void> = []; + const store = new GptDiagnosticsStore({ schedule: (callback) => callback() }); + const bindings = new FakeBindings(); + const publisherSlot = document.createElement('div'); + publisherSlot.id = 'locatable-slot'; + publisherSlot.className = 'publisher-class'; + publisherSlot.style.minHeight = '250px'; + document.body.append(publisherSlot); + const originalMarkup = publisherSlot.outerHTML; + const scrollIntoView = vi.fn(); + publisherSlot.scrollIntoView = scrollIntoView; + vi.spyOn(publisherSlot, 'getBoundingClientRect').mockReturnValue({ + left: 10, + top: 20, + width: 300, + height: 250, + } as DOMRect); + const observedSlot = slot('locatable-slot'); + store.recordSlotRequested(observedSlot); + store.recordSlotRequested(observedSlot); + bindings.set(1, { status: 'bound' }, publisherSlot, true); + let root: ShadowRoot | undefined; + const overlay = new GptDiagnosticsOverlay(store, bindings, { + scheduleFrame: (callback) => frames.push(callback), + onShadowRoot: (createdRoot) => { + root = createdRoot; + }, + }); + runNextFrame(frames); + runNextFrame(frames); + + overlay.selectRequest(1, 1); + runNextFrame(frames); + const selected = root?.querySelector( + '[data-runtime-slot="1"][data-request-number="1"]' + ); + expect(selected?.getAttribute('aria-current')).toBe('true'); + expect(root?.textContent).toContain('Request history (1 previous)'); + const focus = vi.spyOn(HTMLElement.prototype, 'focus'); + selected?.focus(); + const focusCallsBeforeUpdate = focus.mock.calls.length; + store.recordSlotResponseReceived(observedSlot); + runNextFrame(frames); + expect(focus.mock.calls.length).toBeGreaterThan(focusCallsBeforeUpdate); + + button(root!, 'Locate on page').click(); + expect(scrollIntoView).toHaveBeenCalledWith({ + behavior: 'auto', + block: 'center', + inline: 'nearest', + }); + runNextFrame(frames); + expect(root?.querySelector('.tsgd-highlight')).not.toBeNull(); + expect(publisherSlot.outerHTML).toBe(originalMarkup); + overlay.destroy(); + }); + it('does not remove a publisher element that collides with the host ID', async () => { const frames: Array<() => void> = []; const publisherElement = document.createElement('div'); @@ -568,14 +669,14 @@ describe('GptDiagnosticsOverlay', () => { runNextFrame(frames); const content = root!.querySelector('.tsgd-content')!; - const history = root!.querySelector('details')!; + const history = root!.querySelector('.tsgd-slot details')!; history.open = true; content.scrollTop = 42; store.recordSlotResponseReceived(diagnosticSlot); runNextFrame(frames); expect(root!.textContent).toContain('Rendered (fill unknown)'); - expect(root!.querySelector('details')?.open).toBe(true); + expect(root!.querySelector('.tsgd-slot details')?.open).toBe(true); expect(root!.querySelector('.tsgd-content')?.scrollTop).toBe(42); overlay.destroy(); }); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts index 52aef6a7f..22c9760c1 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/store.test.ts @@ -420,7 +420,7 @@ describe('GptDiagnosticsStore', () => { assertCoverageEquation(store); }); - it('evicts least-recently-active slots and re-enters only on a new request', () => { + it('evicts least-recently-active slots and preserves their number on re-entry', () => { let now = 0; const store = new GptDiagnosticsStore({ now: () => ++now }); const slots = Array.from({ length: MAX_DIAGNOSTIC_SLOTS + 1 }, (_, index) => @@ -444,7 +444,7 @@ describe('GptDiagnosticsStore', () => { store.recordSlotRequested(slots[1]); store.recordSlotResponseReceived(slots[1]); const reentered = store.snapshot().slots.find((slot) => slot.slotElementId === 'lru-1'); - expect(reentered).toMatchObject({ runtimeSlotNumber: 66 }); + expect(reentered).toMatchObject({ runtimeSlotNumber: 2 }); expect(reentered?.requests[0]).toMatchObject({ requestNumber: 2 }); expect(reentered?.requests[0].responseAtMs).toBeDefined(); expect(store.snapshot().slots).toHaveLength(MAX_DIAGNOSTIC_SLOTS); @@ -572,6 +572,7 @@ describe('GptDiagnosticsStore', () => { publisher: false, expectedPath: 'trusted_server_direct', expectedOpportunity: 'renderable_candidate', + expectedAuctionType: undefined, }, { name: 'a direct unrenderable candidate', @@ -580,6 +581,7 @@ describe('GptDiagnosticsStore', () => { publisher: false, expectedPath: 'trusted_server_direct', expectedOpportunity: 'unrenderable_candidate', + expectedAuctionType: undefined, }, { name: 'a direct request without a candidate', @@ -588,6 +590,7 @@ describe('GptDiagnosticsStore', () => { publisher: false, expectedPath: 'trusted_server_direct', expectedOpportunity: 'no_candidate', + expectedAuctionType: undefined, }, { name: 'a Prebid refresh', @@ -596,6 +599,7 @@ describe('GptDiagnosticsStore', () => { publisher: false, expectedPath: 'prebid_refresh', expectedOpportunity: undefined, + expectedAuctionType: undefined, }, { name: 'competing direct and Prebid evidence', @@ -604,6 +608,7 @@ describe('GptDiagnosticsStore', () => { publisher: false, expectedPath: 'competing', expectedOpportunity: 'renderable_candidate', + expectedAuctionType: undefined, }, { name: 'an unattributed request', @@ -612,6 +617,7 @@ describe('GptDiagnosticsStore', () => { publisher: false, expectedPath: 'unattributed', expectedOpportunity: undefined, + expectedAuctionType: undefined, }, { name: 'a publisher refresh', @@ -620,14 +626,25 @@ describe('GptDiagnosticsStore', () => { publisher: true, expectedPath: 'publisher_refresh', expectedOpportunity: undefined, + expectedAuctionType: undefined, }, { - name: 'competing Prebid and publisher evidence', + name: 'direct auction with publisher refresh evidence', + direct: 'renderable_candidate', + prebid: false, + publisher: true, + expectedPath: 'competing', + expectedOpportunity: 'renderable_candidate', + expectedAuctionType: undefined, + }, + { + name: 'client-side auction with publisher refresh evidence', direct: undefined, prebid: true, publisher: true, expectedPath: 'competing', expectedOpportunity: undefined, + expectedAuctionType: undefined, }, { name: 'competing all source evidence', @@ -636,10 +653,11 @@ describe('GptDiagnosticsStore', () => { publisher: true, expectedPath: 'competing', expectedOpportunity: 'renderable_candidate', + expectedAuctionType: undefined, }, ] as const)( 'attributes $name without inferring demand ownership', - ({ direct, prebid, publisher, expectedPath, expectedOpportunity }) => { + ({ direct, prebid, publisher, expectedPath, expectedOpportunity, expectedAuctionType }) => { const store = new GptDiagnosticsStore({ now: () => 10, defer: () => undefined }); const slot = fakeSlot('path-slot'); @@ -653,9 +671,136 @@ describe('GptDiagnosticsStore', () => { const cycle = store.snapshot().slots[0].requests[0]; expect(cycle.requestPath).toBe(expectedPath); expect(cycle.trustedServerOpportunity).toBe(expectedOpportunity); + expect(cycle.auctionType).toBe(expectedAuctionType); } ); + it('retains a completed Prebid candidate and only an exact correlated win', () => { + const store = new GptDiagnosticsStore({ now: () => 10, defer: () => undefined }); + const slot = fakeSlot('prebid-facts'); + + store.recordPrebidRefresh([slot]); + store.recordPrebidAuction(slot, 'auction-client-1', { + bidder: ' example-client ', + priceBucket: '2.40', + currency: 'eur', + }); + store.recordSlotRequested(slot); + store.recordPrebidWin(slot, 'other-auction', { + bidder: 'wrong', + priceBucket: '9.99', + }); + store.recordPrebidWin(slot, 'auction-client-1', { + bidder: ' example-client ', + priceBucket: '2.40', + currency: 'eur', + }); + + expect(store.snapshot().slots[0].requests[0]).toMatchObject({ + auctionType: 'client_side', + prebidAuction: { + auctionId: 'auction-client-1', + targetingCandidate: { + bidder: 'example-client', + priceBucket: '2.40', + currency: 'EUR', + }, + win: { bidder: 'example-client', priceBucket: '2.40', currency: 'EUR' }, + }, + }); + }); + + it('retains bounded winner and server timing facts for a Trusted Server auction', () => { + const store = new GptDiagnosticsStore({ now: () => 10, defer: () => undefined }); + const slot = fakeSlot('auction-facts'); + + store.recordTrustedServerOpportunity( + slot, + 'auction-slot', + 'renderable_candidate', + 'auction-123', + [[300, 250]], + { + auctionType: 'trusted_server', + winner: { bidder: ' example-bidder ', priceBucket: '1.20' }, + serverTimings: { + auctionDispatchedMs: 4, + auctionResolvedMs: 84, + auctionCommittedMs: 85, + auctionWaitMs: 80, + auctionWaitPlacement: 'pre_header', + }, + } + ); + store.recordSlotRequested(slot); + + expect(store.snapshot().slots[0].requests[0]).toMatchObject({ + auctionType: 'trusted_server', + auctionWinner: { bidder: 'example-bidder', priceBucket: '1.20' }, + serverAuctionTimings: { + auctionDispatchedMs: 4, + auctionResolvedMs: 84, + auctionCommittedMs: 85, + auctionWaitMs: 80, + auctionWaitPlacement: 'pre_header', + }, + }); + }); + + it('does not treat a Prebid refresh route as client-auction evidence', () => { + const store = new GptDiagnosticsStore({ now: () => 10, defer: () => undefined }); + const slot = fakeSlot('competing-spa-auction'); + + store.recordTrustedServerOpportunity( + slot, + 'auction-slot', + 'renderable_candidate', + undefined, + undefined, + { + auctionType: 'trusted_server', + serverTimings: { auctionDispatchedMs: 0, auctionResolvedMs: 84 }, + } + ); + store.recordPrebidRefresh([slot]); + store.recordSlotRequested(slot); + + expect(store.snapshot().slots[0].requests[0]).toMatchObject({ + auctionType: 'trusted_server', + serverAuctionTimingOrigin: 'spa_auction', + serverAuctionTimings: { auctionDispatchedMs: 0, auctionResolvedMs: 84 }, + }); + }); + + it('drops malformed winner and auction timing fields at the diagnostics boundary', () => { + const store = new GptDiagnosticsStore({ now: () => 10, defer: () => undefined }); + const slot = fakeSlot('invalid-auction-facts'); + + store.recordTrustedServerOpportunity( + slot, + 'auction-slot', + 'renderable_candidate', + undefined, + undefined, + { + auctionType: 'invalid' as never, + winner: { bidder: 'example-bidder', priceBucket: 'not-a-price' }, + serverTimings: { + auctionDispatchedMs: -1, + auctionResolvedMs: Number.NaN, + auctionCommittedMs: 0x1_0000_0000, + auctionWaitPlacement: 'elsewhere' as never, + }, + } + ); + store.recordSlotRequested(slot); + + const cycle = store.snapshot().slots[0].requests[0]; + expect(cycle.auctionType).toBeUndefined(); + expect(cycle.auctionWinner).toBeUndefined(); + expect(cycle.serverAuctionTimings).toBeUndefined(); + }); + it('consumes direct and Prebid markers exactly once', () => { let now = 10; const store = new GptDiagnosticsStore({ now: () => now, defer: () => undefined }); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/types.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/types.test.ts index f4f4c7486..411acf1de 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/types.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt_diagnostics/types.test.ts @@ -81,6 +81,8 @@ describe('GPT diagnostics public types', () => { expectTypeOf().toEqualTypeOf< | 'recordTrustedServerOpportunity' | 'recordPrebidRefresh' + | 'recordPrebidAuction' + | 'recordPrebidWin' | 'recordTrustedServerCreativeRequest' | 'recordTrustedServerCreativeResponse' | 'recordTrustedServerCreativeFailure' diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 0f2906d2b..9211653ae 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -27,6 +27,31 @@ const DEFAULT_BUNDLE_MANIFEST = { userIdModules: ['sharedIdSystem'], }; +// A managed User ID entry is operator configuration, forwarded verbatim. The +// module named here is sample data: nothing in the shim or the server knows +// which identity vendor `identityLink` belongs to. +const MANAGED_USER_ID = { + name: 'identityLink', + params: { pid: '999', notUse3P: false }, + storage: { + type: 'cookie' as const, + name: 'idl_env', + expires: 15, + refreshInSeconds: 1800, + }, +}; + +const EXPECTED_MANAGED_USER_ID = { + name: 'identityLink', + params: { pid: '999', notUse3P: false }, + storage: { + type: 'cookie', + name: 'idl_env', + expires: 15, + refreshInSeconds: 1800, + }, +}; + /** Loose bid shape used by the requestBids shim tests. */ interface TestBid { bidder: string; @@ -47,6 +72,7 @@ interface InjectedPrebidTestConfig { serverSideBidders?: string[]; clientSideBidders?: string[]; excludedGamAdUnitPathSuffixes?: unknown; + managedUserIds?: Array; } interface TestGoogletag { @@ -61,6 +87,7 @@ interface ApsPrebidTestEntry { interface PrebidTestWindow { pbjs?: unknown; + __tcfapi?: unknown; tsjs?: { apsPrebidRenderers?: Record; [key: string]: unknown; @@ -109,6 +136,7 @@ interface TestAdapterSpec { // of mocking module imports. const { mockSetConfig, + mockMergeConfig, mockProcessQueue, mockRequestBids, mockRegisterBidAdapter, @@ -120,6 +148,9 @@ const { mockPbjs, } = vi.hoisted(() => { const mockSetConfig = vi.fn(); + // Prebid's public mergeConfig closes over its internal setConfig rather than + // calling pbjs.setConfig, so wrapping setConfig alone cannot intercept it. + const mockMergeConfig = vi.fn((config: unknown) => mockSetConfig(config)); const mockProcessQueue = vi.fn(); const mockRequestBids = vi.fn(); const mockRegisterBidAdapter = vi.fn(); @@ -140,6 +171,7 @@ const { }); const mockPbjs: { setConfig: typeof mockSetConfig; + mergeConfig: typeof mockMergeConfig; processQueue: typeof mockProcessQueue; requestBids: typeof mockRequestBids; registerBidAdapter: typeof mockRegisterBidAdapter; @@ -152,6 +184,7 @@ const { [key: string]: unknown; } = { setConfig: mockSetConfig, + mergeConfig: mockMergeConfig, processQueue: mockProcessQueue, requestBids: mockRequestBids, registerBidAdapter: mockRegisterBidAdapter, @@ -181,6 +214,7 @@ const { return { mockSetConfig, + mockMergeConfig, mockProcessQueue, mockRequestBids, mockRegisterBidAdapter, @@ -200,8 +234,19 @@ import { installPrebidNpm, installRefreshHandler, } from '../../../src/integrations/prebid/index'; +import { installTsAdInit } from '../../../src/integrations/gpt/index'; import type { AuctionBid } from '../../../src/core/auction'; +import { + claimFirstImpressionForTrustedServer, + consumePublisherFirstImpressionDelivery, + firstImpressionClaim, + observeFirstImpressionGptLifecycle, + registerPublisherFirstImpressionAuctions, + releaseTrustedServerFirstImpressionClaim, + reservePublisherFirstImpressionFallback, +} from '../../../src/core/first_impression'; import { log } from '../../../src/core/log'; +import type { TsjsApi } from '../../../src/core/types'; import { GptDiagnosticsObserver } from '../../../src/integrations/gpt_diagnostics/observer'; import { GptDiagnosticsStore } from '../../../src/integrations/gpt_diagnostics/store'; import envelope from '../../fixtures/aps-renderer-v1.json'; @@ -210,6 +255,10 @@ import envelope from '../../fixtures/aps-renderer-v1.json'; // self-init above already set it), so every test starts from a clean page. beforeEach(() => { delete testWindow.__tsjsPrebidShimInstalled; + mockPbjs.setConfig = mockSetConfig; + mockPbjs.mergeConfig = mockMergeConfig; + mockPbjs.processQueue = mockProcessQueue; + delete mockPbjs['__tsManagedUserIdsSetConfigInstalled']; }); describe('prebid/collectBidders', () => { @@ -404,17 +453,26 @@ describe('prebid/auctionBidsToPrebidBids', () => { describe('prebid/installPrebidNpm', () => { beforeEach(() => { vi.clearAllMocks(); + mockSetConfig.mockReset(); + mockProcessQueue.mockReset(); // Reset requestBids to the mock so each test starts fresh mockPbjs.requestBids = mockRequestBids; + mockPbjs.setConfig = mockSetConfig; + mockPbjs.mergeConfig = mockMergeConfig; + mockPbjs.processQueue = mockProcessQueue; mockPbjs.adUnits = []; + mockPbjs.que = []; + mockPbjs.getConfig = mockGetConfig; mockGetUserIdsAsEids.mockReset(); mockGetUserIdsAsEids.mockReturnValue([]); mockGetConfig.mockReset(); document.cookie = 'ts-eids=; Path=/; Max-Age=0'; delete testWindow.__tsjs_prebid; delete testWindow.__tsjs_prebid_diagnostics; + delete testWindow.__tcfapi; delete testWindow.tsjs; delete mockPbjs['__tsApsBidResponseListenerInstalled']; + delete mockPbjs['__tsManagedUserIdsSetConfigInstalled']; delete mockPbjs.bidderSettings; }); @@ -829,843 +887,1949 @@ describe('prebid/installPrebidNpm', () => { expect(mockProcessQueue).toHaveBeenCalledTimes(1); }); - it('reports the User ID modules selected by the generated bundle', () => { + it('leaves the public config APIs unchanged when no User IDs are managed', () => { + const originalSetConfig = mockPbjs.setConfig; + const originalMergeConfig = mockPbjs.mergeConfig; + testWindow.__tcfapi = vi.fn(); + installPrebidNpm(); - expect(testWindow.__tsjs_prebid_diagnostics.userIdModules).toEqual({ - includedModules: ['sharedIdSystem'], - configuredUserIdNames: [], - missingConfiguredUserIdNames: [], - }); + expect(mockPbjs.setConfig).toBe(originalSetConfig); + expect(mockPbjs.mergeConfig).toBe(originalMergeConfig); + expect(mockSetConfig.mock.calls.some(([value]) => value?.userSync?.userIds)).toBe(false); }); - it('refreshes late User ID config without repeating missing-module warnings', () => { - installPrebidNpm(); + it('activates IAB GDPR consent before managed User IDs and the publisher queue', () => { + testWindow.__tcfapi = vi.fn(); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; mockGetConfig.mockImplementation((key?: string) => - key === 'userSync.userIds' ? [{ name: 'sharedId' }, { name: 'pairId' }] : {} + key === 'userSync.userIds' ? [] : undefined ); - const warnSpy = vi.spyOn(log, 'warn').mockImplementation(() => {}); - mockPbjs.requestBids({ adUnits: [] }); - mockPbjs.requestBids({ adUnits: [] }); + installPrebidNpm(); - expect(testWindow.__tsjs_prebid_diagnostics.userIdModules).toEqual({ - includedModules: ['sharedIdSystem'], - configuredUserIdNames: ['pairId', 'sharedId'], - missingConfiguredUserIdNames: ['pairId'], + const consentCallIndex = mockSetConfig.mock.calls.findIndex( + ([value]) => value?.consentManagement?.gdpr?.cmpApi === 'iab' + ); + const managedCallIndex = mockSetConfig.mock.calls.findIndex( + ([value]) => value?.userSync?.userIds + ); + expect(consentCallIndex).toBeGreaterThanOrEqual(0); + expect(mockSetConfig.mock.calls[consentCallIndex][0]).toEqual({ + consentManagement: { gdpr: { cmpApi: 'iab' } }, }); - expect( - warnSpy.mock.calls.filter(([message]) => String(message).includes('"pairId"')) - ).toHaveLength(1); + expect(consentCallIndex).toBeLessThan(managedCallIndex); + expect(mockSetConfig.mock.invocationCallOrder[consentCallIndex]).toBeLessThan( + mockProcessQueue.mock.invocationCallOrder[0] + ); }); - it('returns the pbjs instance', () => { - const result = installPrebidNpm(); - expect(result).toBe(mockPbjs); - }); + it('does not activate managed consent without a callable TCF API', () => { + testWindow.__tcfapi = true; + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => + key === 'userSync.userIds' ? [] : undefined + ); - it('installs only once per page via the __tsjsPrebidShimInstalled sentinel', () => { - const first = installPrebidNpm(); - const wrappedRequestBids = mockPbjs.requestBids; - const second = installPrebidNpm(); + installPrebidNpm(); - expect(second).toBe(first); - expect(mockRegisterBidAdapter).toHaveBeenCalledTimes(1); - expect(mockPbjs.requestBids).toBe(wrappedRequestBids); - expect(testWindow.__tsjsPrebidShimInstalled).toBe(true); + expect(mockSetConfig.mock.calls.some(([value]) => value?.consentManagement)).toBe(false); }); - it('warns once about an unstamped User ID manifest instead of once per module', () => { - delete testWindow.__tsjs_prebid_bundle; - mockGetConfig.mockImplementation((key?: string) => - key === 'userSync.userIds' ? [{ name: 'sharedId' }, { name: 'pairId' }] : {} - ); - const warnSpy = vi.spyOn(log, 'warn').mockImplementation(() => {}); + it('preserves sibling consent settings when activating managed GDPR consent', () => { + const gpp = { cmpApi: 'iab', timeout: 750 }; + testWindow.__tcfapi = vi.fn(); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => { + if (key === 'consentManagement') return { gpp }; + if (key === 'userSync.userIds') return []; + return undefined; + }); installPrebidNpm(); - mockPbjs.requestBids({ adUnits: [] }); - expect(testWindow.__tsjs_prebid_diagnostics.userIdModules).toEqual({ - includedModules: [], - configuredUserIdNames: ['pairId', 'sharedId'], - missingConfiguredUserIdNames: [], + expect(mockSetConfig).toHaveBeenCalledWith({ + consentManagement: { gpp, gdpr: { cmpApi: 'iab' } }, }); - const manifestWarnings = warnSpy.mock.calls.filter(([message]) => - String(message).includes('did not stamp a User ID module manifest') - ); - expect(manifestWarnings).toHaveLength(1); - const moduleWarnings = warnSpy.mock.calls.filter(([message]) => - String(message).includes('is not included in the external bundle') - ); - expect(moduleWarnings).toHaveLength(0); - - testWindow.__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; }); - describe('adapter spec', () => { - function getAdapterSpec(): TestAdapterSpec { - installPrebidNpm(); - return mockRegisterBidAdapter.mock.calls[0][2] as TestAdapterSpec; - } - - it('isBidRequestValid always returns true', () => { - const spec = getAdapterSpec(); - expect(spec.isBidRequestValid({})).toBe(true); + it.each([ + ['an object', { cmpApi: 'static', timeout: 123 }], + ['null', null], + ['false', false], + ])('preserves an effective publisher-owned GDPR value when it is %s', (_label, gdpr) => { + testWindow.__tcfapi = vi.fn(); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => { + if (key === 'consentManagement') return { gdpr }; + if (key === 'userSync.userIds') return []; + return undefined; }); - it('buildRequests creates a POST request to /auction', () => { - const spec = getAdapterSpec(); - const bidRequests = [ - { - adUnitCode: 'div-gpt-1', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - params: {}, - }, - ]; + installPrebidNpm(); - const result = spec.buildRequests(bidRequests); + expect(mockSetConfig.mock.calls.some(([value]) => value?.consentManagement)).toBe(false); + }); - expect(result.method).toBe('POST'); - expect(result.url).toBe('/auction'); - expect(result.options).toEqual({ contentType: 'application/json' }); + it.each([ + ['null', null], + ['false', false], + ['a string', 'invalid'], + ['an array', []], + ])('does not replace unsafe effective consent state when it is %s', (_label, consent) => { + const errorSpy = vi.spyOn(log, 'error').mockImplementation(() => {}); + testWindow.__tcfapi = vi.fn(); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => { + if (key === 'consentManagement') return consent; + if (key === 'userSync.userIds') return []; + return undefined; + }); - const payload = JSON.parse(result.data); - expect(payload.adUnits).toHaveLength(1); - expect(payload.adUnits[0].code).toBe('div-gpt-1'); - expect(payload.eids).toBeUndefined(); + installPrebidNpm(); + + expect(mockSetConfig.mock.calls.some(([value]) => value?.consentManagement)).toBe(false); + expect(errorSpy).toHaveBeenCalledWith( + '[tsjs-prebid] effective consentManagement configuration is not mergeable' + ); + }); + + it('does not replace consent state when reading it throws', () => { + const errorSpy = vi.spyOn(log, 'error').mockImplementation(() => {}); + testWindow.__tcfapi = vi.fn(); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => { + if (key === 'consentManagement') throw new Error('example consent accessor failure'); + if (key === 'userSync.userIds') return []; + return undefined; }); - it('buildRequests includes current Prebid EIDs in the /auction payload', () => { - const spec = getAdapterSpec(); - mockGetUserIdsAsEids.mockReturnValue([ - { - source: 'id5-sync.com', - uids: [{ id: 'ID5_abc', atype: 1 }], - }, - { - source: 'sharedid.org', - uids: [{ id: 'shared_123' }, { id: 'shared_456', atype: 3 }], - }, - { - source: 'google.com', - uids: [{ id: 'pair_123', atype: 571187 }], - }, - ]); + installPrebidNpm(); - const result = spec.buildRequests([ - { - adUnitCode: 'div-gpt-1', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - params: {}, - }, - ]); + expect(mockSetConfig.mock.calls.some(([value]) => value?.consentManagement)).toBe(false); + expect(errorSpy).toHaveBeenCalledWith( + '[tsjs-prebid] effective consentManagement configuration could not be read', + expect.any(Error) + ); + }); - const payload = JSON.parse(result.data); - expect(payload.eids).toEqual([ - { - source: 'id5-sync.com', - uids: [{ id: 'ID5_abc', atype: 1 }], - }, - { - source: 'sharedid.org', - uids: [{ id: 'shared_123' }, { id: 'shared_456', atype: 3 }], - }, - { - source: 'google.com', - uids: [{ id: 'pair_123', atype: 571187 }], + it('does not break installation when effective consent property access throws', () => { + const errorSpy = vi.spyOn(log, 'error').mockImplementation(() => {}); + const hostileConsent = new Proxy( + {}, + { + getOwnPropertyDescriptor() { + throw new Error('example consent property trap'); }, - ]); + } + ); + testWindow.__tcfapi = vi.fn(); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => { + if (key === 'consentManagement') return hostileConsent; + if (key === 'userSync.userIds') return []; + return undefined; }); - it('buildRequests clears stale ts-eids cookie when current Prebid EIDs are absent', () => { - const spec = getAdapterSpec(); - document.cookie = 'ts-eids=stale-value'; - mockGetUserIdsAsEids.mockReturnValue([]); + expect(() => installPrebidNpm()).not.toThrow(); - spec.buildRequests([ - { - adUnitCode: 'div-gpt-1', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - params: {}, - }, - ]); + expect(mockSetConfig.mock.calls.some(([value]) => value?.consentManagement)).toBe(false); + expect(errorSpy).toHaveBeenCalledWith( + '[tsjs-prebid] effective consentManagement configuration could not be inspected', + expect.any(Error) + ); + }); - expect(document.cookie).toBe(''); + it('lets queued and late publisher consent configuration retain precedence', () => { + const queuedConsent = { gdpr: { cmpApi: 'static', timeout: 321 } }; + const lateConsent = { gdpr: null }; + testWindow.__tcfapi = vi.fn(); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => + key === 'userSync.userIds' ? [] : undefined + ); + mockPbjs.que = [() => mockPbjs.setConfig({ consentManagement: queuedConsent })]; + mockProcessQueue.mockImplementation(() => { + for (const callback of mockPbjs.que.splice(0)) callback(); }); - it('buildRequests preserves uid ext and sanitizes invalid atype values', () => { - const spec = getAdapterSpec(); - mockGetUserIdsAsEids.mockReturnValue([ - { - source: 'adserver.org', - uids: [ - { - id: 'uid-with-ext', - atype: 1, - ext: { provider: 'liveintent.com', rtiPartner: 'TDID' }, - }, - { - id: 'uid-bad-atype', - atype: 2_147_483_648, - ext: { keep: true }, - }, - { - id: 'uid-float-atype', - atype: 1.5, - }, - ], - }, - ]); + installPrebidNpm(); + mockPbjs.mergeConfig({ consentManagement: lateConsent }); - const result = spec.buildRequests([ - { - adUnitCode: 'div-gpt-1', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - params: {}, - }, - ]); + expect(mockSetConfig).toHaveBeenCalledWith({ consentManagement: queuedConsent }); + expect(mockMergeConfig).toHaveBeenCalledWith({ consentManagement: lateConsent }); + expect( + mockSetConfig.mock.calls.filter(([value]) => value?.consentManagement?.gdpr?.cmpApi === 'iab') + ).toHaveLength(1); + }); - const payload = JSON.parse(result.data); - expect(payload.eids).toEqual([ - { - source: 'adserver.org', - uids: [ - { - id: 'uid-with-ext', - atype: 1, - ext: { provider: 'liveintent.com', rtiPartner: 'TDID' }, - }, - { - id: 'uid-bad-atype', - ext: { keep: true }, - }, - { - id: 'uid-float-atype', - }, - ], - }, - ]); - }); + it('retires automatic IAB consent before late setConfig takes GDPR ownership', () => { + const publisherConsent = { gdpr: { cmpApi: 'static', consentData: { tcString: 'example' } } }; + testWindow.__tcfapi = vi.fn(); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => + key === 'userSync.userIds' ? [] : undefined + ); + installPrebidNpm(); + mockSetConfig.mockClear(); - it('buildRequests uses custom endpoint when configured', () => { - mockRegisterBidAdapter.mockClear(); - installPrebidNpm({ endpoint: '/custom/auction' }); - const spec = mockRegisterBidAdapter.mock.calls[0][2]; + mockPbjs.setConfig({ consentManagement: publisherConsent }); - const result = spec.buildRequests([ - { - adUnitCode: 'slot1', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - }, - ]); + expect(mockSetConfig.mock.calls).toEqual([ + [{ consentManagement: { gdpr: { enabled: false } } }], + [{ consentManagement: publisherConsent }], + ]); + }); - expect(result.url).toBe('/custom/auction'); + it('retires automatic IAB consent once before mergeConfig takes GDPR ownership', () => { + const publisherGdpr = { cmpApi: 'static', consentData: { tcString: 'example' } }; + testWindow.__tcfapi = vi.fn(); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => + key === 'userSync.userIds' ? [] : undefined + ); + installPrebidNpm(); + mockSetConfig.mockClear(); + mockMergeConfig.mockClear(); + + mockPbjs.mergeConfig({ consentManagement: { gdpr: publisherGdpr } }); + mockPbjs.mergeConfig({ consentManagement: { gdpr: { timeout: 500 } } }); + + expect(mockSetConfig).toHaveBeenCalledTimes(3); + expect(mockSetConfig.mock.calls[0]).toEqual([ + { consentManagement: { gdpr: { enabled: false } } }, + ]); + expect(mockMergeConfig.mock.calls[0][0]).toEqual({ + consentManagement: { gdpr: { ...publisherGdpr, enabled: true } }, }); + expect(mockMergeConfig.mock.calls[1][0]).toEqual({ + consentManagement: { gdpr: { timeout: 500 } }, + }); + }); - it('interpretResponse parses seatbid and returns Prebid bids', () => { - const spec = getAdapterSpec(); + it('does not replace unknown consent siblings when retirement state cannot be read', () => { + const errorSpy = vi.spyOn(log, 'error').mockImplementation(() => {}); + const publisherConsent = { gdpr: { cmpApi: 'static', consentData: { tcString: 'example' } } }; + let failConsentRead = false; + testWindow.__tcfapi = vi.fn(); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => { + if (key === 'consentManagement' && failConsentRead) { + throw new Error('example late consent read failure'); + } + if (key === 'userSync.userIds') return []; + return undefined; + }); + installPrebidNpm(); + mockSetConfig.mockClear(); + mockMergeConfig.mockClear(); + failConsentRead = true; - const built = spec.buildRequests([ - { - adUnitCode: 'div-gpt-1', - bidId: 'bid-1', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, + mockPbjs.mergeConfig({ consentManagement: publisherConsent }); + + expect(mockSetConfig).toHaveBeenCalledTimes(1); + expect(mockMergeConfig).toHaveBeenCalledWith({ consentManagement: publisherConsent }); + expect(errorSpy).toHaveBeenCalledWith( + '[tsjs-prebid] effective consentManagement configuration could not be read', + expect.any(Error) + ); + }); + + it('restores merged GDPR activation without probing a missing enabled descriptor', () => { + const publisherGdpr = new Proxy( + { cmpApi: 'static', consentData: { tcString: 'example' } }, + { + getOwnPropertyDescriptor(target, property) { + if (property === 'enabled') throw new Error('example enabled descriptor trap'); + return Reflect.getOwnPropertyDescriptor(target, property); }, - ]); + } + ); + testWindow.__tcfapi = vi.fn(); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => + key === 'userSync.userIds' ? [] : undefined + ); + installPrebidNpm(); + mockMergeConfig.mockClear(); - const serverResponse = { - body: { - seatbid: [ - { - seat: 'appnexus', - bid: [ - { - impid: 'div-gpt-1', - price: 4.5, - adm: '
Creative
', - w: 300, - h: 250, - crid: 'cr-789', - adomain: ['advertiser.com'], - }, - ], - }, - ], + expect(() => + mockPbjs.mergeConfig({ consentManagement: { gdpr: publisherGdpr } }) + ).not.toThrow(); + + expect(mockMergeConfig.mock.calls[0][0]).toEqual({ + consentManagement: { + gdpr: { + cmpApi: 'static', + consentData: { tcString: 'example' }, + enabled: true, }, - }; + }, + }); + }); - const bids = spec.interpretResponse(serverResponse, built); + it('does not clean up before a merge whose enabled getter cannot be inspected', () => { + const errorSpy = vi.spyOn(log, 'error').mockImplementation(() => {}); + const publisherGdpr = new Proxy( + { cmpApi: 'static', consentData: { tcString: 'example' } }, + { + get(target, property, receiver) { + if (property === 'enabled') throw new Error('example enabled getter trap'); + return Reflect.get(target, property, receiver); + }, + } + ); + testWindow.__tcfapi = vi.fn(); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => + key === 'userSync.userIds' ? [] : undefined + ); + installPrebidNpm(); + mockSetConfig.mockClear(); + mockMergeConfig.mockClear(); - expect(bids).toHaveLength(1); - expect(bids[0]).toEqual( - expect.objectContaining({ - requestId: 'bid-1', - cpm: 4.5, - width: 300, - height: 250, - ad: '
Creative
', - currency: 'USD', - netRevenue: true, - bidderCode: 'appnexus', - }) - ); - }); + mockPbjs.mergeConfig({ consentManagement: { gdpr: publisherGdpr } }); - it('interpretResponse handles empty/missing seatbid', () => { - const spec = getAdapterSpec(); - const built = spec.buildRequests([]); + expect(mockSetConfig).toHaveBeenCalledTimes(1); + expect(mockMergeConfig).toHaveBeenCalledWith({ + consentManagement: { gdpr: publisherGdpr }, + }); + expect(errorSpy).toHaveBeenCalledWith( + '[tsjs-prebid] publisher consentManagement merge could not be normalized', + expect.any(Error) + ); + }); - expect(spec.interpretResponse({ body: {} }, built)).toEqual([]); - expect(spec.interpretResponse({ body: null }, built)).toEqual([]); - expect(spec.interpretResponse({}, built)).toEqual([]); + it('completes ownership transfer when cleanup throws after applying disabled state', () => { + const errorSpy = vi.spyOn(log, 'error').mockImplementation(() => {}); + const publisherGdpr = { cmpApi: 'static', consentData: { tcString: 'example' } }; + testWindow.__tcfapi = vi.fn(); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => + key === 'userSync.userIds' ? [] : undefined + ); + installPrebidNpm(); + mockSetConfig.mockClear(); + mockMergeConfig.mockClear(); + mockSetConfig.mockImplementation((config: { consentManagement?: { gdpr?: unknown } }) => { + if ((config.consentManagement?.gdpr as { enabled?: unknown })?.enabled === false) { + throw new Error('example cleanup subscriber failure'); + } }); - it('keeps request mapping isolated across overlapping auctions', () => { - const spec = getAdapterSpec(); + mockPbjs.mergeConfig({ consentManagement: { gdpr: publisherGdpr } }); + mockPbjs.mergeConfig({ consentManagement: { gdpr: { timeout: 500 } } }); - const requestA = spec.buildRequests([ - { - adUnitCode: 'slot-a', - bidId: 'bid-a', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - }, - ]); - const requestB = spec.buildRequests([ - { - adUnitCode: 'slot-b', - bidId: 'bid-b', - bidder: 'trustedServer', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - }, - ]); + expect( + mockSetConfig.mock.calls.filter( + ([config]) => config.consentManagement?.gdpr?.enabled === false + ) + ).toHaveLength(1); + expect(mockMergeConfig.mock.calls[0][0]).toEqual({ + consentManagement: { gdpr: { ...publisherGdpr, enabled: true } }, + }); + expect(mockMergeConfig.mock.calls[1][0]).toEqual({ + consentManagement: { gdpr: { timeout: 500 } }, + }); + expect(errorSpy).toHaveBeenCalledWith( + '[tsjs-prebid] automatic IAB consent listener could not be retired', + expect.any(Error) + ); + }); - const responseA = { - body: { - seatbid: [ - { - seat: 'appnexus', - bid: [{ impid: 'slot-a', price: 1.1, adm: '
A
', w: 300, h: 250 }], - }, - ], - }, - }; - const responseB = { - body: { - seatbid: [ - { - seat: 'rubicon', - bid: [{ impid: 'slot-b', price: 2.2, adm: '
B
', w: 300, h: 250 }], - }, - ], - }, - }; + it('preserves effective User ID entries and replaces identityLink exactly once', () => { + mockGetConfig.mockImplementation((key?: string) => + key === 'userSync.userIds' + ? [ + { name: 'sharedId', storage: { name: '_sharedid' } }, + { name: 'identityLink', params: { pid: 'publisher-value' } }, + { name: 'identityLink', params: { pid: 'duplicate-value' } }, + ] + : {} + ); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; - const bidsA = spec.interpretResponse(responseA, requestA); - const bidsB = spec.interpretResponse(responseB, requestB); + installPrebidNpm(); - expect(bidsA[0].requestId).toBe('bid-a'); - expect(bidsB[0].requestId).toBe('bid-b'); + const managedCall = mockSetConfig.mock.calls.find(([value]) => value?.userSync?.userIds); + expect(managedCall?.[0]).toEqual({ + userSync: { + userIds: [{ name: 'sharedId', storage: { name: '_sharedid' } }, EXPECTED_MANAGED_USER_ID], + }, }); + expect( + managedCall?.[0].userSync.userIds.filter( + (entry: { name?: string }) => entry.name === 'identityLink' + ) + ).toHaveLength(1); }); - describe('requestBids shim', () => { - beforeEach(() => { - testWindow.__tsjs_prebid = { - serverSideBidders: ['appnexus', 'rubicon', 'kargo', 'openx'], - }; - }); + it('drops malformed effective User ID state and installs the managed entry', () => { + mockGetConfig.mockImplementation((key?: string) => + key === 'userSync.userIds' ? [null, 'invalid', {}, { name: '' }, { name: 'sharedId' }] : {} + ); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; - it('preserves publisher ts adserverTargeting while adding trustedServer settings', () => { - const publisherTargeting = [{ key: 'ts', val: () => 'publisher-value' }]; - mockPbjs.bidderSettings = { - exampleBidder: { adserverTargeting: publisherTargeting }, - }; - const pbjs = installPrebidNpm(); + expect(() => installPrebidNpm()).not.toThrow(); - pbjs.requestBids({ - adUnits: [{ bids: [{ bidder: 'exampleBidder', params: {} }] }], - } as unknown as RequestBidsArg); + const managedCall = mockSetConfig.mock.calls.find(([value]) => value?.userSync?.userIds); + expect(managedCall?.[0].userSync.userIds).toEqual([ + { name: 'sharedId' }, + EXPECTED_MANAGED_USER_ID, + ]); + }); - const bidderSettings = mockPbjs.bidderSettings as { - exampleBidder: { adserverTargeting: typeof publisherTargeting }; - trustedServer: { - allowAlternateBidderCodes: boolean; - allowedAlternateBidderCodes: string[]; - }; - }; - expect(bidderSettings.exampleBidder.adserverTargeting).toBe(publisherTargeting); - expect(bidderSettings.exampleBidder.adserverTargeting[0].key).toBe('ts'); - expect(bidderSettings.exampleBidder.adserverTargeting[0].val()).toBe('publisher-value'); - expect(bidderSettings.trustedServer).toEqual( - expect.objectContaining({ - allowAlternateBidderCodes: true, - allowedAlternateBidderCodes: ['*'], - }) - ); + it('installs the managed entry before processing the publisher queue', () => { + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + + installPrebidNpm(); + + const managedCallOrder = mockSetConfig.mock.invocationCallOrder.find( + (_, index) => mockSetConfig.mock.calls[index][0]?.userSync?.userIds + ); + expect(managedCallOrder).toBeLessThan(mockProcessQueue.mock.invocationCallOrder[0]); + }); + + it('normalizes queued User ID config before a queued auction observes it', () => { + let observedUserIds: unknown; + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockPbjs.que = [ + () => + mockPbjs.setConfig({ + userSync: { + syncDelay: 50, + userIds: [ + { name: 'sharedId' }, + { name: 'identityLink', params: { pid: 'publisher-value' } }, + ], + }, + auctionOptions: { suppressStaleRender: true }, + }), + () => { + observedUserIds = mockSetConfig.mock.calls.at(-1)?.[0]?.userSync?.userIds; + }, + ]; + mockProcessQueue.mockImplementation(() => { + for (const callback of mockPbjs.que.splice(0)) callback(); }); - it('injects trustedServer bidder into every ad unit', () => { - const pbjs = installPrebidNpm(); + installPrebidNpm(); - const adUnits = [ - { bids: [{ bidder: 'appnexus', params: {} }] }, - { bids: [{ bidder: 'rubicon', params: {} }] }, - ]; - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + expect(observedUserIds).toEqual([{ name: 'sharedId' }, EXPECTED_MANAGED_USER_ID]); + const queuedConfig = mockSetConfig.mock.calls.at(-1)?.[0]; + expect(queuedConfig.userSync.syncDelay).toBe(50); + expect(queuedConfig.auctionOptions).toEqual({ suppressStaleRender: true }); + }); - // Each ad unit should have trustedServer added - for (const unit of adUnits) { - const hasTsBidder = unit.bids.some((b: TestBid) => b.bidder === 'trustedServer'); - expect(hasTsBidder).toBe(true); - } + it('normalizes publisher identityLink updates after processQueue', () => { + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + installPrebidNpm(); - const trustedServerBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer'); - expect(trustedServerBid.params.bidderParams).toEqual({ appnexus: {} }); - expect(adUnits[0].bids.map((b: TestBid) => b.bidder)).toEqual(['trustedServer']); - expect(adUnits[1].bids.map((b: TestBid) => b.bidder)).toEqual(['trustedServer']); + mockPbjs.setConfig({ + userSync: { + userIds: [ + { name: 'id5Id', params: { partner: 1 } }, + { name: 'identityLink', params: { pid: 'publisher-value' } }, + ], + }, + }); - // Should call through to original requestBids - expect(mockRequestBids).toHaveBeenCalled(); + expect(mockSetConfig.mock.calls.at(-1)?.[0].userSync.userIds).toEqual([ + { name: 'id5Id', params: { partner: 1 } }, + EXPECTED_MANAGED_USER_ID, + ]); + }); + + it('normalizes queued identityLink updates made through mergeConfig', () => { + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockPbjs.que = [ + () => + mockPbjs.mergeConfig({ + userSync: { + userIds: [ + { name: 'sharedId' }, + { name: 'identityLink', params: { pid: 'publisher-value' } }, + ], + }, + }), + ]; + mockProcessQueue.mockImplementation(() => { + for (const callback of mockPbjs.que.splice(0)) callback(); }); - it('does not duplicate trustedServer if already present', () => { - const pbjs = installPrebidNpm(); + installPrebidNpm(); - const adUnits = [{ bids: [{ bidder: 'trustedServer', params: {} }] }]; - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + expect(mockMergeConfig.mock.calls.at(-1)?.[0].userSync.userIds).toEqual([ + { name: 'sharedId' }, + EXPECTED_MANAGED_USER_ID, + ]); + }); - const tsCount = adUnits[0].bids.filter((b: TestBid) => b.bidder === 'trustedServer').length; - expect(tsCount).toBe(1); + it('normalizes late identityLink updates made through mergeConfig', () => { + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + installPrebidNpm(); + + mockPbjs.mergeConfig({ + userSync: { + userIds: [ + { name: 'id5Id', params: { partner: 1 } }, + { name: 'identityLink', params: { pid: 'publisher-value' } }, + ], + }, }); - it('folds only authoritative routes across mixed client, PBS, APS, and standard demand', () => { - testWindow.__tsjs_prebid = { - serverSideBidders: ['pbsRoute', 'standardRoute'], - clientSideBidders: ['exampleBrowser'], - }; - const pbjs = installPrebidNpm(); + expect(mockMergeConfig.mock.calls.at(-1)?.[0].userSync.userIds).toEqual([ + { name: 'id5Id', params: { partner: 1 } }, + EXPECTED_MANAGED_USER_ID, + ]); + }); - const adUnits = [ - { - bids: [ - { bidder: 'exampleBrowser', params: { placement: 'browser' } }, - { bidder: 'pbsRoute', params: { placement: 'pbs' } }, - { bidder: 'aps', params: { slot: 'aps' } }, - { bidder: 'standardRoute', params: { placement: 'standard' } }, - { bidder: 'pbs-provider-id', params: { forbidden: true } }, - ], - }, - ]; - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + it('passes unrelated publisher configuration through by reference', () => { + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + installPrebidNpm(); + const publisherConfig = { priceGranularity: 'medium', userSync: { syncDelay: 50 } }; - const trustedServerBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer'); - expect(trustedServerBid?.params?.bidderParams).toEqual({ - pbsRoute: { placement: 'pbs' }, - standardRoute: { placement: 'standard' }, - }); - expect(adUnits[0].bids.map((b: TestBid) => b.bidder)).toEqual([ - 'exampleBrowser', - 'aps', - 'pbs-provider-id', - 'trustedServer', - ]); - }); + mockPbjs.setConfig(publisherConfig); - it('preserves prototype-named server-side bidders as owned JSON properties', () => { - testWindow.__tsjs_prebid = { serverSideBidders: ['__proto__'] }; - const pbjs = installPrebidNpm(); - const adUnits = [ - { - bids: [{ bidder: '__proto__', params: { placement: 'server-owned' } }], - }, - ]; - - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + expect(mockSetConfig.mock.calls.at(-1)?.[0]).toBe(publisherConfig); + }); - const trustedServerBid = adUnits[0].bids.find((bid) => bid.bidder === 'trustedServer'); - const bidderParams = trustedServerBid?.params?.bidderParams as Record; - expect(Object.prototype.hasOwnProperty.call(bidderParams, '__proto__')).toBe(true); - expect(bidderParams['__proto__']).toEqual({ placement: 'server-owned' }); - expect(JSON.parse(JSON.stringify(bidderParams))).toEqual( - Object.fromEntries([['__proto__', { placement: 'server-owned' }]]) - ); - expect(adUnits[0].bids.map((bid) => bid.bidder)).toEqual(['trustedServer']); - }); + it('does not stack the managed User ID config wrappers across shim reinstallations', () => { + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + installPrebidNpm(); + const managedSetConfig = mockPbjs.setConfig; + const managedMergeConfig = mockPbjs.mergeConfig; + delete testWindow.__tsjsPrebidShimInstalled; - it('does not let returned bidder aliases or APS renderer aliases affect folding', () => { - testWindow.__tsjs_prebid = { serverSideBidders: ['configuredRoute'] }; - const pbjs = installPrebidNpm(); - const adUnits = [ - { - bids: [ - { bidder: 'configuredRoute', params: { placement: 1 } }, - { bidder: 'alternateReturnedSeat', params: { placement: 2 } }, - { bidder: 'apsRendererAlias', params: { placement: 3 } }, - ], - }, - ]; + installPrebidNpm(); - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + expect(mockPbjs.setConfig).toBe(managedSetConfig); + expect(mockPbjs.mergeConfig).toBe(managedMergeConfig); + mockPbjs.setConfig({ userSync: { userIds: [] } }); + expect(mockSetConfig.mock.calls.at(-1)?.[0].userSync.userIds).toEqual([ + EXPECTED_MANAGED_USER_ID, + ]); + }); - const trustedServerBid = adUnits[0].bids.find((bid) => bid.bidder === 'trustedServer'); - expect(trustedServerBid?.params?.bidderParams).toEqual({ - configuredRoute: { placement: 1 }, - }); - expect(adUnits[0].bids.map((bid) => bid.bidder)).toEqual([ - 'alternateReturnedSeat', - 'apsRendererAlias', - 'trustedServer', - ]); - }); + it('skips seeding the managed entry when getConfig is unavailable', () => { + // Seeding needs the effective User ID entries. Without getConfig they + // cannot be read, and installing the managed entry alone would silently + // drop every publisher-configured module. + const errorSpy = vi.spyOn(log, 'error').mockImplementation(() => {}); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + delete (mockPbjs as { getConfig?: unknown }).getConfig; - it('preserves captured bidder params when requestBids runs twice on the same ad unit', () => { - const pbjs = installPrebidNpm(); + expect(() => installPrebidNpm()).not.toThrow(); - // First auction: inline server-side params supplied by the publisher. - const adUnits = [ - { - code: 'div-1', - bids: [ - { bidder: 'appnexus', params: { placementId: 123 } }, - { bidder: 'rubicon', params: { accountId: 'abc' } }, - ], - }, - ]; - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + expect(mockSetConfig.mock.calls.some(([value]) => value?.userSync?.userIds)).toBe(false); + expect(errorSpy).toHaveBeenCalledWith( + '[tsjs-prebid] window.pbjs.getConfig is unavailable; managed User ID entries not seeded' + ); - // Second auction (refresh/re-auction) with the SAME ad unit object: the - // server-side bidder entries were already pruned, so the shim must not - // overwrite the captured params with an empty object. - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + // The wrappers are still installed, so the next publisher userIds call + // still gets the managed entry. + mockPbjs.setConfig({ userSync: { userIds: [{ name: 'sharedId' }] } }); + expect(mockSetConfig.mock.calls.at(-1)?.[0].userSync.userIds).toEqual([ + { name: 'sharedId' }, + EXPECTED_MANAGED_USER_ID, + ]); + }); - const trustedServerBid = adUnits[0].bids.find( - (b: TestBid) => b.bidder === 'trustedServer' - ) as TestBid; - expect(trustedServerBid.params.bidderParams).toEqual({ - appnexus: { placementId: 123 }, - rubicon: { accountId: 'abc' }, - }); + it('continues installation when reading effective User ID entries throws', () => { + const errorSpy = vi.spyOn(log, 'error').mockImplementation(() => {}); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + mockGetConfig.mockImplementation((key?: string) => { + if (key === 'userSync.userIds') throw new Error('example User ID accessor failure'); + return undefined; }); - it('adds bids array to ad units that have none', () => { - const pbjs = installPrebidNpm(); + expect(() => installPrebidNpm()).not.toThrow(); - const adUnits = [{ code: 'div-1' }] as TestAdUnit[]; - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + expect(mockSetConfig.mock.calls.some(([value]) => value?.userSync?.userIds)).toBe(false); + expect(mockRegisterBidAdapter).toHaveBeenCalledTimes(1); + expect(mockProcessQueue).toHaveBeenCalledTimes(1); + expect(errorSpy).toHaveBeenCalledWith( + '[tsjs-prebid] effective User ID entries could not be read; managed User ID entries not seeded', + expect.any(Error) + ); - expect(adUnits[0].bids).toHaveLength(1); - expect(adUnits[0].bids[0].bidder).toBe('trustedServer'); - }); + // The wrappers remain installed even when the initial seed read fails. + mockPbjs.setConfig({ userSync: { userIds: [{ name: 'sharedId' }] } }); + expect(mockSetConfig.mock.calls.at(-1)?.[0].userSync.userIds).toEqual([ + { name: 'sharedId' }, + EXPECTED_MANAGED_USER_ID, + ]); + }); - it('normalizes a truthy non-array bids value without throwing', () => { - const pbjs = installPrebidNpm(); - const adUnits = [ - { code: 'example-malformed-slot', bids: { malformed: true } }, - ] as TestAdUnit[]; + it('passes the publisher config through when normalization throws', () => { + const errorSpy = vi.spyOn(log, 'error').mockImplementation(() => {}); + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + installPrebidNpm(); + const hostileConfig = { + userSync: { + get userIds(): never { + throw new Error('example accessor failure'); + }, + }, + }; - expect(() => pbjs.requestBids({ adUnits } as unknown as RequestBidsArg)).not.toThrow(); + mockPbjs.setConfig(hostileConfig as unknown as Parameters[0]); - expect(adUnits[0].bids).toEqual([{ bidder: 'trustedServer', params: { bidderParams: {} } }]); - }); + expect(mockSetConfig.mock.calls.at(-1)?.[0]).toBe(hostileConfig); + expect(errorSpy).toHaveBeenCalledWith( + '[tsjs-prebid] managed User ID entries could not be normalized', + expect.any(Error) + ); + }); - it('preserves the empty stored-request envelope on initial and repeated requests', () => { - const pbjs = installPrebidNpm(); - const adUnits = [ - { - code: 'stored-slot', - bids: [{ bidder: 'trustedServer', params: { bidderParams: {} } }], - }, - ]; + it('hands Prebid a distinct managed entry object per normalization', () => { + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + installPrebidNpm(); - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + mockPbjs.setConfig({ userSync: { userIds: [] } }); + const first = mockSetConfig.mock.calls.at(-1)?.[0].userSync.userIds[0]; + mockPbjs.setConfig({ userSync: { userIds: [] } }); + const second = mockSetConfig.mock.calls.at(-1)?.[0].userSync.userIds[0]; - expect(adUnits[0].bids).toEqual([{ bidder: 'trustedServer', params: { bidderParams: {} } }]); - }); + expect(first).toEqual(EXPECTED_MANAGED_USER_ID); + expect(second).toEqual(EXPECTED_MANAGED_USER_ID); + expect(second).not.toBe(first); + }); - it('includes zone from mediaTypes.banner.name in trustedServer params', () => { - const pbjs = installPrebidNpm(); + it('isolates nested managed params from Prebid and from later normalizations', () => { + // Prebid keeps the entry it receives as `submodule.config` for the life of + // the page. A shallow copy would leave nested params shared with + // window.__tsjs_prebid and with every other entry built from it. + const nestedManagedUserId = { + name: 'exampleId', + params: { pid: '999', ext: { segments: ['a'] } }, + }; + testWindow.__tsjs_prebid = { managedUserIds: [nestedManagedUserId] }; + installPrebidNpm(); - const adUnits = [ - { - code: 'ad-header-0', - mediaTypes: { banner: { name: 'header', sizes: [[728, 90]] } }, - bids: [{ bidder: 'kargo', params: { placementId: '_abc' } }], - }, - { - code: 'ad-fixed_bottom-0', - mediaTypes: { banner: { name: 'fixed_bottom', sizes: [[728, 90]] } }, - bids: [{ bidder: 'kargo', params: { placementId: '_def' } }], - }, - ]; - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + mockPbjs.setConfig({ userSync: { userIds: [] } }); + const first = mockSetConfig.mock.calls.at(-1)?.[0].userSync.userIds[0]; + mockPbjs.setConfig({ userSync: { userIds: [] } }); + const second = mockSetConfig.mock.calls.at(-1)?.[0].userSync.userIds[0]; - const tsBid0 = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; - expect(tsBid0.params.zone).toBe('header'); + // Simulate Prebid mutating the configuration it retained. + (first.params.ext as { segments: string[] }).segments.push('mutated'); - const tsBid1 = adUnits[1].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; - expect(tsBid1.params.zone).toBe('fixed_bottom'); - }); + expect(second.params.ext).toEqual({ segments: ['a'] }); + expect(nestedManagedUserId.params.ext.segments).toEqual(['a']); + }); - it('omits zone when mediaTypes.banner.name is not set', () => { - const pbjs = installPrebidNpm(); + it('reports identityLink missing when its bundle module is absent', () => { + testWindow.__tsjs_prebid = { managedUserIds: [MANAGED_USER_ID] }; + testWindow.__tsjs_prebid_bundle = { userIdModules: [] }; + mockGetConfig.mockImplementation((key?: string) => + key === 'userSync.userIds' ? [EXPECTED_MANAGED_USER_ID] : {} + ); - const adUnits = [ - { - code: 'ad-header-0', - mediaTypes: { banner: { sizes: [[300, 250]] } }, - bids: [{ bidder: 'appnexus', params: {} }], - }, - ]; - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + installPrebidNpm(); - const tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; - expect(tsBid.params.zone).toBeUndefined(); + expect(testWindow.__tsjs_prebid_diagnostics.userIdModules).toEqual({ + includedModules: [], + configuredUserIdNames: ['identityLink'], + missingConfiguredUserIdNames: ['identityLink'], }); + testWindow.__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; + }); - it('omits zone when ad unit has no mediaTypes', () => { - const pbjs = installPrebidNpm(); - - const adUnits = [{ bids: [{ bidder: 'rubicon', params: {} }] }]; - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + it('reports the User ID modules selected by the generated bundle', () => { + installPrebidNpm(); - const tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; - expect(tsBid.params.zone).toBeUndefined(); + expect(testWindow.__tsjs_prebid_diagnostics.userIdModules).toEqual({ + includedModules: ['sharedIdSystem'], + configuredUserIdNames: [], + missingConfiguredUserIdNames: [], }); + }); - it('clears stale zone when existing trustedServer bid is reused', () => { - const pbjs = installPrebidNpm(); + it('refreshes late User ID config without repeating missing-module warnings', () => { + installPrebidNpm(); + mockGetConfig.mockImplementation((key?: string) => + key === 'userSync.userIds' ? [{ name: 'sharedId' }, { name: 'pairId' }] : {} + ); + const warnSpy = vi.spyOn(log, 'warn').mockImplementation(() => {}); - const adUnits = [ - { - code: 'ad-header-0', - mediaTypes: { banner: { name: 'header', sizes: [[300, 250]] } }, - bids: [ - { bidder: 'trustedServer', params: { custom: 'keep' } }, - { bidder: 'kargo', params: { placementId: '_abc' } }, - ], - }, - ]; + mockPbjs.requestBids({ adUnits: [] }); + mockPbjs.requestBids({ adUnits: [] }); - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + expect(testWindow.__tsjs_prebid_diagnostics.userIdModules).toEqual({ + includedModules: ['sharedIdSystem'], + configuredUserIdNames: ['pairId', 'sharedId'], + missingConfiguredUserIdNames: ['pairId'], + }); + expect( + warnSpy.mock.calls.filter(([message]) => String(message).includes('"pairId"')) + ).toHaveLength(1); + }); - let tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; - expect(tsBid.params.zone).toBe('header'); - expect(tsBid.params.custom).toBe('keep'); + it('returns the pbjs instance', () => { + const result = installPrebidNpm(); + expect(result).toBe(mockPbjs); + }); - delete adUnits[0].mediaTypes.banner.name; - pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + it('installs only once per page via the __tsjsPrebidShimInstalled sentinel', () => { + const first = installPrebidNpm(); + const wrappedRequestBids = mockPbjs.requestBids; + const second = installPrebidNpm(); - tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; - expect(tsBid.params.zone).toBeUndefined(); - expect(tsBid.params.custom).toBe('keep'); - }); + expect(second).toBe(first); + expect(mockRegisterBidAdapter).toHaveBeenCalledTimes(1); + expect(mockPbjs.requestBids).toBe(wrappedRequestBids); + expect(testWindow.__tsjsPrebidShimInstalled).toBe(true); + }); - it('falls back to pbjs.adUnits when requestObj has no adUnits', () => { - const pbjs = installPrebidNpm(); + it('warns once about an unstamped User ID manifest instead of once per module', () => { + delete testWindow.__tsjs_prebid_bundle; + mockGetConfig.mockImplementation((key?: string) => + key === 'userSync.userIds' ? [{ name: 'sharedId' }, { name: 'pairId' }] : {} + ); + const warnSpy = vi.spyOn(log, 'warn').mockImplementation(() => {}); - mockPbjs.adUnits = [{ bids: [{ bidder: 'openx', params: {} }] }] as TestAdUnit[]; - pbjs.requestBids({} as RequestBidsArg); + installPrebidNpm(); + mockPbjs.requestBids({ adUnits: [] }); - const hasTsBidder = (mockPbjs.adUnits[0].bids ?? []).some( - (b: TestBid) => b.bidder === 'trustedServer' - ); - expect(hasTsBidder).toBe(true); + expect(testWindow.__tsjs_prebid_diagnostics.userIdModules).toEqual({ + includedModules: [], + configuredUserIdNames: ['pairId', 'sharedId'], + missingConfiguredUserIdNames: [], }); + const manifestWarnings = warnSpy.mock.calls.filter(([message]) => + String(message).includes('did not stamp a User ID module manifest') + ); + expect(manifestWarnings).toHaveLength(1); + const moduleWarnings = warnSpy.mock.calls.filter(([message]) => + String(message).includes('is not included in the external bundle') + ); + expect(moduleWarnings).toHaveLength(0); - it('syncs a structured ts-eids cookie after bidsBackHandler', () => { - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); - mockGetUserIdsAsEids.mockReturnValue([ - { - source: 'sharedid.org', - uids: [ - { id: 'shared_123', atype: 3 }, - { id: 'shared_456', ext: { provider: 'example' } }, - ], - }, - ]); + testWindow.__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; + }); - const pbjs = installPrebidNpm(); - pbjs.requestBids({ - adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], - } as unknown as RequestBidsArg); + describe('adapter spec', () => { + function getAdapterSpec(): TestAdapterSpec { + installPrebidNpm(); + return mockRegisterBidAdapter.mock.calls[0][2] as TestAdapterSpec; + } - const cookieValue = document.cookie.match(/(?:^|; )ts-eids=([^;]+)/)?.[1]; - expect(cookieValue).toBeDefined(); - expect(JSON.parse(atob(cookieValue!))).toEqual([ + it('isBidRequestValid always returns true', () => { + const spec = getAdapterSpec(); + expect(spec.isBidRequestValid({})).toBe(true); + }); + + it('buildRequests creates a POST request to /auction', () => { + const spec = getAdapterSpec(); + const bidRequests = [ { - source: 'sharedid.org', - uids: [ - { id: 'shared_123', atype: 3 }, - { id: 'shared_456', ext: { provider: 'example' } }, - ], + adUnitCode: 'div-gpt-1', + bidder: 'trustedServer', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + params: {}, }, - ]); - }); + ]; - it('clears ts-eids cookie after bidsBackHandler when no current EIDs remain', () => { - document.cookie = `ts-eids=${btoa(JSON.stringify([{ source: 'sharedid.org', uids: [{ id: 'stale' }] }]))}`; - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); - mockGetUserIdsAsEids.mockReturnValue([]); + const result = spec.buildRequests(bidRequests); - const pbjs = installPrebidNpm(); - pbjs.requestBids({ - adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], - } as unknown as RequestBidsArg); + expect(result.method).toBe('POST'); + expect(result.url).toBe('/auction'); + expect(result.options).toEqual({ contentType: 'application/json' }); - expect(document.cookie).toBe(''); + const payload = JSON.parse(result.data); + expect(payload.adUnits).toHaveLength(1); + expect(payload.adUnits[0].code).toBe('div-gpt-1'); + expect(payload.eids).toBeUndefined(); }); - }); -}); - -describe('prebid/installPrebidNpm with server-injected config', () => { - beforeEach(() => { - vi.clearAllMocks(); - mockPbjs.requestBids = mockRequestBids; - mockPbjs.adUnits = []; - mockGetUserIdsAsEids.mockReset(); - mockGetUserIdsAsEids.mockReturnValue([]); - document.cookie = 'ts-eids=; Path=/; Max-Age=0'; - delete testWindow.__tsjs_prebid; - }); - afterEach(() => { - delete testWindow.__tsjs_prebid; - }); - - it('reads timeout and debug from window.__tsjs_prebid', () => { - testWindow.__tsjs_prebid = { timeout: 1500, debug: true }; + it('buildRequests includes current Prebid EIDs in the /auction payload', () => { + const spec = getAdapterSpec(); + mockGetUserIdsAsEids.mockReturnValue([ + { + source: 'id5-sync.com', + uids: [{ id: 'ID5_abc', atype: 1 }], + }, + { + source: 'sharedid.org', + uids: [{ id: 'shared_123' }, { id: 'shared_456', atype: 3 }], + }, + { + source: 'google.com', + uids: [{ id: 'pair_123', atype: 571187 }], + }, + ]); - installPrebidNpm(); + const result = spec.buildRequests([ + { + adUnitCode: 'div-gpt-1', + bidder: 'trustedServer', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + params: {}, + }, + ]); - expect(mockSetConfig).toHaveBeenCalledWith( - expect.objectContaining({ debug: true, bidderTimeout: 1500 }) - ); - }); + const payload = JSON.parse(result.data); + expect(payload.eids).toEqual([ + { + source: 'id5-sync.com', + uids: [{ id: 'ID5_abc', atype: 1 }], + }, + { + source: 'sharedid.org', + uids: [{ id: 'shared_123' }, { id: 'shared_456', atype: 3 }], + }, + { + source: 'google.com', + uids: [{ id: 'pair_123', atype: 571187 }], + }, + ]); + }); - it('keeps browser timeout and debug independent from multiple PBS routes', () => { - testWindow.__tsjs_prebid = { - timeout: 1750, - debug: false, - serverSideBidders: ['pbsPrimaryRoute', 'pbsSecondaryRoute'], - }; + it('forwards the opaque LiveRamp envelope as a liveramp.com EID', () => { + const spec = getAdapterSpec(); + mockGetUserIdsAsEids.mockReturnValue([ + { + source: 'liveramp.com', + uids: [{ id: 'opaque-test-envelope', atype: 3 }], + }, + ]); - installPrebidNpm(); + const request = spec.buildRequests([ + { + adUnitCode: 'div-gpt-1', + bidId: 'bid-1', + bidder: 'trustedServer', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + params: {}, + }, + ]); - expect(mockSetConfig).toHaveBeenCalledWith({ debug: false, bidderTimeout: 1750 }); - }); + expect(JSON.parse(request.data).eids).toEqual([ + { + source: 'liveramp.com', + uids: [{ id: 'opaque-test-envelope', atype: 3 }], + }, + ]); + }); - it('explicit config overrides server-injected values', () => { - testWindow.__tsjs_prebid = { timeout: 1500, debug: true }; + it('drops empty and malformed LiveRamp envelope values', () => { + const spec = getAdapterSpec(); + mockGetUserIdsAsEids.mockReturnValue([ + { + source: 'liveramp.com', + uids: [ + { id: '' }, + { id: undefined as unknown as string }, + { id: 'opaque-valid-envelope', atype: 3 }, + ], + }, + { source: '', uids: [{ id: 'opaque-invalid-source' }] }, + ]); - installPrebidNpm({ timeout: 3000, debug: false }); + const request = spec.buildRequests([ + { + adUnitCode: 'div-gpt-1', + bidder: 'trustedServer', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + params: {}, + }, + ]); - expect(mockSetConfig).toHaveBeenCalledWith( - expect.objectContaining({ debug: false, bidderTimeout: 3000 }) - ); - }); + expect(JSON.parse(request.data).eids).toEqual([ + { + source: 'liveramp.com', + uids: [{ id: 'opaque-valid-envelope', atype: 3 }], + }, + ]); + }); - it('works with no config argument and no injected config', () => { - installPrebidNpm(); + it('never writes an opaque LiveRamp envelope to logs', () => { + const sentinel = 'opaque-envelope-must-not-be-logged'; + const spies = [ + vi.spyOn(log, 'debug').mockImplementation(() => {}), + vi.spyOn(log, 'info').mockImplementation(() => {}), + vi.spyOn(log, 'warn').mockImplementation(() => {}), + vi.spyOn(log, 'error').mockImplementation(() => {}), + ]; + const spec = getAdapterSpec(); + mockGetUserIdsAsEids.mockReturnValue([ + { source: 'liveramp.com', uids: [{ id: sentinel, atype: 3 }] }, + ]); - expect(mockSetConfig).toHaveBeenCalledWith(expect.objectContaining({ debug: false })); - expect(mockProcessQueue).toHaveBeenCalled(); - }); -}); + spec.buildRequests([ + { + adUnitCode: 'div-gpt-1', + bidder: 'trustedServer', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + params: {}, + }, + ]); -describe('prebid/installRefreshHandler', () => { - beforeEach(() => { - vi.clearAllMocks(); - mockRequestBids.mockReset(); - mockPbjs.requestBids = mockRequestBids; - mockPbjs.adUnits = []; - mockPbjs.setTargetingForGPTAsync = undefined; - testWindow.tsjs = undefined; - delete testWindow.googletag; - testWindow.__tsjs_prebid = { - serverSideBidders: ['appnexus', 'rubicon', 'kargo', 'openx', 'exampleServer'], - }; - }); + const logged = spies.flatMap((spy) => spy.mock.calls).flat(); + expect(logged.some((value) => JSON.stringify(value).includes(sentinel))).toBe(false); + }); - afterEach(() => { - testWindow.tsjs = undefined; - delete testWindow.googletag; - delete testWindow.__tsjs_prebid; - }); + it('buildRequests clears stale ts-eids cookie when current Prebid EIDs are absent', () => { + const spec = getAdapterSpec(); + document.cookie = 'ts-eids=stale-value'; + mockGetUserIdsAsEids.mockReturnValue([]); - it('builds refresh ad units from injected slot metadata', () => { - const originalRefresh = vi.fn(); - const gptSlot = { - getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), - getTargeting: vi.fn(() => []), - }; - const pubads = { - refresh: originalRefresh, - getSlots: vi.fn(() => [gptSlot]), - }; - testWindow.googletag = { - cmd: { push: (fn: () => void) => fn() }, - pubads: () => pubads, - }; - testWindow.tsjs = { - adSlots: [ + spec.buildRequests([ { - id: 'homepage_header_ad', - gam_unit_path: '/123/homepage', - div_id: 'div-ad-homepage-header', - formats: [ - [970, 250], - [728, 90], - ], - targeting: { zone: 'homepage', pos: 'atf' }, + adUnitCode: 'div-gpt-1', + bidder: 'trustedServer', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + params: {}, }, - ], - }; + ]); - installRefreshHandler(750); - pubads.refresh(); + expect(document.cookie).toBe(''); + }); - expect(mockRequestBids).toHaveBeenCalledWith( - expect.objectContaining({ - timeout: 750, - adUnits: [ - expect.objectContaining({ + it('buildRequests preserves uid ext and sanitizes invalid atype values', () => { + const spec = getAdapterSpec(); + mockGetUserIdsAsEids.mockReturnValue([ + { + source: 'adserver.org', + uids: [ + { + id: 'uid-with-ext', + atype: 1, + ext: { provider: 'liveintent.com', rtiPartner: 'TDID' }, + }, + { + id: 'uid-bad-atype', + atype: 2_147_483_648, + ext: { keep: true }, + }, + { + id: 'uid-float-atype', + atype: 1.5, + }, + ], + }, + ]); + + const result = spec.buildRequests([ + { + adUnitCode: 'div-gpt-1', + bidder: 'trustedServer', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + params: {}, + }, + ]); + + const payload = JSON.parse(result.data); + expect(payload.eids).toEqual([ + { + source: 'adserver.org', + uids: [ + { + id: 'uid-with-ext', + atype: 1, + ext: { provider: 'liveintent.com', rtiPartner: 'TDID' }, + }, + { + id: 'uid-bad-atype', + ext: { keep: true }, + }, + { + id: 'uid-float-atype', + }, + ], + }, + ]); + }); + + it('buildRequests uses custom endpoint when configured', () => { + mockRegisterBidAdapter.mockClear(); + installPrebidNpm({ endpoint: '/custom/auction' }); + const spec = mockRegisterBidAdapter.mock.calls[0][2]; + + const result = spec.buildRequests([ + { + adUnitCode: 'slot1', + bidder: 'trustedServer', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + }, + ]); + + expect(result.url).toBe('/custom/auction'); + }); + + it('interpretResponse parses seatbid and returns Prebid bids', () => { + const spec = getAdapterSpec(); + + const built = spec.buildRequests([ + { + adUnitCode: 'div-gpt-1', + bidId: 'bid-1', + bidder: 'trustedServer', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + }, + ]); + + const serverResponse = { + body: { + seatbid: [ + { + seat: 'appnexus', + bid: [ + { + impid: 'div-gpt-1', + price: 4.5, + adm: '
Creative
', + w: 300, + h: 250, + crid: 'cr-789', + adomain: ['advertiser.com'], + }, + ], + }, + ], + }, + }; + + const bids = spec.interpretResponse(serverResponse, built); + + expect(bids).toHaveLength(1); + expect(bids[0]).toEqual( + expect.objectContaining({ + requestId: 'bid-1', + cpm: 4.5, + width: 300, + height: 250, + ad: '
Creative
', + currency: 'USD', + netRevenue: true, + bidderCode: 'appnexus', + }) + ); + }); + + it('interpretResponse handles empty/missing seatbid', () => { + const spec = getAdapterSpec(); + const built = spec.buildRequests([]); + + expect(spec.interpretResponse({ body: {} }, built)).toEqual([]); + expect(spec.interpretResponse({ body: null }, built)).toEqual([]); + expect(spec.interpretResponse({}, built)).toEqual([]); + }); + + it('keeps request mapping isolated across overlapping auctions', () => { + const spec = getAdapterSpec(); + + const requestA = spec.buildRequests([ + { + adUnitCode: 'slot-a', + bidId: 'bid-a', + bidder: 'trustedServer', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + }, + ]); + const requestB = spec.buildRequests([ + { + adUnitCode: 'slot-b', + bidId: 'bid-b', + bidder: 'trustedServer', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + }, + ]); + + const responseA = { + body: { + seatbid: [ + { + seat: 'appnexus', + bid: [{ impid: 'slot-a', price: 1.1, adm: '
A
', w: 300, h: 250 }], + }, + ], + }, + }; + const responseB = { + body: { + seatbid: [ + { + seat: 'rubicon', + bid: [{ impid: 'slot-b', price: 2.2, adm: '
B
', w: 300, h: 250 }], + }, + ], + }, + }; + + const bidsA = spec.interpretResponse(responseA, requestA); + const bidsB = spec.interpretResponse(responseB, requestB); + + expect(bidsA[0].requestId).toBe('bid-a'); + expect(bidsB[0].requestId).toBe('bid-b'); + }); + }); + + describe('requestBids shim', () => { + beforeEach(() => { + testWindow.__tsjs_prebid = { + serverSideBidders: ['appnexus', 'rubicon', 'kargo', 'openx'], + }; + }); + + it('limits a global request to opts.adUnitCodes', () => { + const selected = document.createElement('div'); + selected.id = 'selected-global-unit'; + const unselected = document.createElement('div'); + unselected.id = 'unselected-global-unit'; + document.body.append(selected, unselected); + const selectedUnit = { + code: selected.id, + bids: [{ bidder: 'appnexus', params: { placementId: 1 } }], + }; + const unselectedUnit = { + code: unselected.id, + bids: [{ bidder: 'rubicon', params: { accountId: 2 } }], + }; + mockPbjs.adUnits = [selectedUnit, unselectedUnit]; + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ adUnitCodes: [selected.id] } as unknown as RequestBidsArg); + + expect(selectedUnit.bids.map((bid) => bid.bidder)).toEqual(['trustedServer']); + expect(unselectedUnit.bids).toEqual([{ bidder: 'rubicon', params: { accountId: 2 } }]); + expect((testWindow.tsjs as TsjsApi).firstImpression?.slots[selected.id]).toBeDefined(); + expect((testWindow.tsjs as TsjsApi).firstImpression?.slots[unselected.id]).toBeUndefined(); + + selected.remove(); + unselected.remove(); + }); + + it('preserves publisher ts adserverTargeting while adding trustedServer settings', () => { + const publisherTargeting = [{ key: 'ts', val: () => 'publisher-value' }]; + mockPbjs.bidderSettings = { + exampleBidder: { adserverTargeting: publisherTargeting }, + }; + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ bids: [{ bidder: 'exampleBidder', params: {} }] }], + } as unknown as RequestBidsArg); + + const bidderSettings = mockPbjs.bidderSettings as { + exampleBidder: { adserverTargeting: typeof publisherTargeting }; + trustedServer: { + allowAlternateBidderCodes: boolean; + allowedAlternateBidderCodes: string[]; + }; + }; + expect(bidderSettings.exampleBidder.adserverTargeting).toBe(publisherTargeting); + expect(bidderSettings.exampleBidder.adserverTargeting[0].key).toBe('ts'); + expect(bidderSettings.exampleBidder.adserverTargeting[0].val()).toBe('publisher-value'); + expect(bidderSettings.trustedServer).toEqual( + expect.objectContaining({ + allowAlternateBidderCodes: true, + allowedAlternateBidderCodes: ['*'], + }) + ); + }); + + it('injects trustedServer bidder into every ad unit', () => { + const pbjs = installPrebidNpm(); + + const adUnits = [ + { bids: [{ bidder: 'appnexus', params: {} }] }, + { bids: [{ bidder: 'rubicon', params: {} }] }, + ]; + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + // Each ad unit should have trustedServer added + for (const unit of adUnits) { + const hasTsBidder = unit.bids.some((b: TestBid) => b.bidder === 'trustedServer'); + expect(hasTsBidder).toBe(true); + } + + const trustedServerBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer'); + expect(trustedServerBid.params.bidderParams).toEqual({ appnexus: {} }); + expect(adUnits[0].bids.map((b: TestBid) => b.bidder)).toEqual(['trustedServer']); + expect(adUnits[1].bids.map((b: TestBid) => b.bidder)).toEqual(['trustedServer']); + + // Should call through to original requestBids + expect(mockRequestBids).toHaveBeenCalled(); + }); + + it('does not duplicate trustedServer if already present', () => { + const pbjs = installPrebidNpm(); + + const adUnits = [{ bids: [{ bidder: 'trustedServer', params: {} }] }]; + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + const tsCount = adUnits[0].bids.filter((b: TestBid) => b.bidder === 'trustedServer').length; + expect(tsCount).toBe(1); + }); + + it('folds only authoritative routes across mixed client, PBS, APS, and standard demand', () => { + testWindow.__tsjs_prebid = { + serverSideBidders: ['pbsRoute', 'standardRoute'], + clientSideBidders: ['exampleBrowser'], + }; + const pbjs = installPrebidNpm(); + + const adUnits = [ + { + bids: [ + { bidder: 'exampleBrowser', params: { placement: 'browser' } }, + { bidder: 'pbsRoute', params: { placement: 'pbs' } }, + { bidder: 'aps', params: { slot: 'aps' } }, + { bidder: 'standardRoute', params: { placement: 'standard' } }, + { bidder: 'pbs-provider-id', params: { forbidden: true } }, + ], + }, + ]; + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + const trustedServerBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer'); + expect(trustedServerBid?.params?.bidderParams).toEqual({ + pbsRoute: { placement: 'pbs' }, + standardRoute: { placement: 'standard' }, + }); + expect(adUnits[0].bids.map((b: TestBid) => b.bidder)).toEqual([ + 'exampleBrowser', + 'aps', + 'pbs-provider-id', + 'trustedServer', + ]); + }); + + it('preserves prototype-named server-side bidders as owned JSON properties', () => { + testWindow.__tsjs_prebid = { serverSideBidders: ['__proto__'] }; + const pbjs = installPrebidNpm(); + const adUnits = [ + { + bids: [{ bidder: '__proto__', params: { placement: 'server-owned' } }], + }, + ]; + + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + const trustedServerBid = adUnits[0].bids.find((bid) => bid.bidder === 'trustedServer'); + const bidderParams = trustedServerBid?.params?.bidderParams as Record; + expect(Object.prototype.hasOwnProperty.call(bidderParams, '__proto__')).toBe(true); + expect(bidderParams['__proto__']).toEqual({ placement: 'server-owned' }); + expect(JSON.parse(JSON.stringify(bidderParams))).toEqual( + Object.fromEntries([['__proto__', { placement: 'server-owned' }]]) + ); + expect(adUnits[0].bids.map((bid) => bid.bidder)).toEqual(['trustedServer']); + }); + + it('does not let returned bidder aliases or APS renderer aliases affect folding', () => { + testWindow.__tsjs_prebid = { serverSideBidders: ['configuredRoute'] }; + const pbjs = installPrebidNpm(); + const adUnits = [ + { + bids: [ + { bidder: 'configuredRoute', params: { placement: 1 } }, + { bidder: 'alternateReturnedSeat', params: { placement: 2 } }, + { bidder: 'apsRendererAlias', params: { placement: 3 } }, + ], + }, + ]; + + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + const trustedServerBid = adUnits[0].bids.find((bid) => bid.bidder === 'trustedServer'); + expect(trustedServerBid?.params?.bidderParams).toEqual({ + configuredRoute: { placement: 1 }, + }); + expect(adUnits[0].bids.map((bid) => bid.bidder)).toEqual([ + 'alternateReturnedSeat', + 'apsRendererAlias', + 'trustedServer', + ]); + }); + + it('preserves captured bidder params when requestBids runs twice on the same ad unit', () => { + const pbjs = installPrebidNpm(); + + // First auction: inline server-side params supplied by the publisher. + const adUnits = [ + { + code: 'div-1', + bids: [ + { bidder: 'appnexus', params: { placementId: 123 } }, + { bidder: 'rubicon', params: { accountId: 'abc' } }, + ], + }, + ]; + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + // Second auction (refresh/re-auction) with the SAME ad unit object: the + // server-side bidder entries were already pruned, so the shim must not + // overwrite the captured params with an empty object. + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + const trustedServerBid = adUnits[0].bids.find( + (b: TestBid) => b.bidder === 'trustedServer' + ) as TestBid; + expect(trustedServerBid.params.bidderParams).toEqual({ + appnexus: { placementId: 123 }, + rubicon: { accountId: 'abc' }, + }); + }); + + it('adds bids array to ad units that have none', () => { + const pbjs = installPrebidNpm(); + + const adUnits = [{ code: 'div-1' }] as TestAdUnit[]; + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + expect(adUnits[0].bids).toHaveLength(1); + expect(adUnits[0].bids[0].bidder).toBe('trustedServer'); + }); + + it('normalizes a truthy non-array bids value without throwing', () => { + const pbjs = installPrebidNpm(); + const adUnits = [ + { code: 'example-malformed-slot', bids: { malformed: true } }, + ] as TestAdUnit[]; + + expect(() => pbjs.requestBids({ adUnits } as unknown as RequestBidsArg)).not.toThrow(); + + expect(adUnits[0].bids).toEqual([{ bidder: 'trustedServer', params: { bidderParams: {} } }]); + }); + + it('preserves the empty stored-request envelope on initial and repeated requests', () => { + const pbjs = installPrebidNpm(); + const adUnits = [ + { + code: 'stored-slot', + bids: [{ bidder: 'trustedServer', params: { bidderParams: {} } }], + }, + ]; + + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + expect(adUnits[0].bids).toEqual([{ bidder: 'trustedServer', params: { bidderParams: {} } }]); + }); + + it('includes zone from mediaTypes.banner.name in trustedServer params', () => { + const pbjs = installPrebidNpm(); + + const adUnits = [ + { + code: 'ad-header-0', + mediaTypes: { banner: { name: 'header', sizes: [[728, 90]] } }, + bids: [{ bidder: 'kargo', params: { placementId: '_abc' } }], + }, + { + code: 'ad-fixed_bottom-0', + mediaTypes: { banner: { name: 'fixed_bottom', sizes: [[728, 90]] } }, + bids: [{ bidder: 'kargo', params: { placementId: '_def' } }], + }, + ]; + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + const tsBid0 = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; + expect(tsBid0.params.zone).toBe('header'); + + const tsBid1 = adUnits[1].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; + expect(tsBid1.params.zone).toBe('fixed_bottom'); + }); + + it('omits zone when mediaTypes.banner.name is not set', () => { + const pbjs = installPrebidNpm(); + + const adUnits = [ + { + code: 'ad-header-0', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + bids: [{ bidder: 'appnexus', params: {} }], + }, + ]; + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + const tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; + expect(tsBid.params.zone).toBeUndefined(); + }); + + it('omits zone when ad unit has no mediaTypes', () => { + const pbjs = installPrebidNpm(); + + const adUnits = [{ bids: [{ bidder: 'rubicon', params: {} }] }]; + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + const tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; + expect(tsBid.params.zone).toBeUndefined(); + }); + + it('clears stale zone when existing trustedServer bid is reused', () => { + const pbjs = installPrebidNpm(); + + const adUnits = [ + { + code: 'ad-header-0', + mediaTypes: { banner: { name: 'header', sizes: [[300, 250]] } }, + bids: [ + { bidder: 'trustedServer', params: { custom: 'keep' } }, + { bidder: 'kargo', params: { placementId: '_abc' } }, + ], + }, + ]; + + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + let tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; + expect(tsBid.params.zone).toBe('header'); + expect(tsBid.params.custom).toBe('keep'); + + delete adUnits[0].mediaTypes.banner.name; + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; + expect(tsBid.params.zone).toBeUndefined(); + expect(tsBid.params.custom).toBe('keep'); + }); + + it('falls back to pbjs.adUnits when requestObj has no adUnits', () => { + const pbjs = installPrebidNpm(); + + mockPbjs.adUnits = [{ bids: [{ bidder: 'openx', params: {} }] }] as TestAdUnit[]; + pbjs.requestBids({} as RequestBidsArg); + + const hasTsBidder = (mockPbjs.adUnits[0].bids ?? []).some( + (b: TestBid) => b.bidder === 'trustedServer' + ); + expect(hasTsBidder).toBe(true); + }); + + it('syncs a structured ts-eids cookie after bidsBackHandler', () => { + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + mockGetUserIdsAsEids.mockReturnValue([ + { + source: 'sharedid.org', + uids: [ + { id: 'shared_123', atype: 3 }, + { id: 'shared_456', ext: { provider: 'example' } }, + ], + }, + ]); + + const pbjs = installPrebidNpm(); + pbjs.requestBids({ + adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], + } as unknown as RequestBidsArg); + + const cookieValue = document.cookie.match(/(?:^|; )ts-eids=([^;]+)/)?.[1]; + expect(cookieValue).toBeDefined(); + expect(JSON.parse(atob(cookieValue!))).toEqual([ + { + source: 'sharedid.org', + uids: [ + { id: 'shared_123', atype: 3 }, + { id: 'shared_456', ext: { provider: 'example' } }, + ], + }, + ]); + }); + + it('preserves an opaque LiveRamp envelope in the ts-eids cookie', () => { + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + mockGetUserIdsAsEids.mockReturnValue([ + { + source: 'liveramp.com', + uids: [{ id: 'opaque-test-envelope', atype: 3 }], + }, + ]); + + const pbjs = installPrebidNpm(); + pbjs.requestBids({ + adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], + } as unknown as RequestBidsArg); + + const cookieValue = document.cookie.match(/(?:^|; )ts-eids=([^;]+)/)?.[1]; + expect(cookieValue).toBeDefined(); + expect(JSON.parse(atob(cookieValue!))).toEqual([ + { + source: 'liveramp.com', + uids: [{ id: 'opaque-test-envelope', atype: 3 }], + }, + ]); + }); + + it('clears ts-eids cookie after bidsBackHandler when no current EIDs remain', () => { + document.cookie = `ts-eids=${btoa(JSON.stringify([{ source: 'sharedid.org', uids: [{ id: 'stale' }] }]))}`; + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + mockGetUserIdsAsEids.mockReturnValue([]); + + const pbjs = installPrebidNpm(); + pbjs.requestBids({ + adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], + } as unknown as RequestBidsArg); + + expect(document.cookie).toBe(''); + }); + }); +}); + +describe('prebid/installPrebidNpm with server-injected config', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockPbjs.requestBids = mockRequestBids; + mockPbjs.adUnits = []; + mockGetUserIdsAsEids.mockReset(); + mockGetUserIdsAsEids.mockReturnValue([]); + document.cookie = 'ts-eids=; Path=/; Max-Age=0'; + delete testWindow.__tsjs_prebid; + }); + + afterEach(() => { + delete testWindow.__tsjs_prebid; + }); + + it('reads timeout and debug from window.__tsjs_prebid', () => { + testWindow.__tsjs_prebid = { timeout: 1500, debug: true }; + + installPrebidNpm(); + + expect(mockSetConfig).toHaveBeenCalledWith( + expect.objectContaining({ debug: true, bidderTimeout: 1500 }) + ); + }); + + it('keeps browser timeout and debug independent from multiple PBS routes', () => { + testWindow.__tsjs_prebid = { + timeout: 1750, + debug: false, + serverSideBidders: ['pbsPrimaryRoute', 'pbsSecondaryRoute'], + }; + + installPrebidNpm(); + + expect(mockSetConfig).toHaveBeenCalledWith({ debug: false, bidderTimeout: 1750 }); + }); + + it('explicit config overrides server-injected values', () => { + testWindow.__tsjs_prebid = { timeout: 1500, debug: true }; + + installPrebidNpm({ timeout: 3000, debug: false }); + + expect(mockSetConfig).toHaveBeenCalledWith( + expect.objectContaining({ debug: false, bidderTimeout: 3000 }) + ); + }); + + it('works with no config argument and no injected config', () => { + installPrebidNpm(); + + expect(mockSetConfig).toHaveBeenCalledWith(expect.objectContaining({ debug: false })); + expect(mockProcessQueue).toHaveBeenCalled(); + }); +}); + +describe('prebid/installRefreshHandler', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockRequestBids.mockReset(); + mockPbjs.requestBids = mockRequestBids; + mockPbjs.adUnits = []; + mockPbjs.setTargetingForGPTAsync = undefined; + testWindow.tsjs = undefined; + delete testWindow.googletag; + testWindow.__tsjs_prebid = { + serverSideBidders: ['appnexus', 'rubicon', 'kargo', 'openx', 'exampleServer'], + }; + document.body.replaceChildren(); + }); + + afterEach(() => { + testWindow.tsjs = undefined; + delete testWindow.googletag; + delete testWindow.__tsjs_prebid; + document.body.replaceChildren(); + }); + + function attachTestSlot(code: string): void { + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); + } + + it('builds refresh ad units from injected slot metadata', () => { + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + testWindow.tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [ + [970, 250], + [728, 90], + ], + targeting: { zone: 'homepage', pos: 'atf' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + timeout: 750, + adUnits: [ + expect.objectContaining({ code: 'div-ad-homepage-header', mediaTypes: { banner: { - name: 'homepage', - sizes: [ - [970, 250], - [728, 90], - ], + name: 'homepage', + sizes: [ + [970, 250], + [728, 90], + ], + }, + }, + bids: [{ bidder: 'trustedServer', params: { zone: 'homepage' } }], + }), + ], + }) + ); + }); + + it('resolves the exact slot when div_ids share a prefix', () => { + // Regression: a single find() with a startsWith() clause returned the + // first slot whose div_id is a prefix of the element id. With div_ids + // "div-ad" and "div-ad-header", refreshing the "div-ad-header" element + // must resolve to the header slot, not the shorter prefix slot. + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + testWindow.tsjs = { + adSlots: [ + { + id: 'prefix_ad', + gam_unit_path: '/123/prefix', + div_id: 'div-ad', + formats: [[300, 250]], + targeting: { zone: 'prefix' }, + }, + { + id: 'header_ad', + gam_unit_path: '/123/header', + div_id: 'div-ad-header', + formats: [[970, 250]], + targeting: { zone: 'header' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-header', + mediaTypes: { + banner: { + name: 'header', + sizes: [[970, 250]], + }, + }, + }), + ], + }) + ); + }); + + it('scopes the GPT targeting call to the refreshed slot code', () => { + const setTargetingForGPTAsync = vi.fn(); + mockPbjs.setTargetingForGPTAsync = setTargetingForGPTAsync; + // Run the bidsBackHandler synchronously so the targeting call fires. + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const originalRefresh = vi.fn(); + // Only the header slot is refreshed; the footer slot must be untouched. + const headerSlot = { + getSlotElementId: vi.fn(() => 'div-ad-header'), + getTargeting: vi.fn(() => []), + clearTargeting: vi.fn().mockReturnThis(), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [headerSlot]), + }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + testWindow.tsjs = { + adSlots: [ + { + id: 'header_ad', + gam_unit_path: '/123/header', + div_id: 'div-ad-header', + formats: [[728, 90]], + targeting: { zone: 'header' }, + }, + { + id: 'footer_ad', + gam_unit_path: '/123/footer', + div_id: 'div-ad-footer', + formats: [[728, 90]], + targeting: { zone: 'footer' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh([headerSlot]); + + expect(setTargetingForGPTAsync).toHaveBeenCalledTimes(1); + expect(setTargetingForGPTAsync).toHaveBeenCalledWith(['div-ad-header']); + expect(originalRefresh).toHaveBeenCalledWith([headerSlot], undefined); + + mockPbjs.setTargetingForGPTAsync = undefined; + }); + + it('includes every browser-owned bidder in refresh ad units', () => { + testWindow.__tsjs_prebid = { + clientSideBidders: ['rubicon'], + serverSideBidders: ['appnexus', 'exampleServer', 'kargo'], + }; + // Original publisher ad unit carries a client-side rubicon bid. + mockPbjs.adUnits = [ + { + code: 'div-ad-homepage-header', + bids: [ + { bidder: 'trustedServer', params: {} }, + { bidder: 'rubicon', params: { accountId: 1, siteId: 2, zoneId: 3 } }, + { bidder: 'publisherBrowserBidder', params: { placement: 'browser' } }, + ], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + testWindow.tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + bids: [ + { bidder: 'trustedServer', params: { zone: 'homepage' } }, + { bidder: 'rubicon', params: { accountId: 1, siteId: 2, zoneId: 3 } }, + { bidder: 'publisherBrowserBidder', params: { placement: 'browser' } }, + ], + }), + ], + }) + ); + + delete testWindow.__tsjs_prebid; + mockPbjs.adUnits = []; + }); + + it('preserves raw server-side bidder params in refresh ad units', () => { + // Original publisher ad unit carries an inline server-side appnexus bid that + // the initial auction has not yet folded into the trustedServer bid. + mockPbjs.adUnits = [ + { + code: 'div-ad-homepage-header', + bids: [{ bidder: 'appnexus', params: { placementId: 12345 } }], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + testWindow.tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + bids: [ + { + bidder: 'trustedServer', + params: { + zone: 'homepage', + bidderParams: { appnexus: { placementId: 12345 } }, + }, + }, + ], + }), + ], + }) + ); + + mockPbjs.adUnits = []; + }); + + it('recovers params and client-side bids for container-backed slots by injected div_id', () => { + // A TS-owned GPT slot may be defined on `${div_id}-container`, but the + // publisher's Prebid ad unit is keyed by the inner div_id. The synthetic + // refresh code stays the GPT element id (so GPT can match it), while params + // and client-side bids are recovered from the injected div_id candidate. + testWindow.__tsjs_prebid = { + clientSideBidders: ['rubicon'], + serverSideBidders: ['appnexus', 'exampleServer', 'kargo'], + }; + mockPbjs.adUnits = [ + { + code: 'div-ad-x', + bids: [ + { bidder: 'appnexus', params: { placementId: 12345 } }, + { bidder: 'rubicon', params: { accountId: 1 } }, + ], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-x-container'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + testWindow.tsjs = { + adSlots: [ + { + id: 'x_ad', + gam_unit_path: '/123/x', + div_id: 'div-ad-x', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + // Synthetic refresh code stays the GPT element id, not the div_id. + code: 'div-ad-x-container', + bids: [ + { + bidder: 'trustedServer', + params: { + zone: 'homepage', + bidderParams: { appnexus: { placementId: 12345 } }, + }, }, - }, - bids: [{ bidder: 'trustedServer', params: { zone: 'homepage' } }], + { bidder: 'rubicon', params: { accountId: 1 } }, + ], }), ], }) ); + + delete testWindow.__tsjs_prebid; + mockPbjs.adUnits = []; }); - it('resolves the exact slot when div_ids share a prefix', () => { - // Regression: a single find() with a startsWith() clause returned the - // first slot whose div_id is a prefix of the element id. With div_ids - // "div-ad" and "div-ad-header", refreshing the "div-ad-header" element - // must resolve to the header slot, not the shorter prefix slot. + it('recovers server-side bidder params already folded onto the original trustedServer bid', () => { + // After the initial auction, the requestBids shim has folded the publisher's + // server-side params into the original ad unit's trustedServer bid. A later + // refresh must still recover them by code. + mockPbjs.adUnits = [ + { + code: 'div-ad-homepage-header', + bids: [ + { + bidder: 'trustedServer', + params: { bidderParams: { appnexus: { placementId: 12345 } } }, + }, + ], + }, + ]; const originalRefresh = vi.fn(); const gptSlot = { - getSlotElementId: vi.fn(() => 'div-ad-header'), + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), getTargeting: vi.fn(() => []), }; const pubads = { @@ -1679,113 +2843,269 @@ describe('prebid/installRefreshHandler', () => { testWindow.tsjs = { adSlots: [ { - id: 'prefix_ad', - gam_unit_path: '/123/prefix', - div_id: 'div-ad', - formats: [[300, 250]], - targeting: { zone: 'prefix' }, + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + bids: [ + { + bidder: 'trustedServer', + params: { + zone: 'homepage', + bidderParams: { appnexus: { placementId: 12345 } }, + }, + }, + ], + }), + ], + }) + ); + + mockPbjs.adUnits = []; + }); + + it('auctions refreshed TS initial slots and clears stale TS targeting before refresh', () => { + const originalRefresh = vi.fn(); + const slotTargeting = new Map([ + ['ts_initial', ['1']], + ['zone', ['homepage']], + ]); + const clearTargeting = vi.fn((key: string) => { + slotTargeting.delete(key); + }); + const setTargeting = vi.fn((key: string, value: string | string[]) => { + slotTargeting.set(key, Array.isArray(value) ? value : [value]); + }); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn((key: string) => slotTargeting.get(key) ?? []), + getSizes: vi.fn(() => [ + { getWidth: () => 970, getHeight: () => 250 }, + { getWidth: () => 728, getHeight: () => 90 }, + ]), + clearTargeting, + setTargeting, + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + const setTargetingForGPTAsync = vi.fn(() => { + gptSlot.setTargeting('ts', 'prebid-value'); + }); + mockPbjs.setTargetingForGPTAsync = setTargetingForGPTAsync; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + testWindow.tsjs = { + adSlots: [ { - id: 'header_ad', - gam_unit_path: '/123/header', - div_id: 'div-ad-header', - formats: [[970, 250]], - targeting: { zone: 'header' }, + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [ + [970, 250], + [728, 90], + ], + targeting: { zone: 'homepage' }, }, ], }; installRefreshHandler(750); - pubads.refresh(); + pubads.refresh([gptSlot]); expect(mockRequestBids).toHaveBeenCalledWith( expect.objectContaining({ + timeout: 750, adUnits: [ expect.objectContaining({ - code: 'div-ad-header', + code: 'div-ad-homepage-header', mediaTypes: { banner: { - name: 'header', - sizes: [[970, 250]], + name: 'homepage', + sizes: [ + [970, 250], + [728, 90], + ], }, }, + bids: [{ bidder: 'trustedServer', params: { zone: 'homepage' } }], }), ], }) ); + expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); + expect(clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(clearTargeting).toHaveBeenCalledWith('hb_bidder'); + expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(clearTargeting).not.toHaveBeenCalledWith('ts'); + expect(originalRefresh).not.toHaveBeenCalled(); + + const bidsBackHandler = mockRequestBids.mock.calls[0][0].bidsBackHandler; + bidsBackHandler(); + + expect(setTargetingForGPTAsync).toHaveBeenCalled(); + expect(setTargetingForGPTAsync.mock.invocationCallOrder[0]).toBeLessThan( + originalRefresh.mock.invocationCallOrder[0] + ); + expect(slotTargeting.get('ts')).toEqual(['prebid-value']); + expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); + }); + + it('passes an explicitly excluded path directly to GPT after clearing stale targeting', () => { + const originalRefresh = vi.fn(); + const clearTargeting = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-tracking'), + getAdUnitPath: vi.fn(() => '/123/trackingonly'), + getTargeting: vi.fn(() => []), + clearTargeting, + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + testWindow.__tsjs_prebid = { + excludedGamAdUnitPathSuffixes: ['/trackingonly'], + }; + const options = { changeCorrelator: false }; + + installRefreshHandler(750); + pubads.refresh([gptSlot], options); + + expect(mockRequestBids).not.toHaveBeenCalled(); + expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); + expect(clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(clearTargeting).toHaveBeenCalledWith('hb_bidder'); + expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(originalRefresh).toHaveBeenCalledWith([gptSlot], options); + }); + + it('passes an all-excluded global refresh directly to GPT', () => { + const originalRefresh = vi.fn(); + const trackingSlot = { + getSlotElementId: vi.fn(() => 'div-ad-tracking'), + getAdUnitPath: vi.fn(() => '/123/trackingonly'), + getTargeting: vi.fn(() => []), + clearTargeting: vi.fn(), + }; + const measurementSlot = { + getSlotElementId: vi.fn(() => 'div-ad-measurement'), + getAdUnitPath: vi.fn(() => '/123/measurement-only'), + getTargeting: vi.fn(() => []), + clearTargeting: vi.fn(), + }; + const targetSlots = [trackingSlot, measurementSlot]; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => targetSlots), + }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + testWindow.__tsjs_prebid = { + excludedGamAdUnitPathSuffixes: ['/trackingonly', '/measurement-only'], + }; + const options = { changeCorrelator: false }; + + installRefreshHandler(750); + pubads.refresh(undefined, options); + + expect(mockRequestBids).not.toHaveBeenCalled(); + expect(trackingSlot.clearTargeting).toHaveBeenCalled(); + expect(measurementSlot.clearTargeting).toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledWith(undefined, options); }); - it('scopes the GPT targeting call to the refreshed slot code', () => { + it('auctions eligible slots and refreshes every slot in a mixed global refresh', () => { const setTargetingForGPTAsync = vi.fn(); mockPbjs.setTargetingForGPTAsync = setTargetingForGPTAsync; - // Run the bidsBackHandler synchronously so the targeting call fires. mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { opts?.bidsBackHandler?.(); }); const originalRefresh = vi.fn(); - // Only the header slot is refreshed; the footer slot must be untouched. - const headerSlot = { - getSlotElementId: vi.fn(() => 'div-ad-header'), + const displaySlot = { + getSlotElementId: vi.fn(() => 'div-ad-display'), + getAdUnitPath: vi.fn(() => '/123/content'), getTargeting: vi.fn(() => []), - clearTargeting: vi.fn().mockReturnThis(), + clearTargeting: vi.fn(), + }; + const trackingSlot = { + getSlotElementId: vi.fn(() => 'div-ad-tracking'), + getAdUnitPath: vi.fn(() => '/123/trackingonly'), + getTargeting: vi.fn(() => []), + clearTargeting: vi.fn(), }; + const targetSlots = [displaySlot, trackingSlot]; const pubads = { refresh: originalRefresh, - getSlots: vi.fn(() => [headerSlot]), + getSlots: vi.fn(() => targetSlots), }; testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - testWindow.tsjs = { - adSlots: [ - { - id: 'header_ad', - gam_unit_path: '/123/header', - div_id: 'div-ad-header', - formats: [[728, 90]], - targeting: { zone: 'header' }, - }, - { - id: 'footer_ad', - gam_unit_path: '/123/footer', - div_id: 'div-ad-footer', - formats: [[728, 90]], - targeting: { zone: 'footer' }, - }, - ], + testWindow.__tsjs_prebid = { + excludedGamAdUnitPathSuffixes: ['/trackingonly'], }; installRefreshHandler(750); - pubads.refresh([headerSlot]); + pubads.refresh(); - expect(setTargetingForGPTAsync).toHaveBeenCalledTimes(1); - expect(setTargetingForGPTAsync).toHaveBeenCalledWith(['div-ad-header']); - expect(originalRefresh).toHaveBeenCalledWith([headerSlot], undefined); + expect(displaySlot.clearTargeting).toHaveBeenCalled(); + expect(trackingSlot.clearTargeting).toHaveBeenCalled(); + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [expect.objectContaining({ code: 'div-ad-display' })], + }) + ); + expect(setTargetingForGPTAsync).toHaveBeenCalledWith(['div-ad-display']); + expect(originalRefresh).toHaveBeenCalledWith(targetSlots, undefined); mockPbjs.setTargetingForGPTAsync = undefined; }); - it('includes every browser-owned bidder in refresh ad units', () => { - testWindow.__tsjs_prebid = { - clientSideBidders: ['rubicon'], - serverSideBidders: ['appnexus', 'exampleServer', 'kargo'], - }; - // Original publisher ad unit carries a client-side rubicon bid. - mockPbjs.adUnits = [ + it.each([ + ['a missing path getter', {}], + ['a non-string path', { getAdUnitPath: vi.fn(() => 123) }], + [ + 'a throwing path getter', { - code: 'div-ad-homepage-header', - bids: [ - { bidder: 'trustedServer', params: {} }, - { bidder: 'rubicon', params: { accountId: 1, siteId: 2, zoneId: 3 } }, - { bidder: 'publisherBrowserBidder', params: { placement: 'browser' } }, - ], + getAdUnitPath: vi.fn(() => { + throw new Error('path unavailable'); + }), }, - ]; + ], + ])('fails open to an auction for %s', (_description, pathBehavior) => { const originalRefresh = vi.fn(); const gptSlot = { - getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getSlotElementId: vi.fn(() => 'div-ad-display'), getTargeting: vi.fn(() => []), + ...pathBehavior, }; const pubads = { refresh: originalRefresh, @@ -1795,53 +3115,80 @@ describe('prebid/installRefreshHandler', () => { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - testWindow.tsjs = { - adSlots: [ - { - id: 'homepage_header_ad', - gam_unit_path: '/123/homepage', - div_id: 'div-ad-homepage-header', - formats: [[728, 90]], - targeting: { zone: 'homepage' }, - }, - ], + testWindow.__tsjs_prebid = { + excludedGamAdUnitPathSuffixes: ['/trackingonly'], }; installRefreshHandler(750); - pubads.refresh(); + pubads.refresh([gptSlot]); - expect(mockRequestBids).toHaveBeenCalledWith( - expect.objectContaining({ - adUnits: [ - expect.objectContaining({ - code: 'div-ad-homepage-header', - bids: [ - { bidder: 'trustedServer', params: { zone: 'homepage' } }, - { bidder: 'rubicon', params: { accountId: 1, siteId: 2, zoneId: 3 } }, - { bidder: 'publisherBrowserBidder', params: { placement: 'browser' } }, - ], - }), - ], - }) - ); + expect(mockRequestBids).toHaveBeenCalled(); + expect(originalRefresh).not.toHaveBeenCalled(); + }); - delete testWindow.__tsjs_prebid; - mockPbjs.adUnits = []; + it.each([ + ['an empty suffix', ['']], + ['a non-array suffix list', {}], + ])('ignores %s from injected config and runs the refresh auction', (_description, suffixes) => { + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-display'), + getAdUnitPath: vi.fn(() => '/123/content'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + testWindow.__tsjs_prebid = { excludedGamAdUnitPathSuffixes: suffixes }; + + installRefreshHandler(750); + pubads.refresh([gptSlot]); + + expect(mockRequestBids).toHaveBeenCalled(); + expect(originalRefresh).not.toHaveBeenCalled(); }); - it('preserves raw server-side bidder params in refresh ad units', () => { - // Original publisher ad unit carries an inline server-side appnexus bid that - // the initial auction has not yet folded into the trustedServer bid. - mockPbjs.adUnits = [ - { - code: 'div-ad-homepage-header', - bids: [{ bidder: 'appnexus', params: { placementId: 12345 } }], - }, - ]; + it.each(['/123/TrackingOnly', '/123/trackingonly/'])( + 'uses literal case-sensitive suffix matching for %s', + (adUnitPath) => { + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-display'), + getAdUnitPath: vi.fn(() => adUnitPath), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + testWindow.__tsjs_prebid = { + excludedGamAdUnitPathSuffixes: ['/trackingonly'], + }; + + installRefreshHandler(750); + pubads.refresh([gptSlot]); + + expect(mockRequestBids).toHaveBeenCalled(); + expect(originalRefresh).not.toHaveBeenCalled(); + } + ); + + it('passes the adInit internal refresh straight to GPT without a client-side auction', () => { const originalRefresh = vi.fn(); + const clearTargeting = vi.fn(); const gptSlot = { getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), getTargeting: vi.fn(() => []), + clearTargeting, }; const pubads = { refresh: originalRefresh, @@ -1851,853 +3198,1079 @@ describe('prebid/installRefreshHandler', () => { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - testWindow.tsjs = { - adSlots: [ - { - id: 'homepage_header_ad', - gam_unit_path: '/123/homepage', - div_id: 'div-ad-homepage-header', - formats: [[728, 90]], - targeting: { zone: 'homepage' }, - }, - ], - }; + testWindow.tsjs = { adInitRefreshInProgress: true }; installRefreshHandler(750); - pubads.refresh(); - - expect(mockRequestBids).toHaveBeenCalledWith( - expect.objectContaining({ - adUnits: [ - expect.objectContaining({ - code: 'div-ad-homepage-header', - bids: [ - { - bidder: 'trustedServer', - params: { - zone: 'homepage', - bidderParams: { appnexus: { placementId: 12345 } }, - }, - }, - ], - }), - ], - }) - ); + pubads.refresh([gptSlot]); - mockPbjs.adUnits = []; + expect(mockRequestBids).not.toHaveBeenCalled(); + expect(clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); }); - it('recovers params and client-side bids for container-backed slots by injected div_id', () => { - // A TS-owned GPT slot may be defined on `${div_id}-container`, but the - // publisher's Prebid ad unit is keyed by the inner div_id. The synthetic - // refresh code stays the GPT element id (so GPT can match it), while params - // and client-side bids are recovered from the injected div_id candidate. - testWindow.__tsjs_prebid = { - clientSideBidders: ['rubicon'], - serverSideBidders: ['appnexus', 'exampleServer', 'kargo'], - }; - mockPbjs.adUnits = [ - { - code: 'div-ad-x', - bids: [ - { bidder: 'appnexus', params: { placementId: 12345 } }, - { bidder: 'rubicon', params: { accountId: 1 } }, - ], - }, - ]; + it('runs a client-side auction for publisher refreshes after adInit completes', () => { const originalRefresh = vi.fn(); const gptSlot = { - getSlotElementId: vi.fn(() => 'div-ad-x-container'), + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), getTargeting: vi.fn(() => []), + clearTargeting: vi.fn(), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + testWindow.tsjs = { adInitRefreshInProgress: false }; + + installRefreshHandler(750); + pubads.refresh([gptSlot]); + + expect(mockRequestBids).toHaveBeenCalled(); + expect(originalRefresh).not.toHaveBeenCalled(); + }); + + it('keeps nested Prebid refreshes Prebid-only and restores the diagnostics context', () => { + const listeners = new Map void>(); + const store = new GptDiagnosticsStore({ defer: () => undefined }); + const explicitSlot = { + getSlotElementId: () => 'nested-explicit', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const bareSlot = { + getSlotElementId: () => 'nested-bare', + getTargeting: () => [], + clearTargeting: vi.fn(), }; + let throwRefresh = false; + let getSlots: () => object[] = () => []; + const originalRefresh = vi.fn((slots?: unknown[]) => { + for (const slot of slots ?? getSlots()) listeners.get('slotRequested')?.({ slot }); + if (throwRefresh) throw new Error('delegated refresh failed'); + return 'delegated refresh result'; + }); const pubads = { + addEventListener: vi.fn((name: string, listener: (event: { slot: object }) => void) => { + listeners.set(name, listener); + }), refresh: originalRefresh, - getSlots: vi.fn(() => [gptSlot]), + getSlots: vi.fn(() => [bareSlot]), }; + getSlots = pubads.getSlots; testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; testWindow.tsjs = { - adSlots: [ - { - id: 'x_ad', - gam_unit_path: '/123/x', - div_id: 'div-ad-x', - formats: [[728, 90]], - targeting: { zone: 'homepage' }, - }, - ], + gptDiagnosticsRecorder: { + recordPrebidRefresh: (slots: object[]) => store.recordPrebidRefresh(slots), + }, }; + new GptDiagnosticsObserver(store).install(); installRefreshHandler(750); - pubads.refresh(); + const pbjs = installPrebidNpm(); - expect(mockRequestBids).toHaveBeenCalledWith( - expect.objectContaining({ - adUnits: [ - expect.objectContaining({ - // Synthetic refresh code stays the GPT element id, not the div_id. - code: 'div-ad-x-container', - bids: [ - { - bidder: 'trustedServer', - params: { - zone: 'homepage', - bidderParams: { appnexus: { placementId: 12345 } }, - }, - }, - { bidder: 'rubicon', params: { accountId: 1 } }, - ], - }), - ], - }) - ); + const prepareDelivery = (code: string) => { + if (!document.getElementById(code)) attachTestSlot(code); + mockRequestBids.mockImplementationOnce((options) => { + options.bidsBackHandler?.(); + }); + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: vi.fn(), + } as unknown as RequestBidsArg); + }; - delete testWindow.__tsjs_prebid; - mockPbjs.adUnits = []; - }); + prepareDelivery('nested-explicit'); + expect(pubads.refresh([explicitSlot])).toBe('delegated refresh result'); + expect(store.snapshot().slots[0].requests[0].requestPath).toBe('prebid_refresh'); + expect(Object.hasOwn(testWindow.tsjs as object, 'prebidRefreshDispatchInProgress')).toBe(false); - it('recovers server-side bidder params already folded onto the original trustedServer bid', () => { - // After the initial auction, the requestBids shim has folded the publisher's - // server-side params into the original ad unit's trustedServer bid. A later - // refresh must still recover them by code. - mockPbjs.adUnits = [ - { - code: 'div-ad-homepage-header', - bids: [ - { - bidder: 'trustedServer', - params: { bidderParams: { appnexus: { placementId: 12345 } } }, - }, - ], + prepareDelivery('nested-bare'); + expect(pubads.refresh()).toBe('delegated refresh result'); + expect(store.snapshot().slots[1].requests[0].requestPath).toBe('prebid_refresh'); + expect(Object.hasOwn(testWindow.tsjs as object, 'prebidRefreshDispatchInProgress')).toBe(false); + + prepareDelivery('nested-explicit'); + throwRefresh = true; + expect(() => pubads.refresh([explicitSlot])).toThrow('delegated refresh failed'); + expect(Object.hasOwn(testWindow.tsjs as object, 'prebidRefreshDispatchInProgress')).toBe(false); + + testWindow.tsjs = { + adInitRefreshInProgress: true, + gptDiagnosticsRecorder: { + recordPrebidRefresh: (slots: object[]) => store.recordPrebidRefresh(slots), }, - ]; - const originalRefresh = vi.fn(); - const gptSlot = { - getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), - getTargeting: vi.fn(() => []), }; + throwRefresh = false; + expect(pubads.refresh([explicitSlot])).toBe('delegated refresh result'); + expect(store.snapshot().slots[0].requests[2].requestPath).toBe('unattributed'); + }); + + it.each([ + { order: 'diagnostics observer first', diagnosticsFirst: true, expectedPath: 'prebid_refresh' }, + { order: 'Prebid wrapper first', diagnosticsFirst: false, expectedPath: 'competing' }, + ])( + 'attributes a Prebid-consumed refresh as $expectedPath when installed with the $order', + ({ diagnosticsFirst, expectedPath }) => { + const listeners = new Map void>(); + const store = new GptDiagnosticsStore({ defer: () => undefined }); + const slot = { + getSlotElementId: () => 'install-order', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const originalRefresh = vi.fn((slots?: unknown[]) => { + for (const refreshed of slots ?? []) listeners.get('slotRequested')?.({ slot: refreshed }); + return 'delegated refresh result'; + }); + const pubads = { + addEventListener: vi.fn((name: string, listener: (event: { slot: object }) => void) => { + listeners.set(name, listener); + }), + refresh: originalRefresh as (slots?: unknown[], opts?: unknown) => unknown, + getSlots: vi.fn(() => [slot]), + }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + testWindow.tsjs = { + gptDiagnosticsRecorder: { + recordPrebidRefresh: (slots: object[]) => store.recordPrebidRefresh(slots), + }, + }; + + // Only the bundle evaluation order enforces this today, so pin both + // outcomes: the diagnostics wrapper must sit inside the Prebid one to see + // the dispatch context that marks a refresh as Prebid's. + if (diagnosticsFirst) { + new GptDiagnosticsObserver(store).install(); + installRefreshHandler(750); + } else { + installRefreshHandler(750); + new GptDiagnosticsObserver(store).install(); + } + const pbjs = installPrebidNpm(); + attachTestSlot('install-order'); + mockRequestBids.mockImplementationOnce((options) => options.bidsBackHandler?.()); + pbjs.requestBids({ + adUnits: [{ code: 'install-order', bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: vi.fn(), + } as unknown as RequestBidsArg); + + pubads.refresh([slot]); + + expect(store.snapshot().slots[0].requests[0].requestPath).toBe(expectedPath); + } + ); + + it('keeps the outer dispatch context set across a nested Prebid refresh', () => { + const store = new GptDiagnosticsStore({ defer: () => undefined }); + const slot = { + getSlotElementId: () => 'nested-reentrant', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const contextAfterInner: Array = []; + let reentered = false; + const originalRefresh = vi.fn(() => { + if (!reentered) { + reentered = true; + pubads.refresh([slot]); + contextAfterInner.push( + (testWindow.tsjs as { prebidRefreshDispatchInProgress?: boolean }) + .prebidRefreshDispatchInProgress + ); + } + return 'delegated refresh result'; + }); const pubads = { - refresh: originalRefresh, - getSlots: vi.fn(() => [gptSlot]), + refresh: originalRefresh as (slots?: unknown[], opts?: unknown) => unknown, + getSlots: vi.fn(() => [slot]), }; testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; testWindow.tsjs = { - adSlots: [ - { - id: 'homepage_header_ad', - gam_unit_path: '/123/homepage', - div_id: 'div-ad-homepage-header', - formats: [[728, 90]], - targeting: { zone: 'homepage' }, - }, - ], + gptDiagnosticsRecorder: { + recordPrebidRefresh: (slots: object[]) => store.recordPrebidRefresh(slots), + }, }; + const pbjs = installPrebidNpm(); installRefreshHandler(750); - pubads.refresh(); - - expect(mockRequestBids).toHaveBeenCalledWith( - expect.objectContaining({ - adUnits: [ - expect.objectContaining({ - code: 'div-ad-homepage-header', - bids: [ - { - bidder: 'trustedServer', - params: { - zone: 'homepage', - bidderParams: { appnexus: { placementId: 12345 } }, - }, - }, - ], - }), - ], - }) - ); - - mockPbjs.adUnits = []; - }); - - it('auctions refreshed TS initial slots and clears stale TS targeting before refresh', () => { - const originalRefresh = vi.fn(); - const slotTargeting = new Map([ - ['ts_initial', ['1']], - ['zone', ['homepage']], - ]); - const clearTargeting = vi.fn((key: string) => { - slotTargeting.delete(key); - }); - const setTargeting = vi.fn((key: string, value: string | string[]) => { - slotTargeting.set(key, Array.isArray(value) ? value : [value]); - }); - const gptSlot = { - getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), - getTargeting: vi.fn((key: string) => slotTargeting.get(key) ?? []), - getSizes: vi.fn(() => [ - { getWidth: () => 970, getHeight: () => 250 }, - { getWidth: () => 728, getHeight: () => 90 }, - ]), - clearTargeting, - setTargeting, + attachTestSlot('nested-reentrant'); + mockRequestBids.mockImplementation((options) => options.bidsBackHandler?.()); + pbjs.requestBids({ + adUnits: [{ code: 'nested-reentrant', bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: vi.fn(), + } as unknown as RequestBidsArg); + + expect(pubads.refresh([slot])).toBe('delegated refresh result'); + // The inner dispatch owns the flag while it runs and must hand it back, or + // the observer would stop attributing every later publisher refresh. + expect(contextAfterInner).toEqual([true]); + expect( + originalRefresh, + 'the nested refresh must reach the delegated call' + ).toHaveBeenCalledTimes(2); + expect(Object.hasOwn(testWindow.tsjs as object, 'prebidRefreshDispatchInProgress')).toBe(false); + }); + + it('restores diagnostics context when its setter mutates and then throws', () => { + const slot = { + getSlotElementId: () => 'mutating-context-setter', + getTargeting: () => [], + clearTargeting: vi.fn(), }; + const originalRefresh = vi.fn(); const pubads = { refresh: originalRefresh, - getSlots: vi.fn(() => [gptSlot]), + getSlots: vi.fn(() => [slot]), }; - const setTargetingForGPTAsync = vi.fn(() => { - gptSlot.setTargeting('ts', 'prebid-value'); + const contextTarget: Record = {}; + let throwAfterMutation = true; + testWindow.tsjs = new Proxy(contextTarget, { + set(target, property, value) { + Reflect.set(target, property, value); + if (property === 'prebidRefreshDispatchInProgress' && throwAfterMutation) { + throwAfterMutation = false; + throw new Error('example mutating context setter failure'); + } + return true; + }, }); - mockPbjs.setTargetingForGPTAsync = setTargetingForGPTAsync; testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - testWindow.tsjs = { - adSlots: [ - { - id: 'homepage_header_ad', - gam_unit_path: '/123/homepage', - div_id: 'div-ad-homepage-header', - formats: [ - [970, 250], - [728, 90], - ], - targeting: { zone: 'homepage' }, - }, - ], - }; + mockRequestBids.mockImplementation((options) => options.bidsBackHandler?.()); + installPrebidNpm(); installRefreshHandler(750); - pubads.refresh([gptSlot]); + pubads.refresh([slot]); + pubads.refresh([slot]); - expect(mockRequestBids).toHaveBeenCalledWith( - expect.objectContaining({ - timeout: 750, - adUnits: [ - expect.objectContaining({ - code: 'div-ad-homepage-header', - mediaTypes: { - banner: { - name: 'homepage', - sizes: [ - [970, 250], - [728, 90], - ], - }, - }, - bids: [{ bidder: 'trustedServer', params: { zone: 'homepage' } }], - }), - ], - }) - ); - expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); - expect(clearTargeting).toHaveBeenCalledWith('hb_pb'); - expect(clearTargeting).toHaveBeenCalledWith('hb_bidder'); - expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); - expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); - expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); - expect(clearTargeting).not.toHaveBeenCalledWith('ts'); - expect(originalRefresh).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledTimes(2); + expect( + Object.prototype.hasOwnProperty.call(contextTarget, 'prebidRefreshDispatchInProgress') + ).toBe(false); + }); +}); - const bidsBackHandler = mockRequestBids.mock.calls[0][0].bidsBackHandler; - bidsBackHandler(); +describe('prebid publisher snapshots and delivery refreshes', () => { + let deliveryAdIds = new WeakMap(); + let installedGptSlots: Array> = []; + let auctionSequence = 0; - expect(setTargetingForGPTAsync).toHaveBeenCalled(); - expect(setTargetingForGPTAsync.mock.invocationCallOrder[0]).toBeLessThan( - originalRefresh.mock.invocationCallOrder[0] - ); - expect(slotTargeting.get('ts')).toEqual(['prebid-value']); - expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); + beforeEach(() => { + vi.clearAllMocks(); + deliveryAdIds = new WeakMap(); + installedGptSlots = []; + auctionSequence = 0; + mockRequestBids.mockReset(); + mockPbjs.requestBids = mockRequestBids; + mockPbjs.removeAdUnit = mockRemoveAdUnit; + delete (mockPbjs as unknown as Record).__tsRemoveAdUnitWrapped; + delete (mockPbjs as unknown as Record).__tsDiagnosticsBidWonInstalled; + mockPbjs.adUnits = []; + mockGetUserIdsAsEids.mockReset(); + mockGetUserIdsAsEids.mockReturnValue([]); + // By default the manifest declares all adapters compiled in. + (window as unknown as { __tsjs_prebid_bundle?: unknown }).__tsjs_prebid_bundle = + DEFAULT_BUNDLE_MANIFEST; + mockPbjs.setTargetingForGPTAsync = undefined; + testWindow.__tsjs_prebid = { + serverSideBidders: ['exampleServer', 'exampleFallback'], + }; + testWindow.tsjs = undefined; + delete testWindow.googletag; + document.body.replaceChildren(); }); - it('passes an explicitly excluded path directly to GPT after clearing stale targeting', () => { + afterEach(() => { + delete testWindow.__tsjs_prebid; + testWindow.tsjs = undefined; + delete testWindow.googletag; + document.body.replaceChildren(); + }); + + function installGpt(slots: Array>) { + installedGptSlots = slots; + for (const slot of slots) { + if (!slot || typeof slot !== 'object') continue; + const elementId = slot.getSlotElementId?.(); + if (typeof elementId === 'string' && elementId && !document.getElementById(elementId)) { + const element = document.createElement('div'); + element.id = elementId; + document.body.appendChild(element); + } + const originalGetTargeting = slot.getTargeting?.bind(slot); + slot.getTargeting = (key: string) => { + const deliveryAdId = deliveryAdIds.get(slot); + if (key === 'hb_adid' && deliveryAdId) return [deliveryAdId]; + return originalGetTargeting?.(key) ?? []; + }; + } + const originalRefresh = vi.fn(); - const clearTargeting = vi.fn(); - const gptSlot = { - getSlotElementId: vi.fn(() => 'div-ad-tracking'), - getAdUnitPath: vi.fn(() => '/123/trackingonly'), - getTargeting: vi.fn(() => []), - clearTargeting, - }; const pubads = { refresh: originalRefresh, - getSlots: vi.fn(() => [gptSlot]), + getSlots: vi.fn(() => slots), }; testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - testWindow.__tsjs_prebid = { - excludedGamAdUnitPathSuffixes: ['/trackingonly'], + installRefreshHandler(640); + return { originalRefresh, pubads }; + } + + function refreshAdUnitFromLastRequest(): + | (Record & { code?: string; bids?: TestBid[] }) + | undefined { + const lastCall = mockRequestBids.mock.calls[mockRequestBids.mock.calls.length - 1]; + return lastCall?.[0]?.adUnits?.[0]; + } + + function completePublisherAuction( + opts?: { adUnits?: Array<{ code?: string }>; bidsBackHandler?: (...args: unknown[]) => void }, + options: { auctionId?: string; applyTargeting?: boolean } = {} + ): void { + const auctionId = options.auctionId ?? `example-auction-${auctionSequence++}`; + const bidResponses: Record> }> = {}; + + for (const unit of opts?.adUnits ?? []) { + if (!unit.code) continue; + const adId = `${auctionId}-${unit.code}`; + bidResponses[unit.code] = { + bids: [{ adId, adUnitCode: unit.code, auctionId }], + }; + if (options.applyTargeting !== false) { + const slot = installedGptSlots.find((candidate) => { + const elementId = candidate?.getSlotElementId?.(); + return elementId === unit.code || elementId === `${unit.code}-container`; + }); + if (slot) deliveryAdIds.set(slot, adId); + } + } + + opts?.bidsBackHandler?.(bidResponses, false, auctionId); + } + + it('suppresses every publisher auction registered before the first TS delivery', () => { + const element = document.createElement('div'); + element.id = 'overlapping-first-impression'; + document.body.appendChild(element); + const ts = {} as TsjsApi; + claimFirstImpressionForTrustedServer(ts, element, 100); + const first = registerPublisherFirstImpressionAuctions(ts, [element.id], 101).get(element.id); + const second = registerPublisherFirstImpressionAuctions(ts, [element.id], 102).get(element.id); + + expect(consumePublisherFirstImpressionDelivery(ts, first, 103)).toBe(true); + expect(consumePublisherFirstImpressionDelivery(ts, second, 104)).toBe(true); + expect(registerPublisherFirstImpressionAuctions(ts, [element.id], 105)).toEqual(new Map()); + + element.remove(); + }); + + it('suppresses a correlated TS-owned delivery after the five-second lease', () => { + const element = document.createElement('div'); + element.id = 'late-first-impression'; + document.body.appendChild(element); + const ts = {} as TsjsApi; + claimFirstImpressionForTrustedServer(ts, element, 100); + const token = registerPublisherFirstImpressionAuctions(ts, [element.id], 101).get(element.id); + + expect(consumePublisherFirstImpressionDelivery(ts, token, 5_102)).toBe(true); + + element.remove(); + }); + + it('rejects a connected claim whose element is no longer canonical for its ID', () => { + const element = document.createElement('div'); + element.id = 'replaced-canonical-element'; + document.body.appendChild(element); + const ts = {} as TsjsApi; + claimFirstImpressionForTrustedServer(ts, element, 100); + const token = registerPublisherFirstImpressionAuctions(ts, [element.id], 101).get(element.id); + const replacement = document.createElement('div'); + replacement.id = element.id; + document.body.insertBefore(replacement, element); + + expect(document.getElementById(element.id)).toBe(replacement); + expect(consumePublisherFirstImpressionDelivery(ts, token, 102)).toBe(false); + expect(ts.firstImpression?.slots[element.id]).toBeUndefined(); + + replacement.remove(); + element.remove(); + }); + + it('prunes a claim stored under a registry key that does not match its slot element ID', () => { + const element = document.createElement('div'); + element.id = 'malformed-registry-key-slot'; + document.body.appendChild(element); + const ts = {} as TsjsApi; + const claim = claimFirstImpressionForTrustedServer(ts, element, 100)!; + delete ts.firstImpression!.slots[element.id]; + ts.firstImpression!.slots['wrong-registry-key'] = claim; + + expect(firstImpressionClaim(ts, element)).toBeUndefined(); + expect(ts.firstImpression!.slots['wrong-registry-key']).toBeUndefined(); + + element.remove(); + }); + + it('rejects a connected same-ID TS claim from a foreign document', () => { + const element = document.createElement('div'); + element.id = 'foreign-document-claim-slot'; + document.body.appendChild(element); + const foreignDocument = document.implementation.createHTMLDocument('foreign'); + const foreignElement = foreignDocument.createElement('div'); + foreignElement.id = element.id; + foreignDocument.body.appendChild(foreignElement); + const ts = {} as TsjsApi; + const claim = claimFirstImpressionForTrustedServer(ts, element, 100)!; + const token = registerPublisherFirstImpressionAuctions(ts, [element.id], 101).get(element.id); + claim.element = foreignElement; + + expect(foreignElement.isConnected).toBe(true); + expect(consumePublisherFirstImpressionDelivery(ts, token, 102)).toBe(false); + expect(ts.firstImpression?.slots[element.id]).toBeUndefined(); + expect(claimFirstImpressionForTrustedServer(ts, element, 103)?.element).toBe(element); + + element.remove(); + }); + + it('prunes an ordinary expired publisher registration without a reserved fallback', () => { + const element = document.createElement('div'); + element.id = 'ordinary-expired-publisher-slot'; + document.body.appendChild(element); + const ts = {} as TsjsApi; + const token = registerPublisherFirstImpressionAuctions(ts, [element.id], 100).get(element.id); + + expect(consumePublisherFirstImpressionDelivery(ts, token, 5_101)).toBe(false); + expect(ts.firstImpression?.slots[element.id]).toBeUndefined(); + + element.remove(); + }); + + it('clears a failed fallback reservation before a later ordinary publisher claim expires', () => { + vi.useFakeTimers(); + vi.setSystemTime(100); + try { + const element = document.createElement('div'); + element.id = 'failed-fallback-reservation-slot'; + document.body.appendChild(element); + const ts = {} as TsjsApi; + const originalToken = registerPublisherFirstImpressionAuctions(ts, [element.id]).get( + element.id + ); + expect(originalToken).toBeDefined(); + expect(reservePublisherFirstImpressionFallback(ts, element)).toBe(true); + + vi.advanceTimersByTime(5_001); + const fallbackClaim = claimFirstImpressionForTrustedServer(ts, element)!; + expect(fallbackClaim.owner).toBe('trusted_server'); + expect(fallbackClaim.publisherAuctions[originalToken!]?.suppressDelivery).toBe(true); + + releaseTrustedServerFirstImpressionClaim(ts, element, fallbackClaim); + expect(ts.firstImpression?.slots[element.id]).toBeUndefined(); + expect(ts.firstImpression?.fallbackSlots[element.id]).toBeUndefined(); + + const laterToken = registerPublisherFirstImpressionAuctions(ts, [element.id]).get(element.id); + expect(laterToken).toBeDefined(); + vi.advanceTimersByTime(5_001); + expect(consumePublisherFirstImpressionDelivery(ts, laterToken)).toBe(false); + expect(ts.firstImpression?.slots[element.id]).toBeUndefined(); + + const freshClaim = claimFirstImpressionForTrustedServer(ts, element)!; + expect(freshClaim.publisherAuctions).toEqual({}); + + element.remove(); + } finally { + vi.clearAllTimers(); + vi.useRealTimers(); + } + }); + + it('reserves first impression while a publisher refresh auction is pending', () => { + const code = 'pending-publisher-refresh-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), }; - const options = { changeCorrelator: false }; + const { originalRefresh, pubads } = installGpt([slot]); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; + }); + installPrebidNpm(); + + pubads.refresh([slot]); + + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + expect( + claimFirstImpressionForTrustedServer(ts, document.getElementById(code)!) + ).toBeUndefined(); + expect(originalRefresh).not.toHaveBeenCalled(); + + completeRefresh?.(); + + expect(originalRefresh).toHaveBeenCalledOnce(); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + it('suppresses an original publisher delivery after the lease-boundary TS fallback', () => { + vi.useFakeTimers(); + try { + const code = 'lease-boundary-fallback-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + setTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + let originalPublisherAuction: Parameters[0]; + mockRequestBids.mockImplementation((options) => { + if (!originalPublisherAuction) { + originalPublisherAuction = options; + return; + } + completePublisherAuction(options); + }); + const pbjs = installPrebidNpm(); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + ts.servicesEnabled = true; + ts.adSlots = [ + { + id: 'lease-boundary-fallback-ad', + gam_unit_path: '/123/lease-boundary', + div_id: code, + formats: [[300, 250]], + targeting: {}, + }, + ]; + ts.bids = { + 'lease-boundary-fallback-ad': { + hb_pb: '1.00', + hb_adid: 'trusted-server-fallback-ad', + }, + }; - installRefreshHandler(750); - pubads.refresh([gptSlot], options); + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh([slot]), + } as unknown as RequestBidsArg); + installTsAdInit(); + ts.adInit!(); - expect(mockRequestBids).not.toHaveBeenCalled(); - expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); - expect(clearTargeting).toHaveBeenCalledWith('hb_pb'); - expect(clearTargeting).toHaveBeenCalledWith('hb_bidder'); - expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); - expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); - expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); - expect(originalRefresh).toHaveBeenCalledWith([gptSlot], options); + vi.advanceTimersByTime(5001); + observeFirstImpressionGptLifecycle(ts, document.getElementById(code)!, 'requested'); + expect(originalRefresh).toHaveBeenCalledOnce(); + expect(ts.firstImpression?.slots[code]?.owner).toBe('trusted_server'); + expect(ts.firstImpression?.fallbackSlots[code]).toBe(document.getElementById(code)); + expect(Object.values(ts.firstImpression?.slots[code]?.publisherAuctions ?? {})).toEqual([ + expect.objectContaining({ suppressDelivery: true }), + ]); + + completePublisherAuction(originalPublisherAuction); + expect(originalRefresh).toHaveBeenCalledOnce(); + + deliveryAdIds.delete(slot); + pubads.refresh([slot]); + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenCalledTimes(2); + } finally { + vi.clearAllTimers(); + vi.useRealTimers(); + } }); - it('passes an all-excluded global refresh directly to GPT', () => { - const originalRefresh = vi.fn(); - const trackingSlot = { - getSlotElementId: vi.fn(() => 'div-ad-tracking'), - getAdUnitPath: vi.fn(() => '/123/trackingonly'), - getTargeting: vi.fn(() => []), - clearTargeting: vi.fn(), - }; - const measurementSlot = { - getSlotElementId: vi.fn(() => 'div-ad-measurement'), - getAdUnitPath: vi.fn(() => '/123/measurement-only'), - getTargeting: vi.fn(() => []), + it('suppresses a delayed publisher refresh when TS already owns first impression', () => { + const code = 'pending-ts-owned-refresh-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], clearTargeting: vi.fn(), + setTargeting: vi.fn(), }; - const targetSlots = [trackingSlot, measurementSlot]; - const pubads = { - refresh: originalRefresh, - getSlots: vi.fn(() => targetSlots), - }; - testWindow.googletag = { - cmd: { push: (fn: () => void) => fn() }, - pubads: () => pubads, - }; - testWindow.__tsjs_prebid = { - excludedGamAdUnitPathSuffixes: ['/trackingonly', '/measurement-only'], - }; - const options = { changeCorrelator: false }; + const { originalRefresh, pubads } = installGpt([slot]); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + claimFirstImpressionForTrustedServer(ts, document.getElementById(code)!); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; + }); + installPrebidNpm(); - installRefreshHandler(750); - pubads.refresh(undefined, options); + pubads.refresh([slot]); + expect(originalRefresh).not.toHaveBeenCalled(); - expect(mockRequestBids).not.toHaveBeenCalled(); - expect(trackingSlot.clearTargeting).toHaveBeenCalled(); - expect(measurementSlot.clearTargeting).toHaveBeenCalled(); - expect(originalRefresh).toHaveBeenCalledWith(undefined, options); + completeRefresh?.(); + + expect(originalRefresh).not.toHaveBeenCalled(); }); - it('auctions eligible slots and refreshes every slot in a mixed global refresh', () => { - const setTargetingForGPTAsync = vi.fn(); - mockPbjs.setTargetingForGPTAsync = setTargetingForGPTAsync; - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); - const originalRefresh = vi.fn(); - const displaySlot = { - getSlotElementId: vi.fn(() => 'div-ad-display'), - getAdUnitPath: vi.fn(() => '/123/content'), - getTargeting: vi.fn(() => []), + it('filters only the TS-owned slot from a delayed mixed publisher refresh', () => { + const tsCode = 'pending-mixed-ts-slot'; + const publisherCode = 'pending-mixed-publisher-slot'; + const tsSlot = { + getSlotElementId: () => tsCode, + getTargeting: () => [], + getSizes: () => [[300, 250]], clearTargeting: vi.fn(), + setTargeting: vi.fn(), }; - const trackingSlot = { - getSlotElementId: vi.fn(() => 'div-ad-tracking'), - getAdUnitPath: vi.fn(() => '/123/trackingonly'), - getTargeting: vi.fn(() => []), + const publisherSlot = { + getSlotElementId: () => publisherCode, + getTargeting: () => [], + getSizes: () => [[300, 250]], clearTargeting: vi.fn(), }; - const targetSlots = [displaySlot, trackingSlot]; - const pubads = { - refresh: originalRefresh, - getSlots: vi.fn(() => targetSlots), - }; - testWindow.googletag = { - cmd: { push: (fn: () => void) => fn() }, - pubads: () => pubads, - }; - testWindow.__tsjs_prebid = { - excludedGamAdUnitPathSuffixes: ['/trackingonly'], - }; - - installRefreshHandler(750); - pubads.refresh(); + const { originalRefresh, pubads } = installGpt([tsSlot, publisherSlot]); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + claimFirstImpressionForTrustedServer(ts, document.getElementById(tsCode)!); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; + }); + installPrebidNpm(); - expect(displaySlot.clearTargeting).toHaveBeenCalled(); - expect(trackingSlot.clearTargeting).toHaveBeenCalled(); - expect(mockRequestBids).toHaveBeenCalledWith( - expect.objectContaining({ - adUnits: [expect.objectContaining({ code: 'div-ad-display' })], - }) - ); - expect(setTargetingForGPTAsync).toHaveBeenCalledWith(['div-ad-display']); - expect(originalRefresh).toHaveBeenCalledWith(undefined, undefined); + pubads.refresh([tsSlot, publisherSlot]); + completeRefresh?.(); - mockPbjs.setTargetingForGPTAsync = undefined; + expect(originalRefresh).toHaveBeenCalledOnce(); + expect(originalRefresh).toHaveBeenCalledWith([publisherSlot], undefined); }); - it.each([ - ['a missing path getter', {}], - ['a non-string path', { getAdUnitPath: vi.fn(() => 123) }], - [ - 'a throwing path getter', - { - getAdUnitPath: vi.fn(() => { - throw new Error('path unavailable'); - }), - }, - ], - ])('fails open to an auction for %s', (_description, pathBehavior) => { - const originalRefresh = vi.fn(); - const gptSlot = { - getSlotElementId: vi.fn(() => 'div-ad-display'), - getTargeting: vi.fn(() => []), - ...pathBehavior, - }; - const pubads = { - refresh: originalRefresh, - getSlots: vi.fn(() => [gptSlot]), - }; - testWindow.googletag = { - cmd: { push: (fn: () => void) => fn() }, - pubads: () => pubads, + it('filters a TS-owned excluded slot from a delayed mixed publisher refresh', () => { + const eligibleCode = 'pending-mixed-eligible-slot'; + const excludedCode = 'pending-mixed-excluded-slot'; + const eligibleSlot = { + getSlotElementId: () => eligibleCode, + getAdUnitPath: () => '/123/content', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), }; - testWindow.__tsjs_prebid = { - excludedGamAdUnitPathSuffixes: ['/trackingonly'], + const excludedSlot = { + getSlotElementId: () => excludedCode, + getAdUnitPath: () => '/123/trackingonly', + getTargeting: () => [], + getSizes: () => [[1, 1]], + clearTargeting: vi.fn(), + setTargeting: vi.fn(), }; + const { originalRefresh, pubads } = installGpt([eligibleSlot, excludedSlot]); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + claimFirstImpressionForTrustedServer(ts, document.getElementById(excludedCode)!); + testWindow.__tsjs_prebid = { excludedGamAdUnitPathSuffixes: ['/trackingonly'] }; + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; + }); + installPrebidNpm(); - installRefreshHandler(750); - pubads.refresh([gptSlot]); + pubads.refresh([eligibleSlot, excludedSlot]); + completeRefresh?.(); - expect(mockRequestBids).toHaveBeenCalled(); - expect(originalRefresh).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledOnce(); + expect(originalRefresh).toHaveBeenCalledWith([eligibleSlot], undefined); }); - it.each([ - ['an empty suffix', ['']], - ['a non-array suffix list', {}], - ])('ignores %s from injected config and runs the refresh auction', (_description, suffixes) => { - const originalRefresh = vi.fn(); - const gptSlot = { - getSlotElementId: vi.fn(() => 'div-ad-display'), - getAdUnitPath: vi.fn(() => '/123/content'), - getTargeting: vi.fn(() => []), - }; - const pubads = { - refresh: originalRefresh, - getSlots: vi.fn(() => [gptSlot]), + it('drops delayed delivery and auction slots together after SPA navigation', () => { + const deliveryCode = 'pending-navigation-delivery-slot'; + const auctionCode = 'pending-navigation-auction-slot'; + const deliverySlot = { + getSlotElementId: () => deliveryCode, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), }; - testWindow.googletag = { - cmd: { push: (fn: () => void) => fn() }, - pubads: () => pubads, + const auctionSlot = { + getSlotElementId: () => auctionCode, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), }; - testWindow.__tsjs_prebid = { excludedGamAdUnitPathSuffixes: suffixes }; + const { originalRefresh, pubads } = installGpt([deliverySlot, auctionSlot]); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + if (opts?.adUnits?.[0]?.code === deliveryCode) { + completePublisherAuction(opts); + } else { + completeRefresh = opts.bidsBackHandler; + } + }); + const pbjs = installPrebidNpm(); - installRefreshHandler(750); - pubads.refresh([gptSlot]); + pbjs.requestBids({ + adUnits: [{ code: deliveryCode, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh([deliverySlot, auctionSlot]), + } as unknown as RequestBidsArg); + ((testWindow.tsjs ??= {}) as unknown as TsjsApi).navGeneration = 1; + completeRefresh?.(); - expect(mockRequestBids).toHaveBeenCalled(); expect(originalRefresh).not.toHaveBeenCalled(); }); - it.each(['/123/TrackingOnly', '/123/trackingonly/'])( - 'uses literal case-sensitive suffix matching for %s', - (adUnitPath) => { - const originalRefresh = vi.fn(); - const gptSlot = { - getSlotElementId: vi.fn(() => 'div-ad-display'), - getAdUnitPath: vi.fn(() => adUnitPath), - getTargeting: vi.fn(() => []), - }; - const pubads = { - refresh: originalRefresh, - getSlots: vi.fn(() => [gptSlot]), - }; - testWindow.googletag = { - cmd: { push: (fn: () => void) => fn() }, - pubads: () => pubads, - }; - testWindow.__tsjs_prebid = { - excludedGamAdUnitPathSuffixes: ['/trackingonly'], - }; - - installRefreshHandler(750); - pubads.refresh([gptSlot]); - - expect(mockRequestBids).toHaveBeenCalled(); - expect(originalRefresh).not.toHaveBeenCalled(); - } - ); - - it('passes the adInit internal refresh straight to GPT without a client-side auction', () => { - const originalRefresh = vi.fn(); - const clearTargeting = vi.fn(); - const gptSlot = { - getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), - getTargeting: vi.fn(() => []), - clearTargeting, - }; - const pubads = { - refresh: originalRefresh, - getSlots: vi.fn(() => [gptSlot]), - }; - testWindow.googletag = { - cmd: { push: (fn: () => void) => fn() }, - pubads: () => pubads, + it('drops a delayed publisher refresh after SPA navigation', () => { + const code = 'pending-previous-navigation-refresh-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), }; - testWindow.tsjs = { adInitRefreshInProgress: true }; + const { originalRefresh, pubads } = installGpt([slot]); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; + }); + installPrebidNpm(); - installRefreshHandler(750); - pubads.refresh([gptSlot]); + pubads.refresh([slot]); + ((testWindow.tsjs ??= {}) as unknown as TsjsApi).navGeneration = 1; + completeRefresh?.(); - expect(mockRequestBids).not.toHaveBeenCalled(); - expect(clearTargeting).not.toHaveBeenCalled(); - expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); + expect(originalRefresh).not.toHaveBeenCalled(); }); - it('runs a client-side auction for publisher refreshes after adInit completes', () => { - const originalRefresh = vi.fn(); - const gptSlot = { - getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), - getTargeting: vi.fn(() => []), + it('drops a delayed publisher refresh after physical element replacement', () => { + const code = 'pending-replaced-refresh-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], clearTargeting: vi.fn(), }; - const pubads = { - refresh: originalRefresh, - getSlots: vi.fn(() => [gptSlot]), - }; - testWindow.googletag = { - cmd: { push: (fn: () => void) => fn() }, - pubads: () => pubads, - }; - testWindow.tsjs = { adInitRefreshInProgress: false }; + const { originalRefresh, pubads } = installGpt([slot]); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; + }); + installPrebidNpm(); - installRefreshHandler(750); - pubads.refresh([gptSlot]); + pubads.refresh([slot]); + document.getElementById(code)?.remove(); + const replacement = document.createElement('div'); + replacement.id = code; + document.body.appendChild(replacement); + completeRefresh?.(); - expect(mockRequestBids).toHaveBeenCalled(); expect(originalRefresh).not.toHaveBeenCalled(); }); - it('keeps nested Prebid refreshes Prebid-only and restores the diagnostics context', () => { - const listeners = new Map void>(); - const store = new GptDiagnosticsStore({ defer: () => undefined }); - const explicitSlot = { - getSlotElementId: () => 'nested-explicit', + it('keeps a delayed bare refresh scoped to its captured slot list', () => { + const firstCode = 'pending-bare-first-slot'; + const laterCode = 'pending-bare-later-slot'; + const firstSlot = { + getSlotElementId: () => firstCode, getTargeting: () => [], + getSizes: () => [[300, 250]], clearTargeting: vi.fn(), }; - const bareSlot = { - getSlotElementId: () => 'nested-bare', + const laterSlot = { + getSlotElementId: () => laterCode, getTargeting: () => [], + getSizes: () => [[300, 250]], clearTargeting: vi.fn(), }; - let throwRefresh = false; - let getSlots: () => object[] = () => []; - const originalRefresh = vi.fn((slots?: unknown[]) => { - for (const slot of slots ?? getSlots()) listeners.get('slotRequested')?.({ slot }); - if (throwRefresh) throw new Error('delegated refresh failed'); - return 'delegated refresh result'; + const slots = [firstSlot]; + const { originalRefresh, pubads } = installGpt(slots); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; }); - const pubads = { - addEventListener: vi.fn((name: string, listener: (event: { slot: object }) => void) => { - listeners.set(name, listener); - }), - refresh: originalRefresh, - getSlots: vi.fn(() => [bareSlot]), + installPrebidNpm(); + + pubads.refresh(); + slots.push(laterSlot); + completeRefresh?.(); + + expect(originalRefresh).toHaveBeenCalledOnce(); + expect(originalRefresh).toHaveBeenCalledWith([firstSlot], undefined); + }); + + it('allows publisher refreshes that start after the TS first impression request', () => { + const code = 'requested-ts-owned-refresh-slot'; + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); + const ts = {} as TsjsApi; + claimFirstImpressionForTrustedServer(ts, element); + observeFirstImpressionGptLifecycle(ts, element, 'requested'); + + expect(registerPublisherFirstImpressionAuctions(ts, [code])).toEqual(new Map()); + expect(ts.firstImpression?.slots[code]?.publisherRegistrationClosed).toBe(true); + }); + + it('clears a stale GPT handoff when delegating a post-request publisher refresh', () => { + const code = 'post-request-handoff-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), }; - getSlots = pubads.getSlots; + installedGptSlots = [slot]; + const nativeRefresh = vi.fn(); + const ts = (testWindow.tsjs = {} as unknown as PrebidTestWindow['tsjs']) as unknown as TsjsApi; + const handoff = { + gamUnitPath: '/123/post-request', + formats: [[300, 250] as [number, number]], + divIdPrefix: code, + slotElementId: code, + publisherClaimed: true, + suppressPublisherDisplay: false, + suppressPublisherRefresh: true, + }; + ts.gptSlotHandoffs = { [code]: handoff }; + const innerRefresh = vi.fn((slots?: (typeof slot)[]) => { + if (handoff.suppressPublisherRefresh) { + handoff.suppressPublisherRefresh = false; + return; + } + nativeRefresh(slots); + }); + const pubads = { refresh: innerRefresh, getSlots: () => [slot] }; testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - testWindow.tsjs = { - gptDiagnosticsRecorder: { - recordPrebidRefresh: (slots: object[]) => store.recordPrebidRefresh(slots), - }, - }; - - new GptDiagnosticsObserver(store).install(); - installRefreshHandler(750); - const pbjs = installPrebidNpm(); - - const prepareDelivery = (code: string) => { - mockRequestBids.mockImplementationOnce((options) => { - options.bidsBackHandler?.(); - }); - pbjs.requestBids({ - adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], - bidsBackHandler: vi.fn(), - } as unknown as RequestBidsArg); - }; - - prepareDelivery('nested-explicit'); - expect(pubads.refresh([explicitSlot])).toBe('delegated refresh result'); - expect(store.snapshot().slots[0].requests[0].requestPath).toBe('prebid_refresh'); - expect(Object.hasOwn(testWindow.tsjs as object, 'prebidRefreshDispatchInProgress')).toBe(false); - - prepareDelivery('nested-bare'); - expect(pubads.refresh()).toBe('delegated refresh result'); - expect(store.snapshot().slots[1].requests[0].requestPath).toBe('prebid_refresh'); - expect(Object.hasOwn(testWindow.tsjs as object, 'prebidRefreshDispatchInProgress')).toBe(false); + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); + claimFirstImpressionForTrustedServer(ts, element); + observeFirstImpressionGptLifecycle(ts, element, 'requested'); + installRefreshHandler(640); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + installPrebidNpm(); - prepareDelivery('nested-explicit'); - throwRefresh = true; - expect(() => pubads.refresh([explicitSlot])).toThrow('delegated refresh failed'); - expect(Object.hasOwn(testWindow.tsjs as object, 'prebidRefreshDispatchInProgress')).toBe(false); + pubads.refresh([slot]); - testWindow.tsjs = { - adInitRefreshInProgress: true, - gptDiagnosticsRecorder: { - recordPrebidRefresh: (slots: object[]) => store.recordPrebidRefresh(slots), - }, - }; - throwRefresh = false; - expect(pubads.refresh([explicitSlot])).toBe('delegated refresh result'); - expect(store.snapshot().slots[0].requests[2].requestPath).toBe('unattributed'); + expect(handoff.suppressPublisherRefresh).toBe(false); + expect(nativeRefresh).toHaveBeenCalledWith([slot]); }); - it.each([ - { order: 'diagnostics observer first', diagnosticsFirst: true, expectedPath: 'prebid_refresh' }, - { order: 'Prebid wrapper first', diagnosticsFirst: false, expectedPath: 'competing' }, - ])( - 'attributes a Prebid-consumed refresh as $expectedPath when installed with the $order', - ({ diagnosticsFirst, expectedPath }) => { - const listeners = new Map void>(); - const store = new GptDiagnosticsStore({ defer: () => undefined }); - const slot = { - getSlotElementId: () => 'install-order', - getTargeting: () => [], - clearTargeting: vi.fn(), - }; - const originalRefresh = vi.fn((slots?: unknown[]) => { - for (const refreshed of slots ?? []) listeners.get('slotRequested')?.({ slot: refreshed }); - return 'delegated refresh result'; - }); - const pubads = { - addEventListener: vi.fn((name: string, listener: (event: { slot: object }) => void) => { - listeners.set(name, listener); - }), - refresh: originalRefresh as (slots?: unknown[], opts?: unknown) => unknown, - getSlots: vi.fn(() => [slot]), - }; - testWindow.googletag = { - cmd: { push: (fn: () => void) => fn() }, - pubads: () => pubads, - }; - testWindow.tsjs = { - gptDiagnosticsRecorder: { - recordPrebidRefresh: (slots: object[]) => store.recordPrebidRefresh(slots), - }, - }; - - // Only the bundle evaluation order enforces this today, so pin both - // outcomes: the diagnostics wrapper must sit inside the Prebid one to see - // the dispatch context that marks a refresh as Prebid's. - if (diagnosticsFirst) { - new GptDiagnosticsObserver(store).install(); - installRefreshHandler(750); - } else { - installRefreshHandler(750); - new GptDiagnosticsObserver(store).install(); - } - const pbjs = installPrebidNpm(); - mockRequestBids.mockImplementationOnce((options) => options.bidsBackHandler?.()); - pbjs.requestBids({ - adUnits: [{ code: 'install-order', bids: [{ bidder: 'exampleServer', params: {} }] }], - bidsBackHandler: vi.fn(), - } as unknown as RequestBidsArg); + it('suppresses an all-excluded refresh while the TS first impression is pending', () => { + const code = 'pending-all-excluded-slot'; + const slot = { + getSlotElementId: () => code, + getAdUnitPath: () => '/123/trackingonly', + getTargeting: () => [], + getSizes: () => [[1, 1]], + clearTargeting: vi.fn(), + setTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + claimFirstImpressionForTrustedServer(ts, document.getElementById(code)!); + testWindow.__tsjs_prebid = { excludedGamAdUnitPathSuffixes: ['/trackingonly'] }; + installPrebidNpm(); - pubads.refresh([slot]); + pubads.refresh([slot]); - expect(store.snapshot().slots[0].requests[0].requestPath).toBe(expectedPath); - } - ); + expect(mockRequestBids).not.toHaveBeenCalled(); + expect(originalRefresh).not.toHaveBeenCalled(); + }); - it('keeps the outer dispatch context set across a nested Prebid refresh', () => { - const store = new GptDiagnosticsStore({ defer: () => undefined }); + it('delegates an all-excluded refresh after the TS first impression request', () => { + const code = 'requested-all-excluded-slot'; const slot = { - getSlotElementId: () => 'nested-reentrant', + getSlotElementId: () => code, + getAdUnitPath: () => '/123/trackingonly', getTargeting: () => [], + getSizes: () => [[1, 1]], clearTargeting: vi.fn(), }; - const contextAfterInner: Array = []; - let reentered = false; - const originalRefresh = vi.fn(() => { - if (!reentered) { - reentered = true; - pubads.refresh([slot]); - contextAfterInner.push( - (testWindow.tsjs as { prebidRefreshDispatchInProgress?: boolean }) - .prebidRefreshDispatchInProgress - ); + installedGptSlots = [slot]; + const nativeRefresh = vi.fn(); + const ts = (testWindow.tsjs = {} as unknown as PrebidTestWindow['tsjs']) as unknown as TsjsApi; + const handoff = { + gamUnitPath: '/123/trackingonly', + formats: [[1, 1] as [number, number]], + divIdPrefix: code, + slotElementId: code, + publisherClaimed: true, + suppressPublisherDisplay: false, + suppressPublisherRefresh: true, + }; + ts.gptSlotHandoffs = { [code]: handoff }; + const innerRefresh = vi.fn((slots?: (typeof slot)[]) => { + if (handoff.suppressPublisherRefresh) { + handoff.suppressPublisherRefresh = false; + return; } - return 'delegated refresh result'; + nativeRefresh(slots); }); - const pubads = { - refresh: originalRefresh as (slots?: unknown[], opts?: unknown) => unknown, - getSlots: vi.fn(() => [slot]), - }; + const pubads = { refresh: innerRefresh, getSlots: () => [slot] }; testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - testWindow.tsjs = { - gptDiagnosticsRecorder: { - recordPrebidRefresh: (slots: object[]) => store.recordPrebidRefresh(slots), - }, - }; + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); + claimFirstImpressionForTrustedServer(ts, element); + observeFirstImpressionGptLifecycle(ts, element, 'requested'); + testWindow.__tsjs_prebid = { excludedGamAdUnitPathSuffixes: ['/trackingonly'] }; + installRefreshHandler(640); + installPrebidNpm(); - const pbjs = installPrebidNpm(); - installRefreshHandler(750); - mockRequestBids.mockImplementation((options) => options.bidsBackHandler?.()); - pbjs.requestBids({ - adUnits: [{ code: 'nested-reentrant', bids: [{ bidder: 'exampleServer', params: {} }] }], - bidsBackHandler: vi.fn(), - } as unknown as RequestBidsArg); + pubads.refresh([slot]); - expect(pubads.refresh([slot])).toBe('delegated refresh result'); - // The inner dispatch owns the flag while it runs and must hand it back, or - // the observer would stop attributing every later publisher refresh. - expect(contextAfterInner).toEqual([true]); - expect( - originalRefresh, - 'the nested refresh must reach the delegated call' - ).toHaveBeenCalledTimes(2); - expect(Object.hasOwn(testWindow.tsjs as object, 'prebidRefreshDispatchInProgress')).toBe(false); + expect(mockRequestBids).not.toHaveBeenCalled(); + expect(handoff.suppressPublisherRefresh).toBe(false); + expect(nativeRefresh).toHaveBeenCalledWith([slot]); }); - it('restores diagnostics context when its setter mutates and then throws', () => { + it('consumes late-handoff suppression when Prebid suppresses the same delivery', () => { + const code = 'composed-suppression-slot'; + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); const slot = { - getSlotElementId: () => 'mutating-context-setter', + getSlotElementId: () => code, getTargeting: () => [], + getSizes: () => [[300, 250]], clearTargeting: vi.fn(), - }; - const originalRefresh = vi.fn(); - const pubads = { - refresh: originalRefresh, - getSlots: vi.fn(() => [slot]), - }; - const contextTarget: Record = {}; - let throwAfterMutation = true; - testWindow.tsjs = new Proxy(contextTarget, { - set(target, property, value) { - Reflect.set(target, property, value); - if (property === 'prebidRefreshDispatchInProgress' && throwAfterMutation) { - throwAfterMutation = false; - throw new Error('example mutating context setter failure'); - } - return true; - }, + setTargeting: vi.fn(), + }; + installedGptSlots = [slot]; + const nativeRefresh = vi.fn(); + const ts = (testWindow.tsjs = {} as unknown as PrebidTestWindow['tsjs']) as unknown as TsjsApi; + const handoff = { + gamUnitPath: '/123/composed', + formats: [[300, 250] as [number, number]], + divIdPrefix: code, + slotElementId: code, + publisherClaimed: true, + suppressPublisherDisplay: false, + suppressPublisherRefresh: true, + }; + ts.gptSlotHandoffs = { [code]: handoff }; + const innerRefresh = vi.fn((slots?: (typeof slot)[]) => { + if (handoff.suppressPublisherRefresh) { + handoff.suppressPublisherRefresh = false; + return; + } + nativeRefresh(slots); }); + const pubads = { refresh: innerRefresh, getSlots: () => [slot] }; testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - mockRequestBids.mockImplementation((options) => options.bidsBackHandler?.()); + claimFirstImpressionForTrustedServer(ts, element); + installRefreshHandler(640); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh([slot]), + } as unknown as RequestBidsArg); + + expect(handoff.suppressPublisherRefresh).toBe(false); + expect(innerRefresh).not.toHaveBeenCalled(); - installPrebidNpm(); - installRefreshHandler(750); - pubads.refresh([slot]); pubads.refresh([slot]); - expect(originalRefresh).toHaveBeenCalledTimes(2); - expect( - Object.prototype.hasOwnProperty.call(contextTarget, 'prebidRefreshDispatchInProgress') - ).toBe(false); + expect(nativeRefresh).toHaveBeenCalledWith([slot]); }); -}); -describe('prebid publisher snapshots and delivery refreshes', () => { - let deliveryAdIds = new WeakMap(); - let installedGptSlots: Array> = []; - let auctionSequence = 0; - - beforeEach(() => { - vi.clearAllMocks(); - deliveryAdIds = new WeakMap(); - installedGptSlots = []; - auctionSequence = 0; - mockRequestBids.mockReset(); - mockPbjs.requestBids = mockRequestBids; - mockPbjs.removeAdUnit = mockRemoveAdUnit; - delete (mockPbjs as unknown as Record).__tsRemoveAdUnitWrapped; - mockPbjs.adUnits = []; - mockGetUserIdsAsEids.mockReset(); - mockGetUserIdsAsEids.mockReturnValue([]); - // By default the manifest declares all adapters compiled in. - (window as unknown as { __tsjs_prebid_bundle?: unknown }).__tsjs_prebid_bundle = - DEFAULT_BUNDLE_MANIFEST; - mockPbjs.setTargetingForGPTAsync = undefined; - testWindow.__tsjs_prebid = { - serverSideBidders: ['exampleServer', 'exampleFallback'], + it('forwards only unsuppressed excluded slots', () => { + const suppressedCode = 'mixed-suppressed-slot'; + const excludedCode = 'mixed-excluded-slot'; + const suppressedSlot = { + getSlotElementId: () => suppressedCode, + getAdUnitPath: () => '/123/content', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + setTargeting: vi.fn(), }; - testWindow.tsjs = undefined; - delete testWindow.googletag; - }); + const excludedSlot = { + getSlotElementId: () => excludedCode, + getAdUnitPath: () => '/123/trackingonly', + getTargeting: () => [], + getSizes: () => [[1, 1]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([suppressedSlot, excludedSlot]); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + claimFirstImpressionForTrustedServer(ts, document.getElementById(suppressedCode)!); + testWindow.__tsjs_prebid = { excludedGamAdUnitPathSuffixes: ['/trackingonly'] }; + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); - afterEach(() => { - delete testWindow.__tsjs_prebid; - testWindow.tsjs = undefined; - delete testWindow.googletag; - }); + pbjs.requestBids({ + adUnits: [{ code: suppressedCode, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh([suppressedSlot, excludedSlot]), + } as unknown as RequestBidsArg); - function installGpt(slots: Array>) { - installedGptSlots = slots; - for (const slot of slots) { - if (!slot || typeof slot !== 'object') continue; - const originalGetTargeting = slot.getTargeting?.bind(slot); - slot.getTargeting = (key: string) => { - const deliveryAdId = deliveryAdIds.get(slot); - if (key === 'hb_adid' && deliveryAdId) return [deliveryAdId]; - return originalGetTargeting?.(key) ?? []; - }; - } + expect(originalRefresh).toHaveBeenCalledWith([excludedSlot], undefined); + }); - const originalRefresh = vi.fn(); - const pubads = { - refresh: originalRefresh, - getSlots: vi.fn(() => slots), - }; - testWindow.googletag = { - cmd: { push: (fn: () => void) => fn() }, - pubads: () => pubads, + it('rejects pending delivery state from a previous navigation', () => { + const code = 'previous-navigation-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), }; - installRefreshHandler(640); - return { originalRefresh, pubads }; - } + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + } as unknown as RequestBidsArg); + ((testWindow.tsjs ??= {}) as unknown as TsjsApi).navGeneration = 1; - function refreshAdUnitFromLastRequest(): - | (Record & { code?: string; bids?: TestBid[] }) - | undefined { - const lastCall = mockRequestBids.mock.calls[mockRequestBids.mock.calls.length - 1]; - return lastCall?.[0]?.adUnits?.[0]; - } + pubads.refresh([slot]); - function completePublisherAuction( - opts?: { adUnits?: Array<{ code?: string }>; bidsBackHandler?: (...args: unknown[]) => void }, - options: { auctionId?: string; applyTargeting?: boolean } = {} - ): void { - const auctionId = options.auctionId ?? `example-auction-${auctionSequence++}`; - const bidResponses: Record> }> = {}; + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); - for (const unit of opts?.adUnits ?? []) { - if (!unit.code) continue; - const adId = `${auctionId}-${unit.code}`; - bidResponses[unit.code] = { - bids: [{ adId, adUnitCode: unit.code, auctionId }], - }; - if (options.applyTargeting !== false) { - const slot = installedGptSlots.find((candidate) => { - const elementId = candidate?.getSlotElementId?.(); - return elementId === unit.code || elementId === `${unit.code}-container`; - }); - if (slot) deliveryAdIds.set(slot, adId); - } - } + it('rejects pending delivery state after physical element replacement', () => { + const code = 'replaced-physical-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + } as unknown as RequestBidsArg); + document.getElementById(code)?.remove(); + const replacement = document.createElement('div'); + replacement.id = code; + document.body.appendChild(replacement); - opts?.bidsBackHandler?.(bidResponses, false, auctionId); - } + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); function installPrebidRefreshDiagnostics( implementation?: (slots: Array>) => void @@ -2707,6 +4280,134 @@ describe('prebid publisher snapshots and delivery refreshes', () => { return recordPrebidRefresh; } + it('suppresses one publisher delivery after TS claims first and allows a later refresh', () => { + const code = 'example-ts-first-slot'; + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); + try { + const targeting = new Map([ + ['ts_initial', '1'], + ['hb_adid', 'example-ts-ad-id'], + ['hb_pb', '1.25'], + ]); + const slot = { + getSlotElementId: () => code, + getTargeting: (key: string) => { + const value = targeting.get(key); + return value === undefined ? [] : Array.isArray(value) ? value : [value]; + }, + setTargeting: vi.fn((key: string, value: string | string[]) => { + targeting.set(key, value); + return slot; + }), + clearTargeting: vi.fn((key: string) => { + targeting.delete(key); + return slot; + }), + getSizes: () => [[300, 250]], + }; + const ts = (testWindow.tsjs = {} as TsjsApi) as TsjsApi; + const claim = claimFirstImpressionForTrustedServer(ts, element)!; + claim.targeting = Object.fromEntries(targeting); + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh([slot], { changeCorrelator: false }), + } as unknown as RequestBidsArg); + + expect(originalRefresh).not.toHaveBeenCalled(); + expect(slot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); + expect(slot.setTargeting).toHaveBeenCalledWith('hb_adid', 'example-ts-ad-id'); + expect(ts.firstImpression?.slots[code]?.publisherRegistrationClosed).toBe(true); + + pubads.refresh([slot], { changeCorrelator: false }); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenCalledOnce(); + expect(originalRefresh).toHaveBeenCalledWith([slot], { changeCorrelator: false }); + } finally { + element.remove(); + } + }); + + it('leaves Prebid auction identity and event listeners unchanged when diagnostics is inactive', () => { + const getTargeting = vi.fn(() => []); + const slot = { + getSlotElementId: () => 'example-inactive-slot', + getTargeting, + clearTargeting: vi.fn(), + }; + const { pubads } = installGpt([slot]); + mockPbjs.setTargetingForGPTAsync = vi.fn(); + mockRequestBids.mockImplementation((opts) => { + opts.bidsBackHandler?.({}, false, 'example-inactive-auction'); + }); + + pubads.refresh([slot]); + + const request = mockRequestBids.mock.calls[0][0]; + expect(request).not.toHaveProperty('auctionId'); + expect(mockOnEvent).not.toHaveBeenCalledWith('bidWon', expect.any(Function)); + expect(getTargeting).not.toHaveBeenCalledWith('hb_bidder'); + expect(getTargeting).not.toHaveBeenCalledWith('hb_pb'); + expect(getTargeting).not.toHaveBeenCalledWith('hb_cur'); + }); + + it('records completed client auction evidence only for the exact Prebid attempt', () => { + const slot = { + getSlotElementId: () => 'example-client-slot', + getTargeting: (key: string) => + key === 'hb_bidder' ? ['example-client'] : key === 'hb_pb' ? ['2.40'] : [], + clearTargeting: vi.fn(), + }; + const recordPrebidRefresh = vi.fn(); + const recordPrebidAuction = vi.fn(); + const recordPrebidWin = vi.fn(); + testWindow.tsjs = { + gptDiagnosticsRecorder: { + recordPrebidRefresh, + recordPrebidAuction, + recordPrebidWin, + }, + }; + const { pubads } = installGpt([slot]); + mockPbjs.setTargetingForGPTAsync = vi.fn(); + mockRequestBids.mockImplementation((opts) => { + opts.bidsBackHandler?.({}, false, 'example-client-auction'); + }); + + pubads.refresh([slot]); + + expect(mockRequestBids.mock.calls[0][0]).not.toHaveProperty('auctionId'); + expect(mockOnEvent).toHaveBeenCalledWith('bidWon', expect.any(Function)); + expect(recordPrebidRefresh).toHaveBeenCalledWith([slot]); + expect(recordPrebidAuction).toHaveBeenCalledWith(slot, 'example-client-auction', { + bidder: 'example-client', + priceBucket: '2.40', + }); + const bidWon = mockOnEvent.mock.calls.find(([event]) => event === 'bidWon')?.[1]; + expect(bidWon).toBeTypeOf('function'); + bidWon?.({ + auctionId: 'other-auction', + adUnitCode: 'example-client-slot', + adserverTargeting: { hb_bidder: 'wrong-client', hb_pb: '9.99' }, + }); + expect(recordPrebidWin).not.toHaveBeenCalled(); + bidWon?.({ + auctionId: 'example-client-auction', + adUnitCode: 'example-client-slot', + adserverTargeting: { hb_bidder: 'example-client', hb_pb: '2.40' }, + }); + expect(recordPrebidWin).toHaveBeenCalledWith(slot, 'example-client-auction', { + bidder: 'example-client', + priceBucket: '2.40', + }); + }); + it('records a publisher delivery refresh immediately before its GPT request', () => { const slot = { getSlotElementId: () => 'example-delivery-marker', @@ -3572,7 +5273,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(coveredSlot.clearTargeting).not.toHaveBeenCalled(); expect(gamOnlySlot.clearTargeting).toHaveBeenCalledWith('hb_adid'); expect(originalRefresh).toHaveBeenCalledTimes(1); - expect(originalRefresh).toHaveBeenCalledWith(undefined, undefined); + expect(originalRefresh).toHaveBeenCalledWith([coveredSlot, gamOnlySlot], undefined); }); it('keeps explicit unrelated lists synthetic and partitions mixed delivery lists', () => { @@ -3758,6 +5459,10 @@ describe('prebid publisher snapshots and delivery refreshes', () => { clearTargeting: vi.fn(), }; const { originalRefresh, pubads } = installGpt([slot]); + const publisherElement = document.createElement('div'); + publisherElement.id = code; + publisherElement.appendChild(document.getElementById('example-different-gpt-slot')!); + document.body.appendChild(publisherElement); let auctionId = 'example-null-auction'; const setTargetingForGPTAsync = vi.fn(() => { deliveryAdIds.set(slot, `${auctionId}-${code}`); @@ -4023,7 +5728,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); }); - it('consumes all overlapping pending bids for the same ad-unit code', () => { + it('preserves a sibling registration after consuming an exact overlapping delivery', () => { const code = 'example-overlapping-code'; const slot = { getSlotElementId: () => code, @@ -4035,26 +5740,89 @@ describe('prebid publisher snapshots and delivery refreshes', () => { mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); const pbjs = installPrebidNpm(); - pbjs.requestBids({ - adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], - bidsBackHandler: () => {}, - } as unknown as RequestBidsArg); - pbjs.requestBids({ - adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], - bidsBackHandler: () => {}, - } as unknown as RequestBidsArg); + for (let index = 0; index < 2; index += 1) { + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => {}, + } as unknown as RequestBidsArg); + } + deliveryAdIds.set(slot, `example-auction-0-${code}`); + pubads.refresh([slot]); + deliveryAdIds.set(slot, `example-auction-1-${code}`); pubads.refresh([slot]); + expect(mockRequestBids).toHaveBeenCalledTimes(2); expect(slot.clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenNthCalledWith(1, [slot], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [slot], undefined); + }); + + it('does not guess between ordinary overlapping code-only registrations', () => { + const code = 'example-ambiguous-code-only'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + for (let index = 0; index < 2; index += 1) { + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => {}, + } as unknown as RequestBidsArg); + } + + deliveryAdIds.delete(slot); + pubads.refresh([slot]); + expect(mockRequestBids).toHaveBeenCalledTimes(3); deliveryAdIds.set(slot, `example-auction-0-${code}`); pubads.refresh([slot]); expect(mockRequestBids).toHaveBeenCalledTimes(3); - expect(slot.clearTargeting).toHaveBeenCalledWith('hb_adid'); - expect(originalRefresh).toHaveBeenNthCalledWith(1, [slot], undefined); - expect(originalRefresh).toHaveBeenNthCalledWith(2, [slot], undefined); + expect(originalRefresh).toHaveBeenCalledTimes(2); + }); + + it('fails closed without consuming TS-owned ambiguous code-only registrations', () => { + const code = 'example-ts-ambiguous-code-only'; + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + setTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + claimFirstImpressionForTrustedServer(ts, element); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + for (let index = 0; index < 2; index += 1) { + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => {}, + } as unknown as RequestBidsArg); + } + + deliveryAdIds.delete(slot); + pubads.refresh([slot]); + deliveryAdIds.set(slot, `example-auction-0-${code}`); + pubads.refresh([slot]); + deliveryAdIds.set(slot, `example-auction-1-${code}`); + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(originalRefresh).not.toHaveBeenCalled(); + element.remove(); }); it('filters invalid explicit entries without duplicating or leaking a valid delivery', () => { @@ -4192,12 +5960,17 @@ describe('prebid publisher snapshots and delivery refreshes', () => { } }); - it('completes a synthetic refresh when targeting throws', () => { + it('completes a synthetic refresh without recording auction evidence when targeting throws', () => { const slot = { getSlotElementId: () => 'example-throwing-targeting', getTargeting: () => [], clearTargeting: vi.fn(), }; + const recordPrebidRefresh = vi.fn(); + const recordPrebidAuction = vi.fn(); + testWindow.tsjs = { + gptDiagnosticsRecorder: { recordPrebidRefresh, recordPrebidAuction }, + }; const { originalRefresh, pubads } = installGpt([slot]); mockPbjs.setTargetingForGPTAsync = vi.fn(() => { throw new Error('example targeting failure'); @@ -4207,6 +5980,8 @@ describe('prebid publisher snapshots and delivery refreshes', () => { pubads.refresh([slot]); + expect(recordPrebidRefresh).toHaveBeenCalledWith([slot]); + expect(recordPrebidAuction).not.toHaveBeenCalled(); expect(originalRefresh).toHaveBeenCalledTimes(1); expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); }); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts index 2832a4082..1cd0a19b1 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts @@ -4,6 +4,7 @@ import { knownUserIdConfigNames, resolvePrebidUserIdModulesFromEids, } from '../../../src/integrations/prebid/user_id_modules'; +import registry from '../../../src/integrations/prebid/user_id_modules.json'; const sampleEids = [ { source: 'yahoo.com', uids: [{ id: 'connect-id', atype: 3 }] }, @@ -85,6 +86,21 @@ describe('prebid user ID module registry', () => { }); }); + it('maps LiveRamp EIDs to identityLinkIdSystem', () => { + expect( + resolvePrebidUserIdModulesFromEids([ + { source: 'liveramp.com', uids: [{ id: 'opaque-envelope', atype: 3 }] }, + ]) + ).toEqual({ + modules: ['userId', 'identityLinkIdSystem'], + missingSources: [], + }); + }); + + it('includes identityLinkIdSystem in the checked-in default preset', () => { + expect(registry.defaultPreset).toContain('identityLinkIdSystem'); + }); + it('maps unknown LiveIntent provider-backed sources to liveIntentIdSystem', () => { const result = resolvePrebidUserIdModulesFromEids([ { diff --git a/crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs b/crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs index 6b38d93d2..5622daa98 100644 --- a/crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs +++ b/crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs @@ -34,7 +34,7 @@ beforeAll(async () => { '--adapters', 'adf', '--user-id-modules', - 'sharedIdSystem', + 'sharedIdSystem,identityLinkIdSystem', '--out', outputDirectory, ]); @@ -86,7 +86,7 @@ describe('tsjs-prebid shim artifact', () => { // A value-import of Prebid or a private rendering helper would multiply // the shim size; retain a margin above the normal compact shim output. expect(bundleCode.length).toBeGreaterThan(200_000); - expect(shimCode.length).toBeLessThan(30_000); + expect(shimCode.length).toBeLessThan(39_000); expect(shimCode).toContain('markWinningBidAsUsed'); }); }); @@ -135,9 +135,25 @@ describe('external bundle + served shim evaluated together', () => { pageWindow.__tsjs_prebid = { clientSideBidders: [], serverSideBidders: ['appnexus'], + managedUserIds: [ + { + name: 'identityLink', + params: { pid: '999', notUse3P: false }, + storage: { + type: 'cookie', + name: 'idl_env', + expires: 15, + refreshInSeconds: 1800, + }, + }, + ], }; pageWindow.eval(bundleCode); + pageWindow.pbjs.setConfig({ + userSync: { userIds: [{ name: 'sharedId' }] }, + }); + pageWindow.pbjs.setConfig({ userSync: { syncDelay: 41 } }); expect(typeof pageWindow.pbjs.requestBids).toBe('function'); expect(typeof pageWindow.pbjs.registerBidAdapter).toBe('function'); @@ -147,7 +163,10 @@ describe('external bundle + served shim evaluated together', () => { 'adform', 'adformOpenRTB', ]); - expect([...pageWindow.__tsjs_prebid_bundle.userIdModules]).toEqual(['sharedIdSystem']); + expect([...pageWindow.__tsjs_prebid_bundle.userIdModules]).toEqual([ + 'sharedIdSystem', + 'identityLinkIdSystem', + ]); // Count trustedServer registrations across repeated shim evaluations. const originalRegisterBidAdapter = pageWindow.pbjs.registerBidAdapter.bind(pageWindow.pbjs); @@ -157,6 +176,70 @@ describe('external bundle + served shim evaluated together', () => { pageWindow.eval(shimCode); const wrappedRequestBids = pageWindow.pbjs.requestBids; + expect(pageWindow.pbjs.getConfig('userSync.userIds')).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'sharedId' }), + expect.objectContaining({ + name: 'identityLink', + params: { pid: '999', notUse3P: false }, + storage: expect.objectContaining({ name: 'idl_env' }), + }), + ]) + ); + + // Characterize the pinned Prebid artifact: partial userSync updates retain + // its effective User ID list, including the operator-managed entry. + pageWindow.pbjs.setConfig({ userSync: { syncDelay: 50 } }); + + const userIdsAfterPartialUpdate = pageWindow.pbjs.getConfig('userSync.userIds'); + expect(userIdsAfterPartialUpdate.filter(({ name }) => name === 'identityLink')).toEqual([ + expect.objectContaining({ + name: 'identityLink', + params: { pid: '999', notUse3P: false }, + }), + ]); + expect(userIdsAfterPartialUpdate).toEqual( + expect.arrayContaining([expect.objectContaining({ name: 'sharedId' })]) + ); + expect(pageWindow.pbjs.getConfig('userSync.syncDelay')).toBe(50); + + // Exercise the real Prebid mergeConfig implementation. It closes over + // Prebid's internal setConfig, so the shim must guard mergeConfig itself + // to prevent a publisher-owned duplicate from bypassing the setConfig guard. + pageWindow.pbjs.mergeConfig({ + userSync: { + userIds: [ + { name: 'sharedId' }, + { name: 'identityLink', params: { pid: 'publisher-value' } }, + ], + }, + }); + + const mergedUserIds = pageWindow.pbjs.getConfig('userSync.userIds'); + expect(mergedUserIds.filter(({ name }) => name === 'identityLink')).toEqual([ + expect.objectContaining({ + name: 'identityLink', + params: { pid: '999', notUse3P: false }, + }), + ]); + expect(mergedUserIds).toEqual( + expect.arrayContaining([expect.objectContaining({ name: 'sharedId' })]) + ); + + pageWindow.pbjs.mergeConfig({ userSync: { syncDelay: 75 } }); + + const userIdsAfterPartialMerge = pageWindow.pbjs.getConfig('userSync.userIds'); + expect(userIdsAfterPartialMerge.filter(({ name }) => name === 'identityLink')).toEqual([ + expect.objectContaining({ + name: 'identityLink', + params: { pid: '999', notUse3P: false }, + }), + ]); + expect(userIdsAfterPartialMerge).toEqual( + expect.arrayContaining([expect.objectContaining({ name: 'sharedId' })]) + ); + expect(pageWindow.pbjs.getConfig('userSync.syncDelay')).toBe(75); + // A second evaluation (double script inclusion, or a legacy bundle that // still carries a baked-in shim running after this one) must be a no-op. pageWindow.eval(shimCode); diff --git a/crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs b/crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs new file mode 100644 index 000000000..3834e0345 --- /dev/null +++ b/crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs @@ -0,0 +1,360 @@ +// @vitest-environment node + +// Proves the generated external Prebid bundle actually ENFORCES the TCF signal +// it collects. `consentManagementTcf` only retrieves the consent string; the +// activity controls that act on it live in `tcfControl`. Without that module a +// TC string denying Purpose 1 changes nothing: User ID submodules still write +// browser storage and still call their vendor endpoints. +// +// This matters most for the managed LiveRamp entry, which Trusted Server +// configures on the operator's behalf: the publisher never wrote the page code +// that turns it on, so the bundle is the only place enforcement can come from. +// +// Runs in the node environment (vite/esbuild cannot run under jsdom globals) +// and evaluates the artifacts in an explicit JSDOM window instead. + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { JSDOM } from 'jsdom'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; + +import { main } from '../build-prebid-external.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const libDir = path.resolve(__dirname, '..'); + +const LIVE_RAMP_ENVELOPE_HOST = 'api.rlcdn.com'; +const LIVE_RAMP_STORAGE_NAME = 'idl_env'; +// LiveRamp's IAB Global Vendor List ID. +const LIVE_RAMP_GVL_VENDOR_ID = 97; + +/** + * Requests the page made to LiveRamp's envelope endpoint. + * + * Matches the parsed hostname rather than a substring: `includes()` would also + * match an unrelated host that merely carries this one in its name or query + * string, which could let the granted-consent assertion count the wrong + * request. + * + * @returns the matching URLs + */ +function envelopeRequests(urls) { + return urls.filter((url) => { + try { + return new URL(String(url), 'https://pub.example.com').hostname === LIVE_RAMP_ENVELOPE_HOST; + } catch { + return false; + } + }); +} + +let outputDirectory; +let bundleCode; +let shimCode; + +beforeAll(async () => { + outputDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'trusted-server-prebid-consent-')); + + await main([ + '--adapters', + 'adf', + '--user-id-modules', + 'identityLinkIdSystem', + '--out', + outputDirectory, + ]); + const manifest = JSON.parse(fs.readFileSync(path.join(outputDirectory, 'manifest.json'), 'utf8')); + bundleCode = fs.readFileSync(path.join(outputDirectory, manifest.filename), 'utf8'); + + const { build } = await import('vite'); + await build({ + configFile: false, + root: libDir, + build: { + emptyOutDir: false, + outDir: outputDirectory, + assetsDir: '.', + sourcemap: false, + minify: 'esbuild', + rollupOptions: { + input: path.join(libDir, 'src', 'integrations', 'prebid', 'index.ts'), + output: { + format: 'iife', + dir: outputDirectory, + entryFileNames: 'tsjs-prebid.js', + inlineDynamicImports: true, + extend: false, + name: 'tsjs_prebid', + }, + }, + }, + logLevel: 'warn', + }); + shimCode = fs.readFileSync(path.join(outputDirectory, 'tsjs-prebid.js'), 'utf8'); +}, 240_000); + +afterAll(() => { + fs.rmSync(outputDirectory, { recursive: true, force: true }); +}); + +// `tcfControl` reads the CMP's structured `vendorData`, not the encoded string, +// so the purpose and vendor grants below are what the rules actually evaluate. +// The string only has to be present and non-empty. +function tcData( + { purpose1 = true, purpose3 = true, purpose4 = true, vendor97 = true } = {}, + listenerId +) { + return { + gdprApplies: true, + tcString: 'CPexampleTCStringForTests', + eventStatus: 'tcloaded', + cmpStatus: 'loaded', + apiVersion: '2', + purpose: { + consents: { 1: purpose1, 3: purpose3, 4: purpose4 }, + legitimateInterests: {}, + }, + vendor: { + consents: { [LIVE_RAMP_GVL_VENDOR_ID]: vendor97 }, + legitimateInterests: {}, + }, + publisher: { restrictions: {} }, + specialFeatureOptins: {}, + ...(listenerId === undefined ? {} : { listenerId }), + }; +} + +/** + * Evaluates both artifacts on a GDPR page whose CMP grants or denies the + * purpose and vendor grants, then runs one auction. + * + * @returns the URLs the page requested and the cookies it managed to set. + */ +async function runGdprPage( + grants = {}, + { + publisherConsentManagement, + latePublisherConsentManagement, + cmpEventAfterLateConfig, + deferInitialCmpResponse = false, + replaceTcfApiBeforeLateEvent = false, + } = {} +) { + const dom = new JSDOM('', { + url: 'https://pub.example.com/article', + runScripts: 'outside-only', + pretendToBeVisual: true, + }); + const pageWindow = dom.window; + + const requestedUrls = []; + pageWindow.fetch = vi.fn(async (resource) => { + requestedUrls.push(typeof resource === 'string' ? resource : resource?.url); + return new Response(JSON.stringify({ envelope: 'opaque-test-envelope' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }); + pageWindow.Request = class PageRequest extends Request { + constructor(resource, init) { + super( + typeof resource === 'string' ? new URL(resource, 'https://pub.example.com').href : resource, + init + ); + } + }; + pageWindow.Headers = Headers; + pageWindow.Response = Response; + pageWindow.AbortController = AbortController; + if (!('isSecureContext' in pageWindow)) { + pageWindow.isSecureContext = true; + } + + const consentListeners = new Map(); + let nextListenerId = 1; + let removedConsentListenerCount = 0; + let replacementApiRemoveCount = 0; + const consentData = tcData(grants); + pageWindow.__tcfapi = (command, _version, callback, parameter) => { + if (command === 'addEventListener') { + const listenerId = nextListenerId++; + consentListeners.set(listenerId, callback); + if (!deferInitialCmpResponse) { + callback(tcData(grants, listenerId), true); + } + } else if (command === 'getTCData') { + callback(consentData, true); + } else if (command === 'removeEventListener') { + if (consentListeners.delete(parameter)) { + removedConsentListenerCount += 1; + } + callback(true, true); + } + }; + + // Mirror the server's head-injected state, which always precedes the bundle + // script in document order. + pageWindow.eval('window.pbjs = { que: [], cmd: [] };'); + pageWindow.__tsjs_prebid = { + clientSideBidders: [], + managedUserIds: [ + { + name: 'identityLink', + params: { pid: '999', notUse3P: false }, + storage: { + type: 'cookie', + name: 'idl_env', + expires: 15, + refreshInSeconds: 1800, + }, + }, + ], + }; + + pageWindow.eval(bundleCode); + const publisherConfig = { + // Resolve User IDs before the auction so one auction is enough to observe + // whether IdentityLink ran. + userSync: { auctionDelay: 300, syncEnabled: false }, + }; + if (publisherConsentManagement !== undefined) { + publisherConfig.consentManagement = publisherConsentManagement; + } + pageWindow.pbjs.setConfig(publisherConfig); + pageWindow.eval(shimCode); + + if (latePublisherConsentManagement !== undefined) { + pageWindow.pbjs.setConfig({ consentManagement: latePublisherConsentManagement }); + } + if (replaceTcfApiBeforeLateEvent) { + pageWindow.__tcfapi = (command, _version, callback) => { + if (command === 'removeEventListener') { + replacementApiRemoveCount += 1; + callback(true, true); + } + }; + } + if (cmpEventAfterLateConfig !== undefined) { + for (const [listenerId, callback] of consentListeners) { + callback(tcData(cmpEventAfterLateConfig, listenerId), true); + } + } + + pageWindow.pbjs.requestBids({ adUnits: [], bidsBackHandler: () => {} }); + await new Promise((resolve) => setTimeout(resolve, 50)); + await new Promise((resolve) => pageWindow.setTimeout(resolve, 400)); + await new Promise((resolve) => setTimeout(resolve, 50)); + + return { + requestedUrls, + cookies: pageWindow.document.cookie, + consentManagement: pageWindow.pbjs.getConfig('consentManagement'), + removedConsentListenerCount, + replacementApiRemoveCount, + }; +} + +describe('external bundle TCF enforcement', () => { + it('bundles the activity-control module alongside the consent collectors', () => { + // A bundle that collects consent but cannot act on it is the failure mode + // this whole suite exists to prevent. + expect(bundleCode).toContain('consentManagementTcf'); + expect(bundleCode).toContain('tcfControl'); + }); + + it('blocks IdentityLink storage and vendor calls when Purpose 1 is denied', async () => { + const { requestedUrls, cookies } = await runGdprPage({ purpose1: false }); + + expect(envelopeRequests(requestedUrls)).toEqual([]); + expect(cookies).not.toContain(LIVE_RAMP_STORAGE_NAME); + expect(cookies).not.toContain('_lr_retry_request'); + }); + + it('blocks IdentityLink storage and vendor calls when vendor 97 is denied', async () => { + const { requestedUrls, cookies } = await runGdprPage({ vendor97: false }); + + expect(envelopeRequests(requestedUrls)).toEqual([]); + expect(cookies).not.toContain(LIVE_RAMP_STORAGE_NAME); + expect(cookies).not.toContain('_lr_retry_request'); + }); + + it('still resolves IdentityLink when Purpose 3 alone is denied', async () => { + const { requestedUrls, cookies } = await runGdprPage({ purpose3: false }); + + expect(envelopeRequests(requestedUrls)).toHaveLength(1); + expect(cookies).toContain(LIVE_RAMP_STORAGE_NAME); + }); + + it('still resolves IdentityLink when Purpose 4 alone is denied', async () => { + const { requestedUrls, cookies } = await runGdprPage({ purpose4: false }); + + expect(envelopeRequests(requestedUrls)).toHaveLength(1); + expect(cookies).toContain(LIVE_RAMP_STORAGE_NAME); + }); + + it('resolves IdentityLink when all relevant grants are present', async () => { + const { requestedUrls, cookies } = await runGdprPage(); + + expect(envelopeRequests(requestedUrls)).toHaveLength(1); + expect(cookies).toContain(LIVE_RAMP_STORAGE_NAME); + }); + + it('preserves publisher-owned GDPR configuration in the generated bundle', async () => { + const publisherConsentManagement = { + gdpr: { cmpApi: 'iab', timeout: 123, defaultGdprScope: true }, + }; + + const { consentManagement } = await runGdprPage({}, { publisherConsentManagement }); + + expect(consentManagement.gdpr).toEqual(publisherConsentManagement.gdpr); + }); + + it('retires automatic IAB consent when late static GDPR configuration takes ownership', async () => { + const deniedStaticConsent = { + gdpr: { + cmpApi: 'static', + consentData: tcData({ purpose1: false }), + }, + }; + + const { requestedUrls, cookies, removedConsentListenerCount } = await runGdprPage( + {}, + { + latePublisherConsentManagement: deniedStaticConsent, + cmpEventAfterLateConfig: {}, + } + ); + + expect(removedConsentListenerCount).toBe(1); + expect(envelopeRequests(requestedUrls)).toEqual([]); + expect(cookies).not.toContain(LIVE_RAMP_STORAGE_NAME); + expect(cookies).not.toContain('_lr_retry_request'); + }); + + it('ignores a delayed initial IAB response after static GDPR configuration takes ownership', async () => { + const deniedStaticConsent = { + gdpr: { + cmpApi: 'static', + consentData: tcData({ purpose1: false }), + }, + }; + + const { requestedUrls, cookies, replacementApiRemoveCount } = await runGdprPage( + {}, + { + latePublisherConsentManagement: deniedStaticConsent, + cmpEventAfterLateConfig: {}, + deferInitialCmpResponse: true, + replaceTcfApiBeforeLateEvent: true, + } + ); + + expect(replacementApiRemoveCount).toBe(1); + expect(envelopeRequests(requestedUrls)).toEqual([]); + expect(cookies).not.toContain(LIVE_RAMP_STORAGE_NAME); + expect(cookies).not.toContain('_lr_retry_request'); + }); +}); diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 498e2e37a..a95222f11 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -161,6 +161,10 @@ export default withMermaid( text: 'GPT Runtime Diagnostics', link: '/guide/integrations/gpt-diagnostics', }, + { + text: 'GPT Diagnostics Label Dictionary', + link: '/guide/integrations/gpt-diagnostics-dictionary', + }, ], }, { diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index 2fc33796a..20348a971 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -816,8 +816,8 @@ not belong to the browser integration. ### Environment variable overrides The typed `ts config validate`, `ts config diff`, and `ts config push` flows can -override existing scalar leaves. The pinned EdgeZero loader does not create -missing leaves or replace arrays, tables, maps, or rules. Existing configs must add +override existing scalar leaves. EdgeZero's env overlay does not create missing +leaves or replace arrays, tables, maps, or rules. Existing configs must add `rewrite_creatives = true` and `sanitize_creatives = false` before relying on those scalar overrides. Edit and re-push TOML for other values. Provider map keys preserve hyphens, so `pbs-main` uses the `PBS-MAIN` segment and needs diff --git a/docs/guide/cli.md b/docs/guide/cli.md index 775a99146..1e0ca02e6 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -76,6 +76,44 @@ Trusted Server settings JSON. This blob model is intentional because full Trusted Server configs can exceed Fastly limits when split into one config-store entry per setting. +### Diagnose ad-template configuration + +The static `ts config ad-templates` commands evaluate local configuration +without launching a browser: + +| Command | Purpose | +| ---------------------------------------- | ------------------------------------------------------------------------- | +| `lint` | Summarize configuration and report invalid slot page patterns. | +| `match [--details]` | List matching slots; `--details` includes divs, paths, formats/providers. | +| `check --expected-slot ID` | Assert the exact matching slot set; repeat `--expected-slot`. | +| `check --expect-no-slots` | Assert that no slots match. | +| `explain ` | Print every runtime ad-stack gate and its final yes/no verdict. | + +`check --allow-extra-slots` permits matches beyond the repeated +`--expected-slot` values. It conflicts with `--expect-no-slots`. + +`explain` models a GET navigation with consent allowed by default. Use +`--method `, `--non-navigation`, `--prefetch`, `--bot`, or +`--consent-denied` to model another request. Provider configuration is printed +as a separate advisory; it does not change the runtime gate verdict. + +Every `ts config ad-templates ...` and `ts audit ad-templates ...` command +accepts the same config-location flags: + +| Flag | Behavior | +| --------------------- | ----------------------------------------------------------------------------- | +| `--app-config ` | Read this app config instead of deriving `.toml` from the manifest. | +| `--manifest ` | Read this manifest; defaults to `edgezero.toml`. | +| `--no-env` | Disable `TRUSTED_SERVER__...` overlays for read-only commands. | + +The mutating audit generator always edits file-backed values and never writes +environment-only overlays into TOML, including during `--dry-run`. + +For CI-oriented assertions, exit code 0 means the assertion passed, 1 means the +command ran and found drift (`config ad-templates check` or audit verification +with `--strict`), and 2 means argument parsing, configuration, browser launch, +or another tool operation failed. + Reclaim orphaned chunk entries leaked from prior oversized pushes: ```bash @@ -190,7 +228,7 @@ Chrome or Chromium must be installed locally. The command checks common PATH names and standard macOS/Linux install locations. ```bash -ts audit https://publisher.example +ts audit generate https://publisher.example ``` By default, the command writes: @@ -218,13 +256,13 @@ present in HTML processed by Trusted Server. If a config already exists, avoid overwriting it: ```bash -ts audit https://publisher.example --no-config +ts audit generate https://publisher.example --no-config ``` Use custom output paths when reviewing artifacts first: ```bash -ts audit https://publisher.example \ +ts audit generate https://publisher.example \ --js-assets audit/js-assets.toml \ --config audit/trusted-server.toml ``` @@ -232,9 +270,310 @@ ts audit https://publisher.example \ Use `--force` only when replacing existing output files is intentional: ```bash -ts audit https://publisher.example --force +ts audit generate https://publisher.example --force +``` + +The legacy `ts audit ` form remains a compatibility alias for artifact +generation. New automation should use `ts audit generate `. + +## Generate ad-template slots from a live site + +`ts audit ad-templates generate ` discovers the publisher's ad slots and +rewrites the `[creative_opportunities]` slot array in `trusted-server.toml` in +place, preserving every other section and comment. + +```bash +ts audit ad-templates generate https://publisher.example/ +``` + +It samples the site rather than a single page. Ad slots repeat per site +section, so the crawl is sized by the publisher's taxonomy — a dozen sections — +not its catalogue: + +1. Load the requested page and read its links and, from `robots.txt`, its + sitemap. +2. Group both into candidate sections, keeping one landing page and one article + per section. +3. Load those pages, recording each slot's div, sizes, and GAM ad-unit path. +4. Reconcile every slot across the pages it appeared on. +5. Infer a `{section}` ad-unit template if the evidence proves one. +6. Verify the result loads, then write it. + +### What it writes + +Given a site whose ad units track the section, the run produces: + +```toml +[creative_opportunities] +gam_network_id = "99999" +section_root = "homepage" +section_segment = 0 + +[[creative_opportunities.slot]] +id = "ad-header-0" +div_id = "ad-header-0" +gam_unit_path = "/{network_id}/example/{section}" +page_patterns = ["/", "/deals", "/deals/*", "/news", "/news/*"] +formats = [{ width = 728, height = 90 }] +``` + +Each section contributes **two** patterns. `*` crosses `/` in this glob +dialect, so `/news/*` matches `/news/a/b` but not the bare `/news` landing +page; emitting only the star form would drop the landing page from the slot. + +Sizes are unioned across pages, so a format that renders only on articles +survives alongside the homepage's. + +### When it keeps literal paths, and when it refuses + +A wrong ad-unit template makes the publisher bid against inventory that does not +exist, so the command prefers a narrow literal path over a plausible guess. + +| Situation | Result | +| --------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Only one page was crawled | Literal path. One observation cannot distinguish a literal from a template. | +| The ad unit never varied by section | Literal path. | +| A section's slug is not derivable from its URL (`/site-news` requesting `.../sitenews`) | The slot is omitted; the note lists the ad-unit paths it used and says none generalized. | +| No crawled page lacked a section segment, so `section_root` is unwitnessed | No template is written and the reason names the crawl gap. A slot that merely never appears on the root (a sidebar, an in-article unit) still templates, borrowing the `section_root` another slot witnessed; a note says so. | +| Two path segments could both be the section | No template; the ambiguity is reported. | +| The ad unit varies by device, geo, or anything the URL cannot supply | The refused slot is omitted and the reason is written as a note. | +| Crawled pages report different GAM network ids | The run fails; the pages are not one property. | +| More than a quarter of crawled pages return no slots | The run fails. That is the signature of bot protection serving challenge pages, and writing from it would silently narrow the slot set. | +| Several live elements normalize onto one div-id prefix | The whole group is omitted, on every page of the crawl. A prefix resolves to at most one element and the exact ids change per render; the prefix is named in a note. | +| A per-render token sits before the placement part of a div id | The slot is omitted from a single observation and the family prefix is named in a note; no stable prefix identifies one element. | +| A crawled page redirects off the audited origin | The page is skipped on that profile, its path is named in a note, and it stops counting toward profile coverage. There is no override for generation; another site's evidence is never folded into the config. | + +Every run checks that the config it produced still loads before replacing the +file, and `--dry-run` runs the same check — a clean preview is evidence the +config loads, not just that it parses. Dry-run stdout is a zero-context unified +diff containing only the managed creative-opportunity fields; notes and refusal +reasons go to stderr, so unrelated config and secrets are not printed. Crawl +progress also goes to stderr, one line per phase and page — for example +`Auditing desktop [2/17]: /news`. Progress renders the path only, never the +origin, userinfo, query, or fragment, and there is no flag to suppress it. A +`--dry-run` that changes nothing says so on stderr too, leaving stdout an empty +diff. + +### Bounding and steering the crawl + +```bash +# Cover more of a large site. +ts audit ad-templates generate https://publisher.example/ --max-sections 20 --max-pages 41 + +# Audit exactly one page, as earlier releases did. +ts audit ad-templates generate https://publisher.example/ --max-pages 1 + +# Trigger lazy-loaded inventory on every crawled page. +ts audit ad-templates generate https://publisher.example/ --scroll + +# Set the patterns yourself; this disables pattern inference entirely, and the +# run fails outright if any slot's template had to borrow section_root. +ts audit ad-templates generate https://publisher.example/ \ + --page-pattern '/' --page-pattern '/news' --page-pattern '/news/*' + +# Preview without writing. +ts audit ad-templates generate https://publisher.example/ --dry-run +``` + +Re-running merges into the existing slots: a slot seen again keeps its +hand-tuned fields and gains this run's patterns and newly observed formats, and +a hand-written `gam_unit_path` template is preserved. A configured `div_id` is +matched exactly when the crawl observed that exact id; it is treated as a +runtime prefix only when it was never observed as a literal element, so a +configured `ad-sidebar-1` no longer absorbs a discovered `ad-sidebar-10` — the +sibling is appended as its own slot, and when the parent carried a floor price, +targeting, or provider settings, a stderr note names the split because the new +slot does not inherit them. A prefix that does claim several +discovered divs is named in a stderr note, because the runtime resolves a +prefix to at most one element. `--replace` discards existing slots instead, +which also discards any template you wrote by hand. + +`--scroll` performs the same deterministic stepped scroll on every page and +device profile after the initial settle, then waits for the page to settle again +before collecting evidence. It is opt-in because it increases crawl time, ad +requests, and publisher-page side effects. + +During a normal merge, configured slots missing from the current crawl are +preserved and named in a stderr note. Absence is not proof that a slot is stale: +the crawl may have missed a page type, device target, or lazy-loaded placement. +Review coverage and re-run with `--scroll` when appropriate. Only use +`--replace` when intentionally pruning every slot the run did not rediscover. + +A slot that never appeared without a section segment can borrow a +`section_root` witnessed by another slot only while its patterns are derived +from the paths where it was observed. If `--page-pattern` would override those +patterns, generation fails and names the affected div ids; remove the explicit +patterns so the safe per-slot patterns can be derived. + +A merge refuses to change the section policy that preserved `{section}` slots +were written against. If the config has a non-empty `section_root`, an inferred +root or segment mismatch fails and asks for `--replace` as an explicit +migration. An explicitly configured `section_segment` is preserved even when +`section_root` is unset. When the root is unset and the segment is either unset +or agrees with inference, the first merge adopts the inferred root and makes +the otherwise unloadable `{section}` config valid. + +Locale-prefixed sites are inferred at their observed section depth. Only real +ISO 639-1 language codes are read as a locale prefix, so a two-letter _section_ +root such as `/tv` or `/us` keeps sections at the first segment. For +example, `/en/news/story` can produce `section_segment = 1`; generated patterns +retain the locale prefix (`/en/news` and `/en/news/*`). The crawler never +invents an unwitnessed locale or section. + +Behind bot protection, pass a valid clearance cookie. The crawl reuses one +browser session, so clearance earned on the first page carries to the rest, and +`--page-delay-ms` spaces the requests — an unpaced crawl is both discourteous to +the origin and likelier to be challenged partway through: + +```bash +ts audit ad-templates generate https://publisher.example/ \ + --cookie '=' --page-delay-ms 1500 +``` + +Some origins refuse a headless browser outright regardless of the cookie. +`--headful` runs a visible one, which is also the quickest way to _see_ whether +a challenge is being shown: + +```bash +ts audit ad-templates generate https://publisher.example/ --headful ``` +### Sites behind a consent platform + +Publishers gate slot definition behind their consent platform, and the audit +runs in a throwaway browser profile with no consent cookie. Left alone, such a +site defines no slots at all and looks identical to a site with no ad stack. + +The crawl therefore answers the two IAB interfaces every compliant platform +exposes — TCF v2 and US Privacy — as a consenting, out-of-scope reader, before +any page script runs. This changes only what the audit browser sees; it does not +affect the publisher's own readers. Pass `--no-assume-consent` to observe the +un-consented page instead. + +When a page still yields no slots, the run reports GPT's observable state — +whether the library reached `apiReady`, how many queued commands never drained, +how many scripts ran. An empty slot registry has several very different causes, +and that line distinguishes them. + +### Auditing a production hostname served locally + +`ts dev proxy` serves a production hostname from a local Trusted Server. +Auditing through it keeps the page's origin, cookie scope, and any origin checks +in the ad stack matching production rather than `localhost`: + +```bash +ts dev proxy --map www.publisher.example=127.0.0.1:7676 --upstream-plaintext --rewrite-host + +ts audit ad-templates generate https://www.publisher.example/ \ + --browser-proxy 127.0.0.1:18080 --danger-accept-invalid-certs +``` + +`--danger-accept-invalid-certs` covers the proxy's MITM certificate when the +throwaway browser profile does not trust its CA; installing that CA +(`ts dev proxy ca`) is preferable. Against a real origin the flag is dangerous — +the audit sends any `--cookie` session upstream and treats the response as +evidence, so an invalid certificate could mean an impersonator is both +harvesting the session and fabricating the result. + +Note that a local Trusted Server injects its own configured slots into the page, +so a run through the proxy can rediscover config it already has. Slot ids that +are absent from the current config are the publisher's own. + +### Slots that change div id on every render + +Some ad stacks build div ids from a per-render token, so one placement arrives +under a new id on every page. Those ids match nothing at runtime, so the run +declines to write them and reports the group instead: + +```text +note: skipped 3 slot(s) that look like one placement under a per-render div id + on `/123456789/publisher/overlay` (ex_slot_a1_overlay_1, …); + they share the prefix `ex_slot`. Add it once by hand with a div_id prefix + that is stable across renders +``` + +This particular group is detected by evidence, not by recognising token shapes: +candidates share an ad-unit path and formats, and what separates a fragmented +placement from two legitimate siblings on one unit is co-occurrence — real +siblings appear together on a page, fragments never do. (A single id whose +per-render token sits _before_ the placement part is refused on shape alone, +from one observation, as the table above notes.) The suggested prefix is a +starting point only, not written as a `div_id`, because it reaches only as far +as the observed tokens happen to agree. + +### Checking for a device split + +Publishers often serve a different ad unit per device +(`/network/desktop/news` against `/network/mobile/news`). A desktop-only crawl +cannot see that — it infers a template correct for desktop and silently wrong +for every mobile impression. + +```bash +ts audit ad-templates generate https://publisher.example/ --profiles desktop,mobile +``` + +Each page is loaded once per profile. Where the profiles disagree, the slot is +omitted and the diagnostic explains the conflicting paths. The generator does +not fall back to a fabricated default ad unit. + +### Deploy ordering for templated config + +> **A config containing `section_root` or `section_segment` is not +> rollback-safe.** These keys are rejected outright by a Trusted Server binary +> that predates ad-unit templating, and the rejection fails the _entire_ +> configuration load — not just the ad-template section — so every route serves +> an error. This is a full-site outage, not a degraded ad stack. + +When a run reports that it wrote a `{section}` template: + +1. Deploy the template-aware binary **first**. +2. Then `ts config push`. +3. Do **not** roll that binary back while the config is live. + +A run that did not template writes neither key, and leaves the config exactly as +rollback-safe as it was. + +### Audit safety defaults + +Every `ts audit` browser session validates TLS certificates. This matters +because `--cookie` sends a real session to the origin and the page's own +response becomes the audit's evidence, so a certificate-invalid host could both +harvest the session and fabricate what the audit reports. Override only for a +host you control with a known self-signed certificate: + +```bash +ts audit page https://staging.publisher.example --danger-accept-invalid-certs +``` + +`ts audit ad-templates verify` matches configured slots against the +**post-redirect** path, so it refuses a redirect that leaves the requested +origin rather than accepting another site's evidence as verification. Allow it +for a known redirect between your own properties (for example apex to `www`): + +```bash +ts audit ad-templates verify https://publisher.example/ --allow-cross-origin-redirect +``` + +Verification accepts multiple URLs and reuses one browser/profile. Add +`--strict` to return exit 1 when a confirmable slot is missing or partially +confirmed, and `--json` for the stable machine-readable report. Video- and +native-only slots are reported as `unconfirmable`; that records a checker +limitation and does not fail strict mode. A live out-of-page slot with no sizes +against banner-configured formats is reported `partial` and does fail strict +mode. `--scroll` enables the optional second evidence phase and labels evidence +first seen after the deterministic scroll. + +Browser-backed ad-template generation and verification share `--chrome`, +`--headful`, `--browser-proxy`, `--no-assume-consent`, `--scroll`, +`--settle-quiet-ms`, `--settle-max-ms`, and `--danger-accept-invalid-certs`; +`--scroll` runs the same deterministic scroll pass in both, and in verification +it additionally labels the second evidence phase. Verification also accepts +`--browser-profile desktop|mobile`; generation uses `--profiles desktop,mobile` +to compare both profiles. `--cookie NAME=VALUE` is repeatable and creates +host-only, root-path cookies; HTTPS targets also mark them Secure. Verification +refuses cookies when URLs span multiple origins. The quiet settle window must +not exceed the maximum. + `ts audit` is not an EdgeZero adapter command. It has no `--adapter` option and it does not provision resources, push config, build, deploy, or contact platform APIs. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 89dc43aea..11b379d02 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -153,6 +153,8 @@ fail and the service will return its startup-error response. | `[request_signing]` | Ed25519 request signing | | `[auction]` | Auction orchestration | | `[integrations.*]` | Partner integrations (Prebid, Next.js, etc.) | +| `[observability]` | Server-Timing header emission | +| `[tinybird]` | Auction and access telemetry transport | ## Example: Production Setup @@ -212,7 +214,7 @@ base TOML configuration by `ts config validate`, `ts config diff`, and stored in the app-config blob. Changing an environment variable requires rerunning validation and pushing the resolved config, not rebuilding the binary. -The pinned EdgeZero loader only overrides leaves that already exist in the +EdgeZero's env overlay only overrides leaves that already exist in the parsed TOML; it does not create missing fields. Add newly introduced defaulted fields to an existing config before relying on their environment overrides. Secret overlays still contain key names, never secret values. Pass `--no-env` @@ -1337,18 +1339,19 @@ apply when the integration section exists in `trusted-server.toml`. timeout, routing, profile debug/test controls, consent forwarding, bidder-param overrides, and notification suppression belong under `[auction]`. -| Browser field | Type | Default | Description | -| ------------------------------------- | ------------- | ---------------------------------------------------------------------- | ------------------------------------------------------------------------------ | -| `enabled` | Boolean | `true` | Enable browser bundle injection, interception, and the `trustedServer` adapter | -| `account_id` | String | `None` | Optional account value injected into browser Prebid configuration | -| `timeout_ms` | Integer | `1000` | Browser Prebid.js timeout; independent of every server provider timeout | -| `debug` | Boolean | `false` | Browser Prebid.js debug flag; independent of server profile debug | -| `client_side_bidders` | Array[String] | `[]` | Bidders kept on native browser adapters | -| `excluded_gam_ad_unit_path_suffixes` | Array[String] | `[]` | GAM suffixes excluded from Trusted Server refresh auctions | -| `script_patterns` | Array[String] | `["/prebid.js", "/prebid.min.js", "/prebidjs.js", "/prebidjs.min.js"]` | Publisher Prebid script paths intercepted by Trusted Server | -| `external_bundle_url` | String | Required when enabled | HTTPS publisher-specific Prebid.js bundle URL | -| `external_bundle_sha256` / `*_sri` | String | `None` | Optional bundle integrity and cache metadata | -| `bundle.adapters` / `user_id_modules` | Array[String] | CLI selection | Inputs used by `ts prebid bundle` | +| Browser field | Type | Default | Description | +| ------------------------------------- | ------------- | ---------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `enabled` | Boolean | `true` | Enable browser bundle injection, interception, and the `trustedServer` adapter | +| `account_id` | String | `None` | Optional account value injected into browser Prebid configuration | +| `timeout_ms` | Integer | `1000` | Browser Prebid.js timeout; independent of every server provider timeout | +| `debug` | Boolean | `false` | Browser Prebid.js debug flag; independent of server profile debug | +| `client_side_bidders` | Array[String] | `[]` | Bidders kept on native browser adapters | +| `excluded_gam_ad_unit_path_suffixes` | Array[String] | `[]` | GAM suffixes excluded from Trusted Server refresh auctions | +| `script_patterns` | Array[String] | `["/prebid.js", "/prebid.min.js", "/prebidjs.js", "/prebidjs.min.js"]` | Publisher Prebid script paths intercepted by Trusted Server | +| `external_bundle_url` | String | Required when enabled | HTTPS publisher-specific Prebid.js bundle URL | +| `external_bundle_sha256` / `*_sri` | String | `None` | Optional bundle integrity and cache metadata | +| `bundle.adapters` / `user_id_modules` | Array[String] | CLI selection | Inputs used by `ts prebid bundle` | +| `managed_user_ids` | Array[Table] | `[]` | Prebid User ID modules Trusted Server installs and keeps installed; each entry is forwarded to Prebid.js verbatim (see below) | Server-side bidder codes are derived from validated `[auction.bidders.*]` routes and injected into the browser. There is no second server bidder list in @@ -1366,6 +1369,15 @@ client_side_bidders = ["example-browser"] external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" script_patterns = ["/prebid.js", "/prebid.min.js"] +[[integrations.prebid.managed_user_ids]] +name = "sharedId" + +[integrations.prebid.managed_user_ids.storage] +type = "cookie" +name = "_sharedid" +expires = 15 +refresh_in_seconds = 1800 + [proxy] allowed_domains = ["assets.example.com"] @@ -1410,6 +1422,45 @@ Environment overlays only replace existing scalar leaves. Keep `client_side_bidders`, provider profile tables, bidder-parameter overrides, and rules in TOML, then validate and push the edited file. +**Managed User ID modules**: + +Each `[[integrations.prebid.managed_user_ids]]` entry names a Prebid +`userSync.userIds` module that Trusted Server installs on the page and +reinstates whenever publisher JavaScript replaces the User ID configuration. +Trusted Server does not interpret module-specific fields; every registered +module uses the same vendor-neutral surface. The managed `name` must match a +`configNames` entry in the checked-in `user_id_modules.json` registry: + +| Field | Type | Default | Description | +| ---------------------------- | ------- | ------------------------------ | ------------------------------------------------------------------------------------ | +| `name` | String | Required | Prebid `userSync.userIds` entry name, for example `sharedId`. Unique across entries | +| `params` | Table | `{}` | Module-specific parameters, forwarded to Prebid unchanged | +| `storage.type` | String | `cookie` | Browser storage: `cookie` or `html5` | +| `storage.name` | String | Required when `storage` exists | Cookie or local-storage key the module reads and writes | +| `storage.expires` | Integer | Prebid's own default | Storage lifetime in days; must be at least 1. Any per-module ceiling is the module's | +| `storage.refresh_in_seconds` | Integer | Prebid's own default | Seconds before the module may refresh the stored value; must be at least 1 | + +The module must be present in the built bundle. Name it under +`[integrations.prebid.bundle].user_id_modules`, or omit that list to take the +generator's default preset. `ts prebid bundle` resolves each managed `name` +through the checked-in `user_id_modules.json` registry, rejects unknown or +ambiguous names, and confirms the required modules in the newly generated +manifest. A failure identifies the managed name or required module and does not +update the configured bundle hash or SRI. The browser diagnostic remains a +fallback for externally hosted, stale, or modified bundles. Trusted Server core +does not interpret module-specific `params`; it forwards them to Prebid.js +unchanged. + +Persisting a resolved ID into the Edge Cookie identity graph additionally +requires a matching `[[ec.partners]]` entry whose `source_domain` equals the +module's OpenRTB EID source. + +`managed_user_ids` is an array of tables, so it cannot be set through a +`TRUSTED_SERVER__` environment variable; the scalar overlay only replaces leaves +the published TOML already declares. See +[Managed User ID modules](/guide/integrations/prebid#managed-user-id-modules) +for consent, timing, privacy, degraded behavior, and validation guidance. + **Script Pattern Matching**: The `script_patterns` configuration determines which Prebid scripts are intercepted and replaced with empty JavaScript responses. This prevents client-side Prebid.js from loading when using server-side bidding. @@ -2119,6 +2170,143 @@ Rollback to the legacy entry point is no longer controlled by runtime config keys. Use the normal deployment rollback path to restore a pre-cleanup service version if that is required. +## Observability and Access Telemetry Configuration + +Settings for the `Server-Timing` response header and the sampled +access-telemetry sink. Both are off by default and are independent switches: +enabling one does not enable the other. + +### `[observability]` + +| Field | Type | Required | Default | Description | +| ----------------------- | ------- | -------- | ------- | ------------------------------------------------------------------- | +| `server_timing_enabled` | Boolean | No | `false` | Append request-phase timings to the `Server-Timing` response header | + +**Purpose**: Surfaces per-phase request timing (`ts-total` plus recorded +phases such as `ts-appbuild`, `ts-filter`, `ts-geo`, `ts-kv`, `ts-origin`, and +`ts-template-cache`) as a standard `Server-Timing` header, in milliseconds +with one decimal place. An unrecorded phase is omitted from the header +rather than rendered as zero. + +**Emission is conservative**: the header is appended only on responses that +are conclusively private, meaning `Cache-Control` contains `private` or +`no-store`. A response that is heuristically cacheable, carries a bare +`max-age`, or has no cache header at all never receives the header, because a +shared-cache object would otherwise replay one request's timings for its +entire stored lifetime. The long-lived, shared-cacheable `tsjs` asset route is +the concrete case this excludes. The header is appended, never inserted, so +an origin-supplied `Server-Timing` value and any entries the fronting +delivery layer adds are preserved alongside the TS entries. + +The Axum adapter applies the same private-response rule at its own terminal +point before serializing the response, and emits the header only; it does not +send access-telemetry rows. + +**Example**: + +```toml +[observability] +server_timing_enabled = true +``` + +::: warning Client-visible latency disclosure +The `Server-Timing` header is sent to every client on eligible responses, +not only to operators: browsers expose the values to same-origin JavaScript +via `PerformanceResourceTiming.serverTiming`, and any caller can read the +raw header. Enabling it publishes measured per-phase server latency, +including KV read timing on the public identity endpoints (`ts-kv`) and +origin/cache behaviour on publisher pages (`ts-origin`, +`ts-template-cache`). This is standard `Server-Timing` practice and the +values are durations only, but treat the flag as a diagnostic aid to enable +deliberately, not a general always-on toggle, unless disclosing those +timings to all clients is acceptable for the deployment. +::: + +**Environment Override**: + +```bash +TRUSTED_SERVER__OBSERVABILITY__SERVER_TIMING_ENABLED=true +``` + +::: tip Present-but-false by default +`server_timing_enabled` ships as `false` in the base operator config rather +than being left out, even though `false` is also its default. The +environment-variable overlay can only override a leaf that already exists in +the parsed TOML; it cannot create a missing one. Keeping the leaf present lets +`TRUSTED_SERVER__OBSERVABILITY__SERVER_TIMING_ENABLED` take effect without an +extra edit to add the table first. +::: + +### `[tinybird]` access telemetry keys + +`[tinybird]` configures a shared Events API transport (`enabled`, `api_host`, +`secret_store`, and per-sink dataset and token fields) used by two +independent emitters: auction telemetry (`auction_dataset`, +`auction_token_secret`) and access telemetry. The keys below cover the +access-telemetry sink and the shared enable flags. + +| Field | Type | Required | Default | Description | +| -------------------- | ------- | ------------------------------------ | ------- | ------------------------------------------------------------------------------- | +| `enabled` | Boolean | Yes, when `access_enabled` | `false` | Master switch for the shared Tinybird transport (host, store, credentials) | +| `auction_enabled` | Boolean | No | `true` | Independently gates auction telemetry emission, decoupled from access telemetry | +| `access_enabled` | Boolean | No | `false` | Enables the sampled access-telemetry row sent after each response is delivered | +| `access_sample_rate` | Float | Yes (`> 0.0`), when `access_enabled` | `0.0` | Fraction (`0.0`-`1.0`) of requests to emit an access-telemetry row for | + +**Purpose**: `access_enabled` and `auction_enabled` gate the two Tinybird +sinks separately so that turning on one does not silently turn on (or leave +off) the other; a settings test locks this decoupling in both directions. +Setting `access_enabled = true` with `access_sample_rate = 0.0` is rejected at +config load as an armed-but-silent configuration; use `access_enabled` itself +to turn the sink off, not the sample rate. Enabling `access_enabled` also +requires the shared transport fields (`enabled`, non-empty `api_host`, +`secret_store`, `access_dataset`, `access_token_secret`, and a positive +`max_body_bytes`) to already be set. + +**Example**: + +```toml +[tinybird] +enabled = true +api_host = "api.tinybird.example.com" +secret_store = "ts_secrets" +auction_enabled = true + +# Access-log telemetry, decoupled from auction emission. +access_enabled = true +access_dataset = "access_logs_raw" +access_token_secret = "tinybird_access_append_token" +access_sample_rate = 0.05 +max_body_bytes = 1048576 +``` + +**Environment Override**: + +```bash +TRUSTED_SERVER__TINYBIRD__ACCESS_ENABLED=true +TRUSTED_SERVER__TINYBIRD__ACCESS_SAMPLE_RATE=0.05 +TRUSTED_SERVER__TINYBIRD__AUCTION_ENABLED=true +``` + +A sampled request emits one access-telemetry row to `access_dataset` after +the response has already been delivered to the client, so ingest never delays +the response the reader sees. + +### Deploy and rollback ordering + +::: warning `Settings` rejects unknown fields; order matters +Both `[observability].server_timing_enabled` and the new `[tinybird]` access +keys are new fields on a config schema that uses `deny_unknown_fields`, so an +older binary fails to load a config that carries them. + +**Deploying**: upgrade the binary first, then push a config containing the +new fields second. Never push a config with these fields while a +pre-observability binary can still receive it. + +**Rolling back**: reverse the order. Remove the `[observability]` table and +any new `[tinybird]` access keys from the config and push that first, then +roll back the binary second. +::: + ## Validation ### Automatic Validation diff --git a/docs/guide/ec-setup-guide.md b/docs/guide/ec-setup-guide.md index 3ec1d0441..c721b1eaf 100644 --- a/docs/guide/ec-setup-guide.md +++ b/docs/guide/ec-setup-guide.md @@ -19,9 +19,8 @@ This guide covers: ## 1) Required Configuration -Generate the passphrase and partner token independently with -`openssl rand -base64 32`, then set EC configuration in `trusted-server.toml`. -The `replace-with-*` values below are intentionally rejected placeholders: +Configure secret-store key names in `trusted-server.toml`, then provision the +passphrase and partner token independently with `openssl rand -base64 32`: ```toml [ec] @@ -30,7 +29,7 @@ ec_store = "ec_identity_store" [[ec.partners]] name = "Mocktioneer SSP" -source_domain = "formally-vital-lion.edgecompute.app" +source_domain = "ssp.example.com" api_token = "partner_api_token" bidstream_enabled = true ``` @@ -77,7 +76,7 @@ Partners are configured in `trusted-server.toml` and loaded at startup: ```toml [[ec.partners]] name = "Mocktioneer SSP" -source_domain = "formally-vital-lion.edgecompute.app" +source_domain = "ssp.example.com" api_token = "partner_api_token" bidstream_enabled = true ``` diff --git a/docs/guide/error-reference.md b/docs/guide/error-reference.md index 0f7fd26bc..486adf015 100644 --- a/docs/guide/error-reference.md +++ b/docs/guide/error-reference.md @@ -55,9 +55,8 @@ Missing required field: publisher.domain **Cause:** Required configuration field not provided -**Solution:** Add the missing field to `trusted-server.toml`. The secret below -is an intentionally rejected placeholder; replace it with `openssl rand -base64 32` -before validation. +**Solution:** Add the missing field to `trusted-server.toml`. Secret fields name +entries in the Trusted Server secret store. ```toml [publisher] diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index f70cd25b8..9f51c335d 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -147,7 +147,7 @@ ts config init To bootstrap from a public publisher page, run an audit first: ```bash -ts audit https://publisher.example +ts audit generate https://publisher.example ``` The audit command writes `js-assets.toml` plus a draft `trusted-server.toml`. @@ -160,6 +160,7 @@ Edit `trusted-server.toml` to configure: - browser integrations under `[integrations.*]`; - server auction providers under map-shaped `[auction.providers.]`; - server bidder routes under `[auction.bidders.]`; +- ad server integrations; - KV store mappings; - EC configuration; - consent settings (`[gdpr]`); and diff --git a/docs/guide/integrations/aps.md b/docs/guide/integrations/aps.md index 55c9607a9..4f60b63a0 100644 --- a/docs/guide/integrations/aps.md +++ b/docs/guide/integrations/aps.md @@ -287,7 +287,9 @@ In `trusted_server` mode, the TSJS auction client validates the typed renderer d ### GAM and Universal Creative -For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid` and validates the complete envelope. In `trusted_server` mode it returns a static dynamic-renderer program that creates the same opaque renderer iframe. In `publisher_native` mode it instead resolves the publisher div and starts the friendly-frame runner without sending a Universal Creative renderer response. +For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid` and validates the complete envelope. In `trusted_server` mode it returns a static dynamic-renderer program that creates the same opaque renderer iframe. After the response is delivered, the bridge expands an authenticated ordinary display iframe only when its width and height attributes and computed geometry are still 1x1. It resizes that source iframe and every collapsed clipping ancestor through the authenticated slot root to the validated winning dimensions. Ambiguous sources, stale navigation or refresh completions, anchors, interstitials, fixed or sticky frames, invalid dimensions, and already-expanded frames remain unchanged. The same guard applies to APS capabilities, inline `adm`, and PBS Cache responses. + +In `publisher_native` mode the bridge instead resolves the publisher div and starts the friendly-frame runner without sending a Universal Creative renderer response. That renderer replaces the slot through a different owner and does not run the collapsed-shell helper. After the native runner loads, Trusted Server replaces the existing children of the resolved publisher div with the friendly frame. This removes the GAM or Universal Creative iframe when it is inside that div. If the runner fails, the existing iframe remains, but its Universal Creative request receives no response because Trusted Server has already claimed the selected bid. This one-owner behavior avoids a second render path, but GAM impression and viewability reporting must be validated with the APS account team for the controlled cohort. diff --git a/docs/guide/integrations/gpt-diagnostics-dictionary.md b/docs/guide/integrations/gpt-diagnostics-dictionary.md new file mode 100644 index 000000000..605186887 --- /dev/null +++ b/docs/guide/integrations/gpt-diagnostics-dictionary.md @@ -0,0 +1,156 @@ +# GPT Diagnostics Label Dictionary + +This dictionary defines the exact fixed labels and dynamic label prefixes in the GPT Runtime Diagnostics panel and badges. Text after a prefix such as `Server auction winner:` is the bounded observed value. The console reports observations; it does not identify the creative ultimately served unless a listed evidence source explicitly establishes that fact. Browser times use `performance.now()` and server times use the server request's `RequestTimings` clock. The clocks are never subtracted. + +See [GPT Runtime Diagnostics](./gpt-diagnostics.md) for activation and operational details. + +## Identity and controls + +| Label | Badge | Source | Meaning and limits | +| ----------------------------- | ------------ | ---------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| `Ad #N` | `Ad #N` | `runtimeSlotNumber` | Stable number assigned to a GPT slot object in this page. It is not a creative ID. | +| `Request #M` | `Request #M` | `requestNumber` | Monotonic request cycle for that slot, including refreshes. It does not imply fill. | +| `GPT-reported creative` | — | `adManager.creativeId` or `sourceAgnosticCreativeId` | GAM's callback identifier. It is separate from Ad/Request identity and does not identify a demand source. | +| `Filter` | — | panel state | Limits displayed rows. Activating a badge resets a hiding filter to `All`. | +| `Export JSON` | — | `gptDiagnostics.export()` | Downloads the allowlisted V1 snapshot. It issues no ad request. | +| `Label dictionary` | — | documentation link | Opens this keyboard-accessible help page. | +| `How to read this evidence` | — | inline help | Expands a concise explanation that winner, candidate, render, and timing observations have separate meanings and clocks. | +| `Locate on page` | — | exact unique DOM binding | Scrolls only on activation and briefly draws a diagnostics-layer highlight. It never changes publisher attributes, classes, or styles. | +| `Collapse`, `Expand`, `Close` | — | panel state | Presentation controls only. `show()` can reopen a closed panel. | +| `Request history` | — | retained cycles | Earlier retained requests. At most ten cycles are retained per slot. An evicted selection is reported as no longer retained. | +| `Technical details` | — | allowlisted snapshot fields | Expands correlation IDs, coverage, failures, and other non-summary evidence. | + +The per-request panel groups are `Summary`, `Auction evidence`, `Delivery evidence`, `Timing`, and `Size and visibility`. Summary prefixes are `GPT result:` and `Observed auction path:`. Technical prefixes include `Ad unit`, `Request intent:`, `Trusted Server auction:`, and `Prebid auction:`. A fact appears in one detailed group; Technical details does not repeat those grouped facts. + +Panel status is `GPT observed` or `Waiting for GPT`. Filter values are `All`, `Visible`, `Filled`, `Empty`, `Pending/Incomplete`, and `Unbound/Ambiguous`. Empty results are `No GPT slots observed yet.` or `No slots match.` The retained-cycle labels are `Initial request` and `Refresh N`; an evicted selection is announced as `Ad #N, Request #M is no longer retained.` The overview counts use the prefixes `slots`, `callback issues`, and `attribution issues`. Callback coverage uses the suffixes `observed`, `matched`, `unmatched`, and `ambiguous`. + +## GPT lifecycle and delivery + +| Label | Badge | Raw value / source | Meaning and what it does not prove | +| -------------------------------------------------------------------- | ------------------------- | ------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Waiting for request` | — | no request cycle | GPT has not emitted `slotRequested` for the slot. | +| `Requesting` | `Pending` | `slotRequested` | A GPT request callback was observed. Elapsed time alone does not mean failure. | +| `Response received` | — | `slotResponseReceived` | GPT emitted its response callback. It does not prove fill. | +| `GPT result: Filled` / `Filled` | `Filled` | `slotRenderEnded.isEmpty === false` | GPT reported non-empty. It does not prove which bidder or pixels were served. | +| `GPT result: Empty` / `Empty` | `Empty` | `slotRenderEnded.isEmpty === true` | GPT explicitly reported empty. | +| `GPT result: Rendered (fill unknown)` / `Rendered (fill unknown)` | `Rendered (fill unknown)` | `slotRenderEnded` without `isEmpty` | Render callback observed; fill remains unavailable. | +| `Creative markup sent; execution not confirmed` | `TS response sent` | `delivery=trusted_server_response_sent` | The bridge posted markup to PUC. It does not prove execution, visibility, or final served source. | +| `Server bid selected by the creative bridge; response not confirmed` | `TS selected` | `delivery=trusted_server_selected` | A matched PUC request selected the server bid; no successful response post was observed. | +| `Server bid available; selection not confirmed` | `TS unconfirmed` | `delivery=candidate_unconfirmed` | A server candidate existed but no matched selection appeared in the observation window. | +| `Waiting for Trusted Server creative evidence` | `TS candidate (pending)` | `delivery=pending` | The five-second positive-evidence window is open. | +| `No direct Trusted Server candidate` | `No TS candidate` | `delivery=no_candidate` | `adInit` explicitly found no direct candidate for this request. | +| `Delivery status unknown — required evidence was not observed` | `Delivery unknown` | `delivery=unknown` | Evidence exists but cannot establish a more specific delivery state. | +| `Delivery evidence: Not applicable` | — | `delivery=not_applicable` | No positive bridge evidence exists and a delivery conclusion does not apply before render or for an explicitly empty result. | +| `Delivery evidence: Not observed` | — | missing delivery state | No delivery state was captured. | +| `Served bidder not confirmed` | — | a non-empty GPT render without served proof | Server winner, targeting candidate, `bidWon`, and GPT render facts remain separate; none alone confirms the final served creative. This label is not shown for pending or empty requests. | +| `GPT slot onload observed` | — | `slotOnload` | GPT emitted onload. It is not pixel-level or demand-source proof. | +| `GPT impressionViewable observed` | — | `impressionViewable` | GPT emitted its viewability callback. | +| `Incomplete sequence` | `Incomplete sequence` | `incompleteSequence` | An observed callback proves a missing or invalid predecessor. Time alone never sets it. | + +## Auction evidence + +| Label | Badge | Raw value / source | Meaning and limits | +| ----------------------------------- | ---------------- | --------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `SSAT: initial-page server auction` | `SSAT` | `auctionType=ssat` | Explicit completed server-auction evidence from the initial page request. | +| `TS auction: SPA server auction` | `TS auction` | `auctionType=trusted_server` | Explicit completed `/_ts/page-bids` server-auction evidence. | +| `Client-side Prebid auction` | `Prebid auction` | `auctionType=client_side`, exact Prebid callback correlation | A completed Prebid attempt was correlated to this slot and next GPT request. A refresh wrapper call alone does not establish it. | +| `Multiple auction paths observed` | `Multiple paths` | `auctionType=competing` | Completed server and Prebid auction evidence both exist for the request. This does not prove a race, overwrite, or winner. | +| `Auction not observed` | no auction badge | missing or malformed explicit facts | Diagnostics has no qualifying completed-auction evidence. It never defaults to SSAT. | +| `Server auction winner` | — | compatibility `auctionWinner`, server `hb_bidder` | Winner selected by the server auction. It is not the final served bidder. | +| `Server bid price bucket:` | — | server `hb_pb` | Already-bucketed price; raw CPM is never exported. Currency is shown only when supplied. | +| `Prebid targeting candidate:` | — | `prebidAuction.targetingCandidate` after exact `setTargetingForGPTAsync` boundary | Candidate targeting observed on the exact GPT slot for the exact Prebid callback auction ID. It is not a final win. | +| `Prebid candidate price bucket:` | — | candidate `hb_pb` | Bucketed targeting value for the candidate; it is not a winning-price claim. | +| `Prebid bidWon observation:` | — | `prebidAuction.win`, documented `bidWon` payload | A bounded event joined by exact auction ID, ad-unit code, slot object, navigation generation, and retention window. It remains distinct from GPT render and served-source proof. | +| `Prebid win price bucket:` | — | `bidWon.adserverTargeting.hb_pb` | Bucketed value observed on the correlated `bidWon` event; it is not proof of the creative GAM served. | +| `(currency not supplied)` | — | absent validated ISO currency | No verified currency was supplied. The console never assumes USD and does not compare currencies. | + +Bidder names are limited to 128 UTF-8 bytes, numeric bucket strings to 64 bytes, currencies to three ASCII letters, and auction IDs to 256 UTF-8 bytes. Prebid supplies its own auction ID through `bidsBackHandler`; diagnostics does not override Prebid auction identity. Duplicate, ambiguous, expired, late, prior-navigation, and malformed observations are rejected. Only an active diagnostics recorder installs the `bidWon` listener or retains candidate/win state, which is bounded to 128 pending attempts and 30 seconds. No raw CPM, creative markup, targeting dump, or losing bids are retained. + +## Request paths and opportunities + +| Label | Raw value | Meaning | +| -------------------------------------------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `Request path: Trusted Server direct` | `trusted_server_direct` | The direct `adInit` route preceded the request. Route evidence alone is not auction evidence. | +| `Request path: Prebid refresh` | `prebid_refresh` | The installed wrapper delegated the GPT refresh. Timeout and throw fallbacks retain this route label but do not claim a completed auction. | +| `Request path: Publisher refresh` | `publisher_refresh` | Publisher refresh boundary observed. | +| `Request path: Multiple paths observed` | `competing` | More than one route marker preceded the request; competition is possible but unproven. | +| `Request path: Not observed` | `unattributed` | No eligible route marker was consumed. | +| `Server bid available; creative source present` | `renderable_candidate` | Bid targeting plus ad ID and inline markup or complete cache coordinates were present. | +| `Server bid available; creative source incomplete` | `unrenderable_candidate` | Bid targeting existed but the bridge lacked a complete render source. | +| `Direct opportunity: No candidate` | `no_candidate` | `adInit` explicitly observed no direct bid targeting. | +| `Direct opportunity: Not observed` | missing opportunity | No bounded direct-opportunity evidence was captured. It is not negative demand-source evidence. | +| `Request intent:` | `requestIntentId` | Opaque local correlation sequence, not an auction or user ID. | +| `Trusted Server auction:` | `trustedServerAuctionId` | Opaque per-auction correlation token, not a GAM key or visitor identifier. | +| `Prebid auction:` | `prebidAuction.auctionId` | Opaque Prebid-supplied auction correlation token. Diagnostics does not create or replace it. | + +Path markers live for five seconds, are consumed once, and are keyed by GPT slot object identity. + +## Timing + +All values are milliseconds. Missing timing that should apply is `Unavailable`, never zero. Server timing is `Not applicable` when no completed server auction was observed. A displayed zero is a valid immediate observation. + +| Label | Origin and boundaries | Raw field | +| ------------------------------------------- | -------------------------------------------------------------------------------- | ------------------------------------------ | +| `Server request start → auction dispatched` | Server request `RequestTimings` T0 to successful `dispatch_auction` outcome | `serverAuctionTimings.auctionDispatchedMs` | +| `Server request start → auction collected` | Same T0 to completion of `collect_dispatched_auction` | `auctionResolvedMs` | +| `Server request start → bids ready` | Same T0 to winning-bid map commit | `auctionCommittedMs` | +| `Auction collection wait` | Actual duration blocked in collect; placement is `pre-header` or `in stream` | `auctionWaitMs`, `auctionWaitPlacement` | +| `Opportunity → request` | Browser `performance.now()`: recorder observation to matched GPT `slotRequested` | `opportunityToRequestMs` | +| `GAM request → response` | Browser `slotRequested` to `slotResponseReceived` | `durations.requestToResponseMs` | +| `GAM response → render` | Browser `slotResponseReceived` to `slotRenderEnded` | `responseToRenderMs` | +| `GAM request → render` | Browser `slotRequested` to `slotRenderEnded` | `requestToRenderMs` | +| `Render → load` | Browser `slotRenderEnded` to `slotOnload` | `renderToLoadMs` | +| `Render → viewable` | Browser `slotRenderEnded` to `impressionViewable` | `renderToViewableMs` | +| `Replaced rendered request` | Earlier browser render callback to later request callback | `previousRenderToRequestMs` | + +Server offsets are not browser timestamps. `auctionResolvedMs` means collection completed (including timeout handling), not that a network byte arrived at that exact instant. + +## Size, visibility, binding, and GAM fields + +| Label | Raw field / source | Meaning and limits | +| ------------------------------------------------------------------------------ | ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `Requested sizes` | `requestedSlotSizes` | Configured sizes supplied to GPT; ordinary sizes such as 300×250 remain visible. Missing data is shown as `Requested sizes: Not observed`. | +| `GPT-reported size` | `size` | Exact `slotRenderEnded.size`. A 1×1 placeholder is shown as `GPT-reported size: 1×1 placeholder hidden` and retained unchanged in V1 JSON. | +| `Size filled` / `Measured outer slot size` | `observedSlotSize` | CSS outer box of the exact uniquely bound slot after fill. Missing data is shown as `Size filled: Not observed · Measured outer slot size`. | +| `GPT visibility` | current/maximum visibility percentage | Values from GPT visibility callbacks; absence is `GPT visibility: Not observed`. | +| `Binding: Bound` / `Bound` | `binding.status=bound` | Exactly one connected publisher element matched. | +| `Binding: unbound` / `Unbound` | `binding.status=unbound` | No safe element binding; reason is shown. | +| `Binding: ambiguous` / `Ambiguous binding` | `binding.status=ambiguous` | Duplicate ID or GPT-slot evidence prevented a safe choice. No badge or locate action is invented. | +| `Outside viewport` | binding geometry | The exact element does not intersect the viewport; it remains in the panel. | +| `Ad Manager response class: empty/backfill/reservation/unclassified non-empty` | `responseClass` | Source-neutral classification derived from GPT render facts. | +| `Ad Manager reported …` | `adManager` identifiers | GAM-reported line item, order, advertiser, creative, yield-group, and company IDs. They do not identify the demand source. | +| `Ad Manager fields: Not observed` | missing `adManager` | No allowlisted GAM identifier was captured. | +| `Ad Manager response class: Not observed` | missing `responseClass` | GPT did not provide enough render evidence to classify the response. | +| `Backfill yes/no` | `isBackfill` | GPT's callback value. | +| `Slot content changed yes/no` | `slotContentChanged` | GPT's callback value; not proof pixels changed. | +| `Creative changed/unchanged` | retained GAM creative IDs | Comparison only when both cycles supplied an ID. | + +Binding reasons are `missing_slot_element_id`, `missing_element`, `duplicate_dom_id`, `dom_uniqueness_unverifiable`, and `duplicate_gpt_slot_id`. Other exact technical fact prefixes are `Replaced rendered request`, `Creative changed`, `Creative unchanged`, `Trusted Server creative request observed at`, and `Trusted Server markup response sent at`. + +## Failures, attribution, coverage, and retention + +Creative bridge labels are `Creative bridge failure: missing render source`, `Creative bridge failure: cache fetch failed`, `Creative bridge failure: invalid cache payload`, and `Creative bridge failure: response post failed`. They map directly to `missing_render_source`, `cache_fetch_failed`, `invalid_cache_payload`, and `response_post_failed`; they describe only the observed bridge step and contain no URL, markup, payload, or stack trace. + +Attribution issue labels map to `creative_request_without_slot`, `creative_request_without_cycle`, `creative_request_ambiguous_cycle`, `creative_request_on_empty_cycle`, `creative_attempt_capacity`, `creative_attempt_unknown`, `creative_attempt_expired`, and `creative_attempt_evicted`. Callback coverage separately counts `observed`, `matched`, `unmatched`, and `ambiguous` for `slotRequested`, `slotResponseReceived`, `slotRenderEnded`, `slotOnload`, `impressionViewable`, and `slotVisibilityChanged`. + +The V1 export remains additive and compatible: optional Prebid evidence and optional currency fields are new; existing fields keep their meanings. Bounds are 64 slots, ten request cycles per slot, 128 callback issues, 128 attribution issues, 64 direct associations, 16 requested sizes, and 128 creative attempts. Metadata reports dropped callbacks/issues and evicted slots/cycles. + +## Missing-value vocabulary + +- **Not observed**: the relevant callback or explicit evidence was not captured. +- **Unavailable**: the value cannot be calculated or safely retained. +- **Not applicable**: the fact does not apply, for example delivery conclusions before render or for an explicitly empty result. +- **Unknown**: evidence exists but is insufficient to select a more specific state. + +## Acronyms + +- **TS**: Trusted Server. +- **SSAT**: server-side ad targeting on the initial page request. +- **SPA**: single-page application. +- **GPT**: Google Publisher Tag. +- **GAM**: Google Ad Manager. +- **Prebid**: the client-side header bidding library observed here. +- **PBS**: Prebid Server. +- **PUC**: Prebid Universal Creative. +- **CPM**: cost per thousand impressions; diagnostics retains only a bucketed targeting value, never raw CPM. +- **T0**: the start of one server request's `RequestTimings` clock. diff --git a/docs/guide/integrations/gpt-diagnostics.md b/docs/guide/integrations/gpt-diagnostics.md index e419a1a79..7d471bbf8 100644 --- a/docs/guide/integrations/gpt-diagnostics.md +++ b/docs/guide/integrations/gpt-diagnostics.md @@ -9,9 +9,8 @@ GPT Runtime Diagnostics is an opt-in browser console for documented Google Publisher Tag (GPT) lifecycle callbacks and Trusted Server integration evidence. -It groups observations into per-slot request cycles, shows timings and source-neutral -GAM facts, binds slots to exact DOM elements, and downloads the same allowlisted data -as versioned JSON. +It groups observations into per-slot request cycles, shows auction and GPT timings, +links each accessible page badge to the same stable `Ad #N` and `Request #M` in the side panel, binds slots to exact DOM elements, and downloads the same allowlisted data as versioned JSON. See the complete [label dictionary](./gpt-diagnostics-dictionary.md) for every operator-facing term. The console reports positive observations, not inferred ownership. A filled result means only that GPT emitted `slotRenderEnded` with `isEmpty === false`. A Trusted @@ -109,8 +108,12 @@ Visible, Filled, Empty, Pending/Incomplete, and Unbound/Ambiguous slots. Each request cycle can show: +- The same stable `Ad #N` and `Request #M` used by the request's page badge. - The observed request path, request-intent ID, and direct Trusted Server opportunity. +- Explicit auction classification: initial-page server, SPA server, completed client-side Prebid, multiple observed paths, or not observed. - Opaque Trusted Server auction-ID correlation and opportunity-to-request latency when available. +- Source-scoped server-auction winner, Prebid targeting candidate, and documented Prebid win observations. None is presented as the final served creative. +- Server-measured auction dispatch, resolution, commit, and wait timing when available. - Observed replacement of an earlier retained filled render, including GPT creative-ID transitions. - Requesting, Response received, Filled, Empty, or Rendered (fill unknown) GPT lifecycle state. @@ -118,9 +121,11 @@ Each request cycle can show: - Safe, deduplicated creative-bridge failure categories. - Source-neutral GAM response class and identifiers reported by GPT. - GPT slot-onload, impression-viewable, and visibility observations. -- Non-negative request-to-response, response-to-render, render-to-load, and +- Non-negative GAM request-to-response, GAM response-to-render, render-to-load, and render-to-viewable durations. -- GPT-reported rendered size, a separately labelled observed outer slot box when safely bound, backfill, and slot-content-change facts. +- GPT-reported rendered size, a separately labelled `Size filled` outer-slot measurement + when safely bound, backfill, and slot-content-change facts. GPT's ubiquitous `1×1` + placeholder stays in the JSON evidence but is hidden from the panel and badge. - Current DOM binding status and viewport intersection. Elapsed time alone never changes a pending GPT request to Incomplete. Incomplete @@ -129,6 +134,62 @@ step. When `slotRenderEnded` omits `isEmpty`, the result stays Rendered (fill un and `responseClass` remains absent. `unclassified_non_empty` requires an explicit `isEmpty === false` observation. +## Auction Classification and Timing + +Auction labels describe the path observed for that GPT request cycle: + +| Label | Meaning | +| --------------------------------- | ---------------------------------------------------------------------------------------------------- | +| SSAT: initial-page server auction | Explicit completed initial-document server-auction evidence populated the direct request. | +| TS auction: SPA server auction | Explicit completed `/_ts/page-bids` server-auction evidence populated the direct request. | +| Client-side Prebid auction | A completed Prebid attempt was joined by exact auction ID, ad-unit code, GPT slot, and next request. | +| Multiple auction paths observed | Server and completed client-side evidence were both observed. This does not prove a race or winner. | +| Auction not observed | Explicit completed-auction evidence was absent or malformed. It never defaults to SSAT. | + +Publisher refresh evidence does not by itself establish another auction. It remains +visible in the request-path classification but does not turn an SSAT, TS auction, or +client-side auction label into “Competing auctions.” Publisher-only refreshes and +unattributed requests have no auction label because the available evidence does not +establish an auction implementation. + +For synthetic refresh auctions, diagnostics correlates the auction ID supplied by +Prebid's own `bidsBackHandler`; it does not provide or override Prebid auction IDs. The +`bidWon` listener, targeting reads, and bounded correlation state are installed only +when an active diagnostics recorder exists, so an inactive console does not change +Prebid-visible behavior. + +Server auction timing and browser GPT timing use separate clocks and are never +subtracted from each other. Both initial and SPA offsets use their request-scoped `RequestTimings` T0: server request processing start. Diagnostics retains the server origin separately from aggregate classification. The server facts are: + +- `auctionDispatchedMs`: bid dispatch offset from that timing origin. +- `auctionResolvedMs`: offset when auction collection completed, including timeout handling. +- `auctionCommittedMs`: offset when winning bids were available to page state. +- `auctionWaitMs`: actual time blocked in auction collection. +- `auctionWaitPlacement`: whether that wait occurred before response headers or while + the document response was streaming. + +The SPA page-bids response includes these server timings only for an activated +console session and a successfully dispatched auction. Failures before any provider +dispatch do not emit timing evidence. The activation cookie is still removed before +auction and publisher processing; the server retains only the request-scoped activation +decision needed to gate this diagnostics field. + +Browser timings have different boundaries. **GAM request → response** starts at GPT's +`slotRequested` callback and ends at `slotResponseReceived`. **GAM response → render** +starts at `slotResponseReceived` and ends at `slotRenderEnded`. “Render” therefore +means that GPT emitted its render-ended callback; it does not prove inner-iframe pixels +executed or became visible. + +```mermaid +flowchart TB + A[Server timing origin] --> B[Auction dispatched] + B --> C[Auction resolved] + C --> D[Bids committed to page state] + E[GPT slotRequested] --> F[GPT slotResponseReceived] + F --> G[GPT slotRenderEnded] + G --> H[GPT slotOnload / impressionViewable] +``` + ## Request Paths Request-path labels describe integration paths observed immediately before one GPT @@ -162,8 +223,10 @@ context even when the delegated refresh throws, and the Prebid wrapper restores exact prior value. Diagnostics never suppresses or changes a GPT request. For a direct observation, the optional opaque auction ID is retained only after -trimming to a non-empty value no longer than 256 UTF-8 bytes. No auction payload, -targeting map, bid price, markup, network body, or stack trace is exported. The +trimming to a non-empty value no longer than 256 UTF-8 bytes. Winning bidder names are +limited to 128 UTF-8 bytes, and numeric price-bucket strings to 64 bytes. The console +exports only those winning-bid fields and the bounded timing object—not a raw auction +payload, targeting map, exact unbucketed CPM, markup, network body, or stack trace. The reported opportunity-to-request duration is browser-observed only and is omitted for invalid or negative timing. @@ -181,8 +244,8 @@ For the direct path, `adInit` records one opportunity: | `unrenderable_candidate` | Bid targeting was applied, but the current bridge lacked the complete ID/render-source combination needed to serve markup. | | `no_candidate` | `adInit` explicitly observed no direct Trusted Server bid targeting for that configured slot. | -An absent opportunity is displayed as unknown. It must not be converted into a -negative demand-source conclusion. +An absent opportunity is displayed as `Not observed`. It must not be converted into +a negative demand-source conclusion. ## Trusted Server Evidence Ladder @@ -204,9 +267,9 @@ The derived `delivery` value uses these evidence-safe meanings: | Delivery state | Panel wording | | ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `trusted_server_response_sent` | Trusted Server selected; markup response sent to PUC | -| `trusted_server_selected` | Trusted Server selected; no markup response confirmed | -| `candidate_unconfirmed` | Trusted Server candidate unconfirmed — another GAM result or a creative/bridge failure is possible | +| `trusted_server_response_sent` | Creative markup sent; execution not confirmed | +| `trusted_server_selected` | Server bid selected by the creative bridge; response not confirmed | +| `candidate_unconfirmed` | Server bid available; selection not confirmed | | `no_candidate` | adInit observed no direct Trusted Server candidate for this request | | `unknown` | Delivery status unknown — required GPT or direct-candidate evidence was not observed | | `pending` | Waiting for Trusted Server creative evidence | @@ -309,11 +372,12 @@ A concise viewport badge appears only when a slot: - Has a unique, connected exact binding. - Has a non-zero rectangle intersecting the viewport. -A badge summarizes the slot's most recent request cycle: the GPT result (Filled, Empty, -Rendered (fill unknown), or Pending), a short delivery label, a `Competing paths` -marker when the request path is `competing`, the rendered size, and the request-to- -response, response-to-render, and render-to-viewable durations that are available. It -adds `Incomplete sequence` when a callback proved a missing or invalid earlier step. +A badge starts with the slot's stable `Ad #N`, matching the side-panel heading. It then +summarizes the most recent request cycle: the GPT result (Filled, Empty, Rendered (fill +unknown), or Pending), auction type, a short delivery label, a `Competing paths` marker +when the request path is `competing`, sizes, and the GAM request-to-response, GAM +response-to-render, and render-to-viewable durations that are available. It adds +`Incomplete sequence` when a callback proved a missing or invalid earlier step. Badge delivery labels are the same derived states the panel and export report, shortened to fit: @@ -354,10 +418,11 @@ becomes unbound or ambiguous, the last successful sample remains as request-cycl evidence while the binding status reports the current DOM state. It is displayed separately from `size`, which remains the exact GPT-reported `slotRenderEnded.size` fill-size fact. The panel labels the three separate facts as requested slot sizes, -GPT-reported fill size, and observed outer slot box; the badge abbreviates them as -`Req`, `Fill`, and `Box`. The observed box may differ from GPT's reported size (for -example, a flexible APS creative can report `1×1` while its allocated outer slot box -is larger). The measurement describes publisher-page layout, not universal internal +GPT-reported fill size, and `Size filled`; the badge abbreviates the first two as `Req` +and `Fill` and uses `Size filled` for the observed outer box. The observed box may +differ from GPT's reported size. A flexible APS creative, for example, can report GPT's +uninformative `1×1` placeholder while its allocated outer slot box is larger; the UI +suppresses that `1×1` text but the export retains the original `size: [1, 1]` evidence. The measurement describes publisher-page layout, not universal internal creative-pixel dimensions. A collapsed or hidden bound element can report `0×0`, which records the page layout state rather than an invalid measurement. Empty, unbound, missing, or ambiguous slots do not report an observed box; delayed measurements from an older cycle @@ -419,8 +484,10 @@ The allowlisted export contains: - Retained slots, binding facts, visibility, and request cycles. - `requestedSlotSizes` when Trusted Server supplied configured formats for that exact request, plus GPT-reported fill `size` and an optional observed outer `observedSlotSize`. -- Request path, request intent ID, opportunity, creative-progress timestamps, and - safe failure enums. +- Request path, auction type, request intent ID, opportunity, creative-progress + timestamps, and safe failure enums. +- The bounded winning bidder and bucketed price plus server auction timing fields and + their `navigation` or `spa_auction` origin when direct auction evidence was observed. - The per-auction diagnostics token (`trustedServerAuctionId`) and the opportunity-to-request duration, when a direct opportunity was observed. - Replacement facts for a re-rendered slot: `replacedRequestNumber`, @@ -431,9 +498,10 @@ The allowlisted export contains: - Separate callback issues, attribution issues, coverage counters, and retention counters. -It does not contain raw targeting, bid IDs, bid prices, bidder identity, creative -markup, cache URLs, cache payloads, cache or bridge error details, cookies, user -identifiers, query strings, or URL fragments. The exported `trustedServerAuctionId` +It does not contain raw targeting, bid IDs, exact unbucketed bid prices, losing bidder +identity, creative markup, cache URLs, cache payloads, cache or bridge error details, +cookies, user identifiers, query strings, or URL fragments. It does contain the winning +bidder and bucketed `hb_pb` value described above. The exported `trustedServerAuctionId` is the `hb_auction_id` value described in [Auction correlation token](#auction-correlation-token): minted fresh for each server-side auction, not derived from the Edge Cookie ID or any other visitor @@ -463,8 +531,9 @@ inaccessible to JavaScript. - Retained attribution issues: 128. The least-recently-active slot is evicted when the slot bound is exceeded. An evicted -GPT slot can re-enter retention only after a future `slotRequested`; request numbers -remain monotonic. The oldest request cycle or issue is removed at its own bound. +GPT slot can re-enter retention only after a future `slotRequested`; its `Ad #N` identity +and request numbers remain stable and monotonic for the console lifetime. The oldest +request cycle or issue is removed at its own bound. Export metadata reports `evictedSlots`, `evictedRequestCycles`, `droppedCallbacks`, and `droppedAttributionIssues`. diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index dfc8ddb79..201fae4ad 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -34,9 +34,20 @@ external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" # external_bundle_sha256 = "" # external_bundle_sri = "sha384-" +# Optional operator-owned Prebid User ID modules, forwarded to Prebid verbatim. +[[integrations.prebid.managed_user_ids]] +name = "identityLink" +params = { pid = "999", notUse3P = false } + +[integrations.prebid.managed_user_ids.storage] +type = "cookie" +name = "idl_env" +expires = 15 +refresh_in_seconds = 1800 + [integrations.prebid.bundle] adapters = ["example-browser"] -user_id_modules = ["sharedIdSystem"] +user_id_modules = ["sharedIdSystem", "identityLinkIdSystem"] [proxy] allowed_domains = ["assets.example.com"] @@ -75,20 +86,26 @@ provider = "pbs-main" ### Browser configuration options -| Field | Default | Ownership and behavior | -| ------------------------------------ | ---------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | -| `enabled` | `true` | Enables browser bundle injection/interception; it does not create a server provider | -| `account_id` | `None` | Optional browser-injected account value | -| `timeout_ms` | `1000` | Browser Prebid.js timeout only | -| `debug` | `false` | Browser Prebid.js debug only | -| `client_side_bidders` | `[]` | Native browser adapters that are not folded into `trustedServer` | -| `excluded_gam_ad_unit_path_suffixes` | `[]` | GAM suffixes omitted from Trusted Server refresh auctions | -| `script_patterns` | `["/prebid.js", "/prebid.min.js", "/prebidjs.js", "/prebidjs.min.js"]` | Publisher Prebid scripts intercepted to prevent duplicate instances | -| `external_bundle_url` | Required when enabled | HTTPS generated bundle URL; host and redirects must be in `proxy.allowed_domains` | -| `external_bundle_sha256` | `None` | Optional content hash used for versioning, cache policy, and ETag | -| `external_bundle_sri` | `None` | Optional SRI metadata | -| `bundle.adapters` | Required for `ts prebid bundle` | Browser bidder adapters compiled into the external bundle | -| `bundle.user_id_modules` | Generator preset | Browser User ID modules compiled into the external bundle | +| Field | Default | Ownership and behavior | +| ----------------------------------------------- | ---------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| `enabled` | `true` | Enables browser bundle injection/interception; it does not create a server provider | +| `account_id` | `None` | Optional browser-injected account value | +| `timeout_ms` | `1000` | Browser Prebid.js timeout only | +| `debug` | `false` | Browser Prebid.js debug only | +| `client_side_bidders` | `[]` | Native browser adapters that are not folded into `trustedServer` | +| `excluded_gam_ad_unit_path_suffixes` | `[]` | GAM suffixes omitted from Trusted Server refresh auctions | +| `script_patterns` | `["/prebid.js", "/prebid.min.js", "/prebidjs.js", "/prebidjs.min.js"]` | Publisher Prebid scripts intercepted to prevent duplicate instances | +| `external_bundle_url` | Required when enabled | HTTPS generated bundle URL; host and redirects must be in `proxy.allowed_domains` | +| `external_bundle_sha256` | `None` | Optional content hash used for versioning, cache policy, and ETag | +| `external_bundle_sri` | `None` | Optional SRI metadata | +| `bundle.adapters` | Required for `ts prebid bundle` | Browser bidder adapters compiled into the external bundle | +| `bundle.user_id_modules` | Generator preset | Browser User ID modules compiled into the external bundle | +| `managed_user_ids[].name` | Required | Prebid `userSync.userIds` entry name Trusted Server installs and reinstates; unique across entries | +| `managed_user_ids[].params` | `{}` | Module-specific parameters, forwarded to Prebid.js unchanged | +| `managed_user_ids[].storage.type` | `cookie` | Browser storage for the module value: `cookie` or `html5` | +| `managed_user_ids[].storage.name` | Required when `storage` exists | Cookie or local-storage key the module reads and writes | +| `managed_user_ids[].storage.expires` | Prebid default | Storage lifetime in days; at least 1. Any per-module ceiling is the module own | +| `managed_user_ids[].storage.refresh_in_seconds` | Prebid default | Seconds before the module may refresh the stored value; at least 1 | ### Server provider options @@ -165,6 +182,27 @@ regenerating and re-uploading the bundle (and pushing the updated rollout. The shim refuses to install twice on one page via the `window.__tsjsPrebidShimInstalled` sentinel. +The consent modules include Prebid's `tcfControl`, so a regenerated bundle can +enforce the TCF signal it collects rather than only reporting it. Its default +rules are activity-specific and depend on which module a managed entry selects. +For an `identityLink` entry, for example, Purpose 1 and LiveRamp's GVL vendor +consent gate browser resolution and storage; Purpose 3 has no standalone default +rule, and Purpose 4 controls user-provided-data activity rather than +IdentityLink resolution. Validate a regenerated bundle against a live CMP before +rolling it out broadly. + +When managed User IDs are configured and the page exposes a callable +`window.__tcfapi`, the Trusted Server shim activates Prebid's standard IAB GDPR +collector by adding only `consentManagement.gdpr.cmpApi = "iab"`. It does not +set a timeout or force `defaultGdprScope`. An existing publisher-owned `gdpr` +value always wins, sibling consent settings are preserved, and pages without a +TCF API are unchanged. If queued or late publisher configuration later supplies +its own `gdpr` value, the shim first deactivates the collector it created so the +old IAB listener cannot overwrite the publisher's consent state. Ownership +transfers once; the automatic collector is not re-enabled afterward. A delayed +first CMP response is also ignored after transfer and removes its listener when +the CMP finally supplies the listener ID. + ## Debug Mode When `debug = true`, the Prebid integration enables additional diagnostics on both the outgoing OpenRTB request and the incoming response. @@ -506,6 +544,187 @@ Example EID source mapping: User ID module selection is separate from `--adapters`, which controls client-side bidder adapter modules. +## Managed User ID modules + +Trusted Server can own one or more Prebid `userSync.userIds` entries so +operators configure identity centrally instead of asking publishers to edit +their Prebid JavaScript. + +Each `[[integrations.prebid.managed_user_ids]]` entry is forwarded to Prebid.js +verbatim. Trusted Server validates only what Prebid needs to address the module +— a usable entry name and storage key, positive expiry and refresh values — and +never interprets `params`. Supported names come from the checked-in +`user_id_modules.json` registry, so Trusted Server core needs no +vendor-specific code and names no identity vendor itself. + +### Prerequisites + +The module must be present in the built bundle. Name it under +`bundle.user_id_modules`, or omit that list to take the generator's default +preset, which covers the commonly used modules. + +`ts prebid bundle` resolves every managed `name` through the checked-in +`user_id_modules.json` registry. An unknown name, or a name that maps to more +than one module, fails before bundle generation. After generation, the command +reads the new manifest and confirms that every resolved module is present. A +missing module reports both the managed name and required module and leaves the +existing bundle hash and SRI unchanged. + +The browser-side diagnostic remains useful when a bundle is hosted externally, +is stale, or was modified after generation. Core remains vendor-neutral: it +forwards each managed entry's `params` to Prebid.js without interpreting them. + +```toml +[integrations.prebid.bundle] +adapters = ["rubicon"] +user_id_modules = ["identityLinkIdSystem"] + +[[integrations.prebid.managed_user_ids]] +name = "identityLink" +params = { pid = "999", notUse3P = false } + +[integrations.prebid.managed_user_ids.storage] +type = "cookie" +name = "idl_env" +expires = 15 +refresh_in_seconds = 1800 +``` + +Run `ts prebid bundle`, upload the generated content-addressed bundle, copy its +hash metadata into `[integrations.prebid]`, and validate the configuration +before rollout. + +### Worked example: LiveRamp RampID + +The configuration above selects Prebid's `identityLink` submodule, which +resolves a LiveRamp RampID identity envelope and forwards it through the +existing EID path. It is presented here as the reference example; the mechanism +is the same for any User ID module. + +Trusted Server does not collect email addresses, hash identifiers, call a +server-to-server ATS API, or add a new application-facing envelope API. + +Before configuring it, obtain a test or production Placement ID from LiveRamp, +have the exact publisher origin approved by LiveRamp, and confirm the +publisher's CMP and LiveRamp contract permit the intended recognition mode. +`idl_env` is IdentityLink's documented storage key and `pid` its documented +Placement ID parameter; both are operator configuration here, not values Trusted +Server supplies. + +The generated bundle carries Prebid's `tcfControl` module alongside the +`consentManagement*` modules. That pairing is what makes the TCF signal +enforceable: `consentManagement*` retrieves the consent data, while `tcfControl` +registers activity controls that act on it. For managed User IDs, the shim +activates the collector when `window.__tcfapi` is callable and the publisher has +not already supplied a `consentManagement.gdpr` value. Under pinned Prebid's +defaults, a later publisher `gdpr` value takes ownership after the shim removes +its automatically registered IAB listener. Purpose 1 and LiveRamp's GVL vendor +consent (vendor 97) gate IdentityLink resolution and storage. Purpose 3 has no +standalone default rule. Purpose 4 controls user-provided-data activity, but +denying it alone does not block IdentityLink resolution or storage. + +Default EID transmission accepts a qualifying purpose and vendor basis from any +of Purposes 2–10. Publishers can require Purpose 4 specifically by enabling +Prebid's `eidsRequireP4Consent` setting. These are the generated bundle's TCF +defaults; equivalent GPP/US-state browser activity-control modules are not +bundled, so US-state opt-outs remain enforced at Trusted Server's forwarding +gate. + +When entries are configured, Trusted Server owns one deterministic entry per +configured `name` for publisher configuration applied through the public +`pbjs.setConfig` and `pbjs.mergeConfig` APIs. Other publisher-configured User ID +entries are preserved, but calls through those APIs that add, remove, or replace +a managed name are normalized back to the operator-managed values. This is a +configuration-ownership convention, not a security boundary against same-origin +code that retained a pre-wrapper function reference or directly mutates Prebid's +internal configuration. Including a module in a bundle is inert until a managed +entry selects it. + +### Resolution timing and data flow + +IdentityLink resolves asynchronously. A new browser's first auction can run +before RampID is available; later auctions can include it without blocking the +page or auction. When available, the opaque value follows the standard path: + +1. `pbjs.getUserIdsAsEids()` exposes an entry whose source is `liveramp.com`. +2. The current `/auction` request includes that entry. +3. Trusted Server merges and consent-gates it, then forwards it to Prebid + Server as `user.ext.eids`. +4. The browser persists the same opaque value in the bounded `ts-eids` cookie. +5. A later request can ingest it into an EC/KV partner configured with + `source_domain = "liveramp.com"`. + +Trusted Server treats the RampID envelope as an opaque string. Do not log, +decode, publish, or dimension metrics by the value. Source names, counts, +booleans, and status codes are sufficient for diagnostics. + +### Browser network and storage footprint + +With `notUse3P` unset or false, the IdentityLink submodule performs +third-party recognition from the browser. Operators should plan for this before +configuring the entry: + +| Effect | Detail | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| Outbound request | A credentialed `GET` from the page to LiveRamp's envelope endpoint (`api.rlcdn.com`). Trusted Server does not proxy it. | +| Content Security Policy | Publishers running a strict CSP must allow that host in `connect-src`, or recognition fails silently. | +| Browser storage | `idl_env` plus IdentityLink's bookkeeping entries (`idl_env_cst`, `idl_env_last`, `_lr_retry_request`, `_lr_env_src_ats`). | +| Recognition opt-out | `notUse3P = true` suppresses the third-party request. RampID then resolves only where an authenticated envelope is already available on the page. | + +Because the request leaves the browser directly rather than through the edge, +this integration is not a first-party replacement for LiveRamp recognition; it +configures Prebid's client-side submodule on the operator's behalf. Server-side +resolution is tracked separately (see the design document's out-of-scope +section). + +If the publisher's page already loads LiveRamp's ATS library, the submodule +prefers `window.ats.retrieveEnvelope` over the third-party endpoint. That is +the submodule's own behavior — Trusted Server neither loads ATS nor calls a +server-to-server ATS API. + +### Degraded behavior + +| Condition | Result | +| -------------------------------------------------- | ----------------------------------------------------------------------- | +| TCF Purpose 1 or LiveRamp vendor consent is denied | Default `tcfControl` blocks IdentityLink resolution and storage | +| TCF Purpose 3 or 4 alone is denied | Resolution/storage continues under defaults; publisher rules may differ | +| The user opts out under a US state signal | No LiveRamp EID is forwarded; the auction continues | +| LiveRamp cannot recognize the browser | IdentityLink yields no EID; the auction continues | +| LiveRamp network resolution fails | The current auction continues without RampID | +| The managed module is missing from the bundle | Existing diagnostics report the missing module; auctions continue | +| The origin is not approved by LiveRamp | Resolution yields no usable EID; the auction continues | +| EC/KV is unavailable | A current-request EID can still reach `/auction`; persistence degrades | + +The TCF rows assume either the managed-ID automatic setup described above or a +publisher-owned Prebid GDPR configuration. A CMP API and its policy remain +publisher responsibilities; Trusted Server does not synthesize consent or GDPR +applicability. + +### Credential-based validation + +Live validation must run outside CI on a LiveRamp-approved non-production +origin. Never commit a live Placement ID or envelope. Record only the approved +domain, booleans, source names, counts, and status codes: + +1. Build a bundle containing `identityLinkIdSystem` and configure a managed + `identityLink` entry with the test Placement ID. +2. With positive consent, confirm `idl_env` is created or refreshed. +3. Confirm `pbjs.getUserIdsAsEids()` reports source `liveramp.com` without + recording its value. +4. Confirm a controlled Prebid Server request contains that source in + `user.ext.eids`. +5. Confirm a later request ingests the source into the configured + `liveramp.com` EC partner. +6. Repeat with denied consent and confirm the envelope endpoint is not called, + `idl_env` is not written, and no LiveRamp EID is forwarded. +7. Repeat on an unapproved origin and confirm identity resolution degrades + without blocking the auction. + +This integration forwards RampID identity envelopes through the Prebid auction +path. LiveRamp ATS Direct audience segments, including `_lr_atsDirect` storage +and GAM or Prebid segment activation, require a separate integration and are +not passed by this implementation. + ## Identity Forwarding Trusted Server uses a **hybrid EID forwarding model** for Prebid-routed auctions: diff --git a/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md b/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md new file mode 100644 index 000000000..df781e518 --- /dev/null +++ b/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md @@ -0,0 +1,2142 @@ +# Server-Side Ad Template CLI Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build the unified `ts` CLI support for server-side ad-template static diagnostics and browser-backed verification described in `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md`. + +**Architecture:** Keep the CLI host-only and thin: Clap parsing stays in `run.rs` / command adapter modules, shared app-config loading moves to `app_config.rs`, pure ad-template logic lives under `ad_templates/`, and Chrome/Chromium collection lives under `audit/`. Runtime gate rules are extracted into a small pure helper in `trusted-server-core` so the CLI does not duplicate server behavior. + +**Tech Stack:** Rust 2024 workspace, host-target `trusted-server-cli`, `clap`, EdgeZero typed app-config loader, `serde`/`serde_json` for stable JSON, `chromiumoxide` for browser-backed audit collection, local HTML fixture tests, and existing `trusted-server-core::creative_opportunities` matching. + +--- + +## Current State + +- Branch: `feature/ts-cli-ad-templates`. +- Static ad-template commands already exist in `crates/trusted-server-cli/src/config_ad_templates.rs`. +- The current branch does not contain #800 audit files. Port useful #800 pieces into the current #799 code shape; do not resurrect stale `args.rs` or `config_command.rs`. +- The spec was updated after review and is the source of truth: + `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md`. +- Keep `.env` and operator-owned `trusted-server.toml` out of commits. + +## File Map + +### New files + +| File | Responsibility | +| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-cli/src/app_config.rs` | Shared effective app-config loader and shared `AppConfigArgs`. | +| `crates/trusted-server-cli/src/ad_templates/mod.rs` | Re-export focused ad-template CLI modules. | +| `crates/trusted-server-cli/src/ad_templates/expected.rs` | Path/URL normalization and expected-slot projection from runtime slot matching. | +| `crates/trusted-server-cli/src/ad_templates/compare.rs` | Pure DOM/GPT/APS evidence comparison, statuses, warnings, runtime gate output, strict failure decisions. | +| `crates/trusted-server-cli/src/ad_templates/output.rs` | Human and JSON rendering for static diagnostics and browser verification. | +| `crates/trusted-server-cli/src/audit/mod.rs` | Audit namespace entry point. | +| `crates/trusted-server-cli/src/audit/page.rs` | Generic page audit command ported from #800. | +| `crates/trusted-server-cli/src/audit/collector.rs` | Browser collector trait plus collected page/evidence structs. | +| `crates/trusted-server-cli/src/audit/browser.rs` | Chromiumoxide-backed browser collector, init scripts, optional scroll, page-level collection errors. | +| `crates/trusted-server-cli/src/audit/ad_templates.rs` | `ts audit ad-templates verify` orchestration. | +| `crates/trusted-server-cli/src/audit/ad_template_collector.js` | Read-only init script for GPT/APS/DOM evidence collection, included via `include_str!`. | + +### Modified files + +| File | Change summary | +| ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| `Cargo.toml` | Add workspace dependencies missing from this branch: `chromiumoxide`, `serde`, and `serde_json` if not present. | +| `crates/trusted-server-cli/Cargo.toml` | Add host-only CLI dependencies for browser audit and JSON output. | +| `crates/trusted-server-cli/src/lib.rs` | Register new `app_config`, `ad_templates`, and `audit` modules under `cfg(not(target_arch = "wasm32"))`. | +| `crates/trusted-server-cli/src/run.rs` | Add `Audit` command namespace, parser tests, and dispatch. | +| `crates/trusted-server-cli/src/config_ad_templates.rs` | Shrink to Clap adapter using shared loader/expected/output modules. | +| `crates/trusted-server-core/src/creative_opportunities.rs` | Add pure runtime gate helper types/functions shared by runtime and CLI. | +| `crates/trusted-server-core/src/publisher.rs` | Route existing server-side ad-stack gate through the shared helper without changing behavior. | + +## Implementation Rules + +- Use TDD for each task: write a failing test first, run it, implement the minimal code, re-run, then commit. +- Commit after each task using repo style: sentence case, imperative, no semantic prefix. +- Keep `trusted-server-cli` host-only. Do not introduce `tokio`, `chromiumoxide`, or filesystem/browser dependencies into core runtime or wasm adapter crates. +- Do not write real publisher domains or secrets in tests. Use `example.com`, `publisher.example`, and fictional IDs only. +- Prefer pure module tests over browser tests. Browser-backed fixture tests should use local HTML only and no GPT/APS network. + +## Task 0: Baseline And Branch Hygiene + +**Files:** + +- Verify: `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` +- Verify: `docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md` + +- [ ] **Step 1: Confirm branch and working tree** + + Run: + + ```bash + git status --short --branch + git log --oneline --decorate -5 + ``` + + Expected: on `feature/ts-cli-ad-templates`; no unrelated modified files besides the approved spec/plan docs. + +- [ ] **Step 2: Run docs format check before code work** + + Run: + + ```bash + cd docs && npm run format + ``` + + Expected: `All matched files use Prettier code style!` + +- [ ] **Step 3: Commit reviewed spec and plan** + + Run: + + ```bash + git add docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md + git commit -m "Add server-side ad-template CLI implementation plan" + ``` + + Expected: docs-only commit. If the spec commit already exists separately, commit only the plan. + +## Task 1: Share Runtime Ad-Stack Gate Logic + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Write failing core tests for the shared gate helper** + + Add tests near the existing `creative_opportunities` tests: + + ```rust + #[test] + fn ad_stack_gate_passes_for_eligible_navigation() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Yes); + assert!(result.blocking_gates().is_empty()); + } + + #[test] + fn ad_stack_gate_blocks_known_kill_switch() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: false, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::No); + assert!(result.blocking_gates().contains(&AdStackGateName::AuctionEnabled)); + } + + #[test] + fn ad_stack_gate_is_unknown_when_consent_is_unknown() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: None, + auction_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Unknown); + } + + // Locks the spec §5.2 mirror invariant: with Some(consent) supplied for every + // input combination, `expected == Yes` must equal the legacy all-AND boolean. + #[test] + fn ad_stack_gate_with_known_consent_matches_legacy_boolean() { + for bits in 0u8..64 { + let input = AdStackGateInput { + method_get: bits & 1 != 0, + navigation: bits & 2 != 0, + prefetch: bits & 4 != 0, + bot: bits & 8 != 0, + matched_slots: bits & 16 != 0, + consent_allows_auction: Some(bits & 32 != 0), + auction_enabled: bits & 1 == 0, + }; + // Legacy semantics: all positive gates true, both negative gates false. + let legacy = input.method_get + && input.navigation + && !input.prefetch + && !input.bot + && input.matched_slots + && input.consent_allows_auction == Some(true) + && input.auction_enabled; + let got = evaluate_ad_stack_gate(input).expected == RuntimeAdStackExpected::Yes; + assert_eq!(got, legacy, "gate mismatch for bits={bits}"); + } + } + ``` + +- [ ] **Step 2: Run the focused test and verify it fails** + + Run: + + ```bash + # NOTE: trusted-server-core links the `fastly` crate and CANNOT build for the host + # triple — run core tests on the DEFAULT target (wasm32-wasip1 + viceroy runner), + # i.e. no `--target`. Only the host-only `trusted-server-cli` uses `--target `. + cargo test -p trusted-server-core creative_opportunities::tests::ad_stack_gate + ``` + + Expected: compile failure because `AdStackGateInput` / `evaluate_ad_stack_gate` do not exist. + +- [ ] **Step 3: Implement pure gate types and helper** + + Add public, serde-free types to `creative_opportunities.rs`: + + ```rust + #[derive(Debug, Clone, Copy, Eq, PartialEq)] + pub enum RuntimeAdStackExpected { + Yes, + No, + Unknown, + } + + #[derive(Debug, Clone, Copy, Eq, PartialEq)] + pub enum AdStackGateName { + MethodGet, + Navigation, + NotPrefetch, + NotBot, + MatchedSlots, + ConsentAllowsAuction, + AuctionEnabled, + } + + #[derive(Debug, Clone, Copy)] + pub struct AdStackGateInput { + pub method_get: bool, + pub navigation: bool, + pub prefetch: bool, + pub bot: bool, + pub matched_slots: bool, + pub consent_allows_auction: Option, + pub auction_enabled: bool, + } + + #[derive(Debug, Clone, Eq, PartialEq)] + pub struct AdStackGateResult { + pub expected: RuntimeAdStackExpected, + blocking_gates: Vec, + } + + impl AdStackGateResult { + pub fn blocking_gates(&self) -> &[AdStackGateName] { + &self.blocking_gates + } + } + ``` + + Implement `evaluate_ad_stack_gate(input)` so any known blocking boolean gate returns `No`, all known pass plus `Some(true)` consent returns `Yes`, and all known pass plus `None` consent returns `Unknown`. + + Mind the gate polarity, mirroring `should_run_server_side_ad_stack`: `method_get`, + `navigation`, `matched_slots`, and `auction_enabled` block when **false**, while + `prefetch` and `bot` block when **true** (their gate names `NotPrefetch` / `NotBot` + pass when the input bool is false). `consent_allows_auction` is the only tri-state + input: `Some(false)` blocks (No), `Some(true)` passes, `None` yields Unknown only + when no other gate already blocks. + +- [ ] **Step 4: Route `publisher.rs` through the helper** + + Replace the body of `should_run_server_side_ad_stack` with a call to `evaluate_ad_stack_gate`, preserving the existing function signature for low-risk runtime compatibility: + + ```rust + crate::creative_opportunities::evaluate_ad_stack_gate( + crate::creative_opportunities::AdStackGateInput { + method_get: is_get, + navigation: is_navigation, + prefetch: is_prefetch, + bot: is_bot, + matched_slots: has_matched_slots, + consent_allows_auction: Some(consent_allows_auction), + auction_enabled, + }, + ) + .expected + == crate::creative_opportunities::RuntimeAdStackExpected::Yes + ``` + +- [ ] **Step 5: Run focused tests** + + Run: + + ```bash + # Core tests run on the default wasm target via viceroy (no --target). + cargo test -p trusted-server-core publisher::tests + cargo test -p trusted-server-core creative_opportunities + ``` + + Expected: all focused tests pass (including the existing `should_run_server_side_ad_stack` truth-table tests in `publisher::tests`). + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-core/src/creative_opportunities.rs crates/trusted-server-core/src/publisher.rs + git commit -m "Share server-side ad stack gate evaluation" + ``` + +## Task 2: Extract Shared CLI App Config Loader + +**Files:** + +- Create: `crates/trusted-server-cli/src/app_config.rs` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: `crates/trusted-server-cli/src/config_ad_templates.rs` + +- [ ] **Step 1: Write failing loader tests** + + Move the existing temp-project helpers from `config_ad_templates.rs` tests into `app_config.rs` tests and add: + + ```rust + #[test] + fn explicit_missing_app_config_does_not_fall_back() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let missing_path = temp.path().join("missing.toml"); + + let args = AppConfigArgs { + app_config: Some(missing_path.clone()), + manifest: manifest_path, + no_env: true, + }; + + let err = load_settings(&args).expect_err("should reject missing explicit config"); + assert!( + err.contains(missing_path.to_string_lossy().as_ref()), + "error should mention the explicit missing path" + ); + } + ``` + +- [ ] **Step 2: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli app_config --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because `app_config` module is not registered. + +- [ ] **Step 3: Implement `app_config.rs`** + + Move these items out of `config_ad_templates.rs`: + - `AppConfigArgs` + - `LoadedSettings` + - `load_settings` + - `resolve_app_config_path` + + Make the API explicit: + + ```rust + #[derive(Clone, Debug, Args)] + pub struct AppConfigArgs { + #[arg(long)] + pub app_config: Option, + #[arg(long, default_value = "edgezero.toml")] + pub manifest: PathBuf, + #[arg(long)] + pub no_env: bool, + } + + pub struct LoadedSettings { + pub app_config_path: PathBuf, + pub settings: Settings, + } + + pub fn load_settings(args: &AppConfigArgs) -> Result { + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + let app_config_path = + resolve_app_config_path(args.app_config.as_deref(), &args.manifest, &app_name); + + let mut opts = AppConfigLoadOptions::default(); + opts.env_overlay = !args.no_env; + let app_config = app_config::deserialize_app_config_with_options::( + &app_config_path, + &app_name, + &opts, + ) + .map_err(|err| format!("failed to load {}: {err}", app_config_path.display()))?; + + Ok(LoadedSettings { + app_config_path, + settings: app_config.into_settings(), + }) + } + + fn resolve_app_config_path( + explicit: Option<&Path>, + manifest_path: &Path, + app_name: &str, + ) -> PathBuf { + if let Some(path) = explicit { + return path.to_path_buf(); + } + let file_name = format!("{app_name}.toml"); + if let Some(parent) = manifest_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + parent.join(file_name) + } else { + PathBuf::from(file_name) + } + } + ``` + + Include the same top-level imports currently used by these helpers: + `std::path::{Path, PathBuf}`, `clap::Args`, + `edgezero_core::app_config::{self, AppConfigLoadOptions}`, + `edgezero_core::manifest::ManifestLoader`, + `trusted_server_core::config::TrustedServerAppConfig`, and + `trusted_server_core::settings::Settings`. + +- [ ] **Step 4: Register module and update imports** + + In `lib.rs`, add: + + ```rust + #[cfg(not(target_arch = "wasm32"))] + mod app_config; + ``` + + In `config_ad_templates.rs`, import: + + ```rust + use crate::app_config::{load_settings, AppConfigArgs}; + ``` + +- [ ] **Step 5: Run focused CLI tests** + + Run: + + ```bash + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli app_config --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: existing static command behavior remains unchanged. + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-cli/src/app_config.rs crates/trusted-server-cli/src/config_ad_templates.rs crates/trusted-server-cli/src/lib.rs + git commit -m "Extract shared CLI app config loader" + ``` + +## Task 3: Add Expected-Slot Model + +**Files:** + +- Create: `crates/trusted-server-cli/src/ad_templates/mod.rs` +- Create: `crates/trusted-server-cli/src/ad_templates/expected.rs` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: `crates/trusted-server-cli/src/config_ad_templates.rs` + +- [ ] **Step 1: Write failing expected-slot tests** + + Add a test-only dependency to `crates/trusted-server-cli/Cargo.toml` so tests can + deserialize core slot config instead of constructing `CreativeOpportunitySlot` with + its `pub(crate)` `compiled_patterns` cache: + + ```toml + [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] + toml = { workspace = true } + ``` + + In `expected.rs`, add tests for path normalization, full URL normalization, config-order preservation, resolved div ID, resolved GAM unit path, provider names, and matching page patterns: + + ```rust + fn creative_config_with_slots(patterns: &[&str]) -> CreativeOpportunitiesConfig { + let page_patterns = patterns + .iter() + .map(|pattern| format!("\"{pattern}\"")) + .collect::>() + .join(", "); + let toml = format!( + r#" + gam_network_id = "123" + auction_timeout_ms = 500 + price_granularity = "dense" + + [[slot]] + id = "atf" + gam_unit_path = "/123/news/atf" + div_id = "ad-atf-" + page_patterns = [{page_patterns}] + formats = [{{ width = 300, height = 250 }}] + floor_price = 0.50 + targeting = {{ zone = "atf" }} + + [slot.providers.prebid] + bidders = {{}} + "# + ); + let mut config = toml::from_str::(&toml) + .expect("should deserialize creative opportunities config"); + config.compile_slots(); + config + } + + #[test] + fn expected_slots_use_runtime_matcher_and_config_order() { + let config = creative_config_with_slots(["/news/*", "/"].as_slice()); + let expected = expected_slots_for_path("/news/story", &config) + .expect("should build expected slots"); + + assert_eq!(expected.path, "/news/story"); + assert_eq!(expected.slots.iter().map(|slot| slot.id.as_str()).collect::>(), ["atf"]); + assert_eq!(expected.slots[0].div_id, "ad-atf-"); + assert_eq!(expected.slots[0].gam_unit_path, "/123/news/atf"); + assert_eq!(expected.slots[0].providers, ["prebid"]); + } + + #[test] + fn normalize_path_or_url_strips_query_and_fragment() { + assert_eq!(normalize_path_or_url("https://www.example.com/news/story?x=1#top").expect("should normalize"), "/news/story"); + assert_eq!(normalize_path_or_url("news/story?x=1").expect("should normalize"), "/news/story"); + } + ``` + +- [ ] **Step 2: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::expected --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because module/types do not exist. + +- [ ] **Step 3: Implement expected-slot structs** + + Define pure structs that own strings and are stable for output: + + ```rust + #[derive(Debug, Clone, PartialEq)] + pub struct ExpectedSlots { + pub path: String, + pub slots: Vec, + } + + #[derive(Debug, Clone, PartialEq)] + pub struct ExpectedSlot { + pub id: String, + pub div_id: String, + pub gam_unit_path: String, + pub formats: Vec, + pub providers: Vec, + pub page_patterns: Vec, + } + + #[derive(Debug, Clone, PartialEq)] + pub struct ExpectedFormat { + pub width: u32, + pub height: u32, + // Mirrors `MediaType` rendered as a stable string (`"banner"`, `"video"`, `"native"`). + pub media_type: String, + } + ``` + + `div_id` and `gam_unit_path` are resolved (non-optional) strings. The core + `CreativeOpportunitySlot` stores `div_id` / `gam_unit_path` as `Option` + and the GAM unit path is composed with the configured GAM network ID; mirror the + existing `format_slot` resolution in `config_ad_templates.rs` so the CLI does not + invent a second resolution rule. Use + `trusted_server_core::creative_opportunities::match_slots`. Do not compile globs in CLI. + +- [ ] **Step 4: Register `ad_templates` and update static commands** + + In `lib.rs`, add: + + ```rust + #[cfg(not(target_arch = "wasm32"))] + mod ad_templates; + ``` + + Rewire `config_ad_templates.rs` onto the shared module, and remove the now-duplicated + local code so there is no name collision or dead `normalize_path_or_url`: + - delete the private `fn normalize_path_or_url` (currently `config_ad_templates.rs:448`) + and add `use crate::ad_templates::expected::{expected_slots_for_path, normalize_path_or_url};`; + - the existing `config_ad_templates::tests::normalizes_path_or_url_like_runtime_request_path` + test (currently `:661`) calls the local fn via `super::*` — either delete it (Task 3 + Step 1 already adds normalization tests in `expected.rs`) or repoint it at + `crate::ad_templates::expected::normalize_path_or_url`. Pick one so the test crate + still compiles at this commit. + +- [ ] **Step 5: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::expected --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: all pass. + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/ad_templates/mod.rs crates/trusted-server-cli/src/ad_templates/expected.rs crates/trusted-server-cli/src/config_ad_templates.rs crates/trusted-server-cli/src/lib.rs + git commit -m "Add shared ad-template expected slot model" + ``` + +## Task 4: Add Stable Output And JSON Types + +**Files:** + +- Create: `crates/trusted-server-cli/src/ad_templates/output.rs` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `crates/trusted-server-cli/src/ad_templates/mod.rs` + +- [ ] **Step 1: Add CLI JSON dependencies** + + The CLI crate has **no plain `[dependencies]` table** — every runtime dep lives + under `[target.'cfg(not(target_arch = "wasm32"))'.dependencies]` (the workspace + default build target is `wasm32-wasip1` per `.cargo/config.toml`). Add the new deps + to that existing table; do **not** create a `[dependencies]` table, or they compile + for wasm and leak host-only crates into the wasm build: + + ```toml + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] + # ... existing clap/url/etc ... + serde = { workspace = true } + serde_json = { workspace = true } + ``` + + Add workspace dependency `chromiumoxide = "0.9.1"` in `Cargo.toml` only in Task 7 when browser code is introduced. + +- [ ] **Step 2: Write failing JSON output tests** + + In `output.rs`, add tests that construct an in-memory verification result and assert exact JSON values: + + ```rust + #[test] + fn verification_json_contains_gate_state_and_extra_evidence() { + let result = VerificationReport::example_confirmed_with_extra_evidence(); + let value = serde_json::to_value(&result).expect("should serialize"); + + assert_eq!(value["ok"], true); + assert_eq!(value["pages"][0]["requested_path"], "/news/story"); + assert_eq!(value["pages"][0]["runtime_ad_stack_expected"], "unknown"); + assert_eq!(value["pages"][0]["extra_evidence"][0]["kind"], "gpt"); + assert_eq!(value["pages"][0]["warnings"][0]["code"], "redirected"); + } + + // Pins the spec §8 navigation_failed shape: error present, runtime/gates/ + // matched_slot_count keys ABSENT (skipped), final_url/path null. + #[test] + fn page_error_json_matches_navigation_failed_shape() { + let result = VerificationReport::example_navigation_failed(); + let value = serde_json::to_value(&result).expect("should serialize"); + let page = &value["pages"][0]; + + assert_eq!(page["error"]["code"], "navigation_failed"); + assert!(page["final_url"].is_null(), "final_url should be null"); + assert!(page["path"].is_null(), "path should be null"); + assert!(page.get("runtime_ad_stack_expected").is_none(), "runtime field absent on error page"); + assert!(page.get("gates").is_none(), "gates absent on error page"); + assert!(page.get("matched_slot_count").is_none(), "matched_slot_count absent on error page"); + assert_eq!(value["ok"], false); + } + ``` + +- [ ] **Step 3: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::output --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because output model does not exist. + +- [ ] **Step 4: Implement serializable output types** + + Model the **entire** `--json` wire tree from spec §8 (this is the single source of + truth for field names and ordering). Use owned `String` / `Vec` fields and + `#[serde(rename_all = "snake_case")]` so output is stable. Leaf enums: + + ```rust + #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum SlotStatus { Confirmed, Partial, Missing } + + #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum RuntimeAdStackExpectedJson { Yes, No, Unknown } + + impl From for RuntimeAdStackExpectedJson { + fn from(value: trusted_server_core::creative_opportunities::RuntimeAdStackExpected) -> Self { + use trusted_server_core::creative_opportunities::RuntimeAdStackExpected as Core; + match value { + Core::Yes => Self::Yes, + Core::No => Self::No, + Core::Unknown => Self::Unknown, + } + } + } + + #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum GateState { Pass, Fail, Unknown } + ``` + + Top-level tree (field names and nesting must match §8 exactly): + + ```rust + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct VerificationReport { + pub ok: bool, + pub strict: bool, + pub pages: Vec, + pub warnings: Vec, + } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct PageJson { + pub url: String, + pub final_url: Option, + pub requested_path: String, + pub path: Option, + // Field ORDER matters: serde serializes in declaration order. Spec §8 places + // `error` immediately after `path` on the navigation_failed shape, so it must + // be declared here (not last). On normal pages `error` is None and skipped, so + // the runtime/gates/slots run in §8 order; on error pages the runtime/gates/ + // matched_slot_count are None and skipped, leaving url..path, error, slots, + // extra_evidence, warnings — exactly the §8 error shape. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_ad_stack_expected: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub gates: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_slot_count: Option, + pub slots: Vec, + pub extra_evidence: Vec, + pub warnings: Vec, + } + + // One field per gate name from spec §5.2 / §8, each a GateState. + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct Gates { + pub method_get: GateState, + pub navigation: GateState, + pub not_prefetch: GateState, + pub not_bot: GateState, + pub matched_slots: GateState, + pub auction_enabled: GateState, + pub consent_allows_auction: GateState, + } + + // Serialize for output JSON; Deserialize because the browser collector payload + // (Task 8) carries warning objects decoded into `BrowserAdEvidence.warnings`. + #[derive(Debug, Clone, Eq, PartialEq, Serialize, serde::Deserialize)] + pub struct Warning { + pub code: String, + pub message: String, + } + ``` + + Define the remaining nested JSON structs **explicitly** — do not serialize the + compare-module types directly. The compare types (`SlotResult`, `SlotEvidence`, + `GptSlotEvidence`, `ExtraEvidence`) carry a `phase` field and are not `Serialize`; + spec §8's `evidence.gpt` has **no** `phase` key and `configured` excludes `id` + and `page_patterns`. Mismatched reuse would emit extra keys. Wire structs: + + ```rust + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct SlotJson { + pub id: String, + pub status: SlotStatus, + pub phase: EvidencePhaseJson, + pub configured: ConfiguredJson, + pub evidence: SlotEvidenceJson, + pub warnings: Vec, + } + + // §8 `configured`: div_id, gam_unit_path, formats, providers — NO id/page_patterns. + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct ConfiguredJson { + pub div_id: String, + pub gam_unit_path: String, + pub formats: Vec, + pub providers: Vec, + } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct FormatJson { + pub width: u32, + pub height: u32, + pub media_type: String, + } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct SlotEvidenceJson { + pub dom_id: Option, + pub gpt: Option, + } + + // §8 `evidence.gpt`: gam_unit_path, div_id, sizes — NO phase. + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct GptEvidenceJson { + pub gam_unit_path: String, + pub div_id: String, + pub sizes: Vec<[u32; 2]>, + } + + #[derive(Debug, Clone, Copy, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum EvidencePhaseJson { InitialLoad, Scroll } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct ExtraEvidenceJson { + pub kind: String, + pub phase: EvidencePhaseJson, + pub dom_id: Option, + pub gam_unit_path: Option, + pub sizes: Vec<[u32; 2]>, + pub reason: String, + } + ``` + + Note `sizes` serialize as `[[300,250]]` (arrays of two ints), matching §8 — use + `[u32; 2]` here even though the compare module uses `(u32, u32)` tuples; the + Task 9 assembly maps tuple → `[w, h]`. The conversion from the compare + `SlotResult`/`SlotEvidence`/`ExtraEvidence` to these JSON types (dropping `phase` + from `gpt`, dropping `id`/`page_patterns` from `configured`) lives in Task 9 Step 7. + `Warning` is the single warning type for the whole CLI; defined here and re-exported + from `ad_templates::mod` so `compare.rs` reuses it (plain data, not JSON logic). + Keep `example_confirmed_with_extra_evidence()` and similar fixtures behind + `#[cfg(test)]`. + +- [ ] **Step 5: Add verification human-render helpers** + + Add only the **browser-verification** page summary writers here (used by + `audit::ad_templates` in Task 9), writing to `&mut dyn Write`; no `println!` / + `eprintln!`. Do **not** add static match/check/explain writers in this task — + those are the existing `write_match_result`/`format_slot`/etc. functions that + Task 6 Step 3 **moves** out of `config_ad_templates.rs`. Keeping the static + relocation solely in Task 6 avoids two competing copies of the same helpers in + `output.rs`. The verification writers added here may be unused until Task 9 (a + warn-level `dead_code` lint that does not fail `cargo test`); add + `#[allow(dead_code)]` if clippy is run between Task 4 and Task 9. + +- [ ] **Step 6: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::output --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: all pass. + +- [ ] **Step 7: Commit** + + ```bash + git add Cargo.toml crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/ad_templates/mod.rs crates/trusted-server-cli/src/ad_templates/output.rs + git commit -m "Add ad-template CLI output models" + ``` + +## Task 5: Add Pure Evidence Comparison + +**Files:** + +- Create: `crates/trusted-server-cli/src/ad_templates/compare.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/mod.rs` + +- [ ] **Step 1: Write failing comparison tests** + + Cover every spec status and warning case without launching Chrome. Define small + test constructors so tests do not couple to the full `BrowserAdEvidence` field + list (`page_bids` and `warnings` default empty, evidence items default to + `EvidencePhase::InitialLoad`): + + ```rust + fn dom(id: &str) -> DomEvidence { + DomEvidence { dom_id: id.to_string(), phase: EvidencePhase::InitialLoad } + } + + fn gpt_slot(gam_unit_path: &str, div_id: &str, sizes: &[(u32, u32)]) -> GptSlotEvidence { + GptSlotEvidence { + gam_unit_path: gam_unit_path.to_string(), + div_id: div_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn aps(slot_id: &str, sizes: &[(u32, u32)]) -> ApsFetchBidsEvidence { + ApsFetchBidsEvidence { slot_id: slot_id.to_string(), sizes: sizes.to_vec(), phase: EvidencePhase::InitialLoad } + } + + // Non-banner format helper for the unsupported-format test. + fn expected_slot_video(id: &str, div_id: &str, gam_unit_path: &str) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: gam_unit_path.to_string(), + formats: vec![ExpectedFormat { width: 0, height: 0, media_type: "video".to_string() }], + providers: Vec::new(), + page_patterns: Vec::new(), + } + } + + fn evidence(doms: Vec, gpts: Vec, aps: Vec) -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: doms, + gpt_slots: gpts, + aps_calls: aps, + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + #[test] + fn gpt_path_div_and_size_overlap_confirms_slot() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "GPT path+div+size overlap should confirm"); + assert!(result.slots[0].warnings.is_empty(), "confirmed slot should carry no warnings"); + } + + #[test] + fn dom_only_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(vec![dom("ad-atf-0")], Vec::new(), Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "DOM-only evidence should be partial"); + assert!( + result.slots[0].warnings.iter().any(|w| w.code == "dom_without_gpt"), + "DOM-only slot should warn dom_without_gpt" + ); + } + + #[test] + fn no_dom_or_gpt_is_missing() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Missing, "no DOM/GPT evidence should be missing"); + } + + #[test] + fn prefix_dom_resolution_ignores_container_suffix() { + let expected = expected_slot("header", "ad-header-0-", "/123/homepage/header", &[(728, 90)], &[]); + // First candidate ends with `-container` and must be skipped; the framework-suffixed ID resolves. + let evidence = evidence( + vec![dom("ad-header-0--container"), dom("ad-header-0-_R_abc123")], + Vec::new(), + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].evidence.dom_id.as_deref(), Some("ad-header-0-_R_abc123"), "prefix match should skip -container"); + assert_eq!(result.slots[0].status, SlotStatus::Partial, "DOM-only prefix match is partial without GPT"); + } + + #[test] + fn unmatched_gpt_slot_becomes_extra_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![ + gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)]), + gpt_slot("/123/publisher/right-rail", "ad-right-rail-0", &[(300, 250)]), + ], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "matched slot still confirms"); + assert_eq!(result.extra_evidence.len(), 1, "unmatched GPT slot becomes extra evidence"); + assert_eq!(result.extra_evidence[0].kind, "gpt"); + assert!(!result.strict_failed(), "extra evidence alone must not fail strict"); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::auction_disabled()); + + assert_eq!(result.runtime_ad_stack_expected, RuntimeAdStackExpected::No, "auction disabled should set No"); + assert_eq!(result.slots[0].status, SlotStatus::Missing, "static status is still reported"); + assert!(!result.strict_failed(), "missing slot must not fail strict when ad stack expected is No"); + } + + // §5.4: GPT path+div match but no numeric size overlap -> partial + warning. + #[test] + fn gpt_incompatible_sizes_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(728, 90)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "no size overlap should be partial"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "incompatible_sizes")); + } + + // §5.4/§5.6: matched slot with only non-banner formats -> partial + unsupported_format. + #[test] + fn non_banner_only_slot_is_partial() { + let expected = expected_slot_video("video", "ad-video-", "/123/news/video"); + let evidence = evidence( + vec![dom("ad-video-0")], + vec![gpt_slot("/123/news/video", "ad-video-0", &[(640, 480)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "non-banner-only should be partial"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "unsupported_format")); + } + + // §5.4: GPT element ID may be `${resolved_dom_id}-container` and still confirm. + #[test] + fn gpt_container_element_id_confirms() { + let expected = expected_slot("atf", "ad-atf-0", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0"), dom("ad-atf-0-container")], + vec![gpt_slot("/123/news/atf", "ad-atf-0-container", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "container element id is a valid GPT div match"); + } + + // §5.4: out-of-page GPT slot is partial (so it fails strict) plus a warning. + #[test] + fn out_of_page_gpt_slot_warns_and_is_partial() { + let expected = expected_slot("interstitial", "ad-oop-", "/123/news/oop", &[(300, 250)], &[]); + // gpt_slot with empty sizes models an out-of-page slot (no numeric sizes). + let evidence = evidence(vec![dom("ad-oop-0")], vec![gpt_slot("/123/news/oop", "ad-oop-0", &[])], Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "a sizeless slot against banner formats is partial"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "out_of_page_slot")); + } + + // §5.5: matching APS fetchBids -> no provider warning. + #[test] + fn aps_match_adds_no_warning() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + vec![aps("atf", &[(300, 250)])], + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!(!result.slots[0].warnings.iter().any(|w| w.code.starts_with("aps_")), "matching APS should not warn"); + } + + // §5.5: configured aps provider but no APS evidence -> provider warning, still confirmed, strict not failed. + #[test] + fn aps_missing_warns_but_keeps_confirmed() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "missing APS does not flip status"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "aps_evidence_missing")); + assert!(!result.strict_failed(), "provider warning alone must not fail strict"); + } + ``` + + Add a `#[cfg(test)]` constructor in `compare.rs` tests that builds a real + `ExpectedSlot` (the Task 3 type) so comparison tests stay readable: + + ```rust + fn expected_slot(id: &str, div_id: &str, gam_unit_path: &str, sizes: &[(u32, u32)], providers: &[&str]) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: gam_unit_path.to_string(), + formats: sizes.iter().map(|&(width, height)| ExpectedFormat { width, height, media_type: "banner".to_string() }).collect(), + providers: providers.iter().map(|p| p.to_string()).collect(), + page_patterns: Vec::new(), + } + } + ``` + +- [ ] **Step 2: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::compare --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because comparison module does not exist. + +- [ ] **Step 3: Implement browser evidence structs** + + Define the minimum collector-independent input shape plus the comparison result + shape the tests assert against: + + All browser-evidence input structs derive `Debug, Clone` and `serde::Deserialize` + (Task 8 decodes them from the collector's `window.__tsAdTemplateEvidence` JSON); + `EvidencePhase` deserializes from `"initial_load"` / `"scroll"`. The comparison- + result structs derive `Debug` (so the Step 1 `assert_eq!`/`matches!` assertions + compile) and `Clone`. Sizes are `(u32, u32)` tuples internally; deserialize them + from JSON `[w, h]` arrays. + + ```rust + #[derive(Debug, Clone, Copy, Eq, PartialEq, serde::Deserialize)] + #[serde(rename_all = "snake_case")] + pub enum EvidencePhase { + InitialLoad, + Scroll, + } + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct DomEvidence { + pub dom_id: String, + pub phase: EvidencePhase, + } + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct GptSlotEvidence { + pub gam_unit_path: String, + pub div_id: String, + pub sizes: Vec<(u32, u32)>, + pub phase: EvidencePhase, + } + + // APS `apstag.fetchBids` evidence (spec §5.5): configured slot ID + observed sizes. + #[derive(Debug, Clone, serde::Deserialize)] + pub struct ApsFetchBidsEvidence { + pub slot_id: String, + pub sizes: Vec<(u32, u32)>, + pub phase: EvidencePhase, + } + + // DEFERRED in this implementation: `/__ts/page-bids` SPA observation (spec §5.2 + // "when available"). The struct/field are forward scaffolding so the collector and + // JSON can grow it later; Task 8 does NOT populate it and Task 4 JSON does NOT + // surface it in Phase 1. Tracked as a deferred item in Risks. Keep the field so + // `BrowserAdEvidence` deserialization stays forward-compatible (default empty). + #[derive(Debug, Clone, serde::Deserialize)] + pub struct PageBidsEvidence { + pub slot_id: String, + pub phase: EvidencePhase, + } + + // `Warning` is the shared CLI warning type defined in Task 4 (`output.rs`) and + // re-exported from `ad_templates::mod`. It is plain data reused here (not JSON + // logic). Because the collector payload carries warnings, give `Warning` BOTH + // `Serialize` (Task 4 output) and `Deserialize` (Task 8 decode) derives. + use crate::ad_templates::output::Warning; + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct BrowserAdEvidence { + pub dom_ids: Vec, + pub gpt_slots: Vec, + pub aps_calls: Vec, + #[serde(default)] + pub page_bids: Vec, + #[serde(default)] + pub warnings: Vec, + } + + // Comparison output. Uses the core `RuntimeAdStackExpected` enum from Task 1 so + // pure comparison logic does not depend on the output/JSON module. Task 4's + // `RuntimeAdStackExpectedJson` is produced only at serialization time. + #[derive(Debug, Clone)] + pub struct PageVerificationResult { + pub runtime_ad_stack_expected: trusted_server_core::creative_opportunities::RuntimeAdStackExpected, + pub slots: Vec, + pub extra_evidence: Vec, + } + + #[derive(Debug, Clone)] + pub struct SlotResult { + pub id: String, + pub status: SlotStatus, + pub phase: EvidencePhase, + pub evidence: SlotEvidence, + pub warnings: Vec, + } + + #[derive(Debug, Clone)] + pub struct SlotEvidence { + pub dom_id: Option, + pub gpt: Option, + } + + #[derive(Debug, Clone)] + pub struct ExtraEvidence { + pub kind: String, + pub phase: EvidencePhase, + pub dom_id: Option, + pub gam_unit_path: Option, + pub sizes: Vec<(u32, u32)>, + pub reason: String, + } + ``` + + `RuntimeGateSummary` is the third argument to `compare_page_evidence`; it wraps + the core gate result. Provide `RuntimeGateSummary::unknown_allowed()` (expected + `Unknown`) and `RuntimeGateSummary::auction_disabled()` (expected `No`) test + constructors so comparison tests do not rebuild gate inputs by hand. + +- [ ] **Step 4: Implement DOM/GPT/APS rules** + + Status rules: + - DOM exact ID first, then first prefix match, **excluding `-container`** wrappers + (slot-root resolution, spec §5.3). + - GPT confirms when: GAM unit path matches, the GPT slot element ID equals the + resolved DOM ID **or** an existing `${resolved_dom_id}-container` element + (spec §5.4 — note this is the GPT element-ID match, distinct from the §5.3 DOM + root resolution that skips `-container`), and at least one numeric banner size + overlaps. + - GPT path/div match with no numeric size overlap → `partial` (warn `incompatible_sizes`). + - Matched slot whose configured formats are **all non-banner** (video/native) → + `partial` (warn `unsupported_format`); banner is the only Phase-1 confirmable type. + - DOM-only (no GPT) → `partial` (warn `dom_without_gpt`). + - No DOM and no GPT → `missing`. + + Size-compatibility warnings (spec §5.4 — all are warnings, none flip a confirmed + slot to fail): emit a `Warning` for each of: + - `fluid_size_ignored` — non-numeric observed sizes like `"fluid"` ignored for matching; + - `extra_observed_size` — observed GPT sizes not in the configured set; + - `configured_size_not_observed` — configured sizes never observed (when ≥1 was); + - `out_of_page_slot` — out-of-page GPT slot with no sizes observed; the slot is + reported `partial`, which fails `--strict`. + + Provider + extra evidence: + - APS: configured `providers.aps.slot_id` with matching `fetchBids` → no warning; + missing/ambiguous APS evidence → provider warning only (`aps_evidence_missing` / + `aps_evidence_ambiguous`), never flips status or fails `--strict` in Phase 1. + - Unmatched live DOM/GPT/APS evidence → structured `extra_evidence` (never fails strict). + + Define each warning `code` as a stable string constant so output and tests share them. + +- [ ] **Step 5: Implement strict decision method** + + Add an inherent method on the result so tests can call `result.strict_failed()`: + + ```rust + impl PageVerificationResult { + pub fn strict_failed(&self) -> bool { + use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + if self.runtime_ad_stack_expected == RuntimeAdStackExpected::No { + return false; + } + self.slots + .iter() + .any(|slot| matches!(slot.status, SlotStatus::Missing | SlotStatus::Partial)) + } + } + ``` + + - false when `runtime_ad_stack_expected == No`; + - true for any `missing` or `partial` slot when expected is `Yes` or `Unknown`; + - false for provider warnings and extra evidence alone (they are not slot statuses). + +- [ ] **Step 6: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::compare --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: all comparison tests pass. + +- [ ] **Step 7: Commit** + + ```bash + git add crates/trusted-server-cli/src/ad_templates/mod.rs crates/trusted-server-cli/src/ad_templates/compare.rs + git commit -m "Add pure ad-template evidence comparison" + ``` + +## Task 6: Refactor Static Commands Onto Shared Modules + +**Files:** + +- Modify: `crates/trusted-server-cli/src/config_ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/output.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` + +- [ ] **Step 1: Add characterization tests before refactor** + + These guard behavior across the Step 3 move, so each must assert **exact output + substrings** (capture the command's `Vec`/`String` output and + `assert!(out.contains("..."))`), not just run without panicking — a bare + smoke test cannot catch a wording regression. Mirror the existing assertion style + at `config_ad_templates.rs:570-657`. Pin, with concrete expected strings: + - `lint` not configured → e.g. `"creative_opportunities: not configured"`; + - `lint` with slots + auction disabled → slot count line + `"auction: disabled"`; + - `match --details` → slot div ID, GAM unit path, formats, providers lines; + - `check --expect-no-slots` → success message; + - `check` failure with missing and unexpected slots → the exact failure lines; + - `explain` → each gate line (including `"auction providers configured"`) and the + EdgeZero legacy-fallback warning text. + + Run the existing tests first and copy the real emitted strings so the + characterization assertions match current behavior exactly before refactoring. + +- [ ] **Step 2: Run tests before refactor** + + Run: + + ```bash + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: characterization tests pass against the current implementation. + +- [ ] **Step 3: Move formatting into `ad_templates::output`** + + Move these helpers out of `config_ad_templates.rs`: + - `write_match_result` + - `write_gate` + - `format_slot` + - `format_format` + - `format_providers` + - `join_set` + - `plural` + + Keep command functions small: parse args, load config, call expected/gate logic, render. + +- [ ] **Step 4: Reuse shared gate helper in `explain`** + + Build `AdStackGateInput` from explain flags and config: + + ```rust + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get, + navigation: !args.non_navigation, + prefetch: args.prefetch, + bot: args.bot, + matched_slots: !expected.slots.is_empty(), + consent_allows_auction: Some(!args.consent_denied), + auction_enabled: loaded.settings.auction.enabled, + }); + ``` + + Render the seven shared gate names from `gate` rather than hand-rolled boolean chains. + + **Preserve the explain-only provider gate.** The current `run_explain` + (`config_ad_templates.rs:270`) renders an eighth gate, + `"auction providers configured"` (`!loaded.settings.auction.providers.is_empty()`, + line 299), and ANDs it into its local `runs_ad_stack` decision (line 302). The + shared `evaluate_ad_stack_gate` helper and runtime `should_run_server_side_ad_stack` + intentionally have no provider-configured gate. Do not fold this into + `AdStackGateInput`. Keep `"auction providers configured"` as an explain-only + supplementary `write_gate(...)` line rendered alongside the shared result, and + keep it in `explain`'s own `runs_ad_stack` decision: + + ```rust + let providers_configured = !loaded.settings.auction.providers.is_empty(); + render_shared_gates(out, &gate)?; + write_gate(out, "auction providers configured", providers_configured)?; + let runs_ad_stack = + gate.expected == RuntimeAdStackExpected::Yes && providers_configured; + ``` + + This keeps `explain` output and behavior identical to the current implementation + (verified by the Step 1 characterization test) while still sharing the seven core + runtime gates with `publisher.rs`. + +- [ ] **Step 5: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: no output regressions except intentional wording updates covered by tests. + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-cli/src/config_ad_templates.rs crates/trusted-server-cli/src/ad_templates/output.rs crates/trusted-server-cli/src/run.rs + git commit -m "Refactor static ad-template commands" + ``` + +## Task 7: Port Generic Audit Namespace And Browser Collector + +**Files:** + +- Modify: `Cargo.toml` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Create: `crates/trusted-server-cli/src/audit/mod.rs` +- Create: `crates/trusted-server-cli/src/audit/page.rs` +- Create: `crates/trusted-server-cli/src/audit/collector.rs` +- Create: `crates/trusted-server-cli/src/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` + +- [ ] **Step 1: Add browser dependencies** + + Add `chromiumoxide` to the root `[workspace.dependencies]` table (inert until a + crate references it via `{ workspace = true }`): + + ```toml + [workspace.dependencies] + # ... existing entries ... + chromiumoxide = "0.9.1" + ``` + + Add the host deps to the CLI crate under its existing + `[target.'cfg(not(target_arch = "wasm32"))'.dependencies]` table — NOT a plain + `[dependencies]` table (workspace default target is wasm32; an unconditional dep + compiles for wasm and breaks the build / leaks host-only crates): + + ```toml + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] + # ... existing clap/url/serde/etc ... + chromiumoxide = { workspace = true } + futures = { workspace = true } + tempfile = { workspace = true } + tokio = { workspace = true } + which = { workspace = true } + ``` + + Verify the root workspace already provides `futures`, `tempfile`, `tokio`, `which` + (it does on this branch); only `chromiumoxide` is a new workspace entry. + +- [ ] **Step 2: Write failing audit parser tests** + + In `run.rs` tests: + + ```rust + #[test] + fn audit_legacy_url_parses_as_page_alias() { + let args = parse(&["ts", "audit", "https://www.example.com/"]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_page_subcommand_parses() { + let args = parse(&["ts", "audit", "page", "https://www.example.com/"]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_ad_templates_verify_parses() { + let args = parse(&["ts", "audit", "ad-templates", "verify", "https://www.example.com/"]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_ad_templates_is_not_legacy_url() { + assert!(Args::try_parse_from(["ts", "audit", "ad-templates"]).is_err()); + } + ``` + +- [ ] **Step 3: Run parser tests and verify failure** + + Run: + + ```bash + cargo test -p trusted-server-cli audit_ --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because `Audit` command does not exist. + +- [ ] **Step 4: Implement audit Clap namespace in current `run.rs` shape** + + Do not add stale #800 `args.rs`. Add a `Command::Audit(AuditArgs)` variant to the + existing `Command` enum, plus the full audit arg surface. The parser tests in + Step 2 exercise `ad-templates verify`, so the **entire** command surface (including + the verify args) must be defined here for those tests to compile. Task 9 implements + the verify _behavior_ only — it does not redefine these arg types. + + **Visibility:** `audit::run_audit` lives in `audit/mod.rs` and must name these + types in its signature and match their variants, so every audit arg type and its + fields are `pub(crate)` (not private). `PageAuditArgs` (from `audit/page.rs`, Task 7 + Step 5) and `AuditAdTemplatesVerifyArgs` are likewise `pub(crate)`/`pub`. `run.rs` + imports `AuditArgs` for the `Command::Audit(AuditArgs)` variant; everything else is + read by `audit/mod.rs`. (This mirrors `config_ad_templates::AdTemplatesCommand`, + which is `pub` and consumed by `run.rs`.) + + ```rust + // value parser shared by legacy_url and verify urls; rejects non-HTTP(S) schemes. + pub(crate) fn parse_http_url(raw: &str) -> Result { + let url = url::Url::parse(raw).map_err(|error| format!("invalid URL `{raw}`: {error}"))?; + match url.scheme() { + "http" | "https" => Ok(url), + other => Err(format!("unsupported URL scheme `{other}` (expected http or https)")), + } + } + + #[derive(Debug, clap::Args)] + pub(crate) struct AuditArgs { + #[command(subcommand)] + pub(crate) command: Option, + #[arg(value_parser = parse_http_url, hide = true)] + pub(crate) legacy_url: Option, + } + + #[derive(Debug, Subcommand)] + pub(crate) enum AuditSubcommand { + Page(PageAuditArgs), + #[command(name = "ad-templates", subcommand)] + AdTemplates(AuditAdTemplatesCommand), + } + + #[derive(Debug, Subcommand)] + pub(crate) enum AuditAdTemplatesCommand { + Verify(AuditAdTemplatesVerifyArgs), + } + + // Defined here (not Task 9) so parser tests compile. Task 9 fills in the handler. + #[derive(Debug, clap::Args)] + pub(crate) struct AuditAdTemplatesVerifyArgs { + #[command(flatten)] + pub config: AppConfigArgs, + #[arg(required = true, value_parser = parse_http_url)] + pub urls: Vec, + #[arg(long)] + pub strict: bool, + #[arg(long)] + pub json: bool, + #[arg(long)] + pub scroll: bool, + } + ``` + + Dispatch `Command::Audit(args)` to a single `audit::run_audit(args: AuditArgs)` + entry point (in `audit/mod.rs`) that normalizes the namespace: `legacy_url` (if + present) and `Page` both route to the generic page audit; `AdTemplates(Verify(..))` + routes to the verifier (a stub returning `Ok(())` until Task 9). If Clap cannot make + the optional-subcommand-plus-hidden-positional contract unambiguous, implement a + small `AuditArgs::normalize()` that rejects `legacy_url` values that are not HTTP(S). + Decide arg-type home consistently: keep them in `run.rs` as `pub(crate)` (as shown) + and import into `audit/mod.rs`, or move them next to `run_audit` in `audit/mod.rs` + and import `AuditArgs` into `run.rs` — either works, but do not split them. + +- [ ] **Step 5: Port minimal generic page audit** + + Port useful #800 concepts into `audit/page.rs`, but keep output read-only by default for now: + - parse/validate URL; + - call `AuditCollector::collect_page`; + - print summary with final URL, title, script/resource counts, warnings; + - no draft config generation in this PR unless #800 rebase keeps it explicitly. + +- [ ] **Step 6: Implement collector trait and browser collector base** + + `audit/collector.rs` — define the trait plus its concrete request/response types so + Task 9's `FakeCollector` and the verify orchestration have a contract to assert on: + + ```rust + pub trait AuditCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result; + } + + pub struct BrowserCollectRequest { + pub url: url::Url, + // Pre-navigation init scripts (evaluate-on-new-document). Empty for plain page audit; + // Task 8 passes the ad-template collector script here. + pub init_scripts: Vec, + pub scroll: bool, + } + + pub struct CollectedPage { + pub final_url: url::Url, + pub title: String, + // Generic page-audit signals (counts only; no page HTML/cookies/storage). + pub script_count: usize, + pub resource_count: usize, + pub warnings: Vec, + // Present only when an ad-template init script was injected (Task 8); None for + // plain `ts audit page`. This is how `BrowserAdEvidence` rides on a CollectedPage. + pub ad_evidence: Option, + } + ``` + + `BrowserCollectRequest` carries `init_scripts` + `scroll` so ad-template verification + enables evidence hooks without changing the trait later. + + `audit/browser.rs` should port #800's: + - `which` browser lookup; + - isolated `TempDir` profile; + - current-thread Tokio runtime; + - `Browser::launch`; + - `page.goto`; + - `wait_for_navigation_response`; + - settle loop. + +- [ ] **Step 7: Run compile-focused CLI tests** + + Run: + + ```bash + cargo test -p trusted-server-cli audit_ --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: parser and non-browser unit tests pass. No test should require installed Chrome yet. + +- [ ] **Step 8: Commit** + + ```bash + git add Cargo.toml crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/audit crates/trusted-server-cli/src/lib.rs crates/trusted-server-cli/src/run.rs + git commit -m "Add audit namespace and browser collector base" + ``` + +## Task 8: Add Browser Ad-Template Evidence Collector + +**Files:** + +- Create: `crates/trusted-server-cli/src/audit/ad_template_collector.js` +- Modify: `crates/trusted-server-cli/src/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/compare.rs` + +- [ ] **Step 1: Write JS collector contract fixture tests** + + Add Rust unit tests that inspect generated init-script text and decode a mocked `window.__tsAdTemplateEvidence` JSON payload. These should not launch Chrome. + + Prefer **behavioral** assertions over brittle substring matching: where possible, + assert by decoding a mocked `window.__tsAdTemplateEvidence` payload into + `BrowserAdEvidence` and checking fields. For the few structural checks that must + inspect the script text, pin **exact** marker substrings (no "or equivalent", so + the pass condition is deterministic) — choose the markers to match the strings the + implementation will actually emit: + - `build_ad_template_init_script` output contains the literal `__TS_CONFIG` injection; + - contains the chosen googletag-hook marker (pick ONE and pin it, e.g. + `Object.defineProperty(window, "googletag"`); + - contains the `cmd.push` wrap marker; + - contains the `defineSlot` record marker; + - contains the `apstag.fetchBids` wrap marker; + - embeds only the configured div prefixes / provider IDs passed via `__TS_CONFIG` + (assert a non-configured prefix is absent). + +- [ ] **Step 2: Run tests and verify failure** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_template_collector --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: failure because collector script/builder does not exist. + +- [ ] **Step 3: Implement init script builder** + + In Rust, build script as: + + ```rust + pub fn build_ad_template_init_script(config: &AdTemplateCollectorConfig) -> Result { + let config_json = serde_json::to_string(config) + .map_err(|error| format!("failed to serialize ad-template collector config: {error}"))?; + Ok(format!(";(() => {{ const __TS_CONFIG = {config_json};\n{}\n}})();", include_str!("ad_template_collector.js"))) + } + ``` + + Keep the JS file generic; pass configured prefixes and APS slot IDs through `__TS_CONFIG`. + +- [ ] **Step 4: Implement read-only JS evidence collection** + + In `ad_template_collector.js`, write to `window.__tsAdTemplateEvidence`: + - `dom_ids`: matched IDs from configured prefixes, excluding `-container`; + - `gpt_slots`: record `defineSlot` calls observed **both** directly **and** when + dispatched from the `googletag.cmd` queue (wrap `cmd.push` so queued callbacks are + instrumented without changing their order — spec §7), **plus** a post-settle + `googletag.pubads().getSlots()` scrape. For each scraped slot capture + `getAdUnitPath()`, `getSlotElementId()`, and `getSizes()` so `getSlots()`-only + slots still carry numeric `sizes` for the §5.4 overlap rule. Normalize sizes from + both `defineSlot` input and `getSizes()` output: `[300,250]` → one `(300,250)`; + `[[300,250],[728,90]]` → two pairs; non-numeric (`"fluid"`) dropped from numeric + sizes and surfaced as a `fluid_size_ignored` warning; + - `aps_calls`: `fetchBids` payloads (configured slot IDs + sizes); + - `warnings`: collector warnings only ({code, message}), no page HTML/cookies/storage. + + Always call original page functions with unchanged arguments, and never override + `navigator.webdriver` (spec §7). + +- [ ] **Step 5: Add browser collector extraction** + + After settle and after optional scroll, evaluate: + + ```javascript + ;() => window.__tsAdTemplateEvidence || null + ``` + + Decode into `BrowserAdEvidence`. If decode fails, return a page warning rather than failing navigation. + +- [ ] **Step 6: Add deterministic scroll** + + In `audit/browser.rs`, implement `scroll` by evaluating: + + ```javascript + ;async () => { + const height = Math.max( + document.body.scrollHeight, + document.documentElement.scrollHeight + ) + for (const y of [ + Math.floor(height * 0.33), + Math.floor(height * 0.66), + height, + ]) { + window.scrollTo(0, y) + await new Promise((resolve) => setTimeout(resolve, 250)) + } + window.scrollTo(0, 0) + } + ``` + + Then wait for the same settle quiet period and collect evidence with `phase = "scroll"` where the JS script marks new observations. + +- [ ] **Step 7: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_template_collector --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli audit::browser --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: unit tests pass without launching Chrome. + +- [ ] **Step 8: Commit** + + ```bash + git add crates/trusted-server-cli/src/audit/ad_template_collector.js crates/trusted-server-cli/src/audit/browser.rs crates/trusted-server-cli/src/audit/collector.rs crates/trusted-server-cli/src/ad_templates/compare.rs + git commit -m "Collect browser ad-template evidence" + ``` + +## Task 9: Implement `ts audit ad-templates verify` + +**Files:** + +- Create: `crates/trusted-server-cli/src/audit/ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/output.rs` + +- [ ] **Step 1: Write failing orchestration tests with a fake collector** + + Build a fake collector implementing `AuditCollector` and test: + - one confirmed page exits success in default mode; + - strict missing slot returns error; + - `[auction].enabled = false` returns runtime skipped and does not strict-fail missing evidence; + - one page navigation error plus one success sets JSON `ok = false`; + - invalid `ftp://` URL fails before fake collector is called; + - redirect uses final path for expected slots and emits redirect warning. + + Define the test scaffolding explicitly (no dangling helpers): + + ```rust + // Maps each requested URL to a canned outcome so orchestration is tested without Chrome. + struct FakeCollector { + pages: std::collections::HashMap>, + } + + impl FakeCollector { + // Success page: requested -> final_url, carrying the given ad evidence. + fn page(requested: &str, final_url: &str, evidence: BrowserAdEvidence) -> Self { + let mut pages = std::collections::HashMap::new(); + pages.insert( + requested.to_string(), + Ok(CollectedPage { + final_url: url::Url::parse(final_url).expect("valid final url"), + title: String::new(), + script_count: 0, + resource_count: 0, + warnings: Vec::new(), + ad_evidence: Some(evidence), + }), + ); + Self { pages } + } + // Helper to add a failing page for multi-URL tests. + fn with_error(mut self, requested: &str, message: &str) -> Self { + self.pages.insert(requested.to_string(), Err(message.to_string())); + self + } + } + + impl AuditCollector for FakeCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.pages + .get(request.url.as_str()) + .cloned() + .unwrap_or_else(|| Err(format!("no fake page for {}", request.url))) + } + } + + impl BrowserAdEvidence { + // #[cfg(test)] fixture: one confirmed news slot (atf / ad-atf-0 / /123/news/atf, 300x250). + fn confirmed_news_slot() -> Self { + BrowserAdEvidence { + dom_ids: vec![DomEvidence { dom_id: "ad-atf-0".into(), phase: EvidencePhase::InitialLoad }], + gpt_slots: vec![GptSlotEvidence { + gam_unit_path: "/123/news/atf".into(), + div_id: "ad-atf-0".into(), + sizes: vec![(300, 250)], + phase: EvidencePhase::InitialLoad, + }], + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + } + + // Runs the verify orchestration with `--json` over `urls` and returns parsed JSON. + // Loads a #[cfg(test)] effective config whose `/news/*` slot is the atf slot above. + fn run_verify_json(collector: &dyn AuditCollector, urls: impl IntoIterator) -> serde_json::Value { /* impl in test module */ } + + #[test] + fn verify_uses_final_url_for_matching_after_redirect() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + BrowserAdEvidence::confirmed_news_slot(), + ); + let json = run_verify_json(&collector, ["https://www.example.com/"]); + + assert_eq!(json["pages"][0]["path"], "/news/story"); + // Warning order is unspecified; assert presence, not index 0. + let warnings = json["pages"][0]["warnings"].as_array().expect("warnings array"); + assert!( + warnings.iter().any(|w| w["code"] == "redirected"), + "redirect should emit a `redirected` warning" + ); + } + ``` + + `run_verify_json` calls the same `run_verify` entry point used in production but + with the fake collector injected and output captured; define it in the test module + so all six listed cases share it. + +- [ ] **Step 2: Run focused tests and verify failure** + + Run: + + ```bash + cargo test -p trusted-server-cli audit::ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because verifier module does not exist. + +- [ ] **Step 3: Wire the verifier handler** + + `AuditAdTemplatesVerifyArgs` already exists from Task 7, Step 4. Replace the Task 7 + stub so `audit::run_audit` routes `AdTemplates(Verify(args))` into a new + `audit::ad_templates::run_verify(args)`. Do not redefine the arg struct. + +- [ ] **Step 4: Implement verification orchestration** + + For each URL: + 1. Collect browser page with ad-template init script and optional scroll. + 2. Parse final URL and normalize final path. + 3. Build expected slots for final path. + 4. Build gate summary using shared core gate helper with `consent_allows_auction = None`. + 5. Add redirect warning (`code = "redirected"`) if requested path differs from final path. + 6. Compare evidence (`compare_page_evidence`) to get a `PageVerificationResult`. + 7. **Assemble the wire `PageJson`** (Task 4 type) from the pieces the comparison + result does not carry: `url` / `final_url` / `requested_path` / `path`, + `gates` (map the gate summary's per-gate states to `GateState`), + `matched_slot_count`, `runtime_ad_stack_expected` (via the `From` impl on + `RuntimeAdStackExpectedJson`), then the `slots` / `extra_evidence` / `warnings` + from the comparison result. `PageVerificationResult` is intentionally URL- and + gate-agnostic; this step is where per-page request context is joined in. + 8. Preserve page-level errors as a `PageJson` with `error: Some(..)` and continue + remaining URLs. + +- [ ] **Step 5: Implement exit behavior** + - Default auditor-assist mode: return `Ok(())` for missing/partial evidence when no page-level collection errors occur. + - `--strict`: return `Err(String)` when any non-skipped page has missing/partial slot. + - Multi-URL page errors: JSON `ok=false`; command returns `Err(String)` after writing JSON/human output. + - Invalid schemes: fail before browser launch and before any output. + +- [ ] **Step 6: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli audit::ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: verifier and pure comparison tests pass. + +- [ ] **Step 7: Commit** + + ```bash + git add crates/trusted-server-cli/src/audit/ad_templates.rs crates/trusted-server-cli/src/audit/mod.rs crates/trusted-server-cli/src/run.rs crates/trusted-server-cli/src/ad_templates/output.rs + git commit -m "Verify ad-template slots from browser evidence" + ``` + +## Task 10: Add Local Browser Fixture Tests + +**Files:** + +- Modify: `crates/trusted-server-cli/src/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/audit/ad_templates.rs` + +- [ ] **Step 1: Add test-only local HTTP fixture helper** + + In `audit::browser` tests, create a `TcpListener` serving static HTML from strings. Keep it test-only and host-target only. + + Fixture pages: + - direct `googletag.defineSlot`; + - `googletag.cmd.push`; + - late `window.googletag = { cmd: [] }`; + - late `window.apstag`; + - lazy slot created after scroll; + - redirect from `/` to `/news/story`; + - navigation returning 500. + +- [ ] **Step 2: Gate tests when Chrome is unavailable** + + Add helper: + + ```rust + fn chrome_available() -> bool { + ["chrome", "chromium", "google-chrome", "google-chrome-stable"] + .iter() + .any(|name| which::which(name).is_ok()) + } + ``` + + Each browser fixture test should early-return when unavailable. Do not use + `println!` / `eprintln!`; keep the skip reason in the helper name or a skipped + assertion message so clippy stays clean. This keeps CI portable unless Chrome is + installed. + +- [ ] **Step 3: Write fixture tests** + + Tests should assert the collector sees evidence, not real ad network behavior: + - direct GPT evidence confirms; + - command-queue GPT evidence confirms; + - APS `fetchBids` evidence removes APS provider warning; + - lazy slot appears only when `--scroll` is set; + - redirect result uses final path; + - failed page produces page-level error while other pages continue. + +- [ ] **Step 4: Run fixture tests locally** + + Run: + + ```bash + cargo test -p trusted-server-cli browser_fixture --target $(rustc -vV | sed -n 's/^host: //p') -- --nocapture + ``` + + Expected: pass when Chrome/Chromium exists; otherwise tests skip with explicit message. + +- [ ] **Step 5: Commit** + + ```bash + git add crates/trusted-server-cli/src/audit/browser.rs crates/trusted-server-cli/src/audit/ad_templates.rs + git commit -m "Add browser fixtures for ad-template verification" + ``` + +## Task 11: Update Documentation And Help Snapshots + +**Files:** + +- Modify: `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` if implementation decisions differ. +- Modify: `trusted-server.example.toml` only if command examples need harmless fictional config comments. +- Modify: `CLAUDE.md` only if verification commands or CLI command surface need to be documented. + +- [ ] **Step 1: Run CLI help manually** + + Run: + + ```bash + cargo run -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') -- audit --help + cargo run -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') -- audit ad-templates verify --help + cargo run -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') -- config ad-templates --help + ``` + + Expected: nested audit commands are discoverable; hidden legacy `ts audit ` does not dominate help text. + +- [ ] **Step 2: Update docs if help text or behavior differs from spec** + + Keep examples using `https://www.example.com/` only. Do not mention real publisher sites. + +- [ ] **Step 3: Run docs format** + + Run: + + ```bash + cd docs && npm run format + ``` + + Expected: Prettier passes. + +- [ ] **Step 4: Commit** + + ```bash + git add docs trusted-server.example.toml CLAUDE.md + git commit -m "Document ad-template CLI verification" + ``` + + If no docs changed, skip the commit. + +## Task 12: Final Verification + +**Files:** + +- Verify all touched files. + +- [ ] **Step 1: Rust format** + + Run: + + ```bash + cargo fmt --all -- --check + ``` + + Expected: pass. + +- [ ] **Step 2: Host CLI tests** + + Run: + + ```bash + cargo test -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: pass. Browser fixture tests either pass or explicitly skip when Chrome/Chromium is unavailable. + +- [ ] **Step 3: Workspace tests** + + Run: + + ```bash + cargo test --workspace + ``` + + Expected: pass. + +- [ ] **Step 4: Clippy** + + Run: + + ```bash + cargo clippy --workspace --all-targets --all-features -- -D warnings + ``` + + Expected: pass. + +- [ ] **Step 5: Wasm isolation proof** + + `trusted-server-adapter-fastly` does **not** depend on `trusted-server-cli`, so the + adapter build never compiles the CLI crate and cannot detect a CLI-crate dep leak. + The real proof is building the **CLI crate itself** for the wasm target (its modules + are `#[cfg(not(target_arch = "wasm32"))]`, so a wasm build must succeed with the + host-only deps compiled out). Note the workspace default target is already + `wasm32-wasip1`, so Steps 3–4 (`cargo test/clippy --workspace`) also build the CLI + crate for wasm — but make the isolation check explicit: + + ```bash + # Real CLI isolation proof: CLI crate must build for wasm with host deps excluded. + cargo build --package trusted-server-cli --target wasm32-wasip1 + # Adapter still built to confirm the production artifact is unaffected. + cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 + ``` + + Expected: both pass. If `chromiumoxide`/`tokio`/etc. leaked into a non-target-cfg + dependency table, the first command fails — that is the guard. + +- [ ] **Step 6: Docs format** + + Run: + + ```bash + cd docs && npm run format + ``` + + Expected: pass. + +- [ ] **Step 7: Inspect final diff** + + Run: + + ```bash + git status --short + git diff --stat origin/server-side-ad-templates-impl...HEAD + git log --oneline origin/server-side-ad-templates-impl..HEAD + ``` + + Expected: only intended CLI/core/doc files changed; no `.env`, operator `trusted-server.toml`, or generated browser artifacts included. + +## Risks And Watch Points + +- `chromiumoxide` must remain a host-only `trusted-server-cli` dependency. Any wasm build failure here means the dependency leaked. +- `ts audit ` compatibility must not swallow `ts audit ad-templates` as a URL. +- Runtime gate extraction (Task 1) only touches `should_run_server_side_ad_stack` + (the navigation gate). `/__ts/page-bids` is **intentionally NOT routed** through + `evaluate_ad_stack_gate` — its gate semantics differ (bot/prefetch skip the auction + but keep slots; no `is_navigation`/`is_get` gate). Its parity is preserved by + leaving it untouched, not by sharing the helper. Do not reroute page-bids. Keep + existing publisher and page-bids tests passing. +- The browser collector must not capture page HTML, cookies, storage, request bodies, or arbitrary DOM. Only collect configured-prefix DOM IDs and ad-related evidence. Never override `navigator.webdriver`. +- `runtime_ad_stack_expected = "unknown"` is normal for live consent state; do not over-model consent unless the collector can prove it. +- Browser fixture tests must not depend on real GPT/APS network calls. +- **Deferred in this implementation:** `/__ts/page-bids` SPA observation (spec §5.2 + "when available"). `PageBidsEvidence` exists as forward scaffolding but is not + collected (Task 8), surfaced in JSON (Task 4), or tested. Revisit if SPA route + verification is prioritized. +- Keep generation (`ts audit ad-templates generate`) out of this PR. diff --git a/docs/superpowers/plans/2026-08-18-contiguous-generated-slot-tables.md b/docs/superpowers/plans/2026-08-18-contiguous-generated-slot-tables.md new file mode 100644 index 000000000..0fb8977a0 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-contiguous-generated-slot-tables.md @@ -0,0 +1,46 @@ +# Contiguous Generated Slot Tables Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Keep generated creative-opportunity slot and provider tables contiguous with their parent section. + +**Architecture:** Normalize the document positions carried by generated `toml_edit` tables before inserting them into the target document. Anchor the whole generated subtree at the target creative section and rely on stable serialization order. + +**Tech Stack:** Rust, `toml_edit`, Cargo tests + +--- + +### Task 1: Reproduce the position collision + +**Files:** + +- Modify/Test: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` + +- [ ] Add `splice_keeps_generated_slots_and_providers_contiguous` with a late creative section and unrelated tables at colliding positions. +- [ ] Assert no unrelated table header occurs between `[creative_opportunities]`, all generated slots, and their provider subtables. +- [ ] Add `splice_groups_a_new_creative_section_with_its_slots` for an input that has no creative section, proving the newly created parent and generated subtree share the final anchor. +- [ ] Run each focused test with `cargo test_cli_macos -- --exact` and confirm both ordering assertions fail. + +### Task 2: Normalize imported table positions + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` + +- [ ] Add a small recursive helper using `Table::set_position`, `Table::iter_mut`, and `ArrayOfTables::iter_mut` to assign one anchor position to every table in the generated slot subtree. +- [ ] Use the existing creative table's position; for a newly created section, allocate one greater than the greatest parsed position and explicitly assign that anchor to both the new parent and its generated subtree. +- [ ] Run `cargo test_cli_macos commands::audit::generate::slot_toml::tests::splice_keeps_generated_slots_and_providers_contiguous -- --exact` and confirm it passes. +- [ ] Run `cargo test_cli_macos commands::audit::generate::slot_toml::tests::splice_groups_a_new_creative_section_with_its_slots -- --exact` and confirm it passes. +- [ ] Run `cargo test_cli_macos commands::audit::generate::slot_toml::tests` and confirm the complete module suite passes. + +### Task 3: Verify and deliver + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` + +- [ ] Run `./scripts/test-cli.sh`. +- [ ] Run `cargo fmt --all -- --check`. +- [ ] Run `cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets -- -D warnings`. +- [ ] Confirm `trusted-server.toml` and the user's existing `fastly.toml` change remain untouched. +- [ ] Commit the verified generator fix on the current feature branch. diff --git a/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md b/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md new file mode 100644 index 000000000..873168438 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md @@ -0,0 +1,723 @@ +# PR 823 Review Resolution Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve every actionable finding in PR 823 review `4958563121`, verify the branch, publish it, and answer all 28 inline threads. + +**Architecture:** Correct the review findings at four existing seams: core runtime gate APIs, pure CLI projection/comparison, crawl generation and TOML persistence, and the shared browser session. Keep page-controlled work bounded, use one source of truth for runtime/browser behavior, and preserve operator-authored configuration outside the managed creative-opportunities fields (`slot`, `gam_network_id`, `section_root`, and `section_segment`). + +**Tech Stack:** Rust 2024, clap 4, toml_edit 0.23, chromiumoxide 0.9, Tokio current-thread runtime, serde/serde_json, embedded JavaScript collector, mdBook documentation, GitHub CLI. + +--- + +## File Map + +- `crates/trusted-server-core/src/creative_opportunities.rs`: allocation-free gate evaluation, gate diagnostics, pattern validation, consent semantics. +- `crates/trusted-server-core/src/publisher.rs`: named gate input at the runtime call site. +- `crates/trusted-server-cli/src/ad_templates/{expected,compare,output}.rs`: runtime-equivalent projection, typed formats, confirmability, safe output. +- `crates/trusted-server-cli/src/commands/config/ad_templates.rs`: static command validation, gate parity, lint, escaping. +- `crates/trusted-server-cli/src/commands/audit/{collector,browser,ad_templates,ad_template_collector.js}.rs`: shared browser options/session and verifier behavior. +- `crates/trusted-server-cli/src/commands/audit/generate/{browser_collector,evidence,gpt_slots,crawl_plan,page_patterns,unit_template,slot_toml,mod,validate}.rs`: crawl evidence, inference, persistence, and dry-run safety. +- `crates/trusted-server-cli/src/commands/audit/{mod,page}.rs`, `crates/trusted-server-cli/src/run.rs`, `crates/trusted-server-cli/src/main.rs`: clap contracts and exit outcomes. +- `docs/guide/cli.md`, `scripts/test-cli.sh`, `.github/workflows/test.yml`: operator contract and enforced browser CI. + +## Task 1: Make the runtime gate API allocation-free and reusable + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Add failing core tests** + +Add tests that sweep all 64 boolean combinations with `consent_allows_auction: None`, assert the expected `No`/`Unknown` result, assert `blocking_gates()` derives diagnostics without an owned `Vec`, and exercise the specific page-pattern validation error. + +Use a borrowed/static iterator contract: + +```rust +pub fn blocking_gates(self) -> impl Iterator { + AdStackGateName::ALL + .into_iter() + .filter(move |gate| gate.blocks(self.input)) +} + +pub fn validate_page_pattern(pattern: &str) -> Result<(), String> { + compile_page_pattern(pattern).map(|_| ()) +} +``` + +- [ ] **Step 2: Run the narrow tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-core --target "$(rustc -vV | awk '/host:/ {print $2}')" ad_stack_gate -- --nocapture +``` + +Expected: failure because the unknown-consent sweep and allocation-free diagnostic API are not implemented. + +- [ ] **Step 3: Implement the minimal core change** + +Store the original `AdStackGateInput` in `AdStackGateResult`, compute `expected` with boolean expressions rather than `Vec::push`, expose a zero-allocation iterator over a `const ALL`, make `compile_page_pattern` crate-private, and add `validate_page_pattern`. Document that `None` means unknown and differs from denied (`Some(false)`). Preserve the detailed glob error in `compile_patterns`. + +Delete `should_run_server_side_ad_stack`; construct `AdStackGateInput` with named fields in `publisher.rs`. Import the gate types at module scope. + +- [ ] **Step 4: Verify GREEN** + +Run the narrow command again, then: + +```bash +cargo test-fastly creative_opportunities +cargo test-axum creative_opportunities +``` + +Expected: all selected tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/creative_opportunities.rs crates/trusted-server-core/src/publisher.rs +git commit -m "Align ad stack gate diagnostics with runtime" +``` + +## Task 2: Align expected-slot projection and comparison with runtime behavior + +**Files:** + +- Modify: `crates/trusted-server-cli/src/ad_templates/expected.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/compare.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/output.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/ad_templates.rs` + +- [ ] **Step 1: Add failing projection and comparison tests** + +Cover: + +- an unrenderable dynamic slot is omitted from expected slots and does not make `matched_slots` pass; +- the diagnostic says the runtime omits the slot for that path; +- `MediaType` remains typed through comparison; +- video/native-only slots produce `Unconfirmable` and do not fail strict; +- a sizeless out-of-page slot against banner-configured formats is `Partial` and fails strict; +- an incompatible banner is still `Partial` and fails strict; +- a missing slot has `phase: None` and JSON omits `phase`; +- server-side APS configuration alone does not emit `aps_evidence_missing`; +- collector warnings are appended to page warnings; +- human output contains expectation, gates, matched count, extra evidence, and warnings; +- bidi override/isolate characters are escaped. + +The central type changes are: + +```rust +pub struct ExpectedFormat { + pub width: u32, + pub height: u32, + pub media_type: MediaType, +} + +pub enum SlotStatus { + Confirmed, + Partial, + Missing, + Unconfirmable, +} + +pub struct SlotResult { + pub phase: Option, + // existing fields +} +``` + +- [ ] **Step 2: Run the narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" ad_templates::expected +cargo test --package trusted-server-cli --target "$HOST_TARGET" ad_templates::compare +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::ad_templates +``` + +Expected: new assertions fail on current projection/status/warning behavior. + +- [ ] **Step 3: Implement projection, comparison, and output changes** + +Filter `match_slots` with `render_gam_unit_path(...).map(...)` while building `ExpectedSlot`. Remove the unconditional client-side APS check. Compute confirmability before assigning status. Map typed media values to strings only in `to_slot_json`. Make JSON phase `Option` with `skip_serializing_if = "Option::is_none"`. Extend warnings with `evidence.warnings` after decode. + +Extend `is_terminal_control` with `0x202A..=0x202E` and `0x2066..=0x2069`. Apply `escape_terminal_text` to every human-facing page/config-derived field. + +- [ ] **Step 4: Verify GREEN** + +Run all three narrow commands again. + +Expected: all selected tests pass with no warnings. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/ad_templates crates/trusted-server-cli/src/commands/audit/ad_templates.rs +git commit -m "Match ad template verification to runtime behavior" +``` + +## Task 3: Correct static CLI contracts and process exit semantics + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/config/ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` +- Modify: `crates/trusted-server-cli/src/main.rs` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `Cargo.lock` + +- [ ] **Step 1: Add failing parser, normalization, lint, and outcome tests** + +Add tests proving: + +- bare and full-URL forms normalize spaces, dot segments, tabs, queries, and fragments identically; +- `/r?to=https://example.com` remains a bare path; +- `check` requires exactly one expectation mode and rejects `--allow-extra-slots --expect-no-slots` through clap; +- `--method` accepts a valid `http::Method` and uses exact GET semantics; +- `lint` reports each invalid configured pattern; +- `explain` uses `gate.expected` even when providers are empty and prints provider state separately; +- `--edgezero-enabled` is rejected because the unsupported model is removed; +- bare `ts audit` displays help rather than a drifting manual error; +- parser coverage includes lint, explain, generate, verify profiles/options, and the no-`--adapter` contract; +- an assertion outcome maps to exit 1 and a tool error maps to exit 2. + +Use an explicit process outcome: + +```rust +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RunOutcome { + Success, + AssertionFailed, +} + +impl RunOutcome { + pub const fn exit_code(self) -> i32 { + match self { + Self::Success => 0, + Self::AssertionFailed => 1, + } + } +} +``` + +Tool failures remain `Err(String)` and therefore exit 2. Assertion commands write their failure to stderr before returning `AssertionFailed`, avoiding `log::error!` filtering. + +- [ ] **Step 2: Run parser/static tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::config::ad_templates +cargo test --package trusted-server-cli --target "$HOST_TARGET" run::tests +``` + +Expected: current hand-rolled validation, normalization, and exit behavior fail the new tests. + +- [ ] **Step 3: Implement the CLI contract** + +Use a dummy HTTPS base with `Url::options().base_url(...)` for bare paths after anchored scheme detection on the pre-query slice. Add clap `ArgGroup`, `conflicts_with`, `arg_required_else_help`, typed `http::Method`, and browser settle validation. Add `http = { workspace = true }` to the CLI host dependencies. + +Return `RunOutcome` from dispatchable CI commands. Keep edgezero delegated errors as tool errors. Remove the unsupported EdgeZero flag/text and route gate output through `blocking_gates()`. + +- [ ] **Step 4: Verify GREEN** + +Run the two narrow commands again and confirm all tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add Cargo.lock crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/main.rs crates/trusted-server-cli/src/run.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/commands/config/ad_templates.rs +git commit -m "Define ad template CLI assertion contracts" +``` + +## Task 4: Make the injected collector bounded and behavior-preserving + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/ad_template_collector.js` +- Modify: `crates/trusted-server-cli/src/commands/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` + +- [ ] **Step 1: Add failing JavaScript-contract and decoder tests** + +Add tests/fixtures for an out-of-`u32` size beside a valid slot, a truthy `googletag.cmd` without `push`, multiple `cmd.push` arguments, 512-character capture limits, and non-enumerable/closure-local wrapping. Replace the existing `contains("cmd.push")` assertion with assertions that the no-op wrapper is absent. + +The JavaScript bounds are: + +```javascript +const __TS_MAX_STRING = 512 +function __ts_text(value) { + return String(value).slice(0, __TS_MAX_STRING) +} + +if (width > 4294967295 || height > 4294967295) return null +``` + +The setter must always retain the publisher value: + +```javascript +set(value) { + try { + internal = wrap(value) + } catch (error) { + internal = value + __ts_push(__ts_ev.warnings, { + code: "wrap_failed", + message: __ts_text(error), + }) + } +} +``` + +- [ ] **Step 2: Run the narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::collector +cargo test --package trusted-server-cli --target "$HOST_TARGET" collector_payload +``` + +Expected: current script permits oversized integers and retains the behavior-changing wrapper. + +- [ ] **Step 3: Implement minimal collector changes** + +Guard all page-derived strings through `__ts_text`, enforce numeric upper bounds, delete the `cmd.push` wrapper, use a closure-local `WeakSet` for wrapped objects, and install wrapped functions with non-enumerable `Object.defineProperty`. Soften the header claim to “observes without capturing page data.” + +Before serde decode, stringify the evidence inside the page and return a small +sentinel instead of the payload when the serialized string exceeds 1 MiB +(`MAX_EVIDENCE_PAYLOAD_BYTES = 1_048_576`). On the Rust side, the sentinel +produces an `ad_evidence_too_large` warning and `ad_evidence: None`; it does not +fail navigation or the whole collection. This bounds CDP transfer and Rust +decode/allocation while preserving a precise operator diagnostic. + +- [ ] **Step 4: Verify GREEN** + +Run the narrow commands again and confirm all tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/ad_template_collector.js crates/trusted-server-cli/src/commands/audit/collector.rs crates/trusted-server-cli/src/commands/audit/browser.rs +git commit -m "Bound browser ad template evidence collection" +``` + +## Task 5: Unify browser launch, session reuse, and settling + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/page.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Add failing fake-collector and browser configuration tests** + +Cover one launch/session for multiple URLs, root included in the profile batch, page close on success/error, host-only `Path=/` cookies, explicit final-URL failure, same-host HTTP-to-HTTPS acceptance, host/downgrade/port refusal, new-headless 1280x800 defaults, headful/profile/proxy/consent parity, `$CHROME` parity, and generic/legacy default-on consent. + +Extend the trait with a default batch method so fakes remain simple: + +```rust +pub trait AuditCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result; + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + requests.iter().cloned().map(|request| self.collect_page(request)).collect() + } +} +``` + +The real browser implementation overrides `collect_pages` to create one runtime, +temporary profile, browser, handler, and sequentially closed pages. + +- [ ] **Step 2: Run narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::browser +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::ad_templates +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::browser_collector +``` + +Expected: verifier launches per URL, browser defaults diverge, and tabs/cookies/final URL handling fail new assertions. + +- [ ] **Step 3: Implement shared browser configuration and batching** + +Move executable resolution and launch-option construction into `browser.rs` as crate-visible helpers used by both collectors. Flatten shared browser options into generate and verify, while keeping generation-only pacing/crawl flags local. Build cookies with explicit domain from `url.host_str()` and `path = Some("/".to_string())`; do not set `url` simultaneously. + +In each page collector, capture the inner result, always call bounded `page.close().await`, then return the captured result. Batch verify requests via `collect_pages`. Include the root in each profile's batch rather than collecting it in a throwaway session. Use `spawn_blocking` for scraper analysis before folding results. + +- [ ] **Step 4: Bound post-navigation work and correct settle semantics** + +Install `performance.setResourceTimingBufferSize(100000)` before navigation. Make `settle` return warnings and wrap every `evaluate`, URL/title read, scroll operation, and evidence read in a per-operation timeout. Accrue quiet only after `document.readyState` is `interactive` or `complete`; sleep `min(remaining_quiet, 250ms)` so short quiet values are honored. Treat `wait_for_navigation` timeout as a warning after successful `goto`. + +Propagate GPT/link/sitemap evaluation errors as notes, set `await_promise` for sitemap discovery, and warn when only the main frame is inspected while child frames exist. + +- [ ] **Step 5: Verify GREEN** + +Run all three narrow commands again. If Chrome is available, also run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::browser::tests:: -- --ignored --test-threads=1 +``` + +Expected: unit/fake tests pass; browser fixtures execute and pass when Chrome exists. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit +git commit -m "Share browser sessions across ad template audits" +``` + +## Task 6: Preserve crawl evidence and make inference conservative + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Add failing inference tests** + +Add focused tests for: + +- `annonsü1`/`annonsü2` and `ünicode-ad-a`/`ünicode-ad-b` prefixes; +- desktop-empty/mobile-present and the inverse; +- two disjoint unrelated placements retained, two with a useful prefix or three fragments refused; +- same-page normalized UUID collisions retained with raw div IDs and all formats; +- 16+ digit numeric stable segments retained; +- comma-separated SRA `dids` ignored; +- locale `/en` pattern emitted as `/en` and every emitted glob matches its source path; +- glob metacharacters escaped with `glob::Pattern::escape`; +- percent-encoded noise/extension paths and `.html`/`.htm`/`.php` treatment; +- dropped-section notes capped at ten plus “and N more”; +- both ambiguous template rows result in explicit `Refuse`; +- real crawl evidence can infer `section_segment = 1`; +- refused slots do not appear in rendered output and their reasons appear in notes. + +- [ ] **Step 2: Run narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::evidence +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::gpt_slots +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::page_patterns +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::crawl_plan +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::unit_template +``` + +Expected: each new regression reproduces its review finding. + +- [ ] **Step 3: Implement evidence-preserving discovery** + +Use the last matching `char_indices` byte boundary for shared prefixes. Remove an empty-page marker whenever a later profile yields slots. Require `(useful shared prefix || group size >= 3)` before classifying disjoint same-shape slots as fragments; emit an ambiguity diagnostic otherwise. + +Group normalized collisions within a page before deduplication. When a group has multiple raw div IDs, keep raw entries, make their generated IDs unique, and attach a collision note. Restrict ephemeral hex matching to tokens containing at least one `a..f`, or an explicit UUID shape; never treat all-digit identifiers as hashes. Reject gampad fallback when parsed `dids` contains a comma. + +- [ ] **Step 4: Implement conservative patterns/templates** + +Emit the observed short path for locale landing pages, escape literal prefixes, decode only for filtering while retaining encoded request paths for matching, and cap notes. Teach crawl planning to carry/infer the section depth used by page-pattern generation. + +Delete the tautological witness check and move its explanatory invariant into `analyse_slot` docs. Keep the existing conservative `Refuse` result for non-derivable slugs and unwitnessed roots. Filter all `Refuse` decisions before `RenderSlot` creation and push each reason into notes. + +- [ ] **Step 5: Verify GREEN** + +Run all five narrow commands again, then: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate +``` + +Expected: the generate module suite passes. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate +git commit -m "Preserve ad template crawl evidence" +``` + +## Task 7: Make slot persistence and dry-run output safe + +**Files:** + +- Modify: `Cargo.toml` +- Modify: `Cargo.lock` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/validate.rs` + +- [ ] **Step 1: Add failing persistence tests** + +Cover: + +- trailing comments after the final slot; +- a multiline string line beginning `[foo]`; +- an array continuation beginning `[300, 250]`; +- non-contiguous slot tables; +- byte-identical unrelated sections/comments and CRLF preservation; +- end-to-end `--replace` through `run_update_slots`; +- dry-run source file byte identity; +- stdout contains only a zero-context unified diff of managed + creative-opportunities changes and does not contain `admin_password` or + unrelated config; +- notes/rollback warning go to stderr; +- a concurrent source edit between initial read and write is refused; +- rerun unions formats and reports broad-prefix collapse. + +Change `run_update_slots` to accept separate writers: + +```rust +pub(crate) fn run_update_slots( + request: &UpdateSlotsRequest<'_>, + collectors: &[(&str, &dyn AuditCollector)], + out: &mut dyn Write, + err: &mut dyn Write, +) -> CliResult<()>; +``` + +- [ ] **Step 2: Run persistence tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::slot_toml +cargo test --package trusted-server-cli --target "$HOST_TARGET" update_slots +``` + +Expected: current line scanner corrupts/preserves incorrectly and dry-run leaks the complete config. + +- [ ] **Step 3: Implement a TOML-aware managed edit** + +Parse the source as `DocumentMut` and update the complete managed field set: +`creative_opportunities.slot`, `gam_network_id`, `section_root`, and +`section_segment`. Insert the generated array-of-tables and upsert only scalar +values that generation actually inferred. A generated `None` preserves the +existing scalar on both merge and `--replace`; absence of fresh evidence is +never an instruction to delete operator configuration. Retain decorations on +all other items. Before returning, parse both documents and compare canonical +clones with all four managed fields removed; return an error if any other item +differs. Preserve CRLF after serialization. Add regression cases in Step 1 for +an unresolved network ID and literal-only rerun retaining existing +`gam_network_id`/section policy. + +Document `splice_creative_slots` at its definition and remove the orphaned comments. Replace the `let _ = network_id` presence check with `keys.network_id.is_none()` logic. + +- [ ] **Step 4: Implement secret-safe dry-run and stale-read protection** + +Add `similar` as a workspace/CLI dependency and render a zero-context unified +diff between the old and new managed creative-opportunities projection. The +projection contains only `gam_network_id`, `section_root`, `section_segment`, +and the slot array, so every generated scalar change is visible without +including unrelated operator keys: + +```rust +let diff = similar::TextDiff::from_lines(old_managed, new_managed); +writeln!(out, "{}", diff.unified_diff().context_radius(0).header("configured creative opportunities", "generated creative opportunities"))?; +``` + +Send all notes to `err`. Immediately before atomic rename, re-read the config and compare it with the original bytes; refuse on mismatch. Do not perform this check on dry-run because no write occurs. + +In `merge_render_slots`, union discovered formats into a matching existing slot and count how many discovered slots map to each existing prefix; report counts greater than one. + +- [ ] **Step 5: Verify GREEN** + +Run both narrow commands again and confirm all tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add Cargo.toml Cargo.lock crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/commands/audit/generate +git commit -m "Preserve operator config during slot generation" +``` + +## Task 8: Complete documentation, test hygiene, and CI enforcement + +**Files:** + +- Modify: `docs/guide/cli.md` +- Modify: `scripts/test-cli.sh` +- Modify: `.github/workflows/test.yml` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: touched Rust tests and comments under `crates/trusted-server-cli/src/` + +- [ ] **Step 1: Add/restore parser and CI guard tests** + +Restore the `audit` no-`--adapter` parser test. Add a script contract that sets `TS_AUDIT_BROWSER_TESTS=1`; browser fixture tests panic when that variable is set and Chrome cannot be resolved. Configure the workflow with a browser setup action or the runner's installed Chrome path and export `CHROME` before `scripts/test-cli.sh`. + +- [ ] **Step 2: Replace sensitive-looking fixtures and stale assertions** + +Replace sensitive or customer-shaped fixtures introduced by this PR with fictional network IDs, publisher names, URL shapes, and neutral div tokens. Update comments to describe shapes rather than customers. + +Correct all touched `expect` messages to start with `should`, remove redundant crate/file `dead_code` allowances and annotate only genuinely deferred fields, reorder `Audit`, simplify the Prebid query parser so keys—not substrings—are matched, and bind legacy URLs directly without an impossible `expect`. + +- [ ] **Step 3: Document the complete operator contract** + +In `docs/guide/cli.md`, document: + +- `config ad-templates lint|match|check|explain` and every flag; +- shared `--app-config`, `--manifest`, and `--no-env` behavior; +- `audit ad-templates generate|verify` browser/profile/proxy/consent/settle flags; +- dry-run stdout diff versus stderr notes; +- exit 0 success, exit 1 assertion drift, exit 2 tool/configuration error; +- refused slots are omitted with reasons; +- locale-prefixed inference and section depth; +- `Unconfirmable` strict behavior and optional evidence phase. + +Update the existing design/output examples where the wire contract changed. + +- [ ] **Step 4: Run format and focused checks** + +Run: + +```bash +cargo fmt --all -- --check +cd docs && npm run format +``` + +Expected: both commands exit 0. + +- [ ] **Step 5: Commit** + +```bash +git add .github/workflows/test.yml scripts/test-cli.sh docs crates/trusted-server-cli/src +git commit -m "Document and enforce ad template audit contracts" +``` + +## Task 9: Run full verification and repair regressions + +**Files:** + +- Modify only files implicated by a failing check. + +- [ ] **Step 1: Run format and CLI/browser tests** + +```bash +cargo fmt --all -- --check +./scripts/test-cli.sh +``` + +Expected: exit 0; browser fixture output shows tests executed rather than skipped. + +- [ ] **Step 2: Run repository target suites** + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: all suites exit 0. + +- [ ] **Step 3: Run all target-matched clippy gates** + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo clippy --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" --all-targets -- -D warnings +``` + +Expected: all commands exit 0 with no warnings. + +- [ ] **Step 4: Run cross-adapter parity gates** + +```bash +cargo fmt --manifest-path crates/trusted-server-integration-tests/Cargo.toml -- --check +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +cargo clippy --manifest-path crates/trusted-server-integration-tests/Cargo.toml --all-targets -- -D warnings +``` + +Expected: formatting, parity tests, and integration-test clippy exit 0. + +- [ ] **Step 5: Run JavaScript and documentation checks** + +```bash +cd crates/trusted-server-js/lib && npx vitest run && npm run format && node build-all.mjs +cd ../../.. && cd docs && npm run format +``` + +Expected: tests/build/format exit 0. + +- [ ] **Step 6: Inspect the final diff against the review** + +Run: + +```bash +git diff --check origin/main...HEAD +git status --short +``` + +Walk the 28-thread traceability table and every summary category in the design spec. Confirm each has a code/doc/test resolution or an evidence-backed response. + +- [ ] **Step 7: Commit any verification-only corrections** + +If verification required changes, inspect `git diff --name-only`, stage each +listed path explicitly (never `git add .`), and commit them as `Resolve ad +template review regressions`. Record those exact paths in the execution log. +Skip this commit when verification required no changes. + +## Task 10: Publish and answer GitHub review threads + +**Files:** + +- No repository files unless publication reveals a conflict. + +- [ ] **Step 1: Push the verified branch** + +```bash +git push origin feature/ts-cli-ad-templates +``` + +Expected: push succeeds and PR 823 shows the verified head commit. + +- [ ] **Step 2: Correct the PR description** + +Change the legacy alias statement to say bare `ts audit ` aliases to `ts audit generate `. Preserve all unrelated PR-body content. + +- [ ] **Step 3: Reply to every inline thread** + +For each ID in the spec traceability table, post through: + +```bash +gh api repos/IABTechLab/trusted-server/pulls/823/comments//replies -f body='' +``` + +Each reply must name the concrete behavior changed and, where useful, the focused test. For question threads, state the chosen behavior: union formats and diagnose broad prefixes; default consent assumption on; keep conservative refusal and align docs; allow only same-host HTTP-to-HTTPS upgrades; remove the unsupported EdgeZero model. + +- [ ] **Step 4: Verify publication** + +Query PR 823's head SHA, review comments, checks, and unresolved threads. Confirm all 28 inline comments have one reply and no reply claims a fix absent from the pushed diff. + +- [ ] **Step 5: Report the result** + +Summarize commits, verification commands, any environment limitation, PR link, and thread reply count. Do not claim checks pass without fresh output from Task 9. diff --git a/docs/superpowers/plans/2026-08-18-pre-navigation-cookie-install.md b/docs/superpowers/plans/2026-08-18-pre-navigation-cookie-install.md new file mode 100644 index 000000000..c574c4104 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-pre-navigation-cookie-install.md @@ -0,0 +1,49 @@ +# Pre-navigation Cookie Installation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Allow domain/path-scoped operator cookies to be installed before the audit's first navigation. + +**Architecture:** Add one browser-level cookie installation helper beside `host_cookie`, and call it before creating each audit page. Preserve explicit host-only and root-path scope while avoiding `Page::set_cookie`'s `about:blank` validation. + +**Tech Stack:** Rust, chromiumoxide/CDP, Tokio, Cargo tests + +--- + +### Task 1: Reproduce the pre-navigation failure + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` + +- [ ] Add a Chrome-backed test that installs a host-only cookie before navigating away from `about:blank` and asserts it reaches the first document. +- [ ] Exercise the existing `BrowserCollector` end-to-end against a local HTTP fixture, supplying the cookie through `BrowserCollectRequest`, so the RED test compiles before the fix exists. +- [ ] Run `cargo test_cli_macos commands::audit::browser::tests::supplied_cookie_reaches_first_navigation -- --ignored --exact --nocapture` and confirm it fails with `Blank page can not have cookie`. +- [ ] Add a Chrome-backed error test against the wished-for `set_browser_cookies` API, using an invalid cookie name, and assert the error contains the name but not the secret value. +- [ ] Run that error test and confirm RED because `set_browser_cookies` does not exist yet. + +### Task 2: Install cookies at browser scope + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` + +- [ ] Add `set_browser_cookies(&Browser, &[(String, String)], &Url) -> Result<(), String>` beside `host_cookie`; install one cookie per browser call so failures retain name-only context without exposing values. +- [ ] Invoke it before page creation in both collectors and remove page-level cookie installation. +- [ ] Run `cargo test_cli_macos commands::audit::browser::tests::supplied_cookie_reaches_first_navigation -- --ignored --exact --nocapture` and confirm it passes. +- [ ] Run the focused error test and confirm it passes. + +### Task 3: Verify the change + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` + +- [ ] Run `cargo test_cli_macos commands::audit::browser::tests`. +- [ ] Run `cargo test_cli_macos commands::audit::generate::browser_collector::tests`. +- [ ] Run `./scripts/test-cli.sh` to exercise the portable host-target suite and ignored browser fixtures. +- [ ] Run `cargo fmt --all -- --check`. +- [ ] Run `cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets -- -D warnings`. +- [ ] Inspect the diff to confirm no cookie values are logged and `fastly.toml` remains untouched. diff --git a/docs/superpowers/plans/2026-08-19-ad-template-generation-progress.md b/docs/superpowers/plans/2026-08-19-ad-template-generation-progress.md new file mode 100644 index 000000000..ca4245314 --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-ad-template-generation-progress.md @@ -0,0 +1,167 @@ +# Ad-template Generation Progress Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Show immediate, safe, profile-aware progress while `ts audit ad-templates generate` performs a long browser crawl. + +**Architecture:** Add typed progress events to the `AuditCollector` boundary so the browser can report work before buffered page results are returned. Render and flush those events from `run_update_slots` on stderr, using only URL paths. Preserve crawl/progress errors over teardown errors while always closing and waiting for Chrome. + +**Tech Stack:** Rust 2024, `std::io::Write`, existing `url`, `tokio`, `chromiumoxide`, and CLI test helpers; no new dependency. + +--- + +### Task 1: Define and render safe progress events + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing renderer and writer tests** + +Add tests in `generate/mod.rs` for events covering launch, `1/?`, `2/17`, and finalization. Assert that `https://user:pass@publisher.example/news?token=secret#fragment` renders only `/news`, terminal control bytes are escaped, stdout remains untouched, and a counting writer records an explicit `flush()`. Add writers that fail independently on `write()` and `flush()` and assert a CLI output error. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin progress -- --nocapture +``` + +Expected: FAIL because the progress event and renderer do not exist. + +- [ ] **Step 3: Add the progress model and renderer** + +In `collector.rs`, define a small event enum and callback type: + +```rust +pub(crate) enum CollectionProgress<'a> { + Launching, + Loading { + current: usize, + total: Option, + url: &'a Url, + }, + Planning, + Finalizing, +} + +pub(crate) type ProgressSink<'a> = + &'a mut dyn FnMut(CollectionProgress<'_>) -> CliResult<()>; +``` + +Add concise doc comments to the enum, every variant, and the callback alias. The +callback documentation must state that returning an error stops new collection +work but does not bypass an already-launched browser's finalization/close/wait. + +In `generate/mod.rs`, add a `write_collection_progress` helper that accepts a profile label, formats only `url.path()` (or `/` when empty), sanitizes it with `escape_terminal_text`, writes one line to stderr, and immediately calls `flush()`. Render and test `Planning` between the root load and subsequent page loads. + +- [ ] **Step 4: Run the focused tests and confirm GREEN** + +Run the command from Step 2. Expected: all progress renderer/writer tests pass. + +### Task 2: Propagate progress through collectors with teardown-safe failures + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `scripts/test-cli.sh` + +- [ ] **Step 1: Write failing collector tests** + +Test default `collect_pages` and `collect_site` count semantics, including an attempted page whose collection fails. The exact dynamic-site sequence is root `1/?`, planning, then follow-ups `2/total` through `total/total`; totals include the root and failed attempts advance the count. Add a Chrome-backed test whose progress callback fails during collection. It must return the progress error only after the browser teardown path completes. Extend the existing result-combination unit tests to cover first-error preservation across a collection/planning error, a later finalization-progress error, close error, and wait error, while proving finalization, close, and wait were all attempted. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::browser_collector::tests -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::collector::tests -- --nocapture +``` + +Expected: FAIL because collectors do not accept or emit progress callbacks. + +- [ ] **Step 3: Add callbacks to the collector boundary** + +Extend `collect_pages` and `collect_site` with `ProgressSink`. Default collectors emit `Loading` before each page. The root of a dynamically planned site emits `current: 1, total: None`, followed by `Planning`; after planning, default `collect_site` iterates follow-ups itself with an explicit offset so they report `2/total` onward. Fixed batches emit totals including the root, and failed attempts still consume their position. + +Pass the callback into `with_browser`. Adapt `BrowserAuditCollector::collect_page` with an explicit no-op progress sink because single-page artifact generation has no command progress writer. Emit `Launching` before browser launch, `Loading` immediately before each navigation, `Planning` immediately before invoking the root planner, and `Finalizing` before close/wait. Track only the first crawl/progress error: on callback failure, stop scheduling pages, still attempt finalization, `browser.close()`, and `browser.wait()`, then return that first error ahead of teardown errors. + +Extend `scripts/test-cli.sh` with a second ignored-test filter for +`commands::audit::generate::browser_collector::tests::` so the new Chrome-backed +progress-failure test is actually executed under `TS_AUDIT_BROWSER_TESTS=1` and +single-threaded, alongside the existing three browser audit fixtures. + +- [ ] **Step 4: Run unit and Chrome-backed tests and confirm GREEN** + +Run the focused command, then: + +```bash +./scripts/test-cli.sh +``` + +Expected: collector unit tests and all four Chrome-backed tests pass. + +### Task 3: Wire profile-aware progress into generation + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing generation tests** + +Update `run_update_slots` tests to assert the stderr buffer contains progress for the first profile's `1/?` root, planning, later known totals, the second profile's `1/total` root, and finalization. Assert dry-run diff/success output on stdout contains no progress lines. Add an ordering test with a shared observable writer and fake collector: from inside `collect_site`, after invoking and flushing the progress callback but before returning, assert the progress bytes are already visible. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin update_slots -- --nocapture +``` + +Expected: FAIL because `run_update_slots` does not provide progress callbacks. + +- [ ] **Step 3: Connect callbacks and profiles** + +Create a progress closure for the first profile and pass it to `collect_site`. Pass `err` through `crawl_sections`, create a closure for each later profile, and pass it to `collect_pages`. Keep notes and final summary behavior unchanged. + +- [ ] **Step 4: Run the focused tests and confirm GREEN** + +Run the command from Step 2. Expected: all generation tests pass and progress appears only in stderr. + +### Task 4: Verify and ship + +**Files:** + +- Verify all modified files plus the two design documents. + +- [ ] **Step 1: Format and lint** + +```bash +cargo fmt --all -- --check +cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets --all-features -- -D warnings +git diff --check +cd docs && npm run format +``` + +Expected: all commands exit 0. + +- [ ] **Step 2: Run the complete local CLI suite** + +```bash +./scripts/test-cli.sh +``` + +Expected: unit, config, proxy, documentation, and Chrome-backed tests pass. + +- [ ] **Step 3: Review the scoped diff** + +Confirm no cookie values, real publisher data, or changes to the pre-existing `fastly.toml` modification are included. Request an independent code review and address concrete findings. + +- [ ] **Step 4: Commit and push** + +Stage only the progress implementation and its design/plan documents. Commit with `Show ad-template generation progress`, push `feature/ts-cli-ad-templates`, and confirm local HEAD matches the remote branch. diff --git a/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md new file mode 100644 index 000000000..1dca98982 --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md @@ -0,0 +1,78 @@ +# Refuse Volatile Div-ID Collisions Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Prevent `ts audit ad-templates generate --replace` from writing exact per-render div IDs when several live elements normalize to one runtime prefix. + +**Architecture:** Keep collision detection in GPT discovery, where normalized and raw IDs are both available. On the first distinct collision, remove the tentatively accepted normalized slot and mark the group ambiguous; suppress all later members and emit one actionable diagnostic. Carry a separate evidence-present bit into `EvidenceTable` so collision-only pages are not classified as bot challenges. + +**Tech Stack:** Rust, Chromium GPT evidence model, built-in Rust test framework. + +--- + +### Task 1: Specify refusal behavior + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` + +- [ ] Add assertions for documented `DiscoveredSlots::had_slot_evidence` and run one focused test to observe the expected missing-field compile failure. +- [ ] Add only the documented field with its derived/default false value so behavioral tests can compile; do not wire discovery or classification yet. +- [ ] Change the same-page collision test to require zero emitted slots and one refusal diagnostic. +- [ ] Assert the diagnostic names `ad-in_content`, explains that a broad prefix resolves only one element and raw IDs are volatile, and tells the operator to expose distinct stable IDs. +- [ ] Rename the test to `same_page_hex_normalization_collision_is_refused`. +- [ ] Extend `repeated_raw_div_after_a_normalization_collision_is_deduplicated` with repeats of both initial raw IDs and a third distinct ID; require zero slots and one diagnostic. +- [ ] Add `request_normalization_collision_is_refused`; require zero slots, one diagnostic with the same prefix/safety/action content, true evidence, and a surviving request-derived network ID. +- [ ] Add `ambiguous_registry_stem_still_suppresses_request_fallback` and require no slot resurrection. +- [ ] Require every registry/request collision test to assert `had_slot_evidence` is true. +- [ ] Add `collision_only_page_is_not_classified_as_empty` using a discovered collision result. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin same_page_hex_normalization_collision_is_refused` and confirm RED because two raw slots remain. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin repeated_raw_div_after_a_normalization_collision_is_deduplicated` and confirm RED because raw slots remain. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin request_normalization_collision_is_refused` and confirm RED because request-derived raw slots remain. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin ambiguous_registry_stem_still_suppresses_request_fallback` and confirm RED because the ambiguous registry group remains deployable. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin collision_only_page_is_not_classified_as_empty` and confirm RED because collision-only evidence is classified as empty. + +### Task 2: Refuse ambiguous collision groups + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` + +- [ ] Replace raw-ID preservation with a collision result that distinguishes first ambiguity from later members. +- [ ] Remove the initially accepted normalized slot when ambiguity is first proven. +- [ ] Suppress the colliding and subsequent raw members. +- [ ] Emit one message naming the prefix, both unsafe representations, and the publisher-markup action. +- [ ] Set `had_slot_evidence` for any otherwise usable registry/request candidate and use it in empty-page classification. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin normalization_collision` and confirm the registry and request collision tests GREEN. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin ambiguous_registry_stem_still_suppresses_request_fallback` and confirm registry precedence GREEN. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin collision_only_page_is_not_classified_as_empty` and confirm GREEN. + +### Task 3: Verify and deliver + +**Files:** + +- Verify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Verify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` +- Verify: `docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md` +- Verify: `docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md` + +- [ ] Run `cargo fmt --all -- --check`. +- [ ] Run `./scripts/test-cli.sh aarch64-apple-darwin`. +- [ ] Run `cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets --all-features -- -D warnings`. +- [ ] Run `cd docs && npm run format`. +- [ ] Run `git diff --check` and inspect the scoped diff. +- [ ] Commit and push the fix to `feature/ts-cli-ad-templates`. + +### Task 4: Refuse a known single-observation volatile family + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` + +- [ ] Add failing registry and request tests for a single `__` observation. +- [ ] Add a recognizer keyed on the token shape — ten or more leading digits followed by more alphanumerics — in any position that has a non-empty family prefix before it and placement content after it. +- [ ] Omit matching slots while preserving evidence/network discovery and emit one deduplicated actionable diagnostic naming the family prefix. +- [ ] Add negative tests proving IDs with no token, a bare digit run, or a trailing token remain eligible. +- [ ] Run the focused tests, then repeat Task 3 verification and delivery. diff --git a/docs/superpowers/plans/2026-08-21-liveramp-integration.md b/docs/superpowers/plans/2026-08-21-liveramp-integration.md new file mode 100644 index 000000000..3b261db80 --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-liveramp-integration.md @@ -0,0 +1,1199 @@ +# LiveRamp Integration Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +> **Revision, 2026-08-25 — configuration surface superseded.** This plan was +> written against a typed `[integrations.prebid.liveramp]` subsection, which put +> a named identity vendor inside `trusted-server-core`. The delivered +> implementation instead exposes a vendor-neutral +> `[[integrations.prebid.managed_user_ids]]` array that core forwards to +> Prebid.js verbatim, so RampID is enabled by configuration alone and core names +> no vendor. Every task below that references `liveramp`, `placement_id`, +> `PrebidLiveRampConfig`, or `PrebidLiveRampStorageType` is superseded by +> section 6 of the design specification; the sequencing, test strategy, and +> verification steps still apply unchanged. + +**Goal:** Make Trusted Server's existing Prebid RampID EID path first-class by adding validated operator configuration, deterministic `identityLink` setup, bundle diagnostics, tests, and documentation. + +**Architecture:** Add a `managed_user_ids` array to `PrebidIntegrationConfig` and serialize it into the existing `window.__tsjs_prebid` bootstrap. Entries are opaque to core: it validates only what Prebid needs to address a module and forwards `params` uninspected. The TSJS Prebid shim installs idempotent `pbjs.setConfig` and `pbjs.mergeConfig` normalizers before `processQueue()`, synchronously merges the operator-owned entries with effective publisher User ID entries, and preserves all unrelated configuration. Existing `/auction`, OpenRTB `user.ext.eids`, `ts-eids`, consent, and EC/KV paths remain unchanged. + +**Tech Stack:** Rust 2024, Serde, validator, TypeScript, Prebid.js 10, Vitest, JSDOM, Vite, VitePress/Markdown, Cargo workspace aliases. + +**Specification:** `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md` + +--- + +## File structure + +| File | Responsibility | +| ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `crates/trusted-server-core/src/integrations/prebid.rs` | Define and validate vendor-neutral managed User ID settings; inject the browser-safe camel-cased config; host Rust unit tests. | +| `crates/trusted-server-js/lib/src/integrations/prebid/index.ts` | Validate the injected shape, install the managed User ID configuration guard, and preserve publisher User ID settings. | +| `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` | Prove initial, queued, and late Prebid configuration behavior plus diagnostics and EID transport. | +| `crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts` | Characterize the existing `liveramp.com` → `identityLinkIdSystem` registry mapping. | +| `crates/trusted-server-js/lib/test/build-prebid-external.test.mjs` | Prove an external bundle can include and manifest `identityLinkIdSystem`. | +| `crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs` | Exercise the real generated Prebid bundle and served shim together with LiveRamp config. | +| `trusted-server.example.toml` | Show safe, commented operator configuration. | +| `docs/guide/integrations/prebid.md` | Explain LiveRamp prerequisites, configuration, lifecycle, degraded behavior, and verification. | +| `docs/guide/configuration.md` | Add the typed settings reference. | + +No new integration module, route, storage schema, cookie format, or upstream HTTP client is created. + +## Task 1: Add typed Rust configuration and head injection + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/prebid.rs:202` +- Test: `crates/trusted-server-core/src/integrations/prebid.rs:3028` +- Test: `crates/trusted-server-core/src/integrations/prebid.rs:4010` + +- [ ] **Step 1: Write failing configuration tests** + +Add focused tests beside the existing Prebid TOML parsing tests: + +```rust +#[test] +fn liveramp_config_parses_with_documented_defaults() { + let config = parse_prebid_toml( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[integrations.prebid.liveramp] +placement_id = "999" +"#, + ); + + let liveramp = config.liveramp.expect("should parse LiveRamp config"); + assert_eq!(liveramp.placement_id, "999", "should preserve placement ID"); + assert!(!liveramp.not_use_3p, "should allow cookie recognition by default"); + assert_eq!( + liveramp.storage_type, + PrebidLiveRampStorageType::Cookie, + "should default to cookie storage" + ); + assert_eq!(liveramp.expires_days, 15, "should default to conservative expiry"); + assert_eq!( + liveramp.refresh_in_seconds, 1800, + "should default to LiveRamp's recommended refresh" + ); +} + +#[test] +fn liveramp_config_accepts_explicit_supported_values() { + let config = parse_prebid_toml( + r#" +[integrations.prebid] +server_url = "https://prebid.example/openrtb2/auction" + +[integrations.prebid.liveramp] +placement_id = "12345" +not_use_3p = true +storage_type = "html5" +expires_days = 30 +refresh_in_seconds = 3600 +"#, + ); + + let liveramp = config.liveramp.expect("should parse LiveRamp config"); + assert!(liveramp.not_use_3p, "should preserve not_use_3p"); + assert_eq!(liveramp.storage_type, PrebidLiveRampStorageType::Html5); + assert_eq!(liveramp.expires_days, 30); + assert_eq!(liveramp.refresh_in_seconds, 3600); +} +``` + +Add table-driven rejection coverage using `parse_prebid_toml_result` for: + +- an otherwise valid `[integrations.prebid.liveramp]` subsection with + `placement_id` entirely absent; +- empty, whitespace-padded, and nonnumeric `placement_id`; +- `expires_days = 0` and `expires_days = 31`; +- `refresh_in_seconds = 0`; +- unknown `storage_type`; +- unknown fields within `[integrations.prebid.liveramp]`. + +Also assert that omitting the subsection leaves `config.liveramp == None`. + +- [ ] **Step 2: Run the focused Rust tests and verify they fail** + +Run: + +```bash +cargo test-fastly liveramp_config +``` + +Expected: compilation/test failure because `PrebidLiveRampConfig`, +`PrebidLiveRampStorageType`, and `PrebidIntegrationConfig::liveramp` do not yet +exist. + +- [ ] **Step 3: Implement the minimal typed settings** + +Add near `PrebidIntegrationConfig`: + +```rust +const fn default_liveramp_expires_days() -> u16 { + 15 +} + +const fn default_liveramp_refresh_in_seconds() -> u32 { + 1800 +} + +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum PrebidLiveRampStorageType { + #[default] + Cookie, + Html5, +} + +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct PrebidLiveRampConfig { + #[validate(custom(function = "validate_liveramp_placement_id"))] + pub placement_id: String, + #[serde(default)] + pub not_use_3p: bool, + #[serde(default)] + pub storage_type: PrebidLiveRampStorageType, + #[serde(default = "default_liveramp_expires_days")] + #[validate(range(min = 1, max = 30))] + pub expires_days: u16, + #[serde(default = "default_liveramp_refresh_in_seconds")] + #[validate(range(min = 1))] + pub refresh_in_seconds: u32, +} +``` + +Implement `validate_liveramp_placement_id` using a `ValidationError` with a +stable message. Accept only a non-empty, already-trimmed ASCII-digit string. + +Add to `PrebidIntegrationConfig`: + +```rust +#[serde(default)] +#[validate(nested)] +pub liveramp: Option, +``` + +Update every direct `PrebidIntegrationConfig` initializer, especially +`base_config()`, with `liveramp: None`. + +- [ ] **Step 4: Run the focused configuration tests and verify they pass** + +Run: + +```bash +cargo test-fastly liveramp_config +``` + +Expected: all LiveRamp parsing/default/validation tests pass. + +- [ ] **Step 5: Write failing head-injection tests** + +Add tests beside the current head-injector tests: + +```rust +#[test] +fn head_injector_includes_liveramp_config() { + let mut config = base_config(); + config.liveramp = Some(PrebidLiveRampConfig { + placement_id: "999".to_string(), + not_use_3p: true, + storage_type: PrebidLiveRampStorageType::Html5, + expires_days: 30, + refresh_in_seconds: 3600, + }); + let integration = PrebidIntegration::new(config); + let document_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "pub.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + + let script = &integration.head_inserts(&ctx)[0]; + assert!( + script.contains( + r#""liveRamp":{"placementId":"999","notUse3P":true,"storageType":"html5","expiresDays":30,"refreshInSeconds":3600}"# + ), + "should inject camel-cased LiveRamp config: {script}" + ); +} + +#[test] +fn head_injector_omits_liveramp_config_when_absent() { + // Build the normal context with base_config(). + // Assert the first insert does not contain `liveRamp`. +} + +#[test] +fn head_injector_escapes_script_breakout_in_liveramp_config() { + let mut config = base_config(); + config.liveramp = Some(PrebidLiveRampConfig { + placement_id: "1".to_string(), + ..valid_liveramp_config() + }); + + // Build the normal context. Assert the injected payload contains + // `1<\/script>")` + // has count 1: only the insert's legitimate outer closing tag remains. + // This test may build the invalid value directly because it exercises the + // serializer's defense in depth rather than TOML validation. +} +``` + +- [ ] **Step 6: Run both head-injection tests and verify they fail** + +Run: + +```bash +cargo test-fastly head_injector_includes_liveramp_config +cargo test-fastly head_injector_escapes_script_breakout_in_liveramp_config +``` + +Expected: both fail because the injected payload has no `liveRamp` property or +escaped LiveRamp Placement ID. + +- [ ] **Step 7: Inject a browser-specific serialization shape** + +Inside `IntegrationHeadInjector::head_inserts`, define a borrowed injected +shape so TOML remains snake_case while browser JSON is camelCase: + +```rust +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct InjectedPrebidLiveRampConfig<'a> { + placement_id: &'a str, + not_use_3p: bool, + storage_type: PrebidLiveRampStorageType, + expires_days: u16, + refresh_in_seconds: u32, +} +``` + +Add a skipped-when-absent `live_ramp` field to +`InjectedPrebidClientConfig`, map `self.config.liveramp.as_ref()` into the +borrowed shape, and retain the existing ` { + const spec = getAdapterSpec() + mockGetUserIdsAsEids.mockReturnValue([ + { + source: 'liveramp.com', + uids: [{ id: 'opaque-test-envelope', atype: 3 }], + }, + ]) + + const request = spec.buildRequests([ + { + adUnitCode: 'div-gpt-1', + bidId: 'bid-1', + sizes: [[300, 250]], + bidder: 'trustedServer', + params: {}, + }, + ]) + + expect(JSON.parse(request.data).eids).toEqual([ + { + source: 'liveramp.com', + uids: [{ id: 'opaque-test-envelope', atype: 3 }], + }, + ]) +}) +``` + +Add a logging assertion using a sentinel envelope and spies on `log.debug`, +`log.info`, `log.warn`, and `log.error`; no logged argument may contain the +sentinel. + +- [ ] **Step 4: Write the failing real-artifact test before implementation** + +Change the bundle built in `prebid-artifact-integration.test.mjs` to include +both `sharedIdSystem` and `identityLinkIdSystem`. Inject `liveRamp` before +evaluating the served shim, then assert after shim evaluation: + +```javascript +const configuredUserIds = pageWindow.pbjs.getConfig('userSync.userIds') +expect(configuredUserIds).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: 'identityLink', + params: { pid: '999', notUse3P: false }, + storage: expect.objectContaining({ name: 'idl_env' }), + }), + ]) +) +``` + +Retain the current real `/auction` request assertion. Network remains stubbed; +the test must not contact LiveRamp. + +After the initial managed-entry assertion, call the real public merge API and +prove it cannot append a publisher-owned duplicate: + +```javascript +pageWindow.pbjs.mergeConfig({ + userSync: { + userIds: [ + { name: 'sharedId' }, + { name: 'identityLink', params: { pid: 'publisher-value' } }, + ], + }, +}) + +const mergedUserIds = pageWindow.pbjs.getConfig('userSync.userIds') +expect(mergedUserIds.filter(({ name }) => name === 'identityLink')).toEqual([ + expect.objectContaining({ + name: 'identityLink', + params: { pid: '999', notUse3P: false }, + }), +]) +expect(mergedUserIds).toEqual( + expect.arrayContaining([expect.objectContaining({ name: 'sharedId' })]) +) +``` + +- [ ] **Step 5: Run the focused unit and artifact suites and verify failures** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run test/integrations/prebid/index.test.ts +npx vitest run test/prebid-artifact-integration.test.mjs +``` + +Expected: LiveRamp ownership tests fail because the injected shape and public +configuration normalizers do not exist, and the artifact test fails because no +managed entry is installed. The transport-only characterizations may already +pass; retain them as evidence of the pre-existing path. + +- [ ] **Step 6: Add the injected TypeScript types and constants** + +Add: + +```typescript +interface InjectedLiveRampConfig { + placementId: string + notUse3P: boolean + storageType: 'cookie' | 'html5' + expiresDays: number + refreshInSeconds: number +} + +interface InjectedPrebidConfig { + // Existing fields omitted. + liveRamp?: InjectedLiveRampConfig +} + +const IDENTITY_LINK_CONFIG_NAME = 'identityLink' +const IDENTITY_LINK_STORAGE_NAME = 'idl_env' +const LIVE_RAMP_SET_CONFIG_SENTINEL = '__tsLiveRampSetConfigInstalled' +``` + +Keep this internal to the Prebid module; do not add a new global API. + +- [ ] **Step 7: Extract reusable User ID list parsing** + +Refactor the shape handling currently embedded in +`configuredUserIdNamesFromConfig` into a helper that returns validated entry +objects from any of these inputs: + +- the direct array returned by `getConfig('userSync.userIds')`; +- `{ userSync: { userIds: [...] } }`; +- `{ userIds: [...] }`. + +Use explicit record and entry guards. A valid entry is a non-array object with +a non-empty string `name`; filter malformed members rather than forwarding +them. The parser returns an empty array for malformed containers. Keep a +separate `hasUserIdsPath(config)` predicate equivalent to: + +```typescript +function hasUserIdsPath(config: unknown): config is { + userSync: Record & { userIds: unknown } +} { + return ( + isRecord(config) && + isRecord(config.userSync) && + Object.prototype.hasOwnProperty.call(config.userSync, 'userIds') + ) +} +``` + +This distinction is required: an absent path passes through unchanged, while +an explicitly empty `userIds: []` is normalized to the managed entry. Update +`configuredUserIdNamesFromConfig` to derive names from that helper so +diagnostics and LiveRamp normalization agree on supported shapes. + +- [ ] **Step 8: Implement the managed entry and normalizer** + +Implement small focused helpers equivalent to: + +```typescript +function liveRampUserId( + config: InjectedLiveRampConfig +): Record { + return { + name: IDENTITY_LINK_CONFIG_NAME, + params: { pid: config.placementId, notUse3P: config.notUse3P }, + storage: { + type: config.storageType, + name: IDENTITY_LINK_STORAGE_NAME, + expires: config.expiresDays, + refreshInSeconds: config.refreshInSeconds, + }, + } +} + +function withManagedLiveRampUserId( + config: Record, + managedEntry: Record +): Record { + if (!hasUserIdsPath(config)) return config + + const retained = configuredUserIdEntries(config.userSync.userIds).filter( + (entry) => entry.name !== IDENTITY_LINK_CONFIG_NAME + ) + return { + ...config, + userSync: { + ...config.userSync, + userIds: [...retained, managedEntry], + }, + } +} +``` + +`configuredUserIdEntries` must support the three shapes from Step 7 and return +fresh arrays. The spread operations preserve top-level properties and sibling +`userSync` properties. Do not mutate publisher-owned arrays or objects in +place. + +- [ ] **Step 9: Install idempotent public configuration guards before queue processing** + +In `installPrebidNpm`, after confirming the real Prebid API and before the +existing base configuration and `processQueue()` call: + +1. If injected `liveRamp` is absent, do nothing. +2. Capture and bind the current `pbjs.setConfig` and optional + `pbjs.mergeConfig`. +3. Replace both public APIs with wrappers that share one normalizer for calls + containing `userSync.userIds`; pass all other calls through unchanged. +4. Mark the Prebid object with the sentinel so installation cannot stack. +5. Read effective User ID entries through `pbjs.getConfig`. +6. Call the wrapper synchronously with the effective list, producing one + managed entry before any queued auction. +7. Leave both wrappers installed across `processQueue()` and later calls. + +Use logic equivalent to: + +```typescript +const managedPbjs = pbjs as typeof pbjs & Record +if (managedPbjs[LIVE_RAMP_SET_CONFIG_SENTINEL] !== true) { + const originalSetConfig = pbjs.setConfig.bind(pbjs) + const originalMergeConfig = pbjs.mergeConfig?.bind(pbjs) + const managedEntry = liveRampUserId(config.liveRamp) + + const normalizePublisherConfig = (publisherConfig) => { + let nextConfig = publisherConfig + try { + if (hasUserIdsPath(publisherConfig)) { + nextConfig = withManagedLiveRampUserId(publisherConfig, managedEntry) + } + } catch { + log.error('Prebid LiveRamp configuration could not be normalized') + } + return nextConfig + } + + pbjs.setConfig = (publisherConfig) => + originalSetConfig(normalizePublisherConfig(publisherConfig)) + if (originalMergeConfig) { + pbjs.mergeConfig = (publisherConfig) => + originalMergeConfig(normalizePublisherConfig(publisherConfig)) + } + managedPbjs[LIVE_RAMP_SET_CONFIG_SENTINEL] = true + + const effective = configuredUserIdEntries(pbjs.getConfig('userSync.userIds')) + pbjs.setConfig({ userSync: { userIds: effective } }) +} +``` + +Adapt the callback and return types to the repository's actual `pbjs` typing. +Each original method is invoked exactly once, its return value is preserved, +and normalization errors never log values. The sentinel lives on `pbjs`, not +on the page-level shim state: a test must deliberately reset only +`__tsjsPrebidShimInstalled`, reinstall, and prove both wrapper references are +unchanged and publisher calls are normalized once. + +- [ ] **Step 10: Run focused unit and artifact tests and make them pass** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run test/integrations/prebid/index.test.ts +npx vitest run test/prebid-artifact-integration.test.mjs +``` + +Expected: unit tests pass; the real bundle advertises both User ID modules, the +shim configures one `identityLink` entry, a real `mergeConfig` call retains +exactly that managed entry, and the controlled auction still reaches +`/auction`. + +- [ ] **Step 11: Format, lint, and commit Task 2** + +Run: + +```bash +cd crates/trusted-server-js/lib +npm run format +npm run lint +git add src/integrations/prebid/index.ts test/integrations/prebid/index.test.ts test/prebid-artifact-integration.test.mjs +git commit -m "feat: manage LiveRamp identityLink configuration" +``` + +## Task 3: Lock bundle, transport, consent, and EC behavior with regression tests + +**Files:** + +- Test: `crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts` +- Test: `crates/trusted-server-js/lib/test/build-prebid-external.test.mjs` +- Test: `crates/trusted-server-core/src/auction/endpoints.rs` +- Test: `crates/trusted-server-core/src/consent/mod.rs` +- Test: `crates/trusted-server-core/src/ec/prebid_eids.rs` + +- [ ] **Step 1: Add the explicit registry mapping test** + +```typescript +it('maps LiveRamp EIDs to identityLinkIdSystem', () => { + expect( + resolvePrebidUserIdModulesFromEids([ + { source: 'liveramp.com', uids: [{ id: 'opaque-envelope', atype: 3 }] }, + ]) + ).toEqual({ + modules: ['userId', 'identityLinkIdSystem'], + missingSources: [], + }) +}) +``` + +Also load the checked-in registry JSON or expose a narrow helper and assert the +default preset contains `identityLinkIdSystem`. Do not duplicate the registry +as a second production constant. + +- [ ] **Step 2: Add a bundle manifest test for `identityLinkIdSystem`** + +Extend the existing `includes generated User ID metadata` case or add a focused +case that invokes: + +```javascript +await main([ + '--adapters', + 'rubicon', + '--user-id-modules', + 'identityLinkIdSystem', + '--out', + outputDirectory, +]) +``` + +Assert the manifest's `userIdModules` is exactly +`['identityLinkIdSystem']` and the generated bundle contains the module name. + +- [ ] **Step 3: Run the two characterization suites** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run \ + test/integrations/prebid/user_id_modules.test.ts \ + test/build-prebid-external.test.mjs +``` + +Expected: tests pass using the existing registry and generator. If they fail, +fix the single checked-in registry/generator source rather than introducing a +LiveRamp-only bundle path. + +- [ ] **Step 4: Add or rename LiveRamp-specific Rust regression fixtures** + +Add focused tests (or rename/extend an existing generic fixture while keeping +its broader assertions) proving: + +- in `auction/endpoints.rs`, a client `liveramp.com` UID equal to the resolved + KV UID is merged once, with server-resolved metadata winning on conflict; +- in `consent/mod.rs`, a `liveramp.com` EID is removed when consent denies + identity forwarding; +- in `ec/prebid_eids.rs`, a structured `ts-eids` cookie containing an opaque + `liveramp.com` envelope writes that exact opaque string once to a registry + partner whose source domain is `liveramp.com`. + +Use only synthetic values such as `opaque-test-envelope`. Assert that tests do +not decode or inspect an envelope's contents. + +- [ ] **Step 5: Run the LiveRamp Rust regression fixtures** + +Run from the repository root: + +```bash +cargo test-fastly liveramp +``` + +Expected: forwarding, merge/deduplication, consent removal, and later-request +EC ingestion fixtures all pass. + +- [ ] **Step 6: Commit Task 3** + +Run: + +```bash +git add crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts \ + crates/trusted-server-js/lib/test/build-prebid-external.test.mjs \ + crates/trusted-server-core/src/auction/endpoints.rs \ + crates/trusted-server-core/src/consent/mod.rs \ + crates/trusted-server-core/src/ec/prebid_eids.rs +git commit -m "test: cover LiveRamp Prebid bundle support" +``` + +## Task 4: Document configuration, lifecycle, and operational validation + +**Files:** + +- Modify: `trusted-server.example.toml:41` +- Modify: `docs/guide/integrations/prebid.md:50` +- Modify: `docs/guide/integrations/prebid.md:419` +- Modify: `docs/guide/configuration.md:1050` + +- [ ] **Step 1: Add the commented example configuration** + +Add beneath the Prebid bundle configuration in `trusted-server.example.toml`: + +```toml +# Optional managed LiveRamp RampID configuration. The external Prebid bundle +# must contain identityLinkIdSystem. Obtain the Placement ID and approve the +# publisher origin with LiveRamp before enabling. +# [integrations.prebid.liveramp] +# placement_id = "999" +# not_use_3p = false +# storage_type = "cookie" +# expires_days = 15 +# refresh_in_seconds = 1800 +``` + +Do not add a real Placement ID or credential. + +- [ ] **Step 2: Update the configuration reference** + +Add `[integrations.prebid.liveramp]` fields to both Prebid option tables: + +| Field | Type | Default | Description | +| ----------------------------- | ------------------- | ------------------------------- | ------------------------------------------------------------ | +| `liveramp.placement_id` | String | Required when subsection exists | Numeric LiveRamp Placement ID for `identityLink`. | +| `liveramp.not_use_3p` | Boolean | `false` | Disable cookie-recognized RampID envelopes when true. | +| `liveramp.storage_type` | `cookie` or `html5` | `cookie` | Browser storage used by the Prebid module. | +| `liveramp.expires_days` | Integer 1–30 | `15` | Envelope storage lifetime in days. | +| `liveramp.refresh_in_seconds` | Positive integer | `1800` | Interval before retrieving a potentially refreshed envelope. | + +State that storage name `idl_env` is fixed by the integration. + +- [ ] **Step 3: Add the LiveRamp guide section** + +In `docs/guide/integrations/prebid.md`, document: + +- prerequisites: Placement ID, approved origin, CMP/LiveRamp consent posture, + and an external bundle containing `identityLinkIdSystem`; +- the exact TOML example and `ts prebid bundle` selection; +- operator ownership of the single `identityLink` entry for calls through the + supported public `pbjs.setConfig` and `pbjs.mergeConfig` APIs while preserving + other User ID modules; explicitly state that this is not a security boundary + against retained pre-wrapper references or direct internal mutation; +- asynchronous resolution: a new browser's first auction may have no RampID; +- the existing flow through `getUserIdsAsEids()`, `/auction`, + `user.ext.eids`, `ts-eids`, and EC/KV; +- degraded behavior for no consent, no recognition, missing module, LiveRamp + network failure, and KV failure; +- privacy guidance: envelopes are opaque and must not be logged; +- the explicit product boundary: this forwards RampID EIDs, not ATS Direct + audience segments; +- a credential-based manual validation checklist matching Section 11.5 of the + design spec, recording only booleans, counts, source names, and status codes. + +- [ ] **Step 4: Format and verify docs** + +Run: + +```bash +cd docs +npm run format +``` + +Expected: all documentation and TOML examples satisfy Prettier checks. + +- [ ] **Step 5: Commit Task 4** + +Run: + +```bash +git add trusted-server.example.toml docs/guide/integrations/prebid.md docs/guide/configuration.md +git commit -m "docs: explain managed LiveRamp RampID setup" +``` + +## Task 5: Run full verification and prepare live validation handoff + +**Files:** + +- Verify: all files changed in Tasks 1–4 +- Reference: `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md` + +- [ ] **Step 1: Run the complete TSJS test and build gates** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run +npm run format +npm run lint +node build-all.mjs +``` + +Expected: all commands exit 0. + +- [ ] **Step 2: Run Rust formatting and adapter test gates** + +From the repository root, run: + +```bash +cargo fmt --all -- --check +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: all commands exit 0. Do not substitute bare +`cargo test --workspace`. + +- [ ] **Step 3: Run all target-matched clippy gates** + +Run: + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +``` + +Expected: all commands exit 0 with warnings denied. + +- [ ] **Step 4: Re-run documentation formatting and inspect the final diff** + +Run: + +```bash +cd docs +npm run format +cd .. +git diff --check +git status --short +git diff main...HEAD --stat +``` + +Expected: formatting and diff checks pass; status contains only intentional +plan/implementation changes. + +- [ ] **Step 5: Record credential-gated validation status** + +If LiveRamp test configuration is available, execute the guide's manual +validation on an approved non-production origin and report only: + +- approved origin used (domain, not credentials); +- whether `idl_env` was created/refreshed; +- whether `getUserIdsAsEids()` exposed source `liveramp.com`; +- whether the controlled PBS request contained that source; +- whether a later request ingested the EID into the configured + `liveramp.com` EC partner; +- whether opt-out removed it; and +- whether an unapproved origin degraded to no LiveRamp EID without blocking + the auction; and +- status codes/counts without envelope values. + +If credentials remain unavailable, report exactly: “Code complete; live +LiveRamp validation pending IABTechLab/uid2-optout#385.” Do not block automated +verification or add fake live-success evidence. + +In either case, prepare the explicit parent-epic acceptance handoff: “RampID +identity envelopes traverse the existing Prebid auction path; ATS Direct +audience segments are not passed by this implementation.” + +- [ ] **Step 6: Commit any verification-only corrections** + +Only if verification required source changes, repeat the affected focused and +full gates, then commit the minimal correction: + +```bash +git add +git commit -m "fix: address LiveRamp verification findings" +``` + +Do not create an empty verification commit. + +## Correction tasks added after PR #1054 review (2026-08-24) + +These tasks implement the reviewed correction in the specification. Complete +them in order and preserve artifact-level evidence for configuration behavior. + +## Task 6: Characterize partial `userSync` updates + +**Files:** + +- Modify: `crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs:179` + +- [x] **Step 1: Test the review premise against the generated artifact** + +Preconfigure a publisher `sharedId`, install the shim, then call: + +```js +pageWindow.pbjs.setConfig({ userSync: { syncDelay: 50 } }) +``` + +Assert that `getConfig('userSync.userIds')` still contains `sharedId` and exactly +one managed `identityLink`, and that `getConfig('userSync.syncDelay')` is `50`. +This test uses the generated Prebid bundle and generated TSJS shim, not a mock +of `setConfig`. + +- [x] **Step 2: Verify actual pinned behavior** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run test/integrations/prebid/index.test.ts test/prebid-artifact-integration.test.mjs +``` + +Observed: all focused tests pass. The pinned Prebid artifact retains its +effective `userIds` list across the partial update. Mock-only tests that expected +the shim to inject `userIds` into the forwarded argument were discarded because +the mock does not model the shipped artifact's effective configuration behavior. +No production wrapper change is required. + +- [x] **Step 3: Commit the artifact characterization** + +```bash +git add \ + crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs +git commit -m "Characterize partial Prebid userSync updates" +``` + +## Task 7: Characterize exact default TCF enforcement + +**Files:** + +- Modify: `crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs:103-225` +- Modify: `docs/guide/integrations/prebid.md:134-141` +- Modify: `docs/guide/integrations/prebid.md:518-524` +- Modify: `docs/guide/integrations/prebid.md:578-588` + +- [x] **Step 1: Replace the combined consent boolean with independent grants** + +Change the fixture API to accept explicit grants with safe defaults: + +```js +function tcData({ + purpose1 = true, + purpose3 = true, + purpose4 = true, + vendor97 = true, +} = {}) { + return { + // existing CMP fields + purpose: { + consents: { 1: purpose1, 3: purpose3, 4: purpose4 }, + legitimateInterests: {}, + }, + vendor: { + consents: { [LIVE_RAMP_GVL_VENDOR_ID]: vendor97 }, + legitimateInterests: {}, + }, + } +} +``` + +Pass this object through `runGdprPage` without combining the grants. + +- [x] **Step 2: Add four independent artifact cases plus the granted baseline** + +Assert the exact pinned defaults: + +1. Purpose 1 denied alone: no LiveRamp request, no `idl_env`, no retry cookie. +2. Vendor 97 denied alone: no LiveRamp request, no `idl_env`, no retry cookie. +3. Purpose 3 denied alone: one LiveRamp request and `idl_env` written. +4. Purpose 4 denied alone: one LiveRamp request and `idl_env` written. +5. All relevant grants present: one LiveRamp request and `idl_env` written. + +- [x] **Step 3: Run the consent artifact suite** + +Run: + +```bash +cd crates/trusted-server-js/lib +npx vitest run test/prebid-consent-enforcement.test.mjs +``` + +Expected: all five behavioral cases and the module-presence assertion pass +against the real generated artifacts. If a case differs, inspect pinned Prebid +before changing the expected policy. + +- [x] **Step 4: Correct the operator-facing consent claims** + +Document that default client-side resolution/storage is blocked by Purpose 1 +and LiveRamp vendor consent. State that Purpose 3 has no standalone default +rule, Purpose 4 controls UFPD, and default EID transmission accepts qualifying +purpose/vendor basis from any Purpose 2–10 unless the publisher enables +`eidsRequireP4Consent`. Preserve the existing explicit GPP/US-state limitation. + +- [x] **Step 5: Format and verify the focused documentation** + +```bash +cd docs +npx prettier --write guide/integrations/prebid.md +npm run format +``` + +Expected: the guide is formatted and makes no broader enforcement claim than +the artifact matrix proves. + +- [x] **Step 6: Commit the consent characterization** + +```bash +git add \ + crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs \ + docs/guide/integrations/prebid.md +git commit -m "Clarify LiveRamp TCF enforcement defaults" +``` + +## Task 8: Verify and refresh PR #1054 + +**Files:** + +- Verify: all PR files +- Update externally: PR #1054 description + +- [x] **Step 1: Run TypeScript tests, build, and formatting** + +```bash +cd crates/trusted-server-js/lib +npx vitest run +node build-all.mjs +npm run lint +npm run format +``` + +Expected: every command exits 0. + +- [x] **Step 2: Run repository Rust and documentation gates** + +```bash +cd /Users/prk-jr/Desktop/opensource/rust/trusted-server +cargo fmt --all -- --check +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo clippy-cli +cargo clippy-codegen +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +./scripts/test-cli.sh +cd docs && npm run format +``` + +Expected: all commands exit 0. Do not use bare `cargo test --workspace`. + +- [x] **Step 3: Inspect the final branch** + +```bash +git diff --check +git status --short +git diff origin/main...HEAD --stat +git log origin/main..HEAD --oneline +``` + +Expected: no uncommitted source changes and only intentional LiveRamp commits. + +- [x] **Step 4: Request final code review** + +Review the entire `origin/main...HEAD` diff with special attention to the +partial-`userSync` real-artifact case and the independent consent matrix. Fix +all Critical or Important findings and repeat affected gates. + +- [x] **Step 5: Push and update the draft PR description** + +Push without force. Create `/tmp/pr-1054-body.md` with the repository PR +template and these exact sections: + +- Summary: managed RampID configuration, opaque `liveramp.com` EID transport, + exact TCF default behavior, and ATS Direct exclusion. +- Closes: `Closes #355`. +- Status: `Code complete; live LiveRamp validation pending +IABTechLab/uid2-optout#385.` +- Changes table containing every one of these final diff paths and no removed + `crates/trusted-server-core/src/auction/endpoints.rs` row: + - `.cargo/config.toml` + - `CLAUDE.md` + - `crates/trusted-server-cli/src/prebid_bundle.rs` + - `crates/trusted-server-core/src/consent/mod.rs` + - `crates/trusted-server-core/src/ec/prebid_eids.rs` + - `crates/trusted-server-core/src/integrations/prebid.rs` + - `crates/trusted-server-js/lib/build-prebid-external.mjs` + - `crates/trusted-server-js/lib/src/integrations/prebid/index.ts` + - `crates/trusted-server-js/lib/test/build-prebid-external.test.mjs` + - `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` + - `crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts` + - `crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs` + - `crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs` + - `docs/guide/configuration.md` + - `docs/guide/integrations/prebid.md` + - `docs/superpowers/plans/2026-08-21-liveramp-integration.md` + - `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md` + - `trusted-server.example.toml` +- Test plan: check every command completed in Steps 1–2; leave only live + credential validation unchecked. +- Hardening note: no config-derived regex or pattern compilation was added; + invalid enabled LiveRamp config fails typed validation. + +Verify the enumerated paths against +`git diff --name-only origin/main...HEAD` before writing the file. Apply the +body exactly with: + +```bash +git push origin issue-355-liveramp-integration +gh pr edit 1054 \ + --repo IABTechLab/trusted-server \ + --title "Add managed LiveRamp RampID integration" \ + --body-file /tmp/pr-1054-body.md +gh pr view 1054 \ + --repo IABTechLab/trusted-server \ + --json url,isDraft,headRefOid,body,statusCheckRollup +``` + +Do not mark the PR ready for review automatically; report the final readiness +assessment to the user first. diff --git a/docs/superpowers/plans/2026-08-21-pr-823-round-5-review-resolution.md b/docs/superpowers/plans/2026-08-21-pr-823-round-5-review-resolution.md new file mode 100644 index 000000000..9f3735d91 --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-pr-823-round-5-review-resolution.md @@ -0,0 +1,307 @@ +# PR 823 Round-5 Review Resolution Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve every actionable finding in PR 823 review `4989897698` while preserving generation compatibility and enforcing root-less template safety. + +**Architecture:** Keep browser-option defaults and legacy clap compatibility at the CLI boundary, carry borrowed-root evidence through template inference, and reject unsafe overrides before rendering. Improve diagnostics and validation at their existing seams, then pin cross-language and documentation invariants with focused tests. + +**Tech Stack:** Rust 2024, clap 4 derive, `url`, `toml`, embedded JavaScript, mdBook/VitePress documentation. + +--- + +## File Map + +- `crates/trusted-server-cli/src/commands/audit/collector.rs`: generation browser default constants and option defaults. +- `crates/trusted-server-cli/src/commands/audit/mod.rs`: hidden legacy browser arguments, early TOML validation, conversion to generation arguments. +- `crates/trusted-server-cli/src/run.rs`: clap contract tests. +- `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs`: collector defaults and formatting. +- `crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs`: borrowed-root inference metadata and root-gap refusal reasons. +- `crates/trusted-server-cli/src/commands/audit/generate/mod.rs`: redirect output, profile-scoped notes, merge-policy validation, explicit-pattern refusal. +- `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs`: timestamp-shaped volatile token recognition. +- `crates/trusted-server-cli/src/commands/audit/browser.rs`: Rust/JavaScript evidence-cap invariant test. +- `crates/trusted-server-cli/src/commands/audit/page.rs`: accurate final-URL/terminal-escaping test claims. +- `docs/guide/cli.md` and the volatile-collision design/plan: operator and historical documentation corrections. + +### Task 1: Restore generation browser defaults and legacy clap isolation + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` + +- [ ] **Step 1: Add failing clap and default tests** + +Add parser coverage proving that `ts audit --help` does not advertise generation +browser flags, `ts audit --chrome /tmp/chrome generate ...` is rejected, and the +legacy `ts audit --chrome ... --settle-max-ms ...` form still parses and +reaches `GenerateArgs`. Add a generation-option default assertion for 750 ms and +12,000 ms. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin run::tests::audit_ -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::tests::legacy_ -- --nocapture +``` + +Expected: the hidden/help and 12-second assertions fail on the current branch. + +- [ ] **Step 3: Implement one generation-default source and legacy mirror** + +Define generation-specific constants in `collector.rs` and use them in clap +attributes and `GenerateBrowserOpts::default`: + +```rust +pub(crate) const GENERATE_SETTLE_QUIET_MS: u64 = 750; +pub(crate) const GENERATE_SETTLE_MAX_MS: u64 = 12_000; +``` + +Use those constants in `BrowserAuditCollector::default`. Replace the flattened +`GenerateBrowserOpts` under `LegacyGenerateArgs` with `LegacyBrowserOpts`, whose +seven fields each use `hide = true, requires = "legacy_url"`. Implement +`From<&LegacyBrowserOpts> for GenerateBrowserOpts` and use it in +`legacy_generate_args`. Add the missing blank line between collector methods. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run the Step 2 commands and the focused collector default test. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/collector.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/run.rs +git commit -m "Preserve generation browser option contracts" +``` + +### Task 2: Enforce borrowed-root and merge-policy safety + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Add failing inference and end-to-end tests** + +Add tests proving: + +- `InferenceOutcome` identifies `ad-sidebar` as borrowing the root witnessed by + another slot; +- explicit `--page-pattern` values cause `run_update_slots` to fail before the + source config changes when any rendered template borrowed the root; +- no-policy inference gives affected multi-path slots the root-witness reason; +- a configured `section_segment = 1` with no `section_root` refuses inferred + segment 0 when preserved `{section}` slots exist; +- the same segment, or an unset segment, allows adopting the inferred root. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::unit_template::tests -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::tests::merge_ -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::tests::explicit_ -- --nocapture +``` + +Expected: borrowed stems are unavailable, explicit patterns are accepted, and +the configured-segment mismatch is accepted. + +- [ ] **Step 3: Carry borrowed stems and reject unsafe overrides** + +Add an ordered `borrowed_section_root: Vec` field to +`InferenceOutcome`. Populate it only when `RootUnwitnessed` successfully becomes +a template. Before building render slots, reject non-empty explicit patterns if +that vector is non-empty: + +```rust +return cli_error(format!( + "cannot apply --page-pattern to slot(s) {} because their {{section}} templates borrow section_root; remove --page-pattern so patterns can be derived from observed paths", + borrowed.join(", ") +)); +``` + +On the no-policy path, replace the generic multi-path refusal reason for +structurally valid root-unwitnessed slots with the specific missing-root-witness +reason. Preserve structural refusal reasons unchanged. + +Update `validate_merge_policy` so an explicit configured segment is compared +before the empty-root adoption return. Keep the guard limited to preserved +`{section}` slots and allow `--replace`. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run all Step 2 commands. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs crates/trusted-server-cli/src/commands/audit/generate/mod.rs +git commit -m "Protect borrowed section templates during generation" +``` + +### Task 3: Make redirects, warnings, and config errors actionable + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` + +- [ ] **Step 1: Add failing diagnostic tests** + +Strengthen the HTTPS-upgrade assertion to require +`http://publisher.example/` and `https://publisher.example/`. Add a two-profile +warning test whose output names desktop and mobile separately. Add a malformed +whole-document TOML test while retaining tests for unknown valid settings and an +unreadable `[creative_opportunities]` section. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin update_slots_accepts_a_same_host_https_upgrade -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin profile_warning -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin creative_config -- --nocapture +``` + +- [ ] **Step 3: Implement scoped diagnostics and early parse failure** + +Render redirect endpoints as `origin.ascii_serialization() + path`. Thread the +profile label into `fold_collected`; keep the consent-stub warning global, label +page warnings/interstitials with path and profile, and retain the existing +site-wide discovery-warning dedupe. + +Replace `.ok()` in `creative_config` with an error mapping that identifies a +malformed existing TOML document and explains that generation did not start. +Continue parsing into `toml::Value`, not runtime `Settings`, so valid unknown +settings remain tolerated. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run all Step 2 commands. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/mod.rs crates/trusted-server-cli/src/commands/audit/mod.rs +git commit -m "Clarify audit generation diagnostics" +``` + +### Task 4: Pin detector and embedded-collector invariants + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/page.rs` + +- [ ] **Step 1: Add failing invariant tests** + +Add a negative volatile-token test for `promo-20260820a-sidebar`, retain a +positive timestamp-shaped control with at least ten leading digits, and add the +embedded-JavaScript constant assertion: + +```rust +assert!( + AD_TEMPLATE_COLLECTOR_JS.contains(&format!( + "const __ts_max_entries = {MAX_EVIDENCE_ENTRIES}" + )), + "should keep the JS cap equal to MAX_EVIDENCE_ENTRIES" +); +``` + +In the page summary test, assert the exact percent-encoded final URL line and +limit the raw-control assertion's comment to title and warning fields. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin per_render_token -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin evidence_entries -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin page_controlled_text -- --nocapture +``` + +- [ ] **Step 3: Tighten the token shape and correct the test claim** + +Require at least ten leading digits in `is_per_render_token`. Keep the rest of +the recognizer unchanged. Add the evidence-cap test and page assertion without +removing final-URL escaping. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run all Step 2 commands. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs crates/trusted-server-cli/src/commands/audit/browser.rs crates/trusted-server-cli/src/commands/audit/page.rs +git commit -m "Pin audit evidence recognition invariants" +``` + +### Task 5: Align documentation and local style + +**Files:** + +- Modify: `docs/guide/cli.md` +- Modify: `docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md` +- Modify: `docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` + +- [ ] In the volatile-collision design example, remove the section-varying + sidebar from the list of omitted/explained slots because root-less templating + now writes it with a borrowed-root diagnostic. +- [ ] In the volatile-collision implementation plan, state that a recognized + render token must have a non-empty family prefix before it and placement + content after it; remove the broader "in any position" claim. +- [ ] Update the guide to say that a configured segment without a root is + preserved for existing templates, and document the explicit-pattern refusal + for borrowed-root slots. +- [ ] Add the missing `GenerateArgs.browser` doc comment, change the `expect` + message to the required `"should ..."` form, and retain the method-separation + blank line from Task 1. +- [ ] Run `cd docs && npm run format` and `cargo fmt --all -- --check`. +- [ ] Commit: + +```bash +git add docs/guide/cli.md docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md crates/trusted-server-cli/src/commands/audit/generate/mod.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +git commit -m "Align ad-template generation documentation" +``` + +### Task 6: Verify the complete review resolution + +**Files:** + +- Verify all files above. + +- [ ] Run focused audit generation tests: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate -- --nocapture +``` + +- [ ] Run the complete host CLI suite: + +```bash +./scripts/test-cli.sh aarch64-apple-darwin +``` + +- [ ] Run lint and formatting gates: + +```bash +cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets --all-features -- -D warnings +cargo fmt --all -- --check +cd docs && npm run format +git diff --check +``` + +- [ ] Inspect `git status --short`, `git log --oneline -6`, and the complete + diff from `073d5644` to ensure only the approved review resolution is present. +- [ ] Do not push or post GitHub replies without separate user authorization. diff --git a/docs/superpowers/plans/2026-08-24-ad-template-div-id-reconciliation.md b/docs/superpowers/plans/2026-08-24-ad-template-div-id-reconciliation.md new file mode 100644 index 000000000..2ec2f7979 --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-ad-template-div-id-reconciliation.md @@ -0,0 +1,295 @@ +# Ad-template div-ID reconciliation implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Preserve observed numeric sibling creative opportunities during config merge and refuse singleton div IDs containing shorter high-entropy per-render tokens. + +**Architecture:** Reconciliation will use the normalized identities already retained by `EvidenceTable` to distinguish observed literals from intentional configured prefixes. GPT discovery will keep its vendor-neutral, position-aware volatile-family classifier and add a conservative eight-leading-digit/eight-character-suffix alternative without changing existing ten-digit behavior. + +**Tech Stack:** Rust 2024, `BTreeSet`, existing Trusted Server CLI evidence/merge pipeline, Cargo unit and browser integration tests. + +--- + +### Task 1: Preserve observed literal siblings during merge + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` + +- [ ] **Step 1: Write failing numeric-sibling merge tests** + +Add focused tests beside the existing prefix tests: + +```rust +#[test] +fn observed_literal_does_not_claim_numeric_siblings() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-sidebar-1\"\ndiv_id = \"ad-sidebar-1\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10", "ad-sidebar-11"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!(merged.len(), 3); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-10")); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-11")); + assert!(diagnostics.notes.is_empty()); +} +``` + +Add a second regression with an unrelated existing slot and discovered +`ad-sidebar-1` followed by `ad-sidebar-10`. It must prove a newly appended +observed literal cannot absorb a later sibling. Keep +`merge_reports_when_a_broad_prefix_claims_multiple_discovered_divs` unchanged as +the positive intentional-prefix control. + +- [ ] **Step 2: Run the tests and verify RED** + +Run: + +```bash +cargo test -p trusted-server-cli observed_literal_does_not_claim_numeric_siblings -- --nocapture +cargo test -p trusted-server-cli newly_appended_literal_does_not_claim_numeric_sibling -- --nocapture +``` + +Expected: both fail because `ad-sidebar-1` absorbs the longer discovered IDs. + +- [ ] **Step 3: Implement exact-first, evidence-aware prefix matching** + +In `merge_render_slots_with_observed_diagnostics`, build a borrowed set from +`observed_div_ids` once: + +```rust +let observed_literals = observed_div_ids + .iter() + .map(String::as_str) + .collect::>(); +``` + +Thread `&observed_literals` through discovered-slot reconciliation and +observed/unobserved classification. Refactor the matcher so it: + +1. searches all merged slots for an exact stable-key match; +2. returns that exact match immediately; +3. searches for the longest prefix only among prefixes absent from + `observed_literals`; and +4. retains configuration order for equal-length prefix ties. + +Use the same helper for seeding `observed_existing`, so merge behavior and stale +diagnostics cannot disagree. Keep exact matching available for configured slots +that omit `div_id` and therefore resolve through `id`. + +Update the `MergeDiagnostics` field comment from “raw crawl” to “normalized +evidence.” + +- [ ] **Step 4: Add and run the normalization-boundary regression** + +Use `discover_gpt_slots` plus `merge_slots` to show that a live +`ad-header-0-_R_3f_` identity normalizes to `ad-header-0`, and therefore makes +configured `ad-header-0` an observed literal rather than a prefix for a distinct +`ad-header-01` slot. Do not pass collector-level raw IDs into the merge. + +Run: + +```bash +cargo test -p trusted-server-cli normalized_stem_is_the_literal_merge_boundary -- --nocapture +``` + +Expected after implementation: PASS. + +- [ ] **Step 5: Run focused merge tests and verify GREEN** + +Run: + +```bash +cargo test -p trusted-server-cli slot_toml::tests -- --nocapture +``` + +Expected: all merge tests pass, including the existing intentional broad-prefix +test. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +git commit -m "Preserve observed literal ad slot siblings" +``` + +### Task 2: Refuse eight-digit, long-suffix volatile tokens + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` + +- [ ] **Step 1: Add failing shorter-token registry and request tests** + +Add singleton cases using a synthetic shape: + +```rust +const SHORT_VOLATILE_DIV: &str = + "vendor-tag_12345678AbCdEfGhIjKl_slot_overlay_1"; +``` + +Assert both registry and GAMPAD request discovery: + +- retain `had_slot_evidence`; +- produce no writable slots; and +- emit the existing volatile-family warning naming `vendor-tag`. + +- [ ] **Step 2: Run the tests and verify RED** + +Run: + +```bash +cargo test -p trusted-server-cli shorter_high_entropy_singleton -- --nocapture +``` + +Expected: FAIL because the current classifier requires ten leading digits and +accepts the eight-digit token literally. + +- [ ] **Step 3: Add failing classifier boundary tests** + +Extend the table-driven tests so these remain eligible: + +```text +vendor-tag_1234567AbCdEfGh_slot_inarticle_1 # seven leading digits +vendor-tag_12345678AbCdEfG_slot_inarticle_1 # seven-character suffix +promo-20260820a-sidebar # short calendar suffix +vendor-tag_1234567890123456_slot_inarticle_1 # bare numeric segment +``` + +Add `vendor-tag_12345678AbCdEfGh_slot_inarticle_1` to the volatile table. Run +the two boundary tests and confirm only the new 8+8 volatile assertion fails. + +- [ ] **Step 4: Implement the conservative alternative token shape** + +Keep the current all-ASCII-alphanumeric requirement and compute the suffix +length after the leading digit run. A segment is per-render when either: + +```rust +(leading_digits >= 10 && suffix_length >= 1) + || (leading_digits >= 8 && suffix_length >= 8) +``` + +Keep the existing requirement that the token occurs before another div-ID +segment. Do not add a vendor name or family-specific regular expression. + +- [ ] **Step 5: Run GPT discovery tests and verify GREEN** + +Run: + +```bash +cargo test -p trusted-server-cli gpt_slots::tests -- --nocapture +``` + +Expected: all discovery, normalization, collision, registry, request, and +boundary tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +git commit -m "Reject shorter high-entropy ad slot tokens" +``` + +### Task 3: Verify the complete change + +**Files:** + +- No source changes expected. + +- [ ] **Step 1: Run formatting and diff checks** + +```bash +cargo fmt --all -- --check +git diff --check +cd docs && npm run format +``` + +Expected: all exit zero and formatting makes no changes. + +- [ ] **Step 2: Run the complete CLI suite** + +```bash +./scripts/test-cli.sh +``` + +Expected: all unit, config overlay, proxy E2E, and ignored real-Chrome fixtures +pass. The browser portions require permission to bind loopback listeners. + +- [ ] **Step 3: Run host-target CLI clippy** + +```bash +cargo clippy \ + --manifest-path crates/trusted-server-cli/Cargo.toml \ + --target "$(rustc -vV | sed -n 's/^host: //p')" \ + --all-targets -- -D warnings +``` + +Expected: the changed CLI crate and all of its test targets lint without +warnings. The adapter-scoped aliases below do not include this crate. + +- [ ] **Step 4: Run repository target-specific Rust gates** + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: every command exits zero with no warnings promoted to errors. + +- [ ] **Step 5: Run parity and JavaScript/docs gates** + +```bash +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +(cd crates/trusted-server-js/lib && npx vitest run) +(cd crates/trusted-server-js/lib && npm run format) +(cd docs && npm run format) +``` + +Expected: parity, Vitest, and formatting checks pass. + +- [ ] **Step 6: Review branch state** + +```bash +git status --short +git log --oneline --decorate -10 +``` + +Expected: clean feature worktree with the two implementation commits above the +approved design/plan commits. + +- [ ] **Step 7: Validate against the operator's dry-run output** + +Ask the operator to rerun the established desktop/mobile `--scroll --dry-run` +command with a current DataDome cookie. Confirm: + +- there is no `ad-sidebar-1` broad-prefix collision note; +- numeric sidebar siblings are emitted as distinct slots; +- the singleton mobile volatile-family slot is refused; and +- older configured volatile-family slots remain named as preserved but + unobserved until the operator deliberately prunes them. diff --git a/docs/superpowers/plans/2026-08-24-ad-template-generate-scroll-staleness.md b/docs/superpowers/plans/2026-08-24-ad-template-generate-scroll-staleness.md new file mode 100644 index 000000000..db6f22ce2 --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-ad-template-generate-scroll-staleness.md @@ -0,0 +1,358 @@ +# Ad-template Generate Scroll and Staleness Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add opt-in scrolling to `ts audit ad-templates generate` and warn when a normal merge preserves configured slots that the current crawl did not observe. + +**Architecture:** Thread one parsed `--scroll` value through the generation browser session and reuse a shared deterministic scroll primitive before the generator's final evidence scrape. Extend merge reconciliation with structured diagnostics that record unmatched pre-existing slot IDs; format the warning at the command layer so it can account for whether scrolling was already enabled without changing merge behavior. + +**Tech Stack:** Rust 2024, clap, chromiumoxide/CDP, Tokio, existing CLI and Chrome-fixture test harnesses, rustfmt, clippy, Prettier. + +--- + +## File map + +- Create `crates/trusted-server-cli/src/commands/audit/browser_scroll.rs`: shared deterministic scroll primitive. +- Modify `crates/trusted-server-cli/src/commands/audit/mod.rs`: declare the shared module, parse `--scroll`, and wire it into generation. +- Modify `crates/trusted-server-cli/src/commands/audit/browser.rs`: reuse shared scrolling while retaining verifier-only phase marking. +- Modify `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs`: carry scroll state, scroll and re-settle, and test lazy GPT discovery. +- Modify `crates/trusted-server-cli/src/commands/audit/generate/mod.rs`: carry scroll context and render contextual stale-slot notes. +- Modify `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs`: report unmatched preserved slots from the authoritative merge matcher. +- Modify `crates/trusted-server-cli/src/run.rs`: test parsing and defaults. +- Modify `scripts/test-cli.sh`: run the new ignored Chrome fixture. +- Modify `docs/guide/cli.md`: document both behaviors. + +### Task 1: Parse and wire generation scrolling + +**Files:** + +- Modify: `crates/trusted-server-cli/src/run.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing parsing tests** + +Extend `audit_generate_subcommands_use_generation_settle_defaults` with +`assert!(!generate.scroll)`. Add: + +```rust +#[test] +fn audit_ad_templates_generate_parses_scroll() { + let args = parse(&[ + "ts", "audit", "ad-templates", "generate", + "https://www.example.com/", "--scroll", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(generate), + )) = audit.command else { + panic!("expected audit ad-templates generate command"); + }; + assert!(generate.scroll, "--scroll should enable generation scrolling"); +} +``` + +- [ ] **Step 2: Run the focused test and verify it fails** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" audit_ad_templates_generate_parses_scroll +``` + +Expected: compilation fails because `AuditAdTemplatesGenerateArgs` has no +`scroll` field. + +- [ ] **Step 3: Add the flag and session wiring** + +Add to `AuditAdTemplatesGenerateArgs`: + +```rust +/// Perform a deterministic scroll pass after each page initially settles. +#[arg(long)] +pub scroll: bool, +``` + +Add `scroll: bool` to `BrowserAuditCollector` and `SessionSettings`, default it +to false, and add `with_scroll(bool)`. Thread it through `session()`, +`with_browser`, `collect_page_from_browser`, and `collect_open_page`; Task 2 +will use it. + +Add `scroll: bool` to `UpdateSlotsRequest`. In `run_audit`, set both the +collector option and request field from `gen_args.scroll`. Update every test +fixture constructing `UpdateSlotsRequest` with `scroll: false`, except the later +contextual-warning test. + +- [ ] **Step 4: Run parsing/default tests** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" audit_generate_subcommands_use_generation_settle_defaults +cargo test --package trusted-server-cli --target "$HOST_TARGET" audit_ad_templates_generate_parses_scroll +``` + +Expected: both pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/run.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs crates/trusted-server-cli/src/commands/audit/generate/mod.rs +git commit -m "Add scroll option to ad-template generation" +``` + +### Task 2: Share and execute deterministic scrolling + +**Files:** + +- Create: `crates/trusted-server-cli/src/commands/audit/browser_scroll.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `scripts/test-cli.sh` + +- [ ] **Step 1: Add a failing Chrome fixture** + +Add a self-contained tall HTML page whose scroll listener installs a stub GPT +registry and defines `/123/lazy` in `ad-lazy-0` only after `window.scrollY > 0`. +Add this ignored test: + +```rust +#[test] +#[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] +fn collects_lazy_gpt_slot_only_when_scroll_is_enabled() { + if !browser_fixture_available() { + return; + } + let url = lazy_gpt_fixture_url(); + let without_scroll = BrowserAuditCollector::default() + .collect_page(&url, &[]) + .expect("should collect without scrolling"); + let with_scroll = BrowserAuditCollector::default() + .with_scroll(true) + .collect_page(&url, &[]) + .expect("should collect with scrolling"); + + assert!(without_scroll.gpt_slots.is_empty()); + assert!(with_scroll.gpt_slots.iter().any(|slot| { + slot.gam_unit_path == "/123/lazy" && slot.div_id == "ad-lazy-0" + })); +} +``` + +Use loopback HTTP instead of `file://` if Chrome requires it for reliable scroll +events. Change `scripts/test-cli.sh` to run the ignored +`commands::audit::generate::browser_collector::tests::` prefix so lifecycle and +lazy-slot fixtures are both covered. + +- [ ] **Step 2: Run the fixture and verify it fails** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +TS_AUDIT_BROWSER_TESTS=1 cargo test --package trusted-server-cli --target "$HOST_TARGET" collects_lazy_gpt_slot_only_when_scroll_is_enabled -- --ignored --test-threads=1 +``` + +Expected: the scrolled result still lacks `/123/lazy`. + +- [ ] **Step 3: Implement the shared primitive** + +Create `browser_scroll.rs` with a `ScrollFailure` enum (evaluation failure and +timeout) and: + +```rust +pub(crate) async fn scroll_page(page: &chromiumoxide::Page) -> Vec { + let mut failures = Vec::new(); + for fraction in ["0.33", "0.66", "1"] { + let script = format!( + "window.scrollTo(0, Math.floor(Math.max(document.body.scrollHeight, document.documentElement.scrollHeight) * {fraction}))" + ); + evaluate(page, script, &mut failures).await; + tokio::time::sleep(Duration::from_millis(250)).await; + } + evaluate(page, "window.scrollTo(0, 0)".to_string(), &mut failures).await; + failures +} +``` + +Bound each evaluation at five seconds. Declare the module in `audit/mod.rs`. +In `browser.rs`, leave the pre-scroll evidence snapshot and +`window.__tsScrollPhase = true` marker in place, replace the local step loop with +the shared function, and map failures to existing `Warning` output. + +In the generation collector, after initial settle but before final HTML/GPT/ +network/link scraping, call the shared function when `scroll` is true, append +its failures as page warnings, and call `wait_for_page_settle` again. A second +settle timeout is a warning, not a discarded page. + +- [ ] **Step 4: Run browser tests** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::browser::tests:: +TS_AUDIT_BROWSER_TESTS=1 cargo test --package trusted-server-cli --target "$HOST_TARGET" collects_lazy_gpt_slot_only_when_scroll_is_enabled -- --ignored --test-threads=1 +``` + +Expected: all pass and `/123/lazy` appears only with scrolling. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/browser_scroll.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/commands/audit/browser.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs scripts/test-cli.sh +git commit -m "Collect lazy ad slots during generation scroll" +``` + +### Task 3: Report unmatched slots preserved by merge + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing diagnostic tests** + +Next to `merge_keeps_existing_only_slots`, assert that diagnostic merging marks +preserved `sidebar` but not rediscovered `header`; multiple missing IDs retain +configuration order; and full rediscovery, empty existing slots, and +`--replace` produce no stale IDs. + +Add command tests with fake collectors and in-memory writers. Assert non-scroll +wording contains `or --scroll`, scroll wording omits that retry, stdout remains +only diff/summary content, and preserved slots remain in candidate TOML. + +- [ ] **Step 2: Run focused tests and verify they fail** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" merge_reports_preserved_unobserved_slots +cargo test --package trusted-server-cli --target "$HOST_TARGET" update_slots_reports_preserved_unobserved_slots +``` + +Expected: failures because unmatched existing slots are not exposed. + +- [ ] **Step 3: Add structured merge diagnostics** + +Define: + +```rust +#[derive(Debug, Default, PartialEq, Eq)] +pub(super) struct MergeDiagnostics { + pub(super) notes: Vec, + pub(super) unobserved_existing_slot_ids: Vec, +} +``` + +Change `merge_render_slots_with_diagnostics` to return this structure with the +merged slots. Record every matched existing index in a `BTreeSet`, then +collect unmatched existing IDs by enumerating configuration order. Preserve the +current broad-prefix messages in `notes`. The `replace || existing.is_empty()` +early path returns default diagnostics. Keep `merge_render_slots` returning only +the slot vector. + +- [ ] **Step 4: Format the contextual note in `run_update_slots`** + +Extend pending notes with `merge_diagnostics.notes`. If unmatched IDs exist, +append their count and comma-separated IDs. End with: + +```rust +let follow_up = if request.scroll { + "Re-run with broader page/profile coverage; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." +} else { + "Re-run with broader coverage or --scroll; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." +}; +``` + +Do not change the merged configuration. `emit_notes` remains the only terminal +sanitization/output boundary. + +- [ ] **Step 5: Run merge and command tests** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::slot_toml::tests::merge_ +cargo test --package trusted-server-cli --target "$HOST_TARGET" update_slots_reports_preserved_unobserved_slots +``` + +Expected: all pass, with unchanged merged TOML and warnings only on stderr. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs crates/trusted-server-cli/src/commands/audit/generate/mod.rs +git commit -m "Warn about preserved unobserved ad slots" +``` + +### Task 4: Document and verify + +**Files:** + +- Modify: `docs/guide/cli.md` + +- [ ] **Step 1: Document both behaviors** + +Add a generation `--scroll` example under “Bounding and steering the crawl.” +Explain that every page/profile scrolls after initial settle and settles again, +and that it is opt-in because it adds time, requests, and publisher side effects. + +Update merge documentation: missing existing slots are preserved and named on +stderr; absence may reflect coverage, targeting, or lazy loading; only +`--replace` intentionally prunes them. + +- [ ] **Step 2: Format docs and inspect scope** + +```bash +cd docs && npm run format +git diff --check +git diff -- docs/guide/cli.md +``` + +Expected: formatting passes and only intended docs change. + +- [ ] **Step 3: Run the full CLI harness, including Chrome fixtures** + +```bash +./scripts/test-cli.sh +``` + +Expected: all host CLI and configured ignored browser tests pass. + +- [ ] **Step 4: Run formatting and lint gates** + +```bash +cargo fmt --all -- --check +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +``` + +Expected: all exit zero without warnings. + +- [ ] **Step 5: Run adapter regression suites** + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: all pass. Do not use bare `cargo test --workspace`. + +- [ ] **Step 6: Review scope and commit docs** + +```bash +git status --short +git diff --check +git diff HEAD -- crates/trusted-server-cli scripts/test-cli.sh docs/guide/cli.md +``` + +Confirm `fastly.toml` remains untouched and issue #1059 produced no code changes. +Then: + +```bash +git add docs/guide/cli.md +git commit -m "Document generation scroll and stale-slot warnings" +``` diff --git a/docs/superpowers/plans/2026-08-24-request-phase-timing.md b/docs/superpowers/plans/2026-08-24-request-phase-timing.md new file mode 100644 index 000000000..02cda64d8 --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-request-phase-timing.md @@ -0,0 +1,1039 @@ +# Request Phase Timing Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Every application response attributes its own server time by phase via a +Server-Timing header and a sampled Tinybird access-telemetry row. + +**Architecture:** A core `RequestTimings` handle (Arc-shared, infallible recording) +collects phase spans always-on; the Fastly adapter freezes and emits at +`send_edgezero_response` immediately before `into_parts()`; a post-send emitter ships +one NDJSON row to the Tinybird Events API with a bounded, 2xx-validated await. + +**Tech Stack:** Rust 2024, `edgezero` HTTP types, Fastly Compute (wasm32-wasip1, +Viceroy tests), Axum (native tests), Tinybird Events API. + +**Spec:** `docs/superpowers/specs/2026-08-24-request-phase-timing-design.md`: the +plan argues from the spec; executors read both. Spec section numbers are cited per +task. + +## Global Constraints + +- Errors use `error-stack` (`Report`); errors defined with + `derive_more::Display`; never thiserror, never anyhow (except the Spin entry point). +- No `unwrap()` in production code; `expect("should ...")` only. Assertion messages + `"should ..."`. Tests use Arrange-Act-Assert. +- No inline comments; comments on their own line above the code. +- Functions never exceed 7 arguments; use a struct instead (this bit + `ec_finalize_response` in review; the timings handle travels inside existing state). +- No local imports inside functions; `use super::*` only in `#[cfg(test)]`. +- Only example/fictional data in tests and docs (`example.com` domains). +- Recording is infallible: saturating math, lock failure drops the sample, no panics + (spec 5, 13). +- Vendor identity never appears in emitted surfaces: the filter span is `ts-filter` + (spec 3). +- Test commands: `cargo test-axum` (native, fast inner loop), `cargo test-fastly` + (Viceroy) for adapter tasks. Before PR handoff: the full CI gate list in + `CLAUDE.md`. +- Commit style: sentence case, imperative, no prefixes, no trailers. + +## File Structure + +| File | Responsibility | +| ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/request_timing.rs` (new) | `Phase`, `AuctionWaitPlacement`, `RequestTimings`, `PhaseSpan`, `TimingSnapshot`, header rendering | +| `crates/trusted-server-core/src/access_telemetry.rs` (new) | `RouteClass`, `publisher_route_template`, `AccessTelemetrySnapshot`, `AccessEventRow` NDJSON | +| `crates/trusted-server-core/src/geo.rs` (modify) | `GeoLookupState` response-extension type | +| `crates/trusted-server-core/src/settings.rs` (modify) | `ObservabilitySettings`, tinybird flag decoupling, access validation | +| `crates/trusted-server-core/src/publisher.rs` (modify) | `ts-origin`, `ts-template-cache`, auction-wait spans | +| `crates/trusted-server-core/src/ec/kv.rs` (modify) | `ts-kv` at the graph abstraction | +| `crates/trusted-server-adapter-fastly/src/main.rs` (modify) | T0, appbuild span, freeze point, `DeliveryOutcome`, post-send emission ordering | +| `crates/trusted-server-adapter-fastly/src/app.rs` (modify) | filter span, geo span + `GeoLookupState` attach, route class assignment | +| `crates/trusted-server-adapter-fastly/src/middleware.rs` (modify) | finalize consumes `GeoLookupState` | +| `crates/trusted-server-adapter-fastly/src/tinybird.rs` (modify) | access sink with confirmed delivery | +| `crates/trusted-server-adapter-axum/src/` (modify) | terminal freeze layer, header emission | +| `tinybird/datasources/access_logs_raw.datasource` (modify) | phase-column schema, non-null sorting key | +| `trusted-server.example.toml` (modify) | `[observability]`, tinybird keys | + +Out of scope for this plan: the Grafana dashboard JSON (separate telemetry repo, +spec 11) and Cloudflare/Spin emission wiring (spec non-goal). + +--- + +### Task 1: Core `RequestTimings` + +**Files:** + +- Create: `crates/trusted-server-core/src/request_timing.rs` +- Modify: `crates/trusted-server-core/src/lib.rs` (add `pub mod request_timing;`) +- Test: same file, `#[cfg(test)]` + +**Interfaces:** + +- Consumes: nothing (leaf module; `std::time`, `std::sync`). +- Produces (later tasks rely on these exact names): + - `pub enum Phase { AppBuild, Filter, Geo, EcKv, Origin, TemplateCacheLookup, AuctionWait, Stream }` + - `pub enum AuctionWaitPlacement { PreHeader, InStream }` + - `#[derive(Clone)] pub struct RequestTimings` with: + - `pub fn new() -> Self` + - `pub fn record(&self, phase: Phase, dur: Duration)` (saturating accumulate) + - `pub fn record_auction_wait(&self, placement: AuctionWaitPlacement, dur: Duration)` + - `pub fn span(&self, phase: Phase) -> PhaseSpan` (records on drop) + - `pub fn mark_headers_ready(&self)` (first call wins) + - `pub fn mark_request_elapsed(&self)` (first call wins) + - `pub fn set_resp_bytes(&self, bytes: u64)` + - `pub fn server_timing_value(&self) -> Option` + - `pub fn snapshot(&self) -> TimingSnapshot` + - `pub struct TimingSnapshot { pub time_elapsed_ms: Option, pub request_elapsed_ms: Option, pub appbuild_ms: Option, pub filter_ms: Option, pub geo_ms: Option, pub kv_ms: Option, pub origin_ms: Option, pub template_cache_ms: Option, pub auction_wait_ms: Option, pub stream_ms: Option, pub auction_wait_placement: Option, pub resp_bytes: Option }` + +- [ ] **Step 1: Write the failing tests** + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn render_omits_unrecorded_phases_and_orders_total_first() { + let timings = RequestTimings::new(); + timings.record(Phase::Filter, Duration::from_micros(9_100)); + timings.mark_headers_ready(); + let value = timings + .server_timing_value() + .expect("should render after mark_headers_ready"); + assert!( + value.starts_with("ts-total;dur="), + "should lead with ts-total: {value}" + ); + assert!(value.contains("ts-filter;dur=9.1"), "should render one decimal: {value}"); + assert!(!value.contains("ts-geo"), "should omit unrecorded phases: {value}"); + } + + #[test] + fn render_returns_none_before_headers_ready() { + let timings = RequestTimings::new(); + timings.record(Phase::Geo, Duration::from_millis(1)); + assert!(timings.server_timing_value().is_none(), "should require the snapshot"); + } + + #[test] + fn repeated_phases_accumulate_saturating() { + let timings = RequestTimings::new(); + timings.record(Phase::Geo, Duration::from_millis(2)); + timings.record(Phase::Geo, Duration::from_millis(3)); + timings.mark_headers_ready(); + let snapshot = timings.snapshot(); + assert_eq!(snapshot.geo_ms, Some(5), "should accumulate repeats"); + } + + #[test] + fn mark_headers_ready_is_first_call_wins() { + let timings = RequestTimings::new(); + timings.mark_headers_ready(); + let first = timings.snapshot().time_elapsed_ms; + std::thread::sleep(Duration::from_millis(5)); + timings.mark_headers_ready(); + assert_eq!(timings.snapshot().time_elapsed_ms, first, "should not restamp"); + } + + #[test] + fn span_guard_records_on_drop() { + let timings = RequestTimings::new(); + { + let _span = timings.span(Phase::Origin); + std::thread::sleep(Duration::from_millis(2)); + } + timings.mark_headers_ready(); + assert!( + timings.snapshot().origin_ms.expect("should record on drop") >= 1, + "should measure elapsed span time" + ); + } + + #[test] + fn auction_wait_records_placement() { + let timings = RequestTimings::new(); + timings.record_auction_wait(AuctionWaitPlacement::PreHeader, Duration::from_millis(40)); + let snapshot = timings.snapshot(); + assert_eq!(snapshot.auction_wait_ms, Some(40), "should record wait"); + assert_eq!( + snapshot.auction_wait_placement, + Some(AuctionWaitPlacement::PreHeader), + "should record placement" + ); + } + + #[test] + fn rendered_names_never_include_vendor_terms() { + let timings = RequestTimings::new(); + for phase in [Phase::AppBuild, Phase::Filter, Phase::Geo, Phase::EcKv, Phase::Origin, Phase::TemplateCacheLookup] { + timings.record(phase, Duration::from_millis(1)); + } + timings.mark_headers_ready(); + let value = timings.server_timing_value().expect("should render"); + assert!(!value.to_ascii_lowercase().contains("datadome"), "should mask vendors"); + } +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-axum -p trusted-server-core request_timing` +Expected: compile FAIL, module does not exist. + +- [ ] **Step 3: Implement** + +```rust +//! Per-request phase timing collection and Server-Timing rendering. +//! +//! Collection is always-on and infallible: saturating math, lock failure +//! drops the sample, no panics. See the design spec +//! `docs/superpowers/specs/2026-08-24-request-phase-timing-design.md`. + +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +const PHASE_COUNT: usize = 8; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Phase { + AppBuild, + Filter, + Geo, + EcKv, + Origin, + TemplateCacheLookup, + AuctionWait, + Stream, +} + +impl Phase { + fn index(self) -> usize { /* match self -> 0..=7 */ } + + /// Header entry name; row-only phases return None. + fn header_name(self) -> Option<&'static str> { + match self { + Self::AppBuild => Some("ts-appbuild"), + Self::Filter => Some("ts-filter"), + Self::Geo => Some("ts-geo"), + Self::EcKv => Some("ts-kv"), + Self::Origin => Some("ts-origin"), + Self::TemplateCacheLookup => Some("ts-template-cache"), + Self::AuctionWait | Self::Stream => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AuctionWaitPlacement { + PreHeader, + InStream, +} + +struct Inner { + t0: Instant, + phases: [Option; PHASE_COUNT], + headers_ready_total: Option, + request_elapsed: Option, + auction_wait_placement: Option, + resp_bytes: Option, +} + +#[derive(Clone)] +pub struct RequestTimings(Arc>); +``` + +Implementation notes (all bodies in this task, none deferred): + +- Every method takes `if let Ok(mut inner) = self.0.try_lock()` and silently + returns otherwise: contention and poison both drop the sample instead of waiting, + per the infallibility constraint. +- `record` accumulates with `saturating_add` semantics + (`Some(existing.saturating_add(dur))`). +- `mark_headers_ready` and `mark_request_elapsed` write `t0.elapsed()` only when the + slot is `None`. +- `server_timing_value` returns `None` unless `headers_ready_total` is set; renders + `ts-total` first from the stored snapshot, then the six header phases in enum order + with `{:.1}` millisecond formatting (`dur.as_secs_f64() * 1000.0`). +- `PhaseSpan { timings: RequestTimings, phase: Phase, started: Instant }`; `Drop` + calls `record(self.phase, self.started.elapsed())`. +- `TimingSnapshot` converts each `Duration` with + `u32::try_from(dur.as_millis()).unwrap_or(u32::MAX)`. +- `impl Default for RequestTimings` delegates to `new()`. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-axum -p trusted-server-core request_timing` +Expected: all 7 tests PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/request_timing.rs crates/trusted-server-core/src/lib.rs +git commit -m "Add RequestTimings phase collection and Server-Timing rendering" +``` + +--- + +### Task 2: Settings: `[observability]`, tinybird decoupling, access validation + +**Files:** + +- Modify: `crates/trusted-server-core/src/settings.rs` +- Modify: `trusted-server.example.toml` + +**Interfaces:** + +- Produces: + - `pub struct ObservabilitySettings { pub server_timing_enabled: bool }` as + `settings.observability`, `#[serde(default)]` on the field and + `#[serde(skip_serializing_if = "ObservabilitySettings::is_default")]`. + - `TinybirdSettings.auction_enabled: bool` (`#[serde(default = "default_true")]`). + - `prepare_runtime` validation: `access_enabled` requires `enabled`, non-empty + `api_host`, `secret_store`, `access_dataset`, `access_token_secret`, + `max_body_bytes > 0`, and `access_sample_rate > 0.0`. + +- [ ] **Step 1: Write the failing tests** (in `settings.rs` tests module) + +```rust +#[test] +fn observability_defaults_off_and_serializes_away() { + let settings = create_test_settings(); + assert!(!settings.observability.server_timing_enabled, "should default off"); + let toml = toml::to_string(&settings).expect("should serialize settings"); + assert!( + !toml.contains("[observability]"), + "should omit the default table so a prior binary can parse the config" + ); +} + +#[test] +fn access_enabled_requires_positive_sample_rate() { + // access_enabled = true with access_sample_rate = 0 is armed-but-silent: an error. + let err = settings_from_toml_with( + "[tinybird]\nenabled = true\napi_host = \"api.example.com\"\naccess_enabled = true\naccess_sample_rate = 0.0\n", + ) + .expect_err("should reject armed-but-silent access telemetry"); + assert!(format!("{err:?}").contains("access_sample_rate"), "should name the field"); +} + +#[test] +fn access_and_auction_emission_are_independent() { + let settings = settings_from_toml_with( + "[tinybird]\nenabled = true\napi_host = \"api.example.com\"\nauction_enabled = false\naccess_enabled = true\naccess_sample_rate = 1.0\n", + ) + .expect("should accept access without auction"); + assert!(!settings.tinybird.auction_enabled, "should disable auction emission"); + assert!(settings.tinybird.access_enabled, "should enable access emission"); +} + +#[test] +fn auction_enabled_defaults_true_for_existing_configs() { + let settings = settings_from_toml_with("[tinybird]\nenabled = true\napi_host = \"api.example.com\"\n") + .expect("should parse a pre-decoupling config"); + assert!(settings.tinybird.auction_enabled, "should preserve current behavior"); +} +``` + +Also REPLACE the existing rejection test +(`tinybird_access_enabled_is_rejected_until_emitter_is_wired`, `settings.rs:4123`) +with a wiring test asserting a fully-specified access config is accepted. + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-axum -p trusted-server-core observability access_enabled auction_enabled` +Expected: compile FAIL (`observability` field missing). + +- [ ] **Step 3: Implement** + +- Add `ObservabilitySettings` (derive `Debug, Clone, Default, PartialEq, Deserialize, +Serialize`, `#[serde(deny_unknown_fields)]`), with + `fn is_default(&self) -> bool { *self == Self::default() }`. +- Add the `observability` field to `Settings` with the serde attributes above. +- Add `auction_enabled` to `TinybirdSettings` with `default_true()`; update + `Default for TinybirdSettings`. +- Extend `TinybirdSettings::prepare_runtime` with the access validation matrix; error + messages name the failing field (`"tinybird.access_sample_rate must be > 0 when +access_enabled"` and so on). +- `trusted-server.example.toml`: add a commented `[observability]` block with + `server_timing_enabled = false` present-but-false and the env-override note (the + overlay cannot create a missing leaf), plus `auction_enabled`/access keys in the + tinybird section comments. +- Gate the auction sink: in `crates/trusted-server-adapter-fastly/src/app.rs`, + `auction_sink_from_settings` condition becomes + `settings.tinybird.enabled && settings.tinybird.auction_enabled`. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-axum -p trusted-server-core` then `cargo test-fastly` (the sink gate +touches the Fastly adapter). +Expected: PASS, including the replaced wiring test. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/settings.rs trusted-server.example.toml crates/trusted-server-adapter-fastly/src/tinybird.rs crates/trusted-server-adapter-fastly/src/app.rs +git commit -m "Add observability settings and decouple tinybird access and auction emission" +``` + +--- + +### Task 3: Fastly freeze point, header emission, `DeliveryOutcome` + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (entry T0, appbuild span, + `send_edgezero_response`) +- Test: `crates/trusted-server-adapter-fastly/src/app.rs` tests module (route-level + tests run under Viceroy) + +**Interfaces:** + +- Consumes: `RequestTimings`, `Phase` (Task 1); + `trusted_server_core::cache_policy::cache_control_headers_are_private_or_no_store`. +- Produces: + - `RequestTimings` inserted into request extensions at dispatch + (`core_req.extensions_mut().insert(timings.clone())`), alongside the existing + `config_store`/`device_signals`/`client_info` inserts. + - `send_edgezero_response(response, effects, timings) -> DeliveryOutcome` where + `pub(crate) struct DeliveryOutcome { pub bytes: u64, pub result: DeliveryResult }` + and `pub(crate) enum DeliveryResult { Complete, Error }` (streaming partial + detection lands in Task 6). + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn server_timing_emitted_on_private_response_when_enabled() { + // Arrange: settings with observability.server_timing_enabled = true; publisher + // route fixture whose response is Cache-Control: private, no-store. + // Act: dispatch through the full adapter path. + // Assert: + let header = response_header(&response, "server-timing").expect("should emit header"); + assert!(header.contains("ts-total;dur="), "should carry the stored total"); + assert_eq!( + header.matches("ts-total").count(), 1, + "should emit exactly one TS-owned metric set" + ); +} + +#[test] +fn server_timing_absent_when_flag_off() { /* same fixture, flag false: no ts-total */ } + +#[test] +fn server_timing_absent_on_cacheable_responses() { + // tsjs route (public, max-age=31536000, immutable) and a bare max-age=60 response: + // both must carry no ts-total even with the flag on. +} + +#[test] +fn preexisting_server_timing_values_survive() { + // Fixture response already carrying Server-Timing: upstream;dur=1 stays present + // alongside the appended TS set. +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-fastly server_timing` +Expected: FAIL, no header emitted. + +- [ ] **Step 3: Implement** + +In `edgezero_main` (`main.rs`): + +```rust +let timings = RequestTimings::new(); +{ + let _appbuild = timings.span(Phase::AppBuild); + // existing: open_trusted_server_config_store() + build_app_with_state() +} +``` + +Move the config-store open inside the span scope. Insert `timings.clone()` into +request extensions before dispatch. Thread the handle into both send sites and the +error paths by value (it is a cheap clone). + +In `send_edgezero_response`, immediately before `response.into_parts()`: + +```rust +timings.mark_headers_ready(); +let conclusively_private = + cache_control_headers_are_private_or_no_store(response.headers()); +if settings_enabled_server_timing && conclusively_private { + if let Some(value) = timings.server_timing_value() { + match HeaderValue::from_str(&value) { + Ok(header_value) => { + response.headers_mut().append(header::SERVER_TIMING, header_value); + } + Err(error) => log::warn!("skipping server-timing header: {error}"), + } + } +} +``` + +`settings_enabled_server_timing` arrives inside a small +`SendContext { timings: RequestTimings, server_timing_enabled: bool }` so the +function stays at or under seven parameters. Return `DeliveryOutcome` with per-mode +semantics: buffered bodies capture the byte count from the body length before +`send_to_client()` (which returns no delivery result) and report complete-on-return; +the streaming branch gains a counting writer in Task 6. Existing callers ignore the +outcome in this task (Task 8 consumes it). + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-fastly` and `cargo clippy-fastly` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-adapter-fastly/src/main.rs crates/trusted-server-adapter-fastly/src/app.rs +git commit -m "Emit Server-Timing at the send freeze point on conclusively private responses" +``` + +--- + +### Task 4: Filter span and geo span with `GeoLookupState` dedupe + +**Files:** + +- Modify: `crates/trusted-server-core/src/geo.rs` (add `GeoLookupState`) +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` + (`run_pre_route_filters` wrapper, `build_ec_request_state` geo span + state attach) +- Modify: `crates/trusted-server-adapter-fastly/src/middleware.rs` and `main.rs` + (`resolve_geo_for_response` consumes carried state) + +**Interfaces:** + +- Consumes: `RequestTimings` from request extensions (Task 3). +- Produces: + - `pub enum GeoLookupState { NotAttempted, Attempted, Resolved(GeoInfo) }` in + `trusted_server_core::geo`, attached as a response extension on every exit path + that attempted a lookup (including the asset fallback). + - `resolve_geo_for_response` gains the carried state as input: live lookup only on + `NotAttempted`; `Attempted` is never retried; fallback lookups are wrapped in + `timings.span(Phase::Geo)`. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn finalize_reuses_request_phase_geo_without_second_lookup() { + // Counting geo stub: dispatch a publisher route; assert lookup count == 1 and + // x-geo-country still set on the response. +} + +#[test] +fn failed_lookup_is_not_retried() { + // Stub returns None once; assert GeoLookupState::Attempted carried and the + // finalize path performs zero further lookups. +} + +#[test] +fn asset_fallback_carries_geo_state_without_ec_finalize_state() { + // Asset route: response extension holds GeoLookupState, EcFinalizeState absent. +} + +#[test] +fn filter_span_recorded_when_request_filter_runs() { + // Registry fixture with a test request filter; assert snapshot().filter_ms is Some. +} + +#[test] +fn geo_lookup_skipped_for_unauthorized_responses() { + // Existing 401 rule preserved: no lookup, state NotAttempted. +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-fastly geo_ filter_span` +Expected: FAIL (lookup count 2; no `GeoLookupState`). + +- [ ] **Step 3: Implement** + +- `GeoLookupState` derives `Debug, Clone`; store in response extensions from the + dispatch layer right after `build_ec_request_state` resolves (or fails) its lookup. +- Wrap the `build_ec_request_state` lookup and any finalize fallback lookup in + `timings.span(Phase::Geo)` (accumulating slot handles the repeat case). +- Wrap `run_pre_route_filters` (`app.rs:751`) in `timings.span(Phase::Filter)`, + recording only when at least one filter is registered (skip the span when the + registry has no request filters, so the header omits `ts-filter` on unconfigured + deployments). +- `resolve_geo_for_response(response, carried: &GeoLookupState, client_ip, lookup)` + keeps the 401 short-circuit first, then matches the carried state. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-fastly` and `cargo test-axum` +Expected: PASS including untouched existing geo header tests. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/geo.rs crates/trusted-server-adapter-fastly/src/app.rs crates/trusted-server-adapter-fastly/src/middleware.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Record filter and geo spans and dedupe the per-request geo lookup" +``` + +--- + +### Task 5: Core spans: origin, template cache, KV abstraction + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` (origin send ~4496, template + cache lookup ~4391 on current main) +- Modify: `crates/trusted-server-core/src/ec/kv.rs` (graph-level `ts-kv`) +- Test: `publisher.rs` and `ec/kv.rs` test modules + +**Interfaces:** + +- Consumes: `RequestTimings` read from request extensions inside + `handle_publisher_request`; `KvIdentityGraph` gains + `pub fn with_timings(self, timings: RequestTimings) -> Self` (builder-style, + optional field), set where the graph is constructed in `main.rs`. +- Produces: `origin_ms`, `template_cache_ms`, `kv_ms` populated in snapshots. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn origin_span_covers_the_publisher_fetch() { + // Stubbed origin with a small injected delay; assert snapshot().origin_ms is Some. +} + +#[test] +fn template_cache_span_recorded_only_when_lookup_runs() { + // Inline mode fixture: template_cache_ms None. Shared-mode eligible fixture: + // template_cache_ms Some. +} + +#[test] +fn kv_span_accumulates_across_graph_operations() { + // Stub KV recording two operations through a TimedKvStore-wrapped graph; assert + // kv_ms Some and covers both (accumulated, not last-write). +} + +#[test] +fn consent_store_reads_are_timed_and_pull_sync_is_not() { + // Consent read through the decorated RuntimeServices store: kv_ms Some. + // Pull-sync graph built from the untimed store: records nothing. +} + +#[test] +fn ec_finalize_kv_lands_before_freeze() { + // Adapter-level (test-fastly): EC-enabled fixture with eids cookies; assert the + // emitted header contains ts-kv, proving the freeze point sits after finalize. +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-axum -p trusted-server-core origin_span template_cache_span kv_span` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +- `handle_publisher_request` reads the handle: + `let timings = req.extensions().get::().cloned().unwrap_or_default();` + (a defaulted handle records into nothing that ever renders, keeping non-adapter + tests unchanged). +- Origin: `let origin_span = timings.span(Phase::Origin);` immediately before + `services.http_client().send(platform_request).await`; `drop(origin_span)` when the + response headers are available (directly after the `match` arm binds the response). +- Template cache: same guard pattern around + `services.template_cache().lookup_or_reserve(key).await`. +- KV: add `TimedKvStore` (new type in `crates/trusted-server-core/src/platform/`), + a decorator implementing `PlatformKvStore` that wraps `Arc` + plus a `RequestTimings` handle and records `Phase::EcKv` around every trait + method. Every request-path `KvIdentityGraph` construction site (request setup, + identify, admin lookup, batch sync, finalization) receives the timed store; + consent-store access through `RuntimeServices` uses the same decorator; pull-sync + constructs its graph from the untimed store explicitly (add a test asserting the + pull-sync store records nothing). `ec_finalize_response` keeps seven arguments: + the handle rides inside the store the graph already receives. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-axum` then `cargo test-fastly` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs crates/trusted-server-core/src/ec/kv.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Record origin, template cache, and KV phase spans in core" +``` + +--- + +### Task 6: Body-phase capture: stream, auction wait placement, bytes + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` (seam wait + buffered wait) +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (stream drive timing, + `DeliveryOutcome.bytes`) +- Test: `publisher.rs` tests + adapter tests + +**Interfaces:** + +- Consumes: `record_auction_wait` (Task 1), `DeliveryOutcome` (Task 3). +- Produces: `stream_ms`, `auction_wait_ms` + placement, `resp_bytes`, + `mark_request_elapsed()` called by the adapter immediately after the stream drive + returns. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn streaming_seam_wait_records_in_stream_placement() { + // Streaming fixture with a delayed auction: placement InStream, and + // stream_ms >= auction_wait_ms. +} + +#[test] +fn buffered_template_miss_records_pre_header_placement() { + // Shared-template authorized miss (buffered finalizer): placement PreHeader; the + // wait is recorded even though headers had not committed. +} + +#[test] +fn delivery_outcome_reports_bytes_and_request_elapsed_set() { + // Adapter: after send, snapshot has resp_bytes Some(body_len) and + // request_elapsed_ms Some; request_elapsed excludes post-send emitter time by + // construction (asserted by ordering test in Task 8). +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-fastly seam_wait buffered_template delivery_outcome` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +- Streaming path: around the `collect_stream_auction(...)` await inside the body + stream, measure with `Instant::now()` and call + `timings.record_auction_wait(AuctionWaitPlacement::InStream, waited)`. The handle + reaches the stream closure through `OwnedProcessResponseParams`/assembly params (it + is `Clone`; add a field). +- Buffered path (`buffer_publisher_response_async` and the shared-template miss + finalizer): same measurement with `AuctionWaitPlacement::PreHeader`. +- Adapter stream drive: wrap the `block_on(stream_asset_body(...))` region with a + counting writer that tallies bytes and observes truncation/error, record + `Phase::Stream` with the elapsed drive time, populate `DeliveryOutcome` with + bytes and Complete/Partial/Error, call `timings.set_resp_bytes(bytes)` and + `timings.mark_request_elapsed()` immediately after the drive returns, before + anything else post-send. Buffered responses keep the Task 3 complete-on-return + semantics; `body_mode` distinguishes the regimes in the row. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-fastly` and `cargo test-axum` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Capture stream duration, auction wait placement, and response bytes" +``` + +--- + +### Task 7: `AccessTelemetrySnapshot`, route class, route template + +**Files:** + +- Create: `crates/trusted-server-core/src/access_telemetry.rs` +- Modify: `crates/trusted-server-core/src/lib.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (RouteMetadata attach at + handler wrappers), `main.rs` (snapshot build at freeze point), + `crates/trusted-server-core/src/publisher.rs` (typed template-cache state + extension) + +**Interfaces:** + +- Consumes: `TimingSnapshot` (Task 1), `GeoLookupState` (Task 4). +- Produces: + - `pub enum RouteClass { PublisherHtml, Tsjs, IntegrationProxy, Ec, AuctionApi, Other }` + with `pub fn as_str(&self) -> &'static str` (snake_case values from the spec). + - `pub fn publisher_route_template(path: &str) -> String`: `/` plus first segment + filtered to `[a-z0-9_-]`, truncated to 32 chars, plus `/*` when deeper; empty or + disallowed first segments render `/other/*`. + - `pub struct AccessTelemetrySnapshot { pub method: String, pub status: u16, pub route_class: RouteClass, pub route_template: String, pub publisher_domain: String, pub env: String, pub service_id: String, pub pop: String, pub ts_version: String, pub country: String, pub template_cache_state: String, pub body_mode: &'static str, pub sample_rate: f64 }` + - `pub fn access_event_row(snapshot: &AccessTelemetrySnapshot, timings: &TimingSnapshot, event_ts_epoch_ms: u64) -> String` (one NDJSON line). + +- [ ] **Step 1: Write the failing tests** (adversarial, per spec 9) + +```rust +#[test] +fn admin_ec_route_template_never_contains_the_identifier() { + // Named-route template comes from the route table: "/_ts/admin/ec/{id}". + // Assert a row built for that route never contains a 64-hex EC id fixture. +} + +#[test] +fn publisher_paths_normalize_to_coarse_templates() { + assert_eq!(publisher_route_template("/news/some-article-slug"), "/news/*"); + assert_eq!(publisher_route_template("/"), "/"); + assert_eq!( + publisher_route_template("/user@example.com/profile"), + "/other/*", + "should reject non-allowlisted characters" + ); + assert_eq!( + publisher_route_template(&format!("/{}", "a".repeat(500))), + format!("/{}", "a".repeat(32)), + "should bound segment length" + ); + assert_eq!(publisher_route_template("/search terms here"), "/other/*"); +} + +#[test] +fn row_serializes_nulls_for_missing_phases() { + // Sparse TimingSnapshot: absent phases serialize as JSON null, dimension fields + // never null (unknown sentinel). +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-axum -p trusted-server-core access_telemetry route_template` +Expected: compile FAIL. + +- [ ] **Step 3: Implement** + +Row serialization via `serde_json::json!` mapping spec section 9 column names exactly +(`time_elapsed_ms`, `appbuild_ms`, ..., `auction_wait_placement` as +`pre_header|in_stream|none`). `pop`/`service_id` read from Fastly env +(`FASTLY_SERVICE_ID`, `FASTLY_POP`), defaulting `"unknown"`; `env` derived by the +adapter from `FASTLY_IS_STAGING` (the `x-ts-env` input), never from `Settings`. +Route identity travels as a typed `RouteMetadata` response extension +(`pub struct RouteMetadata { pub route_class: RouteClass, pub route_template: String }` +in `access_telemetry.rs`): each named-route handler wrapper attaches its matched +route-table pattern verbatim, and the fallback and tsjs handlers attach their class +plus the coarse template; the freeze point consumes the extension (no `RouteClass` +column in `NAMED_ROUTES`, no reconstruction from a handler enum). Also in this task: +make `TemplateCacheResponseState` a typed response extension in `publisher.rs`, set +at every point that writes `x-ts-template-cache` so header and extension cannot +drift; the row reads the extension. The snapshot is built unconditionally in +`send_edgezero_response` right after `mark_headers_ready()` and returned inside +`DeliveryOutcome` (add field `pub snapshot: AccessTelemetrySnapshot`). + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-axum` and `cargo test-fastly` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/access_telemetry.rs crates/trusted-server-core/src/lib.rs crates/trusted-server-adapter-fastly/src/app.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Add access telemetry snapshot, route classes, and coarse route templates" +``` + +--- + +### Task 8: Access sink with confirmed delivery + post-send ordering + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/tinybird.rs` (access sink) +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (post-send ordering) + +**Interfaces:** + +- Consumes: `AccessTelemetrySnapshot` + `access_event_row` (Task 7), settings flags + (Task 2), `DeliveryOutcome` (Tasks 3/6). +- Produces: `pub(crate) async fn emit_access_event(client: &FastlyPlatformHttpClient, target: &TinybirdEventsTarget, row: String) -> Result<(), Report>`, + sending via the adapter's stateless platform client (the blocking variant, + post-delivery), checking `response.status().is_success()`, warning with status + otherwise. The transport context is adapter-owned and route-independent (target + derived from settings once at entry), so asset, admin, and error responses emit + without `RuntimeServices` or `EcFinalizeState`. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn access_emitter_posts_ndjson_and_validates_2xx() { + // RecordingHttpClient returning 202: assert URI is /v0/events?name=access_logs_raw, + // body is the row, Authorization bearer from the secret stub. +} + +#[test] +fn access_emitter_warns_and_drops_on_non_2xx() { + // RecordingHttpClient returning 422: emit returns Err naming the status; no retry + // request recorded (exactly one request seen). +} + +#[test] +fn sampled_out_requests_emit_nothing() { + // access_sample_rate stub decision false: RecordingHttpClient sees zero requests. +} + +#[test] +fn post_send_order_is_elapsed_then_pull_sync_then_telemetry() { + // Instrumented stubs record call order; assert request_elapsed snapshot precedes + // pull-sync dispatch which precedes the telemetry send. +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-fastly access_emitter post_send_order` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +- Reuse `TinybirdEventsTarget` with a second constructor + `from_access_config(config: TinybirdSettings)` using `access_dataset` and + `access_token_secret`. +- Sampling decision: `fn sampled_in(rate: f64, entropy: u64) -> bool` where entropy is + derived from the event timestamp nanos XOR a per-request counter (no `rand` + dependency; document that uniformity is approximate and sufficient). +- `main.rs` post-send, in order: `timings.mark_request_elapsed()` (already placed in + Task 6), existing pull-sync dispatch unchanged, then when + `settings.tinybird.enabled && settings.tinybird.access_enabled` and sampled in: + build the row from `outcome.snapshot` + `timings.snapshot()`, call + `emit_access_event`, log one warning on `Err`. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-fastly` and `cargo clippy-fastly` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-adapter-fastly/src/tinybird.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Emit confirmed access telemetry rows after pull-sync post-send" +``` + +--- + +### Task 9: Tinybird datasource schema + +**Files:** + +- Modify: `tinybird/datasources/access_logs_raw.datasource` + +**Interfaces:** + +- Consumes: column names exactly as serialized by `access_event_row` (Task 7). +- Produces: the deployed schema contract for the dashboard (separate repo). + +- [ ] **Step 1: Rewrite the schema** per spec section 9: keep + `event_ts DateTime64(3)`, `method`, `status UInt16`, `time_elapsed_ms UInt32`, + `sample_rate Float64` + 30-day TTL; add the columns from spec 9 with + dimension columns non-nullable `LowCardinality(String)` and phase columns + `Nullable(UInt32)`; drop `path` and `cache_state`; set + `ENGINE_SORTING_KEY "toDate(event_ts), service_id, publisher_domain, env, route_class, pop, status"` + (`event_date` was dropped for the sorting-key expression; see spec section 9). + +- [ ] **Step 2: Validate** with the tinybird toolchain if available locally + (`tb check` / project tests under `tinybird/tests`); otherwise assert the file + parses by review and rely on rollout step 4's remote verification. Add a fixture row + in `tinybird/fixtures` matching `access_event_row` output. + +- [ ] **Step 3: Commit** + +```bash +git add tinybird/datasources/access_logs_raw.datasource tinybird/fixtures +git commit -m "Extend access_logs_raw with phase columns and a non-null sorting key" +``` + +--- + +### Task 10: Axum adapter emission + +**Files:** + +- Modify: `crates/trusted-server-adapter-axum/src/` (terminal layer at the response + serialization boundary; locate the equivalent of the Fastly send path) +- Test: axum adapter tests (`cargo test-axum`) + +**Interfaces:** + +- Consumes: `RequestTimings`, header emission helper. Extract the emission block from + Task 3 into a shared core helper so both adapters call one function: + `pub fn append_server_timing_if_private(response: &mut Response, timings: &RequestTimings, enabled: bool)` + in `request_timing.rs` (move the Fastly inline logic here and re-point Task 3's call + site). +- Produces: Axum responses carry the header under the same conservative predicate; + `ts-appbuild` absent by construction (state built at startup); router-generated + 404/405 covered by the terminal layer; `/health` excluded by route match. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn axum_emits_header_on_private_response() { /* flag on, private response: ts-total present, ts-appbuild absent */ } + +#[test] +fn axum_404_carries_header_when_private() { /* router-generated 404 passes through the terminal layer */ } + +#[test] +fn axum_health_is_excluded() { /* /health: no ts-total */ } +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `cargo test-axum axum_emits axum_404 axum_health` +Expected: FAIL. + +- [ ] **Step 3: Implement** an outer service wrapper around the `RouterService` + inside `AxumDevServer` (not router middleware, which router-generated 404/405 + responses bypass and which returns before body serialization): create + `RequestTimings::new()` per request in the wrapper, insert into request + extensions, and on the wrapper's response side call `mark_headers_ready()` + + `append_server_timing_if_private(...)`, skipping the `/health` path by match. + +- [ ] **Step 4: Run to verify pass** + +Run: `cargo test-axum` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-adapter-axum/src crates/trusted-server-core/src/request_timing.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Emit Server-Timing from the Axum terminal layer with adapter-specific semantics" +``` + +--- + +### Task 11: Full gate, docs, and PR + +- [ ] **Step 1: Docs.** Add a short operator section to `docs/guide/configuration.md`: + the `[observability]` flag, the tinybird access keys, the deploy/rollback ordering + from spec section 12 (binary first, config second; config first on rollback), and + the conservative emission rule. Run `cd docs && npm run format`. + +- [ ] **Step 2: Full CI gate list** from `CLAUDE.md`: + `cargo fmt --all -- --check`; all six clippy aliases; `test-fastly`, `test-axum`, + `test-cloudflare`, `test-spin`; the integration-tests parity suite; JS build/test + and formats. Cloudflare/Spin compile the new core modules (collection only), which + is exactly what the non-goal requires. + +- [ ] **Step 3: Commit docs, push the branch, open the implementation PR** referencing + the spec PR #1069 and issue #1068, with the rollout section of the spec quoted as + the deployment checklist (staging pass-through + MISS/HIT replay before production + flag-on). + +--- + +## Self-Review + +- Spec coverage: sections 5 (Task 1), 12 (Task 2), 7 (Tasks 3, 10), 8/8a (Tasks 4, + 10), 6 (Tasks 4-6), 9 (Tasks 7, 9), 10 (Task 8), 13 (Tasks 1, 3, 8), 14 (test + steps throughout), 15 steps 1-4 (Task 11 + deployment checklist). Section 11 + (dashboard) is explicitly out of scope for this repo's plan. +- Type consistency: `RequestTimings`/`TimingSnapshot`/`RouteClass`/ + `AccessTelemetrySnapshot`/`DeliveryOutcome` names and signatures match across + Tasks 1, 3, 6, 7, 8, 10. +- Known intentional deferral: `DeliveryResult::Partial` detection is named in Task 3 + and wired when the stream drive reports bytes in Task 6; no other deferrals. diff --git a/docs/superpowers/plans/2026-08-26-auction-timeline-offsets.md b/docs/superpowers/plans/2026-08-26-auction-timeline-offsets.md new file mode 100644 index 000000000..6dfff6c8c --- /dev/null +++ b/docs/superpowers/plans/2026-08-26-auction-timeline-offsets.md @@ -0,0 +1,64 @@ +# Auction Timeline Offsets Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Record three T0-anchored auction milestones (dispatched, resolved, committed) plus the auction id on `RequestTimings`, and emit them as four additive columns on the `access_logs_raw` row. + +**Architecture:** Follows spec section 18 exactly. All state lives in the existing `RequestTimings` inner (same `try_lock`/first-call-wins/saturating model as `mark_headers_ready`); the row builder reads the values from `TimingSnapshot`, so no new emission path and no adapter changes. + +**Tech Stack:** Rust (core crate only), Tinybird datasource file. + +**Spec:** `docs/superpowers/specs/2026-08-24-request-phase-timing-design.md` section 18. + +## Global Constraints + +- Marks are first-call-wins; `try_lock` only; a contended lock drops the sample. +- Null offsets mean "no auction ran", never zero. `auction_id` sentinel is `none`. +- Column names: `auction_dispatched_ms`, `auction_resolved_ms`, `auction_committed_ms`, `auction_id`; JSONPaths `json:$.`; FORWARD_QUERY extended in the same order. +- Dispatch mark records only on `DispatchAuctionOutcome::Dispatched`; a failed dispatch leaves all three offsets null (the auction dataset still records the failure). +- No header emission, no config surface, no changes outside `trusted-server-core` and `tinybird/`. + +--- + +### Task 1: RequestTimings marks and snapshot fields + +**Files:** + +- Modify: `crates/trusted-server-core/src/request_timing.rs` + +**Interfaces:** + +- Produces: `mark_auction_dispatched(&self, auction_id: String)`, `mark_auction_resolved(&self)`, `mark_auction_committed(&self)`; `TimingSnapshot { auction_dispatched_ms, auction_resolved_ms, auction_committed_ms: Option, auction_id: Option, .. }` + +- [ ] Add `auction_dispatched`, `auction_resolved`, `auction_committed: Option` and `auction_id: Option` to `Inner`; initialize `None`. +- [ ] Add the three mark methods, first-call-wins on their own field, storing `inner.t0.elapsed()`; dispatched also stores the id first-call-wins. +- [ ] Map all four into `TimingSnapshot` via `duration_ms` / clone. +- [ ] Tests: first-call-wins per mark; snapshot maps offsets and id; unmarked snapshot yields all `None`. +- [ ] `cargo test-fastly request_timing`, commit. + +### Task 2: Publisher call sites + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + +**Interfaces:** + +- Consumes: Task 1 methods; `observation.auction_id` (`AuctionObservationContext`), in scope at the dispatch site. + +- [ ] In the `DispatchAuctionOutcome::Dispatched` arm (~line 4341): `timings.mark_auction_dispatched(observation.auction_id.to_string());` +- [ ] After both `record_auction_wait` calls (collect sites ~3952 and ~4012): `mark_auction_resolved()`. +- [ ] After both `write_bids_to_state` calls (~3954 and ~4017): `mark_auction_committed()`. +- [ ] `cargo test-fastly`, commit. + +### Task 3: Row columns and datasource + +**Files:** + +- Modify: `crates/trusted-server-core/src/access_telemetry.rs` +- Modify: `tinybird/datasources/access_logs_raw.datasource` + +- [ ] `access_event_row`: add the three offset keys (nullable) and `auction_id` with `none` sentinel, after the existing phase keys. +- [ ] Extend `row_serializes_nulls_for_missing_phases` and `row_serializes_recorded_phases_as_numbers` for the new keys. +- [ ] Datasource: four schema columns with JSONPaths (`Nullable(UInt32)` ×3, `String`), appended at the end of SCHEMA and FORWARD_QUERY so existing column order stays stable. +- [ ] Full gates: fmt, clippy (all six), test-fastly/axum/cloudflare/spin, parity. Commit. diff --git a/docs/superpowers/plans/2026-08-27-pr-1079-review-remediation.md b/docs/superpowers/plans/2026-08-27-pr-1079-review-remediation.md new file mode 100644 index 000000000..33c6a2832 --- /dev/null +++ b/docs/superpowers/plans/2026-08-27-pr-1079-review-remediation.md @@ -0,0 +1,154 @@ +# PR 1079 Review Remediation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve every review finding on PR 1079 and produce an `rc/202608`-based staging branch containing the corrected implementation. + +**Architecture:** Keep the first-claimant state machine, but make suppression token-local and correlation navigation/element-local. The first suppressed delivery closes registration while preserving every already-registered losing token until navigation or element replacement. Compose GPT/Prebid refresh wrappers explicitly, and centralize pre-response creative freshness validation plus safe authenticated-shell expansion. + +**Tech Stack:** TypeScript, Vitest/jsdom, Playwright, esbuild, Rust workspace validation, Git. + +--- + +### Task 1: First-impression token semantics + +**Files:** + +- Modify: `crates/trusted-server-js/lib/src/core/types.ts` +- Modify: `crates/trusted-server-js/lib/src/core/first_impression.ts` +- Test: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` + +- [ ] **Step 1: Add failing overlap and late-token tests** + +Add tests named `suppresses every publisher auction registered before the first TS delivery` and `suppresses a correlated TS-owned delivery after the five-second lease`. Assert two pre-registered callbacks are both suppressed, a later auction proceeds, and a fake-timer callback after 5 seconds remains suppressed. + +- [ ] **Step 2: Run the focused tests and verify RED** + +Run: `cd crates/trusted-server-js/lib && npx vitest run test/integrations/prebid/index.test.ts -t "registered before|five-second lease"` + +Expected: FAIL because `suppressionConsumed` permits the second delivery and expiry deletes the late token. + +- [ ] **Step 3: Implement token-local suppression** + +Replace `suppressionConsumed` with a claim-level `publisherRegistrationClosed` flag. Set it on the first suppressed delivery; do not consult it when consuming tokens already registered. Retain unresolved TS-owned suppressing tokens as non-evictable tombstones while generation and exact element identity match, including across timeout and auction failure; prune publisher-owned expired tokens and remove suppressing tombstones only on navigation or element replacement. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run the Step 2 command. Expected: PASS. + +- [ ] **Step 5: Commit the state-machine checkpoint** + +Run: `git add crates/trusted-server-js/lib/src/core/types.ts crates/trusted-server-js/lib/src/core/first_impression.ts crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts && git commit -m "fix(js): make first impression suppression auction local"` + +### Task 2: Prebid request and delivery correlation + +**Files:** + +- Modify: `crates/trusted-server-js/lib/src/integrations/prebid/index.ts` +- Test: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` + +- [ ] **Step 1: Add five failing Prebid regressions** + +Add tests named `consumes late-handoff suppression when Prebid suppresses the same delivery`, `limits a global request to opts.adUnitCodes`, `forwards only unsuppressed excluded slots`, `rejects pending delivery state from a previous navigation`, and `rejects pending delivery state after physical element replacement`. Assert the next legitimate refresh survives composed wrappers; only the selected global unit is mutated/claimed/correlated; a suppressed slot is absent from the native mixed refresh; and stale records neither suppress nor directly forward the new physical slot. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: `cd crates/trusted-server-js/lib && npx vitest run test/integrations/prebid/index.test.ts -t "late-handoff|opts.adUnitCodes|unsuppressed excluded|previous navigation|physical element replacement"` + +Expected: FAIL on the current wrapper, scoping, forwarding, and stale-correlation behavior. + +- [ ] **Step 3: Implement scoped, physical correlation** + +When `opts.adUnits` is absent and `opts.adUnitCodes` is an array, filter `pbjs.adUnits` before snapshotting, mutation, claiming, and correlation. Stamp `PendingPublisherBid` and `PendingPublisherCode` with `navGeneration` and the exact resolved `HTMLElement`; accept them only if generation, element identity, connectivity, DOM lookup, and target-slot resolution still match. Retain still-current suppressing correlations as tombstones. When Prebid suppresses a slot, clear the matching `gptSlotHandoffs` one-shot flag. In the no-auction/excluded branch call native GPT with `forwardedSlots`, not the original list. + +- [ ] **Step 4: Run the full Prebid test file and verify GREEN** + +Run: `cd crates/trusted-server-js/lib && npx vitest run test/integrations/prebid/index.test.ts`. Expected: PASS. + +- [ ] **Step 5: Commit the Prebid checkpoint** + +Run: `git add crates/trusted-server-js/lib/src/integrations/prebid/index.ts crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts && git commit -m "fix(js): scope publisher delivery correlation"` + +### Task 3: Creative freshness and nested shell repair + +**Files:** + +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` +- Test: `crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts` +- Test: `crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts` + +- [ ] **Step 1: Add failing stale-response and nested-shell tests** + +Change `does not resize a stale cache response after navigation` to assert zero port posts, zero successful-response evidence, and zero billing beacons. Add `expands every collapsed ancestor through the authenticated slot root`, with iframe -> 1x1 inner wrapper -> 1x1 outer wrapper -> authenticated root. Add/extend the browser scenario to assert all clipping ancestors have the winning dimensions. + +- [ ] **Step 2: Run focused GPT tests and verify RED** + +Run: `cd crates/trusted-server-js/lib && npx vitest run test/integrations/gpt/ad_init.test.ts -t "stale cache response|every collapsed ancestor"` + +Expected: FAIL because stale cache data is posted and only the immediate parent is resized. + +- [ ] **Step 3: Validate before creative side effects** + +Create one helper that checks current generation, winning bid/renderer ownership, authenticated source iframe identity, connectivity, and containment. Invoke it immediately before every APS or ADM `postMessage`; return before successful-response diagnostics, `markUsed`, or billing on failure. + +- [ ] **Step 4: Expand the authenticated shell safely** + +Require finite positive dimensions no larger than 10,000. Require the source iframe to retain its 1x1 attributes and collapsed computed dimensions. Preflight every ancestor through the authenticated root, rejecting detached/foreign roots, `body`/`html`, fixed/sticky positioning, and anchor/vignette/interstitial markers. Then resize the iframe and each ancestor whose width or height remains collapsed; never mutate outside the authenticated root. + +- [ ] **Step 5: Run GPT unit and browser tests** + +Run: `cd crates/trusted-server-js/lib && npx vitest run test/integrations/gpt/ad_init.test.ts` + +Run: `cd crates/trusted-server-integration-tests/browser && npx playwright test tests/shared/aps-renderer.spec.ts` + +Expected: PASS. + +- [ ] **Step 6: Commit the renderer checkpoint** + +Run: `git add crates/trusted-server-js/lib/src/integrations/gpt/index.ts crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts && git commit -m "fix(js): reject stale creatives and expand nested shells"` + +### Task 4: Full verification + +- [ ] **Step 0: Commit the reviewed design and plan** + +Run: `git add docs/superpowers/specs/2026-08-27-pr-1079-review-remediation-design.md docs/superpowers/plans/2026-08-27-pr-1079-review-remediation.md && git commit -m "docs: plan PR 1079 review remediation"`. + +- [ ] **Step 1: Run JS gates** + +Run from `crates/trusted-server-js/lib`: `npm run format && npm run lint && npx vitest run && node build-all.mjs`. Run the relevant Playwright suite with the command established in Task 3. Expected: every command exits 0. + +- [ ] **Step 2: Run repository Rust gates** + +Run: `cargo fmt --all -- --check`, `cargo test-fastly`, `cargo test-axum`, `cargo test-cloudflare`, `cargo test-spin`, `./scripts/test-cli.sh`, `cargo clippy-fastly`, `cargo clippy-axum`, `cargo clippy-cloudflare`, `cargo clippy-cloudflare-wasm`, `cargo clippy-spin-native`, and `cargo clippy-spin-wasm`. Expected: every command exits 0. + +- [ ] **Step 3: Commit formatting or test-only adjustments** + +If verification changed tracked files, review them and commit only scoped changes as `chore: finalize PR 1079 remediation verification`. + +### Task 5: Build the staging branch + +- [ ] **Step 1: Confirm a clean repair branch** + +Run: `git status --short --branch` and record `git rev-parse HEAD`. Expected: branch `fix/gpt-first-impression-aps-shell-review`, no uncommitted changes. + +- [ ] **Step 2: Refresh the remote RC ref** + +Run: `git fetch origin refs/heads/rc/202608:refs/remotes/origin/rc/202608 refs/heads/fix/gpt-first-impression-aps-shell:refs/remotes/origin/fix/gpt-first-impression-aps-shell`. + +- [ ] **Step 3: Create and merge the staging branch** + +Run: `git switch -c staging/202608-pr1079-review origin/rc/202608` then `git merge --no-ff fix/gpt-first-impression-aps-shell-review -m "Merge PR 1079 review remediation for staging"`. Expected: merge succeeds without unresolved conflicts. + +- [ ] **Step 4: Re-run critical post-merge gates** + +Run: `cd crates/trusted-server-js/lib && npm run format && npm run lint && npx vitest run && node build-all.mjs`. + +Run: `cd crates/trusted-server-integration-tests/browser && npx playwright test tests/shared/aps-renderer.spec.ts`. + +Run from the repository root: `cargo fmt --all -- --check && cargo check-fastly && cargo check-axum && cargo check-cloudflare`. + +Expected: every command exits 0 and `git status --short --branch` is clean on `staging/202608-pr1079-review`. + +- [ ] **Step 5: Report deployable refs** + +Record the repair-branch hash, staging merge hash, exact test results, and any non-blocking environment limitations. Do not push unless separately requested. diff --git a/docs/superpowers/plans/2026-08-28-managed-user-id-bundle-validation.md b/docs/superpowers/plans/2026-08-28-managed-user-id-bundle-validation.md new file mode 100644 index 000000000..2265e49b2 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-managed-user-id-bundle-validation.md @@ -0,0 +1,680 @@ +# Managed User ID Bundle Validation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `ts prebid bundle` fail before updating deployable metadata when a configured managed User ID name is unknown, ambiguous, or missing its required module from the freshly generated bundle manifest. + +**Architecture:** Extend the CLI's focused TOML reader with managed User ID names, resolve them through the same checked-in JSON registry used by the JavaScript generator, invalidate any stale output manifest, and validate the newly generated manifest before patching hash/SRI metadata. Keep core vendor-neutral and retain the existing browser diagnostic as defense in depth. + +**Tech Stack:** Rust 2024, Serde/serde_json, TOML/toml_edit, host-target CLI tests, Prettier Markdown formatting. + +**Specification:** `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md` + +**Working tree:** Use the existing `issue-355-liveramp-integration` branch as explicitly requested by the user. Do not create a worktree and do not push without separate authorization. + +--- + +## File structure + +| File | Responsibility | +| ------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-cli/src/prebid_bundle.rs` | Parse managed names, load/resolve the registry, invalidate stale manifests, validate the generated manifest, and contain focused unit/command tests. | +| `docs/guide/integrations/prebid.md` | Explain the registry-backed bundle failure and runtime fallback diagnostic. | +| `docs/guide/configuration.md` | Replace the obsolete “not validated” configuration warning. | +| `trusted-server.example.toml` | Tell operators that the bundle command validates managed-name/module pairing. | + +No core, TypeScript runtime, JavaScript generator, registry schema, manifest producer, or public configuration shape changes are required. + +## Task 1: Strictly parse managed User ID names + +**Files:** + +- Modify: `crates/trusted-server-cli/src/prebid_bundle.rs:32-37` +- Modify: `crates/trusted-server-cli/src/prebid_bundle.rs:182-248` +- Test: `crates/trusted-server-cli/src/prebid_bundle.rs:558-683` + +- [ ] **Step 1: Write failing parser tests** + +Add tests proving an absent list becomes empty, valid entries preserve order, and malformed values fail instead of being skipped: + +```rust +#[test] +fn bundle_config_loader_reads_managed_user_id_names_in_order() { + let (_temp, path) = write_config( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" + +[[integrations.prebid.managed_user_ids]] +name = "identityLink" + +[[integrations.prebid.managed_user_ids]] +name = "pubCommonId" + +[integrations.prebid.bundle] +adapters = ["rubicon"] +"#, + ); + + let config = load_bundle_config(&path).expect("should load managed names"); + + assert_eq!( + config.managed_user_id_names, + ["identityLink", "pubCommonId"], + "should preserve managed entry order" + ); +} + +#[test] +fn bundle_config_loader_rejects_managed_entry_without_string_name() { + let (_temp, path) = write_config( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +managed_user_ids = [{ params = { pid = "999" } }] + +[integrations.prebid.bundle] +adapters = ["rubicon"] +"#, + ); + + let error = load_bundle_config(&path).expect_err("should reject missing managed name"); + + assert!( + error.contains("integrations.prebid.managed_user_ids[0].name"), + "should identify the malformed managed entry: {error}" + ); +} +``` + +Add separate cases for: + +- `managed_user_ids` being a string/table rather than an array; +- an array element being a string rather than a table; +- a missing `name`; +- a non-string `name`; +- an empty or whitespace-only `name`. + +Also extend the existing missing-list test to assert `managed_user_id_names.is_empty()`. + +- [ ] **Step 2: Run the CLI suite and verify the new tests fail** + +Run: + +```bash +./scripts/test-cli.sh +``` + +Expected: FAIL because `PrebidBundleConfig` has no `managed_user_id_names` field and no strict reader exists. + +- [ ] **Step 3: Implement the focused TOML reader** + +Extend the config structure: + +```rust +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PrebidBundleConfig { + pub adapters: Vec, + pub user_id_modules: Option>, + pub managed_user_id_names: Vec, + pub external_bundle_url: Option, +} +``` + +Add a narrow helper; do not deserialize or validate vendor parameters: + +```rust +fn read_managed_user_id_names( + prebid: &toml::Value, + config_path: &Path, +) -> CliResult> { + let Some(value) = prebid.get("managed_user_ids") else { + return Ok(Vec::new()); + }; + let entries = value.as_array().ok_or_else(|| { + report_error(format!( + "{} integrations.prebid.managed_user_ids must be an array of tables", + config_path.display() + )) + })?; + + entries + .iter() + .enumerate() + .map(|(index, entry)| { + let table = entry.as_table().ok_or_else(|| { + report_error(format!( + "{} integrations.prebid.managed_user_ids[{index}] must be a table", + config_path.display() + )) + })?; + let field = format!("integrations.prebid.managed_user_ids[{index}].name"); + let name = table.get("name").and_then(toml::Value::as_str).ok_or_else(|| { + report_error(format!( + "{} {field} must be a non-empty string", + config_path.display() + )) + })?; + if name.trim().is_empty() { + return cli_error(format!( + "{} {field} must be a non-empty string", + config_path.display() + )); + } + Ok(name.to_string()) + }) + .collect() +} +``` + +Call it from `load_bundle_config` and store the result. Keep full token, duplicate-name, params, and storage validation in core; the CLI validates only fields required for bundle consistency. + +- [ ] **Step 4: Run the CLI suite and verify it passes** + +Run: `./scripts/test-cli.sh` + +Expected: all `trusted-server-cli` tests PASS. + +- [ ] **Step 5: Commit locally** + +```bash +git add crates/trusted-server-cli/src/prebid_bundle.rs +git commit -m "Parse managed User ID bundle inputs" +``` + +Do not push. + +## Task 2: Resolve managed names through the shared registry + +**Files:** + +- Modify: `crates/trusted-server-cli/src/prebid_bundle.rs:10-12` +- Modify: `crates/trusted-server-cli/src/prebid_bundle.rs:121-128` +- Test: `crates/trusted-server-cli/src/prebid_bundle.rs:547-924` +- Read-only contract: `crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json` + +- [ ] **Step 1: Write failing registry-resolution tests** + +Define tests around an in-memory registry: + +```rust +#[test] +fn managed_names_resolve_aliases_to_registered_modules() { + let registry = PrebidUserIdModuleRegistry { + modules: vec![PrebidUserIdModuleRegistryEntry { + module_name: "sharedIdSystem".to_string(), + config_names: vec!["sharedId".to_string(), "pubCommonId".to_string()], + }], + }; + let registry_path = Path::new("user_id_modules.json"); + + let required = resolve_managed_user_id_modules( + &["pubCommonId".to_string(), "sharedId".to_string()], + ®istry, + registry_path, + ) + .expect("should resolve aliases"); + + assert_eq!( + required, + [ + RequiredPrebidUserIdModule { + config_name: "pubCommonId".to_string(), + module_name: "sharedIdSystem".to_string(), + }, + RequiredPrebidUserIdModule { + config_name: "sharedId".to_string(), + module_name: "sharedIdSystem".to_string(), + }, + ], + "should retain each managed name while allowing a shared module" + ); +} +``` + +Add cases proving: + +- `identityLink` resolves to `identityLinkIdSystem` from the actual checked-in registry; +- an unknown name fails and identifies the name plus registry path; +- a synthetic name mapped to two distinct modules fails and lists both candidates deterministically; +- an empty managed-name list returns an empty requirement list. + +- [ ] **Step 2: Run the CLI suite and verify the tests fail** + +Run: `./scripts/test-cli.sh` + +Expected: FAIL because the registry types, loader, and resolver do not exist. + +- [ ] **Step 3: Implement registry loading and deterministic resolution** + +Add vendor-neutral types: + +```rust +const USER_ID_REGISTRY_RELATIVE_PATH: &str = + "src/integrations/prebid/user_id_modules.json"; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PrebidUserIdModuleRegistry { + modules: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PrebidUserIdModuleRegistryEntry { + module_name: String, + config_names: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct RequiredPrebidUserIdModule { + config_name: String, + module_name: String, +} +``` + +Load the exact file beneath the already-resolved JS library directory: + +```rust +fn load_user_id_registry( + js_lib_dir: &Path, +) -> CliResult<(PathBuf, PrebidUserIdModuleRegistry)> { + let path = js_lib_dir.join(USER_ID_REGISTRY_RELATIVE_PATH); + let contents = fs::read_to_string(&path).map_err(|error| { + report_error(format!( + "failed to read Prebid User ID registry {}: {error}", + path.display() + )) + })?; + let registry = serde_json::from_str(&contents).map_err(|error| { + report_error(format!( + "failed to parse Prebid User ID registry {}: {error}", + path.display() + )) + })?; + Ok((path, registry)) +} +``` + +Implement `resolve_managed_user_id_modules` with these rules: + +1. Collect matching `module_name` values for every exact `config_names` match. +2. Sort and deduplicate candidate modules for deterministic diagnostics. +3. Zero candidates: fail with managed name and registry path. +4. One candidate: return a requirement retaining both config and module names. +5. More than one candidate: fail with the managed name, registry path, and candidates. + +Do not hardcode `identityLink`, `identityLinkIdSystem`, `liveramp.com`, or any other vendor/module name in production code. + +- [ ] **Step 4: Run the CLI suite and verify it passes** + +Run: `./scripts/test-cli.sh` + +Expected: all CLI tests PASS, including the checked-in registry contract. + +- [ ] **Step 5: Commit locally** + +```bash +git add crates/trusted-server-cli/src/prebid_bundle.rs +git commit -m "Resolve managed User IDs through the bundle registry" +``` + +Do not push. + +## Task 3: Require a fresh manifest containing every managed module + +**Files:** + +- Modify: `crates/trusted-server-cli/src/prebid_bundle.rs:121-180` +- Modify: `crates/trusted-server-cli/src/prebid_bundle.rs:419-454` +- Test: `crates/trusted-server-cli/src/prebid_bundle.rs:797-907` + +- [ ] **Step 1: Make the fake generator express exact manifest behavior** + +Replace `write_manifest: bool` with an optional complete JSON document: + +```rust +struct FakeGenerator { + generate_error: Option, + generate_calls: Vec, + manifest: Option, +} +``` + +When the option is `Some`, write that exact JSON value to `manifest.json`. When +it is `None`, return according to `generate_error` without writing a manifest. +Add a `fake_manifest(user_id_modules: serde_json::Value)` helper that returns the +otherwise-valid manifest object. This lets tests emit a valid array, a non-array +value, or a complete object with `userIdModules` removed. Update existing tests +without changing their intent. + +- [ ] **Step 2: Write failing command-level consistency tests** + +Add command-level tests proving: + +```rust +#[test] +fn run_bundle_rejects_managed_name_when_manifest_omits_required_module() { + let (_temp, config_path) = write_config(&managed_identity_link_config()); + let original = fs::read_to_string(&config_path).expect("should read original config"); + let output_root = tempfile::tempdir().expect("should create output root"); + let mut generator = FakeGenerator { + generate_error: None, + generate_calls: Vec::new(), + manifest: Some(fake_manifest(serde_json::json!(["sharedIdSystem"]))), + }; + let args = PrebidBundleArgs { + config: config_path, + out: output_root.path().join("prebid"), + }; + + let error = run_bundle(&args, &mut generator, &mut Vec::new(), &mut Vec::new()) + .expect_err("should reject missing managed module"); + + assert!(error.contains("identityLink"), "should name managed config: {error}"); + assert!( + error.contains("identityLinkIdSystem"), + "should name required module: {error}" + ); + assert!( + error.contains("integrations.prebid.bundle.user_id_modules"), + "should identify corrective field: {error}" + ); + assert_eq!( + fs::read_to_string(&args.config).expect("should reread config"), + original, + "should not patch metadata after consistency failure" + ); +} +``` + +Add cases proving: + +- the same managed config passes when the manifest contains `identityLinkIdSystem`; +- two managed names require both modules; +- two aliases backed by `sharedIdSystem` both pass with one manifest module; +- omission of `bundle.user_id_modules` passes when the fake generated manifest contains the default module; +- unknown and malformed names fail through `run_bundle` before `generate_calls` + receives an entry, with the entire original config (including existing + hash/SRI metadata) unchanged; +- an ambiguous name fails through the private registry-injected orchestration + seam described in Step 6 before `generate_calls` receives an entry, with the + entire original config (including existing hash/SRI metadata) unchanged; +- fake manifests with a missing or non-array `userIdModules` field fail + manifest parsing; +- a missing required module never changes existing hash/SRI metadata. + +- [ ] **Step 3: Write the failing stale-manifest regression test** + +Prepopulate `/manifest.json` with valid old metadata, make the fake generator return success without writing, then assert: + +- `run_bundle` fails to read the generated manifest; +- the old manifest no longer exists; +- config metadata is unchanged. + +Run: `./scripts/test-cli.sh` + +Expected: FAIL because the current CLI accepts an old manifest and does not validate `userIdModules`. + +- [ ] **Step 4: Extend manifest deserialization and validation** + +```rust +#[derive(Debug, Deserialize)] +struct PrebidBundleManifest { + #[serde(rename = "userIdModules")] + user_id_modules: Vec, + sha256: String, + sri: String, + filename: String, +} + +fn validate_managed_user_id_modules( + requirements: &[RequiredPrebidUserIdModule], + manifest: &PrebidBundleManifest, + config_path: &Path, +) -> CliResult<()> { + for requirement in requirements { + if !manifest + .user_id_modules + .iter() + .any(|module| module == &requirement.module_name) + { + return cli_error(format!( + "{} configures managed User ID {:?}, which requires Prebid module {:?}, but the generated manifest omits it; add {:?} to integrations.prebid.bundle.user_id_modules and rerun `ts prebid bundle`", + config_path.display(), + requirement.config_name, + requirement.module_name, + requirement.module_name, + )); + } + } + Ok(()) +} +``` + +Serde must reject a missing or non-array `userIdModules` field. Do not default it to an empty list. + +- [ ] **Step 5: Invalidate only the exact old manifest before generation** + +```rust +fn invalidate_manifest(path: &Path) -> CliResult<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => cli_error(format!( + "failed to remove stale Prebid manifest {}: {error}", + path.display() + )), + } +} +``` + +Do not delete the output directory or any generated bundle files. + +- [ ] **Step 6: Add a private registry-injected orchestration seam** + +Keep public command behavior in `run_bundle`, but move the post-registry workflow +into a private helper so command ordering can be tested with a synthetic +ambiguous registry: + +```rust +struct PrebidBundleRunContext<'a> { + current_dir: &'a Path, + js_lib_dir: PathBuf, + registry_path: &'a Path, + registry: &'a PrebidUserIdModuleRegistry, +} + +fn run_bundle_with_context( + args: &PrebidBundleArgs, + config: PrebidBundleConfig, + context: PrebidBundleRunContext<'_>, + generator: &mut dyn PrebidBundleGenerator, + out: &mut dyn Write, + err: &mut dyn Write, +) -> CliResult<()> { + // Resolve requirements before output mutation or generator invocation, + // then perform generation, manifest validation, and metadata patching. +} +``` + +`run_bundle` must load config, determine the current/JS directories, load the +real registry, then delegate. Tests may pass a synthetic registry only to this +private helper. + +- [ ] **Step 7: Wire the command in the specified order** + +Implement the helper workflow in this order: + +1. Load focused config. +2. Locate the JS directory. +3. Load the shared registry. +4. Resolve all managed names; return before generator invocation on failure. +5. Ensure the output directory is writable. +6. Invalidate only `/manifest.json`. +7. Invoke the generator. +8. Load the newly created manifest. +9. Validate all resolved requirements. +10. Patch hash/SRI metadata. + +Keep `external_bundle_url` output behavior unchanged. Add the synthetic +ambiguous-registry command test now and assert the fake generator has zero +calls and the original config remains byte-for-byte unchanged. + +- [ ] **Step 8: Run the CLI suite and verify it passes** + +Run: `./scripts/test-cli.sh` + +Expected: all CLI tests PASS. + +- [ ] **Step 9: Run formatting and CLI lint** + +Run: + +```bash +cargo fmt --all -- --check +cargo clippy-cli +``` + +Expected: both commands exit 0 with no warnings. + +- [ ] **Step 10: Commit locally** + +```bash +git add crates/trusted-server-cli/src/prebid_bundle.rs +git commit -m "Reject incomplete managed User ID bundles" +``` + +Do not push. + +## Task 4: Document the build-time guard + +**Files:** + +- Modify: `docs/guide/integrations/prebid.md:487-527` +- Modify: `docs/guide/configuration.md:1270-1293` +- Modify: `trusted-server.example.toml:420-445` + +- [ ] **Step 1: Replace obsolete unvalidated-pairing guidance** + +Document these exact semantics in both guides: + +- `ts prebid bundle` resolves each managed name through `user_id_modules.json`; +- unknown or ambiguous config names fail; +- the command confirms required modules in the newly generated manifest; +- failure identifies the managed name/module and does not update hash/SRI; +- the browser diagnostic remains useful for external, stale, or modified bundles; +- core remains vendor-neutral and continues to forward `params` opaquely. + +Replace the example-file warning with concise wording such as: + +```toml +# `ts prebid bundle` resolves every managed name through the checked-in User ID +# registry and fails if the generated manifest omits its required module. +``` + +- [ ] **Step 2: Verify documentation no longer claims the pairing is unvalidated** + +Run: + +```bash +rg -n "Nothing validates|not validated|pairing is not validated" \ + docs/guide/integrations/prebid.md \ + docs/guide/configuration.md \ + trusted-server.example.toml +``` + +Expected: no matches. + +- [ ] **Step 3: Format and check documentation** + +Run: + +```bash +cd docs +npm run format:write +npm run format +``` + +Expected: Prettier writes any required formatting changes, then reports all +documentation files formatted. + +- [ ] **Step 4: Commit locally** + +```bash +git add docs/guide/integrations/prebid.md docs/guide/configuration.md trusted-server.example.toml +git commit -m "Document managed User ID bundle validation" +``` + +Do not push. + +## Task 5: Final verification and local review + +**Files:** + +- Review: all files changed since `origin/issue-355-liveramp-integration` + +- [ ] **Step 1: Run the focused gate** + +```bash +./scripts/test-cli.sh +cargo clippy-cli +cargo fmt --all -- --check +cd docs && npm run format +``` + +Expected: every command exits 0; no test, lint, or formatting failures. + +- [ ] **Step 2: Run the broader PR regression gate** + +The branch also contains core and JS LiveRamp work, so run the repository-required relevant suites: + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo clippy-codegen +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +cd crates/trusted-server-js/lib +npx vitest run +node build-all.mjs +npm run format +``` + +Expected: all commands exit 0. If an environment-dependent suite cannot run, record the exact command and error rather than claiming it passed. + +- [ ] **Step 3: Review the complete local diff** + +```bash +git status --short --branch +git diff --check origin/issue-355-liveramp-integration...HEAD +git diff --stat origin/issue-355-liveramp-integration...HEAD +git log --oneline origin/issue-355-liveramp-integration..HEAD +``` + +Confirm: + +- production CLI code contains no vendor name; +- registry and manifest are the only mapping/inclusion sources; +- malformed input cannot be silently skipped; +- stale manifests cannot be reused; +- metadata is patched only after validation; +- unrelated `main` changes are present only through the local merge commit; +- no secrets, Placement IDs, or envelope values were added. + +- [ ] **Step 4: Request code review** + +Invoke `@superpowers:requesting-code-review` against the final local diff and address any verified findings one at a time. + +- [ ] **Step 5: Stop before remote mutation** + +Report the local commits, verification evidence, and any remaining live-validation work. Do not push, update PR #1054, reply to GitHub comments, or change the draft state without explicit user authorization. diff --git a/docs/superpowers/plans/2026-08-31-managed-user-id-consent-activation.md b/docs/superpowers/plans/2026-08-31-managed-user-id-consent-activation.md new file mode 100644 index 000000000..1ce7cc2a8 --- /dev/null +++ b/docs/superpowers/plans/2026-08-31-managed-user-id-consent-activation.md @@ -0,0 +1,316 @@ +# Managed User ID Consent Activation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ensure server-managed Prebid User IDs activate the existing TCF enforcement modules when an IAB CMP is present, without overwriting publisher-owned consent configuration. + +**Architecture:** Add one focused browser-side helper in the existing Prebid shim. It reads effective Prebid consent configuration, recognizes any existing own `gdpr` property as publisher-owned, and otherwise installs only `gdpr.cmpApi = "iab"` when managed User IDs and a callable `window.__tcfapi` are present. The `setConfig`/`mergeConfig` wrappers preserve publisher precedence; when a later call first claims GDPR ownership, they retire the automatically created IAB collector before forwarding the publisher value so stale CMP events cannot overwrite it. + +**Tech Stack:** TypeScript, Prebid.js 10.26.0, Vitest, JSDOM, generated external Prebid bundle, Markdown. + +--- + +## File structure + +- Modify `crates/trusted-server-js/lib/src/integrations/prebid/index.ts`: detect and install the minimum managed-ID TCF configuration before managed IDs are seeded. +- Modify `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts`: unit-test activation conditions, merge semantics, malformed values, and publisher precedence. +- Modify `crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs`: prove the real generated bundle blocks IdentityLink without publisher-side Prebid consent setup. +- Modify `docs/guide/integrations/prebid.md`: document automatic activation and ownership boundaries. +- Modify `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md`: mark the consent hardening and bundle guard as implemented and record sanitized live-validation results accurately. + +### Task 1: Add failing shim unit tests + +**Files:** + +- Modify: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts:65-105` +- Modify: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts:244-252` +- Test: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts:835-1130` + +- [ ] **Step 1: Extend the test window and reset state** + +Add an optional `__tcfapi` function to `PrebidTestWindow`. Delete it in the shared `beforeEach`, and reset `mockGetConfig` so tests do not leak effective configuration. + +- [ ] **Step 2: Write the activation-condition tests** + +Add tests that install the shim and assert the original `mockSetConfig` receives: + +```ts +{ + consentManagement: { + gdpr: { cmpApi: 'iab' }, + }, +} +``` + +only when `managedUserIds` is non-empty, `window.__tcfapi` is callable, and effective `consentManagement` has no own `gdpr` property. Assert this call precedes the managed `userSync.userIds` call and `processQueue()`. + +- [ ] **Step 3: Write preservation and degraded-behavior tests** + +Cover: + +- no managed IDs; +- missing and non-callable `__tcfapi`; +- sibling `gpp` configuration preserved; +- effective own `gdpr` object, `null`, and `false` preserved without an automatic GDPR call; +- root `null`, `false`, strings, arrays, and throwing effective consent state log a diagnostic and are not replaced; +- queued and late publisher `setConfig`/`mergeConfig` consent fields pass through unchanged; +- automatic configuration is applied only once. + +- [ ] **Step 4: Run the focused unit tests and verify RED** + +Run: + +```bash +cd crates/trusted-server-js/lib +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH \ + npx vitest run test/integrations/prebid/index.test.ts +``` + +Expected: the new activation assertion fails because no automatic `consentManagement.gdpr` call exists. + +- [ ] **Step 5: Remove the masking publisher consent setup from the artifact test** + +Change the primary denied-consent harness cases in +`test/prebid-consent-enforcement.test.mjs` so they do not call publisher-side +`pbjs.setConfig({ consentManagement: ... })`. Keep only the +`userSync.auctionDelay` setup required to resolve IDs during one auction. Add an +optional publisher consent configuration for later preservation coverage. + +- [ ] **Step 6: Run the generated-artifact test and verify RED** + +```bash +cd crates/trusted-server-js/lib +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH \ + npx vitest run test/prebid-consent-enforcement.test.mjs +``` + +Expected: the denied-consent case fails because IdentityLink makes its envelope +request or writes storage when Prebid consent management is not activated. + +- [ ] **Step 7: Commit the failing tests** + +```bash +git add crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts \ + crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs +git commit -m "Test managed User ID consent activation" +``` + +### Task 2: Implement the minimum non-clobbering TCF setup + +**Files:** + +- Modify: `crates/trusted-server-js/lib/src/integrations/prebid/index.ts:130-210` +- Modify: `crates/trusted-server-js/lib/src/integrations/prebid/index.ts:1187-1241` +- Test: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` + +- [ ] **Step 1: Add the focused helper** + +Add a private helper receiving the original `setConfig`, `getConfig`, and managed entries. It must: + +1. return unless managed entries exist and `typeof window.__tcfapi === 'function'`; +2. read `getConfig('consentManagement')` inside `try/catch`; +3. preserve any record with an own `gdpr` property; +4. merge record-valued sibling settings with `gdpr: { cmpApi: 'iab' }`; +5. treat every defined non-record value, including `null`, or a thrown accessor as publisher-owned/unsafe, log once, and return; +6. call the original Prebid `setConfig` exactly once, without adding timeout or `defaultGdprScope`. + +- [ ] **Step 2: Invoke it before managed ID seeding** + +Call the helper after capturing the original config APIs and before the first managed `userSync.userIds` update. Do not add a new core/TOML field and do not alter pages without managed IDs. + +- [ ] **Step 3: Run the focused unit tests and verify GREEN** + +Run the Task 1 command. Expected: all tests in `index.test.ts` pass. + +- [ ] **Step 4: Run formatting and type-aware JS tests** + +```bash +cd crates/trusted-server-js/lib +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH npm run format +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH npx vitest run test/integrations/prebid/index.test.ts +``` + +- [ ] **Step 5: Commit the implementation** + +```bash +git add crates/trusted-server-js/lib/src/integrations/prebid/index.ts \ + crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +git commit -m "Activate TCF for managed User IDs" +``` + +### Task 3: Complete generated-artifact enforcement coverage + +**Files:** + +- Modify: `crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs:115-205` + +- [ ] **Step 1: Add artifact-level preservation coverage** + +Use the harness option introduced in Task 1 and prove an existing custom GDPR +object remains effective after the shim loads. + +- [ ] **Step 2: Run the generated-artifact suite and verify GREEN** + +```bash +cd crates/trusted-server-js/lib +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH \ + npx vitest run test/prebid-consent-enforcement.test.mjs +``` + +Expected: denied Purpose 1 and denied vendor 97 make no envelope request and write no LiveRamp storage; granted consent still makes one request and writes storage. + +- [ ] **Step 3: Commit the completed artifact regression test** + +```bash +git add crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs +git commit -m "Prove managed-only TCF enforcement" +``` + +### Task 4: Align documentation and PR handoff text + +**Files:** + +- Modify: `docs/guide/integrations/prebid.md:130-145` +- Modify: `docs/guide/integrations/prebid.md:540-565` +- Modify: `docs/guide/integrations/prebid.md:620-635` +- Modify: `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md:9` +- Modify: `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md:151-205` +- Modify: `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md:620-720` + +- [ ] **Step 1: Document exact browser consent ownership** + +State that callable `__tcfapi` plus managed IDs activates only `gdpr.cmpApi = "iab"`, existing publisher GDPR configuration wins, and pages without TCF remain unchanged. + +- [ ] **Step 2: Update implementation and validation status** + +Mark the registry-driven bundle guard and managed consent hardening implemented. Record only sanitized live evidence: anonymous/unresolved browser returned 204/no EID; resolvable test identity returned 200, stored an envelope, and exposed one `liveramp.com` EID. Do not record Placement IDs, cookies, or envelope values. Do not claim the unperformed PBS/EC follow-on checks are complete. + +Keep the full live-validation acceptance criterion explicitly pending. List the +remaining external checks: denied-consent behavior on an approved live origin, +unapproved-origin degradation, controlled PBS `user.ext.eids` forwarding, and +later EC/KV ingestion. These require publisher/LiveRamp test conditions and are +not replaced by automated artifact tests. + +- [ ] **Step 3: Prepare corrected PR description text** + +Prepare a concise handoff in the final response replacing vendor-specific core wording, removing the unrelated credential-blocked status, recording completed browser validation, and describing ATS server-side work as deferred pending team confirmation. Do not mutate GitHub. + +- [ ] **Step 4: Format documentation** + +```bash +cd docs +npm run format:write +npm run format +``` + +- [ ] **Step 5: Commit documentation** + +```bash +git add docs/guide/integrations/prebid.md \ + docs/superpowers/specs/2026-08-21-liveramp-integration-design.md +git commit -m "Align LiveRamp consent and validation status" +``` + +### Task 5: Full verification and final review + +**Files:** + +- Review: all changes from the pre-plan HEAD through the final HEAD + +- [ ] **Step 1: Run the full JavaScript suite with the pinned Node version** + +```bash +cd crates/trusted-server-js/lib +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH npx vitest run +``` + +- [ ] **Step 2: Run repository formatting checks** + +```bash +cargo fmt --all -- --check +cd crates/trusted-server-js/lib +env PATH=/Users/prk-jr/.nvm/versions/node/v24.12.0/bin:$PATH npm run format +cd ../../.. && cd docs +npm run format +``` + +- [ ] **Step 3: Run the repository test matrix** + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +./scripts/test-cli.sh +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +``` + +- [ ] **Step 4: Run the repository lint matrix** + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo clippy-cli +cargo clippy-codegen +``` + +- [ ] **Step 5: Inspect repository and PR diff hygiene** + +```bash +git diff --check origin/main...HEAD +git status --short +git diff --stat origin/main...HEAD +``` + +Confirm `fastly.toml` remains the user's uncommitted local file and no Placement ID, cookie, or envelope value entered the committed diff. + +- [ ] **Step 6: Request final code review** + +Review the complete diff for correctness, privacy regressions, scope, stale documentation, and test gaps. Fix any blocking finding test-first and rerun the relevant verification. + +- [ ] **Step 7: Report readiness without pushing** + +Summarize commits, verification evidence, remaining external steps, and corrected PR-description text. Do not push or mark the PR ready. + +### Task 6: Retire automatic consent ownership safely + +**Files:** + +- Modify: `crates/trusted-server-js/lib/src/integrations/prebid/index.ts` +- Modify: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` +- Modify: `crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs` +- Modify: `docs/guide/integrations/prebid.md` +- Modify: `docs/superpowers/specs/2026-08-21-liveramp-integration-design.md` + +- [ ] **Step 1: Reproduce the stale-listener failure in the real bundle** + +Start with shim-owned IAB consent, apply late publisher static denial, emit a +later granting CMP event, and verify the test fails because the original CMP +listener remains registered. + +- [ ] **Step 2: Add focused ownership-transfer tests** + +Require cleanup before publisher `setConfig`, cleanup exactly once before +publisher `mergeConfig`, restoration of the normal enabled default for a merged +object-valued GDPR config, and safe degradation for throwing effective consent +accessors. + +- [ ] **Step 3: Implement one-time collector retirement** + +Track successful automatic activation. When a later publisher call claims GDPR +ownership, send `gdpr.enabled = false` through the original Prebid `setConfig` +before forwarding the publisher call. Preserve sibling consent state, avoid +leaking the temporary disabled flag through `mergeConfig`, and never reactivate +the automatic collector. Guard the automatically registered callback so a +delayed first CMP response cannot bypass transfer before a listener ID exists; +prepare merge normalization before cleanup, and skip replacement cleanup when +unknown sibling state or the publisher merge cannot be inspected safely. + +- [ ] **Step 4: Verify focused and full JavaScript suites** + +Run the focused shim and generated-artifact suites, formatting, and the full +Vitest suite with pinned Node 24.12.0. Then repeat diff hygiene and final review. diff --git a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md index 8617ef877..147323675 100644 --- a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md +++ b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md @@ -68,10 +68,14 @@ across every navigation in the user's clickstream rather than once per session. pipeline. The GAM call (`securepubads.g.doubleclick.net`) moving server-side is aspirational, contingent on Google agreement, and is not committed for any phase (see §9.6). -- Eliminating Prebid entirely — a stripped-down Prebid bundle (_slim-Prebid_) is +- Eliminating Prebid entirely. A stripped-down Prebid bundle (_slim-Prebid_) is lazy-loaded post-`window.load` to handle scroll/refresh auctions and userID - enrichment. **TS owns the first impression; Prebid owns subsequent refresh - auctions.** + enrichment. **The first valid claimant owns each navigation's first impression.** + A publisher auction, GPT request, or GPT render consumes the claim before late + page-bids data can target or refresh that slot. If TS claims first, it suppresses + one correlated losing publisher delivery during a bounded lease. Later publisher + refresh auctions proceed normally. Strict TS-first delivery would require holding + publisher delivery and remains a separate design choice. - Dynamic slot discovery (reading the DOM) — this design commits to pre-defined, URL-matched slot templates. Smart Slots' dynamic injection behavior is replaced by server knowledge. diff --git a/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md new file mode 100644 index 000000000..98af3e176 --- /dev/null +++ b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md @@ -0,0 +1,842 @@ +# Server-Side Ad Template CLI Design + +**Date:** 2026-06-26 +**Status:** Draft design +**Scope:** Static and browser-backed CLI diagnostics for server-side ad templates + +## 1. Goal + +Add Trusted Server CLI support for server-side ad-template onboarding and +verification without resurrecting the stale standalone `ts-config` design. + +The CLI must answer two operator questions: + +1. Given an effective `trusted-server.toml`, which configured ad-template slots + match this path? +2. Given one or more live publisher URLs, are the configured slots for the + final navigated paths actually present on the page according to DOM, GPT, + and provider evidence, and do any runtime gates explain why Trusted Server + would not inject or auction for that page? + +The command surface is split by whether the command is local-config-only or +browser-backed: + +```bash +ts config ad-templates lint +ts config ad-templates match +ts config ad-templates check +ts config ad-templates explain + +ts audit ad-templates verify ... +``` + +Static commands live under `ts config` because they only load local effective +app config. Browser-backed verification lives under `ts audit` because it loads +public publisher pages in Chrome/Chromium and observes live page behavior. + +## 2. Context + +This design replaces the stale PR #724 direction. + +PR #724 designed a standalone `ts-config` binary around a +`creative-opportunities.toml` file. That is no longer the project shape: + +- Trusted Server configuration now flows through the unified `ts` CLI from PR + #799. +- Server-side ad-template slots live under `[creative_opportunities]` / + `[[creative_opportunities.slot]]` in `trusted-server.toml`. +- Effective config can include EdgeZero app-config environment overlays unless + `--no-env` is passed. +- Operator-owned `trusted-server.toml` is ignored; the repository tracks + `trusted-server.example.toml`. + +PR #799 is the CLI base. It owns the `ts` binary, EdgeZero lifecycle delegates, +and typed app-config validation/push/diff behavior. + +PR #800 is the audit dependency. It adds the generic browser-backed +`ts audit ` collector using local Chrome/Chromium. At the time this spec +was written, PR #800 was stale relative to the latest #799 head, so this work +depends on the #800 audit collector after it is rebased onto the latest #799 +typed blob-config model. + +## 3. Non-Goals + +- Do not add a standalone `ts-config` binary. +- Do not reintroduce `creative-opportunities.toml`. +- Do not implement browser-backed generation in Phase 1. +- Do not mutate `trusted-server.toml` from `verify`. +- Do not probe PBS, GAM, or APS management APIs. +- Do not require EdgeZero platform adapters for local static diagnostics. +- Do not make `ts audit ad-templates verify` push, provision, deploy, or update + platform resources. +- Do not rely on real GPT or APS network calls in tests. + +Browser-backed generation is a later phase: + +```bash +ts audit ad-templates generate ... +``` + +That phase needs separate rules for slot ID derivation, page-pattern inference, +multi-URL merging, TOML ordering, and whether the command emits a patch, a draft +file, or full config blocks. + +## 4. Command Surface + +### 4.1 Shared Config Flags + +All `ts config ad-templates ...` commands and +`ts audit ad-templates verify` accept the same local app-config flags: + +```bash +--app-config +--manifest +--no-env +``` + +Defaults match PR #799: + +| Option | Default | +| -------------- | ------------------------------------------------ | +| `--app-config` | `.toml`, resolved from `edgezero.toml` | +| `--manifest` | `edgezero.toml` | +| `--no-env` | `false`; app-config env overlay is applied | + +If an explicit `--app-config` path is supplied and missing, the command reports +that path as the error. It must not silently fall back to an environment or +manifest-derived path. + +### 4.2 Static Config Diagnostics + +```bash +ts config ad-templates lint [--app-config ] [--manifest ] [--no-env] +``` + +Reports whether `[creative_opportunities]` is configured, how many slots exist, +GAM network ID, auction timeout, auction enablement, configured auction +providers, and whether current EdgeZero routing will fall back to the legacy +path when configured slots are present. + +```bash +ts config ad-templates match [--details] ... +``` + +Normalizes a path or full URL to a path and reports the slots matched by the +runtime `creative_opportunities::match_slots` logic. `--details` includes slot +div ID, GAM unit path, page patterns, formats, and configured providers. + +```bash +ts config ad-templates check \ + (--expected-slot ... | --expect-no-slots) \ + [--allow-extra-slots] ... +``` + +CI-friendly assertion wrapper around the same matching logic. + +```bash +ts config ad-templates explain \ + [--method GET] \ + [--non-navigation] \ + [--prefetch] \ + [--bot] \ + [--consent-denied] \ + [--edgezero-enabled] ... +``` + +Explains the major runtime gates that decide whether the server-side ad stack +would run for a page request. This is a local model, not a live request replay. + +### 4.3 Browser-Backed Verification + +```bash +ts audit ad-templates verify ... \ + [--app-config ] \ + [--manifest ] \ + [--no-env] \ + [--strict] \ + [--json] \ + [--scroll] +``` + +Behavior: + +- Accept one or more `http` or `https` URLs. +- Reject all other schemes before launching a browser. +- Load the effective Trusted Server app config. +- For each URL, navigate first, collect the final URL, normalize the final URL + to a path, and call `creative_opportunities::match_slots`. +- Preserve the requested URL/path separately from the final URL/path. +- Emit a redirect warning when the final path differs from the requested path. +- Expect only the slots matched for the final URL path to be present on that + live page. +- Report live DOM/GPT/APS ad-slot evidence that does not correspond to a + matched configured slot as structured extra evidence. +- Launch Chrome/Chromium through the audit collector from the rebased #800 work. +- Inject a read-only ad-template collector before publisher scripts run. +- Compare configured matched slots against DOM, GPT, and APS evidence. +- Report runtime ad-stack gate evidence separately from placement evidence. +- Print human output by default. +- Emit stable machine-readable output with `--json`. +- Exit `0` by default for missing or partial live evidence; this is an + auditor-assist mode. +- Exit non-zero under `--strict` when a matched configured slot is missing or + only partially confirmed. + +`--scroll` performs a deterministic scroll pass after initial load and settle. +It is opt-in because it is slower and can trigger additional page behavior. +Slots first observed during scroll count as confirmed when the GPT evidence is +otherwise sufficient. + +## 5. Confirmation Model + +The verifier compares configured expected slots to live page evidence. + +It must keep three concepts separate: + +1. **Static slot matching:** which configured slots match a URL path according + to `creative_opportunities::match_slots`. +2. **Runtime ad-stack eligibility:** whether Trusted Server would run its + server-side ad stack for the audited navigation. This mirrors + `should_run_server_side_ad_stack` for the initial publisher request and the + `/__ts/page-bids` kill-switch/consent behavior for SPA route updates. +3. **Live placement evidence:** what the browser actually observes on the + rendered page through DOM, GPT, and APS evidence. + +`verify` is primarily a live placement verifier. `--strict` fails when matched +configured slots for an eligible page are missing or partial. Runtime gates are +reported so operators can distinguish "the slot is not on the page" from "the +current request/config would intentionally suppress Trusted Server ad-template +injection or page-bids slot output". + +### 5.1 Expected Slots + +For each input URL: + +1. Navigate the browser to the requested URL. +2. Record `requested_url`, `requested_path`, `final_url`, and `final_path`. +3. Match configured slots through the core runtime matcher using `final_path`. +4. Build an expected-slot record for each matched slot: + - slot ID; + - resolved div ID; + - resolved GAM unit path; + - configured formats; + - configured providers; + - matching page patterns. + +Only these expected slots are verified for that page. For example, slots whose +only pattern is `/` are expected for the homepage path, not for `/news/story`. + +When a navigation redirects, `verify` uses the final path for expected slots and +reports the requested path in output. This matches runtime behavior: Trusted +Server evaluates the actual publisher request path it handles, not the URL the +operator typed before redirects. + +### 5.2 Runtime Gate Evidence + +For each page result, `verify` reports a local runtime-gate model: + +| Gate | Source | +| ------------------------ | -------------------------------------------------------------------------------------------------------- | +| `method_get` | Browser navigation request; expected to pass for normal `verify`. | +| `navigation` | Browser navigation request; expected to pass for normal `verify`. | +| `not_prefetch` | Browser request headers; expected to pass unless the collector is extended with prefetch simulation. | +| `not_bot` | Browser User-Agent checked against the runtime bot fragments. | +| `matched_slots` | Final-path slot matching. | +| `auction_enabled` | Effective `[auction].enabled` / orchestrator enablement from app config. | +| `consent_allows_auction` | `unknown` unless the collector can prove a consent-allowed or consent-denied state for the live request. | + +`runtime_ad_stack_expected` is a three-state value: `yes`, `no`, or `unknown`. +Known blocking gates produce page warnings and set +`runtime_ad_stack_expected = "no"`. Unknown gates set +`runtime_ad_stack_expected = "unknown"` but do not by themselves fail +`--strict`. + +If `runtime_ad_stack_expected = "no"` because of a known config/request gate +such as `[auction].enabled = false`, strict mode does not fail missing GPT/APS +evidence for that page. The page result is reported as skipped for runtime +verification while still showing the static expected slots and any live +placement evidence that was observed. + +If `runtime_ad_stack_expected = "yes"` or `"unknown"`, strict mode applies the +normal missing/partial placement rules from §5.6. + +For SPA routes, `/__ts/page-bids` returns no slots when the ad-stack kill switch +or consent gate blocks the stack. Browser verification should report observed +page-bids responses when available, but it must not require real partner bids in +tests. + +Live ad-slot evidence that does not map to a matched expected slot is reported +as structured extra evidence. Extra evidence can identify publisher-owned slots +that have not yet moved into server-side ad templates, slots whose +`page_patterns` are too narrow, or slots that should stay outside Trusted +Server. It does not make `--strict` fail in Phase 1. + +### 5.3 DOM Slot Resolution + +The verifier must mirror the runtime GPT bootstrap's slot-root resolution: + +1. Try `document.getElementById(slot.div_id)`. +2. If absent, find the first element with an ID that starts with `slot.div_id`. +3. Ignore elements whose ID ends with `-container`. + +This is required because `div_id` may intentionally be a stable prefix for +framework-generated IDs, for example `ad-header-0-`. + +### 5.4 GPT Evidence + +A slot is confirmed by GPT evidence when the live page exposes a GPT slot whose: + +- ad unit path equals the configured resolved GAM unit path; +- slot element ID equals the resolved DOM element ID or an existing + `${resolved_dom_id}-container` element used by Trusted Server when defining + its own slot; +- configured sizes are compatible with the observed GPT sizes. + +The collector should observe both direct `googletag.defineSlot` calls and +post-load `googletag.pubads().getSlots()` state. + +Size compatibility is defined for Phase 1 as follows: + +- Normalize configured sizes from `CreativeOpportunityFormat` values where + `media_type = "banner"` into `(width, height)` pairs. +- Normalize observed GPT sizes from `defineSlot` input and `getSizes()` output: + - `[300, 250]` becomes one `(300, 250)` pair. + - `[[300, 250], [728, 90]]` becomes two pairs. + - non-numeric values such as `"fluid"` are ignored for numeric matching and + reported as warnings. +- A GPT slot's sizes are compatible when the configured banner size set and the + observed numeric GPT size set have at least one pair in common. +- Extra observed GPT sizes do not block confirmation, but they are reported as + warnings so auditors can decide whether to add formats to config. +- Configured banner sizes that are not observed do not block confirmation when + at least one configured size was observed, but they are reported as warnings. +- If ad unit path and div match but no numeric size overlap exists, the slot is + `partial`, not `confirmed`. +- Configured `video` and `native` formats are not used for Phase 1 GPT size + confirmation. A matched slot with only non-banner formats is `unconfirmable` + with an unsupported-format warning and does not fail `--strict`. +- A sizeless live GPT slot is `partial` when the config declares banner sizes, + because that is observable drift and must fail `--strict`. + +### 5.5 APS Evidence + +Phase 1 does not wrap or collect `apstag.fetchBids`: APS is server-side provider +configuration and client-side calls are neither required nor authoritative for +the runtime ad-template decision. + +### 5.6 Statuses + +| Status | Meaning | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `confirmed` | GPT evidence matches the configured GAM unit path, div resolution, and compatible sizes. | +| `partial` | The page has some evidence for the configured slot, but not enough to confirm it. This includes DOM-only evidence, GPT path/div matches with incompatible sizes, GPT path/div matches for unsupported non-banner-only configured formats, and other non-confirming GPT evidence. | +| `missing` | No DOM or GPT evidence confirms the configured slot. | +| `unconfirmable` | The checker cannot evaluate the configured format with Phase 1 evidence, such as a non-banner-only slot. This is reported but does not fail strict mode. | + +In `--strict` mode: + +- `missing` fails. +- `partial` fails. +- `unconfirmable` does not fail. + +Provider issues are not statuses. They are warnings attached to the slot result. +For example, a slot can be `confirmed` and still carry a warning that configured +APS evidence was missing or ambiguous. Provider warnings do not fail `--strict` +unless a future `--strict-providers` flag is added. + +## 6. Architecture + +The architecture should keep command parsing thin and move ad-template behavior +into pure, testable modules. + +```text +crates/trusted-server-cli/src/ + app_config.rs + ad_templates/ + mod.rs + expected.rs + compare.rs + output.rs + config_ad_templates.rs + audit/ + page.rs + browser.rs + ad_templates.rs +``` + +### 6.1 `app_config.rs` + +Shared loader for effective Trusted Server app config. + +Responsibilities: + +- read `edgezero.toml` through EdgeZero manifest helpers; +- resolve the default `.toml` path; +- apply EdgeZero app-config env overlay unless `--no-env`; +- return `TrustedServerAppConfig` / `Settings`; +- report errors in the same terms as #799 config commands. + +This avoids duplicating config path and env-overlay behavior between +`ts config ad-templates ...` and `ts audit ad-templates verify`. + +The current branch already has a private loader in `config_ad_templates.rs`. +Before adding browser-backed verification, move that behavior into this shared +module and route the existing static commands through it so both command +families load the same effective config. + +### 6.2 `ad_templates::expected` + +Pure local expected-slot model. + +Responsibilities: + +- normalize path-or-URL input; +- call `creative_opportunities::match_slots`; +- convert matched slots into stable expected-slot structs; +- preserve deterministic ordering by slot order from config. + +This module must not compile glob patterns independently or duplicate matching +semantics. + +If richer pattern diagnostics are needed, add a small helper to +`trusted-server-core::creative_opportunities` and use it from both runtime and +CLI. + +### 6.3 `ad_templates::compare` + +Pure comparison between expected slots and collected browser evidence. + +Responsibilities: + +- implement DOM prefix matching rules; +- compare GPT path, div, and size evidence; +- compare APS evidence; +- collect unmatched live DOM/GPT/APS ad-slot evidence as structured + `extra_evidence`; +- assign `confirmed`, `partial`, `missing`, and provider warning details; +- decide strict failure status. + +This module should be testable without launching Chrome. + +### 6.4 `ad_templates::output` + +Human and JSON output model. + +Responsibilities: + +- serialize stable JSON output; +- keep arrays ordered by input URL, then configured slot order, then provider + name; +- render concise human summaries; +- avoid leaking page HTML, cookies, local storage, or arbitrary page data. + +### 6.5 `config_ad_templates.rs` + +Thin Clap adapter for `ts config ad-templates ...`. + +Responsibilities: + +- parse command arguments; +- call `app_config` and `ad_templates::expected`; +- delegate formatting to `ad_templates::output`; +- keep no browser-specific logic. + +### 6.6 `audit::browser` + +Shared browser utility extracted from or aligned with the rebased #800 audit +collector. + +Responsibilities: + +- locate Chrome/Chromium; +- launch an isolated profile; +- reject non-HTTP(S) URLs before navigation; +- set bounded navigation and settle timeouts; +- run optional init scripts; +- perform optional deterministic scroll; +- collect final URL, title, rendered scripts, resource entries, and optional + ad-template evidence. + +The generic `ts audit ` command from #800 should continue to work without +ad-template verification enabled. + +### 6.7 `audit::ad_templates` + +Browser-backed verifier orchestration. + +Responsibilities: + +- parse `ts audit ad-templates verify`; +- load effective config through `app_config`; +- compute expected slots for each URL; +- run the browser collector with ad-template evidence enabled; +- call `ad_templates::compare`; +- print human or JSON output; +- apply default auditor-assist exit behavior and `--strict` behavior. + +## 7. Browser Collector + +The ad-template collector is injected before page scripts run. It is read-only: +it records evidence and calls original page functions with unchanged arguments. + +The rebased #800 collector must grow a pre-navigation init-script hook before it +can satisfy this spec. The stale #800 collector only navigates, waits, and reads +post-load page state; that is insufficient for GPT/APS call evidence. + +Instrumentation requirements: + +- install the collector through the browser's "evaluate on new document" / + init-script mechanism before navigation; +- serialize only configured div prefixes and provider IDs needed for matching; +- observe pages that create `window.googletag = { cmd: [] }` after injection; +- wrap `googletag.cmd.push` callbacks without changing callback order; +- record direct `googletag.defineSlot` calls and calls executed from the GPT + command queue; +- read final `googletag.pubads().getSlots()` state after settle and after + scroll; +- observe pages that assign `window.apstag` after injection and wrap + `apstag.fetchBids` when present; +- tolerate pages that never load GPT or APS and report warnings instead of + throwing collector errors. + +Evidence to collect: + +- DOM elements with IDs relevant to configured slot div prefixes; +- calls to `googletag.defineSlot`; +- final `googletag.pubads().getSlots()` state after settle and after scroll; +- calls to `apstag.fetchBids`; +- timestamps or phases indicating whether evidence was observed during + `initial_load` or `scroll`. + +The collector must not: + +- block, rewrite, or suppress publisher scripts; +- override `navigator.webdriver`; +- capture cookies, local storage, session storage, request bodies, or arbitrary + page data; +- require real GPT/APS network calls in test fixtures. + +## 8. JSON Output Contract + +`--json` emits deterministic JSON. Shape: + +```json +{ + "ok": true, + "strict": false, + "pages": [ + { + "url": "https://www.example.com/news/story", + "final_url": "https://www.example.com/news/story", + "requested_path": "/news/story", + "path": "/news/story", + "runtime_ad_stack_expected": "unknown", + "gates": { + "method_get": "pass", + "navigation": "pass", + "not_prefetch": "pass", + "not_bot": "pass", + "matched_slots": "pass", + "auction_enabled": "pass", + "consent_allows_auction": "unknown" + }, + "matched_slot_count": 1, + "slots": [ + { + "id": "atf", + "status": "confirmed", + "phase": "initial_load", + "configured": { + "div_id": "ad-atf-", + "gam_unit_path": "/123/news/atf", + "formats": [ + { "width": 300, "height": 250, "media_type": "banner" } + ], + "providers": ["aps"] + }, + "evidence": { + "dom_id": "ad-atf-0", + "gpt": { + "gam_unit_path": "/123/news/atf", + "div_id": "ad-atf-0", + "sizes": [[300, 250]] + } + }, + "warnings": [] + } + ], + "extra_evidence": [], + "warnings": [] + } + ], + "warnings": [] +} +``` + +Warning entries are objects with stable `code` and human-readable `message` +fields. Human output may print only the message. JSON consumers must not need to +parse warning strings. + +Extra live evidence is structured: + +```json +{ + "kind": "gpt", + "phase": "initial_load", + "dom_id": "ad-right-rail-0", + "gam_unit_path": "/123/publisher/right-rail", + "sizes": [[300, 250]], + "reason": "no_configured_slot_matched" +} +``` + +Allowed `kind` values for Phase 1 are `dom` and `gpt`. + +Strict-mode failures with page results use the same shape and set `ok` to +`false`. Example partial slot: + +```json +{ + "ok": false, + "strict": true, + "pages": [ + { + "url": "https://www.example.com/", + "final_url": "https://www.example.com/", + "requested_path": "/", + "path": "/", + "runtime_ad_stack_expected": "unknown", + "gates": { + "method_get": "pass", + "navigation": "pass", + "not_prefetch": "pass", + "not_bot": "pass", + "matched_slots": "pass", + "auction_enabled": "pass", + "consent_allows_auction": "unknown" + }, + "matched_slot_count": 1, + "slots": [ + { + "id": "homepage-header", + "status": "partial", + "phase": "initial_load", + "configured": { + "div_id": "ad-header-0-", + "gam_unit_path": "/123/homepage/header", + "formats": [{ "width": 728, "height": 90, "media_type": "banner" }], + "providers": ["aps"] + }, + "evidence": { + "dom_id": "ad-header-0-_R_abc123", + "gpt": null + }, + "warnings": [ + { + "code": "dom_without_gpt", + "message": "DOM element matched, but no GPT slot evidence was observed" + } + ] + } + ], + "extra_evidence": [], + "warnings": [] + } + ], + "warnings": [] +} +``` + +For errors that occur before any page result can be produced, the command exits +non-zero and prints the normal CLI error. JSON error output can be added later +if the base CLI standardizes it. + +For multi-URL runs, browser/navigation failures after argument validation are +page-level failures when possible. The command continues to the remaining URLs, +sets top-level `ok` to `false`, and includes a page result: + +```json +{ + "url": "https://www.example.com/broken", + "final_url": null, + "requested_path": "/broken", + "path": null, + "error": { + "code": "navigation_failed", + "message": "failed to read main document navigation response" + }, + "slots": [], + "extra_evidence": [], + "warnings": [] +} +``` + +Invalid schemes are still rejected before browser launch for the whole command, +because they are argument errors rather than page collection results. + +## 9. Error Handling + +Static commands fail when: + +- config cannot be loaded; +- `[creative_opportunities]` is malformed; +- CLI assertions in `check` fail. + +Browser verification fails when: + +- config cannot be loaded; +- any URL is not HTTP(S); +- Chrome/Chromium cannot be found or launched; +- all navigations fail before any page result can be collected; +- at least one page-level error occurs in a multi-URL run; +- command output cannot be written; +- `--strict` is set, runtime verification is not skipped by a known gate, and + at least one matched slot is missing or partial. `unconfirmable` is excluded. + +Browser collection can still produce a page result with warnings when: + +- page settle times out; +- a navigation redirects before final URL matching; +- scroll evidence is incomplete; +- GPT is not loaded; +- extra live DOM/GPT ad-slot evidence has no matched configured slot; +- no slots match the URL. + +## 10. Testing + +Static tests: + +- parse every `ts config ad-templates` command; +- load temp `edgezero.toml` and temp `trusted-server.toml`; +- verify `--app-config`, `--manifest`, and `--no-env` behavior; +- verify `/`, `/news/*`, and full URL normalization behavior; +- verify `check` success and failure output. +- verify the existing static command loader uses the shared `app_config` module. + +Pure comparison tests: + +- exact DOM ID match; +- prefix DOM ID match for framework-generated suffixes; +- ignore `-container` elements; +- GPT confirms by GAM unit path, div ID, and compatible sizes; +- DOM-only creates `partial`; +- no DOM/GPT creates `missing`; +- APS match creates no provider warning; +- APS missing/ambiguous creates provider warnings; +- `--strict` fails only missing and partial slots. + +Browser fixture tests: + +- local HTML fixture with direct `googletag.defineSlot`; +- fixture using `googletag.cmd.push`; +- fixture assigning `window.googletag` after collector injection; +- fixture with delayed/lazy slot observed only with `--scroll`; +- fixture with APS `fetchBids`; +- fixture assigning `window.apstag` after collector injection; +- redirect fixture that matches expected slots on final path; +- multi-URL fixture where one URL fails and one URL returns page results; +- fixture where `[auction].enabled = false` reports runtime skipped instead of + strict missing-slot failure; +- invalid non-HTTP(S) URL rejection before browser launch; +- JSON contract tests for warning codes, `extra_evidence`, page errors, + deterministic ordering, `partial`, `missing`, and strict failures; +- fixture with no real GPT/APS network dependency. + +Verification commands: + +```bash +cargo test --workspace +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test --package trusted-server-cli --target +``` + +## 11. Branch And PR Plan + +The implementation should not be built on stale #724. + +Recommended dependency order: + +1. Land or rebase PR #799 as the CLI base. +2. Rebase PR #800 onto the latest #799 head so `ts audit` uses the current typed + blob app-config model. +3. Harden and refactor the existing static `ts config ad-templates ...` + diagnostics on top of the current server-side ad-template branch and #799: + extract the private config loader into `app_config`, move pure expected-slot + logic into `ad_templates::expected`, and keep existing behavior covered by + tests. +4. Extend the rebased #800 collector with pre-navigation init scripts, + ad-template evidence hooks, optional scroll, page-level errors, and bounded + structured output. +5. Build `ts audit ad-templates verify` on top of that collector and the + server-side ad-template branch. +6. Keep `generate` for a separate Phase 2 spec and PR. + +If delivery needs to be split, static diagnostics can land before browser-backed +verification. Browser-backed verification should not duplicate the #800 browser +collector. + +## 12. CLI Namespace Decision + +`ts audit ad-templates verify` is the final command shape for browser-backed +ad-template verification. + +When this work is combined with the rebased #800 audit command, `ts audit` +should become a subcommand namespace: + +```bash +ts audit page +ts audit generate +ts audit ad-templates verify ... +``` + +The existing #800 `ts audit ` behavior should be preserved as a +compatibility alias for `ts audit generate ` during the transition, +including its artifact output flags. This avoids a successful but silent +behavior change for existing onboarding scripts. + +Parsing contract: + +- `ts audit page ` is the canonical generic page-audit command. +- `ts audit generate ` is the canonical artifact-generation command. +- `ts audit ad-templates verify ...` is the canonical ad-template verifier. +- `ts audit ` is a hidden compatibility alias for + `ts audit generate ` and is accepted only when `` parses as `http` + or `https`. +- `ts audit ad-templates` must never be treated as a legacy URL positional. +- `ts audit page` without a URL must fail with the normal Clap missing-argument + error. + +Implementation shape: + +```rust +#[derive(Debug, clap::Args)] +struct AuditArgs { + #[command(subcommand)] + command: Option, + #[arg(value_parser = parse_http_url, hide = true)] + legacy_url: Option, +} + +#[derive(Debug, clap::Subcommand)] +enum AuditSubcommand { + Page(PageAuditArgs), + #[command(name = "ad-templates", subcommand)] + AdTemplates(AuditAdTemplatesCommand), +} +``` + +If Clap cannot enforce the optional-subcommand plus hidden positional contract +cleanly, implement a small custom dispatcher for the `audit` argv tail and test +it directly. Required parser tests: + +- `ts audit https://www.example.com/` dispatches to artifact generation; +- `ts audit page https://www.example.com/` dispatches to page audit; +- `ts audit ad-templates verify https://www.example.com/` dispatches to + ad-template verification; +- `ts audit ad-templates` does not parse as a URL; +- `ts audit ftp://www.example.com/` fails before browser launch. + +JSON error output is intentionally left to the broader CLI output contract. This +spec only standardizes successful verification result JSON and strict-mode +verification failure JSON where page results exist. diff --git a/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md b/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md index 68e1cf75e..6a42bcec6 100644 --- a/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md +++ b/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md @@ -21,9 +21,10 @@ A fix must keep both implementations in sync. 1. A configured placement has at most one initial GPT slot and ad request when TS runs before a publisher defines its inner div. -2. Apply TS targeting and the `ts_initial=1` marker before that single initial - request. -3. Continue reusing a slot that the publisher has already defined. +2. Apply TS targeting and the `ts_initial=1` marker only when TS owns that single + initial request. +3. Continue reusing a slot that the publisher has already defined without changing + its targeting after a publisher auction, GPT request, or GPT render claims it. 4. Keep the TS-only fallback: if the publisher never defines the placement, TS still displays it and makes exactly one initial request. 5. Preserve `disableInitialLoad()`, SPA targeting cleanup, and the rule that TS does @@ -35,11 +36,36 @@ A fix must keep both implementations in sync. - Deduplicating by GAM ad-unit path. Multiple visible placements may validly share a path. - Changing publisher GAM configuration, line items, or refresh policy. -- Delaying the initial TS request while waiting an arbitrary amount of time for - framework hydration. A time-based grace period cannot distinguish a slow - publisher-owned slot from a placement that the publisher will never define. +- Delaying the initial TS request while waiting for a publisher that has not made a + concrete claim. A time-based grace period cannot distinguish a slow publisher-owned + slot from a placement that the publisher will never define. An actual publisher + `requestBids()` call receives a bounded lease instead. - General interception of unrelated GPT slots. +## Decision: first claimant owns delivery + +The first valid claimant owns each physical slot's first impression for the current +navigation. A real publisher `requestBids()` call claims before native Prebid starts. +A GPT `slotRequested` or `slotRenderEnded` event also claims for the publisher when TS +has not claimed first. `adInit()` may write `ts_initial=1`, apply `hb_*` targeting, and +request an existing slot only after it atomically claims an untouched slot. + +Publisher auction claims use unique, expiring registration tokens. The matching +callback associates returned ad IDs with only its registration in Prebid's delivery +correlation state. Overlapping auctions cannot clear each other's tokens. Exact ad-ID +delivery consumes only its matching registration. A code-only delivery consumes a registration only +when exactly one current candidate matches; ambiguous ordinary deliveries run a new +auction, while ambiguous TS-owned suppressing deliveries fail closed without deleting +their tombstones. If TS claimed first, the GPT refresh wrapper filters one correlated +losing publisher delivery and restores the TS targeting snapshot. It forwards every +unaffected slot and the original refresh options exactly once. The one-shot state is +then consumed, so later publisher refresh auctions remain eligible. + +If a publisher claim expires without a GPT request, `adInit()` retries only that slot +after checking the navigation generation, DOM element identity, and ownership again. +It never reruns whole-page initialization. Strict TS-first delivery is outside this +design because it would require holding publisher delivery while page-bids settles. + ## Decision: one inner-div slot with late-definition handoff TS will define its fallback slot on the **actual inner div**, never on its outer diff --git a/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md b/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md new file mode 100644 index 000000000..200a8abbe --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md @@ -0,0 +1,15 @@ +# Contiguous Generated Slot Tables Design + +## Problem + +`splice_creative_slots` parses rendered slots in a temporary `toml_edit::DocumentMut` and moves its `ArrayOfTables` into the target document. Parsed tables retain document-local numeric positions. Those positions collide with positions in the target document, so serialization can interleave generated slot and provider tables with unrelated top-level tables even though the resulting TOML remains semantically valid. + +## Design + +Before insertion, assign the generated slot tables and all nested provider tables the target `[creative_opportunities]` table's document position. `toml_edit` performs a stable position sort, so equal positions retain traversal order: the creative table, each slot, and that slot's provider tables remain contiguous. For a newly created creative section, allocate an anchor after the greatest existing parsed-table position. + +The update continues to preserve unrelated values, comments, line endings, and semantic table ownership. It does not reformat existing operator-authored content or modify slot inference. + +## Testing + +Add a regression fixture with a late `[creative_opportunities]` section and unrelated tables whose positions overlap those from the temporary generated document. Assert that the parent, generated slots, and provider subtables serialize contiguously before the next unrelated table. Retain the existing semantic-preservation and CRLF tests, then run the CLI test suite, formatting, and native CLI clippy. diff --git a/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md new file mode 100644 index 000000000..5ff4d0707 --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md @@ -0,0 +1,254 @@ +# PR 823 Review Resolution Design + +## Goal + +Resolve the actionable findings in review `4958563121` on PR 823 without +unrelated refactoring, verify the complete branch, publish the fixes, and reply +to every inline review thread with concrete resolution evidence. + +## Scope + +The implementation covers all 28 inline threads and all actionable items in the +review summary. The summary's explicitly out-of-scope pre-existing +partially-invalid `page_patterns` behavior is not expanded into this PR unless a +fix is required by another in-scope change. The PR description's stale legacy +alias sentence is corrected after the branch changes are published. + +Each reviewer suggestion is verified against the current code. A suggestion is +implemented when it is correct for this repository. Where repository evidence +contradicts a suggestion, the implementation retains the correct behavior and +the review response explains the evidence. + +## Design Principles + +- Preserve operator-authored configuration, comments, ordering, and unrelated + sections byte-for-byte wherever possible. +- Never print secrets or whole effective configuration documents as diagnostic + output. +- Never turn uncertain crawl evidence into a runnable fabricated ad-unit path. +- Treat browser navigation as a session, not a sequence of isolated launches. +- Keep `generate`, `verify`, static CLI commands, and runtime matching on shared + domain rules instead of parallel reimplementations. +- Bound all page-controlled data and browser operations. +- Use test-first changes for behavior corrections and minimal annotations for + code-quality-only corrections. + +## Component Design + +### 1. Configuration integrity and command output + +`slot_toml` will replace the line-oriented slot-boundary heuristic with a +TOML-aware edit strategy. The resulting document must preserve every top-level +item outside the managed creative-opportunity fields and preserve comments +adjacent to or between operator sections. Non-contiguous slot declarations, +multiline values, arrays whose continuation lines begin with `[`, trailing +comments, CRLF input, and inline-slot conversion receive regression coverage. +The updater will reject a candidate if preservation cannot be proven. + +Generation will re-read the source config immediately before the atomic write +and refuse to overwrite a concurrently edited file. `--dry-run` will emit only +the managed creative-opportunities change, never the complete config. Notes and +rollback warnings go to stderr so machine-readable stdout remains clean. Tests +will prove that dry-run leaves the source file byte-identical and does not expose +unrelated secret-bearing keys. + +Merge behavior remains add-only for operator-authored data: existing templated +unit paths are retained, newly observed formats are unioned, and multiple +discovered placements absorbed by one broad configured div prefix produce an +operator note. + +### 2. Crawl evidence and inference + +Inference will preserve evidence instead of silently collapsing it: + +- Non-ASCII shared-prefix computation uses UTF-8 byte boundaries. +- Same-page normalization collisions retain distinct raw placements and emit a + diagnostic rather than silently dropping formats. Numeric-only stable tokens + are not classified as hexadecimal hash noise. +- Multi-slot SRA request fallbacks are ignored when `dids` names more than one + slot. +- A page is considered empty only when no audited profile found slots there. +- Fragment detection requires stronger evidence: a useful shared prefix, or at + least three disjoint fragments. Ambiguous two-slot groups are retained with a + note. +- Locale landing paths are emitted literally when they are shorter than the + inferred section depth, and literal path segments are escaped before being + interpolated into globs. +- Refused template decisions are omitted from generated slots and surfaced with + their reasons. The documentation and tests will consistently describe these + cases as refusal, not literal fallback. +- The redundant witness rule is removed or made independently meaningful. The + actual crawler will support the section depth that inference can produce; + locale-prefixed behavior will not exist only in hand-built evidence tests. +- Dropped-section diagnostics are capped, percent-encoded paths are normalized + before filtering, and page-like extensions are classified consistently. + +The root page and section pages for a device profile are collected in one +browser session. Page analysis that parses full HTML is moved off the +current-thread CDP event pump. Each page/tab is closed on every success and +error path. + +### 3. Shared browser behavior + +The browser collectors will share executable discovery and launch/session +configuration. Browser options exposed to operators will have one meaning in +`page`, `verify`, and `generate`: Chrome override, settling, headful/headless +mode, device profile/viewport, proxy, consent assumption, cookies, and TLS +policy. + +`verify` will reuse one browser/runtime/profile across its URLs so clearance and +session state survive. The generic/legacy generator will default to the same +consent assumption as ad-template generation and expose the opt-out rather than +depending on `derive(Default)`. + +Cookie parameters are explicitly host-only with `Path=/`. A same-host +`http`-to-`https` upgrade is accepted with a redirect note; host changes, +downgrades, and unexpected port changes remain cross-origin refusals. Failure to +read or parse the final browser URL fails closed instead of substituting the +requested URL. + +Every post-navigation evaluation is time-bounded. The collector enlarges the +resource timing buffer before navigation, waits for an interactive or complete +document before accruing quiet time, honors sub-poll quiet windows, validates +`quiet <= max`, and reports saturation. Navigation load-event timeout is a +warning after a successful `goto`; it does not discard readable page evidence. +Evidence payload bytes and captured string lengths are capped before expensive +decode/allocation. + +Init-script and page-evaluation failures become explicit warnings or errors +rather than empty evidence. Promise-returning sitemap evaluation awaits its +result. Main-frame-only collection is disclosed when frames are skipped. + +The injected collector will be behavior-preserving: size pairs enforce the +`u32` range, the `googletag` setter is total, the unused non-variadic `cmd.push` +wrapper is removed, wrapping markers are closure-local/non-enumerable, and +page-derived warning text is terminal-safe. + +### 4. Runtime and static-command parity + +Expected-slot projection uses the runtime's renderability rule. Slots the +runtime omits for a path do not count as matched verification slots; diagnostics +state that the runtime omits the slot on that path rather than claiming the +whole config is rejected. + +Configured media type remains a typed `MediaType` through comparison and is +rendered to a string only at the output boundary. Slots that the phase-one +checker cannot confirm (video/native-only) are represented as unconfirmable and +do not fail `--strict`; genuinely partial or missing confirmable slots still +fail, including a live out-of-page slot with no sizes matched against +banner-configured formats, which is partial. Slot phase is absent when no +evidence exists. +The server-side APS compatibility field no longer creates unconditional +client-side `fetchBids` warnings. + +Collector warnings are included in page results. Human output includes the +runtime expectation, gate summary, matched count, extra evidence, and warnings +already present in JSON. Output escaping covers Unicode bidi controls and all +config-derived strings. + +`explain` reports exactly the shared runtime gate result. Provider configuration +is a separate advisory. The unsupported `--edgezero-enabled` model and stale +legacy-fallback claim are removed because no runtime condition backs them. +Gate diagnostics consume the shared gate result instead of rebuilding lists by +hand. The hot runtime gate avoids heap allocation, the seven-boolean wrapper is +removed, and the consent tri-state is documented and exhaustively tested. + +`compile_page_pattern` becomes crate-private and a public validation-only API is +used by the CLI. `lint` explicitly reports every configured page pattern the +runtime would drop, while the broader pre-existing runtime acceptance policy +remains out of scope. Specific compile failures are retained in logs. HTTP +methods use `http::Method` parsing so CLI semantics match the runtime. + +Full URLs and bare path inputs pass through the same URL normalization rules: +percent-encoding, dot-segment resolution, query/fragment removal, and leading +slash behavior must be identical. Scheme detection is anchored to the path +portion before `?`, so an absolute URL inside a query value does not cause a +bare path to be parsed as a full URL. + +### 5. CLI contracts, documentation, and CI + +Clap owns argument validation: URL parsing happens at the value parser, the +audit namespace uses help-on-missing-subcommand, `check` uses an argument group +and conflicts, and settle bounds are rejected during parsing. Parser tests cover +the visible command shapes and legacy restrictions. + +CI-oriented assertion failures exit 1; tool/configuration/navigation failures +exit 2. Assertion text is written directly and cannot disappear behind a log +filter. The guide documents all four `ts config ad-templates` commands, all +flags, shared config-loading flags, browser flags, consent/profile behavior, +dry-run output, and exit codes. + +Browser fixture CI either installs/resolves Chrome and requires the tests to +execute, or explicitly opts into a mode that fails when Chrome is unavailable; +it may not report success after silently skipping every browser assertion. + +All real-looking customer identifiers and names introduced by this PR are +replaced with fictional values in tests, comments, and documentation. Stale +module-level lint suppressions, inaccurate docs, assertion messages, enum +ordering, dead query matching, and orphaned comments are corrected without +unrelated cleanup. + +## Inline Review Traceability + +| Thread | Resolution area | +| -------------------------- | ------------------------------------------------------------------ | +| `3802056460`, `3802056470` | TOML-aware splice and comment/value preservation | +| `3802056474` | Secret-safe dry-run and stderr diagnostics | +| `3802056481` | Omit and explain refused slots | +| `3802056488` | UTF-8-safe div prefix calculation | +| `3802056494` | Same-page normalized-div collisions | +| `3802056497` | Locale landing-page patterns | +| `3802056502` | Multi-profile empty-page accounting | +| `3802056508` | Close every browser tab | +| `3802056513` | Enforce JavaScript-to-Rust `u32` bounds | +| `3802056521`, `3802056529` | Total GPT hook and removal of behavior-changing `cmd.push` wrapper | +| `3802056539` | Shared faithful browser launch configuration | +| `3802056549`, `3802056555` | Correct settling and load-timeout handling | +| `3802056559` | Preserve injected collector warnings | +| `3802056564`, `3802056571` | Runtime renderability parity and accurate diagnostics | +| `3802056580`, `3802056584` | Unconfirmable status and removal of false APS warning | +| `3802056586` | Identical URL and bare-path normalization | +| `3802056593` | Fictional committed examples | +| `3802056599` | Browser fixture CI must execute or fail loudly | +| `3802056605` | Add-only merge of formats with broad-prefix diagnostics | +| `3802056614` | Consent parity for generic and legacy generation | +| `3802056623` | Refusal behavior, tests, and documentation agree | +| `3802056628` | Safe same-host HTTP-to-HTTPS redirect handling | +| `3802056638` | Remove ungrounded EdgeZero fallback model | + +## Error Handling and Compatibility + +All new Rust fallible paths use the repository's existing `CliResult` / +`error-stack` conventions. Browser failures identify the operation and URL but +do not include cookies, configuration values, or page payloads. Best-effort +cleanup must not replace an earlier collection error. + +JSON compatibility is preserved where possible. New distinctions are additive +or correct semantically invalid fields: unconfirmable status is explicit, and +phase may be omitted when there was no evidence. Documentation is updated with +the exact wire behavior. + +## Verification Strategy + +Each behavioral issue follows red-green-refactor: + +1. Add the smallest unit, parser, orchestration, or fixture test reproducing the + review finding. +2. Run the narrow test and confirm the expected failure. +3. Implement the minimal correction. +4. Re-run the narrow test and the affected crate suite. + +Final verification runs the repository-required commands relevant to the +changed surface: CLI tests through `scripts/test-cli.sh`, target-matched Rust +tests, JS tests when the collector script changes, `cargo fmt --all -- --check`, +all target-matched clippy aliases, documentation formatting, and browser fixture +tests with an available Chrome. Any environment-dependent test that cannot run +is reported explicitly and is not described as passing. + +## Review Replies and Publication + +Changes are grouped into reviewable commits by component, then pushed to the PR +branch after final verification. Each inline reply is posted in its existing +thread and states the concrete change, relevant test, or evidence-backed reason +for retaining behavior. Replies avoid generic acknowledgements. Threads are not +replied to as fixed until the corresponding commit is visible on GitHub. diff --git a/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md b/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md new file mode 100644 index 000000000..e9025174d --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md @@ -0,0 +1,15 @@ +# Pre-navigation Cookie Installation Design + +## Problem + +The audit collectors open `about:blank` so initialization scripts can be installed before publisher code runs. Cookies are explicitly scoped by domain and `/`, but `chromiumoxide::Page::set_cookie` rejects cookies without a URL while the page is still `about:blank`. Consequently, any audit using `--cookie` fails before navigation; audits without cookies are unaffected. + +## Design + +Build the same host-only, root-scoped `CookieParam` values, then install them through `Browser::set_cookies` before creating the page. Browser-level installation sends the explicit domain/path cookie directly to Chrome without deriving scope from the current page URL. Both verification and generation collectors use one shared helper so their behavior cannot drift. + +Cookie-installation errors remain fatal and identify the affected cookie without logging its value. Page initialization, first-request authentication, browser-session reuse, and cookie scope remain unchanged. + +## Testing + +Add a Chrome-backed regression test that starts with `about:blank`, installs a cookie through the shared browser helper, navigates to a local HTTP fixture, and verifies the cookie is visible on the first loaded document. Run the focused CLI tests, formatting, and lint checks required for the touched crate. diff --git a/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md b/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md new file mode 100644 index 000000000..8f75d0dcf --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md @@ -0,0 +1,59 @@ +# Ad-template generation progress design + +## Problem + +`ts audit ad-templates generate` audits up to the configured page budget for +each selected device profile (17 pages by default). Navigation and page settling +are intentionally bounded but can still take tens of seconds per page. The +browser collector buffers page results until the browser session closes, so the +command currently emits no output during most of that work and appears stuck. + +## Design + +Emit line-oriented progress on stderr while collection is running. Progress +must identify the device profile, current page, known total, and safe page +location. It must also identify non-page phases where a noticeable pause can +occur: launching the browser, planning the crawl after the root page, and +finalizing the browser session. + +Progress is an explicit collector callback rather than direct terminal output +inside the browser implementation. This keeps output policy in the command +layer, makes the behavior testable with in-memory writers, and lets non-browser +collectors preserve the same contract. Each line is flushed immediately. + +The first profile's root navigation has no final total because follow-up pages +are planned from the rendered root. It is reported as `1/?`; once planning +finishes, subsequent pages use a stable `current/total` count. Later profiles +receive the complete target list and report the root as `1/total`. Totals include +the root, and every attempted page advances the current count even if collection +fails. + +Progress never prints a full URL. It renders only the origin-free path, omitting +userinfo, query, and fragment data, then applies the CLI's existing terminal-text +sanitizer. An empty path is rendered as `/`. + +Stdout remains reserved for the generated diff or success summary. This keeps +`--dry-run` and shell redirection stable. Progress is intentionally plain text, +not an animated spinner, so it remains useful in logs and does not add a terminal +UI dependency. + +## Error handling + +Failure to write or flush progress is returned as a normal CLI output error. A +callback failure during a browser session stops further collection but does not +skip finalization, browser close, or process wait. An earlier collection or +planning error takes precedence over a later progress error; either takes +precedence over teardown errors. Close and wait are still attempted +independently. No cookie values, URL credentials, query values, fragments, or +browser credentials are included in progress. + +## Tests + +Unit tests will verify that progress is emitted before collection completes, +contains the specified profile-aware page counts, keeps stdout unchanged, +redacts URL credentials/query/fragment data, sanitizes paths, and reports +finalization. Writer tests will cover write failure, flush failure, and explicit +flush invocation. Collector tests will verify teardown still runs after progress +failure and that collection/planning errors, progress errors, and teardown errors +retain the stated precedence. The existing CLI and Chrome-backed suites will +verify the collector behavior and browser lifecycle remain intact. diff --git a/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md new file mode 100644 index 000000000..bdc9c5b9c --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md @@ -0,0 +1,95 @@ +# Refuse Volatile Div-ID Collisions + +## Problem + +GPT discovery normalizes per-render div IDs such as +`ad-in_content--in_content-0` to the stable prefix `ad-in_content`. +When several live elements on the same page normalize to that prefix, the +runtime cannot represent them safely: one prefix resolves at most one element, +while each exact raw ID changes on a later render. The current collision path +preserves the raw IDs, causing `--replace` to write unusable literal slots. + +## Design + +Treat a source-local normalized collision as ambiguous and refuse the entire +group. The first observation remains tentatively accepted. When a second raw div +ID that describes a _different element_ normalizes to the same prefix, remove +the first slot, record the group as ambiguous, and suppress every later member. +Emit one diagnostic when the group first becomes ambiguous, naming the +normalized prefix and explaining that neither a single prefix nor volatile exact +IDs are safe. Tell the operator to expose distinct stable div IDs or prefixes in +publisher markup before configuring the placements. + +Two raw IDs sharing a stem are not by themselves two elements. One element +re-rendered under a fresh framework token produces exactly that shape, and +absorbing it is what normalization is for: a React publisher reports +`ad-header-0-_R_3f_` from the server render and `ad-header-0-_r_0_` from the +client one, and refusing that pair would generate no slots at all. The two cases +are separated by comparing what the ephemeral markers did _not_ cover — the +marker spans are excised and the remaining parts compared, so identical +residues mean one element observed twice, while `-in_content-0` against +`-in_content-1` means two siblings and is refused. + +The verdict is site-wide, not page-local. Article pages carry several in-content +units and refuse the shared prefix while a landing page carries one, so a +page-local refusal would let crawl sampling decide whether the ambiguous prefix +reaches the config. `DiscoveredSlots` therefore carries the refused stems, +`EvidenceTable` unions them across pages, and the slot iterator the writer reads +suppresses them regardless of which page contributed them. + +Registry and request-derived evidence retain separate collision maps, matching +the current source precedence: even an ambiguous registry stem continues to +suppress request fallback for that stem. Network-ID discovery is unaffected. + +`DiscoveredSlots` records whether any otherwise usable GPT slot evidence was +seen independently of how many safe slots remain. `EvidenceTable::fold_page` +uses that signal when classifying empty pages, so a collision-only page is not +mistaken for a bot challenge. Cross-page slot inference, merging, and +`--replace` otherwise remain unchanged because ambiguous slots never enter +those stages. + +Some ad stacks build IDs as `__`, where the +render token — at least ten leading digits followed by more alphanumerics, +that is, a millisecond timestamp plus entropy — sits _before_ the part that +distinguishes one placement from the next. Such an ID can be written neither +literally nor as a prefix: the only stable prefix stops at the token and reaches +every placement in the family at once. Discovery refuses a single otherwise +usable registry or request observation of that shape, preserves the page/network +evidence, and emits one diagnostic naming the family prefix. The shape decides +rather than a vendor name, so any stack with this layout is covered without a +code change, and every placement after the token is covered rather than an +enumerated few. A token in trailing position is _not_ this case — everything +before it still identifies the element — and is left to normalization and the +collision check. + +## Safety and Output + +The generator prefers omission over a configuration that cannot match future +renders. For an observed desktop crawl of a site with this mix, replacement +output should therefore contain the stable `ad-header-0` and `ad-fixed_bottom-0` +slots, while the in-content collision group and the volatile-token family are +explained in notes. + +## Tests + +- A two-element same-page normalization collision yields no slots and one + diagnostic containing the prefix, both unsafe alternatives, and operator + action. +- Two renders of one element (identical residues either side of the marker, + including a React server/client pair) collapse to one slot with no diagnostic. +- Repeats of the first and second IDs plus a third distinct ID after a collision + remain suppressed and do not create additional diagnostics. +- Request-derived collisions follow the same policy. +- An ambiguous registry stem still suppresses request fallback, and network-ID + discovery survives when every collided slot is omitted. +- A stem refused on one page stays refused after a later page contributes a + single member of the group. +- A collision-only page is recorded as having evidence rather than as an empty + challenge page. +- Single registry- and request-derived render-token observations are omitted + while retaining evidence and any parseable network ID, for every placement + suffix after the token. +- IDs with no render token, with a bare digit run, or with a trailing token stay + eligible. +- Existing normalization, request fallback, fragment detection, and full CLI + tests remain green. diff --git a/docs/superpowers/specs/2026-08-21-liveramp-integration-design.md b/docs/superpowers/specs/2026-08-21-liveramp-integration-design.md new file mode 100644 index 000000000..c4611ff08 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-liveramp-integration-design.md @@ -0,0 +1,856 @@ +# LiveRamp Integration Design + +**Issue:** [#355 — Investigate and document LiveRamp integration](https://github.com/IABTechLab/trusted-server/issues/355) + +**Parent epic:** [#354 — LiveRamp integration](https://github.com/IABTechLab/trusted-server/issues/354) + +**Initiative:** [#55 — Monetization integrations](https://github.com/IABTechLab/trusted-server/issues/55) + +**Status:** Draft PR implemented; external live validation partially complete + +**Date:** 2026-08-21 + +**Revised:** 2026-08-31 + +## 1. Executive summary + +LiveRamp integration is feasible in two distinct forms, but they must not be +treated as one protocol: + +1. **RampID identity-envelope forwarding through Prebid.js is feasible now.** + Trusted Server already bundles Prebid's `identityLinkIdSystem`, reads + `liveramp.com` EIDs through `pbjs.getUserIdsAsEids()`, sends them to + `/auction`, merges them with EC/KV identities, applies consent gating, and + forwards them to Prebid Server as OpenRTB `user.ext.eids`. +2. **LiveRamp ATS Direct audience segments are not part of that EID flow.** ATS + Direct returns a separate segment envelope and has separate subscription, + storage, TTL, deal-approval, and activation requirements. The cited + LiveRamp documentation describes activating these values as GAM `atsd` + targeting, not as a `liveramp.com` EID. + +The first implementation makes the existing RampID path operationally complete +through vendor-neutral `managed_user_ids` configuration under the Prebid +integration. The generated bundle command must resolve every managed config +name through the checked-in User ID registry and reject a manifest that omits +the corresponding module. Native server-to-server ATS resolution and ATS +Direct segment activation remain separate follow-up decisions. + +## 2. Issue hierarchy and collected requirements + +The GitHub issue hierarchy is: + +```text +#55 Initiative: Monetization integrations +└── #354 Epic: LiveRamp integration + └── #355 Task: Investigate and document LiveRamp integration +``` + +The work also references `IABTechLab/uid2-optout#385`, which tracked access to +LiveRamp test credentials. It is a related cross-repository dependency, not part +of the trusted-server issue hierarchy. + +The three trusted-server issues have empty or placeholder bodies, so their +comments and linked documentation define the operative requirements. + +### 2.1 Issue #354 + +The only comment asks the team to confirm whether LiveRamp segments are passed +to auction requests through the Prebid.js integration. This specification must +therefore distinguish identity envelopes from segment data and answer both +questions explicitly. + +### 2.2 Issue #355 + +The comments establish the following sequence and requirements: + +1. Review LiveRamp's Real-Time Identity Service (RTIS) tag documentation. +2. Wait for LiveRamp to clarify the integration. +3. Test the documentation LiveRamp supplied. +4. Review the ATS Envelope API page LiveRamp recommended. +5. Write a specification and determine feasibility. + +The issue's direct deliverable is an evidence-backed specification. If the +recommended path is feasible, implementation follows the approved design. + +### 2.3 Credential dependency + +The work references the cross-repository tracking issue +[IABTechLab/uid2-optout#385](https://github.com/IABTechLab/uid2-optout/issues/385), +named “Get test credentials from LR team.” The implementation owner has since +confirmed access to a test Placement ID and a MITM-assisted browser validation +environment. Those values remain outside the repository. + +Automated tests must not depend on LiveRamp configuration. A live Placement ID +and a LiveRamp-approved test origin remain necessary for the outstanding live +validation matrix, but their availability is no longer an implementation +blocker. + +## 3. Terminology and product boundaries + +### 3.1 RampID identity envelope + +Prebid's LiveRamp module is named `identityLinkIdSystem`, its configuration name +is `identityLink`, and its EID source is `liveramp.com`. It resolves an encrypted +RampID envelope into Prebid's identity APIs. The envelope identifies a user to +authorized demand partners; Trusted Server treats the value as opaque. + +### 3.2 RTIS + +LiveRamp's Real-Time Identity Service tag is a pixel or JavaScript tag that uses +LiveRamp cookie recognition and redirects a RampID to an endpoint registered +with LiveRamp. It requires LiveRamp to configure a tag ID and callback endpoint. +Trusted Server has no RTIS callback route today. + +RTIS is not selected for the first implementation because the managed Prebid +module already provides the browser-to-bidstream path, while a new callback +would require correlation, endpoint authentication, storage, abuse protection, +and a LiveRamp-specific server contract. + +### 3.3 ATS Envelope API + +The ATS Envelope API resolves hashed email, hashed phone, or configured custom +IDs into one or more encrypted envelopes. A server-to-server call requires a +Placement ID, a privacy-approved Origin, consent parameters where applicable, +and the browser's client IP in `X-Forwarded-For`. + +The ordinary ATS response contains an identity envelope with `type: 19` and +`source: "envelopeLiveramp"`. A no-consent response is HTTP 204. Configuration, +authorization, service, and geographic/consent failures use distinct 4xx +statuses. + +### 3.4 ATS Direct segments + +ATS Direct is a separate product layered onto an approved ATS placement and +subscription. Its V2 response can include `type: 26`, `source: "atsDirect"`, +whose value represents matching deal/segment IDs. LiveRamp documents storing +this in `_lr_atsDirect`, maintaining a region-dependent TTL, refreshing it, and +applying selected deal IDs to GAM under the `atsd` targeting key. + +An ATS Direct segment envelope is not a RampID and must not be placed in +`user.ext.eids` under `liveramp.com`. + +## 4. Current Trusted Server capabilities + +The following capabilities already exist on `main`: + +- `crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json` + includes `identityLinkIdSystem` in the default preset, maps the Prebid config + name `identityLink`, and maps EID source `liveramp.com`. +- `crates/trusted-server-js/lib/src/integrations/prebid/index.ts` reads + `pbjs.getUserIdsAsEids()`, validates EID structure, and includes valid EIDs in + the current `/auction` request. +- The same TSJS module persists structured OpenRTB-style EIDs in the first-party + `ts-eids` cookie after auction completion. +- `crates/trusted-server-core/src/auction/endpoints.rs` parses current-request + EIDs, loads server-resolved EIDs from the EC/KV graph, merges and deduplicates + them, and applies centralized consent gating. +- `crates/trusted-server-core/src/integrations/prebid.rs` serializes the merged + set to Prebid Server as OpenRTB `user.ext.eids`. +- `crates/trusted-server-core/src/ec/prebid_eids.rs` ingests `ts-eids` on a + later request and maps configured sources such as `liveramp.com` into the + EC/KV identity graph. +- The external bundle manifest and runtime diagnostics already identify which + Prebid User ID modules were compiled into the bundle. + +### 4.1 Bundle consistency + +The implementation validates managed entries against the checked-in User ID +module registry during `ts prebid bundle`. The CLI resolves each managed config +name through the registry, rejects unknown or ambiguous names, invalidates any +stale manifest before generation, and confirms that the fresh manifest contains +every required module before updating deployable hash metadata. Runtime +diagnostics retain the same defense for externally supplied or stale artifacts. + +### 4.2 Browser consent activation + +Bundling Prebid's consent collector and activity-control modules makes browser +enforcement available, but does not activate it. Prebid activates the TCF path +only after `consentManagement.gdpr` is configured. The managed User ID path +therefore initializes the standard IAB collector when all of the following are +true: + +- at least one managed User ID entry is configured; +- the publisher has not already configured `consentManagement.gdpr`; and +- the page exposes the IAB `__tcfapi`. + +The shim performs this check before seeding managed User IDs and before +`processQueue()`. It preserves every publisher-owned consent setting and does +not force GDPR scope or add a CMP configuration on pages without the TCF API. +This keeps the browser behavior vendor-neutral and avoids imposing GDPR latency +or defaults on non-TCF publishers. + +The effective value returned by `pbjs.getConfig("consentManagement")` is the +source of truth at installation time. If it is an object with its own `gdpr` +property, that property is publisher-owned and the shim leaves it untouched +regardless of its value, including `null`, `false`, or a partial object. If the +effective value is absent, or is an object without its own `gdpr` property, the +shim adds only: + +```js +{ + consentManagement: { + ...existingConsentManagement, + gdpr: { cmpApi: "iab" }, + }, +} +``` + +Prebid's timeout and `defaultGdprScope` defaults remain authoritative. Existing +sibling settings such as `gpp` are copied into the update. A non-object or +throwing effective value is not safe to merge: the shim logs a diagnostic and +does not replace it. + +The automatic update uses the original Prebid `setConfig` function and records +that the shim owns the resulting IAB collector. Publisher configuration already +applied before the shim therefore wins immediately. If a queued or late +`setConfig` or `mergeConfig` call later supplies an own `gdpr` value, including +`null` or `false`, ownership transfers to the publisher. Before forwarding that +call, the shim sends `gdpr.enabled = false` through the original `setConfig` API. +This invokes Prebid's supported consent reset path and removes the CMP event +listener when its ID is already known. Together with the callback guard below, +it prevents a later IAB event from overwriting publisher-owned static or custom +consent. + +Prebid cannot remove an IAB listener before the CMP returns its listener ID. To +cover that interval, the shim guards only the callback registered by its own +automatic activation. After ownership transfers, a delayed first response is +not forwarded into Prebid's consent handler; when it carries a listener ID, the +guard asks the TCF API to remove that stale subscription. The page's current +callable `__tcfapi` is used for removal, with the function captured at activation +as a fallback for pages whose API disappears. The page's global `__tcfapi` +function is restored immediately after activation, so publisher and CMP calls +outside that subscription are unchanged. + +The cleanup update preserves effective sibling consent settings. A following +publisher `setConfig` call retains its normal replacement semantics. Because +Prebid's `mergeConfig` deep-merges with the temporary disabled value, the shim +adds `enabled = true` only when the publisher supplied an object-valued `gdpr` +whose `enabled` value is missing or `undefined`; this restores Prebid's normal +enabled default without changing an explicit boolean publisher choice. The +publisher merge is prepared before the cleanup update. If it cannot be safely +inspected, cleanup is skipped so the temporary disabled value cannot leak into +the publisher's effective configuration. The transfer occurs at most once, +sibling-only consent updates do not claim GDPR ownership, and the shim never +re-applies its automatic minimum afterward. Throwing configuration accessors +are caught and logged rather than breaking shim installation. If effective +consent state cannot be read or copied during transfer, the shim does not issue +a replacement cleanup update that could erase unknown sibling state; the +guarded callback still rejects stale automatic responses, and the publisher call +is forwarded unchanged. + +## 5. Approaches considered + +### 5.1 Selected: vendor-neutral managed User IDs with bundle validation + +Add `managed_user_ids` to `PrebidIntegrationConfig`, inject the opaque entries +through `window.__tsjs_prebid`, and let the TSJS Prebid shim install and protect +each operator-owned Prebid User ID configuration before queued work is +processed. At bundle time, the CLI reads the same registry as the JavaScript +generator, resolves each managed config name, and confirms that the freshly +generated manifest contains every required module before updating hash/SRI +metadata. + +Benefits: + +- Uses the existing module, bundle generator, EID transport, consent gate, and + EC/KV ingestion path. +- Keeps browser identity configuration beside the Prebid bundle that consumes + it. +- Adds no new upstream route or PII-bearing server API. +- Fails an unusable managed-name/module pairing during the bundle command. +- Can be fully tested without external credentials, with a separate live + verification gate. + +Trade-offs: this only resolves identities visible to browser modules, and the +CLI must deserialize the registry's vendor-neutral module/config-name schema. +It does not add server-side HEM resolution or ATS Direct segments. + +### 5.2 Alternative boundary: standalone LiveRamp integration + +A new `integrations/liveramp` module could own browser and server APIs. This is +not needed for the browser path implemented by the current PR, while the ATS +API input contract and ATS Direct product scope remain unresolved. It would +also duplicate Prebid lifecycle and bundle validation responsibilities if added +before a server-side consumer is confirmed. + +Revisit this boundary if a future approved design adds server-to-server ATS +resolution or a non-Prebid LiveRamp consumer. + +### 5.3 Deferred: RTIS callback endpoint + +An RTIS endpoint would introduce a new unauthenticated redirect/callback +surface and a correlation problem without improving the already-supported +Prebid identity path. LiveRamp also requires per-endpoint configuration. It is +not implemented by the current PR and requires explicit team confirmation +before being treated as a follow-up requirement. + +### 5.4 Deferred: native server-to-server ATS resolution + +Native resolution is technically possible with Trusted Server's platform HTTP +abstractions, consent context, geo context, and client IP access. It is not +implementation-ready because: + +- Trusted Server has no approved source for hashed email, hashed phone, or a + LiveRamp custom ID. +- Sending a hashed identifier to LiveRamp is a privacy and publisher-contract + decision, not merely a transport detail. +- ATS API enablement and placement configuration for server-to-server use are + not confirmed by the browser Placement ID alone. +- Rate limits, timeout policy, caching, envelope refresh, and identifier + deletion semantics are not confirmed. +- [#630 — HEM Resolution (LiveRamp)](https://github.com/IABTechLab/trusted-server/issues/630) + was closed as not planned and must not be silently revived. + +## 6. Proposed configuration + +> **Revision, 2026-08-25.** An earlier draft of this section specified a typed +> `[integrations.prebid.liveramp]` subsection, which named a single identity +> vendor inside `trusted-server-core`. It is superseded by the vendor-neutral +> `managed_user_ids` surface below. RampID is now a configuration choice, not a +> type in core. + +Managed Prebid User ID modules are optional and nested under the existing Prebid +integration. Each entry is an opaque passthrough: core validates only what +Prebid needs to address the module, and never interprets `params`. + +```toml +[integrations.prebid] +enabled = true +server_url = "https://prebid.example.com/openrtb2/auction" +external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-.js" +external_bundle_sha256 = "" +external_bundle_sri = "sha384-" + +# RampID, expressed purely as operator configuration. +[[integrations.prebid.managed_user_ids]] +name = "identityLink" +params = { pid = "999", notUse3P = false } + +[integrations.prebid.managed_user_ids.storage] +type = "cookie" +name = "idl_env" +expires = 15 +refresh_in_seconds = 1800 +``` + +The Rust representation names no vendor: + +```rust +pub struct PrebidIntegrationConfig { + // Existing fields omitted. + pub managed_user_ids: Vec, +} + +pub struct PrebidManagedUserIdConfig { + pub name: String, + pub params: serde_json::Map, + pub storage: Option, +} + +pub struct PrebidManagedUserIdStorage { + pub storage_type: PrebidUserIdStorageType, + pub name: String, + pub expires: Option, + pub refresh_in_seconds: Option, +} + +pub enum PrebidUserIdStorageType { + Cookie, + Html5, +} +``` + +Validation: + +| Field | Rule | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Required. Non-empty, untrimmed-free ASCII token of letters, digits, `_`, `-`, or `.`. Unique across entries: Prebid keys `userSync.userIds` by name, so a repeat gives one submodule two conflicting configurations | +| `params` | Optional. Any TOML table; forwarded to Prebid without inspection | +| `storage.type` | Optional. `cookie` (default) or `html5` | +| `storage.name` | Required when `storage` exists. Same token rule as `name` | +| `storage.expires` | Optional. At least 1 when present; omitted leaves Prebid's default. No upper bound — a ceiling is the module's | +| `storage.refresh_in_seconds` | Optional. At least 1 when present; omitted leaves Prebid's default | + +Values that used to be typed defaults in core — `notUse3P = false`, +`idl_env`, 15 days, 1800 seconds — are now operator-supplied, because each is a +property of the module the operator selected rather than of Trusted Server. + +The operator selects both managed entries and bundle modules, but `ts prebid +bundle` validates that selection. It reads +`crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json`, the +same registry used by the JavaScript generator, and joins each managed `name` +against the registry's `configNames`. No vendor-specific mapping is compiled +into the CLI, and registry additions extend both the generator and validation. + +### 6.1 Bundle consistency validation + +The CLI performs focused validation in this order: + +1. Parse the managed User ID names alongside the existing bundle inputs. An + absent key becomes an empty list; a non-array value, non-table entry, or + missing, empty, or non-string `name` fails instead of being skipped. +2. Locate the JavaScript library and load its checked-in User ID registry. +3. Resolve every managed name to exactly one `moduleName`. Unknown or + ambiguously mapped names fail before generator invocation. +4. Remove only the exact `/manifest.json` file when it already exists. A + generator that returns success without writing a new manifest must not reuse + stale metadata from an earlier build. +5. Generate the external Prebid bundle normally. +6. Deserialize `userIdModules` from the newly written manifest. +7. Confirm that every resolved module appears in the manifest. +8. Update `external_bundle_sha256` and `external_bundle_sri` only after the + consistency check succeeds. + +Unknown-name errors identify the managed name and registry path. Ambiguous-name +errors additionally list the candidate modules. Missing-manifest-module errors +identify the managed name, its required module, and the corrective +`integrations.prebid.bundle.user_id_modules` field. A failed consistency check +leaves the existing config metadata unchanged. The browser-side warning remains +as defense in depth for externally built, stale, or modified artifacts. + +An empty `managed_user_ids` preserves current behavior and emits no managed User +ID configuration. + +## 7. Browser configuration and ordering + +The Rust Prebid head injector extends `window.__tsjs_prebid` with a camel-cased +`managedUserIds` array containing the validated entries. A Placement ID is an +operator identifier rather than a secret, but diagnostics must not copy +envelope values. + +The TSJS Prebid shim translates the injected object into: + +```javascript +{ + userSync: { + userIds: [ + { + name: 'identityLink', + params: { + pid: '999', + notUse3P: false, + }, + storage: { + type: 'cookie', + name: 'idl_env', + expires: 15, + refreshInSeconds: 1800, + }, + }, + ] + } +} +``` + +Publisher commands may already be waiting in `window.pbjs.que`, including a +`requestBids` command. Appending the managed configuration would be too late: +Prebid processes existing commands in insertion order, so a publisher auction +could run before the new entry. + +When one or more managed entries are configured, the shim instead installs +narrowly scoped, idempotent wrappers around the public `pbjs.setConfig` and +`pbjs.mergeConfig` APIs before calling `pbjs.processQueue()`: + +1. Capture and bind the real `pbjs.setConfig` and, when present, + `pbjs.mergeConfig` implementations. +2. Replace both public APIs with wrappers that normalize every call containing + a `userSync` object. Calls without `userSync` pass through unchanged. +3. Build a set containing every configured managed name. For a call with an + explicit `userSync.userIds`, preserve entries whose names are outside that + set, remove every publisher-supplied entry whose name is managed, and append + one fresh copy of each operator-managed entry in configuration order. + Preserve sibling `userSync` and top-level fields. +4. Calls whose `userSync` object omits `userIds` pass through unchanged. The + pinned generated Prebid artifact retains its effective `userIds` defaults + across partial `setConfig` and `mergeConfig` updates, so injecting a copied + list in the shim would duplicate Prebid behavior and make the wrapper depend + on a mocked configuration model that does not match the shipped artifact. + A real-artifact characterization test protects this pinned behavior. +5. During initial installation, read the already-effective + `pbjs.getConfig('userSync.userIds')` value, + normalize its supported array/config shape, preserve entries whose names are + not managed, append fresh copies of every managed entry, and apply that + merged list synchronously through the captured function. This covers + publisher configuration that ran after the external Prebid bundle loaded + but before the deferred TSJS shim. An absent or malformed effective list + degrades to an empty publisher list. Complete this step before processing + any existing queue entries. +6. Call `pbjs.processQueue()`. Queued publisher `setConfig` and `mergeConfig` + calls flow through the wrappers, so a later queued `requestBids` observes + the managed entry. +7. Keep the wrappers installed after queue processing so later publisher calls + through either public configuration API cannot silently replace or delete + operator-owned managed entries. Repeated TSJS installation must not stack + wrappers. + +This is configuration ownership for supported Prebid API usage, not a security +boundary against adversarial same-origin JavaScript that retained an earlier +function reference or mutates internal configuration objects directly. + +This policy gives the operator ownership of every configured managed entry. +Publishers retain ownership of all other Prebid and User ID configuration. +Omitting `managed_user_ids` installs no wrapper and preserves current publisher +behavior exactly. + +After queue processing, existing runtime diagnostics repeat the registry-backed +module check against the browser bundle stamp. This is a fallback for bundles +that were built externally, became stale, or were modified after `ts prebid +bundle`; a bundle created by the CLI has already passed the build-time check. + +## 8. Data flow + +```mermaid +sequenceDiagram + participant O as Operator config + participant TS as Trusted Server + participant B as Browser + participant LR as LiveRamp + participant PBS as Prebid Server + participant KV as EC identity graph + + O->>TS: Configure integrations.prebid.managed_user_ids + TS-->>B: Inject managed User ID config and Prebid bundle + B->>B: Guard setConfig/mergeConfig and merge managed entries + B->>LR: Prebid identityLink module resolves/refreshes envelope + LR-->>B: Opaque RampID envelope + B->>B: pbjs.getUserIdsAsEids() + B->>TS: POST /auction with source=liveramp.com EID + TS->>TS: Validate, merge, deduplicate, consent-gate EIDs + TS->>PBS: OpenRTB user.ext.eids + B->>B: Persist structured EIDs in ts-eids after auction + B->>TS: Later request with ts-eids + ts-ec + TS->>KV: Upsert configured liveramp.com partner UID +``` + +Identity resolution is asynchronous. The design does not promise a LiveRamp +EID in the first auction on a new browser. Current-request forwarding applies +as soon as `getUserIdsAsEids()` exposes the envelope; `ts-eids` and EC/KV +ingestion provide reuse on later requests. + +## 9. Consent, privacy, and security + +- Trusted Server continues to apply its centralized consent gate before EIDs + reach providers. No LiveRamp-specific bypass is introduced. +- Prebid's User ID and consent-management modules remain responsible for + deciding whether the browser may call LiveRamp. LiveRamp must be configured + correctly in the publisher's CMP/GVL posture. +- When managed User IDs are active and the publisher exposes `__tcfapi`, the + Trusted Server shim activates Prebid's standard IAB GDPR collector if the + publisher has not already configured one. Existing publisher + `consentManagement.gdpr` settings always win. Pages without `__tcfapi` are + unchanged, and Trusted Server does not synthesize GDPR applicability. +- Correction applied during implementation: `consentManagementTcf` only + _retrieves_ the TC string. Enforcement lives in Prebid's `tcfControl` + activity-control module, which the generated external bundle did not carry. + Without it a denied Purpose 1 still permitted the vendor call and the + `idl_env` write; only EID _forwarding_ was gated, server-side. The bundle now + imports `tcfControl`, covered by + `crates/trusted-server-js/lib/test/prebid-consent-enforcement.test.mjs`. + Equivalent GPP/US-state activity controls (`gppControl_usnat`, + `gppControl_usstates`) remain unbundled; US opt-outs are still enforced only + at the server's forwarding gate. +- Pinned Prebid's default `tcfControl` rules do not treat every denied purpose + identically. Purpose 1 plus the module's GVL vendor consent controls + IdentityLink device access, resolution, and storage. Purpose 2 controls bid + fetching. Purpose 3 has no standalone default `tcfControl` rule. Purpose 4 + controls user-provided-data activity. With the default + `eidsRequireP4Consent: false`, EID transmission is permitted when any Purpose + 2–10 has the required purpose/legal-interest and vendor basis; publishers may + opt into requiring Purpose 4 specifically. Therefore a Purpose 3 or Purpose + 4 denial alone does not establish that the LiveRamp vendor request or + `idl_env` write is blocked. Automated artifact tests must vary Purpose 1, + Purposes 3/4, and vendor 97 independently, and the operator guide must + describe these exact defaults rather than claiming that every denied purpose + blocks resolution. +- LiveRamp envelope values are opaque identifiers. They must never appear in + logs, public diagnostics, error bodies, or telemetry dimensions. +- The implementation does not collect plaintext or hashed email and does not + add an API for publishers to submit either value. +- The managed configuration preserves unrelated publisher User ID entries but + owns every configured managed name. A managed `identityLink` entry therefore + prevents ambiguous duplicate LiveRamp configurations without special-casing + LiveRamp in core. +- Existing EID size limits, source/UID validation, cookie caps, merge rules, + and consent withdrawal behavior remain authoritative. +- Live credentials and Placement IDs must not be committed to fixtures or + repository configuration. + +## 10. Error and degraded behavior + +| Condition | Behavior | +| ------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| `managed_user_ids` absent | Preserve current behavior; configure no operator-managed User ID entries. | +| Managed name is absent or ambiguous in the registry | Fail `ts prebid bundle` before updating config metadata. | +| Required module is absent from the generated manifest | Fail `ts prebid bundle` and name both the config name and required module. | +| An externally supplied runtime bundle omits a required module | Emit existing browser diagnostics; continue auctions without that module's EID. | +| LiveRamp network or recognition failure | Prebid module yields no EID; continue auction normally. | +| TCF Purpose 1 or LiveRamp vendor consent denied | Default `tcfControl` blocks IdentityLink resolution/storage; continue auction normally. | +| TCF Purpose 3 or 4 denied alone | Default rules do not prove resolution/storage is blocked; publisher policy may add stricter rules. | +| US-state opt-out | Server forwarding gate drops the LiveRamp EID; browser activity controls remain a documented gap. | +| Malformed LiveRamp EID | Existing client/server EID sanitizers drop it. | +| Oversized `ts-eids` payload | Existing bounded cookie behavior truncates whole UID/source entries; no partial UID is written. | +| EC/KV unavailable | Current-request EID can still reach `/auction`; persistence degrades without blocking the auction. | + +Trusted Server does not parse LiveRamp envelope contents and therefore cannot +distinguish authenticated ATS envelopes from cookie-recognized RTIS envelopes. +That distinction remains inside LiveRamp's module and encrypted envelope. + +## 11. Testing strategy + +Implementation follows test-driven development. + +### 11.1 Rust configuration tests + +Add tests in `crates/trusted-server-core/src/integrations/prebid.rs` and the +settings tests to prove: + +- managed entries deserialize with opaque nested `params`; +- documented storage defaults are applied; +- blank or whitespace-padded managed and storage names fail; +- invalid expiry and zero refresh values fail; +- unknown storage types fail; +- duplicate managed names fail; +- omission remains backward-compatible; +- serialized head configuration uses the expected camel-cased keys; +- script-breaking input cannot escape the injected script element. + +### 11.2 TypeScript unit tests + +Add tests in +`crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` proving: + +- no managed config produces no operator-owned entry; +- managed config creates the exact documented Prebid object; +- unrelated publisher `userIds` entries are preserved; +- a publisher-provided entry with a managed name is replaced, not duplicated; +- with at least two managed names, `setConfig` and `mergeConfig` preserve + unrelated publisher entries, replace publisher duplicates of both managed + names exactly once, and append fresh managed copies in configuration order; +- managed configuration is active before an already-queued publisher + `requestBids` call; +- queued publisher `setConfig` followed by `requestBids` preserves other User + ID entries while the auction observes the managed `identityLink` entry; +- User ID entries already effective before TSJS installation are preserved + while the managed `identityLink` entry is added; +- malformed pre-install `userSync.userIds` state degrades to the managed entry + without throwing; +- queued and late publisher `identityLink` updates through `mergeConfig` are + normalized back to the operator-managed values; +- a publisher `identityLink` update through `setConfig` after `processQueue()` + is normalized back to the operator-managed values; +- repeated installation does not stack either configuration wrapper; +- configuration calls without an explicit `userIds` list pass through + unchanged; +- missing `identityLinkIdSystem` appears in existing diagnostics; +- `getUserIdsAsEids()` output for `liveramp.com` enters the current auction; +- malformed and empty envelope values are dropped; +- envelope values are not written to logs or diagnostics; +- the existing `ts-eids` persistence path preserves the opaque value without + decoding it. + +### 11.3 Bundle tests + +Extend external bundle tests to prove: + +- the default preset contains `identityLinkIdSystem`; +- explicitly selecting it stamps the module into the manifest; +- the manifest stamps the exact selected User ID module list; +- a generated-real-bundle case that denies only Purpose 1 while granting + Purposes 3/4 and vendor 97 produces no LiveRamp request and no `idl_env`; +- a separate case that denies only vendor 97 while granting Purposes 1/3/4 + produces no LiveRamp request and no `idl_env`; +- separate cases that deny only Purpose 3 or only Purpose 4 while granting + Purpose 1 and vendor 97 still produce one LiveRamp request and write + `idl_env` under pinned Prebid's default rules. +- a generated-real-bundle case proves a partial `userSync` update retains the + publisher entry and exactly one managed `identityLink` entry. + +### 11.4 CLI bundle consistency tests + +Add focused tests in `crates/trusted-server-cli/src/prebid_bundle.rs` proving: + +- no managed entries preserve existing bundle behavior; +- a known name passes when its resolved module is in the manifest; +- a known name fails when its resolved module is absent; +- multiple managed names are checked; +- `pubCommonId` resolves to `sharedIdSystem`; +- multiple aliases may resolve to the same required module; +- an unknown name fails with an actionable registry error; +- an ambiguously mapped synthetic name fails deterministically; +- malformed `managed_user_ids` containers, entries, and names fail instead of + being skipped; +- unknown, ambiguous, and malformed-name failures occur before generator + invocation and leave hash/SRI metadata unchanged; +- a missing or malformed `userIdModules` manifest field fails; +- a pre-existing manifest is invalidated before generation, so a fake generator + that succeeds without writing a replacement cannot reuse stale metadata; +- omission of `bundle.user_id_modules` works with the generated default preset; +- failed consistency validation does not update hash/SRI config metadata; +- the checked-in registry maps `identityLink` to `identityLinkIdSystem`. + +### 11.5 Rust auction/EC regression tests + +Existing generic EID tests cover most transport behavior. Add or retain a +LiveRamp-named fixture proving that a `liveramp.com` EID: + +- is forwarded as `user.ext.eids` to the Prebid provider; +- merges without duplication against the EC/KV version; +- is removed when consent denies identity forwarding; +- is ingested into the configured `liveramp.com` EC partner namespace on a + later request. + +### 11.6 Managed browser-consent activation + +Extend the generated-artifact test before changing production code. The matrix +must prove: + +- managed User IDs plus a callable `__tcfapi`, with no publisher-side Prebid + consent configuration, activates `gdpr.cmpApi = "iab"` and blocks the + IdentityLink request and storage when Purpose 1 or vendor 97 is denied; +- no managed User IDs results in no automatic consent configuration; +- a missing or non-callable `__tcfapi` results in no automatic consent + configuration; +- an already-effective publisher `gdpr` value is preserved, including an + object and an explicit non-object value; +- existing sibling consent settings are preserved when the minimum is added; +- queued and late publisher GDPR configuration retains precedence; and +- the shim does not re-apply the automatic minimum after publisher changes. + +### 11.7 Live configuration validation + +Run outside CI against a LiveRamp-approved non-production origin: + +1. Obtain a test Placement ID and confirm the origin is approved. +2. Generate a Prebid bundle containing `identityLinkIdSystem`. +3. Configure a managed `identityLink` entry with the test Placement ID. +4. Load the publisher page with positive consent. +5. Confirm `idl_env` is created or refreshed according to the selected storage. +6. Confirm `pbjs.getUserIdsAsEids()` returns a `liveramp.com` entry without + recording its value. +7. Inspect a controlled Prebid Server request and confirm the same source is + present in `user.ext.eids`. +8. Confirm a later request can ingest the EID into the configured EC partner. +9. Repeat with opt-out/no-consent and confirm no LiveRamp EID is forwarded. +10. Repeat with an unapproved origin and document the expected degraded result. + +Record only booleans, source names, counts, and status codes. Do not capture or +publish live envelopes. + +Sanitized browser validation completed on the approved publisher origin: + +- an unresolved browser identity returned HTTP 204 and exposed no LiveRamp EID; +- a resolvable test identity returned HTTP 200, stored an envelope, and exposed + one `liveramp.com` EID; and +- automated generated-bundle coverage proves denied Purpose 1 or vendor 97 + consent suppresses the IdentityLink request and browser storage without + publisher-side Prebid consent configuration. + +The full live-validation acceptance criterion remains pending. A controlled +environment must still confirm live denied-consent behavior, unapproved-origin +degradation, the resulting `user.ext.eids` on the Prebid Server request, and +later EC/KV ingestion. These checks require publisher and LiveRamp test +conditions and are not replaced by the automated artifact suite. + +## 12. Documentation changes + +Implementation updates: + +- `trusted-server.example.toml` with a commented managed User ID example; +- `docs/guide/integrations/prebid.md` with configuration, lifecycle, bundle, + consent, troubleshooting, and verification guidance; +- `docs/guide/configuration.md` with the vendor-neutral managed field reference; +- optionally a short `docs/guide/integrations/liveramp.md` page if the Prebid + guide would become difficult to navigate. The first implementation should + avoid duplicating the authoritative Prebid flow across two pages. + +The documentation must state that: + +- RampID envelopes, not audience segments, are forwarded as EIDs; +- a Placement ID and LiveRamp-approved origin are operational prerequisites; +- the first auction may not contain a newly resolved identity; +- a module included in a bundle is inert until configured; +- ATS Direct segments require separate enablement and implementation. + +## 13. Rollout and observability + +1. Land configuration and tests with `managed_user_ids` empty by default. +2. Generate and publish a test bundle that includes `identityLinkIdSystem`. +3. Validate on a non-production approved origin with debug logging restricted + to source names/counts. +4. Enable for a canary publisher property. +5. Monitor missing-module diagnostics, LiveRamp EID presence counts, auction + error rates, and cookie/header size truncation counts. Never dimension + metrics by envelope value. +6. Validate opt-out behavior before broader rollout. +7. Document the tested Placement/origin configuration in operator-owned, + non-repository deployment records. + +No database or KV migration is required. Removing the managed entries provides +an immediate configuration rollback. + +## 14. Acceptance criteria + +Issue #355's implementation portion is complete when: + +- operators can configure LiveRamp RampID through vendor-neutral Trusted Server + config; +- invalid configuration fails before serving traffic; +- managed configuration preserves non-LiveRamp publisher User ID modules and + owns one deterministic `identityLink` entry; +- `ts prebid bundle` rejects unknown or ambiguous managed names and a generated + manifest that omits `identityLinkIdSystem` for `identityLink`; +- runtime bundle diagnostics retain the same missing-module defense for + externally supplied or stale artifacts; +- valid `liveramp.com` EIDs follow the existing browser → `/auction` → Prebid + Server path without exposing envelope contents; +- existing consent, validation, merge, cookie, and EC/KV behavior is preserved; +- automated Rust and TypeScript tests pass; +- a generated-bundle test proves that a denied TCF signal blocks managed + IdentityLink network access and storage without a publisher-side Prebid + consent configuration; +- operator documentation explains setup, timing, privacy, failure behavior, + and live verification; +- live configuration validation is completed and recorded without Placement ID + or envelope values; and +- the parent epic receives the explicit answer: RampID identity envelopes can + be passed through the Prebid auction path; ATS Direct segments are not passed + by this implementation. + +## 15. Out of scope and follow-up work + +### 15.1 Server-to-server ATS resolution + +Create or reopen a dedicated issue only after product approval. Its design must +define the hashed-identifier source, origin approval, consent mapping, +`X-Forwarded-For` handling, timeout/cache/refresh policy, geographic failure +behavior, data deletion, and credential storage. It must also reconcile the +decision that closed #630 as not planned. + +### 15.2 ATS Direct audience segments + +Create a separate issue if publishers require LiveRamp segment activation. It +must define: + +- ATS Direct subscription and approved-deal prerequisites; +- whether the integration calls the API or consumes existing browser storage; +- `_lr_atsDirect` and TTL ownership; +- refresh behavior and regional TTL rules; +- whether activation targets GAM (`atsd`), Prebid first-party data, a Prebid + real-time-data module, or more than one destination; +- consent and deletion behavior; and +- the exact evidence needed to confirm segment delivery. + +### 15.3 RTIS callback + +Do not add an RTIS callback unless a concrete non-Prebid use case demonstrates +that the browser module is insufficient and LiveRamp approves the endpoint +contract. + +## 16. Authoritative references + +- [LiveRamp: Implementing the Real-Time Identity Service Tag](https://docs.liveramp.com/identity/en/implementing-liveramp-s-real-time-identity-service-tag.html) +- [LiveRamp: Call the ATS Envelope API](https://developers.liveramp.com/authenticatedtraffic-api/docs/4-call-the-ats-envelope-api) +- [LiveRamp: Retrieving Envelope Endpoints](https://developers.liveramp.com/authenticatedtraffic-api/v1.0/docs/about-the-ats-api) +- [LiveRamp: ATS Direct](https://developers.liveramp.com/authenticatedtraffic-api/docs/implement-ats-direct-via-api) +- [Prebid: LiveRamp RampID User ID module](https://docs.prebid.org/dev-docs/modules/userid-submodules/ramp.html) +- [Prebid: User ID module](https://docs.prebid.org/dev-docs/modules/userId.html) diff --git a/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md b/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md new file mode 100644 index 000000000..8962f4862 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md @@ -0,0 +1,98 @@ +# PR 823 Round-5 Review Resolution + +## Goal + +Resolve review `4989897698` on PR 823 without weakening the generator's safety +rules, silently changing existing CLI defaults, or expanding the change beyond +the audit CLI and its documentation. + +## Browser and CLI Compatibility + +The hidden `ts audit ` compatibility form keeps accepting the same browser +flags as `ts audit generate `, but those flags must remain hidden and must +require the legacy URL positional. A dedicated `LegacyBrowserOpts` mirrors the +seven generation browser fields and converts into `GenerateBrowserOpts` when the +legacy command is dispatched. Consequently, flags placed before a real audit +subcommand are rejected instead of parsed and ignored. + +Generation retains its established 750 ms quiet period and 12-second maximum +settle wait. Generation defaults have one source of truth shared by clap, +`GenerateBrowserOpts::default`, and `BrowserAuditCollector::default`; applying +parsed options must not silently shorten the collector's maximum. The generic +page/verification collector keeps its existing independent 10-second default. + +Redirect notes show the origin and path for both requested and final URLs. This +makes scheme and host changes visible without exposing URL userinfo, queries, or +fragments. + +## Root-Less Template Safety + +Template inference records which slot stems borrowed the config-level +`section_root` because those slots were never witnessed on a path without the +configured section segment. Such a template is safe only while its page patterns +are derived from the paths where the slot was observed. + +Operator-supplied `--page-pattern` values replace those derived patterns for +every slot. If inference contains any borrowed-root slot and explicit patterns +were supplied, generation fails before rendering or writing a candidate config. +The error identifies the affected slots, explains that explicit patterns cannot +prove the borrowed-root invariant, and directs the operator to remove +`--page-pattern`. Failing the command is preferable to silently omitting real +inventory or attempting an unsound glob intersection. + +When no config-level section policy can be inferred because every otherwise +templatable slot lacks a root witness, each affected slot's refusal reason names +that crawl gap rather than claiming that its paths failed to generalize. + +## Merge Policy + +An explicitly configured `section_segment` is operator intent even when +`section_root` is currently unset. If preserved `{section}` slots exist and an +inferred policy would change that configured segment, merge fails and requires +`--replace` for the migration. If the configured segment matches, or is unset, +the inferred `section_root` may be adopted so the previously incomplete config +becomes loadable. + +## Diagnostics and Early Validation + +Warnings produced while folding a collected page include the device-profile +label as well as the path. Identical warnings from desktop and mobile therefore +remain distinguishable. The consent-stub warning remains a single unscoped +run-level note, and site-wide discovery warnings remain deduplicated. + +The existing config is parsed as TOML before Chrome starts. A whole-document +syntax error is returned immediately; a valid document with settings unknown to +the CLI still permits extraction of `[creative_opportunities]`; and a present +but unreadable creative section remains an error. + +The volatile div-id token recognizer requires at least ten leading digits plus +an alphanumeric suffix. This continues to recognize timestamp-like generated +tokens while preventing an eight-digit calendar date followed by a stable +letter from causing a single-observation family refusal. + +## Consistency Corrections + +Tests pin the Rust evidence cap to the embedded JavaScript collector constant. +The terminal-escaping test claims only controls it can actually inject; URL's +own percent-encoding is covered by an exact final-URL assertion rather than +presented as evidence for terminal escaping. Existing code escaping the final +URL remains as defense in depth. + +The affected guide, prior volatile-collision spec and plan, documentation +comments, `expect` message, and method spacing are corrected to describe the +implemented behavior exactly. The root-less templating behavior and this review +resolution are documented by this design and its paired implementation plan. + +## Testing and Delivery + +Every behavioral correction starts with a focused regression test that fails on +the current branch. Tests cover hidden legacy flags, the 12-second generation +default, complete redirect notes, borrowed-root rejection with explicit +patterns, configured-segment preservation, profile-specific warnings, +whole-document TOML failure, the evidence-cap invariant, and the calendar-date +token control. + +After focused tests pass, verification runs the host-target CLI suite and +audit/generate tests, CLI clippy with warnings denied, Rust formatting, docs +formatting, and `git diff --check`. No GitHub replies or push are part of this +change unless separately requested. diff --git a/docs/superpowers/specs/2026-08-24-ad-template-div-id-reconciliation-design.md b/docs/superpowers/specs/2026-08-24-ad-template-div-id-reconciliation-design.md new file mode 100644 index 000000000..9290e6e5c --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-ad-template-div-id-reconciliation-design.md @@ -0,0 +1,118 @@ +# Ad-template div-ID reconciliation design + +## Goal + +Prevent `ts audit ad-templates generate` from losing numeric sibling creative +opportunities during merge or persisting a singleton div ID whose middle token +is demonstrably per-render. + +This follows a live validation crawl. The crawl observed +`ad-sidebar-1`, `ad-sidebar-10`, and other siblings, but the merge treated the +configured literal `ad-sidebar-1` as a prefix and absorbed the longer IDs. It +also proposed one `vendor-tag_12345678AbCdEfGhIjKl_slot_overlay_1`-shaped slot +because the volatile-token classifier recognizes ten leading digits but this +token has eight. + +## Scope + +The change is limited to div-ID identity and volatility classification during +generation: + +- Preserve every distinct normalized, usable div identity retained by the + evidence table when an existing configured div ID was itself observed + exactly. +- Preserve intentional configured prefix behavior when that prefix was not + observed as a literal element ID. +- Refuse singleton IDs with a conservative eight-digit-plus-long-suffix token + shape in a non-trailing segment. +- Keep the existing warning and refusal behavior for ambiguous and fragmented + placements. + +This does not implement the broader cross-page-type preservation requested by +GitHub issue #1059, change crawl planning, or change runtime slot resolution. + +## Exact versus prefix reconciliation + +The generator already carries the div identities from `EvidenceTable::slots()` +into the TOML merge. This is intentionally not collector-level raw DOM input: +the identities have passed per-page normalization and usability checks, while +slots later rejected by template inference or cross-page fragmentation remain +present. Page-local volatile and ambiguous identities already refused by GPT +discovery do not re-enter reconciliation. + +The merge will classify a configured or newly appended slot as an observed +literal when its resolved div identity appears exactly in that normalized +evidence set. + +Matching proceeds in this order: + +1. Prefer an exact stable-key match. +2. Otherwise consider configured-prefix matches whose prefix was not observed + as a literal normalized div identity during this crawl. +3. Choose the longest remaining prefix, retaining configuration order for + equal-length ties. +4. Append the discovered slot when neither exact nor eligible prefix matching + succeeds. + +Consequently, `ad-sidebar-1` matches itself but cannot claim +`ad-sidebar-10`. A hand-authored broad prefix such as `ad-`, absent as a literal +DOM ID, retains its existing merge behavior. Newly appended discovered slots +are also protected because the decision is based on the normalized evidence +set, not only the original configuration indexes. + +The same reconciliation rules will drive observed/unobserved diagnostics so a +slot cannot be merged one way and classified for staleness another way. + +## Volatile token classification + +The existing vendor-neutral classifier refuses a div ID when a non-trailing +segment contains a per-render token before the placement suffix. It currently +recognizes a segment with at least ten leading digits followed by alphanumerics. + +Retain that rule and add a narrower alternative for shorter counters: + +- at least eight leading ASCII digits; and +- at least eight trailing ASCII alphanumeric characters in the same segment. + +The token must still occur before another div-ID segment. This catches the +`12345678AbCdEfGhIjKl` shape without claiming: + +- bare numeric placement IDs; +- seven-digit counters with long suffixes; +- eight-digit values with fewer than eight trailing characters, including + calendar-like `20260820a`; or +- trailing tokens whose preceding prefix can still identify the element. + +The warning remains vendor-neutral and names the stable family prefix. The slot +continues to count as evidence of an ad stack but is not rendered into config. + +## Diagnostics and failure behavior + +No new command failure is introduced. Unsafe singleton volatile slots are +skipped with the existing volatile-family note. Literal numeric siblings are +written separately and no longer produce the broad-prefix collision note. +Truly intentional broad prefixes can still produce that note when they claim +multiple observed divs. + +Normal merge continues to preserve configured slots. `--replace` retains its +existing replacement semantics. + +## Testing + +Use test-driven development with focused regressions: + +- A merge containing configured `ad-sidebar-1` and normalized observations for + `ad-sidebar-1`, `ad-sidebar-10`, and `ad-sidebar-11` must produce three slots. +- A configured `ad-` prefix that was not observed literally must continue to + merge multiple matching discovered divs and emit its collision note. +- Newly appended observed literals must not absorb later numeric siblings. +- A framework-bearing DOM ID normalized to a stable stem must classify the + matching configured stem as literal; identities refused during per-page GPT + discovery must not be reintroduced solely for merge classification. +- Registry and request evidence containing a singleton shorter high-entropy token + must be refused with the volatile-family warning. +- Boundary tests cover seven leading digits, eight digits with a seven-character + suffix, eight digits with an eight-character suffix, bare digits, and the + existing calendar-shaped example. +- Run the complete CLI suite, including the real-Chrome scrolling fixture, plus + formatting and the repository's target-specific verification gates. diff --git a/docs/superpowers/specs/2026-08-24-ad-template-generate-scroll-staleness-design.md b/docs/superpowers/specs/2026-08-24-ad-template-generate-scroll-staleness-design.md new file mode 100644 index 000000000..8709f6330 --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-ad-template-generate-scroll-staleness-design.md @@ -0,0 +1,98 @@ +# Ad-template generation scroll and staleness diagnostics design + +## Problem + +`ts audit ad-templates generate` currently collects each page only after its +initial settle. Unlike `ts audit page` and `ts audit ad-templates verify`, it +cannot request the deterministic scroll pass that triggers lazy ad inventory. +On a lazy-loading publisher site this produced fewer observable frames than a +scrolled page audit of the same page. + +Generation also merges by default, deliberately preserving configured slots +that the current crawl did not rediscover. That safety behavior is correct, but +it is silent: stale slots look as though the latest crawl confirmed them. + +## Scope + +Add opt-in scrolling to `ts audit ad-templates generate` and report configured +slots that a merge preserved without observing during the current crawl. + +This change does not prune slots automatically, enable scrolling by default, +alter crawl planning or budgets, change volatile-div refusal, or implement +GitHub issue #1059. `--replace` remains the only intentional pruning mode. + +## Command behavior + +`ts audit ad-templates generate` accepts a boolean `--scroll` option. Its +default is false, preserving current crawl cost and side effects. When enabled, +every page on every selected device profile performs the same deterministic +stepped scroll used by the existing page audit: scroll to 33%, 66%, and 100% of +the document, pause between steps, return to the top, then wait for the page to +settle again before reading HTML, GPT registry entries, and network evidence. + +The browser collector carries the option as session configuration so root, +planned section, desktop, and mobile page loads all behave consistently. Scroll +evaluation failures are best-effort page warnings; they do not discard evidence +that was already available after the initial settle. + +The implementation will share the deterministic scroll primitive with the +existing browser audit rather than maintain a second sequence of scroll steps. +Verifier-only evidence-phase bookkeeping remains in the verifier call path. + +## Merge diagnostics + +During a normal merge, generation tracks which pre-existing configured slots +matched at least one discovered slot. After processing all discovered slots, it +reports every unmatched pre-existing slot in configuration order. Those slots +remain unchanged in the output. + +The diagnostic is explicit about the limits of negative crawl evidence. Its +human-readable form for a non-scrolling run is equivalent to: + +```text +note: preserved 2 configured slot(s) not observed during this crawl: ad-header-0, ad-fixed_bottom-0. Re-run with broader coverage or --scroll; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover. +``` + +When the current run already used `--scroll`, the follow-up omits that redundant +suggestion and recommends broader page/profile coverage before intentional +pruning. + +No staleness diagnostic is emitted when all configured slots were rediscovered, +when there were no existing slots, or under `--replace`, because that mode does +not preserve unmatched slots. Matching uses the same reconciliation logic as +the merge itself, avoiding a second definition of slot identity. + +Diagnostics go to stderr through the existing generation-note path. Stdout +remains limited to the dry-run diff or successful write summary, so redirection +and machine comparison remain stable. + +## Safety and compatibility + +The default command behavior, merge result, and generated TOML remain unchanged +unless `--scroll` discovers additional evidence. The warning never mutates or +deletes operator configuration. It names only configured slot IDs and does not +include cookies, URL credentials, query strings, or fragments. + +Scrolling can trigger additional ad requests and publisher behavior, which is +why it remains explicit. Existing page-delay, settle-window, browser-proxy, +certificate, cookie, and device-profile behavior applies unchanged. + +## Tests + +CLI parsing tests cover `--scroll` and its false default. Browser-collector tests +use a deterministic local page that defines a GPT slot only after scrolling and +prove that generation captures it with the option enabled but not without it. +Existing browser lifecycle and settle tests continue to cover teardown and +timeouts. + +Merge unit tests cover multiple unmatched configured slots, stable diagnostic +ordering, partial rediscovery, full rediscovery, an empty existing config, and +`--replace`. Command-level tests verify that the warning reaches stderr while +stdout and the preserved generated configuration retain their existing +contracts. + +Verification will run the host CLI test suite and relevant Chrome-backed CLI +tests, followed by the repository-required formatting and CLI lint gates. A +manual dry run against a live publisher site may be used when a fresh +bot-protection cookie and proxy are available, but network-dependent behavior +is not a required CI test. diff --git a/docs/superpowers/specs/2026-08-24-request-phase-timing-design.md b/docs/superpowers/specs/2026-08-24-request-phase-timing-design.md new file mode 100644 index 000000000..e3d9d6e50 --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-request-phase-timing-design.md @@ -0,0 +1,673 @@ +# Request phase timing: Server-Timing subtimings and access telemetry + +**Date:** 2026-08-24 +**Status:** Approved design, revised for review rounds 1 and 2, pending implementation +plan. +**Scope:** `trusted-server-core`, Fastly and Axum adapters, `tinybird/` schema, +performance dashboard (separate repo). + +--- + +## 1. Problem + +On 2026-08-21 a production deployment (publisher redacted, `prospect-a.example`) showed +an episodic stall: for a window of roughly 40 minutes, every request that reached the +application path carried a uniform extra ~600 ms of Fastly `time-elapsed`, and then +recovered to 20-50 ms with no deploy or config change we could observe. `/health` +(2-4 ms, short-circuits before app construction) and `/_ts/debug/ja4` (6-9 ms, settings +load only) stayed fast throughout, so the stall lived between app construction and +response send. + +Attributing that window required a live probing session: route-by-route bisection, +cookie-deletion experiments, and an eight-agent code trace. The trace found no +unconditional await on the path that could cost 570 ms, and exactly two +config-conditional candidates (the pre-route request filter's synchronous verification +POST, and EC identity KV writes before send), plus one dependency shared by every +application route (two geo hostcalls per request). We could not tell which one stalled, +because nothing in the response says where server time went. + +The Compute CPU budget is ~50 ms per request, so a large `time-elapsed` strongly +suggests wall-clock time outside active guest CPU: dependency awaits are the leading +explanation, with platform scheduling and hostcall queueing as the residual ones. The +comparison figure here is the fronting delivery layer's `time-elapsed` Server-Timing +entry, observed at its deliver phase. Either way, these are exactly the numbers a +response can carry about itself. + +## 2. Goals + +1. Every normal application response attributes its own server time by phase in a + standard header. Browsers expose the values to same-origin JavaScript via + `PerformanceResourceTiming.serverTiming`, so RUM tooling that reads that API can + surface the breakdown. Whether a given vendor or the publisher's own monitoring + extension actually collects it is verified separately in rollout; the publisher + extension needs a small change to render it. +2. The same numbers flow to Tinybird so we hold p50/p95/p99 per phase, per route class, + per PoP, per deployed version, and a future stall window self-diagnoses in one query. +3. No additional awaited I/O before first byte. The pre-send cost is a handful of + monotonic clock reads, one small allocation at entry, and rendering one header; + telemetry emission happens strictly after the last body byte. + +Scope note: phases cover the application lifecycle after T0. The `/health` and +`/_ts/debug/ja4` short-circuits, config-store open failures, and request-conversion +failures bypass the lifecycle and emit nothing. Requests served entirely by the +fronting cache never reach the guest and produce neither header entries nor rows. + +## 3. Non-goals + +- No trailer-based Server-Timing for body-phase spans (browsers do not expose trailer + values to JavaScript). +- No per-filter naming in any emitted surface. The request-filter span is `ts-filter` + regardless of which filter runs; vendor identity stays out of headers and telemetry. +- No Cloudflare or Spin emission wiring in v1. Core collection is adapter-neutral; those + adapters can wire emission later without core changes. +- No Tinybird endpoint pipe and no rollup materialized views in v1. Grafana queries the + datasource through the ClickHouse connector, matching the auction dashboards; rollups + only if panel latency demands them. +- No sampling of the header. The header is all-traffic when enabled; only Tinybird rows + sample. +- No cross-request circuit breaker for telemetry emission. Compute runs one isolate per + request; there is no shared mutable state to hold breaker state. The controls are the + bounded per-request cost and the `access_sample_rate` lever (section 10). + +## 4. Design overview + +``` +adapter entry (T0) + | RequestTimings::new() -> shared handle + v +app construction ................ ts-appbuild (adapter) +pre-route request filters ....... ts-filter (adapter wrapper) +geo lookup (single, deduped) .... ts-geo (adapter; result carried forward) +template cache lookup ........... ts-template-cache (core: publisher.rs) +origin fetch to resp headers .... ts-origin (core: publisher.rs) +EC identity KV, pre-send ........ ts-kv (core: KV abstraction) +auction wait, buffered mode ..... auction_wait_ms (row only; pre-header in this mode) + | +send_edgezero_response, immediately before into_parts(): + mark_headers_ready() snapshot (unconditional) + build AccessTelemetrySnapshot (unconditional) + append Server-Timing header (flag-gated, only on conclusively private responses) + | +headers committed; body streams + auction hold at seam .......... auction_wait_ms (row only; in-stream in this mode) + stream duration, bytes ........ stream_ms, resp_bytes (row only) + | +post-send (adapter main): + request_elapsed snapshot, then existing pull-sync, then: + sample gate -> one NDJSON row -> Tinybird Events API + bounded response await, 2xx validated +``` + +Collection is always-on and flag-free, including the `mark_headers_ready()` snapshot. +Two independent flags gate emission: the header (`observability.server_timing_enabled`) +and the telemetry row (`tinybird.access_enabled`). + +## 5. `RequestTimings` (core) + +New module `crates/trusted-server-core/src/request_timing.rs`. + +- `Phase`: a closed enum: `AppBuild`, `Filter`, `Geo`, `EcKv`, `Origin`, + `TemplateCacheLookup`, `AuctionWait`, `Stream`. Header rendering covers the first six + plus the stored total; the last two are row-only. +- Inner state: one fixed-size array of `Option` slots indexed by phase, + `t0: Instant`, `headers_ready_total: Option`, + `auction_wait_placement: Option` (`PreHeader` or `InStream`), + and `resp_bytes: Option`. Phases that repeat within a request (geo, KV) + accumulate by saturating addition into the same slot. +- `mark_headers_ready()`: stores `t0.elapsed()` once at the response-commit boundary, + unconditionally, before either emission flag is consulted. The header renders this + stored value as `ts-total`; the telemetry row reads the same stored value as + `time_elapsed_ms`. The two surfaces cannot disagree, and the row stays correct when + the header flag is off. Full request duration is captured separately as + `request_elapsed_ms`, snapshotted immediately after the body-stream drive returns and + before any other post-send work, so pull-sync and telemetry emission are never + included in it. +- Sharing: `RequestTimings` is a cheap-clone handle, `Arc>`. It crosses + three boundaries: adapter entry to core handlers, the streaming body closure (records + body-phase spans after the response object has been handed off), and the adapter's + post-send emission read. Access is exclusively `try_lock()`: a contended or + poisoned lock drops the sample immediately rather than waiting, so recording can + never delay a request. +- Recording API: `timings.record(Phase::Geo, dur)` and a scope guard + `timings.span(Phase::Origin)` that records on drop. Guards use saturating duration + math; a non-monotonic reading records zero rather than panicking. The auction-wait + recorder takes the placement explicitly so the two modes cannot be conflated. +- Rendering: `server_timing_value(&self) -> Option` produces + `ts-total;dur=41.2, ts-appbuild;dur=18.4, ts-filter;dur=9.1` with durations in + milliseconds at one decimal. Phases never recorded are omitted. Returns `None` when + `mark_headers_ready()` has not run. + +`Instant` is already used freely in the guest (`publisher.rs`, `auction/telemetry.rs`), +so no new clock abstraction is needed. + +## 6. Span taxonomy and recording sites + +| Entry | Measures | Site | +| ------------------- | ------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| `ts-total` | T0 to `mark_headers_ready()` at the response-commit boundary | stored snapshot | +| `ts-appbuild` | config-store open, Settings parse, orchestrator + registry + router build | Fastly `main.rs` around `open_trusted_server_config_store()` + `build_app_with_state()` | +| `ts-filter` | pre-route request filters, end to end (backend ensure, secret read, POST) | around `run_pre_route_filters` (`app.rs:751`) | +| `ts-geo` | geo hostcall (single after dedupe; accumulates if any path still repeats) | `build_ec_request_state` (`app.rs:410`) and timed finalizer fallback lookups | +| `ts-kv` | EC identity KV operations before response send (see enumeration below) | the shared KV abstraction | +| `ts-origin` | publisher backend send to response headers available (read-through cache hit or miss) | `publisher.rs` around the origin `send` | +| `ts-template-cache` | template cache `lookup_or_reserve`, including the hit-path full-body read | `publisher.rs` around the lookup | + +Naming follows the completed template-cache terminology migration (`x-ts-template-cache` +is the emitted header on `main`; `c2` naming is retired). + +`ts-kv` is instrumented by a timing decorator implementing `PlatformKvStore` that +wraps the store handed to request-scoped consumers, because no single existing +abstraction covers the taxonomy: EC graph operations go through `KvIdentityGraph` +while consent persistence uses `PlatformKvStore` directly, and graphs are constructed +independently in request setup, identify, admin lookup, batch sync, and finalization. +Every request-path graph construction receives the timed store; pull-sync explicitly +constructs its graph from an untimed store. Consent-store reads pass through the same +decorator and are timed like any other store call. Included pre-send operations: EC +generation `create_or_revive`, identify-path graph reads and evaluation, finalize-path +`ingest_eid_cookies`/`upsert_partner_ids` and withdrawal tombstones, consent-store +reads on consent routes, and batch-sync graph access when it runs before send. +Explicitly excluded: pull-sync work, which runs strictly after `send_to_client` and is +invisible to both surfaces. The decorator measures store-call latency only: no value +passing through it is read, parsed, or recorded, and the emitted surfaces carry no +consent or identity payloads. This feature therefore needs no consent gate; it is the +site measuring its own infrastructure, not processing user data. The timings handle +reaches `ec_finalize_response` inside the graph it already receives; that function +keeps the repository maximum of seven arguments and does not gain an eighth. + +Row-only fields: + +| Field | Measures | Site | +| -------------------- | --------------------------------------------------------- | ------------------------------------------------ | +| `auction_wait_ms` | wait on the dispatched auction (placement varies by mode) | seam hold (streaming) or buffered finalizer wait | +| `body_mode` | `streamed` or `buffered` response assembly | set where the response body is built | +| `stream_ms` | headers committed to last body byte | adapter around the body-stream drive | +| `resp_bytes` | bytes written to the client body | same | +| `request_elapsed_ms` | T0 to immediately after the body-stream drive returns | post-send snapshot, before pull-sync | + +Auction-wait placement is not universal. On the ordinary streaming path the wait +happens at the `` seam inside the body stream and nests inside `stream_ms`. On +buffered paths (the Fastly shared-template authorized miss, which buffers the full +transform and auction before returning a response, and every Axum response) the wait +completes before headers commit. The row therefore carries `body_mode` plus +`auction_wait_placement` (`pre_header` or `in_stream`), and derivations are +conditional: + +- `in_stream`: `stream_other_ms = greatest(coalesce(stream_ms, 0) - coalesce(auction_wait_ms, 0), 0)`. +- `pre_header`: `auction_wait_ms` joins the pre-header phase set, and `stream_other_ms = coalesce(stream_ms, 0)`. + +`unattributed_ms = greatest(coalesce(time_elapsed_ms, 0) - (coalesce(appbuild_ms, 0) + +coalesce(filter_ms, 0) + coalesce(geo_ms, 0) + coalesce(kv_ms, 0) + +coalesce(origin_ms, 0) + coalesce(template_cache_ms, 0) + pre-header auction wait), 0)`. +Every phase column is nullable, so every query-time formula wraps each term in +`coalesce(column, 0)` and every subtraction in `greatest(..., 0)`; query tests cover +sparse phase combinations. + +## 7. Freeze point and header emission + +The freeze-and-emit point is `send_edgezero_response` (Fastly `main.rs`), immediately +before `response.into_parts()`. This is the single choke point every send path shares, +and it runs after everything that can still mutate the response: the router middleware, +entry-point finalize (`apply_finalize_headers`, asset-policy reapplication), EC +finalization and its KV work, and terminal filter/privacy effects. +`apply_finalize_headers` itself does not emit; the `HEADER_X_TS_FINALIZED` sentinel +marks middleware finalization, not header commitment, and must not be treated as the +timing boundary. + +At the freeze point, in order: `mark_headers_ready()` (unconditional), the +`AccessTelemetrySnapshot` build (unconditional, section 10), then, gated on +`observability.server_timing_enabled`, append one `Server-Timing` header from +`server_timing_value()`. Append semantics, never insert: an origin-supplied +Server-Timing survives, and the fronting delivery layer's own entries (`time-elapsed`, +`hit-state`) are additive per the header's list semantics. + +Header emission is conservative: it happens only when the response is conclusively +non-storable by any shared cache, meaning `Cache-Control` contains `private` or +`no-store` (the existing `cache_control_headers_are_private_or_no_store` predicate). +Anything else, including bare `max-age`, `s-maxage` without `private`, +heuristically-cacheable responses with no cache header at all, and anything a fronting +cache override might store, emits no header, because a stored object would replay one +request's timings for its full lifetime. The long-lived immutable `tsjs` asset route +is the concrete excluded case. The snapshot and the telemetry row are unaffected by +this skip, so excluded routes still report through Tinybird. + +The Axum adapter applies the same emission rule at its terminal point before response +serialization, with adapter-specific phase semantics (section 8a). + +## 8. Geo lookup dedupe (rider) + +Today every dispatched request pays two geo hostcalls for one answer: request-phase in +`build_ec_request_state` (`app.rs:410`) and response-phase in +`FinalizeResponseMiddleware` (`middleware.rs:83`, alternate site `main.rs:285`). + +Plain request extensions cannot carry the result out: the middleware moves the request +context into `next.run(ctx)` and holds only the response afterward. The resolved geo +travels on a dedicated `GeoLookupState` response extension, attached on every exit +path that attempted a lookup, including the asset fallback, which runs +`build_ec_request_state` and then returns without `EcFinalizeState` (which is why +`EcFinalizeState` is not an acceptable carrier). States: `NotAttempted`, +`Attempted(None)` (lookup ran and failed, do not retry), and `Resolved(GeoInfo)`. The +finalize path consumes the carried value and performs a live lookup only in the +`NotAttempted` state; those legitimate fallback lookups (admin, batch, error paths) +are themselves timed into `ts-geo` so degraded geo cannot hide inside +`unattributed_ms`. The 401 rule (`resolve_geo_for_response` skips lookup for +unauthorized responses) is preserved. + +## 8a. Adapter phase semantics + +The Fastly adapter is the reference implementation of the taxonomy. Axum differs +structurally and its emissions are defined accordingly rather than pretending parity: + +- `ts-appbuild` is absent: Axum builds application state once at startup. +- `body_mode` is always `buffered`: the Axum HTTP client buffers upstream bodies, so + `stream_ms` measures buffered-body write-out and `auction_wait_placement` is always + `pre_header`. +- The freeze point is an outer service wrapper around the `RouterService` inside + `AxumDevServer`, not router middleware: router-generated 404/405 responses bypass + router middleware, and middleware returns before Axum serializes the body. The + wrapper sees every response including router-generated ones; `/health` is excluded + by path match inside the wrapper. +- Axum emits the header only; no Tinybird rows in v1 (unchanged). + +Cloudflare and Spin: collection compiles, no emission wiring in v1 (unchanged). + +## 9. Access telemetry row + +Extends the reserved `tinybird/datasources/access_logs_raw.datasource`. + +Kept columns: `event_ts`, `method`, `status`, `time_elapsed_ms` (defined as the +`mark_headers_ready()` snapshot; nullable because a contended lock drop can lose the +snapshot), `sample_rate`, 30-day TTL. (`event_date` was later dropped for the +`toDate(event_ts)` sorting-key expression; see section 9's schema note.) + +Removed: raw `path`. Route identifiers like `/_ts/admin/ec/{id}` would otherwise put +EC identifiers into a 30-day dataset, and publisher paths carry unbounded cardinality +and user-generated content (search terms, usernames, emails in slugs). Replaced by +`route_template`: + +- Named routes: the matched route-table pattern verbatim, parameters left as + placeholders. +- Publisher fallback: a coarse fixed template, `/` plus the first path segment + restricted to a bounded allowlisted charset, plus `/*` when deeper (for example + `/news/*`). The auction-telemetry normalizer is explicitly not sufficient here: it + redacts long tokens but preserves short identifiers and arbitrary slugs. +- Rejection is whole-segment, never truncation: a segment is dropped to `/other/*` + when it fails the charset allowlist, exceeds 32 characters, carries more than 7 + ASCII digits, or is the only segment in the path. Depth is what makes a first + segment a section name: single-segment paths are documents (WordPress + `/%postname%/` puts every article at depth 1), so they reject wholesale, root + landing pages included. The character allowlist alone does not bound identity (`[a-z0-9_-]` + is exactly the alphabet of UUIDs, hex ids, and reset tokens), and a truncated + prefix of any of those is still identifying, so the length and digit bounds reject + the segment outright. +- Tests are adversarial, not just the happy path: a literal EC identifier on the admin + route, an email address in a path segment, search-term-shaped segments, overlong + segments, UUIDs, hex ids, reset tokens, and full article slugs must all normalize + to bounded, content-free templates. + +Added columns (all dimension columns non-nullable with an `unknown` sentinel, because +ClickHouse sorting keys cannot contain nullable columns): + +``` +`service_id` LowCardinality(String), -- FASTLY_SERVICE_ID; immutable deployment identity +`publisher_domain` LowCardinality(String), -- matches auction schema +`env` LowCardinality(String), -- adapter-derived: production | staging | unknown +`route_class` LowCardinality(String), -- publisher_html | tsjs | integration_proxy | ec | auction_api | other +`route_template` String, -- bounded, normalized; replaces path +`body_mode` LowCardinality(String), -- streamed | buffered +`auction_wait_placement` LowCardinality(String), -- pre_header | in_stream | none +`appbuild_ms` Nullable(UInt32), +`filter_ms` Nullable(UInt32), +`geo_ms` Nullable(UInt32), +`kv_ms` Nullable(UInt32), +`origin_ms` Nullable(UInt32), +`template_cache_ms` Nullable(UInt32), +`auction_wait_ms` Nullable(UInt32), +`stream_ms` Nullable(UInt32), +`request_elapsed_ms` Nullable(UInt32), +`resp_bytes` Nullable(UInt64), +`template_cache_state` LowCardinality(String), -- from the typed response extension, not the public header +`country` LowCardinality(String), +`ts_version` LowCardinality(String), +`pop` LowCardinality(String) -- FASTLY_POP, 'unknown' when absent +``` + +The matched route pattern does not survive dispatch today, so a typed +`RouteMetadata` response extension carries `route_class` and `route_template`: each +named-route handler wrapper attaches its route-table pattern verbatim (handlers +serving multiple patterns attach the one that matched), and the fallback and tsjs +handlers attach their class plus the coarse template. The freeze point consumes the +extension; nothing reconstructs routes from a handler enum or path regex. + +Typed sources only: `env` is adapter-owned, derived from the same Fastly +`FASTLY_IS_STAGING` input that drives `x-ts-env` (`Settings` has no environment +field and does not gain one). `template_cache_state` comes from a typed response +extension, not the `x-ts-template-cache` header (operator-configured response +headers can override managed headers): the currently private +`TemplateCacheResponseState` in `publisher.rs` becomes a typed response extension, +and every state transition sets the managed header and the extension together so +the two can never drift. `service_id` and `pop` come from the Fastly environment. `cache_state` +from the reserved schema is dropped: the guest cannot observe the fronting cache, and +guest-visible cache behavior is already carried by `template_cache_state` and +`origin_ms`. Rows exist only for guest-handled requests; fronting-cache hits are +invisible by construction and the dashboard documentation says so. + +Sorting key: `(toDate(event_ts), service_id, publisher_domain, env, route_class, +pop, status)`. Every column carries a `json:$.` path (the Events API rejects +NDJSON into a datasource without JSONPaths, discovered live); `event_date` was +dropped in favor of the sorting-key expression because a DEFAULT column cannot +carry a JSONPath the producer never sends. Grafana time filtering uses `$__timeFilter(event_ts)` and every panel query +also carries a `toDate(event_ts)` predicate so the primary index prunes; rollout validates +the panel queries with `EXPLAIN` before the dashboard is committed. This replaces the +reserved key `(event_date, path, status, method)`. Rollout step 4 verifies whether the +reserved datasource was ever deployed to the remote workspace; if it was, this schema +ships as a versioned replacement datasource with a cutover, not an in-place edit. + +## 10. Emission mechanics + +- `AccessTelemetrySnapshot`: built unconditionally at the freeze point, before + `into_parts()` consumes the response. It captures method, status, route metadata + (from the `RouteMetadata` extension), and typed dimension states (`env`, + `template_cache_state`, geo country). It exists because nothing else survives to + post-send on every path: the request is consumed by dispatch, the response by + `into_parts()`, and `EcFinalizeState` is absent on asset, admin, and error paths. +- The emitter's transport context is adapter-owned and route-independent: the + Events API target (backend spec, secret store name, dataset, token secret, sample + rate) derives from settings once at entry in `main.rs`, and the HTTP client is the + adapter's stateless platform client. Asset, admin, and error responses therefore + emit without `RuntimeServices` or `EcFinalizeState`. +- `send_edgezero_response` returns a delivery outcome instead of `()`, with + per-mode semantics because the two body paths observe different things. Streamed + bodies: a counting writer reports bytes written and distinguishes complete, + partial (truncated), and error outcomes. Buffered bodies: `send_to_client()` + returns no delivery result, so the byte count is captured from the body length + before the send and the outcome is complete-on-return with no partial detection; + `body_mode` in the row keeps the two regimes distinguishable in analysis. +- Ordering after the body-stream drive returns: snapshot `request_elapsed_ms` first, + run the existing pull-sync dispatch unchanged, then telemetry emission last, so + pull-sync is never delayed behind the ingest await and never included in + `request_elapsed_ms`. +- Sampling: uniform per-request decision against `tinybird.access_sample_rate`. No + client stickiness. Sampled-out requests are silent; every other drop (row build + failure, send failure, non-2xx) logs one warning naming the reason. There is no + cross-request warning suppression (per-request isolates hold no shared state); the + overload controls are the 2 s bounded await, the single-warning-per-request cap, and + `access_sample_rate` pushed down by config as the operational abort lever. Ingest + health is monitored from the Tinybird side via ingestion freshness on the + datasource, which catches quarantine and schema rejection that per-request warnings + cannot. +- Transport: one NDJSON row to the Tinybird Events API: same `api_host`, reserved + `access_dataset` and `access_token_secret`, 2 s first-byte and between-bytes + timeouts, `max_body_bytes` guard, no retry. +- Delivery confirmation: unlike the auction sink, which starts `send_async` and drops + the pending response (it runs before delivery completes and cannot afford to wait), + the access emitter runs after the client has the full response and therefore awaits + the bounded ingest response and validates 2xx. A non-2xx or timeout logs a warning + with the status. +- Budget: at `access_sample_rate = 1.0` this adds one backend request per request to + the service, after delivery; during a Tinybird outage each such request holds its + sandbox for up to the bounded timeout. The sample rate is the budget control; 1.0 is + a diagnosis setting, not a steady state, and rollout treats sustained emission + warnings as the signal to dial it down. +- Axum adapter: emits the header only; no Tinybird rows in v1. + +## 11. Dashboard and query model + +No endpoint pipe in v1. Grafana queries `access_logs_raw` directly through the +ClickHouse connector with `$__timeFilter(event_ts)` plus a `toDate(event_ts)` +predicate, matching the auction dashboards. + +Dashboard: a new standalone `grafana/dashboards/edge-performance.json` in the +telemetry repo (`trusted-server-tinybird`), performance only, no panels shared with +the revenue and auction dashboards. Panels: + +- Phase percentiles (p50/p95/p99) by `route_class`, per phase column. +- Stacked phase breakdown over time using the non-overlapping set: `appbuild_ms`, + `filter_ms`, `geo_ms`, `kv_ms`, `origin_ms`, `template_cache_ms`, pre-header + auction wait (where `auction_wait_placement = 'pre_header'`), and derived + `unattributed_ms`. In-stream auction wait and derived `stream_other_ms` chart in a + separate body-phase panel and never stack with pre-header phases. +- PoP split, `ts_version` overlay, template-cache state rates. +- Stall panel: rows with `request_elapsed_ms > 500` (post-body total, so body-only + stalls are caught) grouped by dominant phase, where `unattributed_ms` competes as a + phase so the panel cannot confidently blame a small measured span while most time is + uninstrumented. + +All derivations use the `coalesce`/`greatest` forms from section 6; query tests cover +sparse phase combinations and both `auction_wait_placement` modes. + +Sampling semantics for every aggregate: `sample_rate` must be operationally stable +within any queried window. Quantile panels filter strictly to a single `sample_rate` +value. Volume panels weight each row by `1.0 / sample_rate` (the inverse-probability +estimator is `sum(1.0 / sample_rate)` over emitted rows; `count() / rate` is valid +only when the query is already filtered to one rate). Pooled unweighted quantiles +across a rate change are documented as invalid. + +## 12. Config surface + +```toml +[observability] +# Append TS phase timings to the Server-Timing response header. +server_timing_enabled = false # example default +``` + +New `ObservabilitySettings` struct with the single boolean, default off, standard +environment override (`TRUSTED_SERVER__OBSERVABILITY__SERVER_TIMING_ENABLED`). +Collection has no flag: the flags gate the two emission surfaces independently. + +Tinybird flag structure: `tinybird.enabled` today arms the auction sink by itself, so +"enable Tinybird for access telemetry" would silently enable auction emission too. The +master flag is demoted to transport-only (host, store, credentials), and each emitter +gets its own switch: a new `tinybird.auction_enabled` defaulting to `true` (preserving +current behavior for existing configs) and the reserved `tinybird.access_enabled` +defaulting to `false`. A settings test locks the decoupling in both directions. + +Validation when `access_enabled = true`: `tinybird.enabled`, non-empty `api_host`, +non-empty `secret_store`, `access_dataset`, and `access_token_secret`, a positive +`max_body_bytes`, and `access_sample_rate > 0`. An armed-but-silent configuration +(`access_enabled = true`, `access_sample_rate = 0`) is a configuration error, not a +valid state; disabling is done with the flag, not the rate. + +Rollback and compatibility, because `Settings` is `deny_unknown_fields`: + +- Deployment order is binary first, config second. Rollback order is config first + (remove the `[observability]` table and any new tinybird keys), binary second. A + config containing the new fields must never be pushed while a pre-observability + binary can still run. +- Config serialization omits the table when it equals the default, so round-tripping a + config through tooling does not inject a field an older binary rejects. A + compatibility test asserts the serialized default config parses under the previous + schema. +- The environment-variable overlay cannot create a missing leaf, so the key ships + present-but-false in the base operator TOML (the same pattern the GPT integration + documents in `trusted-server.example.toml`) and is flipped by config push. + +## 13. Error handling + +- Recording is infallible: saturating math, lock-failure drops the sample, no panics. +- Header rendering failure (defensive `HeaderValue::from_str` error) logs and skips + the header. +- Row emission failure logs one warning naming the reason and drops the row. The + response has already been delivered; there is nothing to degrade. + +## 14. Testing + +- Core unit tests: phase accumulation, saturating math, `mark_headers_ready()` + idempotence and both-surface consistency, render format (one decimal, omission of + unrecorded phases), row serialization shape, auction-wait placement recording. +- Adapter tests (Fastly via Viceroy, Axum native): header present and well-formed on a + conclusively-private publisher route with the flag on; absent with the flag off; + absent on the shared-cacheable tsjs route and on a bare `max-age` response with the + flag on; exactly one TS-owned metric set (a single `ts-total`) with every + pre-existing Server-Timing value preserved, across all send paths; `ts-kv` captures + EC finalize work (proving the freeze point sits after it). +- Body-mode tests: ordinary streaming (in-stream wait nested in `stream_ms`), + Fastly shared-template authorized miss (buffered, pre-header wait), and Axum + (always buffered), each asserting placement and non-negative derivations. +- Geo dedupe: finalize consumes `Resolved`; no retry on `Attempted(None)`; live + lookup only on `NotAttempted`; fallback lookups timed into `ts-geo`; asset-fallback + path carries `GeoLookupState` without `EcFinalizeState`; 401 skip preserved. +- Route template: adversarial normalization tests (literal EC identifier on the admin + route, email address in a segment, search-term segments, overlong segments) all + producing bounded content-free templates. +- Settings: the access validation matrix including the armed-but-silent rejection; + auction/access flag decoupling in both directions; the former rejection test becomes + the wiring test; the serialized-default-config compatibility test against the + previous schema. +- Sink tests: `RecordingHttpClient` pattern; assert URI, NDJSON body shape, token + header, 2xx validation and warning on non-2xx, skip when sampled out, ordering after + pull-sync. +- Query tests: derivation formulas against sparse rows and both placements. + +## 15. Rollout and verification + +1. Land collection + freeze point + header emission behind the flag, off everywhere. + Full CI gate. +2. Staging deploy with the flag on. Delivery-layer verification is two-sided: a + pass-through request confirming the appended Server-Timing survives the fronting + VCL, and a MISS-then-HIT replay against a cacheable route confirming no stale + timing header is ever served from cache. Fallback if the VCL clobbers the header: a + one-line VCL change on the delivery service, or mirroring the value to + `x-ts-timing` while that lands. +3. Production flag on. Confirm + `performance.getEntriesByType('navigation')[0].serverTiming` shows `ts-*` entries + in a real browser session, and separately confirm what the publisher's RUM tooling + actually collects; the publisher monitoring extension renders it only after a small + change on their side. +4. Verify whether `access_logs_raw` exists in the remote Tinybird workspace. If yes, + ship the schema as a versioned replacement with cutover; if no, edit in place. + Validate the dashboard panel queries with `EXPLAIN` against the sorting key. Then + land the row schema, sink, and settings changes; sample at 1.0 during stall + diagnosis with ingestion-freshness monitoring on the datasource; then the + dashboard. +5. Success criterion: the next stall window is attributable from one response header + or one dashboard query, with no live probing session. + +## 16. Overhead + +Roughly ten monotonic clock reads, two stored snapshots, and one ~130-byte header per +request; one sampled HTTP POST with a bounded await after the response has fully +streamed. No allocation in the hot path beyond the one `Arc` at entry, the +`AccessTelemetrySnapshot` at the freeze point, and the rendered header string. + +## 17. Decisions and open questions + +- **Public exposure is a decision, not an open question.** The header is all-traffic + when enabled. Rationale: values are durations only; the delivery layer already + exposes `hit-state` and `time-elapsed` publicly on every response; filter vendor + identity is masked; emission is restricted to conclusively-private responses so no + cache can replay stale timings. Revisit (quantization or gating) only if a concrete + abuse surfaces. +- The fronting delivery layer's Server-Timing pass-through is unverified until the + first staging deploy (step 2). This is the only known external dependency. +- Body-phase capture threads the timings handle into the streaming closure in + `publisher.rs`; the exact seam is an implementation-plan detail, with the constraint + that a dropped handle (error paths, early client disconnect) must still yield a + valid row with null body-phase fields and a recorded delivery outcome. +- The stall window itself remains unattributed until this ships. If it recurs first, + the bisection runbook from 2026-08-21 (cookie-free curl UA request, static-asset + path versus HTML path) is the fallback. + +## 18. Auction timeline offsets (follow-up increment) + +Status: spec amendment for a follow-up PR; not part of the initial implementation +(#1074). Builds only on machinery that spec sections 5, 9, and 10 already define. + +### Problem + +The pipeline has two clocks that never meet. The auction dataset +(`auction_events_raw`, PR #813) measures the auction internally: `total_time_ms` +from auction start to terminal, `provider_response_time_ms` per bidder call. Its +clock starts when the auction observation is created, so nothing places those +numbers on the request timeline. The access row is T0-anchored but records only +`auction_wait_ms`: time the handler was blocked at collect, deliberately not the +auction's own timeline. + +That leaves three questions unanswerable today: + +1. At what request-relative time did the auction start (dispatch leave the edge)? +2. At what request-relative time did the auction resolve (final bid or timeout)? +3. At what request-relative time were the results committed toward GAM? + +These are the overlap-proof questions. A client-side wrapper cannot dispatch until +the browser boots (t≈3000ms on measured prospect pages); the server-side auction +dispatches while the origin fetch is in flight. Proving that requires all +milestones on one clock. + +### Design + +Three first-call-wins marks on `RequestTimings`, in the style of +`mark_headers_ready()`, each storing `Option` since T0: + +| Mark | Recorded at | Meaning | +| --------------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | +| `mark_auction_dispatched()` | immediately before `orchestrator.dispatch_auction` returns control to the caller (`publisher.rs` dispatch site) | bid requests have left the edge | +| `mark_auction_resolved()` | immediately after `collect_dispatched_auction` returns, both collect sites | final bid returned or auction timed out; terminal either way | +| `mark_auction_committed()` | immediately after `write_bids_to_state` returns, both call sites | winning bids are in page state, available to the response pipeline | + +Notes on the definitions: + +- "Committed toward GAM" is defined as `write_bids_to_state` returning: the common + point in buffered and streaming modes where targeting becomes part of the + response. TS never calls GAM server-side; the browser's GPT call carries the + targeting, and that half of the timeline belongs to client-side measurement. + The edge proves when targeting was available; the client proves when GAM saw it. +- First-call-wins on all three marks. A request produces at most one publisher-path + auction today; if a second auction ever occurs in one request, the row describes + the first and the auction dataset still carries both in full. +- Same locking and failure model as every other `RequestTimings` write: `try_lock`, + drop on contention, saturating conversion at serialization. + +### Row changes + +Four additive columns on `access_logs_raw`, all populated from the +`TimingSnapshot` at the existing freeze/emission points (no new emission path): + +``` +`auction_dispatched_ms` Nullable(UInt32), `json:$.auction_dispatched_ms` +`auction_resolved_ms` Nullable(UInt32), `json:$.auction_resolved_ms` +`auction_committed_ms` Nullable(UInt32), `json:$.auction_committed_ms` +`auction_id` String, `json:$.auction_id` +``` + +- The three offsets are null when no auction ran (the common case: assets, EC + endpoints, auction-disabled deployments). Null means "no auction", never "zero". +- `auction_id` is the telemetry auction UUID already present on every + `auction_events_raw` row, carried onto the access row as the join key between + the T0 timeline and per-bidder detail. Sentinel `none` when no auction ran, + matching the non-nullable-dimension convention of section 9. It is a random + UUID, not identity-bearing; unbounded cardinality is accepted for the same + reason it is accepted in the auction dataset. +- Schema evolution is additive with JSONPaths on every new column and + `FORWARD_QUERY` carrying the existing columns, per the deployed datasource's + established evolution path. Verified with `tb --cloud deploy --check` before + deploy. + +### Interpretation model + +Combined with existing columns, one access row now reads as a timeline: + +``` +t=0 ......... request entry +t=D ......... auction_dispatched_ms (bids out; origin fetch typically in flight) +t=R ......... auction_resolved_ms (R - D ~ auction duration; join auction_id + for the per-bidder long pole) +t=C ......... auction_committed_ms (targeting in page state) +t=H ......... time_elapsed_ms (headers committed) +``` + +Derivations the dashboard can add without schema help: auction duration on the +request clock (`R - D`), commit latency (`C - R`), and overlap ratio (share of +`R - D` that ran concurrently with `ts-origin`). `auction_wait_ms` keeps its +existing meaning (blocked time only) and is now interpretable next to the +timeline: `R - D` minus `auction_wait_ms` approximates how much of the auction +was absorbed by work the request needed anyway. + +### Scope + +- Fastly emits; Axum, Cloudflare, and Spin collect the marks but do not emit, + matching section 8a adapter semantics. +- No header emission for any of these values: they are post-hoc analysis fields, + and two of the three are typically unknown at the header freeze point in + streaming mode. +- No config surface: the marks are always-on collection like every other phase, + gated at emission by the existing `tinybird.access_enabled`. diff --git a/docs/superpowers/specs/2026-08-27-pr-1079-review-remediation-design.md b/docs/superpowers/specs/2026-08-27-pr-1079-review-remediation-design.md new file mode 100644 index 000000000..f3603e767 --- /dev/null +++ b/docs/superpowers/specs/2026-08-27-pr-1079-review-remediation-design.md @@ -0,0 +1,80 @@ +# PR 1079 Review Remediation Design + +## Goal + +Make the first-impression ownership and APS creative bridge safe under overlapping +publisher auctions, late callbacks, SPA navigation, mixed GPT refresh lists, and +nested 1x1 GAM shells. Preserve PR 1079's first-claimant policy: Trusted Server may +win an untouched physical slot, but must neither overwrite a publisher impression +nor let a stale response affect a later navigation. + +## Ownership model + +First-impression state remains keyed by navigation generation and exact physical +element identity. Each publisher auction gets an independent token whose +suppression decision is fixed when the auction is registered. When Trusted Server +commits its request, registration closes for new losing publisher auctions, while +already-registered losing tokens remain suppressible. Those tokens remain as +tombstones for the lifetime of the same navigation and exact physical element. +Unresolved suppressing tombstones are never evicted or removed by timeout or +auction failure; only navigation change or physical element replacement removes +them. The existing per-slot registration limit bounds the set before registration +closes, so an arbitrarily late correlated callback cannot become unrelated. + +Prebid's pending bid/code correlation records carry the navigation generation and +physical element identity captured at registration. A record is usable only while +both still match, and consuming one exact ad-ID delivery removes only its auction's +registration. A code-only delivery consumes a record only when exactly one current +registration matches. Ambiguous ordinary code-only deliveries run an independent +auction rather than guessing; ambiguous TS-owned suppressing deliveries fail closed +without deleting their tombstones. Scoped `requestBids({ adUnitCodes })` calls +inspect, mutate, claim, and correlate only those requested global ad units. + +## Refresh suppression + +The Prebid delivery wrapper is the owner of first-impression delivery suppression. +When it suppresses a GPT slot, it also consumes any equivalent late-handoff +one-shot flag so the inner GPT wrapper cannot suppress the next legitimate +refresh. When it delegates a permitted GPT request, it consumes that flag at the +delegation boundary so the inner wrapper cannot silently drop the request. Mixed +refresh calls always forward the already-filtered slot list, including the path +where every remaining slot is excluded from a Prebid auction. That all-excluded +path performs the same ownership registration and consumption synchronously +before delegating. A bare refresh delayed by an auction becomes an explicit list +at callback time, preventing slots added after the snapshot from joining it. + +A publisher-triggered GPT refresh that starts a synthetic Prebid auction registers +its own per-slot first-impression tokens before waiting for the asynchronous +callback. A publisher-first token reserves the slot so TS cannot claim it while +the auction is pending. A token registered against an earlier TS claim is consumed +at callback time, filtering that slot from the eventual GPT request. When TS emits +its first GPT request, registration closes for new losing publisher tokens so +ordinary later publisher refreshes continue normally. Mixed callbacks forward +only their unsuppressed slots and scope Prebid targeting to the same filtered set. +The callback also revalidates the captured navigation generation and exact +physical element, dropping stale work rather than refreshing a replacement slot. + +## Creative bridge + +Every asynchronous renderer/cache result is revalidated before posting a creative +response or recording successful response/billing evidence. A stale result may be +recorded as safe failure telemetry, but is never recorded as a response or win. +Validation covers navigation generation, winning bid identity, authenticated +source iframe identity, DOM connectivity, and containment in the authenticated +slot root. When a configured prefix matches several roots, the requesting frame +may disambiguate them only when exactly one candidate root owns that source. + +After a valid response is posted, a collapsed 1x1 source iframe is expanded to the +winning creative size. The bridge walks all collapsed ancestors through the +authenticated slot root and expands each clipping shell. It refuses all resizing +for fixed/sticky, anchor, vignette, interstitial, detached, oversized, or +otherwise unauthenticated shells. + +## Verification + +Regression tests cover all seven review findings, including wrapper composition, +scoped ad-unit requests, mixed excluded refreshes, stale SPA callbacks, +overlapping auctions, stale cache responses with no successful response/billing +evidence, and two nested +collapsed ancestors. Existing JS unit/browser suites, formatting, lint, build, +and repository Rust verification remain the completion gates. diff --git a/fastly.toml b/fastly.toml index a3812999b..5d1113969 100644 --- a/fastly.toml +++ b/fastly.toml @@ -78,6 +78,8 @@ build = """ [local_server.config_stores.edgezero_runtime_env] format = "inline-toml" [local_server.config_stores.edgezero_runtime_env.contents] + # Viceroy reports this fixed synthetic service id. EdgeZero scopes + # Fastly runtime mappings by service id. EDGEZERO__SERVICES__0000000000000000000000__STORES__SECRETS__TRUSTED_SERVER_SECRETS__NAME = "ts_secrets" [local_server.config_stores.trusted_server_config] diff --git a/scripts/template-cache-local-test.sh b/scripts/template-cache-local-test.sh index 58d714cf8..53e86837f 100755 --- a/scripts/template-cache-local-test.sh +++ b/scripts/template-cache-local-test.sh @@ -238,6 +238,7 @@ s = replace_once( f'origin_url = "http://127.0.0.1:{origin_port}"', "publisher origin", ) +# The example publisher domains are reserved placeholders that validation rejects. s = replace_once( s, 'domain = "example.com"', @@ -250,6 +251,7 @@ s = replace_once( 'cookie_domain = ".local-harness.example"', "publisher cookie domain", ) + # A real auction points at the slow HTTPS stub so the timings mean something. s = replace_once( s, diff --git a/scripts/test-cli.sh b/scripts/test-cli.sh index eef9e2f7d..7b562c96e 100755 --- a/scripts/test-cli.sh +++ b/scripts/test-cli.sh @@ -19,3 +19,20 @@ if ! rustup target list --installed | awk -v target="$HOST_TARGET" '$0 == target fi cargo test --package trusted-server-cli --target "$HOST_TARGET" +export TS_AUDIT_BROWSER_TESTS=1 +AUDIT_BROWSER_TEST_FILTERS=( + "commands::audit::browser::tests::" + "commands::audit::generate::browser_collector::tests::" +) +for AUDIT_BROWSER_TEST_FILTER in "${AUDIT_BROWSER_TEST_FILTERS[@]}"; do + AUDIT_BROWSER_TEST_COUNT="$({ + cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --list + } | awk '/: test$/ { count += 1 } END { print count + 0 }')" + if [ "$AUDIT_BROWSER_TEST_COUNT" -eq 0 ]; then + echo "No ignored browser audit fixtures matched $AUDIT_BROWSER_TEST_FILTER" >&2 + exit 1 + fi + cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --test-threads=1 +done diff --git a/tinybird/datasources/access_logs_raw.datasource b/tinybird/datasources/access_logs_raw.datasource index 42f214e07..4441de384 100644 --- a/tinybird/datasources/access_logs_raw.datasource +++ b/tinybird/datasources/access_logs_raw.datasource @@ -1,19 +1,43 @@ DESCRIPTION > - Optional sampled Trusted Server access telemetry rows. Disabled by default in Fastly config. + Per-request phase-timing telemetry rows, sampled and emitted post-send by the edge service. SCHEMA > - `event_ts` DateTime64(3), - `method` LowCardinality(String), - `path` String, - `status` UInt16, - `time_elapsed_ms` UInt32, - `cache_state` LowCardinality(Nullable(String)), - `country` LowCardinality(String), - `sample_rate` Float64, - `event_date` Date DEFAULT toDate(event_ts) + `event_ts` DateTime64(3) `json:$.event_ts`, + `method` LowCardinality(String) `json:$.method`, + `status` UInt16 `json:$.status`, + `time_elapsed_ms` Nullable(UInt32) `json:$.time_elapsed_ms`, + `sample_rate` Float64 `json:$.sample_rate`, + `service_id` LowCardinality(String) `json:$.service_id`, + `publisher_domain` LowCardinality(String) `json:$.publisher_domain`, + `env` LowCardinality(String) `json:$.env`, + `route_class` LowCardinality(String) `json:$.route_class`, + `route_template` String `json:$.route_template`, + `body_mode` LowCardinality(String) `json:$.body_mode`, + `auction_wait_placement` LowCardinality(String) `json:$.auction_wait_placement`, + `appbuild_ms` Nullable(UInt32) `json:$.appbuild_ms`, + `filter_ms` Nullable(UInt32) `json:$.filter_ms`, + `geo_ms` Nullable(UInt32) `json:$.geo_ms`, + `kv_ms` Nullable(UInt32) `json:$.kv_ms`, + `origin_ms` Nullable(UInt32) `json:$.origin_ms`, + `template_cache_ms` Nullable(UInt32) `json:$.template_cache_ms`, + `auction_wait_ms` Nullable(UInt32) `json:$.auction_wait_ms`, + `stream_ms` Nullable(UInt32) `json:$.stream_ms`, + `request_elapsed_ms` Nullable(UInt32) `json:$.request_elapsed_ms`, + `resp_bytes` Nullable(UInt64) `json:$.resp_bytes`, + `template_cache_state` LowCardinality(String) `json:$.template_cache_state`, + `country` LowCardinality(String) `json:$.country`, + `ts_version` LowCardinality(String) `json:$.ts_version`, + `pop` LowCardinality(String) `json:$.pop`, + `auction_dispatched_ms` Nullable(UInt32) `json:$.auction_dispatched_ms`, + `auction_resolved_ms` Nullable(UInt32) `json:$.auction_resolved_ms`, + `auction_committed_ms` Nullable(UInt32) `json:$.auction_committed_ms`, + `auction_id` String `json:$.auction_id` ENGINE "MergeTree" -ENGINE_SORTING_KEY "event_date, path, status, method" -TTL "event_date + INTERVAL 30 DAY" +ENGINE_SORTING_KEY "toDate(event_ts), service_id, publisher_domain, env, route_class, pop, status" +TTL "toDate(event_ts) + INTERVAL 30 DAY" + +FORWARD_QUERY > + SELECT event_ts, method, status, time_elapsed_ms, sample_rate, service_id, publisher_domain, env, route_class, route_template, body_mode, auction_wait_placement, appbuild_ms, filter_ms, geo_ms, kv_ms, origin_ms, template_cache_ms, auction_wait_ms, stream_ms, request_elapsed_ms, resp_bytes, template_cache_state, country, ts_version, pop, CAST(NULL AS Nullable(UInt32)) AS auction_dispatched_ms, CAST(NULL AS Nullable(UInt32)) AS auction_resolved_ms, CAST(NULL AS Nullable(UInt32)) AS auction_committed_ms, 'none' AS auction_id TOKEN ts_access_ingest APPEND diff --git a/tinybird/fixtures/access_logs_raw.ndjson b/tinybird/fixtures/access_logs_raw.ndjson new file mode 100644 index 000000000..3c82c5ca6 --- /dev/null +++ b/tinybird/fixtures/access_logs_raw.ndjson @@ -0,0 +1 @@ +{"event_ts":"2026-06-23 12:00:00.000","method":"GET","status":200,"time_elapsed_ms":145,"sample_rate":0.1,"service_id":"abc123","publisher_domain":"test-publisher.com","env":"production","route_class":"publisher_html","route_template":"/news/*","body_mode":"streamed","auction_wait_placement":"in_stream","appbuild_ms":12,"filter_ms":5,"geo_ms":3,"kv_ms":8,"origin_ms":25,"template_cache_ms":10,"auction_wait_ms":45,"stream_ms":18,"request_elapsed_ms":145,"resp_bytes":8192,"template_cache_state":"hit","country":"US","ts_version":"v1.2.3","pop":"SFO"} diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 33984ab81..8ccbe5640 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -319,6 +319,12 @@ provider = "pbs-main" # inventory_domain = "publisher.example" # inventory_page_origin = "https://www.publisher.example" +[observability] +# Keep this leaf present so the environment override can apply; the overlay +# cannot create a missing configuration leaf. The Server-Timing header stays +# off until enabled. +server_timing_enabled = false + # Server-side ad slot templates + creative-opportunity auction. Kept active. [creative_opportunities] # Set false to disable server-side ad templates while keeping slot definitions @@ -409,8 +415,14 @@ auction_timeout_ms = 500 # [tinybird] # enabled = true # api_host = "api.us-east.tinybird.example" # required when enabled; host only -# auction_dataset = "auction_events" # Events API datasource name +# auction_enabled = true # emit auction telemetry +# auction_dataset = "auction_events_raw" # auction Events API datasource # auction_token_secret = "tinybird_auction_append_token" # Key in trusted_server_secrets +# access_enabled = false # emit sampled access telemetry +# access_dataset = "access_logs_raw" # access Events API datasource +# access_token_secret = "tinybird_access_append_token" # Key in trusted_server_secrets +# access_sample_rate = 0.0 # fraction from 0.0 through 1.0 +# max_body_bytes = 1048576 # maximum NDJSON request body # Debug endpoints (all default false — never enable in production). # [debug] @@ -472,6 +484,33 @@ client_side_bidders = [] # bidders running via native Prebid.js adapter # adapters = ["rubicon"] # user_id_modules = ["sharedIdSystem"] +# Prebid User ID modules that Trusted Server installs and keeps installed, so +# operators can manage identity centrally without publisher JavaScript changes. +# Each entry is forwarded to Prebid.js verbatim: `name` is a +# `userSync.userIds` entry name, `params` and `storage` are whatever that module +# documents. Trusted Server does not interpret them; supported names come from +# the checked-in User ID registry. Each `name` must appear only once. +# +# The module must be present in the built bundle: name it under +# [integrations.prebid.bundle].user_id_modules, or omit that list to take the +# generator's default preset. `ts prebid bundle` resolves every managed name +# through the checked-in User ID registry and fails if the generated manifest +# omits its required module without updating the configured hash or SRI. +# +# Persisting a resolved ID into the EC identity graph additionally needs a +# matching partner under [[ec.partners]] whose source_domain equals the module's +# OpenRTB EID source; without one the ID still reaches the auction but is never +# written to KV. +# +# [[integrations.prebid.managed_user_ids]] +# name = "sharedId" +# +# [integrations.prebid.managed_user_ids.storage] +# type = "cookie" # or "html5" +# name = "_sharedid" +# expires = 15 # days; omit to keep Prebid's default +# refresh_in_seconds = 1800 # omit to keep Prebid's default + # Next.js first-party rewriting for App Router / RSC payloads. # [integrations.nextjs] # enabled = true