diff --git a/README.md b/README.md index f0b5175..b216f3b 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ Errors go to stderr and return a non-zero exit code. With `--json`, errors also ## Commands ```text -brainpod detect [] [--rules ] +brainpod detect [] [--rules ] [--with-env [--with-source]] brainpod describe [...] brainpod describe resource [] brainpod login @@ -271,11 +271,19 @@ Inspect a directory without credentials, network access, subprocesses, or execut ```sh brainpod detect . brainpod detect ./services --json +brainpod detect . --with-env --json +brainpod detect . --with-env --with-source --json brainpod detect . --rules ./detection.toml --json ``` The result contains sorted project paths, manifest names, languages and version declarations, priority-ranked detections with evidence and `uses`/`provides` framework roles, workspace members, diagnostics, and scan statistics. Optional `runtimes` records Deno declarations separately from JavaScript/TypeScript versions; `workspaces` records each manager’s ecosystem, source, and members, while `workspaceMembers` remains their union. Framework versions are attached to dependency evidence. JavaScript versions describe Node.js; TypeScript versions describe the compiler. C# versions distinguish `language` (LangVersion), `target` (TFM), `runtime`, and `sdk`; Java distinguishes compiler `source`/`target`/`release`, toolchains, build-JDK metadata, and pins. Requirements and pins retain their source and kind: these are declarations, not a claim about installed versions. Missing versions remain unknown. +Add `--with-env` for name-only environment declarations from dotenv files (including examples/templates), Compose, Dockerfile `ENV` and `ARG` (build-scoped), conventional Kubernetes manifests, GitHub Actions workflow/job/step `env` maps, and deployment manifests of other hosting platforms: Heroku/Dokku `app.json`, Scalingo `scalingo.json`, `heroku.yml`, Render `render.yaml`, Fly.io `fly.toml`, DigitalOcean `.do/app.yaml`/`.do/deploy.template.yaml`, Vercel `vercel.json`/`vercel.toml`, Netlify `netlify.toml`, Google App Engine `app.yaml` and AWS Copilot `copilot/*/manifest.yml`. Platform records add optional `required`, `secret`, `generated`, `hasDefault` (a literal value exists; never the value) and `scope` (`runtime`, `build`, `build-and-runtime`) fields only where the manifest states them, and database/service/secret wiring as a `reference`. + +`--with-env` reads only manifest, config and deployment files, never application source. Add `--with-source` (requires `--with-env`) to also **read application source files** (JavaScript/TypeScript, Vue/Svelte/Astro, Python, Ruby, Go, Rust, PHP, Java/Kotlin, C#, Elixir; existing directory exclusions and the 1 MiB limit apply) and search them literally for a cataloged list of platform-injected names (for example `VERCEL_URL`, `RENDER_GIT_COMMIT`, `HEROKU_SLUG_COMMIT`, `CF_PAGES_URL`) and framework prefixes (`NEXT_PUBLIC_*`, `VITE_*`, `REACT_APP_*`, `NUXT_PUBLIC_*`, `EXPO_PUBLIC_*`, …). This is not a general read-site scanner: arbitrary `process.env` reads are not inferred. Each `(project, name)` becomes one `code-reference` record with `origin`, `provider`, `scope` and up to 20 file/line occurrences, never code text. On the pinned corpus `--with-source` reads about 40 times as many files as `--with-env` alone: 277,283 files / 1.77 GB versus 6,822 files / 15.1 MB. + +JSON adds `environment`; text shows names, source/format/location, project ownership, unresolved references and the optional fields. Values are never exposed, loaded or evaluated. References are not followed, and declarations are not a claim about usage or installed/effective values. Without the flag, output and read counts are unchanged. See [supported syntax, per-platform mapping, code-reference patterns, privacy/scoping limits and offline validation](tests/detect-env.md). + Built-in coverage includes Laravel, Symfony, CakePHP, CodeIgniter, Composer; Go modules/workspaces and Gin/Echo/Fiber; Cargo workspaces, axum, Rocket, Actix; Bundler, Rails, Sinatra, Hanami; Python requirements, PEP 621, Poetry, Pipenv, uv workspaces, Django, FastAPI, Flask, Starlette, Celery; npm/pnpm and Deno workspaces and catalogs, Fresh, Oak, Hono, Next.js, Nuxt, Astro, SvelteKit, Remix, Angular, Express, React, Vue, Vite; C#/.NET projects and solutions, ASP.NET Core/classic, Aspire, FastEndpoints; Java/JVM Maven and Gradle projects, Spring/Spring Boot, Quarkus, Micronaut; and standalone source files in all nine languages. Rust workspace dependency aliases and inherited `rust-version` are understood. Ancestor runtime pin/toolchain files are included with provenance. Poetry names, literal gemspec metadata, and Rails component dependencies are recognized. pnpm catalog references retain the original declaration and add the locally resolved constraint with its YAML source; unresolved references produce diagnostics. Deno supports `deno.json`/`deno.jsonc` (comments and trailing commas), Deno-only, hybrid, and npm-only workspace members, globs/exclusions, and shared imports with member alias overrides. Registry targets (`npm:`, `jsr:`) and supported `deno.land/x`/`esm.sh` URLs supply dependency evidence without downloading modules. Local relative import maps are bounded and cannot escape the scan root or follow symlinks. `.deno-version`, `.tool-versions` and npm `engines.deno` retain explicit runtime version provenance; a JSR package’s `version` is never treated as a Deno runtime version. Peer and development dependency constraints remain separate evidence. pnpm-only membership is not inferred as Deno membership. Scoped import-map overrides and non-relative/remote import-map files are diagnosed, not fetched or guessed; source imports, lockfiles, Docker/CI version hints, and tasks are not evaluated. diff --git a/src/detect/catalog.rs b/src/detect/catalog.rs index 423b1d2..7f15b77 100644 --- a/src/detect/catalog.rs +++ b/src/detect/catalog.rs @@ -41,6 +41,8 @@ pub(super) enum Resolver { #[serde(default, deny_unknown_fields)] pub(super) struct CatalogSpec { manifests: Vec, + environment: Vec, + code_references: super::environment::references::Spec, scan: Scan, families: BTreeMap, inheritance: Vec, @@ -249,6 +251,8 @@ struct WorkspaceField { pub(super) struct Catalog { spec: CatalogSpec, + pub environment: super::environment::Catalog, + pub references: super::environment::references::Catalog, exact: HashMap>, wildcard: Vec<(FilePattern, usize)>, extensions: HashMap>, @@ -260,6 +264,8 @@ pub(super) struct Catalog { impl CatalogSpec { pub fn extend(&mut self, extra: Self) { self.manifests.extend(extra.manifests); + self.environment.extend(extra.environment); + self.code_references.extend(extra.code_references); self.scan.ignore.extend(extra.scan.ignore); self.scan.prune.extend(extra.scan.prune); self.families.extend(extra.families); @@ -271,8 +277,14 @@ impl CatalogSpec { } impl Catalog { - pub fn new(spec: CatalogSpec) -> Result { + pub fn new(mut spec: CatalogSpec) -> Result { + let environment = super::environment::Catalog::new(&spec.environment)?; + let references = super::environment::references::Catalog::new(std::mem::take( + &mut spec.code_references, + ))?; let mut catalog = Self { + environment, + references, spec, exact: HashMap::new(), wildcard: Vec::new(), diff --git a/src/detect/environment.rs b/src/detect/environment.rs new file mode 100644 index 0000000..ed261e7 --- /dev/null +++ b/src/detect/environment.rs @@ -0,0 +1,772 @@ +mod platforms; +pub(super) mod references; + +use std::path::Path; + +use anyhow::{Result, bail}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::rules::{FilePattern, path_glob}; +use super::{Diagnostic, relative_file}; + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "kebab-case")] +pub enum Format { + Dotenv, + Compose, + Dockerfile, + Kubernetes, + GithubActions, + AppJson, + HerokuYml, + Render, + Fly, + Digitalocean, + Vercel, + Netlify, + AppEngine, + Copilot, + /// Names found by the catalog's literal code-reference patterns; never routed to files. + CodeReference, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Spec { + format: Format, + files: Vec, + #[serde(default)] + paths: Vec, + #[serde(default)] + exclude: Vec, +} + +struct Entry { + format: Format, + files: Vec, + paths: Vec, + exclude: Vec, +} + +pub(super) struct Catalog(Vec); + +impl Catalog { + pub fn new(specs: &[Spec]) -> Result { + let mut entries = Vec::new(); + for spec in specs { + if spec.format == Format::CodeReference + || spec.files.is_empty() + || spec + .files + .iter() + .chain(&spec.exclude) + .any(|file| file.is_empty() || file.contains(['/', '\\'])) + || spec.paths.iter().any(|path| { + path.is_empty() || path.starts_with('/') || path.split('/').any(|p| p == "..") + }) + { + bail!("invalid environment catalog file/path pattern"); + } + entries.push(Entry { + format: spec.format, + files: spec.files.iter().map(|p| FilePattern::new(p)).collect(), + paths: spec.paths.clone(), + exclude: spec.exclude.iter().map(|p| FilePattern::new(p)).collect(), + }); + } + Ok(Self(entries)) + } + + pub fn matching(&self, directory: &Path, file: &str) -> Option { + self.0 + .iter() + .find(|entry| { + entry.files.iter().any(|pattern| pattern.matches(file)) + && !entry.exclude.iter().any(|pattern| pattern.matches(file)) + && (entry.paths.is_empty() + || entry + .paths + .iter() + .any(|pattern| path_glob(pattern, &relative_file(directory, file)))) + }) + .map(|entry| entry.format) + } +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "camelCase")] +pub struct Declaration { + pub source: String, + pub format: Format, + pub location: String, + pub project: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + pub state: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub reference: Option, + #[serde(flatten)] + pub semantics: Semantics, + #[serde(flatten)] + pub code: Option, +} + +/// Provenance of a `code-reference` record: which catalog pattern matched and where. +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "camelCase")] +pub struct CodeReference { + pub origin: Origin, + pub provider: String, + /// Sorted `(source, line)` occurrences, capped at `references::MAX_OCCURRENCES`. + pub occurrences: Vec, + pub occurrence_count: usize, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Occurrence { + pub source: String, + pub line: usize, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "kebab-case")] +pub enum Origin { + Platform, + Framework, +} + +/// Platform-declared facts about a declaration. Each field is present only when +/// the metadata format states it; literal values are never retained. +#[derive(Clone, Copy, Debug, Default, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "camelCase")] +pub struct Semantics { + #[serde(skip_serializing_if = "Option::is_none")] + pub required: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub secret: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub generated: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_default: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub scope: Option, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "kebab-case")] +pub enum Scope { + Runtime, + Build, + BuildAndRuntime, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Reference { + pub kind: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub target: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub prefix: Option, +} + +struct Parser<'a> { + source: &'a str, + format: Format, + declarations: Vec, + diagnostics: &'a mut Vec, +} + +impl Parser<'_> { + fn emit( + &mut self, + location: &str, + name: Option<&str>, + state: &str, + reference: Option, + ) { + self.declare(location, name, state, reference, Semantics::default()); + } + + fn declare( + &mut self, + location: &str, + name: Option<&str>, + state: &str, + reference: Option, + semantics: Semantics, + ) { + if name.is_some_and(|name| !identifier(name)) { + self.warn("unsupported or dynamic environment variable name"); + return; + } + self.declarations.push(Declaration { + source: self.source.to_owned(), + format: self.format, + location: location.to_owned(), + project: None, + name: name.map(str::to_owned), + state: state.to_owned(), + reference, + semantics, + code: None, + }); + } + + fn warn(&mut self, message: &str) { + if !self + .diagnostics + .iter() + .any(|d| d.path == self.source && d.message == message) + { + self.diagnostics.push(Diagnostic { + path: self.source.to_owned(), + message: message.to_owned(), + }); + } + } + + fn env_map(&mut self, value: &Value, location: &str, compose: bool) { + match value { + Value::Object(map) => { + for (name, value) in map { + if !(value.is_null() + || value.is_string() + || value.is_number() + || value.is_boolean()) + { + self.warn("environment value must be a scalar; declaration skipped"); + continue; + } + self.emit( + &format!("{location}/{}", pointer(name)), + Some(name), + if compose && value.is_null() { + "external" + } else { + "declared" + }, + None, + ); + } + } + Value::Array(list) if compose => { + for (index, value) in list.iter().enumerate() { + if let Some(value) = value.as_str() { + let (name, state) = value + .split_once('=') + .map_or((value, "external"), |(name, _)| (name, "declared")); + self.emit(&format!("{location}/{index}"), Some(name), state, None); + } else { + self.warn("environment list entry must be a string"); + } + } + } + Value::String(_) => self.emit( + location, + None, + "unresolved", + Some(reference("dynamic-env", None)), + ), + _ => self.warn("environment declaration must be a mapping or supported list"), + } + } +} + +pub(super) fn parse( + format: Format, + source: &str, + text: &str, + diagnostics: &mut Vec, +) -> Vec { + let mut parser = Parser { + source, + format, + declarations: Vec::new(), + diagnostics, + }; + match format { + Format::Dotenv => dotenv(&mut parser, text), + Format::Dockerfile => dockerfile(&mut parser, text), + _ if source.ends_with(".toml") => match toml::from_str::(text) { + Ok(doc) => document(&mut parser, &doc, "document:0"), + Err(_) => parser.warn("environment TOML is invalid"), + }, + // Platform JSON manifests follow JSON (last duplicate key wins) like their platforms. + Format::AppJson | Format::Vercel if source.ends_with(".json") => { + match serde_json::from_str::(text) { + Ok(doc) => document(&mut parser, &doc, "document:0"), + Err(_) => parser.warn("environment JSON is invalid"), + } + } + _ => { + let options = serde_saphyr::options! { + budget: serde_saphyr::budget! { + max_depth: 48, + max_documents: 256, + max_events: 100_000, + max_nodes: 50_000, + max_aliases: 128, + max_anchors: 128, + max_recorded_anchor_events: 100_000, + max_total_scalar_bytes: 2 * 1024 * 1024, + max_inclusion_depth: 0, + }, + strict_booleans: true, + reject_unsupported_tags: true, + }; + match serde_saphyr::from_multiple_with_options::(text, options) { + Ok(documents) => { + for (index, doc) in documents.iter().enumerate() { + document(&mut parser, doc, &format!("document:{index}")); + } + } + Err(_) => parser.warn("environment YAML is invalid or exceeds parser limits"), + } + } + } + parser.declarations +} + +fn document(parser: &mut Parser<'_>, doc: &Value, location: &str) { + match parser.format { + Format::Compose => compose(parser, doc, location), + Format::Kubernetes => kubernetes(parser, doc, location), + Format::GithubActions => actions(parser, doc, location), + _ => platforms::parse(parser, doc, location), + } +} + +fn identifier(name: &str) -> bool { + let mut chars = name.bytes(); + chars + .next() + .is_some_and(|ch| ch.is_ascii_alphabetic() || ch == b'_') + && chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, b'_' | b'.' | b'-')) +} + +fn metadata_name(name: &str) -> bool { + !name.is_empty() + && name + .bytes() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, b'_' | b'.' | b'-')) +} + +fn pointer(value: &str) -> String { + value.replace('~', "~0").replace('/', "~1") +} + +fn reference(kind: &str, target: Option<&str>) -> Reference { + Reference { + kind: kind.to_owned(), + target: target + .filter(|value| metadata_name(value)) + .map(str::to_owned), + key: None, + prefix: None, + } +} + +fn dotenv(parser: &mut Parser<'_>, text: &str) { + let mut lines = text.trim_start_matches('\u{feff}').lines().enumerate(); + while let Some((index, line)) = lines.next() { + let mut line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + if let Some(rest) = line + .strip_prefix("export") + .filter(|rest| rest.starts_with(char::is_whitespace)) + { + line = rest.trim_start(); + } + let Some((name, value)) = line.split_once('=') else { + parser.warn("unsupported dotenv declaration; expected NAME=value"); + continue; + }; + let name = name.trim(); + let value = value.trim_start(); + let mut valid = true; + if let Some(quote) = value + .chars() + .next() + .filter(|ch| matches!(ch, '\'' | '"' | '`')) + { + let mut tail = closing_quote(&value[quote.len_utf8()..], quote); + while tail.is_none() { + let Some((_, next)) = lines.next() else { + break; + }; + tail = closing_quote(next, quote); + } + valid = tail + .is_some_and(|tail| tail.trim().is_empty() || tail.trim_start().starts_with('#')); + } + if valid { + parser.emit(&format!("line:{}", index + 1), Some(name), "declared", None); + } else { + parser.warn("unterminated or malformed quoted dotenv declaration"); + } + } +} + +fn closing_quote(value: &str, quote: char) -> Option<&str> { + let mut escaped = false; + for (index, ch) in value.char_indices() { + if escaped { + escaped = false; + } else if ch == '\\' { + escaped = true; + } else if ch == quote { + return Some(&value[index + ch.len_utf8()..]); + } + } + None +} + +fn words(text: &str, escape: char) -> Option> { + let mut result = Vec::new(); + let mut word = String::new(); + let mut quote = None; + let mut escaped = false; + for ch in text.chars() { + if escaped { + word.push(ch); + escaped = false; + } else if ch == escape { + escaped = true; + } else if quote == Some(ch) { + quote = None; + } else if quote.is_none() && matches!(ch, '\'' | '"') { + quote = Some(ch); + } else if quote.is_none() && ch.is_whitespace() { + if !word.is_empty() { + result.push(std::mem::take(&mut word)); + } + } else { + word.push(ch); + } + } + if quote.is_some() || escaped { + return None; + } + if !word.is_empty() { + result.push(word); + } + Some(result) +} + +fn dockerfile(parser: &mut Parser<'_>, text: &str) { + let mut escape = '\\'; + let mut directives = true; + let mut stage = 0; + let mut lines = text.lines().enumerate(); + while let Some((index, raw)) = lines.next() { + let line = raw.trim(); + if directives { + if let Some((key, value)) = line.strip_prefix('#').and_then(|s| s.split_once('=')) { + if key.trim().eq_ignore_ascii_case("escape") { + match value.trim() { + "`" => escape = '`', + "\\" => escape = '\\', + _ => { + parser.warn("unsupported Dockerfile escape directive"); + return; + } + } + continue; + } + if key.trim().eq_ignore_ascii_case("syntax") + || key.trim().eq_ignore_ascii_case("check") + { + continue; + } + } + directives = false; + } + if line.is_empty() || line.starts_with('#') { + continue; + } + let mut instruction = line.to_owned(); + while instruction.trim_end().ends_with(escape) { + instruction.truncate(instruction.trim_end().len() - escape.len_utf8()); + let Some((_, next)) = lines + .by_ref() + .find(|(_, line)| !line.trim().is_empty() && !line.trim_start().starts_with('#')) + else { + parser.warn("incomplete Dockerfile continuation"); + return; + }; + instruction.push_str(next.trim()); + } + let (command, body) = instruction + .split_once(char::is_whitespace) + .unwrap_or((&instruction, "")); + if command.eq_ignore_ascii_case("FROM") { + stage += 1; + } + if (command.eq_ignore_ascii_case("RUN") + || command.eq_ignore_ascii_case("COPY") + || command.eq_ignore_ascii_case("ADD")) + && body.contains("<<") + { + parser.warn("Dockerfile heredoc and subsequent instructions not inspected"); + return; + } + let location = format!("stage:{stage}/line:{}", index + 1); + if command.eq_ignore_ascii_case("ARG") { + match words(body, escape) { + Some(words) if !words.is_empty() => { + for (index, word) in words.iter().enumerate() { + let (name, has_default) = word + .split_once('=') + .map_or((word.as_str(), false), |(name, _)| (name, true)); + let semantics = Semantics { + has_default: Some(has_default), + scope: Some(Scope::Build), + ..Semantics::default() + }; + parser.declare( + &format!("{location}/{index}"), + Some(name), + "declared", + None, + semantics, + ); + } + } + _ => parser.warn("malformed Dockerfile ARG instruction"), + } + continue; + } + if !command.eq_ignore_ascii_case("ENV") { + continue; + } + let Some(words) = words(body, escape) else { + parser.warn("malformed Dockerfile ENV instruction"); + continue; + }; + let Some(first) = words.first() else { + parser.warn("empty Dockerfile ENV instruction"); + continue; + }; + if first.starts_with("--") { + parser.warn("Dockerfile ENV flags are not supported"); + } else if first.contains('=') { + if words.iter().any(|word| !word.contains('=')) { + parser.warn("malformed Dockerfile ENV assignment list"); + continue; + } + for (index, word) in words.iter().enumerate() { + if let Some((name, _)) = word.split_once('=') { + parser.emit(&format!("{location}/{index}"), Some(name), "declared", None); + } + } + } else if words.len() >= 2 { + parser.emit(&location, Some(first), "declared", None); + } else { + parser.warn("Dockerfile ENV requires a value"); + } + } +} + +fn compose(parser: &mut Parser<'_>, doc: &Value, location: &str) { + let Some(services) = doc.get("services").and_then(Value::as_object) else { + return; + }; + for (service, config) in services { + if !identifier(service) { + parser.warn("unsupported Compose service name"); + continue; + } + let location = format!("{location}/services/{}", pointer(service)); + if let Some(env) = config.get("environment") { + parser.env_map(env, &format!("{location}/environment"), true); + } + if let Some(files) = config.get("env_file") { + let list = files + .as_array() + .map_or_else(|| vec![files], |files| files.iter().collect()); + for (index, file) in list.iter().enumerate() { + let path = file + .as_str() + .or_else(|| file.get("path").and_then(Value::as_str)); + let mut reference = reference("env-file", None); + reference.target = path + .filter(|path| safe_reference_path(path)) + .map(str::to_owned); + parser.emit( + &format!("{location}/env_file/{index}"), + None, + "unresolved", + Some(reference), + ); + } + } + } +} + +fn safe_reference_path(path: &str) -> bool { + !path.is_empty() + && !path.starts_with('/') + && !path.contains(['$', '{', '}', '\\', ':']) + && !path.chars().any(char::is_control) +} + +fn actions(parser: &mut Parser<'_>, doc: &Value, location: &str) { + if let Some(env) = doc.get("env") { + parser.env_map(env, &format!("{location}/env"), false); + } + if let Some(jobs) = doc.get("jobs").and_then(Value::as_object) { + for (job, config) in jobs { + if !identifier(job) { + parser.warn("unsupported Actions job name"); + continue; + } + let location = format!("{location}/jobs/{}", pointer(job)); + if let Some(env) = config.get("env") { + parser.env_map(env, &format!("{location}/env"), false); + } + if let Some(steps) = config.get("steps").and_then(Value::as_array) { + for (index, step) in steps.iter().enumerate() { + if let Some(env) = step.get("env") { + parser.env_map(env, &format!("{location}/steps/{index}/env"), false); + } + } + } + } + } +} + +fn kubernetes(parser: &mut Parser<'_>, doc: &Value, location: &str) { + let Some(api) = doc.get("apiVersion").and_then(Value::as_str) else { + return; + }; + let Some(kind) = doc.get("kind").and_then(Value::as_str) else { + return; + }; + let supported_api = match kind { + "Pod" | "List" | "ReplicationController" => api == "v1", + "Deployment" | "StatefulSet" | "DaemonSet" | "ReplicaSet" => api == "apps/v1", + "Job" | "CronJob" => api == "batch/v1", + _ => false, + }; + if !supported_api { + return; + } + if kind == "List" { + if let Some(items) = doc.get("items").and_then(Value::as_array) { + for (index, item) in items.iter().enumerate() { + kubernetes(parser, item, &format!("{location}/items/{index}")); + } + } + return; + } + let path = match kind { + "Pod" => "/spec", + "Deployment" + | "StatefulSet" + | "DaemonSet" + | "ReplicaSet" + | "ReplicationController" + | "Job" => "/spec/template/spec", + "CronJob" => "/spec/jobTemplate/spec/template/spec", + _ => return, + }; + let Some(spec) = doc.pointer(path) else { + return; + }; + for containers in ["containers", "initContainers", "ephemeralContainers"] { + if let Some(list) = spec.get(containers).and_then(Value::as_array) { + for (index, container) in list.iter().enumerate() { + let location = format!("{location}{path}/{containers}/{index}"); + if let Some(env) = container.get("env") { + if let Some(env) = env.as_array() { + for (index, env) in env.iter().enumerate() { + let Some(name) = env.get("name").and_then(Value::as_str) else { + parser.warn("Kubernetes env entry requires a name"); + continue; + }; + let reference = env.get("valueFrom").map(kube_reference); + parser.emit( + &format!("{location}/env/{index}"), + Some(name), + if reference.is_some() { + "unresolved" + } else { + "declared" + }, + reference, + ); + } + } else { + parser.warn("Kubernetes env must be a list"); + } + } + if let Some(env) = container.get("envFrom") { + if let Some(env) = env.as_array() { + for (index, env) in env.iter().enumerate() { + let mut reference = kube_reference(env); + reference.prefix = env + .get("prefix") + .and_then(Value::as_str) + .filter(|p| identifier(p)) + .map(str::to_owned); + parser.emit( + &format!("{location}/envFrom/{index}"), + None, + "unresolved", + Some(reference), + ); + } + } else { + parser.warn("Kubernetes envFrom must be a list"); + } + } + } + } + } +} + +fn kube_reference(value: &Value) -> Reference { + for (field, kind) in [ + ("secretKeyRef", "secret-key"), + ("configMapKeyRef", "config-map-key"), + ("secretRef", "secret"), + ("configMapRef", "config-map"), + ] { + if let Some(value) = value.get(field) { + let mut result = reference(kind, value.get("name").and_then(Value::as_str)); + result.key = value + .get("key") + .and_then(Value::as_str) + .filter(|key| metadata_name(key)) + .map(str::to_owned); + return result; + } + } + if let Some(field) = value.get("fieldRef") { + let mut result = reference("field", None); + result.key = field + .get("fieldPath") + .and_then(Value::as_str) + .filter(|key| metadata_name(key)) + .map(str::to_owned); + return result; + } + if let Some(field) = value.get("resourceFieldRef") { + let mut result = reference( + "resource-field", + field.get("containerName").and_then(Value::as_str), + ); + result.key = field + .get("resource") + .and_then(Value::as_str) + .filter(|key| metadata_name(key)) + .map(str::to_owned); + return result; + } + reference("unknown", None) +} diff --git a/src/detect/environment/platforms.rs b/src/detect/environment/platforms.rs new file mode 100644 index 0000000..502ba6a --- /dev/null +++ b/src/detect/environment/platforms.rs @@ -0,0 +1,571 @@ +//! Environment declarations in deployment manifests of other hosting platforms. +//! +//! Each adapter reads only the documented, statically declared variable fields. +//! Literal values are inspected solely to classify them (scalar, secret/binding +//! reference) and are never copied into declarations or diagnostics. + +use serde_json::{Map, Value}; + +use super::{ + Format, Parser, Reference, Scope, Semantics, metadata_name, pointer, reference, + safe_reference_path, +}; + +pub(super) fn parse(parser: &mut Parser<'_>, doc: &Value, location: &str) { + match parser.format { + Format::AppJson => app_json(parser, doc, location), + Format::HerokuYml => heroku_yml(parser, doc, location), + Format::Render => render(parser, doc, location), + Format::Fly => literal_map( + parser, + doc.get("env"), + &format!("{location}/env"), + scoped(Scope::Runtime), + ), + Format::Digitalocean => digitalocean(parser, doc, location), + Format::Vercel => vercel(parser, doc, location), + Format::Netlify => netlify(parser, doc, location), + Format::AppEngine => app_engine(parser, doc, location), + Format::Copilot => copilot(parser, doc, location), + _ => (), + } +} + +fn scalar(value: &Value) -> bool { + value.is_string() || value.is_number() || value.is_boolean() +} + +fn child(location: &str, key: &str) -> String { + format!("{location}/{}", pointer(key)) +} + +fn scoped(scope: Scope) -> Semantics { + Semantics { + scope: Some(scope), + ..Semantics::default() + } +} + +fn string<'a>(value: &'a Value, field: &str) -> Option<&'a str> { + value.get(field).and_then(Value::as_str) +} + +fn list<'a>(value: &'a Value, field: &str) -> impl Iterator { + value + .get(field) + .and_then(Value::as_array) + .into_iter() + .flatten() + .enumerate() +} + +fn object<'a>( + parser: &mut Parser<'_>, + value: &'a Value, + message: &str, +) -> Option<&'a Map> { + let map = value.as_object(); + if map.is_none() { + parser.warn(message); + } + map +} + +/// Named sub-tables (contexts, environments, sidecars) whose keys become location segments. +fn named<'a>(parser: &mut Parser<'_>, value: Option<&'a Value>) -> Vec<(&'a String, &'a Value)> { + let mut entries = value + .and_then(Value::as_object) + .into_iter() + .flatten() + .collect::>(); + if entries.iter().any(|(name, _)| !metadata_name(name)) { + parser.warn("unsupported platform environment or context name"); + entries.retain(|(name, _)| metadata_name(name)); + } + entries +} + +/// `NAME = literal` maps: Fly `[env]`, Netlify/App Engine/heroku.yml/Vercel maps. +fn literal_map(parser: &mut Parser<'_>, value: Option<&Value>, location: &str, base: Semantics) { + let Some(value) = value else { + return; + }; + let Some(map) = object(parser, value, "environment declaration must be a mapping") else { + return; + }; + for (name, value) in map { + if !(value.is_null() || scalar(value)) { + parser.warn("environment value must be a scalar; declaration skipped"); + continue; + } + parser.declare( + &child(location, name), + Some(name), + "declared", + None, + Semantics { + has_default: Some(!value.is_null()), + ..base + }, + ); + } +} + +/// Heroku `app.json` (also read by Dokku) and Scalingo `scalingo.json`. +fn app_json(parser: &mut Parser<'_>, doc: &Value, location: &str) { + app_json_env(parser, doc.get("env"), &format!("{location}/env")); + for (name, environment) in named(parser, doc.get("environments")) { + app_json_env( + parser, + environment.get("env"), + &format!("{location}/environments/{}/env", pointer(name)), + ); + } +} + +fn app_json_env(parser: &mut Parser<'_>, value: Option<&Value>, location: &str) { + let Some(value) = value else { + return; + }; + let Some(map) = object(parser, value, "app.json env must be an object") else { + return; + }; + for (name, entry) in map { + let semantics = if scalar(entry) { + Semantics { + has_default: Some(true), + ..Semantics::default() + } + } else if entry.is_object() { + let generator = string(entry, "generator"); + Semantics { + required: entry.get("required").and_then(Value::as_bool), + secret: (generator == Some("secret")).then_some(true), + generated: generator.map(|_| true), + has_default: generator + .is_none() + .then(|| entry.get("value").is_some_and(scalar)), + scope: None, + } + } else { + parser.warn("app.json env entry must be a string or object; declaration skipped"); + continue; + }; + parser.declare( + &child(location, name), + Some(name), + "declared", + None, + semantics, + ); + } +} + +/// Heroku `heroku.yml`: provisioning config vars and build-time variables. +fn heroku_yml(parser: &mut Parser<'_>, doc: &Value, location: &str) { + literal_map( + parser, + doc.pointer("/setup/config"), + &format!("{location}/setup/config"), + scoped(Scope::Runtime), + ); + literal_map( + parser, + doc.pointer("/build/config"), + &format!("{location}/build/config"), + scoped(Scope::Build), + ); +} + +/// Render Blueprint `render.yaml`. +fn render(parser: &mut Parser<'_>, doc: &Value, location: &str) { + render_resources(parser, doc, location); + if let Some(ungrouped) = doc.get("ungrouped") { + render_resources(parser, ungrouped, &format!("{location}/ungrouped")); + } + for (project_index, project) in list(doc, "projects") { + for (index, environment) in list(project, "environments") { + render_resources( + parser, + environment, + &format!("{location}/projects/{project_index}/environments/{index}"), + ); + } + } +} + +fn render_resources(parser: &mut Parser<'_>, value: &Value, location: &str) { + for (field, group) in [("services", false), ("envVarGroups", true)] { + for (index, resource) in list(value, field) { + if let Some(vars) = resource.get("envVars") { + render_env_vars( + parser, + vars, + &format!("{location}/{field}/{index}/envVars"), + group, + ); + } + } + } +} + +fn render_env_vars(parser: &mut Parser<'_>, vars: &Value, location: &str, group: bool) { + let Some(vars) = vars.as_array() else { + parser.warn("Render envVars must be a list"); + return; + }; + for (index, var) in vars.iter().enumerate() { + let location = format!("{location}/{index}"); + if let Some(name) = var.get("fromGroup") { + parser.emit( + &location, + None, + "unresolved", + Some(reference("env-group", name.as_str())), + ); + continue; + } + let Some(name) = string(var, "key") else { + parser.warn("Render envVars entry requires a key"); + continue; + }; + if let Some(database) = var.get("fromDatabase") { + let reference = keyed( + "database", + string(database, "name"), + string(database, "property"), + ); + parser.emit(&location, Some(name), "unresolved", Some(reference)); + } else if let Some(service) = var.get("fromService") { + let target = string(service, "name"); + let reference = match string(service, "envVarKey") { + Some(key) => keyed("service-env", target, Some(key)), + None => keyed( + if matches!(string(service, "type"), Some("keyvalue" | "redis")) { + "key-value" + } else { + "service" + }, + target, + string(service, "property"), + ), + }; + parser.emit(&location, Some(name), "unresolved", Some(reference)); + } else if var.get("generateValue").and_then(Value::as_bool) == Some(true) { + let semantics = Semantics { + secret: Some(true), + generated: Some(true), + ..Semantics::default() + }; + parser.declare(&location, Some(name), "declared", None, semantics); + } else if var.get("sync").and_then(Value::as_bool) == Some(false) { + if group { + parser + .warn("Render ignores sync: false in environment groups; declaration skipped"); + continue; + } + let semantics = Semantics { + secret: Some(true), + has_default: Some(false), + ..Semantics::default() + }; + parser.declare(&location, Some(name), "declared", None, semantics); + } else if var.get("value").is_some_and(scalar) { + let semantics = Semantics { + has_default: Some(true), + ..Semantics::default() + }; + parser.declare(&location, Some(name), "declared", None, semantics); + } else { + parser.warn("unsupported Render envVars entry; declaration skipped"); + } + } +} + +fn keyed(kind: &str, target: Option<&str>, key: Option<&str>) -> Reference { + let mut result = reference(kind, target); + result.key = key.filter(|key| metadata_name(key)).map(str::to_owned); + result +} + +/// DigitalOcean App Platform app spec (`.do/app.yaml`) and Deploy button +/// template (`.do/deploy.template.yaml`, spec nested under `spec`). +fn digitalocean(parser: &mut Parser<'_>, doc: &Value, location: &str) { + let (spec, location) = match doc.get("spec") { + Some(spec) if spec.is_object() => (spec, format!("{location}/spec")), + _ => (doc, location.to_owned()), + }; + let databases = list(spec, "databases") + .filter_map(|(_, database)| string(database, "name")) + .collect::>(); + if let Some(envs) = spec.get("envs") { + digitalocean_envs(parser, envs, &format!("{location}/envs"), &databases); + } + for kind in ["services", "static_sites", "workers", "jobs", "functions"] { + for (index, component) in list(spec, kind) { + if let Some(envs) = component.get("envs") { + digitalocean_envs( + parser, + envs, + &format!("{location}/{kind}/{index}/envs"), + &databases, + ); + } + } + } +} + +fn digitalocean_envs(parser: &mut Parser<'_>, envs: &Value, location: &str, databases: &[&str]) { + let Some(envs) = envs.as_array() else { + parser.warn("DigitalOcean envs must be a list"); + return; + }; + for (index, env) in envs.iter().enumerate() { + let location = format!("{location}/{index}"); + let Some(name) = string(env, "key") else { + parser.warn("DigitalOcean env entry requires a key"); + continue; + }; + let secret = match string(env, "type") { + Some("SECRET") => Some(true), + Some("GENERAL") => Some(false), + _ => None, + }; + let mut semantics = Semantics { + secret, + scope: match string(env, "scope") { + Some("RUN_TIME") => Some(Scope::Runtime), + Some("BUILD_TIME") => Some(Scope::Build), + Some("RUN_AND_BUILD_TIME") => Some(Scope::BuildAndRuntime), + _ => None, + }, + ..Semantics::default() + }; + // Encrypted SECRET values are opaque; only non-secret values may be bindings. + let binding = + string(env, "value").filter(|value| secret != Some(true) && value.contains("${")); + if let Some(binding) = binding { + let reference = bindable(binding, databases); + parser.declare( + &location, + Some(name), + "unresolved", + Some(reference), + semantics, + ); + } else { + semantics.has_default = Some(env.get("value").is_some_and(scalar)); + parser.declare(&location, Some(name), "declared", None, semantics); + } + } +} + +/// Whole-value `${APP_URL}`, `${component.VAR}` or `${database.pool.VAR}` bindings. +/// Partial interpolation is reported without echoing any part of the value. +fn bindable(value: &str, databases: &[&str]) -> Reference { + let inner = value + .strip_prefix("${") + .and_then(|value| value.strip_suffix('}')) + .filter(|inner| !inner.contains(['$', '{', '}'])); + let parts = inner.map(|inner| inner.split('.').collect::>()); + match parts.as_deref() { + Some([key]) => keyed("app", None, Some(key)), + Some([target, key] | [target, _, key]) => { + let database = databases + .iter() + .any(|name| name == target || (target.len() == 32 && name.starts_with(target))); + keyed( + if database { "database" } else { "service" }, + Some(target), + Some(key), + ) + } + _ => reference("interpolation", None), + } +} + +/// Legacy `vercel.json`/`vercel.toml` `env` (functions) and `build.env` (builds). +fn vercel(parser: &mut Parser<'_>, doc: &Value, location: &str) { + for (path, scope) in [("/env", Scope::Runtime), ("/build/env", Scope::Build)] { + let Some(value) = doc.pointer(path) else { + continue; + }; + let Some(map) = object(parser, value, "Vercel env must be an object") else { + continue; + }; + for (name, value) in map { + if !scalar(value) { + parser.warn("environment value must be a scalar; declaration skipped"); + continue; + } + let location = child(&format!("{location}{path}"), name); + match value.as_str().and_then(|value| value.strip_prefix('@')) { + Some(secret) => parser.declare( + &location, + Some(name), + "unresolved", + Some(reference("platform-secret", Some(secret))), + Semantics { + secret: Some(true), + scope: Some(scope), + ..Semantics::default() + }, + ), + None => parser.declare( + &location, + Some(name), + "declared", + None, + Semantics { + has_default: Some(true), + scope: Some(scope), + ..Semantics::default() + }, + ), + } + } + } +} + +/// `netlify.toml` build/context environments and Deploy button template variables. +fn netlify(parser: &mut Parser<'_>, doc: &Value, location: &str) { + literal_map( + parser, + doc.pointer("/build/environment"), + &format!("{location}/build/environment"), + scoped(Scope::Build), + ); + for (name, context) in named(parser, doc.get("context")) { + literal_map( + parser, + context.get("environment"), + &format!("{location}/context/{}/environment", pointer(name)), + scoped(Scope::Build), + ); + } + let Some(template) = doc.pointer("/template/environment") else { + return; + }; + let Some(map) = object( + parser, + template, + "Netlify template environment must be a table", + ) else { + return; + }; + let location = format!("{location}/template/environment"); + for name in map.keys() { + // Template values are UI labels for required user input, not defaults. + let semantics = Semantics { + required: Some(true), + has_default: Some(false), + ..Semantics::default() + }; + parser.declare( + &child(&location, name), + Some(name), + "declared", + None, + semantics, + ); + } +} + +/// Google App Engine `app.yaml`; requires the mandatory `runtime` element. +fn app_engine(parser: &mut Parser<'_>, doc: &Value, location: &str) { + if string(doc, "runtime").is_none() { + return; + } + literal_map( + parser, + doc.get("env_variables"), + &format!("{location}/env_variables"), + scoped(Scope::Runtime), + ); + literal_map( + parser, + doc.get("build_env_variables"), + &format!("{location}/build_env_variables"), + scoped(Scope::Build), + ); +} + +/// AWS Copilot workload manifests (`copilot//manifest.yml`). +fn copilot(parser: &mut Parser<'_>, doc: &Value, location: &str) { + if string(doc, "type").is_none_or(|kind| kind == "Environment") { + return; + } + copilot_workload(parser, doc, location); + for (name, environment) in named(parser, doc.get("environments")) { + copilot_workload( + parser, + environment, + &format!("{location}/environments/{}", pointer(name)), + ); + } +} + +/// The main container plus its sidecars; also the shape of an `environments` override. +fn copilot_workload(parser: &mut Parser<'_>, value: &Value, location: &str) { + copilot_container(parser, value, location); + for (name, sidecar) in named(parser, value.get("sidecars")) { + copilot_container( + parser, + sidecar, + &format!("{location}/sidecars/{}", pointer(name)), + ); + } +} + +fn copilot_container(parser: &mut Parser<'_>, value: &Value, location: &str) { + for (field, secret) in [("variables", false), ("secrets", true)] { + let Some(map) = value.get(field) else { + continue; + }; + let Some(map) = object(parser, map, "Copilot variables and secrets must be maps") else { + continue; + }; + for (name, value) in map { + let location = child(&format!("{location}/{field}"), name); + let semantics = Semantics { + secret: Some(secret), + scope: Some(Scope::Runtime), + ..Semantics::default() + }; + let reference = if let Some(export) = value.get("from_cfn") { + Some(reference("cloudformation-export", export.as_str())) + } else if let Some(secret) = value.get("secretsmanager") { + Some(reference("secrets-manager", secret.as_str())) + } else if secret && value.is_string() { + Some(reference("ssm-parameter", value.as_str())) + } else if !secret && (value.is_null() || scalar(value)) { + None + } else { + parser.warn("unsupported Copilot variable or secret entry; declaration skipped"); + continue; + }; + if reference.is_some() { + parser.declare(&location, Some(name), "unresolved", reference, semantics); + } else { + let semantics = Semantics { + has_default: Some(!value.is_null()), + ..semantics + }; + parser.declare(&location, Some(name), "declared", None, semantics); + } + } + } + if let Some(file) = value.get("env_file") { + let mut reference = reference("env-file", None); + reference.target = file + .as_str() + .filter(|path| safe_reference_path(path)) + .map(str::to_owned); + parser.emit( + &format!("{location}/env_file"), + None, + "unresolved", + Some(reference), + ); + } +} diff --git a/src/detect/environment/references.rs b/src/detect/environment/references.rs new file mode 100644 index 0000000..68f5b98 --- /dev/null +++ b/src/detect/environment/references.rs @@ -0,0 +1,297 @@ +//! Literal search for catalog-declared environment names in source files. +//! +//! This is not a read-site scanner: only names that a platform injects or a +//! framework exposes by documented convention are recognised, by exact name or +//! by `PREFIX[A-Z0-9_]+` on identifier boundaries. Bytes are inspected for +//! ASCII identifiers only; no source text, value or context is retained. + +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::{Result, bail}; +use serde::Deserialize; + +use super::super::rules::FilePattern; +use super::{CodeReference, Declaration, Format, Occurrence, Origin, Scope, Semantics}; + +/// Occurrences listed per `(project, name)` record; `occurrenceCount` keeps the total. +pub const MAX_OCCURRENCES: usize = 20; + +#[derive(Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub(in super::super) struct Spec { + extensions: Vec, + exclude: Vec, + patterns: Vec, +} + +impl Spec { + pub fn extend(&mut self, extra: Self) { + self.extensions.extend(extra.extensions); + self.exclude.extend(extra.exclude); + self.patterns.extend(extra.patterns); + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PatternSpec { + origin: Origin, + provider: String, + #[serde(default)] + prefixes: Vec, + #[serde(default)] + names: Vec, + #[serde(default)] + scope: Option, + /// Basenames; when set, matches count only if one of them exists in the scan. + #[serde(default)] + signals: Vec, + /// Detection ids; when set, the owning project must have one of them. + #[serde(default)] + requires: Vec, + /// Primary documentation for the names/prefixes; validated, not emitted. + docs: Vec, +} + +pub(in super::super) struct Catalog { + patterns: Vec, + extensions: BTreeSet, + exclude: Vec, + /// Exact names and prefixes map to candidate patterns in catalog order. + names: BTreeMap>, + /// Longest prefix first, so `NUXT_PUBLIC_` wins over `NUXT_`. + prefixes: Vec<(String, Vec)>, + signals: BTreeSet, +} + +/// Matches in one scan directory, keyed by name; storage is bounded per name. +#[derive(Debug, Default, PartialEq, Eq)] +pub(in super::super) struct Hits(BTreeMap); + +impl Hits { + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +/// Unique `(source, line)` count plus the smallest `MAX_OCCURRENCES` of them. +#[derive(Debug, Default, PartialEq, Eq)] +pub(in super::super) struct Occurrences { + count: usize, + listed: Vec, +} + +impl Occurrences { + /// Counts a `(source, line)` the caller has not added before. + fn add(&mut self, source: &str, line: usize) { + self.count += 1; + self.insert( + || Occurrence { + source: source.to_owned(), + line, + }, + (source, line), + ); + } + + /// Merges occurrences from disjoint sources. + fn merge(&mut self, other: &Self) { + self.count += other.count; + for occurrence in &other.listed { + self.insert(|| occurrence.clone(), (&occurrence.source, occurrence.line)); + } + } + + fn insert(&mut self, make: impl FnOnce() -> Occurrence, key: (&str, usize)) { + let index = self + .listed + .partition_point(|o| (o.source.as_str(), o.line) < key); + if index < MAX_OCCURRENCES { + self.listed.insert(index, make()); + self.listed.truncate(MAX_OCCURRENCES); + } + } +} + +fn upper(value: &str) -> bool { + value.as_bytes().first().is_some_and(u8::is_ascii_uppercase) + && value + .bytes() + .all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit() || ch == b'_') +} + +impl Catalog { + pub fn new(spec: Spec) -> Result { + let mut names = BTreeMap::>::new(); + let mut prefixes = Vec::<(String, Vec)>::new(); + let mut signals = BTreeSet::new(); + for (index, pattern) in spec.patterns.iter().enumerate() { + if pattern.provider.is_empty() + || pattern.docs.iter().all(|url| !url.starts_with("https://")) + || (pattern.names.is_empty() && pattern.prefixes.is_empty()) + || !pattern + .names + .iter() + .chain(&pattern.prefixes) + .all(|n| upper(n)) + || pattern + .signals + .iter() + .any(|s| s.is_empty() || s.contains(['/', '\\', '*', '?'])) + { + bail!( + "invalid environment code-reference pattern: {}", + pattern.provider + ); + } + for name in &pattern.names { + names.entry(name.clone()).or_default().push(index); + } + for prefix in &pattern.prefixes { + match prefixes.iter_mut().find(|(existing, _)| existing == prefix) { + Some((_, candidates)) => candidates.push(index), + None => prefixes.push((prefix.clone(), vec![index])), + } + } + signals.extend(pattern.signals.iter().cloned()); + } + prefixes.sort_by(|a, b| b.0.len().cmp(&a.0.len()).then(a.0.cmp(&b.0))); + if spec + .extensions + .iter() + .any(|e| e.is_empty() || e.contains(['.', '/', '\\', '*'])) + || spec + .exclude + .iter() + .any(|e| e.is_empty() || e.contains(['/', '\\'])) + { + bail!("invalid environment code-reference file pattern"); + } + Ok(Self { + extensions: spec.extensions.into_iter().collect(), + exclude: spec.exclude.iter().map(|p| FilePattern::new(p)).collect(), + patterns: spec.patterns, + names, + prefixes, + signals, + }) + } + + /// Whether a file of this basename is searched for references. + pub fn scans(&self, file: &str) -> bool { + !self.patterns.is_empty() + && file.rsplit_once('.').is_some_and(|(stem, extension)| { + !stem.is_empty() && self.extensions.contains(extension) + }) + && !self.exclude.iter().any(|pattern| pattern.matches(file)) + } + + pub fn is_signal(&self, file: &str) -> bool { + self.signals.contains(file) + } + + /// Candidate patterns: exact names first, then the longest matching prefix. + fn candidates(&self, token: &str) -> &[usize] { + if let Some(candidates) = self.names.get(token) { + return candidates; + } + self.prefixes + .iter() + .find(|(prefix, _)| { + token.len() > prefix.len() && token.starts_with(prefix.as_str()) && upper(token) + }) + .map_or(&[], |(_, candidates)| candidates) + } + + /// Whole ASCII identifier tokens (`[A-Za-z0-9_]+`) that match a pattern. + pub fn find(&self, source: &str, bytes: &[u8], hits: &mut Hits) { + let word = |ch: u8| ch.is_ascii_alphanumeric() || ch == b'_'; + // Last line counted per name in this file, so a line counts once. + let mut last = BTreeMap::<&str, usize>::new(); + let mut line = 1; + let mut index = 0; + while index < bytes.len() { + let ch = bytes[index]; + if ch == b'\n' { + line += 1; + index += 1; + } else if word(ch) { + let start = index; + while index < bytes.len() && word(bytes[index]) { + index += 1; + } + if ch.is_ascii_uppercase() + && let Ok(token) = std::str::from_utf8(&bytes[start..index]) + && !self.candidates(token).is_empty() + && last.insert(token, line) != Some(line) + { + match hits.0.get_mut(token) { + Some(occurrences) => occurrences.add(source, line), + None => hits + .0 + .entry(token.to_owned()) + .or_default() + .add(source, line), + } + } + } else { + index += 1; + } + } + } + + /// Builds one record per `(project, name)`, choosing the first candidate + /// pattern whose signal and framework gates pass. Each input is one scan + /// directory's hits with its owning project and that project's detections. + pub fn records<'a>( + &self, + hits: impl IntoIterator, &'a BTreeSet, &'a Hits)>, + signals: &BTreeSet, + ) -> Vec { + let mut grouped = BTreeMap::<(Option, &str), (usize, Occurrences)>::new(); + for (project, detections, hits) in hits { + for (name, occurrences) in &hits.0 { + let Some(&index) = self.candidates(name).iter().find(|&&index| { + let pattern = &self.patterns[index]; + (pattern.signals.is_empty() + || pattern.signals.iter().any(|s| signals.contains(s))) + && (pattern.requires.is_empty() + || pattern.requires.iter().any(|id| detections.contains(id))) + }) else { + continue; + }; + let (pattern, merged) = grouped + .entry((project.clone(), name.as_str())) + .or_insert_with(|| (index, Occurrences::default())); + *pattern = (*pattern).min(index); + merged.merge(occurrences); + } + } + grouped + .into_iter() + .map(|((project, name), (index, occurrences))| { + let pattern = &self.patterns[index]; + let first = occurrences.listed[0].clone(); + Declaration { + source: first.source, + format: Format::CodeReference, + location: format!("line:{}", first.line), + project, + name: Some(name.to_owned()), + state: "referenced".to_owned(), + reference: None, + semantics: Semantics { + scope: pattern.scope, + ..Semantics::default() + }, + code: Some(CodeReference { + origin: pattern.origin, + provider: pattern.provider.clone(), + occurrences: occurrences.listed, + occurrence_count: occurrences.count, + }), + } + }) + .collect() + } +} diff --git a/src/detect/environment_platform_tests.rs b/src/detect/environment_platform_tests.rs new file mode 100644 index 0000000..c2db987 --- /dev/null +++ b/src/detect/environment_platform_tests.rs @@ -0,0 +1,799 @@ +use super::environment::{Declaration, Format, Scope, Semantics, parse}; +use super::tests::fixture; +use super::{Report, render, scan, scan_options}; + +fn with_env(files: &[(&str, &str)]) -> Report { + let dir = fixture(files); + scan_options(dir.path(), None, super::EnvScan::Declarations).unwrap() +} + +fn declarations(report: &Report) -> &[Declaration] { + report.environment.as_deref().unwrap() +} + +fn find<'a>(report: &'a Report, source: &str, name: &str) -> &'a Declaration { + declarations(report) + .iter() + .find(|d| d.source == source && d.name.as_deref() == Some(name)) + .unwrap_or_else(|| panic!("{source}: {name} missing: {report:#?}")) +} + +fn names<'a>(report: &'a Report, source: &str) -> Vec<&'a str> { + declarations(report) + .iter() + .filter(|d| d.source == source) + .filter_map(|d| d.name.as_deref()) + .collect() +} + +fn adapter(format: Format, source: &str, text: &str) -> (Vec, Vec) { + let mut diagnostics = Vec::new(); + let declarations = parse(format, source, text, &mut diagnostics); + (declarations, diagnostics) +} + +fn assert_redacted(report: &Report) { + let json = serde_json::to_string(report).unwrap(); + let text = render(&serde_json::to_value(report).unwrap()).join("\n"); + for output in [json, text] { + assert!(!output.contains("DO_NOT_DISCLOSE"), "{output}"); + } +} + +const NONE: Semantics = Semantics { + required: None, + secret: None, + generated: None, + has_default: None, + scope: None, +}; + +#[test] +fn app_json_and_scalingo_map_required_generated_and_default_presence() { + let app = r#"{ + "name": "example", + "stack": "heroku-22", + "stack": "heroku-24", + "env": { + "PLAIN": "DO_NOT_DISCLOSE", + "TOKEN": {"description": "API token", "required": true}, + "OPTIONAL": {"value": "DO_NOT_DISCLOSE", "required": false}, + "SECRET_KEY_BASE": {"generator": "secret"}, + "EMPTY_DEFAULT": {"value": ""} + }, + "environments": {"test": {"env": {"PLAIN": "DO_NOT_DISCLOSE"}}}, + "addons": ["heroku-postgresql"] +}"#; + let scalingo = + r#"{"env": {"ADMIN_URL": {"generator": "url", "template": "%URL%/DO_NOT_DISCLOSE"}}}"#; + let report = with_env(&[ + ("app.json", app), + ("scalingo.json", scalingo), + ("mobile/app.json", r#"{"expo": {"name": "not-a-manifest"}}"#), + ]); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert_eq!( + names(&report, "app.json"), + [ + "EMPTY_DEFAULT", + "OPTIONAL", + "PLAIN", + "SECRET_KEY_BASE", + "TOKEN", + "PLAIN" + ] + ); + assert!(names(&report, "mobile/app.json").is_empty()); + let semantics = |source, name| find(&report, source, name).semantics; + assert_eq!( + semantics("app.json", "PLAIN"), + Semantics { + has_default: Some(true), + ..NONE + } + ); + assert_eq!( + semantics("app.json", "TOKEN"), + Semantics { + required: Some(true), + has_default: Some(false), + ..NONE + } + ); + assert_eq!( + semantics("app.json", "OPTIONAL"), + Semantics { + required: Some(false), + has_default: Some(true), + ..NONE + } + ); + assert_eq!( + semantics("app.json", "SECRET_KEY_BASE"), + Semantics { + secret: Some(true), + generated: Some(true), + ..NONE + } + ); + assert_eq!( + semantics("scalingo.json", "ADMIN_URL"), + Semantics { + generated: Some(true), + ..NONE + } + ); + let test = declarations(&report) + .iter() + .filter(|d| d.name.as_deref() == Some("PLAIN")) + .map(|d| d.location.as_str()) + .collect::>(); + assert_eq!( + test, + [ + "document:0/env/PLAIN", + "document:0/environments/test/env/PLAIN" + ] + ); + assert!( + declarations(&report) + .iter() + .all(|d| d.format == Format::AppJson) + ); + assert_redacted(&report); +} + +#[test] +fn render_blueprint_maps_references_generated_prompted_groups_and_projects() { + let yaml = r#" +services: + - type: web + name: api + envVars: + - key: API_BASE_URL + value: DO_NOT_DISCLOSE + - key: APP_SECRET + generateValue: true + - key: STRIPE_KEY + sync: false + - key: DATABASE_URL + fromDatabase: {name: elephant, property: connectionString} + - key: REDIS_URL + fromService: {type: keyvalue, name: lightning, property: connectionString} + - key: WORKER_HOST + fromService: {type: pserv, name: worker, property: hostport} + - key: MINIO_PASSWORD + fromService: {type: pserv, name: minio, envVarKey: MINIO_ROOT_PASSWORD} + - fromGroup: shared-settings +databases: + - name: elephant +envVarGroups: + - name: shared-settings + envVars: + - key: CONCURRENCY + value: 2 + - key: IGNORED + sync: false +projects: + - name: app + environments: + - name: production + services: + - name: web + envVars: [{key: PROJECT_SCOPED, value: DO_NOT_DISCLOSE}] +"#; + let report = with_env(&[("render.yaml", yaml)]); + assert_eq!( + names(&report, "render.yaml"), + [ + "API_BASE_URL", + "APP_SECRET", + "STRIPE_KEY", + "DATABASE_URL", + "REDIS_URL", + "WORKER_HOST", + "MINIO_PASSWORD", + "CONCURRENCY", + "PROJECT_SCOPED" + ] + ); + assert_eq!(report.diagnostics.len(), 1, "{report:#?}"); + assert!(report.diagnostics[0].message.contains("sync: false")); + let get = |name| find(&report, "render.yaml", name); + assert_eq!( + get("API_BASE_URL").semantics, + Semantics { + has_default: Some(true), + ..NONE + } + ); + assert_eq!( + get("APP_SECRET").semantics, + Semantics { + secret: Some(true), + generated: Some(true), + ..NONE + } + ); + assert_eq!( + get("STRIPE_KEY").semantics, + Semantics { + secret: Some(true), + has_default: Some(false), + ..NONE + } + ); + for (name, kind, target, key) in [ + ("DATABASE_URL", "database", "elephant", "connectionString"), + ("REDIS_URL", "key-value", "lightning", "connectionString"), + ("WORKER_HOST", "service", "worker", "hostport"), + ( + "MINIO_PASSWORD", + "service-env", + "minio", + "MINIO_ROOT_PASSWORD", + ), + ] { + let declaration = get(name); + assert_eq!(declaration.state, "unresolved"); + assert_eq!(declaration.semantics, NONE); + let reference = declaration.reference.as_ref().unwrap(); + assert_eq!( + ( + reference.kind.as_str(), + reference.target.as_deref(), + reference.key.as_deref() + ), + (kind, Some(target), Some(key)) + ); + } + let group = declarations(&report) + .iter() + .find(|d| d.name.is_none()) + .unwrap(); + assert_eq!(group.location, "document:0/services/0/envVars/7"); + let reference = group.reference.as_ref().unwrap(); + assert_eq!(reference.kind, "env-group"); + assert_eq!(reference.target.as_deref(), Some("shared-settings")); + assert_eq!( + get("PROJECT_SCOPED").location, + "document:0/projects/0/environments/0/services/0/envVars/0" + ); + assert_redacted(&report); +} + +#[test] +fn toml_and_json_platform_files_record_scopes_templates_and_vercel_secrets() { + let report = with_env(&[ + ( + "fly.toml", + "app = 'example'\n[env]\n LOG_LEVEL = 'DO_NOT_DISCLOSE'\n PORT = 8080\n[build.args]\n NOT_ENV = 'DO_NOT_DISCLOSE'\n", + ), + ("fly.staging.toml", "[env]\nSTAGING = 'DO_NOT_DISCLOSE'\n"), + ( + "netlify.toml", + r#" +[build] + command = "make" + environment = { NODE_VERSION = "DO_NOT_DISCLOSE" } +[context.production.environment] + API_URL = "DO_NOT_DISCLOSE" +[context.deploy-preview] + environment = { API_URL = "DO_NOT_DISCLOSE" } +[template.environment] + SECRET_TOKEN = "DO_NOT_DISCLOSE label" +[[redirects]] + from = "/a" + to = "/b" + [redirects.headers] + X-Api-Key = "DO_NOT_DISCLOSE" +"#, + ), + ( + "web/vercel.json", + r#"{"env": {"MY_KEY": "DO_NOT_DISCLOSE", "SECRET": "@my-secret-name"}, "build": {"env": {"BUILD_KEY": "DO_NOT_DISCLOSE"}}}"#, + ), + ("site/vercel.toml", "[env]\nTOML_KEY = 'DO_NOT_DISCLOSE'\n"), + ]); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert_eq!(names(&report, "fly.toml"), ["LOG_LEVEL", "PORT"]); + assert_eq!(names(&report, "fly.staging.toml"), ["STAGING"]); + assert_eq!( + find(&report, "fly.toml", "PORT").semantics, + Semantics { + has_default: Some(true), + scope: Some(Scope::Runtime), + ..NONE + } + ); + assert_eq!( + names(&report, "netlify.toml"), + ["NODE_VERSION", "API_URL", "API_URL", "SECRET_TOKEN"] + ); + let netlify = declarations(&report) + .iter() + .filter(|d| d.source == "netlify.toml") + .map(|d| (d.location.as_str(), d.semantics)) + .collect::>(); + let build = Semantics { + has_default: Some(true), + scope: Some(Scope::Build), + ..NONE + }; + assert_eq!( + netlify, + [ + ("document:0/build/environment/NODE_VERSION", build), + ( + "document:0/context/deploy-preview/environment/API_URL", + build + ), + ("document:0/context/production/environment/API_URL", build), + ( + "document:0/template/environment/SECRET_TOKEN", + Semantics { + required: Some(true), + has_default: Some(false), + ..NONE + } + ), + ] + ); + let secret = find(&report, "web/vercel.json", "SECRET"); + assert_eq!(secret.state, "unresolved"); + assert_eq!( + secret.semantics, + Semantics { + secret: Some(true), + scope: Some(Scope::Runtime), + ..NONE + } + ); + let reference = secret.reference.as_ref().unwrap(); + assert_eq!(reference.kind, "platform-secret"); + assert_eq!(reference.target.as_deref(), Some("my-secret-name")); + assert_eq!( + find(&report, "web/vercel.json", "BUILD_KEY") + .semantics + .scope, + Some(Scope::Build) + ); + assert_eq!(names(&report, "site/vercel.toml"), ["TOML_KEY"]); + assert_redacted(&report); +} + +#[test] +fn digitalocean_specs_map_type_scope_and_bindable_references() { + let spec = r#" +name: example +envs: + - key: APP_WIDE + value: DO_NOT_DISCLOSE + scope: RUN_AND_BUILD_TIME +services: + - name: api + envs: + - key: DATABASE_URL + value: ${elephant.DATABASE_URL} + scope: RUN_TIME + - key: POOL_URL + value: ${elephant.pool.DATABASE_URL} + - key: PUBLIC_URL + value: ${APP_URL} + - key: SELF_URL + value: ${_self.PRIVATE_URL} + - key: PARTIAL + value: postgres://DO_NOT_DISCLOSE@${elephant.HOSTNAME}/db + - key: API_TOKEN + value: EV[1:DO_NOT_DISCLOSE] + type: SECRET + - key: OPAQUE_SECRET + value: ${DO_NOT_DISCLOSE.DO_NOT_DISCLOSE} + type: SECRET +static_sites: + - name: web + envs: + - key: BUILD_ONLY + value: DO_NOT_DISCLOSE + scope: BUILD_TIME + type: GENERAL +databases: + - name: elephant + engine: PG +"#; + let template = r#" +spec: + name: sample + services: + - name: web + envs: + - key: EMAIL + scope: RUN_TIME + - key: DB_PASSWORD + value: DO_NOT_DISCLOSE + type: SECRET +"#; + let report = with_env(&[ + (".do/app.yaml", spec), + (".do/deploy.template.yaml", template), + ("other/deploy.template.yaml", template), + ]); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert!(names(&report, "other/deploy.template.yaml").is_empty()); + let get = |name| find(&report, ".do/app.yaml", name); + assert_eq!( + get("APP_WIDE").semantics, + Semantics { + has_default: Some(true), + scope: Some(Scope::BuildAndRuntime), + ..NONE + } + ); + for (name, kind, target, key) in [ + ( + "DATABASE_URL", + "database", + Some("elephant"), + Some("DATABASE_URL"), + ), + ( + "POOL_URL", + "database", + Some("elephant"), + Some("DATABASE_URL"), + ), + ("PUBLIC_URL", "app", None, Some("APP_URL")), + ("SELF_URL", "service", Some("_self"), Some("PRIVATE_URL")), + ("PARTIAL", "interpolation", None, None), + ] { + let declaration = get(name); + assert_eq!(declaration.state, "unresolved", "{name}"); + assert_eq!(declaration.semantics.has_default, None, "{name}"); + let reference = declaration.reference.as_ref().unwrap(); + assert_eq!( + ( + reference.kind.as_str(), + reference.target.as_deref(), + reference.key.as_deref() + ), + (kind, target, key), + "{name}" + ); + } + assert_eq!(get("DATABASE_URL").semantics.scope, Some(Scope::Runtime)); + for name in ["API_TOKEN", "OPAQUE_SECRET"] { + assert_eq!(get(name).state, "declared"); + assert_eq!( + get(name).semantics, + Semantics { + secret: Some(true), + has_default: Some(true), + ..NONE + } + ); + } + assert_eq!( + get("BUILD_ONLY").semantics, + Semantics { + secret: Some(false), + has_default: Some(true), + scope: Some(Scope::Build), + ..NONE + } + ); + let template = |name| find(&report, ".do/deploy.template.yaml", name); + assert_eq!( + template("EMAIL").location, + "document:0/spec/services/0/envs/0" + ); + assert_eq!( + template("EMAIL").semantics, + Semantics { + has_default: Some(false), + scope: Some(Scope::Runtime), + ..NONE + } + ); + assert_eq!(template("DB_PASSWORD").semantics.secret, Some(true)); + assert_redacted(&report); +} + +#[test] +fn app_engine_requires_runtime_and_heroku_yml_separates_build_config() { + let report = with_env(&[ + ( + "app.yaml", + "runtime: python312\nenv_variables:\n BUCKET_NAME: DO_NOT_DISCLOSE\nbuild_env_variables:\n GOOGLE_NODE_RUN_SCRIPTS: ''\n", + ), + ( + "config/app.yaml", + "env_variables: {NOT_APP_ENGINE: DO_NOT_DISCLOSE}", + ), + ( + "heroku.yml", + "setup:\n addons: [{plan: heroku-postgresql, as: DATABASE}]\n config: {S3_BUCKET: DO_NOT_DISCLOSE}\nbuild:\n docker: {web: Dockerfile}\n config: {RAILS_ENV: DO_NOT_DISCLOSE}\nrun:\n web: echo NOT_ENV=DO_NOT_DISCLOSE\n", + ), + ]); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert_eq!( + names(&report, "app.yaml"), + ["BUCKET_NAME", "GOOGLE_NODE_RUN_SCRIPTS"] + ); + assert!(names(&report, "config/app.yaml").is_empty()); + assert_eq!( + find(&report, "app.yaml", "BUCKET_NAME").semantics.scope, + Some(Scope::Runtime) + ); + assert_eq!( + find(&report, "app.yaml", "GOOGLE_NODE_RUN_SCRIPTS").semantics, + Semantics { + has_default: Some(true), + scope: Some(Scope::Build), + ..NONE + } + ); + assert_eq!(names(&report, "heroku.yml"), ["S3_BUCKET", "RAILS_ENV"]); + assert_eq!( + find(&report, "heroku.yml", "RAILS_ENV").semantics.scope, + Some(Scope::Build) + ); + assert!(declarations(&report).iter().all(|d| d.project.is_none())); + assert_redacted(&report); +} + +#[test] +fn copilot_workloads_map_variables_secrets_overrides_and_env_files() { + let manifest = r#" +name: api +type: Load Balanced Web Service +variables: + LOG_LEVEL: DO_NOT_DISCLOSE + QUEUE_URL: {from_cfn: queue-export} + UNSET: +secrets: + GITHUB_TOKEN: GH_TOKEN_SECRET + NESTED_PATH: /copilot/DO_NOT_DISCLOSE/secret + DB_SECRET: {secretsmanager: mysql} +env_file: ./api.env +sidecars: + proxy: + variables: {PROXY_MODE: DO_NOT_DISCLOSE} +environments: + prod: + variables: {LOG_LEVEL: DO_NOT_DISCLOSE} + sidecars: + proxy: + variables: {PROXY_OVERRIDE: DO_NOT_DISCLOSE} +"#; + let report = with_env(&[ + ("copilot/api/manifest.yml", manifest), + ( + "copilot/environments/prod/manifest.yml", + "name: prod\ntype: Environment\nvariables: {NOT_A_WORKLOAD: DO_NOT_DISCLOSE}\n", + ), + ("services/manifest.yml", manifest), + ]); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let source = "copilot/api/manifest.yml"; + assert_eq!( + names(&report, source), + [ + "LOG_LEVEL", + "QUEUE_URL", + "UNSET", + "DB_SECRET", + "GITHUB_TOKEN", + "NESTED_PATH", + "PROXY_MODE", + "LOG_LEVEL", + "PROXY_OVERRIDE" + ] + ); + assert!(names(&report, "copilot/environments/prod/manifest.yml").is_empty()); + assert!(names(&report, "services/manifest.yml").is_empty()); + let runtime_secret = Semantics { + secret: Some(true), + scope: Some(Scope::Runtime), + ..NONE + }; + assert_eq!( + find(&report, source, "UNSET").semantics, + Semantics { + secret: Some(false), + has_default: Some(false), + scope: Some(Scope::Runtime), + ..NONE + } + ); + for (name, kind, target) in [ + ("GITHUB_TOKEN", "ssm-parameter", Some("GH_TOKEN_SECRET")), + ("NESTED_PATH", "ssm-parameter", None), + ("DB_SECRET", "secrets-manager", Some("mysql")), + ] { + let declaration = find(&report, source, name); + assert_eq!(declaration.semantics, runtime_secret); + let reference = declaration.reference.as_ref().unwrap(); + assert_eq!( + (reference.kind.as_str(), reference.target.as_deref()), + (kind, target) + ); + } + assert_eq!( + find(&report, source, "QUEUE_URL") + .reference + .as_ref() + .unwrap() + .kind, + "cloudformation-export" + ); + assert_eq!( + find(&report, source, "PROXY_MODE").semantics, + Semantics { + secret: Some(false), + has_default: Some(true), + scope: Some(Scope::Runtime), + ..NONE + } + ); + assert_eq!( + find(&report, source, "PROXY_OVERRIDE").location, + "document:0/environments/prod/sidecars/proxy/variables/PROXY_OVERRIDE" + ); + let file = declarations(&report) + .iter() + .find(|d| d.name.is_none()) + .unwrap(); + assert_eq!( + file.reference.as_ref().unwrap().target.as_deref(), + Some("./api.env") + ); + assert_redacted(&report); +} + +#[test] +fn malformed_platform_inputs_are_redacted_diagnostics() { + for (format, source, text) in [ + ( + Format::AppJson, + "app.json", + r#"{"env": {"KEY": ["DO_NOT_DISCLOSE"]}}"#, + ), + (Format::AppJson, "app.json", r#"{"env": "DO_NOT_DISCLOSE"}"#), + ( + Format::AppJson, + "app.json", + r#"{"env": {"KEY": "DO_NOT_DISCLOSE""#, + ), + ( + Format::Render, + "render.yaml", + "services: [{envVars: [{value: DO_NOT_DISCLOSE}]}]", + ), + ( + Format::Render, + "render.yaml", + "services: [{envVars: {KEY: DO_NOT_DISCLOSE}}]", + ), + (Format::Fly, "fly.toml", "[env]\nKEY = 'DO_NOT_DISCLOSE"), + (Format::Fly, "fly.toml", "[env]\nKEY = ['DO_NOT_DISCLOSE']"), + ( + Format::Netlify, + "netlify.toml", + "[build]\nenvironment = 'DO_NOT_DISCLOSE'", + ), + ( + Format::Vercel, + "vercel.json", + r#"{"env": {"DO NOT DISCLOSE": "x"}}"#, + ), + ( + Format::Digitalocean, + ".do/app.yaml", + "envs: [{value: DO_NOT_DISCLOSE}]", + ), + ( + Format::AppEngine, + "app.yaml", + "runtime: go\nenv_variables: [DO_NOT_DISCLOSE]", + ), + ( + Format::Copilot, + "copilot/a/manifest.yml", + "type: Worker Service\nsecrets: {KEY: [DO_NOT_DISCLOSE]}", + ), + ( + Format::HerokuYml, + "heroku.yml", + "build: {config: {KEY: {nested: DO_NOT_DISCLOSE}}}", + ), + ] { + let (declarations, diagnostics) = adapter(format, source, text); + assert!( + !diagnostics.is_empty(), + "{format:?} {text}: {declarations:?}" + ); + let output = format!("{declarations:?}{diagnostics:?}"); + assert!(!output.contains("DO_NOT_DISCLOSE"), "{output}"); + } + let nested = format!("[env]\nKEY = {}1{}", "[".repeat(5000), "]".repeat(5000)); + let (declarations, diagnostics) = adapter(Format::Fly, "fly.toml", &nested); + assert!(declarations.is_empty()); + assert_eq!(diagnostics.len(), 1); +} + +#[test] +fn platform_routing_is_opt_in_deterministic_and_keeps_existing_formats() { + let files = [ + ("package.json", "{}"), + (".env", "EXISTING=DO_NOT_DISCLOSE"), + ("fly.toml", "[env]\nFLY=DO_NOT_DISCLOSE\n"), + ( + "render.yaml", + "services: [{envVars: [{key: RENDER, sync: false}]}]", + ), + ("app.json", r#"{"env": {"HEROKU": {"required": false}}}"#), + ( + "k8s/app.yaml", + "apiVersion: v1\nkind: Pod\nruntime: not-app-engine\nspec: {containers: [{env: [{name: KUBE, value: DO_NOT_DISCLOSE}]}]}", + ), + ]; + let dir = fixture(&files); + let default = scan(dir.path(), None).unwrap(); + assert_eq!(default.stats.files_read, 1); + assert!( + serde_json::to_value(&default) + .unwrap() + .get("environment") + .is_none() + ); + let report = scan_options(dir.path(), None, super::EnvScan::Declarations).unwrap(); + assert_eq!( + scan_options(dir.path(), None, super::EnvScan::Declarations).unwrap(), + report + ); + assert_eq!(report.stats.files_read, 6); + assert_eq!(default.projects, report.projects); + let kube = declarations(&report) + .iter() + .find(|d| d.name.as_deref() == Some("KUBE")) + .unwrap(); + assert_eq!(kube.format, Format::Kubernetes); + let json = serde_json::to_value(&report).unwrap(); + let existing = json["environment"] + .as_array() + .unwrap() + .iter() + .find(|d| d["name"] == "EXISTING") + .unwrap(); + let mut keys = existing + .as_object() + .unwrap() + .keys() + .cloned() + .collect::>(); + keys.sort(); + assert_eq!( + keys, + ["format", "location", "name", "project", "source", "state"] + ); + let render_record = json["environment"] + .as_array() + .unwrap() + .iter() + .find(|d| d["name"] == "RENDER") + .unwrap(); + assert_eq!(render_record["secret"], true); + assert_eq!(render_record["hasDefault"], false); + let text = render(&json); + let line = text + .iter() + .position(|line| line.starts_with("env RENDER\t")) + .unwrap(); + assert_eq!(text[line + 1], " secret: true, hasDefault: false"); + let existing = text + .iter() + .position(|line| line.starts_with("env EXISTING\t")) + .unwrap(); + assert!(text[existing + 1].starts_with("env ")); + assert_redacted(&report); +} diff --git a/src/detect/environment_reference_tests.rs b/src/detect/environment_reference_tests.rs new file mode 100644 index 0000000..5e01323 --- /dev/null +++ b/src/detect/environment_reference_tests.rs @@ -0,0 +1,489 @@ +use std::fs; + +use super::environment::{Declaration, Format, Origin, Scope, parse}; +use super::tests::fixture; +use super::{Report, render, scan, scan_options}; + +fn with_env(files: &[(&str, &str)]) -> Report { + let dir = fixture(files); + scan_options(dir.path(), None, super::EnvScan::Source).unwrap() +} + +fn references(report: &Report) -> Vec<&Declaration> { + report + .environment + .as_ref() + .unwrap() + .iter() + .filter(|d| d.format == Format::CodeReference) + .collect() +} + +fn names(report: &Report) -> Vec<(&str, &str)> { + references(report) + .iter() + .map(|d| { + ( + d.name.as_deref().unwrap(), + d.code.as_ref().unwrap().provider.as_str(), + ) + }) + .collect() +} + +fn find<'a>(report: &'a Report, name: &str) -> &'a Declaration { + references(report) + .into_iter() + .find(|d| d.name.as_deref() == Some(name)) + .unwrap_or_else(|| panic!("{name} missing: {report:#?}")) +} + +fn assert_redacted(report: &Report) { + let json = serde_json::to_string(report).unwrap(); + let text = render(&serde_json::to_value(report).unwrap()).join("\n"); + for output in [json, text] { + for forbidden in ["DO_NOT_DISCLOSE", "process.env", "import.meta"] { + assert!(!output.contains(forbidden), "{output}"); + } + } +} + +#[test] +fn dockerfile_args_are_build_scoped_names_with_default_presence_only() { + let mut diagnostics = Vec::new(); + let env = parse( + Format::Dockerfile, + "Dockerfile", + "ARG BASE=DO_NOT_DISCLOSE\nFROM node:${BASE}\nARG TOKEN NODE_ENV=DO_NOT_DISCLOSE\nENV PATH=/app\nARG\nARG \"BROKEN=DO_NOT_DISCLOSE\n", + &mut diagnostics, + ); + let args = env + .iter() + .filter(|d| d.semantics.scope == Some(Scope::Build)) + .map(|d| { + ( + d.name.as_deref().unwrap(), + d.location.as_str(), + d.semantics.has_default, + ) + }) + .collect::>(); + assert_eq!( + args, + [ + ("BASE", "stage:0/line:1/0", Some(true)), + ("TOKEN", "stage:1/line:3/0", Some(false)), + ("NODE_ENV", "stage:1/line:3/1", Some(true)), + ] + ); + assert_eq!(env.len(), 4); + assert_eq!(diagnostics.len(), 1, "{diagnostics:?}"); + assert!(!format!("{env:?}{diagnostics:?}").contains("DO_NOT_DISCLOSE")); +} + +#[test] +fn framework_prefixes_match_whole_uppercase_identifiers_per_project() { + let report = with_env(&[ + ( + "web/package.json", + r#"{"dependencies": {"next": "15.0.0"}}"#, + ), + ( + "web/app/page.tsx", + "const a = process.env.NEXT_PUBLIC_API_URL + \"DO_NOT_DISCLOSE\";\nconst b = process.env.NEXT_PUBLIC_;\nconst c = NEXT_PUBLIC_lower + MY_NEXT_PUBLIC_HIDDEN + NEXT_PUBLIC_Mixed;\n// NEXT_PUBLIC_API_URL again\n", + ), + ( + "web/lib/env.ts", + "export const url = process.env.NEXT_PUBLIC_API_URL;\n", + ), + ( + "spa/src/App.vue", + "\n", + ), + ("spa/package.json", "{}"), + ( + "scripts/run.py", + "os.environ['REACT_APP_TITLE']\nprint('$EXPO_PUBLIC_KEY')\n", + ), + ("docs/README.md", "NEXT_PUBLIC_DOCS_ONLY"), + ("web/node_modules/pkg/index.js", "NEXT_PUBLIC_VENDORED"), + ("web/dist/app.js", "NEXT_PUBLIC_BUILD_OUTPUT"), + ("web/public/app.min.js", "NEXT_PUBLIC_MINIFIED"), + ( + "nuxt/server/api.ts", + "useRuntimeConfig(); process.env.NUXT_PUBLIC_SITE_URL; NUXT_API_SECRET", + ), + ]); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let mut found = names(&report); + found.sort(); + assert_eq!( + found, + [ + ("EXPO_PUBLIC_KEY", "expo"), + ("NEXT_PUBLIC_API_URL", "nextjs"), + ("NUXT_API_SECRET", "nuxt"), + ("NUXT_PUBLIC_SITE_URL", "nuxt"), + ("REACT_APP_TITLE", "create-react-app"), + ("VITE_API_BASE", "vite"), + ] + ); + let next = find(&report, "NEXT_PUBLIC_API_URL"); + assert_eq!(next.project.as_deref(), Some("web")); + assert_eq!(next.source, "web/app/page.tsx"); + assert_eq!(next.location, "line:1"); + assert_eq!(next.state, "referenced"); + assert_eq!(next.semantics.scope, Some(Scope::Build)); + let code = next.code.as_ref().unwrap(); + assert_eq!(code.origin, Origin::Framework); + assert_eq!(code.occurrence_count, 3); + assert_eq!( + code.occurrences + .iter() + .map(|o| (o.source.as_str(), o.line)) + .collect::>(), + [ + ("web/app/page.tsx", 1), + ("web/app/page.tsx", 4), + ("web/lib/env.ts", 1) + ] + ); + assert_eq!( + find(&report, "NUXT_PUBLIC_SITE_URL").semantics.scope, + Some(Scope::Runtime) + ); + assert_eq!( + find(&report, "VITE_API_BASE").project.as_deref(), + Some("spa") + ); + assert_redacted(&report); + let json = serde_json::to_value(&report).unwrap(); + let record = json["environment"] + .as_array() + .unwrap() + .iter() + .find(|d| d["name"] == "NEXT_PUBLIC_API_URL") + .unwrap(); + assert_eq!(record["format"], "code-reference"); + assert_eq!(record["origin"], "framework"); + assert_eq!(record["provider"], "nextjs"); + assert_eq!(record["occurrenceCount"], 3); + let text = render(&json); + let line = text + .iter() + .position(|l| l.starts_with("env NEXT_PUBLIC_API_URL\t")) + .unwrap(); + assert_eq!( + text[line + 1], + " scope: build, origin: framework, provider: nextjs, occurrenceCount: 3" + ); +} + +#[test] +fn generic_prefixes_require_the_framework_and_generic_names_a_platform_file() { + let files = [ + ( + "astro/package.json", + r#"{"dependencies": {"astro": "5.0.0"}}"#, + ), + ( + "astro/src/pages/index.astro", + "import.meta.env.PUBLIC_SITE_KEY", + ), + ("plain/package.json", "{}"), + ( + "plain/index.js", + "const PUBLIC_KEY = crypto.PUBLIC_KEY_PEM;", + ), + ( + "edge/handler.ts", + "VERCEL_URL; VERCEL_NOT_DOCUMENTED; DEPLOY_URL; URL; CONTEXT; FLY_REGION; PRIMARY_REGION; RENDER_EXTERNAL_URL; RENDER; DYNO; CF_PAGES_URL; K_SERVICE", + ), + ]; + let report = with_env(&files); + let mut found = names(&report); + found.sort(); + assert_eq!( + found, + [ + ("CF_PAGES_URL", "cloudflare-pages"), + ("DYNO", "heroku"), + ("FLY_REGION", "fly"), + ("K_SERVICE", "cloud-run"), + ("PUBLIC_SITE_KEY", "astro"), + ("RENDER_EXTERNAL_URL", "render"), + ("VERCEL_URL", "vercel"), + ] + ); + let vercel = find(&report, "VERCEL_URL"); + assert_eq!(vercel.code.as_ref().unwrap().origin, Origin::Platform); + assert_eq!(vercel.semantics.scope, Some(Scope::BuildAndRuntime)); + let mut gated = files.to_vec(); + gated.extend([ + ("netlify.toml", "[build]\ncommand = 'make'\n"), + ("deploy/fly.toml", "app = 'x'\n"), + ("render.yaml", "services: []\n"), + ]); + let report = with_env(&gated); + let found = names(&report); + for expected in [ + ("DEPLOY_URL", "netlify"), + ("PRIMARY_REGION", "fly"), + ("RENDER", "render"), + ] { + assert!(found.contains(&expected), "{found:?}"); + } + for never in ["URL", "CONTEXT", "VERCEL_NOT_DOCUMENTED", "PUBLIC_KEY"] { + assert!(found.iter().all(|(name, _)| *name != never), "{found:?}"); + } +} + +#[test] +fn code_references_are_opt_in_capped_bounded_deterministic_and_redacted() { + // Two matches per line count once; `lib/` sorts first and lives in another directory. + let many = (0..25) + .map(|i| format!("v{i} = NEXT_PUBLIC_MANY + NEXT_PUBLIC_MANY // DO_NOT_DISCLOSE\n")) + .collect::(); + let dir = fixture(&[ + ("package.json", "{}"), + ("src/many.js", &many), + ("lib/first.js", "\nNEXT_PUBLIC_MANY"), + ("src/a.ts", "NEXT_PUBLIC_SHARED"), + ]); + fs::write( + dir.path().join("src/latin1.js"), + b"// caf\xe9 DO_NOT_DISCLOSE\nVITE_LATIN1_OK\n", + ) + .unwrap(); + fs::write( + dir.path().join("src/huge.js"), + format!( + "NEXT_PUBLIC_TOO_LARGE {}", + "x".repeat(super::MAX_FILE_BYTES as usize) + ), + ) + .unwrap(); + let default = scan(dir.path(), None).unwrap(); + assert_eq!(default.stats.files_read, 1); + assert!(default.diagnostics.is_empty()); + let report = scan_options(dir.path(), None, super::EnvScan::Source).unwrap(); + assert_eq!( + scan_options(dir.path(), None, super::EnvScan::Source).unwrap(), + report + ); + assert_eq!(report.stats.files_read, 6); + assert_eq!(default.projects, report.projects); + let record = find(&report, "NEXT_PUBLIC_MANY"); + assert_eq!( + (record.source.as_str(), record.location.as_str()), + ("lib/first.js", "line:2") + ); + let many = record.code.as_ref().unwrap(); + assert_eq!(many.occurrence_count, 26); + assert_eq!( + many.occurrences.len(), + super::environment::references::MAX_OCCURRENCES + ); + assert_eq!( + many.occurrences.last().unwrap(), + &super::environment::Occurrence { + source: "src/many.js".to_owned(), + line: 19, + } + ); + assert!( + references(&report) + .iter() + .any(|d| d.name.as_deref() == Some("VITE_LATIN1_OK")) + ); + assert!( + references(&report) + .iter() + .all(|d| d.name.as_deref() != Some("NEXT_PUBLIC_TOO_LARGE")) + ); + assert_eq!(report.diagnostics.len(), 1, "{report:#?}"); + assert_eq!(report.diagnostics[0].path, "src/huge.js"); + assert_redacted(&report); + + let rules = super::Rules::builtin().unwrap(); + for index in 0..80 { + let path = dir.path().join(format!("services/s{index}/src")); + fs::create_dir_all(&path).unwrap(); + fs::write(path.join("index.ts"), "NEXT_PUBLIC_SERVICE").unwrap(); + } + let serial = super::inventory(dir.path(), rules, 1, super::EnvScan::Source).unwrap(); + let parallel = super::inventory(dir.path(), rules, 4, super::EnvScan::Source).unwrap(); + assert_eq!(serial.stats, parallel.stats); + for (a, b) in serial.nodes.iter().zip(¶llel.nodes) { + assert_eq!(a.code_hits, b.code_hits); + } +} + +#[test] +fn code_reference_catalog_is_data_validated_and_extensible() { + let dir = fixture(&[("src/app.ts", "ACME_PUBLIC_TOKEN; NEXT_PUBLIC_STILL")]); + let extra = r#" +[[catalog.code_references.patterns]] +origin = "framework" +provider = "acme" +prefixes = ["ACME_PUBLIC_"] +scope = "build" +docs = ["https://example.com/acme#env"] +"#; + let report = scan_options(dir.path(), Some(extra), super::EnvScan::Source).unwrap(); + let mut found = names(&report); + found.sort(); + assert_eq!( + found, + [ + ("ACME_PUBLIC_TOKEN", "acme"), + ("NEXT_PUBLIC_STILL", "nextjs") + ] + ); + for invalid in [ + extra.replace("ACME_PUBLIC_", "acme_public_"), + extra.replace("https://example.com/acme#env", "not-a-url"), + extra.replace("origin = \"framework\"", "origin = \"guess\""), + format!("{extra}signals = [\"dir/file\"]\n"), + "[[catalog.environment]]\nformat = \"code-reference\"\nfiles = [\"*.ts\"]\n".to_owned(), + ] { + assert!( + scan_options(dir.path(), Some(&invalid), super::EnvScan::Source).is_err(), + "{invalid}" + ); + } +} + +#[test] +fn every_cataloged_provider_matches_a_documented_name() { + let cases = [ + ("NEXT_PUBLIC_A", "nextjs", Origin::Framework), + ("VITE_A", "vite", Origin::Framework), + ("REACT_APP_A", "create-react-app", Origin::Framework), + ("NUXT_PUBLIC_A", "nuxt", Origin::Framework), + ("EXPO_PUBLIC_A", "expo", Origin::Framework), + ("GATSBY_A", "gatsby", Origin::Framework), + ("VUE_APP_A", "vue-cli", Origin::Framework), + ("REDWOOD_ENV_A", "redwood", Origin::Framework), + ("STORYBOOK_A", "storybook", Origin::Framework), + ("SANITY_STUDIO_A", "sanity", Origin::Framework), + ("UMI_APP_A", "umi", Origin::Framework), + ("PLASMO_PUBLIC_A", "plasmo", Origin::Framework), + ("WXT_A", "wxt", Origin::Framework), + ("TAURI_ENV_A", "tauri", Origin::Framework), + ("QUASAR_A", "quasar", Origin::Framework), + ("RENDERER_VITE_A", "electron-vite", Origin::Framework), + ("VERCEL_GIT_COMMIT_SHA", "vercel", Origin::Platform), + ("DEPLOY_PRIME_URL", "netlify", Origin::Platform), + ("RENDER_GIT_COMMIT", "render", Origin::Platform), + ("RAILWAY_PUBLIC_DOMAIN", "railway", Origin::Platform), + ("HEROKU_APP_NAME", "heroku", Origin::Platform), + ("CF_PAGES_BRANCH", "cloudflare-pages", Origin::Platform), + ("GAE_SERVICE", "app-engine", Origin::Platform), + ("KOYEB_PUBLIC_DOMAIN", "koyeb", Origin::Platform), + ("K_REVISION", "cloud-run", Origin::Platform), + ("FLY_APP_NAME", "fly", Origin::Platform), + ]; + let source = cases + .iter() + .map(|(name, _, _)| format!("read({name});\n")) + .collect::(); + let report = with_env(&[("src/all.ts", &source)]); + let mut found = names(&report); + found.sort(); + let mut expected = cases + .iter() + .map(|(name, provider, _)| (*name, *provider)) + .collect::>(); + expected.sort(); + assert_eq!(found, expected); + for (name, provider, origin) in cases { + assert_eq!( + find(&report, name).code.as_ref().unwrap().origin, + origin, + "{provider}" + ); + } +} + +#[test] +fn with_env_alone_reads_no_source_files_and_emits_no_code_references() { + let dir = fixture(&[ + ("package.json", r#"{"dependencies": {"next": "15.0.0"}}"#), + (".env.example", "NEXT_PUBLIC_DECLARED=DO_NOT_DISCLOSE"), + ("netlify.toml", "[build.environment]\nNODE_VERSION = '22'\n"), + ( + "app/page.tsx", + "process.env.NEXT_PUBLIC_API_URL; VERCEL_URL", + ), + ("lib/util.py", "os.environ['REACT_APP_TITLE']"), + ( + "src/huge.js", + &"x".repeat(super::MAX_FILE_BYTES as usize + 10), + ), + ]); + let declarations = scan_options(dir.path(), None, super::EnvScan::Declarations).unwrap(); + assert_eq!(declarations.stats.files_read, 3); + assert!(declarations.diagnostics.is_empty(), "{declarations:#?}"); + let environment = declarations.environment.as_ref().unwrap(); + assert!( + environment + .iter() + .all(|d| d.format != Format::CodeReference) + ); + assert_eq!( + environment + .iter() + .filter_map(|d| d.name.as_deref()) + .collect::>(), + ["NEXT_PUBLIC_DECLARED", "NODE_VERSION"] + ); + let source = scan_options(dir.path(), None, super::EnvScan::Source).unwrap(); + assert_eq!(source.stats.files_read, 6); + assert_eq!(source.projects, declarations.projects); + let mut found = names(&source); + found.sort(); + assert_eq!( + found, + [ + ("NEXT_PUBLIC_API_URL", "nextjs"), + ("REACT_APP_TITLE", "create-react-app"), + ("VERCEL_URL", "vercel") + ] + ); + assert_redacted(&source); +} + +#[test] +fn with_source_requires_with_env() { + use clap::Parser; + let parse = |args: &[&str]| { + crate::Opts::try_parse_from(args).map(|opts| match opts.command { + crate::cmd::Command::Detect(args) => (args.with_env, args.with_source), + _ => panic!("not detect"), + }) + }; + assert_eq!(parse(&["brainpod", "detect"]).unwrap(), (false, false)); + assert_eq!( + parse(&["brainpod", "detect", "--with-env"]).unwrap(), + (true, false) + ); + assert_eq!( + parse(&[ + "brainpod", + "detect", + ".", + "--with-env", + "--with-source", + "--json" + ]) + .unwrap(), + (true, true) + ); + let error = parse(&["brainpod", "detect", "--with-source"]).unwrap_err(); + assert_eq!( + error.kind(), + clap::error::ErrorKind::MissingRequiredArgument + ); + assert!(error.to_string().contains("--with-env"), "{error}"); +} diff --git a/src/detect/environment_tests.rs b/src/detect/environment_tests.rs new file mode 100644 index 0000000..5e0d00a --- /dev/null +++ b/src/detect/environment_tests.rs @@ -0,0 +1,589 @@ +use std::fs; + +use super::environment::{Declaration, Format, parse}; +use super::tests::fixture; +use super::{Report, render, scan, scan_options}; + +fn with_env(files: &[(&str, &str)]) -> Report { + let dir = fixture(files); + scan_options(dir.path(), None, super::EnvScan::Declarations).unwrap() +} + +fn names(report: &Report) -> Vec<&str> { + report + .environment + .as_ref() + .unwrap() + .iter() + .filter_map(|d| d.name.as_deref()) + .collect() +} + +fn adapter(format: Format, text: &str) -> (Vec, Vec) { + let mut diagnostics = Vec::new(); + let declarations = parse(format, "metadata", text, &mut diagnostics); + (declarations, diagnostics) +} + +#[test] +fn dotenv_names_quotes_comments_multiline_duplicates_and_templates() { + let report = with_env(&[ + ( + ".env.example", + "\u{feff}# comment\n export ONE = secret # comment\nTWO='a#b'\nTHREE=\"a\\\"b\"\nMULTI=\"private\nFAKE=not-a-key\nend\"\nBACK=`text\nALSO_FAKE=no\nend`\nEMPTY=\nONE=second\n", + ), + ("config/test.env.template", "TEMPLATE=hidden\n"), + (".env.production.local", "PRODUCTION=hidden\n"), + (".env.enc", "ENCRYPTED=no\n"), + (".env.vault", "VAULT=no\n"), + ("source.ts", "process.env.SOURCE_INFERENCE_IS_FORBIDDEN"), + ]); + assert_eq!( + names(&report), + [ + "ONE", + "TWO", + "THREE", + "MULTI", + "BACK", + "EMPTY", + "ONE", + "PRODUCTION", + "TEMPLATE" + ] + ); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert_eq!(report.stats.files_read, 3); + assert!( + report + .environment + .as_ref() + .unwrap() + .iter() + .all(|d| d.project.as_deref() == Some(".")) + ); +} + +#[test] +fn malformed_inputs_never_echo_secret_values() { + for (format, text) in [ + ( + Format::Dotenv, + "BAD NAME=DO_NOT_DISCLOSE\nBROKEN='DO_NOT_DISCLOSE\nFAKE=DO_NOT_DISCLOSE", + ), + (Format::Dockerfile, "ENV KEY=\"DO_NOT_DISCLOSE\nENV KEY\n"), + ( + Format::Compose, + "services: {web: {environment: {KEY: [DO_NOT_DISCLOSE]}}}", + ), + ( + Format::Compose, + "services: {web: {environment: {KEY: DO_NOT_DISCLOSE, KEY: DO_NOT_DISCLOSE}}}", + ), + (Format::Kubernetes, "apiVersion: DO_NOT_DISCLOSE\nkind: ["), + ( + Format::GithubActions, + "env: {KEY: !unknown DO_NOT_DISCLOSE}", + ), + ] { + let (declarations, diagnostics) = adapter(format, text); + assert!(!diagnostics.is_empty(), "{format:?}: {declarations:?}"); + let output = format!("{declarations:?}{diagnostics:?}"); + assert!(!output.contains("DO_NOT_DISCLOSE"), "{output}"); + } +} + +#[test] +fn compose_maps_lists_external_values_and_unresolved_files() { + let report = with_env(&[( + "compose.override.yaml", + r#" +services: + api: + environment: + TOKEN: secret + EMPTY: '' + INHERITED: + env_file: + - ./private.env + - path: '${PATH:-DO_NOT_DISCLOSE}' + required: false + worker: + environment: + - TOKEN=other-secret + - INHERITED + - MULTILINE=first + - BAD NAME=hidden + labels: + DO_NOT_INFER: hidden +"#, + )]); + assert_eq!( + names(&report), + [ + "EMPTY", + "INHERITED", + "TOKEN", + "TOKEN", + "INHERITED", + "MULTILINE" + ] + ); + let env = report.environment.as_ref().unwrap(); + assert_eq!(env.iter().filter(|d| d.state == "external").count(), 2); + let references = env + .iter() + .filter_map(|d| d.reference.as_ref()) + .collect::>(); + assert_eq!(references.len(), 2); + assert_eq!(references[0].target.as_deref(), Some("./private.env")); + assert_eq!(references[1].target, None); + assert!( + !serde_json::to_string(&report) + .unwrap() + .contains("DO_NOT_DISCLOSE") + ); +} + +#[test] +fn docker_env_and_build_args_not_run_and_records_all_stages_not_effective_image() { + let (env, diagnostics) = adapter( + Format::Dockerfile, + "FROM alpine AS base\nARG BUILD_ONLY=hidden\nRUN echo ENV FAKE=hidden\nENV A=one B=\"two words\" \\\n C=three\\ words\nENV OLD legacy value D=not-a-declaration\nFROM base\nenv A=overridden\n", + ); + assert!(diagnostics.is_empty(), "{diagnostics:?}"); + assert_eq!( + env.iter() + .filter_map(|d| d.name.as_deref()) + .collect::>(), + ["BUILD_ONLY", "A", "B", "C", "OLD", "A"] + ); + assert!(env[0].location.starts_with("stage:1/")); + assert_eq!( + env[0].semantics.scope, + Some(super::environment::Scope::Build) + ); + assert_eq!(env[0].semantics.has_default, Some(true)); + assert_eq!(env[1].semantics, super::environment::Semantics::default()); + assert!(env[5].location.starts_with("stage:2/")); + let (env, diagnostics) = adapter( + Format::Dockerfile, + "FROM alpine\nRUN <> $GITHUB_ENV + - env: '${{ fromJSON(secrets.DO_NOT_DISCLOSE) }}' + with: {NOT_ENV: hidden} +"#; + let report = with_env(&[ + (".github/workflows/test.yml", yaml), + ("not-a-workflow.yml", yaml), + ]); + assert_eq!(names(&report), ["TOP", "JOB", "TOP", "STEP"]); + assert_eq!(report.environment.as_ref().unwrap().len(), 5); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let output = serde_json::to_string(&report).unwrap(); + assert!(!output.contains("DO_NOT_DISCLOSE")); + assert!(!output.contains("SOURCE_INFERENCE_FORBIDDEN")); +} + +#[test] +fn kubernetes_workloads_lists_multidoc_and_references_without_secret_data() { + let report = with_env(&[( + "deploy/app.yaml", + r#" +apiVersion: v1 +kind: Secret +metadata: {name: credentials} +stringData: {NEVER_INFER: DO_NOT_DISCLOSE} +--- +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + initContainers: + - name: init + env: [{name: INIT, value: hidden}] + containers: + - name: app + env: + - {name: TOKEN, valueFrom: {secretKeyRef: {name: credentials, key: password}}} + - {name: CONFIG, valueFrom: {configMapKeyRef: {name: config, key: data}}} + - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} + envFrom: + - secretRef: {name: credentials} + prefix: APP_ + - configMapRef: {name: config} +--- +apiVersion: v1 +kind: List +items: + - apiVersion: batch/v1 + kind: CronJob + spec: + jobTemplate: + spec: + template: + spec: + containers: + - env: [{name: CRON, value: hidden}] +"#, + )]); + assert_eq!( + names(&report), + ["TOKEN", "CONFIG", "POD_NAME", "INIT", "CRON"] + ); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let env = report.environment.as_ref().unwrap(); + assert_eq!(env.iter().filter(|d| d.name.is_none()).count(), 2); + let secret = env + .iter() + .find(|d| d.name.as_deref() == Some("TOKEN")) + .unwrap() + .reference + .as_ref() + .unwrap(); + assert_eq!(secret.target.as_deref(), Some("credentials")); + assert_eq!(secret.key.as_deref(), Some("password")); + assert!(env.iter().any(|d| { + d.reference + .as_ref() + .is_some_and(|r| r.prefix.as_deref() == Some("APP_")) + })); + assert!( + !serde_json::to_string(&report) + .unwrap() + .contains("DO_NOT_DISCLOSE") + ); +} + +#[test] +fn yaml_anchors_merge_keys_flow_and_block_scalars() { + let report = with_env(&[( + "compose.yaml", + r#" +x-env: &common + KEY: | + secret + FAKE=not-a-declaration +services: + api: + environment: + <<: *common + EXTRA: >- + multiline secret + worker: + environment: *common +"#, + )]); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert_eq!(names(&report), ["EXTRA", "KEY", "KEY"]); +} + +#[test] +fn nearest_project_owns_declarations_without_workspace_inheritance() { + let report = with_env(&[ + ("package.json", r#"{"workspaces":["apps/*"]}"#), + (".env", "ROOT=secret"), + ("apps/web/package.json", "{}"), + ("apps/web/.env", "LOCAL=secret"), + ("apps/web/config/.env", "CONFIG=secret"), + ("apps/api/deno.json", "{}"), + ("apps/api/.env.example", "LOCAL=secret"), + ("shared/.env", "SHARED=secret"), + ]); + let owners = report + .environment + .as_ref() + .unwrap() + .iter() + .map(|d| (d.name.as_deref().unwrap(), d.project.as_deref().unwrap())) + .collect::>(); + assert_eq!( + owners, + [ + ("ROOT", "."), + ("LOCAL", "apps/api"), + ("LOCAL", "apps/web"), + ("CONFIG", "apps/web"), + ("SHARED", ".") + ] + ); + let report = with_env(&[ + ("unowned/.env", "UNOWNED=secret"), + ("api/package.json", "{}"), + ]); + assert_eq!(report.environment.as_ref().unwrap()[0].project, None); +} + +#[test] +fn opt_in_preserves_default_reads_output_and_diagnostics() { + let dir = fixture(&[ + ("package.json", "{}"), + (".env", "INVALID SECRET"), + ("Dockerfile", "ENV MALFORMED"), + ("deployment.yaml", "[invalid YAML"), + ]); + let default = scan(dir.path(), None).unwrap(); + assert_eq!(default.stats.files_read, 1); + assert!(default.diagnostics.is_empty()); + let json = serde_json::to_value(&default).unwrap(); + assert!(json.get("environment").is_none()); + assert_eq!(render(&json), [".\tjavascript\tnode"]); + fs::remove_file(dir.path().join(".env")).unwrap(); + fs::remove_file(dir.path().join("Dockerfile")).unwrap(); + fs::remove_file(dir.path().join("deployment.yaml")).unwrap(); + let clean = scan(dir.path(), None).unwrap(); + assert_eq!(default.projects, clean.projects); + assert_eq!(default.stats.files_read, clean.stats.files_read); + assert_eq!(default.stats.bytes_read, clean.stats.bytes_read); +} + +#[test] +fn deterministic_json_text_and_no_evaluation_or_values() { + let dir = fixture(&[ + ("package.json", "{}"), + ( + ".env", + "NAME=$(touch SHOULD_NOT_EXIST)\nOTHER=${NAME:-DO_NOT_DISCLOSE}\nQUOTED='DO_NOT_DISCLOSE'", + ), + ( + ".github/workflows/ci.yml", + "env: {CI_NAME: '${{ secrets.DO_NOT_DISCLOSE }}'}", + ), + ]); + fs::write( + dir.path().join(".env"), + format!( + "NAME=$(touch {})\nOTHER=${{NAME:-DO_NOT_DISCLOSE}}\nQUOTED='DO_NOT_DISCLOSE'", + dir.path().join("SHOULD_NOT_EXIST").display() + ), + ) + .unwrap(); + let report = scan_options(dir.path(), None, super::EnvScan::Declarations).unwrap(); + assert_eq!( + scan_options(dir.path(), None, super::EnvScan::Declarations).unwrap(), + report + ); + let value = serde_json::to_value(&report).unwrap(); + let text = render(&value).join("\n"); + for output in [serde_json::to_string(&report).unwrap(), text] { + assert!(output.contains("NAME")); + for forbidden in [ + "DO_NOT_DISCLOSE", + "touch", + "SHOULD_NOT_EXIST", + "${", + "secrets.", + ] { + assert!(!output.contains(forbidden), "{output}"); + } + } + assert!(!dir.path().join("SHOULD_NOT_EXIST").exists()); +} + +#[test] +fn size_and_yaml_depth_alias_limits_are_bounded_and_redacted() { + let dir = fixture(&[( + ".env", + &format!("KEY={}", "s".repeat(super::MAX_FILE_BYTES as usize + 100)), + )]); + let report = scan_options(dir.path(), None, super::EnvScan::Declarations).unwrap(); + assert!(report.environment.as_ref().unwrap().is_empty()); + assert_eq!(report.stats.bytes_read, super::MAX_FILE_BYTES as usize + 1); + assert_eq!(report.diagnostics.len(), 1); + let deeply_nested = format!("{}secret{}", "[".repeat(60), "]".repeat(60)); + let (_, diagnostics) = adapter(Format::Kubernetes, &deeply_nested); + assert_eq!(diagnostics.len(), 1); + let aliases = format!("x: &x [secret]\ny: [{}]", vec!["*x"; 140].join(",")); + let (_, diagnostics) = adapter(Format::Compose, &aliases); + assert_eq!(diagnostics.len(), 1); +} + +#[cfg(unix)] +#[test] +fn no_symlinks_no_referenced_file_reads_no_root_escape() { + use std::os::unix::fs::symlink; + let outside = fixture(&[("private.env", "ESCAPED=DO_NOT_DISCLOSE")]); + let dir = fixture(&[( + "compose.yaml", + "services: {app: {env_file: [../../private.env, /private.env, '${PRIVATE:-DO_NOT_DISCLOSE}']}}", + )]); + symlink(outside.path().join("private.env"), dir.path().join(".env")).unwrap(); + symlink(outside.path(), dir.path().join("linked")).unwrap(); + let report = scan_options(dir.path(), None, super::EnvScan::Declarations).unwrap(); + assert!(names(&report).is_empty()); + assert_eq!(report.stats.files_read, 1); + assert!( + !serde_json::to_string(&report) + .unwrap() + .contains("DO_NOT_DISCLOSE") + ); +} + +#[test] +fn directory_depth_limit_applies_to_environment_metadata() { + let dir = fixture(&[(".env", "ROOT=secret")]); + let deep = (0..130).fold(dir.path().to_path_buf(), |path, _| path.join("d")); + fs::create_dir_all(&deep).unwrap(); + fs::write(deep.join(".env"), "TOO_DEEP=secret").unwrap(); + let report = scan_options(dir.path(), None, super::EnvScan::Declarations).unwrap(); + assert_eq!(names(&report), ["ROOT"]); + assert!( + report + .diagnostics + .iter() + .any(|d| d.message.contains("depth limit")) + ); +} + +#[test] +fn cli_flag_is_opt_in_and_accepts_paths_with_json() { + use clap::Parser; + for (args, expected) in [ + (vec!["brainpod", "detect"], false), + ( + vec!["brainpod", "detect", "some/project", "--with-env", "--json"], + true, + ), + ] { + let opts = crate::Opts::try_parse_from(args).unwrap(); + let crate::cmd::Command::Detect(args) = opts.command else { + panic!("not detect"); + }; + assert_eq!(args.with_env, expected); + } +} + +#[test] +fn declarative_environment_catalog_can_register_another_path_without_code() { + let dir = fixture(&[ + ("config/application.vars", "EXPLICIT=secret"), + ("elsewhere.vars", "NOT_SELECTED=secret"), + ]); + let extra = r#" +[[catalog.environment]] +format = "dotenv" +files = ["*.vars"] +paths = ["config/**"] +"#; + let report = scan_options(dir.path(), Some(extra), super::EnvScan::Declarations).unwrap(); + assert_eq!(names(&report), ["EXPLICIT"]); + assert_eq!(report.stats.files_read, 1); + assert_eq!( + scan_options(dir.path(), Some(extra), super::EnvScan::Off) + .unwrap() + .stats + .files_read, + 0 + ); + for invalid in [ + extra.replace("*.vars", "bad/*.vars"), + extra.replace("config/**", "../**"), + ] { + assert!(scan_options(dir.path(), Some(&invalid), super::EnvScan::Declarations).is_err()); + } +} + +#[test] +fn kubernetes_json_only_in_declared_metadata_paths_and_known_api_groups() { + let pod = r#"{"apiVersion":"v1","kind":"Pod","spec":{"ephemeralContainers":[{"env":[{"name":"DEBUG","value":"secret"}]}]}}"#; + let report = with_env(&[ + ("k8s/pod.json", pod), + ("app.k8s.json", pod), + ("data.json", pod), + ( + "manifests/not-kubernetes.yaml", + "apiVersion: example.com/v1\nkind: Deployment\nspec: {template: {spec: {containers: [{env: [{name: NOT_ENV}]}]}}}", + ), + ]); + assert_eq!(names(&report), ["DEBUG", "DEBUG"]); + assert_eq!(report.stats.files_read, 3); + assert!(report.diagnostics.is_empty()); +} + +#[test] +fn docker_continuations_skip_comments_without_inferring_nested_instructions() { + let (declarations, diagnostics) = adapter( + Format::Dockerfile, + "FROM alpine\nENV ONE=secret \\\n# comment\n\n TWO=secret\nRUN echo \\\n ENV FAKE=secret\n", + ); + assert!(diagnostics.is_empty(), "{diagnostics:?}"); + assert_eq!( + declarations + .iter() + .filter_map(|d| d.name.as_deref()) + .collect::>(), + ["ONE", "TWO"] + ); +} + +#[cfg(unix)] +#[test] +fn rejects_explicit_symlink_scan_root() { + let dir = fixture(&[("project/.env", "KEY=secret")]); + std::os::unix::fs::symlink(dir.path().join("project"), dir.path().join("link")).unwrap(); + assert!(scan_options(&dir.path().join("link"), None, super::EnvScan::Declarations).is_err()); +} + +#[test] +fn parallel_environment_inventory_is_identical_to_serial_and_retains_no_values() { + let dir = tempfile::tempdir().unwrap(); + for index in 0..128 { + let path = dir.path().join(format!("services/s{index}/config/deep")); + fs::create_dir_all(&path).unwrap(); + fs::write(path.join(".env"), "KEY=DO_NOT_DISCLOSE").unwrap(); + } + let rules = super::Rules::builtin().unwrap(); + let serial = super::inventory(dir.path(), rules, 1, super::EnvScan::Declarations).unwrap(); + let parallel = super::inventory(dir.path(), rules, 4, super::EnvScan::Declarations).unwrap(); + assert_eq!(serial.stats, parallel.stats); + assert_eq!(serial.nodes.len(), parallel.nodes.len()); + for (a, b) in serial.nodes.iter().zip(¶llel.nodes) { + assert_eq!(a.environment, b.environment); + assert!(a.contents.is_empty()); + assert!(!format!("{:?}", a.environment).contains("DO_NOT_DISCLOSE")); + } + assert_eq!(serial.stats.files_read, 128); +} + +#[test] +fn yaml_include_is_rejected_without_reading_its_target() { + let report = with_env(&[ + ( + ".github/workflows/test.yml", + "env: !include ../../private.data", + ), + ("private.data", "DO_NOT_DISCLOSE=DO_NOT_DISCLOSE"), + ]); + assert!(names(&report).is_empty()); + assert_eq!(report.stats.files_read, 1); + assert_eq!(report.diagnostics.len(), 1); + assert!( + !serde_json::to_string(&report) + .unwrap() + .contains("DO_NOT_DISCLOSE") + ); +} diff --git a/src/detect/mod.rs b/src/detect/mod.rs index 5f45bde..842d6be 100644 --- a/src/detect/mod.rs +++ b/src/detect/mod.rs @@ -14,6 +14,13 @@ mod deno; #[cfg(test)] mod deno_tests; mod dotnet; +mod environment; +#[cfg(test)] +mod environment_platform_tests; +#[cfg(test)] +mod environment_reference_tests; +#[cfg(test)] +mod environment_tests; mod gradle; mod java; mod jsonc; @@ -38,6 +45,33 @@ pub struct DetectArgs { /// Additional TOML detection rules; IDs must be unique #[arg(long, value_name = "PATH")] pub rules: Option, + /// Discover environment declarations and references without exposing values + #[arg(long)] + pub with_env: bool, + /// Also search application source files for cataloged platform/framework variable names + #[arg(long, requires = "with_env")] + pub with_source: bool, +} + +/// How much environment metadata a scan collects. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum EnvScan { + /// No environment adapters and no additional reads. + Off, + /// Manifest, config and deployment files only. + Declarations, + /// Declarations plus the `catalog.code_references` source search. + Source, +} + +impl EnvScan { + fn declarations(self) -> bool { + self != Self::Off + } + + fn source(self) -> bool { + self == Self::Source + } } #[derive(Debug, Serialize, PartialEq, Eq)] @@ -47,6 +81,8 @@ pub struct Report { pub projects: Vec, pub diagnostics: Vec, pub stats: Stats, + #[serde(skip_serializing_if = "Option::is_none")] + pub environment: Option>, } #[derive(Debug, Serialize, PartialEq, Eq)] @@ -134,6 +170,8 @@ pub struct Stats { #[derive(Default)] struct Node { path: PathBuf, + environment: Vec, + code_hits: environment::references::Hits, parent: Option, files: BTreeSet, contents: BTreeMap, @@ -231,7 +269,12 @@ pub fn handle(args: DetectArgs) -> Result { .map(fs::read_to_string) .transpose() .context("could not read detection rules")?; - let report = scan(&args.path, extra.as_deref())?; + let env = match (args.with_env, args.with_source) { + (false, _) => EnvScan::Off, + (true, false) => EnvScan::Declarations, + (true, true) => EnvScan::Source, + }; + let report = scan_options(&args.path, extra.as_deref(), env)?; Ok(crate::output::CommandOutput::new( serde_json::to_value(report)?, crate::output::View::Detect, @@ -243,6 +286,8 @@ struct Inventory { nodes: Vec, diagnostics: Vec, stats: Stats, + /// Code-reference signal basenames seen anywhere in the scan (`--with-env` only). + signals: BTreeSet, } impl Inventory { @@ -252,6 +297,7 @@ impl Inventory { path: &Path, depth: usize, rules: &Rules, + env: EnvScan, ) -> Result> { let mut node = Node::default(); self.stats.directories += 1; @@ -290,25 +336,60 @@ impl Inventory { .or_insert_with(|| name.clone()); } let readable = rules.catalog.readable(&name); - if readable { + let format = env + .declarations() + .then(|| rules.catalog.environment.matching(path, &name)) + .flatten(); + let code = env.source() && rules.catalog.references.scans(&name); + if env.source() && rules.catalog.references.is_signal(&name) { + self.signals.insert(name.clone()); + } + if readable || format.is_some() || code { let file = File::open(entry.path())?; let mut bytes = Vec::new(); file.take(MAX_FILE_BYTES + 1).read_to_end(&mut bytes)?; self.stats.files_read += 1; self.stats.bytes_read += bytes.len(); - let content = if bytes.len() > MAX_FILE_BYTES as usize { - Err("manifest exceeds 1 MiB read limit".to_owned()) + let oversized = bytes.len() > MAX_FILE_BYTES as usize; + if code && !oversized { + rules.catalog.references.find( + &relative_file(path, &name), + &bytes, + &mut node.code_hits, + ); + } + let content = if !(readable || format.is_some()) { + oversized.then(|| { + Err("source exceeds 1 MiB read limit; environment references not scanned" + .to_owned()) + }) + } else if oversized { + Some(Err("manifest exceeds 1 MiB read limit".to_owned())) } else { - String::from_utf8(bytes).map_err(|_| "manifest is not UTF-8".to_owned()) + Some( + String::from_utf8(bytes) + .map_err(|_| "manifest is not UTF-8".to_owned()), + ) }; match content { - Ok(content) => { - node.contents.insert(name.clone(), content); + Some(Ok(content)) => { + if let Some(format) = format { + node.environment.extend(environment::parse( + format, + &relative_file(path, &name), + &content, + &mut self.diagnostics, + )); + } + if readable { + node.contents.insert(name.clone(), content); + } } - Err(message) => self.diagnostics.push(Diagnostic { + Some(Err(message)) => self.diagnostics.push(Diagnostic { path: relative_file(path, &name), message, }), + None => (), } } if readable || rules.needs_filename(&name) { @@ -326,7 +407,11 @@ impl Inventory { } !prune }); - if !node.files.is_empty() || !node.sources.is_empty() { + if !node.files.is_empty() + || !node.sources.is_empty() + || !node.environment.is_empty() + || !node.code_hits.is_empty() + { node.path = path.to_path_buf(); self.nodes.push(node); } @@ -338,9 +423,10 @@ impl Inventory { root: &Path, rules: &Rules, mut pending: Vec<(PathBuf, usize)>, + env: EnvScan, ) -> Result<()> { while let Some((path, depth)) = pending.pop() { - pending.extend(self.read_directory(root, &path, depth, rules)?); + pending.extend(self.read_directory(root, &path, depth, rules, env)?); } Ok(()) } @@ -353,16 +439,17 @@ impl Inventory { self.stats.files_read += other.stats.files_read; self.stats.bytes_read += other.stats.bytes_read; self.stats.skipped_directories += other.stats.skipped_directories; + self.signals.extend(other.signals); } } -fn inventory(root: &Path, rules: &Rules, workers: usize) -> Result { +fn inventory(root: &Path, rules: &Rules, workers: usize, env: EnvScan) -> Result { let mut inventory = Inventory::default(); let mut pending = vec![(PathBuf::new(), 0)]; for _ in 0..3 { let mut next = Vec::new(); for (path, depth) in pending { - next.extend(inventory.read_directory(root, &path, depth, rules)?); + next.extend(inventory.read_directory(root, &path, depth, rules, env)?); } pending = next; if pending.is_empty() { @@ -377,7 +464,7 @@ fn inventory(root: &Path, rules: &Rules, workers: usize) -> Result { scope.spawn(|| -> Result { let mut inventory = Inventory::default(); while let Some(task) = pending.get(next.fetch_add(1, Ordering::Relaxed)) { - inventory.walk(root, rules, vec![task.clone()])?; + inventory.walk(root, rules, vec![task.clone()], env)?; } Ok(inventory) }) @@ -396,7 +483,7 @@ fn inventory(root: &Path, rules: &Rules, workers: usize) -> Result { inventory.merge(result); } } else { - inventory.walk(root, rules, pending)?; + inventory.walk(root, rules, pending, env)?; } index_parents(&mut inventory.nodes); Ok(inventory) @@ -417,7 +504,12 @@ fn index_parents(nodes: &mut [Node]) { } } +#[cfg(test)] pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { + scan_options(root, extra_rules, EnvScan::Off) +} + +fn scan_options(root: &Path, extra_rules: Option<&str>, env: EnvScan) -> Result { let custom = extra_rules .map(|extra| Rules::new(Some(extra))) .transpose()?; @@ -425,6 +517,9 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { Some(rules) => rules, None => Rules::builtin()?, }; + if env.declarations() && fs::symlink_metadata(root)?.file_type().is_symlink() { + bail!("environment scan root must not be a symlink"); + } let root = root .canonicalize() .with_context(|| format!("could not open {}", root.display()))?; @@ -436,13 +531,15 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { projects: Vec::new(), diagnostics: Vec::new(), stats: Stats::default(), + environment: env.declarations().then(Vec::new), }; let workers = thread::available_parallelism().map_or(1, |n| n.get().min(4)); let Inventory { mut nodes, diagnostics, stats, - } = inventory(&root, rules, workers)?; + signals, + } = inventory(&root, rules, workers, env)?; report.diagnostics = diagnostics; report.stats = stats; let original_nodes = nodes.len(); @@ -637,6 +734,42 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { project.workspace_members.sort(); project.workspace_members.dedup(); } + if let Some(environment) = &mut report.environment { + let mut code_hits = Vec::new(); + for (index, node) in nodes.iter().enumerate() { + let mut ancestor = Some(index); + let mut owner = None; + while let Some(i) = ancestor { + if let Some(project) = project_indices[i] { + owner = Some(project); + break; + } + ancestor = nodes[i].parent; + } + let path = owner.map(|project| projects[project].1.path.clone()); + environment.extend(node.environment.iter().cloned().map(|mut declaration| { + declaration.project = path.clone(); + declaration + })); + if !node.code_hits.is_empty() { + let detections = owner + .into_iter() + .flat_map(|project| &projects[project].1.detections) + .map(|detection| detection.id.clone()) + .collect::>(); + code_hits.push((path, detections, &node.code_hits)); + } + } + environment.extend( + rules.catalog.references.records( + code_hits + .iter() + .map(|(path, detections, hits)| (path.clone(), detections, *hits)), + &signals, + ), + ); + environment.sort_by(|a, b| a.source.cmp(&b.source)); + } projects.sort_by(|a, b| nodes[a.0].path.cmp(&nodes[b.0].path)); report.projects = projects.into_iter().map(|(_, project)| project).collect(); report @@ -704,6 +837,52 @@ pub fn render(value: &serde_json::Value) -> Vec { } } } + for declaration in value["environment"].as_array().into_iter().flatten() { + lines.push(format!( + "env {}\t{} ({}, {}, {}, project: {})", + declaration["name"] + .as_str() + .unwrap_or(""), + declaration["source"].as_str().unwrap_or(""), + declaration["format"].as_str().unwrap_or(""), + declaration["location"].as_str().unwrap_or(""), + declaration["state"].as_str().unwrap_or(""), + declaration["project"].as_str().unwrap_or("unowned") + )); + if let Some(reference) = declaration.get("reference") { + lines.push(format!( + " reference {} (target: {}, key: {}, prefix: {})", + reference["kind"].as_str().unwrap_or(""), + reference["target"].as_str().unwrap_or("unresolved"), + reference["key"].as_str().unwrap_or("-"), + reference["prefix"].as_str().unwrap_or("-") + )); + } + let facts = [ + "required", + "secret", + "generated", + "hasDefault", + "scope", + "origin", + "provider", + "occurrenceCount", + ] + .into_iter() + .filter_map(|key| { + let value = declaration.get(key)?; + Some(format!( + "{key}: {}", + value + .as_str() + .map_or_else(|| value.to_string(), str::to_owned) + )) + }) + .collect::>(); + if !facts.is_empty() { + lines.push(format!(" {}", facts.join(", "))); + } + } for diagnostic in value["diagnostics"].as_array().into_iter().flatten() { lines.push(format!( "warning: {}: {}", diff --git a/src/detect/rules.toml b/src/detect/rules.toml index e95d768..752454c 100644 --- a/src/detect/rules.toml +++ b/src/detect/rules.toml @@ -999,3 +999,349 @@ language = "typescript" kind = "framework" priority = 100 any = ["dep:npm:hono", "identity:npm:hono", "dep:jsr:@hono/hono", "identity:jsr:@hono/hono", "dep:deno:hono"] + +[[catalog.environment]] +format = "dotenv" +files = [".env", ".env.*", "*.env", "*.env.example", "*.env.sample", "*.env.template"] +exclude = ["*.enc", ".env.vault"] + +[[catalog.environment]] +format = "compose" +files = ["compose.yml", "compose.yaml", "compose.*.yml", "compose.*.yaml", "docker-compose.yml", "docker-compose.yaml", "docker-compose.*.yml", "docker-compose.*.yaml"] + +[[catalog.environment]] +format = "dockerfile" +files = ["Dockerfile", "Dockerfile.*", "*.Dockerfile"] + +[[catalog.environment]] +format = "github-actions" +files = ["*.yml", "*.yaml"] +paths = ["**/.github/workflows/*.yml", "**/.github/workflows/*.yaml"] + +[[catalog.environment]] +format = "kubernetes" +files = ["*.yml", "*.yaml", "*.json"] +paths = ["**/k8s/**", "**/kubernetes/**", "**/kube/**", "**/manifests/**", "**/deploy/**", "**/deployment/**", "**/deployments/**"] + +[[catalog.environment]] +format = "kubernetes" +files = ["pod.yaml", "pod.yml", "deployment.yaml", "deployment.yml", "*.k8s.yaml", "*.k8s.yml", "*.kube.yaml", "*.kube.yml", "kubernetes-*.yaml", "kubernetes-*.yml"] + +[[catalog.environment]] +format = "kubernetes" +files = ["*.k8s.json", "*.kube.json"] + +# Platform deployment manifests. Listed after the generic entries so a file in a +# Kubernetes metadata directory keeps its Kubernetes routing. +[[catalog.environment]] +format = "app-json" +files = ["app.json", "scalingo.json"] + +[[catalog.environment]] +format = "heroku-yml" +files = ["heroku.yml"] + +[[catalog.environment]] +format = "render" +files = ["render.yaml"] + +[[catalog.environment]] +format = "fly" +files = ["fly.toml", "fly.*.toml"] + +[[catalog.environment]] +format = "digitalocean" +files = ["app.yaml", "deploy.template.yaml"] +paths = ["**/.do/app.yaml", "**/.do/deploy.template.yaml"] + +[[catalog.environment]] +format = "vercel" +files = ["vercel.json", "vercel.toml"] + +[[catalog.environment]] +format = "netlify" +files = ["netlify.toml"] + +[[catalog.environment]] +format = "app-engine" +files = ["app.yaml"] + +[[catalog.environment]] +format = "copilot" +files = ["manifest.yml", "manifest.yaml"] +paths = ["**/copilot/*/manifest.yml", "**/copilot/*/manifest.yaml"] + +# Literal code references to environment names that platforms inject or +# frameworks expose by documented convention (`--with-env` only). Patterns match +# whole identifier tokens: exact `names`, or `prefixes` followed by [A-Z0-9_]+. +# `signals` gate generic names on a platform file existing anywhere in the scan; +# `requires` gates on a detection id of the owning project. First passing +# candidate wins. `docs` cites the primary source for each list. +[catalog.code_references] +extensions = ["js", "jsx", "mjs", "cjs", "ts", "tsx", "mts", "cts", "vue", "svelte", "astro", "py", "rb", "go", "rs", "php", "java", "kt", "kts", "cs", "ex", "exs"] +exclude = ["*.min.js", "*.min.mjs", "*.min.cjs", "*.bundle.js"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "nextjs" +prefixes = ["NEXT_PUBLIC_"] +scope = "build" +docs = ["https://nextjs.org/docs/app/guides/environment-variables#bundling-environment-variables-for-the-browser"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "vite" +prefixes = ["VITE_"] +scope = "build" +docs = ["https://vite.dev/guide/env-and-mode#env-variables", "https://docs.solidjs.com/configuration/environment-variables#public-environment-variables", "https://tanstack.com/start/latest/docs/framework/react/guide/environment-variables"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "create-react-app" +prefixes = ["REACT_APP_"] +scope = "build" +docs = ["https://create-react-app.dev/docs/adding-custom-environment-variables/"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "nuxt" +prefixes = ["NUXT_PUBLIC_", "NUXT_"] +scope = "runtime" +docs = ["https://nuxt.com/docs/4.x/guide/going-further/runtime-config#environment-variables"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "sveltekit" +prefixes = ["PUBLIC_"] +requires = ["sveltekit"] +docs = ["https://svelte.dev/docs/kit/$env-static-public", "https://svelte.dev/docs/kit/$env-dynamic-public"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "astro" +prefixes = ["PUBLIC_"] +scope = "build" +requires = ["astro"] +docs = ["https://docs.astro.build/en/guides/environment-variables/#vites-built-in-support"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "expo" +prefixes = ["EXPO_PUBLIC_"] +scope = "build" +docs = ["https://docs.expo.dev/guides/environment-variables/#reading-environment-variables-from-env-files"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "gatsby" +prefixes = ["GATSBY_"] +scope = "build" +docs = ["https://www.gatsbyjs.com/docs/how-to/local-development/environment-variables/#accessing-environment-variables-in-the-browser"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "vue-cli" +prefixes = ["VUE_APP_"] +scope = "build" +docs = ["https://cli.vuejs.org/guide/mode-and-env.html#environment-variables"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "redwood" +prefixes = ["REDWOOD_ENV_"] +scope = "build" +docs = ["https://docs.redwoodjs.com/docs/environment-variables#option-2-prefixing-with-redwood_env_"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "storybook" +prefixes = ["STORYBOOK_"] +scope = "build" +docs = ["https://storybook.js.org/docs/configure/environment-variables"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "sanity" +prefixes = ["SANITY_STUDIO_"] +scope = "build" +docs = ["https://www.sanity.io/docs/studio/environment-variables#exposed-variables"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "umi" +prefixes = ["UMI_APP_"] +scope = "build" +docs = ["https://umijs.org/en-US/docs/guides/env-variables#use-environment-variables-in-browser"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "plasmo" +prefixes = ["PLASMO_PUBLIC_"] +scope = "build" +docs = ["https://docs.plasmo.com/framework/env#custom-environment-variables"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "wxt" +prefixes = ["WXT_"] +scope = "build" +docs = ["https://wxt.dev/guide/essentials/config/environment-variables#dotenv-files"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "tauri" +prefixes = ["TAURI_ENV_"] +scope = "build" +docs = ["https://v2.tauri.app/reference/environment-variables/#tauri-cli-hook-commands"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "quasar" +prefixes = ["QCLI_", "QUASAR_"] +scope = "build" +docs = ["https://quasar.dev/quasar-cli-vite/handling-import-meta-env#from-dotenv-files", "https://quasar.dev/quasar-cli-vite/handling-import-meta-env#values-provided-by-quasar-cli"] + +[[catalog.code_references.patterns]] +origin = "framework" +provider = "electron-vite" +prefixes = ["MAIN_VITE_", "PRELOAD_VITE_", "RENDERER_VITE_"] +scope = "build" +docs = ["https://electron-vite.org/guide/env-and-mode#global-env-variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "vercel" +names = ["VERCEL_OIDC_TOKEN", "VERCEL_HASH_SALT", "VERCEL_GIT_PREVIOUS_SHA"] +scope = "build" +docs = ["https://vercel.com/docs/environment-variables/system-environment-variables#system-environment-variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "vercel" +names = ["VERCEL", "VERCEL_ENV", "VERCEL_TARGET_ENV", "VERCEL_URL", "VERCEL_BRANCH_URL", "VERCEL_PROJECT_PRODUCTION_URL", "VERCEL_DEPLOYMENT_ID", "VERCEL_PROJECT_ID", "VERCEL_PROJECT_NAME", "VERCEL_SKEW_PROTECTION_ENABLED", "VERCEL_AUTOMATION_BYPASS_SECRET", "VERCEL_GIT_PROVIDER", "VERCEL_GIT_REPO_SLUG", "VERCEL_GIT_REPO_OWNER", "VERCEL_GIT_REPO_ID", "VERCEL_GIT_COMMIT_REF", "VERCEL_GIT_COMMIT_SHA", "VERCEL_GIT_COMMIT_MESSAGE", "VERCEL_GIT_COMMIT_AUTHOR_LOGIN", "VERCEL_GIT_COMMIT_AUTHOR_NAME", "VERCEL_GIT_PULL_REQUEST_ID"] +scope = "build-and-runtime" +docs = ["https://vercel.com/docs/environment-variables/system-environment-variables#system-environment-variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "vercel" +names = ["VERCEL_REGION"] +scope = "runtime" +docs = ["https://vercel.com/docs/environment-variables/system-environment-variables#system-environment-variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "netlify" +names = ["NETLIFY", "NETLIFY_PREVIEW_SERVER", "CACHED_COMMIT_REF", "DEPLOY_PRIME_URL", "INCOMING_HOOK_TITLE", "INCOMING_HOOK_URL", "INCOMING_HOOK_BODY"] +scope = "build" +docs = ["https://docs.netlify.com/build/configure-builds/environment-variables/#read-only-variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "render" +names = ["RENDER_CPU_COUNT", "RENDER_DISCOVERY_SERVICE", "RENDER_EXTERNAL_HOSTNAME", "RENDER_EXTERNAL_URL", "RENDER_GIT_BRANCH", "RENDER_GIT_COMMIT", "RENDER_GIT_REPO_SLUG", "RENDER_INSTANCE_ID", "RENDER_SERVICE_ID", "RENDER_SERVICE_NAME", "RENDER_SERVICE_TYPE"] +scope = "build-and-runtime" +docs = ["https://render.com/docs/environment-variables#all-runtimes"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "render" +names = ["RENDER_WEB_CONCURRENCY"] +scope = "runtime" +docs = ["https://render.com/docs/environment-variables#all-runtimes"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "railway" +names = ["RAILWAY_PUBLIC_DOMAIN", "RAILWAY_PRIVATE_DOMAIN", "RAILWAY_TCP_PROXY_DOMAIN", "RAILWAY_TCP_PROXY_PORT", "RAILWAY_TCP_APPLICATION_PORT", "RAILWAY_PROJECT_NAME", "RAILWAY_PROJECT_ID", "RAILWAY_ENVIRONMENT_NAME", "RAILWAY_ENVIRONMENT_ID", "RAILWAY_SERVICE_NAME", "RAILWAY_SERVICE_ID", "RAILWAY_REPLICA_ID", "RAILWAY_REPLICA_REGION", "RAILWAY_DEPLOYMENT_ID", "RAILWAY_SNAPSHOT_ID", "RAILWAY_VOLUME_NAME", "RAILWAY_VOLUME_MOUNT_PATH", "RAILWAY_GIT_COMMIT_SHA", "RAILWAY_GIT_AUTHOR", "RAILWAY_GIT_BRANCH", "RAILWAY_GIT_REPO_NAME", "RAILWAY_GIT_REPO_OWNER", "RAILWAY_GIT_COMMIT_MESSAGE"] +scope = "build-and-runtime" +docs = ["https://docs.railway.com/variables/reference#railway-provided-variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "heroku" +names = ["DYNO", "HEROKU_APP_ID", "HEROKU_APP_NAME", "HEROKU_APP_DEFAULT_DOMAIN_NAME", "HEROKU_DYNO_ID", "HEROKU_RELEASE_CREATED_AT", "HEROKU_RELEASE_VERSION", "HEROKU_SLUG_COMMIT", "HEROKU_SLUG_DESCRIPTION", "HEROKU_BUILD_COMMIT", "HEROKU_BUILD_DESCRIPTION"] +scope = "runtime" +docs = ["https://devcenter.heroku.com/articles/dyno-metadata#dyno-metadata"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "cloudflare-pages" +names = ["CF_PAGES", "CF_PAGES_COMMIT_SHA", "CF_PAGES_BRANCH", "CF_PAGES_URL"] +scope = "build" +docs = ["https://developers.cloudflare.com/pages/configuration/build-configuration/#environment-variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "app-engine" +names = ["GAE_APPLICATION", "GAE_DEPLOYMENT_ID", "GAE_ENV", "GAE_INSTANCE", "GAE_MEMORY_MB", "GAE_RUNTIME", "GAE_SERVICE", "GAE_VERSION"] +scope = "runtime" +docs = ["https://cloud.google.com/appengine/docs/standard/nodejs/runtime#environment_variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "koyeb" +names = ["KOYEB_APP_NAME", "KOYEB_APP_ID", "KOYEB_ORGANIZATION_NAME", "KOYEB_ORGANIZATION_ID", "KOYEB_SERVICE_NAME", "KOYEB_SERVICE_ID", "KOYEB_SERVICE_PRIVATE_DOMAIN", "KOYEB_PUBLIC_DOMAIN", "KOYEB_PRIVILEGED", "KOYEB_DOCKER_REF", "KOYEB_GIT_SHA", "KOYEB_GIT_BRANCH", "KOYEB_GIT_REPOSITORY", "KOYEB_GIT_COMMIT_MESSAGE", "KOYEB_GIT_COMMIT_AUTHOR"] +scope = "build-and-runtime" +docs = ["https://www.koyeb.com/docs/build-and-deploy/environment-variables#koyeb-provided-variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "koyeb" +names = ["KOYEB_REGION", "KOYEB_REGIONAL_DEPLOYMENT_ID", "KOYEB_INSTANCE_ID", "KOYEB_INSTANCE_TYPE", "KOYEB_INSTANCE_MEMORY_MB", "KOYEB_REPLICA_INDEX", "KOYEB_HYPERVISOR_ID", "KOYEB_DC", "KOYEB_GPU_COUNT"] +scope = "runtime" +docs = ["https://www.koyeb.com/docs/build-and-deploy/environment-variables#koyeb-provided-variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "cloud-run" +names = ["K_SERVICE", "K_REVISION", "K_CONFIGURATION", "CLOUD_RUN_JOB", "CLOUD_RUN_EXECUTION", "CLOUD_RUN_TASK_INDEX", "CLOUD_RUN_TASK_ATTEMPT", "CLOUD_RUN_TASK_COUNT", "CLOUD_RUN_WORKER_POOL", "CLOUD_RUN_REVISION", "FUNCTION_TARGET", "FUNCTION_SIGNATURE_TYPE"] +scope = "runtime" +docs = ["https://cloud.google.com/run/docs/container-contract#env-vars"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "fly" +names = ["FLY_APP_NAME", "FLY_MACHINE_ID", "FLY_ALLOC_ID", "FLY_REGION", "FLY_PUBLIC_IP", "FLY_IMAGE_REF", "FLY_MACHINE_VERSION", "FLY_PRIVATE_IP", "FLY_PROCESS_GROUP", "FLY_VM_MEMORY_MB", "FLY_TEST_MACHINE_IP"] +scope = "runtime" +docs = ["https://fly.io/docs/machines/runtime-environment/#environment-variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "netlify" +names = ["BUILD_ID", "REPOSITORY_URL", "COMMIT_REF", "PULL_REQUEST", "REVIEW_ID", "DEPLOY_URL", "DEPLOY_ID"] +scope = "build" +signals = ["netlify.toml"] +docs = ["https://docs.netlify.com/build/configure-builds/environment-variables/#read-only-variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "netlify" +names = ["SITE_NAME", "SITE_ID"] +scope = "build-and-runtime" +signals = ["netlify.toml"] +docs = ["https://docs.netlify.com/build/configure-builds/environment-variables/#read-only-variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "render" +names = ["RENDER", "IS_PULL_REQUEST"] +scope = "build-and-runtime" +signals = ["render.yaml"] +docs = ["https://render.com/docs/environment-variables#all-runtimes"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "fly" +names = ["PRIMARY_REGION", "RELEASE_COMMAND"] +scope = "runtime" +signals = ["fly.toml"] +docs = ["https://fly.io/docs/machines/runtime-environment/#environment-variables"] + +[[catalog.code_references.patterns]] +origin = "platform" +provider = "app-engine" +names = ["GOOGLE_CLOUD_PROJECT"] +scope = "runtime" +signals = ["app.yaml"] +docs = ["https://cloud.google.com/appengine/docs/standard/nodejs/runtime#environment_variables"] diff --git a/src/detect/tests.rs b/src/detect/tests.rs index b509fd2..c10531c 100644 --- a/src/detect/tests.rs +++ b/src/detect/tests.rs @@ -962,8 +962,8 @@ fn parallel_inventory_matches_serial_inventory() { fs::write(dir.path().join(format!("services/s{i}/package.json")), "{}").unwrap(); } let rules = super::Rules::builtin().unwrap(); - let serial = super::inventory(dir.path(), rules, 1).unwrap(); - let parallel = super::inventory(dir.path(), rules, 4).unwrap(); + let serial = super::inventory(dir.path(), rules, 1, super::EnvScan::Off).unwrap(); + let parallel = super::inventory(dir.path(), rules, 4, super::EnvScan::Off).unwrap(); assert_eq!(serial.stats, parallel.stats); assert_eq!(serial.nodes.len(), parallel.nodes.len()); for (a, b) in serial.nodes.iter().zip(¶llel.nodes) { diff --git a/src/detect/xml.rs b/src/detect/xml.rs index c91f42e..731e05a 100644 --- a/src/detect/xml.rs +++ b/src/detect/xml.rs @@ -181,7 +181,7 @@ mod tests { use anyhow::{Result, ensure}; use super::read; - use crate::detect::{MAX_FILE_BYTES, inventory, rules::Rules}; + use crate::detect::{EnvScan, MAX_FILE_BYTES, inventory, rules::Rules}; fn kind(name: &str) -> Option<&'static str> { if name == "pom.xml" { @@ -194,7 +194,7 @@ mod tests { } fn files(root: &Path) -> Result> { - let inventory = inventory(root, Rules::builtin()?, 1)?; + let inventory = inventory(root, Rules::builtin()?, 1, EnvScan::Off)?; for diagnostic in &inventory.diagnostics { eprintln!("inventory: {}: {}", diagnostic.path, diagnostic.message); } diff --git a/tests/audit_detect.py b/tests/audit_detect.py index c321cc3..70ad211 100644 --- a/tests/audit_detect.py +++ b/tests/audit_detect.py @@ -86,7 +86,19 @@ def snapshot(entry, cache, fetch): return destination -def check(projects, expected): +def check(projects, expected, environment=None): + if expected["kind"] == "environment": + fields = { + key: value + for key, value in expected.items() + if key not in {"kind", "category", "absent"} + } + actual = [ + declaration + for declaration in environment or [] + if all(declaration.get(key) == value for key, value in fields.items()) + ] + return (not actual if expected.get("absent") else bool(actual)), actual project = projects.get(expected["path"]) if project is None: return False, "project missing" @@ -176,14 +188,34 @@ def check(projects, expected): raise ValueError(f"unknown check kind: {kind}") +def detect_command(args, path): + command = [str(args.binary), "detect", str(path), "--json"] + if args.with_env: + command.append("--with-env") + if args.with_source: + command.append("--with-source") + return command + + +def expected_checks(entry, args): + """Declaration checks run in both opt-in modes; code-reference + `sourceChecks` only when application source is searched.""" + checks = list(entry["checks"]) + if args.with_env: + checks.extend(entry.get("environmentChecks", [])) + if args.with_source: + checks.extend(entry.get("sourceChecks", [])) + return checks + + def audit(entry, args): - path = snapshot(entry, args.cache, args.fetch) + path = snapshot(entry, args.cache, args.fetch).resolve(strict=True) samples, first = [], None deterministic = True for _ in range(args.runs): start = time.perf_counter() process = subprocess.run( - [str(args.binary), "detect", str(path), "--json"], + detect_command(args, path), capture_output=True, timeout=90, check=False, @@ -200,8 +232,8 @@ def audit(entry, args): raise ValueError("unexpected detector schema version") projects = {project["path"]: project for project in report["projects"]} checks = [] - for expected in entry["checks"]: - passed, actual = check(projects, expected) + for expected in expected_checks(entry, args): + passed, actual = check(projects, expected, report.get("environment")) checks.append({"expected": expected, "passed": passed, "actual": actual}) return { "repository": entry["repository"], @@ -214,6 +246,7 @@ def audit(entry, args): "stats": report["stats"], "projectCount": len(projects), "diagnostics": report["diagnostics"], + **({"environmentCount": len(report["environment"])} if args.with_env else {}), "checks": checks, } @@ -233,6 +266,16 @@ def main(): parser.add_argument( "--cache", type=Path, help="Snapshot cache; required unless --list" ) + parser.add_argument( + "--with-env", + action="store_true", + help="Include name-only environment declarations in each scan", + ) + parser.add_argument( + "--with-source", + action="store_true", + help="Also search application source for cataloged code references (requires --with-env)", + ) parser.add_argument( "--fetch", action="store_true", @@ -262,6 +305,8 @@ def main(): return 0 if args.cache is None or args.output is None: parser.error("--cache and --output are required unless --list") + if args.with_source and not args.with_env: + parser.error("--with-source requires --with-env") if args.runs < 2: parser.error("--runs must be at least 2") args.binary = args.binary.resolve(strict=True) diff --git a/tests/detect-corpus.json b/tests/detect-corpus.json index ff59a6a..b72f1ae 100644 --- a/tests/detect-corpus.json +++ b/tests/detect-corpus.json @@ -81,6 +81,20 @@ "value": "1.27.0", "source": "go.mod" } + ], + "environmentChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "netlify", + "source": "docs/netlify.toml", + "name": "HUGO_VERSION", + "project": "docs", + "state": "declared", + "hasDefault": true, + "scope": "build", + "location": "document:0/build/environment/HUGO_VERSION" + } ] }, { @@ -301,6 +315,23 @@ "id": "node-workspace", "member": "packages/next" } + ], + "environmentChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "vercel", + "source": "examples/with-firebase/vercel.json", + "name": "FIREBASE_API_KEY", + "project": "examples/with-firebase", + "state": "unresolved", + "secret": true, + "scope": "build", + "reference": { + "kind": "platform-secret", + "target": "firebase-api-key" + } + } ] }, { @@ -456,6 +487,57 @@ "path": ".", "id": "laravel" } + ], + "environmentChecks": [ + { + "format": "dotenv", + "name": "APP_ENV", + "project": ".", + "source": ".env.development.example", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "compose", + "name": "APP_DEBUG", + "project": ".", + "source": "docker-compose.dev-multi.yml", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "dockerfile", + "name": "COOLIFY_VERSION", + "project": ".", + "source": "docker/production/Dockerfile", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "kind": "environment", + "category": "environment", + "format": "dockerfile", + "name": "USER_ID", + "source": "docker/development/Dockerfile", + "project": ".", + "state": "declared", + "scope": "build", + "hasDefault": false + }, + { + "kind": "environment", + "category": "environment", + "format": "dockerfile", + "name": "NGINX_VERSION", + "source": "docker/development/Dockerfile", + "project": ".", + "state": "declared", + "scope": "build", + "hasDefault": true + } ] }, { @@ -654,6 +736,44 @@ "path": "autogpt_platform/frontend", "id": "nextjs" } + ], + "environmentChecks": [ + { + "format": "dotenv", + "name": "POSTGRES_HOST", + "project": "autogpt_platform", + "source": "autogpt_platform/.env.default", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "compose", + "name": "CLAMAV_NO_FRESHCLAMD", + "project": "autogpt_platform/backend", + "source": "autogpt_platform/backend/docker-compose.test.yaml", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "dockerfile", + "name": "CGO_ENABLED", + "project": null, + "source": ".github/actions/start-minio/Dockerfile", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "github-actions", + "name": "ARG1", + "project": null, + "source": ".github/workflows/batch-command-handler.yml", + "state": "declared", + "kind": "environment", + "category": "environment" + } ] }, { @@ -758,6 +878,44 @@ "value": "npm:@typescript/typescript6@6.0.2", "source": "pnpm-workspace.yaml" } + ], + "environmentChecks": [ + { + "format": "dotenv", + "name": "SECRET_KEY", + "project": "api", + "source": "api/.env.example", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "dotenv", + "name": "NEXT_PUBLIC_API_PREFIX", + "project": "web", + "source": "web/.env.example", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "compose", + "name": "MYSQL_DATABASE", + "project": "docker", + "source": "docker/docker-compose.middleware.yaml", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "dockerfile", + "name": "FLASK_APP", + "project": "api", + "source": "api/Dockerfile", + "state": "declared", + "kind": "environment", + "category": "environment" + } ] }, { @@ -806,6 +964,32 @@ "path": "src/backend/base", "id": "starlette" } + ], + "environmentChecks": [ + { + "format": "kubernetes", + "name": "LANGFLOW_DATABASE_URL", + "project": ".", + "reference": { + "key": "database-url", + "kind": "secret-key", + "target": "langflow-secrets" + }, + "source": "deploy/listeners/kubernetes-deployment.yaml", + "state": "unresolved", + "kind": "environment", + "category": "environment" + }, + { + "kind": "environment", + "category": "environment", + "format": "render", + "source": "render.yaml", + "name": "LANGFLOW_DATABASE_URL", + "project": ".", + "state": "declared", + "hasDefault": true + } ] }, { @@ -928,6 +1112,30 @@ "path": ".", "id": "rails" } + ], + "environmentChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "app-json", + "source": "app.json", + "name": "LOCAL_DOMAIN", + "project": ".", + "state": "declared", + "required": true, + "hasDefault": false + }, + { + "kind": "environment", + "category": "environment", + "format": "app-json", + "source": "scalingo.json", + "name": "SECRET_KEY_BASE", + "project": ".", + "state": "declared", + "generated": true, + "secret": true + } ] }, { @@ -964,6 +1172,19 @@ "path": ".", "id": "rails" } + ], + "environmentChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "app-json", + "source": "app.json", + "name": "APP_SECRET_TOKEN", + "project": ".", + "state": "declared", + "generated": true, + "secret": true + } ] }, { @@ -1056,6 +1277,47 @@ "id": "cargo-workspace", "member": "macros" } + ], + "environmentChecks": [ + { + "format": "dotenv", + "name": "DC_ENV_FILE", + "project": "playwright", + "source": "playwright/.env.template", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "compose", + "project": "playwright", + "reference": { + "kind": "env-file" + }, + "source": "playwright/docker-compose.yml", + "state": "unresolved", + "kind": "environment", + "category": "environment", + "name": null + }, + { + "format": "dockerfile", + "name": "LANG", + "project": ".", + "source": "docker/Dockerfile.alpine", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "github-actions", + "name": "RUSTFLAGS", + "project": ".", + "source": ".github/workflows/build.yml", + "state": "declared", + "kind": "environment", + "category": "environment" + } ] }, { @@ -1146,6 +1408,54 @@ "id": "cargo-workspace", "member": "crates/router" } + ], + "environmentChecks": [ + { + "format": "compose", + "name": "HYPERSWITCH_SERVER_URL", + "project": ".", + "source": "docker-compose.yml", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "dockerfile", + "name": "CARGO_INCREMENTAL", + "project": ".", + "source": "Dockerfile", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "github-actions", + "name": "CARGO_INCREMENTAL", + "project": ".", + "source": ".github/workflows/CI-pr.yml", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "kind": "environment", + "category": "environment", + "name": "EXTRA_FEATURES", + "source": "Dockerfile", + "format": "dockerfile", + "absent": true, + "scope": null + }, + { + "kind": "environment", + "category": "environment", + "name": "EXTRA_FEATURES", + "source": "Dockerfile", + "format": "dockerfile", + "state": "declared", + "scope": "build", + "hasDefault": true + } ] }, { @@ -1269,6 +1579,35 @@ "id": "node-workspace", "member": "api" } + ], + "environmentChecks": [ + { + "format": "dotenv", + "name": "MONGOHQ_URL", + "project": ".", + "source": "sample.env", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "compose", + "name": "GATSBY_HOST", + "project": ".", + "source": ".devcontainer/docker-compose.yml", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "github-actions", + "name": "CROWDIN_PERSONAL_TOKEN", + "project": ".", + "source": ".github/workflows/crowdin-download.client-ui.yml", + "state": "declared", + "kind": "environment", + "category": "environment" + } ] }, { @@ -1417,6 +1756,37 @@ "path": "apps/api", "id": "express" } + ], + "environmentChecks": [ + { + "format": "kubernetes", + "name": "PORT", + "project": "examples", + "source": "examples/kubernetes/cluster-install/api.yaml", + "state": "declared", + "kind": "environment", + "category": "environment" + }, + { + "format": "kubernetes", + "project": "examples", + "reference": { + "kind": "config-map", + "target": "firecrawl-config" + }, + "source": "examples/kubernetes/cluster-install/api.yaml", + "state": "unresolved", + "kind": "environment", + "category": "environment", + "name": null + }, + { + "kind": "environment", + "category": "environment", + "name": "apiKey", + "format": "kubernetes", + "absent": true + } ] }, { @@ -1474,6 +1844,20 @@ "id": "node-workspace", "member": "packages/excalidraw" } + ], + "sourceChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "code-reference", + "state": "referenced", + "name": "VITE_APP_DISABLE_PREVENT_UNLOAD", + "project": "excalidraw-app", + "source": "excalidraw-app/App.tsx", + "origin": "framework", + "provider": "vite", + "scope": "build" + } ] }, { @@ -1844,6 +2228,19 @@ "id": "gradle-workspace", "member": "core/spring-boot" } + ], + "sourceChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "code-reference", + "state": "referenced", + "name": "DYNO", + "project": "core/spring-boot", + "origin": "platform", + "provider": "heroku", + "scope": "runtime" + } ] }, { @@ -2521,6 +2918,302 @@ "topicRank": 60, "selectionQuery": "topic:deno archived:false fork:false", "supplemental": true + }, + { + "repository": "documenso/documenso", + "language": "TypeScript", + "stars": 15710, + "revision": "fdd0337bdd8f6afac64faf97113c65ed2dd0e2a2", + "platform": "render", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node-workspace" + } + ], + "environmentChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "render", + "source": "render.yaml", + "name": "NEXT_PRIVATE_DATABASE_URL", + "project": ".", + "state": "unresolved", + "reference": { + "kind": "database", + "target": "documenso-db", + "key": "connectionString" + } + }, + { + "kind": "environment", + "category": "environment", + "format": "render", + "source": "render.yaml", + "name": "NEXTAUTH_URL", + "project": ".", + "state": "unresolved", + "reference": { + "kind": "service-env", + "target": "documenso-app", + "key": "RENDER_EXTERNAL_URL" + } + }, + { + "kind": "environment", + "category": "environment", + "format": "render", + "source": "render.yaml", + "name": "NEXTAUTH_SECRET", + "project": ".", + "state": "declared", + "generated": true, + "secret": true + }, + { + "kind": "environment", + "category": "environment", + "format": "render", + "source": "render.yaml", + "name": "NEXT_PRIVATE_SMTP_HOST", + "project": ".", + "state": "declared", + "secret": true, + "hasDefault": false + } + ], + "supplemental": true, + "webRank": null, + "sourceChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "code-reference", + "state": "referenced", + "name": "NEXT_PUBLIC_WEBAPP_URL", + "project": "packages/lib", + "source": "packages/lib/constants/app.ts", + "origin": "framework", + "provider": "nextjs", + "scope": "build" + } + ] + }, + { + "repository": "Unleash/unleash", + "language": "TypeScript", + "stars": 14293, + "revision": "d55a55f2181627c14cb51220390fdc13c45499aa", + "platform": "digitalocean", + "checks": [ + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "express" + } + ], + "environmentChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "digitalocean", + "source": ".do/deploy.template.yaml", + "name": "DATABASE_URL", + "project": ".", + "state": "unresolved", + "scope": "runtime", + "reference": { + "kind": "database", + "target": "unleash-db", + "key": "DATABASE_URL" + } + }, + { + "kind": "environment", + "category": "environment", + "format": "digitalocean", + "source": ".do/deploy.template.yaml", + "name": "UNLEASH_URL", + "project": ".", + "state": "unresolved", + "scope": "runtime", + "reference": { + "kind": "app", + "key": "APP_URL" + } + }, + { + "kind": "environment", + "category": "environment", + "format": "app-json", + "source": "app.json", + "name": "NPM_CONFIG_PRODUCTION", + "project": ".", + "state": "declared", + "hasDefault": true + } + ], + "supplemental": true, + "webRank": null + }, + { + "repository": "elie222/inbox-zero", + "language": "TypeScript", + "stars": 12969, + "revision": "9d7cf0758a9caa23daec728ffaafa63568b166be", + "platform": "copilot", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node-workspace" + } + ], + "environmentChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "copilot", + "source": "copilot/inbox-zero-ecs/manifest.yml", + "name": "DATABASE_URL", + "project": ".", + "state": "unresolved", + "secret": true, + "scope": "runtime", + "reference": { + "kind": "ssm-parameter" + } + }, + { + "kind": "environment", + "category": "environment", + "format": "copilot", + "source": "copilot/inbox-zero-ecs/manifest.yml", + "name": "NEXT_PUBLIC_BASE_URL", + "project": ".", + "state": "declared", + "secret": false, + "hasDefault": false, + "scope": "runtime" + }, + { + "kind": "environment", + "category": "environment", + "format": "copilot", + "source": "copilot/inbox-zero-ecs/manifest.yml", + "name": "HOSTNAME", + "project": ".", + "state": "declared", + "secret": false, + "hasDefault": true, + "scope": "runtime" + } + ], + "supplemental": true, + "webRank": null, + "sourceChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "code-reference", + "state": "referenced", + "name": "VERCEL_URL", + "project": "apps/web", + "source": "apps/web/env.ts", + "origin": "platform", + "provider": "vercel", + "scope": "build-and-runtime" + } + ] + }, + { + "repository": "twilio/twilio-video-app-react", + "language": "TypeScript", + "stars": 1819, + "revision": "40c5e765c61794efaa214331fde19e1959ad4fc3", + "platform": "app-engine", + "checks": [ + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "react" + } + ], + "environmentChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "app-engine", + "source": "app.yaml", + "name": "REACT_APP_SET_AUTH", + "project": ".", + "state": "declared", + "hasDefault": true, + "scope": "runtime" + } + ], + "supplemental": true, + "webRank": null + }, + { + "repository": "chatwoot/chatwoot", + "language": "Ruby", + "stars": 39564, + "revision": "af2a870c6d51aecf2a76388cf7b181dcf934c0fa", + "platform": "app-json", + "checks": [ + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "rails" + } + ], + "environmentChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "app-json", + "source": "app.json", + "name": "SECRET_TOKEN", + "project": ".", + "state": "declared", + "generated": true, + "secret": true + }, + { + "kind": "environment", + "category": "environment", + "format": "app-json", + "source": "app.json", + "name": "RAILS_ENV", + "project": ".", + "state": "declared", + "hasDefault": true + } + ], + "supplemental": true, + "webRank": null, + "sourceChecks": [ + { + "kind": "environment", + "category": "environment", + "format": "code-reference", + "state": "referenced", + "name": "HEROKU_SLUG_COMMIT", + "project": ".", + "source": "config/initializers/git_sha.rb", + "origin": "platform", + "provider": "heroku", + "scope": "runtime" + } + ] } ], "extensions": [ @@ -2545,6 +3238,13 @@ "honojs/middleware" ], "note": "Deno is a runtime ecosystem group, not a new language. Primary-language checks stay javascript/typescript." + }, + { + "selectedAt": "2026-11-10T16:30:00+00:00", + "purpose": "Opt-in environment coverage for deployment manifests of other hosting platforms (--with-env only).", + "selection": "Existing corpus repositories with platform manifests received environmentChecks. Five popular non-archived web applications were added because the corpus had no Render reference wiring, DigitalOcean, AWS Copilot, App Engine or duplicate-key app.json example; stars and pinned default-branch heads were read through the GitHub API on the selection date.", + "note": "Fly.io fly.toml, heroku.yml and Vercel/Netlify template variables have synthetic Rust coverage only; no eligible pinned repository with those declarations was selected.", + "codeReferences": "`sourceChecks` sample code references found by the catalog's literal search (framework prefixes VITE_ and NEXT_PUBLIC_; platform-injected HEROKU_SLUG_COMMIT, DYNO, VERCEL_URL) and run only with --with-env --with-source. Dockerfile ARG declaration checks are ordinary environmentChecks." } ] } diff --git a/tests/detect-corpus.md b/tests/detect-corpus.md index c826f56..cf9728b 100644 --- a/tests/detect-corpus.md +++ b/tests/detect-corpus.md @@ -176,6 +176,10 @@ All **295/295 assertions across 51 repositories** pass on the extended engine, w Official behavior references: [configuration](https://docs.deno.com/runtime/fundamentals/configuration/), [workspaces](https://docs.deno.com/runtime/fundamentals/workspaces/), and [config discovery implementation](https://github.com/denoland/deno_config/blob/main/src/deno_json/mod.rs) (`deno.json` precedes `deno.jsonc`). These document current behavior; pinned repository SHAs, not moving documentation or CI/Docker versions, define the assertion evidence. +## Platform manifest supplement (2026-11-10) + +Five non-archived web applications are pinned as `supplemental` entries for `--with-env` platform-manifest coverage. The existing corpus had no Render reference wiring, DigitalOcean spec, AWS Copilot manifest, App Engine `app.yaml` or `app.json` with a duplicate key: [documenso/documenso](https://github.com/documenso/documenso/tree/fdd0337bdd8f6afac64faf97113c65ed2dd0e2a2), [Unleash/unleash](https://github.com/Unleash/unleash/tree/d55a55f2181627c14cb51220390fdc13c45499aa), [elie222/inbox-zero](https://github.com/elie222/inbox-zero/tree/9d7cf0758a9caa23daec728ffaafa63568b166be), [twilio/twilio-video-app-react](https://github.com/twilio/twilio-video-app-react/tree/40c5e765c61794efaa214331fde19e1959ad4fc3) and [chatwoot/chatwoot](https://github.com/chatwoot/chatwoot/tree/af2a870c6d51aecf2a76388cf7b181dcf934c0fa). Each carries one reviewed default project/framework check, so the default audit now covers **300/300 assertions across 56 repositories**. Their environment checks and results are in [`detect-env.md`](detect-env.md#local-results-2026-11-10). + ## Reproduce ```sh diff --git a/tests/detect-env.md b/tests/detect-env.md new file mode 100644 index 0000000..e94f9af --- /dev/null +++ b/tests/detect-env.md @@ -0,0 +1,206 @@ +# Name-only environment detection + +```sh +brainpod detect . --with-env +brainpod detect ./services --with-env --json +brainpod detect . --with-env --with-source --json +``` + +Without the flag, environment adapters are not run, no additional files are read, and the `environment` field is absent. With it, the field is present even when empty. Project/framework/runtime detection is unchanged. + +**`--with-env` reads metadata files only**: the base five formats and the platform manifests below. It never reads application source. **`--with-source`** (accepted only together with `--with-env`) adds [code references](#code-references): every source file with a cataloged extension is searched for a fixed list of platform-injected names and framework prefixes. This multiplies reads; see [local results](#local-results-2026-11-10). Without flags, reads and output are byte-identical to the base branch. + +## Why the base five + +This is a practical initial selection, **not a measured global top-five ranking**. Research on 2026-11-10 used public, read-only documentation and GitHub repository metadata: + +- **Dotenv:** the cross-language `.env` convention; [motdotla/dotenv](https://github.com/motdotla/dotenv) had 20,563 stars. Its [parser rules](https://github.com/motdotla/dotenv#what-rules-does-the-parsing-engine-follow) explicitly describe assignments, comments, quotes and multiline values. +- **Compose:** [docker/compose](https://github.com/docker/compose) had 38,674 stars. The [Compose specification](https://github.com/compose-spec/compose-spec/blob/main/spec.md#environment) explicitly distinguishes `environment`, pass-through keys and `env_file` declarations. +- **Dockerfile:** Docker is described as near-universal in the [2025 Stack Overflow technology survey](https://survey.stackoverflow.co/2025/technology/). The [official frontend reference](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#env) distinguishes persisted `ENV` declarations from build-only `ARG` (BuildKit had 10,382 stars). +- **Kubernetes:** [kubernetes/kubernetes](https://github.com/kubernetes/kubernetes) had 128,969 stars. Its [environment documentation](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/) explicitly defines `env` and `envFrom` without requiring cluster queries. +- **GitHub Actions:** an explicit, declarative CI format with [documented workflow/job/step `env` maps](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#env). [actions/runner](https://github.com/actions/runner) had 6,386 stars; the pinned local corpus independently supplied environment declarations in 42 of 51 repositories. This sample is web-project-biased, not market-share evidence. + +Stars are adoption proxies, not comparable format-usage counts. No private repository content was sent to a service. Context7 was unavailable; primary documentation was read directly. + +## Exact coverage + +Filename/path routing is declared in `catalog.environment` in [`rules.toml`](../src/detect/rules.toml), compiled alongside the existing catalog. First matching entry wins. Syntax stays in the five base adapters in [`environment.rs`](../src/detect/environment.rs) and the platform adapters in [`environment/platforms.rs`](../src/detect/environment/platforms.rs). No executable configuration language or config-key guessing is involved. Platform entries come after the base entries, so a platform file inside a Kubernetes metadata directory (for example `deploy/app.yaml`) keeps Kubernetes routing. + +| Format | Candidates | Supported declarations | +|---|---|---| +| `dotenv` | `.env`, `.env.*`, `*.env`, `*.env.example`, `*.env.sample`, `*.env.template`; excludes `*.enc` and `.env.vault` | `NAME=value`, optional `export`, whitespace, empty values, comments, single/double/backtick quotes, escaped quotes and quoted multiline values. BOM/CRLF accepted. Bare names, colon assignment and shell programs are not supported. | +| `compose` | `compose.yml/yaml`, `docker-compose.yml/yaml` and dotted variants such as `compose.override.yaml` | `services.*.environment` scalar maps or string lists (`NAME=value` or bare `NAME`); null/bare entries are `external`. `env_file` string/list/path-mapping entries become unresolved references, not file reads. | +| `dockerfile` | `Dockerfile`, `Dockerfile.*`, `*.Dockerfile` | Case-insensitive `ENV` and `ARG`, multiple assignments, legacy `ENV NAME value`, quotes, escaped spaces, backslash/backtick continuations, continued comments, and `escape` directives. Each `FROM` advances a numbered stage; `ARG` before the first `FROM` is stage 0. [`ARG`](https://docs.docker.com/reference/dockerfile/#arg) records carry `scope: "build"` and `hasDefault` (whether `=default` is present; the default is never returned). `ENV` records are unchanged. `RUN` assignments and base-image environments are not inferred. | +| `kubernetes` | YAML/JSON in directories named `k8s`, `kubernetes`, `kube`, `manifests`, `deploy`, `deployment`, `deployments`; also `pod.yml/yaml`, `deployment.yml/yaml`, `kubernetes-*.yml/yaml`, `*.k8s.yml/yaml/json`, `*.kube.yml/yaml/json` | Core `v1` Pod/List/ReplicationController, `apps/v1` Deployment/StatefulSet/DaemonSet/ReplicaSet, `batch/v1` Job/CronJob. Known pod-spec paths only; regular/init/ephemeral containers; explicit `env[].name`, `valueFrom`, `envFrom` and prefixes. Lists and multi-document streams supported. | +| `github-actions` | `.github/workflows/*.yml/yaml` at any repository depth | Only workflow `env`, `jobs.*.env` and `jobs.*.steps[].env` maps. Whole-map expressions are unresolved references. `with`, matrix keys, scripts and writes to `$GITHUB_ENV` are not inferred. | +| `app-json` | `app.json`, `scalingo.json` | `env` and `environments..env`. A string is a literal value; an object may state `required`, `value` and `generator`. Files without `env` (for example Expo `app.json`) produce nothing. Parsed as JSON; a duplicate key keeps the last value, as JSON parsers on these platforms do. | +| `heroku-yml` | `heroku.yml` | `setup.config` (provisioning config vars, `runtime`) and `build.config` (build-time only, `build`). | +| `render` | `render.yaml` | `envVars` of `services` and `envVarGroups` at the root, in `ungrouped`, and in `projects[].environments[]`. | +| `fly` | `fly.toml`, `fly.*.toml` | `[env]` (`runtime`). `[build.args]` are Docker build arguments and are not read. | +| `digitalocean` | `.do/app.yaml`, `.do/deploy.template.yaml` at any depth | App-level `envs` and the `envs` of `services`, `static_sites`, `workers`, `jobs` and `functions`. A Deploy button template's spec under `spec` is supported. | +| `vercel` | `vercel.json`, `vercel.toml` | Legacy `env` (functions, `runtime`) and `build.env` (`build`). | +| `netlify` | `netlify.toml` | `build.environment`, `context..environment` (`build`) and `template.environment`. | +| `app-engine` | `app.yaml` with a top-level `runtime` | `env_variables` (`runtime`) and `build_env_variables` (`build`). | +| `copilot` | `copilot//manifest.yml/yaml` at any depth, except `type: Environment` | `variables`, `secrets` and `env_file` at the top level, in `sidecars.`, in `environments.` overrides and in their `sidecars.`. | + +TOML (`fly.toml`, `netlify.toml`, `vercel.toml`) uses the existing `toml` dependency, which already parses detection rules and manifests; its parser rejects excessive nesting (a 5,000-level array is a redacted diagnostic, not a crash). `app.json`, `scalingo.json` and `vercel.json` use `serde_json`, whose recursion limit is 128. YAML supports flow/block collections, quoted/block scalars and bounded anchors/merge keys. Duplicate mapping keys, unsupported tags, malformed input or budget breaches reject the entire YAML stream with a redacted diagnostic. No includes or property interpolation are enabled. Kubernetes candidates are intentionally narrower than all YAML: application data, Rails fixtures, Spring properties, arbitrary JSON, CRDs, Helm rendering and Kustomize evaluation are not scanned/inferred as environments. Use an extra catalog entry for a nonconventional manifest location. + +Names use the conservative common subset `[A-Za-z_][A-Za-z0-9_.-]*`. Unsupported/dynamic names are skipped with diagnostics. This is not a claim to accept every name allowed by every ecosystem. Kubernetes resource/key references additionally allow a leading digit. Simple downward-API field/resource paths are retained as reference keys; complex field selectors are left unresolved. + +A `RUN`/`COPY`/`ADD` body containing `<<` conservatively stops inspection of that Dockerfile at the containing instruction with a diagnostic, preventing script bodies from becoming false `ENV` declarations. Instructions after the heredoc and experimental `ENV` flags are not covered. Custom Dockerfile frontends, Compose includes/extends/overlays, precedence, image stages/inheritance, workflow activation and environment installation are not evaluated. + +## Platform deployment manifests + +Deployment files written for other hosts are the best structured source of per-variable intent. They often state whether a variable is required, secret, generated, or wired to a database or service. Primary documentation was read on 2026-11-10. A format is included only when its documentation defines variable declarations inside the repository. + +Included: + +- **Heroku, Dokku and Scalingo `app.json` / `scalingo.json`:** [Heroku app.json schema](https://devcenter.heroku.com/articles/app-json-schema#env), [Dokku app.json](https://dokku.com/docs/appendices/file-formats/app-json/#env), [Scalingo manifest](https://doc.scalingo.com/platform/app/app-manifest) and [scalingo.json schema](https://developers.scalingo.com/scalingo-json-schema). Scalingo also reads `app.json` when no `scalingo.json` exists. +- **Heroku `heroku.yml`:** [setup config vars and build-time variables](https://devcenter.heroku.com/articles/build-docker-images-heroku-yml#setup-define-your-app-s-environment). +- **Render `render.yaml`:** [Blueprint reference, environment variables](https://render.com/docs/blueprint-spec#environment-variables). +- **Fly.io `fly.toml`:** [the `[env]` section](https://fly.io/docs/reference/configuration/#the-env-variables-section). Secrets are set with `fly secrets` outside the repository. +- **DigitalOcean App Platform:** [app spec `envs`](https://docs.digitalocean.com/products/app-platform/reference/app-spec/), [Deploy button template](https://docs.digitalocean.com/products/app-platform/how-to/add-deploy-do-button/#configure-environment-variables), [bindable variables](https://docs.digitalocean.com/products/app-platform/how-to/use-environment-variables/#use-bindable-variables), and the [dev config default `.do/app.yaml`](https://docs.digitalocean.com/products/app-platform/reference/dev-config/). +- **Vercel `vercel.json` / `vercel.toml`:** the [legacy `env` and `build.env`](https://vercel.com/docs/project-configuration/vercel-json#env) properties, which Vercel recommends against but still supports. [`vercel.toml` supports the same fields](https://vercel.com/docs/project-configuration/vercel-toml). +- **Netlify `netlify.toml`:** [build and context environments](https://docs.netlify.com/build/configure-builds/file-based-configuration/#deploy-contexts), [which are available to builds but not Functions](https://docs.netlify.com/build/environment-variables/overview/#configuration-options-and-limitations), and [template variables for the Deploy button](https://docs.netlify.com/deploy/create-deploys/#file-based-template-configuration). +- **Google App Engine `app.yaml`:** [`env_variables` and `build_env_variables`](https://cloud.google.com/appengine/docs/standard/reference/app-yaml#runtime_and_app_elements). The mandatory `runtime` element distinguishes App Engine files from other `app.yaml` files. +- **AWS Copilot workload manifests:** [`variables`, `secrets` and `env_file`](https://aws.github.io/copilot-cli/docs/manifest/lb-web-service/) and [secrets](https://aws.github.io/copilot-cli/docs/developing/secrets/). Copilot CLI reached end of support on 2026-06-12, but existing manifests remain in repositories. + +Excluded: + +- **Railway:** `railway.json`/`railway.toml` [Config as Code](https://docs.railway.com/reference/config-as-code) only carries build/deploy settings, has no variables, and is deprecated (no longer read after 2026-12-01). Its replacement, [Infrastructure as Code](https://docs.railway.com/infrastructure-as-code), is executable TypeScript/Python/Go evaluated by the CLI. Executing or interpreting it is out of scope. +- **Koyeb:** no in-repository manifest. The [Deploy to Koyeb button](https://www.koyeb.com/docs/build-and-deploy/deploy-to-koyeb-button) passes `env[NAME]=value` URL parameters. +- **README Deploy-button URLs** (Vercel `env` query parameters, Netlify URL hashes, Koyeb `env[...]`): unstructured links, not manifests. **`vercel.ts`** is executable configuration. +- **Fly.io `fly.json`/`fly.yaml`:** alternate formats produced by `fly config save --json/--yaml`, with no fixed documented filename. They are not routed. +- **Heroku add-ons:** add-ons such as `heroku-postgresql` set config vars (for example `DATABASE_URL`) on the platform side. `app.json` does not name them, so nothing is inferred. + +### Semantic fields + +Platform records add the following optional fields. Each is present **only when the manifest states it**, and existing fields keep their meaning. Base-format records only gain them for Dockerfile `ARG` (above). + +| Field | Meaning | Set by | +|---|---|---| +| `required` | Stated requirement. | `app.json`/`scalingo.json` `required`. Heroku and Dokku document a default of `true` when it is omitted, and Scalingo documents none; that default is **not** applied. Netlify `template.environment` (documented as required) gives `true`. | +| `secret` | The platform treats the value as sensitive. `false` means the format explicitly classifies it as plain. | `true`: Render `sync: false` ([prompted secret values](https://render.com/docs/blueprint-spec#prompting-for-secret-values)) and `generateValue`, `app.json` `generator: "secret"`, DigitalOcean `type: SECRET`, Copilot `secrets`, Vercel `@name` secret references. `false`: DigitalOcean `type: GENERAL`, Copilot `variables`. | +| `generated` | The platform generates the value. | `app.json` `generator` (Heroku/Dokku `secret`; Scalingo `secret`, `url`, `template`), Render `generateValue: true`. | +| `hasDefault` | Whether the file supplies a literal value (fixed or default). **The value is never returned.** `false` means the entry exists without a value and must come from elsewhere. | `true`: literal map entries, `app.json` strings and `value`, Render `value`, DigitalOcean `value`, Dockerfile `ARG NAME=…`. `false`: `app.json` objects without `value`, Render `sync: false`, DigitalOcean entries without `value`, YAML `NAME:` with a null value, Netlify `template.environment` (its value is a UI label), `ARG NAME`. Omitted for references and generated values. | +| `scope` | `runtime`, `build` or `build-and-runtime`, when the docs fix it. | DigitalOcean `RUN_TIME`/`BUILD_TIME`/`RUN_AND_BUILD_TIME`; Vercel `env`/`build.env`; Netlify toml environments; Fly `[env]`; App Engine `env_variables`/`build_env_variables`; `heroku.yml` `setup.config`/`build.config`; Copilot containers; Dockerfile `ARG`. | + +Wiring to another resource uses the existing `reference` shape with `state: "unresolved"`: + +- **Render:** `fromDatabase` → `database` (target = database name, key = property such as `connectionString`). `fromService` with `property` → `service`, or `key-value` for `type: keyvalue`/`redis`. `fromService` with `envVarKey` → `service-env`. `fromGroup` → a nameless `env-group` record; group contents in other files are not followed. Render ignores `sync: false` inside environment groups, so those entries are skipped with a diagnostic. +- **DigitalOcean:** a whole-value binding `${db.VAR}` becomes `database` when `db` is listed in the spec's `databases` (including 32-character prefixes). Otherwise `${component.VAR}`/`${_self.VAR}` becomes `service`, and `${APP_URL}` becomes `app` with key `APP_URL`. The pool form `${db.pool.VAR}` keeps the database and the variable. Any other value containing `${` becomes `interpolation` with no target. `type: SECRET` values are never inspected. +- **Vercel** `@name` → `platform-secret`. **Copilot** string secrets → `ssm-parameter`, `secretsmanager` → `secrets-manager`, `from_cfn` → `cloudformation-export`, `env_file` → `env-file`. +- A target or key is kept only when it is a simple name (`[A-Za-z0-9_.-]`). SSM parameter paths, ARNs and anything with `:` or `/` are omitted. + +Environment, context and sidecar names used as location segments must be simple names; others are skipped with a diagnostic. + +## Code references + +Some variables are never declared in repository files: platforms inject them at build or run time, and frameworks expose prefixed variables to client code by convention. `--with-env --with-source` therefore runs a literal search over source files; `--with-env` alone does not, and does not read source files at all. This is **not** a read-site scanner. Only the names and prefixes in `catalog.code_references` in [`rules.toml`](../src/detect/rules.toml) are recognised; arbitrary `process.env.X`/`os.environ[...]` reads are not. Patterns are data: an extra `--rules` file can add patterns, extensions and exclusions without code changes. + +- **Files:** basenames with extensions `js jsx mjs cjs ts tsx mts cts vue svelte astro py rb go rs php java kt kts cs ex exs`, excluding `*.min.js`, `*.min.mjs`, `*.min.cjs` and `*.bundle.js`. The existing directory exclusions apply (`node_modules`, `vendor`, `dist`, `build`, `.next`, `.nuxt`, `.output`, `target`, virtualenvs and similar), as do symlink skipping, the 128-level depth limit and the 1 MiB per-file limit. Oversized source files are skipped with the diagnostic `source exceeds 1 MiB read limit; environment references not scanned`. Bytes are searched directly, so non-UTF-8 files are scanned without a diagnostic. +- **Matching:** the file is split into whole ASCII identifier tokens (`[A-Za-z0-9_]+`, bounded by any other byte, so `$VITE_X` and `env.VITE_X` match). A token matches an exact catalog name, or starts with a catalog prefix and continues with one or more `[A-Z0-9_]` characters (`\bPREFIX[A-Z0-9_]+\b`). The prefix alone, lowercase or mixed-case continuations, and longer identifiers that merely contain a prefix (`MY_NEXT_PUBLIC_X`) do not match. Exact names win over prefixes, and the longest prefix wins (`NUXT_PUBLIC_` before `NUXT_`). +- **Generic names:** names that collide with ordinary code are never cataloged: `URL`, `CONTEXT`, `HEAD`, `BRANCH`, `PORT`, `CI`, `NODE_ENV`, `HOSTNAME`, `DATABASE_URL`, `WEB_CONCURRENCY`, `ACCOUNT_ID` and the OpenTelemetry names. Moderately generic documented names are cataloged with `signals`: they are reported only when the platform's manifest basename exists anywhere in the scan (for example Netlify `DEPLOY_URL` requires a `netlify.toml`). The generic `PUBLIC_` prefix uses `requires` and is reported only when the owning project is detected as SvelteKit or Astro. Rsbuild, Qwik, Farm and Modern.js prefixes have no detection rule to gate on and are not cataloged. +- **Records:** one record per `(project, name)` with `format: "code-reference"`, `state: "referenced"`, `origin` (`platform` or `framework`), `provider` (the pattern id) and `scope` when documented. `source`/`location` (`line:N`) point to the first occurrence. `occurrences` lists sorted `{source, line}` pairs, **capped at 20**, and `occurrenceCount` is the uncapped total. Project ownership is the nearest detected ancestor, as for declarations. No code text, surrounding context or value is retained or returned. +- **Attribution is by convention, not proof:** a `NEXT_PUBLIC_*` reference in a non-Next.js package is still attributed to `nextjs`. Vercel's framework-prefixed system variables (such as `NEXT_PUBLIC_VERCEL_URL`) are reported under the framework prefix. Fly documents `FLY_` as a reserved prefix, but only its documented names are cataloged because users also set `FLY_*` CLI tokens. + +Patterns and their sources (research 2026-11-10): + +| Origin | Provider | Matches (scope) | Gate | Documentation | +|---|---|---|---|---| +| framework | `nextjs` | `NEXT_PUBLIC_*` (build) | — | [1](https://nextjs.org/docs/app/guides/environment-variables#bundling-environment-variables-for-the-browser) | +| framework | `vite` | `VITE_*` (build) | — | [1](https://vite.dev/guide/env-and-mode#env-variables) [2](https://docs.solidjs.com/configuration/environment-variables#public-environment-variables) [3](https://tanstack.com/start/latest/docs/framework/react/guide/environment-variables) | +| framework | `create-react-app` | `REACT_APP_*` (build) | — | [1](https://create-react-app.dev/docs/adding-custom-environment-variables/) | +| framework | `nuxt` | `NUXT_PUBLIC_*`, `NUXT_*` (runtime) | — | [1](https://nuxt.com/docs/4.x/guide/going-further/runtime-config#environment-variables) | +| framework | `sveltekit` | `PUBLIC_*` (unstated) | owning project detected as `sveltekit` | [1](https://svelte.dev/docs/kit/$env-static-public) [2](https://svelte.dev/docs/kit/$env-dynamic-public) | +| framework | `astro` | `PUBLIC_*` (build) | owning project detected as `astro` | [1](https://docs.astro.build/en/guides/environment-variables/#vites-built-in-support) | +| framework | `expo` | `EXPO_PUBLIC_*` (build) | — | [1](https://docs.expo.dev/guides/environment-variables/#reading-environment-variables-from-env-files) | +| framework | `gatsby` | `GATSBY_*` (build) | — | [1](https://www.gatsbyjs.com/docs/how-to/local-development/environment-variables/#accessing-environment-variables-in-the-browser) | +| framework | `vue-cli` | `VUE_APP_*` (build) | — | [1](https://cli.vuejs.org/guide/mode-and-env.html#environment-variables) | +| framework | `redwood` | `REDWOOD_ENV_*` (build) | — | [1](https://docs.redwoodjs.com/docs/environment-variables#option-2-prefixing-with-redwood_env_) | +| framework | `storybook` | `STORYBOOK_*` (build) | — | [1](https://storybook.js.org/docs/configure/environment-variables) | +| framework | `sanity` | `SANITY_STUDIO_*` (build) | — | [1](https://www.sanity.io/docs/studio/environment-variables#exposed-variables) | +| framework | `umi` | `UMI_APP_*` (build) | — | [1](https://umijs.org/en-US/docs/guides/env-variables#use-environment-variables-in-browser) | +| framework | `plasmo` | `PLASMO_PUBLIC_*` (build) | — | [1](https://docs.plasmo.com/framework/env#custom-environment-variables) | +| framework | `wxt` | `WXT_*` (build) | — | [1](https://wxt.dev/guide/essentials/config/environment-variables#dotenv-files) | +| framework | `tauri` | `TAURI_ENV_*` (build) | — | [1](https://v2.tauri.app/reference/environment-variables/#tauri-cli-hook-commands) | +| framework | `quasar` | `QCLI_*`, `QUASAR_*` (build) | — | [1](https://quasar.dev/quasar-cli-vite/handling-import-meta-env#from-dotenv-files) [2](https://quasar.dev/quasar-cli-vite/handling-import-meta-env#values-provided-by-quasar-cli) | +| framework | `electron-vite` | `MAIN_VITE_*`, `PRELOAD_VITE_*`, `RENDERER_VITE_*` (build) | — | [1](https://electron-vite.org/guide/env-and-mode#global-env-variables) | +| platform | `vercel` | `VERCEL_OIDC_TOKEN`, `VERCEL_HASH_SALT`, `VERCEL_GIT_PREVIOUS_SHA` (build); `VERCEL`, `VERCEL_ENV`, `VERCEL_TARGET_ENV`, `VERCEL_URL`, `VERCEL_BRANCH_URL`, `VERCEL_PROJECT_PRODUCTION_URL`, `VERCEL_DEPLOYMENT_ID`, `VERCEL_PROJECT_ID`, `VERCEL_PROJECT_NAME`, `VERCEL_SKEW_PROTECTION_ENABLED`, `VERCEL_AUTOMATION_BYPASS_SECRET`, `VERCEL_GIT_PROVIDER`, `VERCEL_GIT_REPO_SLUG`, `VERCEL_GIT_REPO_OWNER`, `VERCEL_GIT_REPO_ID`, `VERCEL_GIT_COMMIT_REF`, `VERCEL_GIT_COMMIT_SHA`, `VERCEL_GIT_COMMIT_MESSAGE`, `VERCEL_GIT_COMMIT_AUTHOR_LOGIN`, `VERCEL_GIT_COMMIT_AUTHOR_NAME`, `VERCEL_GIT_PULL_REQUEST_ID` (build-and-runtime); `VERCEL_REGION` (runtime) | — | [1](https://vercel.com/docs/environment-variables/system-environment-variables#system-environment-variables) | +| platform | `netlify` | `NETLIFY`, `NETLIFY_PREVIEW_SERVER`, `CACHED_COMMIT_REF`, `DEPLOY_PRIME_URL`, `INCOMING_HOOK_TITLE`, `INCOMING_HOOK_URL`, `INCOMING_HOOK_BODY` (build); `BUILD_ID`, `REPOSITORY_URL`, `COMMIT_REF`, `PULL_REQUEST`, `REVIEW_ID`, `DEPLOY_URL`, `DEPLOY_ID` (build); `SITE_NAME`, `SITE_ID` (build-and-runtime) | names `BUILD_ID`, `REPOSITORY_URL`, `COMMIT_REF`, `PULL_REQUEST`, `REVIEW_ID`, `DEPLOY_URL`, `DEPLOY_ID` need `netlify.toml`; names `SITE_NAME`, `SITE_ID` need `netlify.toml` | [1](https://docs.netlify.com/build/configure-builds/environment-variables/#read-only-variables) | +| platform | `render` | `RENDER_CPU_COUNT`, `RENDER_DISCOVERY_SERVICE`, `RENDER_EXTERNAL_HOSTNAME`, `RENDER_EXTERNAL_URL`, `RENDER_GIT_BRANCH`, `RENDER_GIT_COMMIT`, `RENDER_GIT_REPO_SLUG`, `RENDER_INSTANCE_ID`, `RENDER_SERVICE_ID`, `RENDER_SERVICE_NAME`, `RENDER_SERVICE_TYPE` (build-and-runtime); `RENDER_WEB_CONCURRENCY` (runtime); `RENDER`, `IS_PULL_REQUEST` (build-and-runtime) | names `RENDER`, `IS_PULL_REQUEST` need `render.yaml` | [1](https://render.com/docs/environment-variables#all-runtimes) | +| platform | `railway` | `RAILWAY_PUBLIC_DOMAIN`, `RAILWAY_PRIVATE_DOMAIN`, `RAILWAY_TCP_PROXY_DOMAIN`, `RAILWAY_TCP_PROXY_PORT`, `RAILWAY_TCP_APPLICATION_PORT`, `RAILWAY_PROJECT_NAME`, `RAILWAY_PROJECT_ID`, `RAILWAY_ENVIRONMENT_NAME`, `RAILWAY_ENVIRONMENT_ID`, `RAILWAY_SERVICE_NAME`, `RAILWAY_SERVICE_ID`, `RAILWAY_REPLICA_ID`, `RAILWAY_REPLICA_REGION`, `RAILWAY_DEPLOYMENT_ID`, `RAILWAY_SNAPSHOT_ID`, `RAILWAY_VOLUME_NAME`, `RAILWAY_VOLUME_MOUNT_PATH`, `RAILWAY_GIT_COMMIT_SHA`, `RAILWAY_GIT_AUTHOR`, `RAILWAY_GIT_BRANCH`, `RAILWAY_GIT_REPO_NAME`, `RAILWAY_GIT_REPO_OWNER`, `RAILWAY_GIT_COMMIT_MESSAGE` (build-and-runtime) | — | [1](https://docs.railway.com/variables/reference#railway-provided-variables) | +| platform | `heroku` | `DYNO`, `HEROKU_APP_ID`, `HEROKU_APP_NAME`, `HEROKU_APP_DEFAULT_DOMAIN_NAME`, `HEROKU_DYNO_ID`, `HEROKU_RELEASE_CREATED_AT`, `HEROKU_RELEASE_VERSION`, `HEROKU_SLUG_COMMIT`, `HEROKU_SLUG_DESCRIPTION`, `HEROKU_BUILD_COMMIT`, `HEROKU_BUILD_DESCRIPTION` (runtime) | — | [1](https://devcenter.heroku.com/articles/dyno-metadata#dyno-metadata) | +| platform | `cloudflare-pages` | `CF_PAGES`, `CF_PAGES_COMMIT_SHA`, `CF_PAGES_BRANCH`, `CF_PAGES_URL` (build) | — | [1](https://developers.cloudflare.com/pages/configuration/build-configuration/#environment-variables) | +| platform | `app-engine` | `GAE_APPLICATION`, `GAE_DEPLOYMENT_ID`, `GAE_ENV`, `GAE_INSTANCE`, `GAE_MEMORY_MB`, `GAE_RUNTIME`, `GAE_SERVICE`, `GAE_VERSION` (runtime); `GOOGLE_CLOUD_PROJECT` (runtime) | names `GOOGLE_CLOUD_PROJECT` need `app.yaml` | [1](https://cloud.google.com/appengine/docs/standard/nodejs/runtime#environment_variables) | +| platform | `koyeb` | `KOYEB_APP_NAME`, `KOYEB_APP_ID`, `KOYEB_ORGANIZATION_NAME`, `KOYEB_ORGANIZATION_ID`, `KOYEB_SERVICE_NAME`, `KOYEB_SERVICE_ID`, `KOYEB_SERVICE_PRIVATE_DOMAIN`, `KOYEB_PUBLIC_DOMAIN`, `KOYEB_PRIVILEGED`, `KOYEB_DOCKER_REF`, `KOYEB_GIT_SHA`, `KOYEB_GIT_BRANCH`, `KOYEB_GIT_REPOSITORY`, `KOYEB_GIT_COMMIT_MESSAGE`, `KOYEB_GIT_COMMIT_AUTHOR` (build-and-runtime); `KOYEB_REGION`, `KOYEB_REGIONAL_DEPLOYMENT_ID`, `KOYEB_INSTANCE_ID`, `KOYEB_INSTANCE_TYPE`, `KOYEB_INSTANCE_MEMORY_MB`, `KOYEB_REPLICA_INDEX`, `KOYEB_HYPERVISOR_ID`, `KOYEB_DC`, `KOYEB_GPU_COUNT` (runtime) | — | [1](https://www.koyeb.com/docs/build-and-deploy/environment-variables#koyeb-provided-variables) | +| platform | `cloud-run` | `K_SERVICE`, `K_REVISION`, `K_CONFIGURATION`, `CLOUD_RUN_JOB`, `CLOUD_RUN_EXECUTION`, `CLOUD_RUN_TASK_INDEX`, `CLOUD_RUN_TASK_ATTEMPT`, `CLOUD_RUN_TASK_COUNT`, `CLOUD_RUN_WORKER_POOL`, `CLOUD_RUN_REVISION`, `FUNCTION_TARGET`, `FUNCTION_SIGNATURE_TYPE` (runtime) | — | [1](https://cloud.google.com/run/docs/container-contract#env-vars) | +| platform | `fly` | `FLY_APP_NAME`, `FLY_MACHINE_ID`, `FLY_ALLOC_ID`, `FLY_REGION`, `FLY_PUBLIC_IP`, `FLY_IMAGE_REF`, `FLY_MACHINE_VERSION`, `FLY_PRIVATE_IP`, `FLY_PROCESS_GROUP`, `FLY_VM_MEMORY_MB`, `FLY_TEST_MACHINE_IP` (runtime); `PRIMARY_REGION`, `RELEASE_COMMAND` (runtime) | names `PRIMARY_REGION`, `RELEASE_COMMAND` need `fly.toml` | [1](https://fly.io/docs/machines/runtime-environment/#environment-variables) | + +## Output, provenance and privacy + +Each `environment` record has: + +- `name` when explicitly known; otherwise an unresolved reference record without a name. +- `source`: scan-root-relative metadata filename (for code references, the first source file); `format`: one of the base five, a platform format above, or `code-reference`. +- `location`: one-based `line:N` for dotenv and code references, numbered stage/one-based line/assignment index for Dockerfile, or zero-based `document:N` plus escaped JSON-pointer-style paths for YAML/JSON/TOML. +- `project`: nearest detected ancestor project/workspace path, or `null`. This is directory ownership, **not evidence that the application uses the variable**. Workflow files normally belong to the repository root, not projects mentioned in scripts. +- `state`: `declared`, `external` (Compose pass-through), `unresolved`, or `referenced` (code reference). +- Optional `reference`: kind and statically known target/key/prefix. Dynamic expressions and absolute/dynamic file targets are omitted, not interpolated or echoed. +- Optional `required`, `secret`, `generated`, `hasDefault` and `scope` ([semantic fields](#semantic-fields)), and for code references `origin`, `provider`, `occurrences` and `occurrenceCount`. The JSON schema is additive: existing records keep exactly their previous fields. + +Records are stably sorted by source; within a source they retain adapter order (JSON/YAML/TOML mapping keys are sorted). Code references follow declarations of the same source. Repeated list entries, repeated lines and the same name in different files/services/stages/scopes remain separate. There is no last-wins merge, collision resolution or workspace-to-member copying. Existing `project.workspaces` supplies workspace membership separately. Identical names do not establish dataflow between declarations. + +Literal environment values, including defaults and generated or secret entries, are never returned in JSON/text, diagnostics or retained declaration facts. Source-file bytes are only tokenised into identifiers; code text is never retained. File bytes and YAML values exist transiently in parser memory, then are discarded. No values are loaded into the process, expanded, sourced, executed, fetched or saved. Current process environment variables are not inspected. ConfigMap/Secret data keys are not enumerated, even when those resources are present locally; `envFrom` remains an unresolved declaration reference. Referenced env files are not followed; matching files independently discovered in the inventory retain their own source/project provenance. + +The existing directory exclusions, symlink skipping, 128-level depth limit and 1 MiB per-file read limit apply to manifests and source files alike. Explicit symlink scan roots are rejected with the flag. Reads stop at 1 MiB + 1 byte for oversized-file diagnostics; no total repository-size cap is imposed. YAML is limited to depth 48, 256 documents, 100,000 events, 50,000 nodes, 128 anchors/aliases and 2 MiB of expanded scalar content per stream. Diagnostics never quote environment input and are deduplicated per source/message. Static metadata scanning assumes the directory tree is not concurrently replaced during traversal. + +## Reproduction + +No dependencies or downloaded project code need to be installed/executed. The local Cargo cache lacked `serde-saphyr` offline, so the Rust commands below ran without `--offline`. The five repositories added for platform manifests need one explicit `--fetch` into a snapshot cache; later runs are offline: + +```sh +cargo test --locked -j 4 +cargo clippy --locked -j 4 --all-targets -- -D warnings +cargo fmt --all -- --check +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_audit_detect.py' +cargo build --release --locked -j 4 +python3 tests/audit_detect.py --list + +cache=/tmp/bp-envplat/cache +python3 tests/audit_detect.py --binary target/release/brainpod --cache "$cache" \ + --runs 5 --output /tmp/detect-default.json # add --fetch once for a fresh cache +python3 tests/audit_detect.py --binary target/release/brainpod --cache "$cache" \ + --with-env --runs 5 --output /tmp/detect-with-env.json +python3 tests/audit_detect.py --binary target/release/brainpod --cache "$cache" \ + --with-env --with-source --runs 5 --output /tmp/detect-with-source.json +``` + +`environmentChecks` in the pinned corpus run in both opt-in modes. They sample reviewed names/sources/formats/project ownership, unresolved references, platform semantic fields and Dockerfile `ARG` in mixed repositories. Code-reference checks are kept separately as `sourceChecks` and run only with `--with-source`. The audit runner resolves verified cache directory aliases before passing an explicit scan root. Synthetic Rust tests cover malformed input, secret and default redaction, no evaluation, quoting/multiline declarations, duplicate/scoped names, symlinks, read/depth/parser limits, every platform format's semantic mapping, `ARG`, every code-reference pattern family, generic-name gating, identifier boundaries, exclusions, the occurrence cap and serial/parallel determinism. + +## Local results (2026-11-10) + +Validated on this branch: 231 Rust tests passed (3 ignored benchmarks not run), 13 Python audit-runner tests passed, formatting passed and the release build succeeded. `cargo clippy --all-targets -- -D warnings` with clippy 0.1.98 reports four errors in unchanged code: `clippy::question_mark` in `src/detect/deno.rs` and `clippy::result_large_err` in generated tunnel protobuf code. The base branch reports the same four with this toolchain. With only those two lints allowed, clippy is clean. Native OS CI and the Brainpod starter branches were not rerun. + +- **56/56 pinned repositories** (51 existing plus five added for platform manifests), five deterministic runs per mode: + - No flags: **300/300** assertions. + - `--with-env`: **348/348**, including **48/48** declaration checks (22 new). + - `--with-env --with-source`: **353/353**, adding the **5/5** `sourceChecks`. +- All 56 no-flag JSON reports are byte-identical to the base binary at `27f09ed`, and the `projects` arrays in both opt-in modes equal the no-flag arrays. +- Reads and latency (sum over the 56 repositories; median and mean of per-repository five-run warm medians, timed under concurrent host load including CLI startup and serialization): + +| Mode | Files read | Bytes read | Median ms | Mean ms | +|---|---:|---:|---:|---:| +| No flags | 5,239 | 7,344,517 | 15.66 | 33.13 | +| `--with-env` on the base branch (#21) | 6,756 | 14,383,727 | 17.49 | 35.90 | +| `--with-env` | 6,822 | 15,090,783 | 18.04 | 36.39 | +| `--with-env --with-source` | 277,283 | 1,769,906,284 | 78.29 | 115.13 | + + `--with-env` reads 66 more files than #21: the platform manifests. +- Records with `--with-env`: 7,494 Actions, 3,177 dotenv, 2,545 Compose, 1,127 Dockerfile (`ENV` plus `ARG`), 22 Kubernetes, 70 Render, 55 `app-json`, 15 Copilot, 7 Vercel, 5 Netlify, 2 DigitalOcean and 1 App Engine. `--with-source` adds 818 code references in 24 repositories. These counts include examples, duplicate names, unresolved references and separate scopes; they are **not** distinct-variable counts or precision/recall measurements. Fly.io, `heroku.yml` and Netlify template variables have synthetic coverage only; no pinned repository declares them. +- New diagnostics compared with #21's `--with-env`: + - `--with-env`: two unsupported `ARG` names, both in Rails ERB `Dockerfile.tt` templates. + - `--with-source`: those two plus 24 source files over 1 MiB (generated bundles and fixtures). + - The `app.json` that #21's YAML path rejected for a duplicate key now parses as JSON. + +Raw local results are in `/tmp/bp-envplat/{detect-default,detect-with-env,detect-with-source,compare}.json`, and the comparison script is `/tmp/bp-envplat/compare.py`. These cache-dependent artifacts are not committed. diff --git a/tests/test_audit_detect.py b/tests/test_audit_detect.py index 7471dec..829314c 100644 --- a/tests/test_audit_detect.py +++ b/tests/test_audit_detect.py @@ -1,3 +1,4 @@ +import argparse import io import json import tarfile @@ -184,6 +185,47 @@ def test_runtime_declarations_are_not_language_or_package_versions(self): )[0] ) + def test_environment_checks_preserve_source_format_and_project_scope(self): + declaration = { + "source": ".env.example", "format": "dotenv", "project": "api", "name": "TOKEN" + } + expected = {"kind": "environment", "category": "environment", **declaration} + self.assertTrue(audit_detect.check({}, expected, [declaration])[0]) + for wrong in [ + {"source": ".env"}, {"project": "web"}, {"format": "compose"}, {"name": "SECRET"} + ]: + self.assertFalse(audit_detect.check({}, {**expected, **wrong}, [declaration])[0]) + self.assertFalse(audit_detect.check({}, expected)[0]) + self.assertTrue(audit_detect.check({}, {**expected, "absent": True})[0]) + unowned = {**declaration, "project": None} + self.assertTrue(audit_detect.check({}, {**expected, "project": None}, [unowned])[0]) + + def test_source_checks_and_flag_only_in_source_mode(self): + entry = { + "checks": [{"kind": "language"}], + "environmentChecks": [{"kind": "environment", "name": "DECLARED"}], + "sourceChecks": [{"kind": "environment", "format": "code-reference"}], + } + for with_env, with_source, flags, count in [ + (False, False, [], 1), + (True, False, ["--with-env"], 2), + (True, True, ["--with-env", "--with-source"], 3), + ]: + args = argparse.Namespace( + binary=Path("/bin/brainpod"), with_env=with_env, with_source=with_source + ) + self.assertEqual(len(audit_detect.expected_checks(entry, args)), count) + self.assertEqual( + audit_detect.detect_command(args, Path("/repo")), + ["/bin/brainpod", "detect", "/repo", "--json", *flags], + ) + with ( + patch("sys.argv", ["audit_detect.py", "--with-source", "--cache", "c", "--output", "o"]), + patch("sys.stderr", new_callable=io.StringIO), + self.assertRaises(SystemExit), + ): + audit_detect.main() + def test_listing_needs_no_binary_cache_or_network(self): with ( tempfile.TemporaryDirectory() as directory,